@queria-io/fdl-js 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/parquet.d.ts +4 -1
- package/dist/parquet.js +189 -41
- package/dist/parquet.js.map +1 -1
- package/package.json +1 -1
- package/src/parquet.ts +215 -39
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
|
|
4
4
|
|
|
5
|
-
Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column added after a file was written holds its default (or null) in that file's rows.
|
|
5
|
+
Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column or `struct` field added after a file was written holds its default (or null) in that file's rows. Fields inside a `struct`, a `list` or a `map` are found the same way. Defaults of every type DuckLake takes a default for are typed as hyparquet returns the column: a `Date` for dates and timestamps, microseconds (nanoseconds for `time_ns`) as a `bigint` for times, with a `timetz` moved to UTC. A table with renamed columns whose `files.json` has no `columns` (written before fdl-cli 0.24) is refused; read it with DuckDB.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
package/dist/parquet.d.ts
CHANGED
|
@@ -18,12 +18,15 @@ export declare const cacheApiFooterCache: (cache: Cache, base?: string) => Foote
|
|
|
18
18
|
export declare const memoryFooterCache: () => FooterCache;
|
|
19
19
|
/**
|
|
20
20
|
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
21
|
-
* written before the column was added)
|
|
21
|
+
* written before the column was added), the value rows of such a file hold, and the conversion
|
|
22
|
+
* from the file's values to the column as it is now (renamed or added struct fields, integers
|
|
23
|
+
* widened to int64)
|
|
22
24
|
*/
|
|
23
25
|
export type FileColumn = {
|
|
24
26
|
name: string;
|
|
25
27
|
physical: string | undefined;
|
|
26
28
|
fill: () => unknown;
|
|
29
|
+
convert: (value: unknown) => unknown;
|
|
27
30
|
};
|
|
28
31
|
export type ParquetFile = {
|
|
29
32
|
key: string;
|
package/dist/parquet.js
CHANGED
|
@@ -47,47 +47,173 @@ const readFooter = async (file, footerBytes) => {
|
|
|
47
47
|
throw new Error("Parquet footer is longer than the file");
|
|
48
48
|
return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
|
|
49
49
|
};
|
|
50
|
-
/** The
|
|
51
|
-
const
|
|
52
|
-
const
|
|
53
|
-
let i = 1;
|
|
54
|
-
while (i < metadata.schema.length) {
|
|
50
|
+
/** The file's schema as a tree. schema[0] is the root */
|
|
51
|
+
const schemaTree = (metadata) => {
|
|
52
|
+
const build = (i) => {
|
|
55
53
|
const s = metadata.schema[i];
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
54
|
+
const node = {
|
|
55
|
+
name: s.name,
|
|
56
|
+
...(s.field_id !== undefined ? { id: s.field_id } : {}),
|
|
57
|
+
children: [],
|
|
58
|
+
};
|
|
59
|
+
let next = i + 1;
|
|
60
|
+
for (let n = 0; n < (s.num_children ?? 0); n++) {
|
|
61
|
+
const [child, after] = build(next);
|
|
62
|
+
node.children.push(child);
|
|
63
|
+
next = after;
|
|
64
|
+
}
|
|
65
|
+
return [node, next];
|
|
66
|
+
};
|
|
67
|
+
return build(0)[0];
|
|
64
68
|
};
|
|
69
|
+
/** A direct child for a column: by field_id, or by name when the file has no field_ids */
|
|
70
|
+
const childFor = (node, column) => node.children.find((n) => n.id === column.id) ??
|
|
71
|
+
node.children.find((n) => n.id === undefined && n.name === column.name);
|
|
72
|
+
/**
|
|
73
|
+
* The node of a list's element or a map's value: below the repeated group that Parquet puts
|
|
74
|
+
* between them and the column
|
|
75
|
+
*/
|
|
76
|
+
const nestedFor = (node, column) => node.children.flatMap((g) => g.children).find((n) => n.id === column.id) ??
|
|
77
|
+
node.children
|
|
78
|
+
.flatMap((g) => g.children)
|
|
79
|
+
.find((n) => n.id === undefined && n.name === column.name);
|
|
65
80
|
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
81
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
82
|
+
// hyparquet returns DOUBLE, which DuckDB writes for 128-bit integers, and decimals as numbers
|
|
83
|
+
const NUMBER = /^(float32|float64|int128|uint128|decimal\(.*\))$/;
|
|
84
|
+
const TIMESTAMP = /^timestamp(tz|_s|_ms|_ns)?$/;
|
|
85
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
86
|
+
const SPECIAL = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
87
|
+
const float = (text) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
88
|
+
/**
|
|
89
|
+
* A date or timestamp as DuckDB writes it ("2020-01-02 03:04:05.123456", "...+09",
|
|
90
|
+
* "0044-03-15 (BC) 12:00:00", "10000-01-01"), as a Date. Like hyparquet, the microseconds since
|
|
91
|
+
* the epoch are divided by 1000 toward zero, so times before 1970 land on the same millisecond
|
|
92
|
+
*/
|
|
93
|
+
const timestamp = (text) => {
|
|
94
|
+
const m = /^(\d{4,})-(\d\d)-(\d\d)( \(BC\))?(?:[ T](\d\d):(\d\d):(\d\d)(\.\d+)?)?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
|
|
95
|
+
if (!m)
|
|
96
|
+
return new Date(text);
|
|
97
|
+
const [, y, mo, d, bc, h = "0", mi = "0", s = "0", fraction = "", sign, oh = "0", om = "0"] = m;
|
|
98
|
+
const base = new Date(0);
|
|
99
|
+
// DuckDB writes year 0 as 0001 (BC)
|
|
100
|
+
base.setUTCFullYear(bc ? 1 - Number(y) : Number(y), Number(mo) - 1, Number(d));
|
|
101
|
+
base.setUTCHours(Number(h), Number(mi), Number(s), 0);
|
|
102
|
+
const offset = (sign === "-" ? -1 : 1) * (Number(oh) * 60 + Number(om)) * 60_000;
|
|
103
|
+
const micros = BigInt(base.getTime() - offset) * 1000n + BigInt(fraction.slice(1).padEnd(6, "0").slice(0, 6));
|
|
104
|
+
return new Date(Number(micros / 1000n));
|
|
105
|
+
};
|
|
106
|
+
/**
|
|
107
|
+
* A time of day ("03:04:05.123456", "03:04:05+09") in units of 1e-digits seconds, as hyparquet
|
|
108
|
+
* returns TIME. A time with a time zone is moved to UTC, as DuckDB writes it
|
|
109
|
+
*/
|
|
110
|
+
const timeOfDay = (text, digits) => {
|
|
111
|
+
const m = /^(\d\d):(\d\d):(\d\d)(?:\.(\d+))?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
|
|
112
|
+
if (!m)
|
|
113
|
+
throw new FdlError(`not a time: ${text}`);
|
|
114
|
+
const [, h, mi, sec, fraction = "", sign, oh = "0", om = "0"] = m;
|
|
115
|
+
const unit = 10n ** BigInt(digits);
|
|
116
|
+
const day = 86400n * unit;
|
|
117
|
+
const offset = (sign === "-" ? -1n : 1n) * BigInt(Number(oh) * 3600 + Number(om) * 60) * unit;
|
|
118
|
+
const value = BigInt((Number(h) * 60 + Number(mi)) * 60 + Number(sec)) * unit +
|
|
119
|
+
BigInt(fraction.padEnd(digits, "0").slice(0, digits)) -
|
|
120
|
+
(sign ? offset : 0n);
|
|
121
|
+
return ((value % day) + day) % day;
|
|
122
|
+
};
|
|
123
|
+
/** A blob as DuckDB writes it (\\x01 for a byte), decoded as hyparquet decodes BYTE_ARRAY */
|
|
124
|
+
const blob = (text) => new TextDecoder().decode(new Uint8Array(text
|
|
125
|
+
.split(/(\\x[0-9a-fA-F]{2})/)
|
|
126
|
+
.flatMap((part) => /^\\x[0-9a-fA-F]{2}$/.test(part)
|
|
127
|
+
? [parseInt(part.slice(2), 16)]
|
|
128
|
+
: [...new TextEncoder().encode(part)])));
|
|
129
|
+
/**
|
|
130
|
+
* The value a column's default text stands for, typed as hyparquet returns the column. This
|
|
131
|
+
* covers every type DuckLake takes a default for; DuckLake refuses defaults of struct, list and
|
|
132
|
+
* map columns, so another type means a writer this reader does not know
|
|
133
|
+
*/
|
|
134
|
+
const parseDefault = ({ name, type }, text) => {
|
|
135
|
+
if (INTEGER.has(type))
|
|
136
|
+
return Number(text);
|
|
137
|
+
if (BIGINT.has(type))
|
|
138
|
+
return BigInt(text);
|
|
139
|
+
if (NUMBER.test(type))
|
|
140
|
+
return float(text);
|
|
74
141
|
if (type === "boolean")
|
|
75
|
-
return
|
|
76
|
-
if (type === "
|
|
77
|
-
return ()
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
142
|
+
return text === "true";
|
|
143
|
+
if (type === "date" || TIMESTAMP.test(type))
|
|
144
|
+
return timestamp(text);
|
|
145
|
+
if (type === "time" || type === "timetz")
|
|
146
|
+
return timeOfDay(text, 6);
|
|
147
|
+
if (type === "time_ns")
|
|
148
|
+
return timeOfDay(text, 9);
|
|
149
|
+
if (type === "blob")
|
|
150
|
+
return blob(text);
|
|
151
|
+
if (type === "json")
|
|
152
|
+
return JSON.parse(text);
|
|
153
|
+
// hyparquet returns UUIDs as lower-case text, as DuckDB writes them
|
|
154
|
+
if (type === "varchar" || type === "uuid")
|
|
155
|
+
return text;
|
|
156
|
+
throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
|
|
157
|
+
};
|
|
158
|
+
const defaultValue = (column) => {
|
|
159
|
+
const text = column.default;
|
|
160
|
+
return text === undefined ? () => null : () => parseDefault(column, text);
|
|
161
|
+
};
|
|
162
|
+
/**
|
|
163
|
+
* Converts a file's value of a column to the column as it is now. Struct fields are found by
|
|
164
|
+
* field_id, so renamed fields take their new names and fields added later their defaults
|
|
165
|
+
*/
|
|
166
|
+
const converter = (column, node) => {
|
|
167
|
+
const children = column.children ?? [];
|
|
168
|
+
if (column.type === "struct") {
|
|
169
|
+
const fields = children.map((c) => {
|
|
170
|
+
const n = childFor(node, c);
|
|
171
|
+
return {
|
|
172
|
+
name: c.name,
|
|
173
|
+
physical: n?.name,
|
|
174
|
+
convert: n && converter(c, n),
|
|
175
|
+
fill: defaultValue(c),
|
|
176
|
+
};
|
|
177
|
+
});
|
|
178
|
+
return (value) => value === null || typeof value !== "object"
|
|
179
|
+
? value
|
|
180
|
+
: Object.fromEntries(fields.map((f) => [
|
|
181
|
+
f.name,
|
|
182
|
+
f.physical === undefined || !f.convert
|
|
183
|
+
? f.fill()
|
|
184
|
+
: f.convert(value[f.physical]),
|
|
185
|
+
]));
|
|
186
|
+
}
|
|
187
|
+
if (column.type === "list") {
|
|
188
|
+
const element = children[0] && nestedFor(node, children[0]);
|
|
189
|
+
const convert = element && converter(children[0], element);
|
|
190
|
+
return (value) => (convert && Array.isArray(value) ? value.map(convert) : value);
|
|
191
|
+
}
|
|
192
|
+
if (column.type === "map") {
|
|
193
|
+
const valueColumn = children[1];
|
|
194
|
+
const valueNode = valueColumn && nestedFor(node, valueColumn);
|
|
195
|
+
const convert = valueNode && converter(valueColumn, valueNode);
|
|
196
|
+
return (value) => convert && value !== null && typeof value === "object" && !Array.isArray(value)
|
|
197
|
+
? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, convert(v)]))
|
|
198
|
+
: value;
|
|
199
|
+
}
|
|
200
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
201
|
+
if (BIGINT.has(column.type))
|
|
202
|
+
return (v) => (typeof v === "number" ? BigInt(v) : v);
|
|
203
|
+
return (v) => v;
|
|
81
204
|
};
|
|
82
205
|
/** Finds each column of the table in the file by field_id (by name when the file has none) */
|
|
83
206
|
const fileColumns = (metadata, columns) => {
|
|
84
|
-
const
|
|
85
|
-
return columns.map((c) =>
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
207
|
+
const root = schemaTree(metadata);
|
|
208
|
+
return columns.map((c) => {
|
|
209
|
+
const node = childFor(root, c);
|
|
210
|
+
return {
|
|
211
|
+
name: c.name,
|
|
212
|
+
physical: node?.name,
|
|
213
|
+
fill: defaultValue(c),
|
|
214
|
+
convert: node ? converter(c, node) : (v) => v,
|
|
215
|
+
};
|
|
216
|
+
});
|
|
91
217
|
};
|
|
92
218
|
const deletedPositions = async (store, deletes, options) => {
|
|
93
219
|
const parquet = await openParquet(store, deletes, options);
|
|
@@ -122,12 +248,32 @@ export const openParquet = async (store, { key, bytes, footerBytes, deletes, col
|
|
|
122
248
|
...(deleted ? { deleted } : {}),
|
|
123
249
|
};
|
|
124
250
|
};
|
|
251
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
125
252
|
const columnIndex = (metadata, column) => {
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
253
|
+
let leaf = 0;
|
|
254
|
+
let i = 1;
|
|
255
|
+
while (i < metadata.schema.length) {
|
|
256
|
+
const s = metadata.schema[i];
|
|
257
|
+
if (!s.num_children) {
|
|
258
|
+
if (s.name === column)
|
|
259
|
+
return leaf;
|
|
260
|
+
leaf++;
|
|
261
|
+
i++;
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (s.name === column)
|
|
265
|
+
throw new Error(`${column} is not a leaf column`);
|
|
266
|
+
// Skip the group's subtree, counting its leaves
|
|
267
|
+
let pending = s.num_children;
|
|
268
|
+
i++;
|
|
269
|
+
while (pending > 0) {
|
|
270
|
+
const child = metadata.schema[i++].num_children ?? 0;
|
|
271
|
+
if (child === 0)
|
|
272
|
+
leaf++;
|
|
273
|
+
pending += child - 1;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
throw new Error(`no column ${column}`);
|
|
131
277
|
};
|
|
132
278
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
133
279
|
// and above) after U+E000-U+FFFF
|
|
@@ -195,8 +341,10 @@ export const readRows = async ({ file, metadata, columns: table, deleted }, { co
|
|
|
195
341
|
const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
|
|
196
342
|
const rows = physical.length > 0
|
|
197
343
|
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
198
|
-
: Array.from({
|
|
199
|
-
|
|
344
|
+
: Array.from({
|
|
345
|
+
length: Math.max(0, Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start),
|
|
346
|
+
}, () => ({}));
|
|
347
|
+
return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.convert(row[c.physical])])));
|
|
200
348
|
};
|
|
201
349
|
// INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
|
|
202
350
|
// by value, and unlike BigInt() it does not throw for 1.5
|
package/dist/parquet.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAoBF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjF,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7F,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,EAAE,EAC/D,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AA8BF,yDAAyD;AACzD,MAAM,UAAU,GAAG,CAAC,QAAsB,EAAQ,EAAE;IAClD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAkB,EAAE;QAC1C,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,MAAM,IAAI,GAAS;YACjB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;QACD,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACtB,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,CAAC,IAAU,EAAE,MAAc,EAAoB,EAAE,CAChE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;AAE1E;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,IAAU,EAAE,MAAc,EAAoB,EAAE,CACjE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC;IACxE,IAAI,CAAC,QAAQ;SACV,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;AAE/D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC5C,8FAA8F;AAC9F,MAAM,MAAM,GAAG,kDAAkD,CAAC;AAClE,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAEhD,0FAA0F;AAC1F,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACvF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;AAE5E;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAQ,EAAE;IACvC,MAAM,CAAC,GACL,uGAAuG,CAAC,IAAI,CAC1G,IAAI,CACL,CAAC;IACJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,QAAQ,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACzB,oCAAoC;IACpC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IACjF,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjG,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;AAC1C,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,MAAc,EAAU,EAAE;IACzD,MAAM,CAAC,GAAG,kEAAkE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAO,GAAG,IAAI,CAAC;IAC3B,MAAM,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IAC9F,MAAM,KAAK,GACT,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI;QAC/D,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvB,OAAO,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACrC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,IAAI,GAAG,CAAC,IAAY,EAAU,EAAE,CACpC,IAAI,WAAW,EAAE,CAAC,MAAM,CACtB,IAAI,UAAU,CACZ,IAAI;KACD,KAAK,CAAC,qBAAqB,CAAC;KAC5B,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAChB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;IAC9B,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CACxC,CACJ,CACF,CAAC;AAEJ;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAU,EAAE,IAAY,EAAW,EAAE;IACrE,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,KAAK,MAAM,CAAC;IAC/C,IAAI,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7C,oEAAoE;IACpE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAc,EAAmB,EAAE;IACvD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;IAC5B,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC5E,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,IAAU,EAAiC,EAAE;IAC9E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;IACvC,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,EAAE,IAAI;gBACjB,OAAO,EAAE,CAAC,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7B,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;aACtB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,EAAE,EAAE,CACf,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YACzC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBAChB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO;oBACpC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;oBACV,CAAC,CAAC,CAAC,CAAC,OAAO,CAAE,KAAiC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC9D,CAAC,CACH,CAAC;IACV,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,OAAO,CAAC,CAAC;QAC5D,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,WAAW,IAAI,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,SAAS,IAAI,SAAS,CAAC,WAAY,EAAE,SAAS,CAAC,CAAC;QAChE,OAAO,CAAC,KAAK,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC7E,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC,CAAC,KAAK,CAAC;IACd,CAAC;IACD,0FAA0F;IAC1F,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/B,OAAO;YACL,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,QAAQ,EAAE,IAAI,EAAE,IAAI;YACpB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;SAC9C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,2FAA2F;AAC3F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YACnC,IAAI,EAAE,CAAC;YACP,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QACzE,gDAAgD;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC;QAC7B,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,CAAC;gBAAE,IAAI,EAAE,CAAC;YACxB,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR;YACE,MAAM,EAAE,IAAI,CAAC,GAAG,CACd,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CACtE;SACF,EACD,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC9F,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
package/src/parquet.ts
CHANGED
|
@@ -83,9 +83,16 @@ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<Arra
|
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
85
|
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
86
|
-
* written before the column was added)
|
|
86
|
+
* written before the column was added), the value rows of such a file hold, and the conversion
|
|
87
|
+
* from the file's values to the column as it is now (renamed or added struct fields, integers
|
|
88
|
+
* widened to int64)
|
|
87
89
|
*/
|
|
88
|
-
export type FileColumn = {
|
|
90
|
+
export type FileColumn = {
|
|
91
|
+
name: string;
|
|
92
|
+
physical: string | undefined;
|
|
93
|
+
fill: () => unknown;
|
|
94
|
+
convert: (value: unknown) => unknown;
|
|
95
|
+
};
|
|
89
96
|
|
|
90
97
|
export type ParquetFile = {
|
|
91
98
|
key: string;
|
|
@@ -99,46 +106,192 @@ export type ParquetFile = {
|
|
|
99
106
|
|
|
100
107
|
export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
|
|
101
108
|
|
|
102
|
-
/**
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
109
|
+
/** An element of the file's schema with its children */
|
|
110
|
+
type Node = { name: string; id?: number; children: Node[] };
|
|
111
|
+
|
|
112
|
+
/** The file's schema as a tree. schema[0] is the root */
|
|
113
|
+
const schemaTree = (metadata: FileMetaData): Node => {
|
|
114
|
+
const build = (i: number): [Node, number] => {
|
|
107
115
|
const s = metadata.schema[i]!;
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
116
|
+
const node: Node = {
|
|
117
|
+
name: s.name,
|
|
118
|
+
...(s.field_id !== undefined ? { id: s.field_id } : {}),
|
|
119
|
+
children: [],
|
|
120
|
+
};
|
|
121
|
+
let next = i + 1;
|
|
122
|
+
for (let n = 0; n < (s.num_children ?? 0); n++) {
|
|
123
|
+
const [child, after] = build(next);
|
|
124
|
+
node.children.push(child);
|
|
125
|
+
next = after;
|
|
126
|
+
}
|
|
127
|
+
return [node, next];
|
|
128
|
+
};
|
|
129
|
+
return build(0)[0];
|
|
115
130
|
};
|
|
116
131
|
|
|
132
|
+
/** A direct child for a column: by field_id, or by name when the file has no field_ids */
|
|
133
|
+
const childFor = (node: Node, column: Column): Node | undefined =>
|
|
134
|
+
node.children.find((n) => n.id === column.id) ??
|
|
135
|
+
node.children.find((n) => n.id === undefined && n.name === column.name);
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The node of a list's element or a map's value: below the repeated group that Parquet puts
|
|
139
|
+
* between them and the column
|
|
140
|
+
*/
|
|
141
|
+
const nestedFor = (node: Node, column: Column): Node | undefined =>
|
|
142
|
+
node.children.flatMap((g) => g.children).find((n) => n.id === column.id) ??
|
|
143
|
+
node.children
|
|
144
|
+
.flatMap((g) => g.children)
|
|
145
|
+
.find((n) => n.id === undefined && n.name === column.name);
|
|
146
|
+
|
|
117
147
|
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
148
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
149
|
+
// hyparquet returns DOUBLE, which DuckDB writes for 128-bit integers, and decimals as numbers
|
|
150
|
+
const NUMBER = /^(float32|float64|int128|uint128|decimal\(.*\))$/;
|
|
151
|
+
const TIMESTAMP = /^timestamp(tz|_s|_ms|_ns)?$/;
|
|
118
152
|
|
|
119
|
-
/**
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
153
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
154
|
+
const SPECIAL: Record<string, number> = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
155
|
+
const float = (text: string) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* A date or timestamp as DuckDB writes it ("2020-01-02 03:04:05.123456", "...+09",
|
|
159
|
+
* "0044-03-15 (BC) 12:00:00", "10000-01-01"), as a Date. Like hyparquet, the microseconds since
|
|
160
|
+
* the epoch are divided by 1000 toward zero, so times before 1970 land on the same millisecond
|
|
161
|
+
*/
|
|
162
|
+
const timestamp = (text: string): Date => {
|
|
163
|
+
const m =
|
|
164
|
+
/^(\d{4,})-(\d\d)-(\d\d)( \(BC\))?(?:[ T](\d\d):(\d\d):(\d\d)(\.\d+)?)?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(
|
|
165
|
+
text,
|
|
166
|
+
);
|
|
167
|
+
if (!m) return new Date(text);
|
|
168
|
+
const [, y, mo, d, bc, h = "0", mi = "0", s = "0", fraction = "", sign, oh = "0", om = "0"] = m;
|
|
169
|
+
const base = new Date(0);
|
|
170
|
+
// DuckDB writes year 0 as 0001 (BC)
|
|
171
|
+
base.setUTCFullYear(bc ? 1 - Number(y) : Number(y), Number(mo) - 1, Number(d));
|
|
172
|
+
base.setUTCHours(Number(h), Number(mi), Number(s), 0);
|
|
173
|
+
const offset = (sign === "-" ? -1 : 1) * (Number(oh) * 60 + Number(om)) * 60_000;
|
|
174
|
+
const micros =
|
|
175
|
+
BigInt(base.getTime() - offset) * 1000n + BigInt(fraction.slice(1).padEnd(6, "0").slice(0, 6));
|
|
176
|
+
return new Date(Number(micros / 1000n));
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* A time of day ("03:04:05.123456", "03:04:05+09") in units of 1e-digits seconds, as hyparquet
|
|
181
|
+
* returns TIME. A time with a time zone is moved to UTC, as DuckDB writes it
|
|
182
|
+
*/
|
|
183
|
+
const timeOfDay = (text: string, digits: number): bigint => {
|
|
184
|
+
const m = /^(\d\d):(\d\d):(\d\d)(?:\.(\d+))?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
|
|
185
|
+
if (!m) throw new FdlError(`not a time: ${text}`);
|
|
186
|
+
const [, h, mi, sec, fraction = "", sign, oh = "0", om = "0"] = m;
|
|
187
|
+
const unit = 10n ** BigInt(digits);
|
|
188
|
+
const day = 86_400n * unit;
|
|
189
|
+
const offset = (sign === "-" ? -1n : 1n) * BigInt(Number(oh) * 3600 + Number(om) * 60) * unit;
|
|
190
|
+
const value =
|
|
191
|
+
BigInt((Number(h) * 60 + Number(mi)) * 60 + Number(sec)) * unit +
|
|
192
|
+
BigInt(fraction.padEnd(digits, "0").slice(0, digits)) -
|
|
193
|
+
(sign ? offset : 0n);
|
|
194
|
+
return ((value % day) + day) % day;
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
/** A blob as DuckDB writes it (\\x01 for a byte), decoded as hyparquet decodes BYTE_ARRAY */
|
|
198
|
+
const blob = (text: string): string =>
|
|
199
|
+
new TextDecoder().decode(
|
|
200
|
+
new Uint8Array(
|
|
201
|
+
text
|
|
202
|
+
.split(/(\\x[0-9a-fA-F]{2})/)
|
|
203
|
+
.flatMap((part) =>
|
|
204
|
+
/^\\x[0-9a-fA-F]{2}$/.test(part)
|
|
205
|
+
? [parseInt(part.slice(2), 16)]
|
|
206
|
+
: [...new TextEncoder().encode(part)],
|
|
207
|
+
),
|
|
208
|
+
),
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* The value a column's default text stands for, typed as hyparquet returns the column. This
|
|
213
|
+
* covers every type DuckLake takes a default for; DuckLake refuses defaults of struct, list and
|
|
214
|
+
* map columns, so another type means a writer this reader does not know
|
|
215
|
+
*/
|
|
216
|
+
const parseDefault = ({ name, type }: Column, text: string): unknown => {
|
|
217
|
+
if (INTEGER.has(type)) return Number(text);
|
|
218
|
+
if (BIGINT.has(type)) return BigInt(text);
|
|
219
|
+
if (NUMBER.test(type)) return float(text);
|
|
220
|
+
if (type === "boolean") return text === "true";
|
|
221
|
+
if (type === "date" || TIMESTAMP.test(type)) return timestamp(text);
|
|
222
|
+
if (type === "time" || type === "timetz") return timeOfDay(text, 6);
|
|
223
|
+
if (type === "time_ns") return timeOfDay(text, 9);
|
|
224
|
+
if (type === "blob") return blob(text);
|
|
225
|
+
if (type === "json") return JSON.parse(text);
|
|
226
|
+
// hyparquet returns UUIDs as lower-case text, as DuckDB writes them
|
|
227
|
+
if (type === "varchar" || type === "uuid") return text;
|
|
228
|
+
throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
|
|
229
|
+
};
|
|
230
|
+
|
|
231
|
+
const defaultValue = (column: Column): (() => unknown) => {
|
|
232
|
+
const text = column.default;
|
|
233
|
+
return text === undefined ? () => null : () => parseDefault(column, text);
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Converts a file's value of a column to the column as it is now. Struct fields are found by
|
|
238
|
+
* field_id, so renamed fields take their new names and fields added later their defaults
|
|
239
|
+
*/
|
|
240
|
+
const converter = (column: Column, node: Node): ((value: unknown) => unknown) => {
|
|
241
|
+
const children = column.children ?? [];
|
|
242
|
+
if (column.type === "struct") {
|
|
243
|
+
const fields = children.map((c) => {
|
|
244
|
+
const n = childFor(node, c);
|
|
245
|
+
return {
|
|
246
|
+
name: c.name,
|
|
247
|
+
physical: n?.name,
|
|
248
|
+
convert: n && converter(c, n),
|
|
249
|
+
fill: defaultValue(c),
|
|
250
|
+
};
|
|
251
|
+
});
|
|
252
|
+
return (value) =>
|
|
253
|
+
value === null || typeof value !== "object"
|
|
254
|
+
? value
|
|
255
|
+
: Object.fromEntries(
|
|
256
|
+
fields.map((f) => [
|
|
257
|
+
f.name,
|
|
258
|
+
f.physical === undefined || !f.convert
|
|
259
|
+
? f.fill()
|
|
260
|
+
: f.convert((value as Record<string, unknown>)[f.physical]),
|
|
261
|
+
]),
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
if (column.type === "list") {
|
|
265
|
+
const element = children[0] && nestedFor(node, children[0]);
|
|
266
|
+
const convert = element && converter(children[0]!, element);
|
|
267
|
+
return (value) => (convert && Array.isArray(value) ? value.map(convert) : value);
|
|
268
|
+
}
|
|
269
|
+
if (column.type === "map") {
|
|
270
|
+
const valueColumn = children[1];
|
|
271
|
+
const valueNode = valueColumn && nestedFor(node, valueColumn);
|
|
272
|
+
const convert = valueNode && converter(valueColumn!, valueNode);
|
|
273
|
+
return (value) =>
|
|
274
|
+
convert && value !== null && typeof value === "object" && !Array.isArray(value)
|
|
275
|
+
? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, convert(v)]))
|
|
276
|
+
: value;
|
|
277
|
+
}
|
|
278
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
279
|
+
if (BIGINT.has(column.type)) return (v) => (typeof v === "number" ? BigInt(v) : v);
|
|
280
|
+
return (v) => v;
|
|
129
281
|
};
|
|
130
282
|
|
|
131
283
|
/** Finds each column of the table in the file by field_id (by name when the file has none) */
|
|
132
284
|
const fileColumns = (metadata: FileMetaData, columns: Column[]): FileColumn[] => {
|
|
133
|
-
const
|
|
134
|
-
return columns.map((c) =>
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
285
|
+
const root = schemaTree(metadata);
|
|
286
|
+
return columns.map((c) => {
|
|
287
|
+
const node = childFor(root, c);
|
|
288
|
+
return {
|
|
289
|
+
name: c.name,
|
|
290
|
+
physical: node?.name,
|
|
291
|
+
fill: defaultValue(c),
|
|
292
|
+
convert: node ? converter(c, node) : (v) => v,
|
|
293
|
+
};
|
|
294
|
+
});
|
|
142
295
|
};
|
|
143
296
|
|
|
144
297
|
const deletedPositions = async (
|
|
@@ -186,11 +339,29 @@ export const openParquet = async (
|
|
|
186
339
|
export type RowRange = { rowStart: number; rowEnd: number };
|
|
187
340
|
type Value = string | number | bigint;
|
|
188
341
|
|
|
342
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
189
343
|
const columnIndex = (metadata: FileMetaData, column: string): number => {
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
344
|
+
let leaf = 0;
|
|
345
|
+
let i = 1;
|
|
346
|
+
while (i < metadata.schema.length) {
|
|
347
|
+
const s = metadata.schema[i]!;
|
|
348
|
+
if (!s.num_children) {
|
|
349
|
+
if (s.name === column) return leaf;
|
|
350
|
+
leaf++;
|
|
351
|
+
i++;
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
if (s.name === column) throw new Error(`${column} is not a leaf column`);
|
|
355
|
+
// Skip the group's subtree, counting its leaves
|
|
356
|
+
let pending = s.num_children;
|
|
357
|
+
i++;
|
|
358
|
+
while (pending > 0) {
|
|
359
|
+
const child = metadata.schema[i++]!.num_children ?? 0;
|
|
360
|
+
if (child === 0) leaf++;
|
|
361
|
+
pending += child - 1;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
throw new Error(`no column ${column}`);
|
|
194
365
|
};
|
|
195
366
|
|
|
196
367
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
@@ -271,12 +442,17 @@ export const readRows = async <T = Record<string, unknown>>(
|
|
|
271
442
|
physical.length > 0
|
|
272
443
|
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
273
444
|
: Array.from(
|
|
274
|
-
{
|
|
445
|
+
{
|
|
446
|
+
length: Math.max(
|
|
447
|
+
0,
|
|
448
|
+
Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start,
|
|
449
|
+
),
|
|
450
|
+
},
|
|
275
451
|
(): Record<string, unknown> => ({}),
|
|
276
452
|
);
|
|
277
453
|
return keep(rows).map((row) =>
|
|
278
454
|
Object.fromEntries(
|
|
279
|
-
wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]]),
|
|
455
|
+
wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.convert(row[c.physical])]),
|
|
280
456
|
),
|
|
281
457
|
) as T[];
|
|
282
458
|
};
|