@queria-io/fdl-js 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
4
4
 
5
- Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column added after a file was written holds its default (or null) in that file's rows. Renamed fields inside a `struct` keep their names in the file. A table with renamed columns whose `files.json` has no `columns` (written before fdl-cli 0.24) is refused; read it with DuckDB.
5
+ Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column or `struct` field added after a file was written holds its default (or null) in that file's rows. Fields inside a `struct`, a `list` or a `map` are found the same way. Defaults of every type DuckLake takes a default for are typed as hyparquet returns the column: a `Date` for dates and timestamps, microseconds (nanoseconds for `time_ns`) as a `bigint` for times, with a `timetz` moved to UTC. A table with renamed columns whose `files.json` has no `columns` (written before fdl-cli 0.24) is refused; read it with DuckDB.
6
6
 
7
7
  ## Install
8
8
 
package/dist/parquet.d.ts CHANGED
@@ -18,12 +18,15 @@ export declare const cacheApiFooterCache: (cache: Cache, base?: string) => Foote
18
18
  export declare const memoryFooterCache: () => FooterCache;
19
19
  /**
20
20
  * A table's column as one data file holds it: its name in the file (undefined when the file was
21
- * written before the column was added) and the value rows of such a file hold
21
+ * written before the column was added), the value rows of such a file hold, and the conversion
22
+ * from the file's values to the column as it is now (renamed or added struct fields, integers
23
+ * widened to int64)
22
24
  */
23
25
  export type FileColumn = {
24
26
  name: string;
25
27
  physical: string | undefined;
26
28
  fill: () => unknown;
29
+ convert: (value: unknown) => unknown;
27
30
  };
28
31
  export type ParquetFile = {
29
32
  key: string;
package/dist/parquet.js CHANGED
@@ -47,47 +47,173 @@ const readFooter = async (file, footerBytes) => {
47
47
  throw new Error("Parquet footer is longer than the file");
48
48
  return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
49
49
  };
50
- /** The top-level columns of a file, with their field_id. schema[0] is the root */
51
- const topLevel = (metadata) => {
52
- const out = [];
53
- let i = 1;
54
- while (i < metadata.schema.length) {
50
+ /** The file's schema as a tree. schema[0] is the root */
51
+ const schemaTree = (metadata) => {
52
+ const build = (i) => {
55
53
  const s = metadata.schema[i];
56
- out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
57
- // Skip the element's subtree
58
- let pending = s.num_children ?? 0;
59
- i++;
60
- while (pending > 0)
61
- pending += (metadata.schema[i++].num_children ?? 0) - 1;
62
- }
63
- return out;
54
+ const node = {
55
+ name: s.name,
56
+ ...(s.field_id !== undefined ? { id: s.field_id } : {}),
57
+ children: [],
58
+ };
59
+ let next = i + 1;
60
+ for (let n = 0; n < (s.num_children ?? 0); n++) {
61
+ const [child, after] = build(next);
62
+ node.children.push(child);
63
+ next = after;
64
+ }
65
+ return [node, next];
66
+ };
67
+ return build(0)[0];
64
68
  };
69
+ /** A direct child for a column: by field_id, or by name when the file has no field_ids */
70
+ const childFor = (node, column) => node.children.find((n) => n.id === column.id) ??
71
+ node.children.find((n) => n.id === undefined && n.name === column.name);
72
+ /**
73
+ * The node of a list's element or a map's value: below the repeated group that Parquet puts
74
+ * between them and the column
75
+ */
76
+ const nestedFor = (node, column) => node.children.flatMap((g) => g.children).find((n) => n.id === column.id) ??
77
+ node.children
78
+ .flatMap((g) => g.children)
79
+ .find((n) => n.id === undefined && n.name === column.name);
65
80
  const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
66
- /** The value a column's default text stands for, typed as hyparquet returns the column */
67
- const defaultValue = ({ name, type, default: text }) => {
68
- if (text === undefined)
69
- return () => null;
70
- if (INTEGER.has(type) || type === "float32" || type === "float64")
71
- return () => Number(text);
72
- if (type === "int64" || type === "uint64")
73
- return () => BigInt(text);
81
+ const BIGINT = new Set(["int64", "uint64"]);
82
+ // hyparquet returns DOUBLE, which DuckDB writes for 128-bit integers, and decimals as numbers
83
+ const NUMBER = /^(float32|float64|int128|uint128|decimal\(.*\))$/;
84
+ const TIMESTAMP = /^timestamp(tz|_s|_ms|_ns)?$/;
85
+ /** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
86
+ const SPECIAL = { inf: Infinity, "-inf": -Infinity, nan: NaN };
87
+ const float = (text) => SPECIAL[text.toLowerCase()] ?? Number(text);
88
+ /**
89
+ * A date or timestamp as DuckDB writes it ("2020-01-02 03:04:05.123456", "...+09",
90
+ * "0044-03-15 (BC) 12:00:00", "10000-01-01"), as a Date. Like hyparquet, the microseconds since
91
+ * the epoch are divided by 1000 toward zero, so times before 1970 land on the same millisecond
92
+ */
93
+ const timestamp = (text) => {
94
+ const m = /^(\d{4,})-(\d\d)-(\d\d)( \(BC\))?(?:[ T](\d\d):(\d\d):(\d\d)(\.\d+)?)?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
95
+ if (!m)
96
+ return new Date(text);
97
+ const [, y, mo, d, bc, h = "0", mi = "0", s = "0", fraction = "", sign, oh = "0", om = "0"] = m;
98
+ const base = new Date(0);
99
+ // DuckDB writes year 0 as 0001 (BC)
100
+ base.setUTCFullYear(bc ? 1 - Number(y) : Number(y), Number(mo) - 1, Number(d));
101
+ base.setUTCHours(Number(h), Number(mi), Number(s), 0);
102
+ const offset = (sign === "-" ? -1 : 1) * (Number(oh) * 60 + Number(om)) * 60_000;
103
+ const micros = BigInt(base.getTime() - offset) * 1000n + BigInt(fraction.slice(1).padEnd(6, "0").slice(0, 6));
104
+ return new Date(Number(micros / 1000n));
105
+ };
106
+ /**
107
+ * A time of day ("03:04:05.123456", "03:04:05+09") in units of 1e-digits seconds, as hyparquet
108
+ * returns TIME. A time with a time zone is moved to UTC, as DuckDB writes it
109
+ */
110
+ const timeOfDay = (text, digits) => {
111
+ const m = /^(\d\d):(\d\d):(\d\d)(?:\.(\d+))?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
112
+ if (!m)
113
+ throw new FdlError(`not a time: ${text}`);
114
+ const [, h, mi, sec, fraction = "", sign, oh = "0", om = "0"] = m;
115
+ const unit = 10n ** BigInt(digits);
116
+ const day = 86400n * unit;
117
+ const offset = (sign === "-" ? -1n : 1n) * BigInt(Number(oh) * 3600 + Number(om) * 60) * unit;
118
+ const value = BigInt((Number(h) * 60 + Number(mi)) * 60 + Number(sec)) * unit +
119
+ BigInt(fraction.padEnd(digits, "0").slice(0, digits)) -
120
+ (sign ? offset : 0n);
121
+ return ((value % day) + day) % day;
122
+ };
123
+ /** A blob as DuckDB writes it (\\x01 for a byte), decoded as hyparquet decodes BYTE_ARRAY */
124
+ const blob = (text) => new TextDecoder().decode(new Uint8Array(text
125
+ .split(/(\\x[0-9a-fA-F]{2})/)
126
+ .flatMap((part) => /^\\x[0-9a-fA-F]{2}$/.test(part)
127
+ ? [parseInt(part.slice(2), 16)]
128
+ : [...new TextEncoder().encode(part)])));
129
+ /**
130
+ * The value a column's default text stands for, typed as hyparquet returns the column. This
131
+ * covers every type DuckLake takes a default for; DuckLake refuses defaults of struct, list and
132
+ * map columns, so another type means a writer this reader does not know
133
+ */
134
+ const parseDefault = ({ name, type }, text) => {
135
+ if (INTEGER.has(type))
136
+ return Number(text);
137
+ if (BIGINT.has(type))
138
+ return BigInt(text);
139
+ if (NUMBER.test(type))
140
+ return float(text);
74
141
  if (type === "boolean")
75
- return () => text === "true";
76
- if (type === "varchar" || type === "json")
77
- return () => text;
78
- return () => {
79
- throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
80
- };
142
+ return text === "true";
143
+ if (type === "date" || TIMESTAMP.test(type))
144
+ return timestamp(text);
145
+ if (type === "time" || type === "timetz")
146
+ return timeOfDay(text, 6);
147
+ if (type === "time_ns")
148
+ return timeOfDay(text, 9);
149
+ if (type === "blob")
150
+ return blob(text);
151
+ if (type === "json")
152
+ return JSON.parse(text);
153
+ // hyparquet returns UUIDs as lower-case text, as DuckDB writes them
154
+ if (type === "varchar" || type === "uuid")
155
+ return text;
156
+ throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
157
+ };
158
+ const defaultValue = (column) => {
159
+ const text = column.default;
160
+ return text === undefined ? () => null : () => parseDefault(column, text);
161
+ };
162
+ /**
163
+ * Converts a file's value of a column to the column as it is now. Struct fields are found by
164
+ * field_id, so renamed fields take their new names and fields added later their defaults
165
+ */
166
+ const converter = (column, node) => {
167
+ const children = column.children ?? [];
168
+ if (column.type === "struct") {
169
+ const fields = children.map((c) => {
170
+ const n = childFor(node, c);
171
+ return {
172
+ name: c.name,
173
+ physical: n?.name,
174
+ convert: n && converter(c, n),
175
+ fill: defaultValue(c),
176
+ };
177
+ });
178
+ return (value) => value === null || typeof value !== "object"
179
+ ? value
180
+ : Object.fromEntries(fields.map((f) => [
181
+ f.name,
182
+ f.physical === undefined || !f.convert
183
+ ? f.fill()
184
+ : f.convert(value[f.physical]),
185
+ ]));
186
+ }
187
+ if (column.type === "list") {
188
+ const element = children[0] && nestedFor(node, children[0]);
189
+ const convert = element && converter(children[0], element);
190
+ return (value) => (convert && Array.isArray(value) ? value.map(convert) : value);
191
+ }
192
+ if (column.type === "map") {
193
+ const valueColumn = children[1];
194
+ const valueNode = valueColumn && nestedFor(node, valueColumn);
195
+ const convert = valueNode && converter(valueColumn, valueNode);
196
+ return (value) => convert && value !== null && typeof value === "object" && !Array.isArray(value)
197
+ ? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, convert(v)]))
198
+ : value;
199
+ }
200
+ // A column widened from a narrower integer type keeps numbers in the files written before
201
+ if (BIGINT.has(column.type))
202
+ return (v) => (typeof v === "number" ? BigInt(v) : v);
203
+ return (v) => v;
81
204
  };
82
205
  /** Finds each column of the table in the file by field_id (by name when the file has none) */
83
206
  const fileColumns = (metadata, columns) => {
84
- const inFile = topLevel(metadata);
85
- return columns.map((c) => ({
86
- name: c.name,
87
- physical: (inFile.find((f) => f.id === c.id) ??
88
- inFile.find((f) => f.id === undefined && f.name === c.name))?.name,
89
- fill: defaultValue(c),
90
- }));
207
+ const root = schemaTree(metadata);
208
+ return columns.map((c) => {
209
+ const node = childFor(root, c);
210
+ return {
211
+ name: c.name,
212
+ physical: node?.name,
213
+ fill: defaultValue(c),
214
+ convert: node ? converter(c, node) : (v) => v,
215
+ };
216
+ });
91
217
  };
92
218
  const deletedPositions = async (store, deletes, options) => {
93
219
  const parquet = await openParquet(store, deletes, options);
@@ -122,12 +248,32 @@ export const openParquet = async (store, { key, bytes, footerBytes, deletes, col
122
248
  ...(deleted ? { deleted } : {}),
123
249
  };
124
250
  };
251
+ /** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
125
252
  const columnIndex = (metadata, column) => {
126
- // schema[0] is the root; leaves follow in column order (flat tables only)
127
- const i = metadata.schema.findIndex((s, n) => n > 0 && s.name === column) - 1;
128
- if (i < 0)
129
- throw new Error(`no column ${column}`);
130
- return i;
253
+ let leaf = 0;
254
+ let i = 1;
255
+ while (i < metadata.schema.length) {
256
+ const s = metadata.schema[i];
257
+ if (!s.num_children) {
258
+ if (s.name === column)
259
+ return leaf;
260
+ leaf++;
261
+ i++;
262
+ continue;
263
+ }
264
+ if (s.name === column)
265
+ throw new Error(`${column} is not a leaf column`);
266
+ // Skip the group's subtree, counting its leaves
267
+ let pending = s.num_children;
268
+ i++;
269
+ while (pending > 0) {
270
+ const child = metadata.schema[i++].num_children ?? 0;
271
+ if (child === 0)
272
+ leaf++;
273
+ pending += child - 1;
274
+ }
275
+ }
276
+ throw new Error(`no column ${column}`);
131
277
  };
132
278
  // A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
133
279
  // and above) after U+E000-U+FFFF
@@ -195,8 +341,10 @@ export const readRows = async ({ file, metadata, columns: table, deleted }, { co
195
341
  const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
196
342
  const rows = physical.length > 0
197
343
  ? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
198
- : Array.from({ length: (range.rowEnd ?? Number(metadata.num_rows)) - start }, () => ({}));
199
- return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]])));
344
+ : Array.from({
345
+ length: Math.max(0, Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start),
346
+ }, () => ({}));
347
+ return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.convert(row[c.physical])])));
200
348
  };
201
349
  // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
202
350
  // by value, and unlike BigInt() it does not throw for 1.5
@@ -1 +1 @@
1
- {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAoBF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjF,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7F,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,EAAE,EAC/D,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
1
+ {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AA8BF,yDAAyD;AACzD,MAAM,UAAU,GAAG,CAAC,QAAsB,EAAQ,EAAE;IAClD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAkB,EAAE;QAC1C,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,MAAM,IAAI,GAAS;YACjB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,QAAQ,EAAE,EAAE;SACb,CAAC;QACF,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;YACnC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;QACD,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACtB,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrB,CAAC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,QAAQ,GAAG,CAAC,IAAU,EAAE,MAAc,EAAoB,EAAE,CAChE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC;IAC7C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;AAE1E;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,IAAU,EAAE,MAAc,EAAoB,EAAE,CACjE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC;IACxE,IAAI,CAAC,QAAQ;SACV,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;SAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;AAE/D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC5C,8FAA8F;AAC9F,MAAM,MAAM,GAAG,kDAAkD,CAAC;AAClE,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAEhD,0FAA0F;AAC1F,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACvF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;AAE5E;;;;GAIG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAQ,EAAE;IACvC,MAAM,CAAC,GACL,uGAAuG,CAAC,IAAI,CAC1G,IAAI,CACL,CAAC;IACJ,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,QAAQ,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACzB,oCAAoC;IACpC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IACjF,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,GAAG,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjG,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;AAC1C,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,MAAc,EAAU,EAAE;IACzD,MAAM,CAAC,GAAG,kEAAkE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC;IAClD,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,GAAG,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,IAAI,GAAG,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,MAAO,GAAG,IAAI,CAAC;IAC3B,MAAM,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC;IAC9F,MAAM,KAAK,GACT,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI;QAC/D,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACvB,OAAO,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACrC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,MAAM,IAAI,GAAG,CAAC,IAAY,EAAU,EAAE,CACpC,IAAI,WAAW,EAAE,CAAC,MAAM,CACtB,IAAI,UAAU,CACZ,IAAI;KACD,KAAK,CAAC,qBAAqB,CAAC;KAC5B,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAChB,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;IAC9B,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/B,CAAC,CAAC,CAAC,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CACxC,CACJ,CACF,CAAC;AAEJ;;;;GAIG;AACH,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAU,EAAE,IAAY,EAAW,EAAE;IACrE,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,KAAK,MAAM,CAAC;IAC/C,IAAI,IAAI,KAAK,MAAM,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7C,oEAAoE;IACpE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IACvD,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAc,EAAmB,EAAE;IACvD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;IAC5B,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC5E,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,SAAS,GAAG,CAAC,MAAc,EAAE,IAAU,EAAiC,EAAE;IAC9E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;IACvC,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5B,OAAO;gBACL,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,EAAE,IAAI;gBACjB,OAAO,EAAE,CAAC,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7B,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;aACtB,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,KAAK,EAAE,EAAE,CACf,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;YACzC,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBAChB,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,OAAO;oBACpC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;oBACV,CAAC,CAAC,CAAC,CAAC,OAAO,CAAE,KAAiC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;aAC9D,CAAC,CACH,CAAC;IACV,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,OAAO,CAAC,CAAC;QAC5D,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,SAAS,GAAG,WAAW,IAAI,SAAS,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,SAAS,IAAI,SAAS,CAAC,WAAY,EAAE,SAAS,CAAC,CAAC;QAChE,OAAO,CAAC,KAAK,EAAE,EAAE,CACf,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC7E,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC,CAAC,KAAK,CAAC;IACd,CAAC;IACD,0FAA0F;IAC1F,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC/B,OAAO;YACL,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,QAAQ,EAAE,IAAI,EAAE,IAAI;YACpB,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;YACrB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;SAC9C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,2FAA2F;AAC3F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YACnC,IAAI,EAAE,CAAC;YACP,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QACzE,gDAAgD;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC;QAC7B,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,CAAC;gBAAE,IAAI,EAAE,CAAC;YACxB,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR;YACE,MAAM,EAAE,IAAI,CAAC,GAAG,CACd,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CACtE;SACF,EACD,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC9F,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@queria-io/fdl-js",
3
- "version": "0.10.0",
3
+ "version": "0.11.0",
4
4
  "description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
5
5
  "keywords": [
6
6
  "cloudflare-workers",
package/src/parquet.ts CHANGED
@@ -83,9 +83,16 @@ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<Arra
83
83
 
84
84
  /**
85
85
  * A table's column as one data file holds it: its name in the file (undefined when the file was
86
- * written before the column was added) and the value rows of such a file hold
86
+ * written before the column was added), the value rows of such a file hold, and the conversion
87
+ * from the file's values to the column as it is now (renamed or added struct fields, integers
88
+ * widened to int64)
87
89
  */
88
- export type FileColumn = { name: string; physical: string | undefined; fill: () => unknown };
90
+ export type FileColumn = {
91
+ name: string;
92
+ physical: string | undefined;
93
+ fill: () => unknown;
94
+ convert: (value: unknown) => unknown;
95
+ };
89
96
 
90
97
  export type ParquetFile = {
91
98
  key: string;
@@ -99,46 +106,192 @@ export type ParquetFile = {
99
106
 
100
107
  export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
101
108
 
102
- /** The top-level columns of a file, with their field_id. schema[0] is the root */
103
- const topLevel = (metadata: FileMetaData): { name: string; id?: number }[] => {
104
- const out: { name: string; id?: number }[] = [];
105
- let i = 1;
106
- while (i < metadata.schema.length) {
109
+ /** An element of the file's schema with its children */
110
+ type Node = { name: string; id?: number; children: Node[] };
111
+
112
+ /** The file's schema as a tree. schema[0] is the root */
113
+ const schemaTree = (metadata: FileMetaData): Node => {
114
+ const build = (i: number): [Node, number] => {
107
115
  const s = metadata.schema[i]!;
108
- out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
109
- // Skip the element's subtree
110
- let pending = s.num_children ?? 0;
111
- i++;
112
- while (pending > 0) pending += (metadata.schema[i++]!.num_children ?? 0) - 1;
113
- }
114
- return out;
116
+ const node: Node = {
117
+ name: s.name,
118
+ ...(s.field_id !== undefined ? { id: s.field_id } : {}),
119
+ children: [],
120
+ };
121
+ let next = i + 1;
122
+ for (let n = 0; n < (s.num_children ?? 0); n++) {
123
+ const [child, after] = build(next);
124
+ node.children.push(child);
125
+ next = after;
126
+ }
127
+ return [node, next];
128
+ };
129
+ return build(0)[0];
115
130
  };
116
131
 
132
+ /** A direct child for a column: by field_id, or by name when the file has no field_ids */
133
+ const childFor = (node: Node, column: Column): Node | undefined =>
134
+ node.children.find((n) => n.id === column.id) ??
135
+ node.children.find((n) => n.id === undefined && n.name === column.name);
136
+
137
+ /**
138
+ * The node of a list's element or a map's value: below the repeated group that Parquet puts
139
+ * between them and the column
140
+ */
141
+ const nestedFor = (node: Node, column: Column): Node | undefined =>
142
+ node.children.flatMap((g) => g.children).find((n) => n.id === column.id) ??
143
+ node.children
144
+ .flatMap((g) => g.children)
145
+ .find((n) => n.id === undefined && n.name === column.name);
146
+
117
147
  const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
148
+ const BIGINT = new Set(["int64", "uint64"]);
149
+ // hyparquet returns DOUBLE, which DuckDB writes for 128-bit integers, and decimals as numbers
150
+ const NUMBER = /^(float32|float64|int128|uint128|decimal\(.*\))$/;
151
+ const TIMESTAMP = /^timestamp(tz|_s|_ms|_ns)?$/;
118
152
 
119
- /** The value a column's default text stands for, typed as hyparquet returns the column */
120
- const defaultValue = ({ name, type, default: text }: Column): (() => unknown) => {
121
- if (text === undefined) return () => null;
122
- if (INTEGER.has(type) || type === "float32" || type === "float64") return () => Number(text);
123
- if (type === "int64" || type === "uint64") return () => BigInt(text);
124
- if (type === "boolean") return () => text === "true";
125
- if (type === "varchar" || type === "json") return () => text;
126
- return () => {
127
- throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
128
- };
153
+ /** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
154
+ const SPECIAL: Record<string, number> = { inf: Infinity, "-inf": -Infinity, nan: NaN };
155
+ const float = (text: string) => SPECIAL[text.toLowerCase()] ?? Number(text);
156
+
157
+ /**
158
+ * A date or timestamp as DuckDB writes it ("2020-01-02 03:04:05.123456", "...+09",
159
+ * "0044-03-15 (BC) 12:00:00", "10000-01-01"), as a Date. Like hyparquet, the microseconds since
160
+ * the epoch are divided by 1000 toward zero, so times before 1970 land on the same millisecond
161
+ */
162
+ const timestamp = (text: string): Date => {
163
+ const m =
164
+ /^(\d{4,})-(\d\d)-(\d\d)( \(BC\))?(?:[ T](\d\d):(\d\d):(\d\d)(\.\d+)?)?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(
165
+ text,
166
+ );
167
+ if (!m) return new Date(text);
168
+ const [, y, mo, d, bc, h = "0", mi = "0", s = "0", fraction = "", sign, oh = "0", om = "0"] = m;
169
+ const base = new Date(0);
170
+ // DuckDB writes year 0 as 0001 (BC)
171
+ base.setUTCFullYear(bc ? 1 - Number(y) : Number(y), Number(mo) - 1, Number(d));
172
+ base.setUTCHours(Number(h), Number(mi), Number(s), 0);
173
+ const offset = (sign === "-" ? -1 : 1) * (Number(oh) * 60 + Number(om)) * 60_000;
174
+ const micros =
175
+ BigInt(base.getTime() - offset) * 1000n + BigInt(fraction.slice(1).padEnd(6, "0").slice(0, 6));
176
+ return new Date(Number(micros / 1000n));
177
+ };
178
+
179
+ /**
180
+ * A time of day ("03:04:05.123456", "03:04:05+09") in units of 1e-digits seconds, as hyparquet
181
+ * returns TIME. A time with a time zone is moved to UTC, as DuckDB writes it
182
+ */
183
+ const timeOfDay = (text: string, digits: number): bigint => {
184
+ const m = /^(\d\d):(\d\d):(\d\d)(?:\.(\d+))?(?:([+-])(\d\d)(?::?(\d\d))?)?$/.exec(text);
185
+ if (!m) throw new FdlError(`not a time: ${text}`);
186
+ const [, h, mi, sec, fraction = "", sign, oh = "0", om = "0"] = m;
187
+ const unit = 10n ** BigInt(digits);
188
+ const day = 86_400n * unit;
189
+ const offset = (sign === "-" ? -1n : 1n) * BigInt(Number(oh) * 3600 + Number(om) * 60) * unit;
190
+ const value =
191
+ BigInt((Number(h) * 60 + Number(mi)) * 60 + Number(sec)) * unit +
192
+ BigInt(fraction.padEnd(digits, "0").slice(0, digits)) -
193
+ (sign ? offset : 0n);
194
+ return ((value % day) + day) % day;
195
+ };
196
+
197
+ /** A blob as DuckDB writes it (\\x01 for a byte), decoded as hyparquet decodes BYTE_ARRAY */
198
+ const blob = (text: string): string =>
199
+ new TextDecoder().decode(
200
+ new Uint8Array(
201
+ text
202
+ .split(/(\\x[0-9a-fA-F]{2})/)
203
+ .flatMap((part) =>
204
+ /^\\x[0-9a-fA-F]{2}$/.test(part)
205
+ ? [parseInt(part.slice(2), 16)]
206
+ : [...new TextEncoder().encode(part)],
207
+ ),
208
+ ),
209
+ );
210
+
211
+ /**
212
+ * The value a column's default text stands for, typed as hyparquet returns the column. This
213
+ * covers every type DuckLake takes a default for; DuckLake refuses defaults of struct, list and
214
+ * map columns, so another type means a writer this reader does not know
215
+ */
216
+ const parseDefault = ({ name, type }: Column, text: string): unknown => {
217
+ if (INTEGER.has(type)) return Number(text);
218
+ if (BIGINT.has(type)) return BigInt(text);
219
+ if (NUMBER.test(type)) return float(text);
220
+ if (type === "boolean") return text === "true";
221
+ if (type === "date" || TIMESTAMP.test(type)) return timestamp(text);
222
+ if (type === "time" || type === "timetz") return timeOfDay(text, 6);
223
+ if (type === "time_ns") return timeOfDay(text, 9);
224
+ if (type === "blob") return blob(text);
225
+ if (type === "json") return JSON.parse(text);
226
+ // hyparquet returns UUIDs as lower-case text, as DuckDB writes them
227
+ if (type === "varchar" || type === "uuid") return text;
228
+ throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
229
+ };
230
+
231
+ const defaultValue = (column: Column): (() => unknown) => {
232
+ const text = column.default;
233
+ return text === undefined ? () => null : () => parseDefault(column, text);
234
+ };
235
+
236
+ /**
237
+ * Converts a file's value of a column to the column as it is now. Struct fields are found by
238
+ * field_id, so renamed fields take their new names and fields added later their defaults
239
+ */
240
+ const converter = (column: Column, node: Node): ((value: unknown) => unknown) => {
241
+ const children = column.children ?? [];
242
+ if (column.type === "struct") {
243
+ const fields = children.map((c) => {
244
+ const n = childFor(node, c);
245
+ return {
246
+ name: c.name,
247
+ physical: n?.name,
248
+ convert: n && converter(c, n),
249
+ fill: defaultValue(c),
250
+ };
251
+ });
252
+ return (value) =>
253
+ value === null || typeof value !== "object"
254
+ ? value
255
+ : Object.fromEntries(
256
+ fields.map((f) => [
257
+ f.name,
258
+ f.physical === undefined || !f.convert
259
+ ? f.fill()
260
+ : f.convert((value as Record<string, unknown>)[f.physical]),
261
+ ]),
262
+ );
263
+ }
264
+ if (column.type === "list") {
265
+ const element = children[0] && nestedFor(node, children[0]);
266
+ const convert = element && converter(children[0]!, element);
267
+ return (value) => (convert && Array.isArray(value) ? value.map(convert) : value);
268
+ }
269
+ if (column.type === "map") {
270
+ const valueColumn = children[1];
271
+ const valueNode = valueColumn && nestedFor(node, valueColumn);
272
+ const convert = valueNode && converter(valueColumn!, valueNode);
273
+ return (value) =>
274
+ convert && value !== null && typeof value === "object" && !Array.isArray(value)
275
+ ? Object.fromEntries(Object.entries(value).map(([k, v]) => [k, convert(v)]))
276
+ : value;
277
+ }
278
+ // A column widened from a narrower integer type keeps numbers in the files written before
279
+ if (BIGINT.has(column.type)) return (v) => (typeof v === "number" ? BigInt(v) : v);
280
+ return (v) => v;
129
281
  };
130
282
 
131
283
  /** Finds each column of the table in the file by field_id (by name when the file has none) */
132
284
  const fileColumns = (metadata: FileMetaData, columns: Column[]): FileColumn[] => {
133
- const inFile = topLevel(metadata);
134
- return columns.map((c) => ({
135
- name: c.name,
136
- physical: (
137
- inFile.find((f) => f.id === c.id) ??
138
- inFile.find((f) => f.id === undefined && f.name === c.name)
139
- )?.name,
140
- fill: defaultValue(c),
141
- }));
285
+ const root = schemaTree(metadata);
286
+ return columns.map((c) => {
287
+ const node = childFor(root, c);
288
+ return {
289
+ name: c.name,
290
+ physical: node?.name,
291
+ fill: defaultValue(c),
292
+ convert: node ? converter(c, node) : (v) => v,
293
+ };
294
+ });
142
295
  };
143
296
 
144
297
  const deletedPositions = async (
@@ -186,11 +339,29 @@ export const openParquet = async (
186
339
  export type RowRange = { rowStart: number; rowEnd: number };
187
340
  type Value = string | number | bigint;
188
341
 
342
+ /** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
189
343
  const columnIndex = (metadata: FileMetaData, column: string): number => {
190
- // schema[0] is the root; leaves follow in column order (flat tables only)
191
- const i = metadata.schema.findIndex((s, n) => n > 0 && s.name === column) - 1;
192
- if (i < 0) throw new Error(`no column ${column}`);
193
- return i;
344
+ let leaf = 0;
345
+ let i = 1;
346
+ while (i < metadata.schema.length) {
347
+ const s = metadata.schema[i]!;
348
+ if (!s.num_children) {
349
+ if (s.name === column) return leaf;
350
+ leaf++;
351
+ i++;
352
+ continue;
353
+ }
354
+ if (s.name === column) throw new Error(`${column} is not a leaf column`);
355
+ // Skip the group's subtree, counting its leaves
356
+ let pending = s.num_children;
357
+ i++;
358
+ while (pending > 0) {
359
+ const child = metadata.schema[i++]!.num_children ?? 0;
360
+ if (child === 0) leaf++;
361
+ pending += child - 1;
362
+ }
363
+ }
364
+ throw new Error(`no column ${column}`);
194
365
  };
195
366
 
196
367
  // A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
@@ -271,12 +442,17 @@ export const readRows = async <T = Record<string, unknown>>(
271
442
  physical.length > 0
272
443
  ? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
273
444
  : Array.from(
274
- { length: (range.rowEnd ?? Number(metadata.num_rows)) - start },
445
+ {
446
+ length: Math.max(
447
+ 0,
448
+ Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start,
449
+ ),
450
+ },
275
451
  (): Record<string, unknown> => ({}),
276
452
  );
277
453
  return keep(rows).map((row) =>
278
454
  Object.fromEntries(
279
- wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]]),
455
+ wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.convert(row[c.physical])]),
280
456
  ),
281
457
  ) as T[];
282
458
  };