@queria-io/fdl-js 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
4
4
 
5
- Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. It reads only tables whose files can be read as they are: a table with deleted rows or renamed columns is refused, since reading its Parquet files without the catalog would return deleted rows or miss columns. Read those with DuckDB.
5
+ Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column added after a file was written holds its default (or null) in that file's rows. Renamed fields inside a `struct` keep their names in the file. A table with renamed columns whose `files.json` has no `columns` (written before fdl-cli 0.24) is refused; read it with DuckDB.
6
6
 
7
7
  ## Install
8
8
 
package/dist/parquet.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { type AsyncBuffer, type Compressors, type FileMetaData } from "hyparquet";
2
- import type { DataFile } from "./repo.js";
2
+ import { type DataFile } from "./repo.js";
3
3
  import type { Store } from "./store.js";
4
4
  /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
5
5
  export declare const compressors: Compressors;
@@ -16,17 +16,33 @@ export type FooterCache = {
16
16
  export declare const cacheApiFooterCache: (cache: Cache, base?: string) => FooterCache;
17
17
  /** A FooterCache in memory, for one process or isolate */
18
18
  export declare const memoryFooterCache: () => FooterCache;
19
+ /**
20
+ * A table's column as one data file holds it: its name in the file (undefined when the file was
21
+ * written before the column was added) and the value rows of such a file hold
22
+ */
23
+ export type FileColumn = {
24
+ name: string;
25
+ physical: string | undefined;
26
+ fill: () => unknown;
27
+ };
19
28
  export type ParquetFile = {
20
29
  key: string;
21
30
  file: AsyncBuffer;
22
31
  metadata: FileMetaData;
32
+ /** The table's columns, in files.json's order, when the DataFile gave them */
33
+ columns?: FileColumn[];
34
+ /** Positions of the deleted rows, when the DataFile has a delete file */
35
+ deleted?: Set<number>;
23
36
  };
24
37
  export type OpenOptions = {
25
38
  cache?: FooterCache;
26
39
  onRead?: OnRead;
27
40
  };
28
- /** Opens a data file: reads its footer, from the cache when it is there */
29
- export declare const openParquet: (store: Store, { key, bytes, footerBytes }: DataFile, { cache, onRead }?: OpenOptions) => Promise<ParquetFile>;
41
+ /**
42
+ * Opens a data file: reads its footer, from the cache when it is there, and the positions in its
43
+ * delete file when it has one
44
+ */
45
+ export declare const openParquet: (store: Store, { key, bytes, footerBytes, deletes, columns }: DataFile, options?: OpenOptions) => Promise<ParquetFile>;
30
46
  export type RowRange = {
31
47
  rowStart: number;
32
48
  rowEnd: number;
@@ -37,8 +53,13 @@ type Value = string | number | bigint;
37
53
  * statistics are kept. Adjacent row groups are merged into one range
38
54
  */
39
55
  export declare const rowRangesFor: (metadata: FileMetaData, column: string, value: Value) => RowRange[];
40
- /** Reads the rows given (all of them by default), the columns given (all by default) */
41
- export declare const readRows: <T = Record<string, unknown>>({ file, metadata }: ParquetFile, { columns, ...range }?: Partial<RowRange> & {
56
+ /**
57
+ * Reads the rows given (all of them by default), the columns given (all by default), leaving out
58
+ * deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
59
+ * read under its new name, a dropped one is left out, and one added after the file was written
60
+ * holds its default (or null). Without them, columns the file lacks are left out
61
+ */
62
+ export declare const readRows: <T = Record<string, unknown>>({ file, metadata, columns: table, deleted }: ParquetFile, { columns, ...range }?: Partial<RowRange> & {
42
63
  columns?: string[];
43
64
  }) => Promise<T[]>;
44
65
  /**
package/dist/parquet.js CHANGED
@@ -2,6 +2,7 @@
2
2
  // files never change), then only the row groups whose min/max can hold the value asked for
3
3
  import { decompress } from "fzstd";
4
4
  import { parquetMetadata, parquetReadObjects, } from "hyparquet";
5
+ import { FdlError } from "./repo.js";
5
6
  /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
6
7
  export const compressors = { ZSTD: (input) => decompress(input) };
7
8
  /** A data file as hyparquet's AsyncBuffer */
@@ -46,16 +47,80 @@ const readFooter = async (file, footerBytes) => {
46
47
  throw new Error("Parquet footer is longer than the file");
47
48
  return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
48
49
  };
49
- /** Opens a data file: reads its footer, from the cache when it is there */
50
- export const openParquet = async (store, { key, bytes, footerBytes }, { cache, onRead } = {}) => {
50
+ /** The top-level columns of a file, with their field_id. schema[0] is the root */
51
+ const topLevel = (metadata) => {
52
+ const out = [];
53
+ let i = 1;
54
+ while (i < metadata.schema.length) {
55
+ const s = metadata.schema[i];
56
+ out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
57
+ // Skip the element's subtree
58
+ let pending = s.num_children ?? 0;
59
+ i++;
60
+ while (pending > 0)
61
+ pending += (metadata.schema[i++].num_children ?? 0) - 1;
62
+ }
63
+ return out;
64
+ };
65
+ const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
66
+ /** The value a column's default text stands for, typed as hyparquet returns the column */
67
+ const defaultValue = ({ name, type, default: text }) => {
68
+ if (text === undefined)
69
+ return () => null;
70
+ if (INTEGER.has(type) || type === "float32" || type === "float64")
71
+ return () => Number(text);
72
+ if (type === "int64" || type === "uint64")
73
+ return () => BigInt(text);
74
+ if (type === "boolean")
75
+ return () => text === "true";
76
+ if (type === "varchar" || type === "json")
77
+ return () => text;
78
+ return () => {
79
+ throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
80
+ };
81
+ };
82
+ /** Finds each column of the table in the file by field_id (by name when the file has none) */
83
+ const fileColumns = (metadata, columns) => {
84
+ const inFile = topLevel(metadata);
85
+ return columns.map((c) => ({
86
+ name: c.name,
87
+ physical: (inFile.find((f) => f.id === c.id) ??
88
+ inFile.find((f) => f.id === undefined && f.name === c.name))?.name,
89
+ fill: defaultValue(c),
90
+ }));
91
+ };
92
+ const deletedPositions = async (store, deletes, options) => {
93
+ const parquet = await openParquet(store, deletes, options);
94
+ const rows = await readRows(parquet, { columns: ["pos"] });
95
+ return new Set(rows.map((r) => Number(r.pos)));
96
+ };
97
+ /**
98
+ * Opens a data file: reads its footer, from the cache when it is there, and the positions in its
99
+ * delete file when it has one
100
+ */
101
+ export const openParquet = async (store, { key, bytes, footerBytes, deletes, columns }, options = {}) => {
102
+ const { cache, onRead } = options;
51
103
  const file = asyncBuffer(store, key, bytes, onRead);
52
- // The cache only saves reads: when it fails, the footer is read from the file
53
- const hit = await cache?.get(key).catch(() => undefined);
54
- if (hit)
55
- return { key, file, metadata: parquetMetadata(hit) };
56
- const footer = await readFooter(file, footerBytes);
57
- await cache?.put(key, footer).catch(() => { });
58
- return { key, file, metadata: parquetMetadata(footer) };
104
+ const footer = async () => {
105
+ // The cache only saves reads: when it fails, the footer is read from the file
106
+ const hit = await cache?.get(key).catch(() => undefined);
107
+ if (hit)
108
+ return hit;
109
+ const read = await readFooter(file, footerBytes);
110
+ await cache?.put(key, read).catch(() => { });
111
+ return read;
112
+ };
113
+ const [metadata, deleted] = await Promise.all([
114
+ footer().then(parquetMetadata),
115
+ deletes ? deletedPositions(store, deletes, options) : undefined,
116
+ ]);
117
+ return {
118
+ key,
119
+ file,
120
+ metadata,
121
+ ...(columns ? { columns: fileColumns(metadata, columns) } : {}),
122
+ ...(deleted ? { deleted } : {}),
123
+ };
59
124
  };
60
125
  const columnIndex = (metadata, column) => {
61
126
  // schema[0] is the root; leaves follow in column order (flat tables only)
@@ -104,17 +169,34 @@ export const rowRangesFor = (metadata, column, value) => {
104
169
  };
105
170
  }, { start: 0, ranges: [] }).ranges;
106
171
  };
107
- /** Reads the rows given (all of them by default), the columns given (all by default) */
108
- export const readRows = ({ file, metadata }, { columns, ...range } = {}) => {
109
- const names = new Set(metadata.schema.slice(1).map((s) => s.name));
110
- return parquetReadObjects({
111
- file,
112
- metadata,
113
- compressors,
114
- ...range,
115
- // Columns the file lacks are left out, so a reader can ask for a column added later
116
- ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
117
- });
172
+ /**
173
+ * Reads the rows given (all of them by default), the columns given (all by default), leaving out
174
+ * deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
175
+ * read under its new name, a dropped one is left out, and one added after the file was written
176
+ * holds its default (or null). Without them, columns the file lacks are left out
177
+ */
178
+ export const readRows = async ({ file, metadata, columns: table, deleted }, { columns, ...range } = {}) => {
179
+ const start = range.rowStart ?? 0;
180
+ const keep = (rows) => (deleted ? rows.filter((_, i) => !deleted.has(start + i)) : rows);
181
+ if (!table) {
182
+ const names = new Set(metadata.schema.slice(1).map((s) => s.name));
183
+ const rows = await parquetReadObjects({
184
+ file,
185
+ metadata,
186
+ compressors,
187
+ ...range,
188
+ // Columns the file lacks are left out, so a reader can ask for a column added later
189
+ ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
190
+ });
191
+ return keep(rows);
192
+ }
193
+ // Columns the table lacks are left out, so a reader can ask for a column added later
194
+ const wanted = columns ? columns.flatMap((name) => table.filter((t) => t.name === name)) : table;
195
+ const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
196
+ const rows = physical.length > 0
197
+ ? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
198
+ : Array.from({ length: (range.rowEnd ?? Number(metadata.num_rows)) - start }, () => ({}));
199
+ return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]])));
118
200
  };
119
201
  // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
120
202
  // by value, and unlike BigInt() it does not throw for 1.5
@@ -129,7 +211,15 @@ const same = (a, b) => a === b ||
129
211
  * the other rows. Works best on files sorted by column
130
212
  */
131
213
  export const readWhere = async (parquet, column, value, columns) => {
132
- const ranges = rowRangesFor(parquet.metadata, column, value);
214
+ const c = parquet.columns?.find((t) => t.name === column);
215
+ if (parquet.columns && !c)
216
+ throw new Error(`no column ${column}`);
217
+ // A column added after the file was written holds the same value in every row
218
+ if (c && c.physical === undefined && !same(c.fill(), value))
219
+ return [];
220
+ const ranges = c && c.physical === undefined
221
+ ? [{ rowStart: 0, rowEnd: Number(parquet.metadata.num_rows) }]
222
+ : rowRangesFor(parquet.metadata, c?.physical ?? column, value);
133
223
  const wanted = columns && [column, ...columns.filter((c) => c !== column)];
134
224
  const rows = await Promise.all(ranges.map((r) => readRows(parquet, { ...r, ...(wanted ? { columns: wanted } : {}) })));
135
225
  return rows.flat().filter((row) => same(row[column], value));
@@ -1 +1 @@
1
- {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AAInB,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAMF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAY,EACrC,EAAE,KAAK,EAAE,MAAM,KAAkB,EAAE,EACb,EAAE;IACxB,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,8EAA8E;IAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,IAAI,GAAG;QAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;AAC1D,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,EAAE,IAAI,EAAE,QAAQ,EAAe,EAC/B,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO,kBAAkB,CAAC;QACxB,IAAI;QACJ,QAAQ;QACR,WAAW;QACX,GAAG,KAAK;QACR,oFAAoF;QACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrE,CAAiB,CAAC;AACrB,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
1
+ {"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAoBF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEjF,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7F,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,QAAQ;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;KACtB,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,EAAE,EAC/D,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CACnF,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
package/dist/repo.d.ts CHANGED
@@ -35,9 +35,21 @@ export type FileEntry = {
35
35
  bytes: number;
36
36
  };
37
37
  };
38
+ /** A column of a table in files.json. id is the field_id DuckLake writes in every data file */
39
+ export type Column = {
40
+ id: number;
41
+ name: string;
42
+ type: string;
43
+ nullable: boolean;
44
+ /** The value, as text, that rows of files written before the column was added hold */
45
+ default?: string;
46
+ children?: Column[];
47
+ };
38
48
  export type FileList = {
39
49
  tables: Record<string, {
40
50
  files: FileEntry[];
51
+ /** The table's columns, written from fdl-cli 0.24 */
52
+ columns?: Column[];
41
53
  renamed_columns?: boolean;
42
54
  /** The table's fdl.primary_key tag, when it has one */
43
55
  primary_key?: string[];
@@ -57,6 +69,14 @@ export type DataFile = {
57
69
  rows: number;
58
70
  bytes: number;
59
71
  footerBytes?: number;
72
+ /** The delete file that lists the positions of the file's deleted rows */
73
+ deletes?: {
74
+ key: string;
75
+ rows: number;
76
+ bytes: number;
77
+ };
78
+ /** The table's columns, to find them in the file by field_id and fill those it lacks */
79
+ columns?: Column[];
60
80
  };
61
81
  /** Whether a value is a version ID: a lower-case UUIDv7 */
62
82
  export declare const isVersionId: (value: unknown) => value is string;
@@ -82,8 +102,9 @@ export declare const open: (store: Store, rev?: string) => Promise<Version>;
82
102
  export declare const tableNames: (version: Version) => string[];
83
103
  /**
84
104
  * The data files of a table (schema.table), or undefined when the version has no such table.
85
- * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
86
- * would return deleted rows or miss columns
105
+ * Each carries its delete file and the table's columns, so openParquet and readRows leave out
106
+ * deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
107
+ * whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
87
108
  */
88
109
  export declare const tableFiles: (version: Version, table: string) => DataFile[] | undefined;
89
110
  export type RefreshingOptions = {
package/dist/repo.js CHANGED
@@ -120,27 +120,31 @@ export const open = async (store, rev = "HEAD") => {
120
120
  export const tableNames = (version) => Object.keys(version.files.tables);
121
121
  /**
122
122
  * The data files of a table (schema.table), or undefined when the version has no such table.
123
- * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
124
- * would return deleted rows or miss columns
123
+ * Each carries its delete file and the table's columns, so openParquet and readRows leave out
124
+ * deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
125
+ * whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
125
126
  */
126
127
  export const tableFiles = (version, table) => {
127
128
  const entry = version.files.tables[table];
128
129
  if (!entry)
129
130
  return undefined;
130
- if (entry.renamed_columns)
131
- throw new FdlError(`${table} has renamed columns`);
132
- return entry.files.map((f) => {
133
- if (f.deletes)
134
- throw new FdlError(`${table} has deleted rows`);
135
- if (!safe(f.path))
136
- throw new FdlError(`${table} names a file outside the repository: ${f.path}`);
137
- return {
138
- key: version.config.data_dir + f.path,
139
- rows: f.rows,
140
- bytes: f.bytes,
141
- ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
142
- };
143
- });
131
+ if (entry.renamed_columns && !entry.columns)
132
+ throw new FdlError(`${table} has renamed columns, and files.json does not list its columns`);
133
+ const key = (path) => {
134
+ if (!safe(path))
135
+ throw new FdlError(`${table} names a file outside the repository: ${path}`);
136
+ return version.config.data_dir + path;
137
+ };
138
+ return entry.files.map((f) => ({
139
+ key: key(f.path),
140
+ rows: f.rows,
141
+ bytes: f.bytes,
142
+ ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
143
+ ...(f.deletes
144
+ ? { deletes: { key: key(f.deletes.path), rows: f.deletes.rows, bytes: f.deletes.bytes } }
145
+ : {}),
146
+ ...(entry.columns ? { columns: entry.columns } : {}),
147
+ }));
144
148
  };
145
149
  /**
146
150
  * Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
package/dist/repo.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AA2CtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC3D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACnB,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACzE,KAAK;SACF,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAElF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAY,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACtE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACrB,MAAM,IAAI,QAAQ,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5F,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC7D,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,sBAAsB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3B,IAAI,CAAC,CAAC,OAAO;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YACf,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,IAAI;YACrC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiEtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC3D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACnB,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACzE,KAAK;SACF,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAElF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAY,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACtE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACrB,MAAM,IAAI,QAAQ,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5F,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC7D,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,OAAO;QACzC,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,gEAAgE,CAAC,CAAC;IAC/F,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;QAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,IAAI,EAAE,CAAC,CAAC;QAC7F,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAChB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,CAAC,CAAC,OAAO;YACX,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE;YACzF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrD,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@queria-io/fdl-js",
3
- "version": "0.9.0",
3
+ "version": "0.10.0",
4
4
  "description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
5
5
  "keywords": [
6
6
  "cloudflare-workers",
package/src/parquet.ts CHANGED
@@ -8,7 +8,7 @@ import {
8
8
  parquetMetadata,
9
9
  parquetReadObjects,
10
10
  } from "hyparquet";
11
- import type { DataFile } from "./repo.js";
11
+ import { type Column, type DataFile, FdlError } from "./repo.js";
12
12
  import type { Store } from "./store.js";
13
13
 
14
14
  /** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
@@ -81,23 +81,106 @@ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<Arra
81
81
  return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
82
82
  };
83
83
 
84
- export type ParquetFile = { key: string; file: AsyncBuffer; metadata: FileMetaData };
84
+ /**
85
+ * A table's column as one data file holds it: its name in the file (undefined when the file was
86
+ * written before the column was added) and the value rows of such a file hold
87
+ */
88
+ export type FileColumn = { name: string; physical: string | undefined; fill: () => unknown };
89
+
90
+ export type ParquetFile = {
91
+ key: string;
92
+ file: AsyncBuffer;
93
+ metadata: FileMetaData;
94
+ /** The table's columns, in files.json's order, when the DataFile gave them */
95
+ columns?: FileColumn[];
96
+ /** Positions of the deleted rows, when the DataFile has a delete file */
97
+ deleted?: Set<number>;
98
+ };
85
99
 
86
100
  export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
87
101
 
88
- /** Opens a data file: reads its footer, from the cache when it is there */
102
+ /** The top-level columns of a file, with their field_id. schema[0] is the root */
103
+ const topLevel = (metadata: FileMetaData): { name: string; id?: number }[] => {
104
+ const out: { name: string; id?: number }[] = [];
105
+ let i = 1;
106
+ while (i < metadata.schema.length) {
107
+ const s = metadata.schema[i]!;
108
+ out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
109
+ // Skip the element's subtree
110
+ let pending = s.num_children ?? 0;
111
+ i++;
112
+ while (pending > 0) pending += (metadata.schema[i++]!.num_children ?? 0) - 1;
113
+ }
114
+ return out;
115
+ };
116
+
117
+ const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
118
+
119
+ /** The value a column's default text stands for, typed as hyparquet returns the column */
120
+ const defaultValue = ({ name, type, default: text }: Column): (() => unknown) => {
121
+ if (text === undefined) return () => null;
122
+ if (INTEGER.has(type) || type === "float32" || type === "float64") return () => Number(text);
123
+ if (type === "int64" || type === "uint64") return () => BigInt(text);
124
+ if (type === "boolean") return () => text === "true";
125
+ if (type === "varchar" || type === "json") return () => text;
126
+ return () => {
127
+ throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
128
+ };
129
+ };
130
+
131
+ /** Finds each column of the table in the file by field_id (by name when the file has none) */
132
+ const fileColumns = (metadata: FileMetaData, columns: Column[]): FileColumn[] => {
133
+ const inFile = topLevel(metadata);
134
+ return columns.map((c) => ({
135
+ name: c.name,
136
+ physical: (
137
+ inFile.find((f) => f.id === c.id) ??
138
+ inFile.find((f) => f.id === undefined && f.name === c.name)
139
+ )?.name,
140
+ fill: defaultValue(c),
141
+ }));
142
+ };
143
+
144
+ const deletedPositions = async (
145
+ store: Store,
146
+ deletes: NonNullable<DataFile["deletes"]>,
147
+ options: OpenOptions,
148
+ ): Promise<Set<number>> => {
149
+ const parquet = await openParquet(store, deletes, options);
150
+ const rows = await readRows<{ pos: number | bigint }>(parquet, { columns: ["pos"] });
151
+ return new Set(rows.map((r) => Number(r.pos)));
152
+ };
153
+
154
+ /**
155
+ * Opens a data file: reads its footer, from the cache when it is there, and the positions in its
156
+ * delete file when it has one
157
+ */
89
158
  export const openParquet = async (
90
159
  store: Store,
91
- { key, bytes, footerBytes }: DataFile,
92
- { cache, onRead }: OpenOptions = {},
160
+ { key, bytes, footerBytes, deletes, columns }: DataFile,
161
+ options: OpenOptions = {},
93
162
  ): Promise<ParquetFile> => {
163
+ const { cache, onRead } = options;
94
164
  const file = asyncBuffer(store, key, bytes, onRead);
95
- // The cache only saves reads: when it fails, the footer is read from the file
96
- const hit = await cache?.get(key).catch(() => undefined);
97
- if (hit) return { key, file, metadata: parquetMetadata(hit) };
98
- const footer = await readFooter(file, footerBytes);
99
- await cache?.put(key, footer).catch(() => {});
100
- return { key, file, metadata: parquetMetadata(footer) };
165
+ const footer = async () => {
166
+ // The cache only saves reads: when it fails, the footer is read from the file
167
+ const hit = await cache?.get(key).catch(() => undefined);
168
+ if (hit) return hit;
169
+ const read = await readFooter(file, footerBytes);
170
+ await cache?.put(key, read).catch(() => {});
171
+ return read;
172
+ };
173
+ const [metadata, deleted] = await Promise.all([
174
+ footer().then(parquetMetadata),
175
+ deletes ? deletedPositions(store, deletes, options) : undefined,
176
+ ]);
177
+ return {
178
+ key,
179
+ file,
180
+ metadata,
181
+ ...(columns ? { columns: fileColumns(metadata, columns) } : {}),
182
+ ...(deleted ? { deleted } : {}),
183
+ };
101
184
  };
102
185
 
103
186
  export type RowRange = { rowStart: number; rowEnd: number };
@@ -157,20 +240,45 @@ export const rowRangesFor = (metadata: FileMetaData, column: string, value: Valu
157
240
  ).ranges;
158
241
  };
159
242
 
160
- /** Reads the rows given (all of them by default), the columns given (all by default) */
161
- export const readRows = <T = Record<string, unknown>>(
162
- { file, metadata }: ParquetFile,
243
+ /**
244
+ * Reads the rows given (all of them by default), the columns given (all by default), leaving out
245
+ * deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
246
+ * read under its new name, a dropped one is left out, and one added after the file was written
247
+ * holds its default (or null). Without them, columns the file lacks are left out
248
+ */
249
+ export const readRows = async <T = Record<string, unknown>>(
250
+ { file, metadata, columns: table, deleted }: ParquetFile,
163
251
  { columns, ...range }: Partial<RowRange> & { columns?: string[] } = {},
164
252
  ): Promise<T[]> => {
165
- const names = new Set(metadata.schema.slice(1).map((s) => s.name));
166
- return parquetReadObjects({
167
- file,
168
- metadata,
169
- compressors,
170
- ...range,
171
- // Columns the file lacks are left out, so a reader can ask for a column added later
172
- ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
173
- }) as Promise<T[]>;
253
+ const start = range.rowStart ?? 0;
254
+ const keep = <R>(rows: R[]) => (deleted ? rows.filter((_, i) => !deleted.has(start + i)) : rows);
255
+ if (!table) {
256
+ const names = new Set(metadata.schema.slice(1).map((s) => s.name));
257
+ const rows = await parquetReadObjects({
258
+ file,
259
+ metadata,
260
+ compressors,
261
+ ...range,
262
+ // Columns the file lacks are left out, so a reader can ask for a column added later
263
+ ...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
264
+ });
265
+ return keep(rows) as T[];
266
+ }
267
+ // Columns the table lacks are left out, so a reader can ask for a column added later
268
+ const wanted = columns ? columns.flatMap((name) => table.filter((t) => t.name === name)) : table;
269
+ const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
270
+ const rows =
271
+ physical.length > 0
272
+ ? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
273
+ : Array.from(
274
+ { length: (range.rowEnd ?? Number(metadata.num_rows)) - start },
275
+ (): Record<string, unknown> => ({}),
276
+ );
277
+ return keep(rows).map((row) =>
278
+ Object.fromEntries(
279
+ wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : row[c.physical]]),
280
+ ),
281
+ ) as T[];
174
282
  };
175
283
 
176
284
  // INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
@@ -193,7 +301,14 @@ export const readWhere = async <T = Record<string, unknown>>(
193
301
  value: Value,
194
302
  columns?: string[],
195
303
  ): Promise<T[]> => {
196
- const ranges = rowRangesFor(parquet.metadata, column, value);
304
+ const c = parquet.columns?.find((t) => t.name === column);
305
+ if (parquet.columns && !c) throw new Error(`no column ${column}`);
306
+ // A column added after the file was written holds the same value in every row
307
+ if (c && c.physical === undefined && !same(c.fill(), value)) return [];
308
+ const ranges =
309
+ c && c.physical === undefined
310
+ ? [{ rowStart: 0, rowEnd: Number(parquet.metadata.num_rows) }]
311
+ : rowRangesFor(parquet.metadata, c?.physical ?? column, value);
197
312
  const wanted = columns && [column, ...columns.filter((c) => c !== column)];
198
313
  const rows = await Promise.all(
199
314
  ranges.map((r) =>
package/src/repo.ts CHANGED
@@ -28,11 +28,24 @@ export type FileEntry = {
28
28
  footer_bytes?: number;
29
29
  deletes?: { path: string; rows: number; bytes: number };
30
30
  };
31
+ /** A column of a table in files.json. id is the field_id DuckLake writes in every data file */
32
+ export type Column = {
33
+ id: number;
34
+ name: string;
35
+ type: string;
36
+ nullable: boolean;
37
+ /** The value, as text, that rows of files written before the column was added hold */
38
+ default?: string;
39
+ children?: Column[];
40
+ };
41
+
31
42
  export type FileList = {
32
43
  tables: Record<
33
44
  string,
34
45
  {
35
46
  files: FileEntry[];
47
+ /** The table's columns, written from fdl-cli 0.24 */
48
+ columns?: Column[];
36
49
  renamed_columns?: boolean;
37
50
  /** The table's fdl.primary_key tag, when it has one */
38
51
  primary_key?: string[];
@@ -46,7 +59,16 @@ export type FileList = {
46
59
  export type Version = { config: Config; record: VersionRecord; files: FileList };
47
60
 
48
61
  /** A data file to read: its key in the store, rows and sizes from files.json */
49
- export type DataFile = { key: string; rows: number; bytes: number; footerBytes?: number };
62
+ export type DataFile = {
63
+ key: string;
64
+ rows: number;
65
+ bytes: number;
66
+ footerBytes?: number;
67
+ /** The delete file that lists the positions of the file's deleted rows */
68
+ deletes?: { key: string; rows: number; bytes: number };
69
+ /** The table's columns, to find them in the file by field_id and fill those it lacks */
70
+ columns?: Column[];
71
+ };
50
72
 
51
73
  const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
52
74
 
@@ -175,24 +197,29 @@ export const tableNames = (version: Version): string[] => Object.keys(version.fi
175
197
 
176
198
  /**
177
199
  * The data files of a table (schema.table), or undefined when the version has no such table.
178
- * Throws for a table with delete files or renamed columns: reading its Parquet files as they are
179
- * would return deleted rows or miss columns
200
+ * Each carries its delete file and the table's columns, so openParquet and readRows leave out
201
+ * deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
202
+ * whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
180
203
  */
181
204
  export const tableFiles = (version: Version, table: string): DataFile[] | undefined => {
182
205
  const entry = version.files.tables[table];
183
206
  if (!entry) return undefined;
184
- if (entry.renamed_columns) throw new FdlError(`${table} has renamed columns`);
185
- return entry.files.map((f) => {
186
- if (f.deletes) throw new FdlError(`${table} has deleted rows`);
187
- if (!safe(f.path))
188
- throw new FdlError(`${table} names a file outside the repository: ${f.path}`);
189
- return {
190
- key: version.config.data_dir + f.path,
191
- rows: f.rows,
192
- bytes: f.bytes,
193
- ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
194
- };
195
- });
207
+ if (entry.renamed_columns && !entry.columns)
208
+ throw new FdlError(`${table} has renamed columns, and files.json does not list its columns`);
209
+ const key = (path: string) => {
210
+ if (!safe(path)) throw new FdlError(`${table} names a file outside the repository: ${path}`);
211
+ return version.config.data_dir + path;
212
+ };
213
+ return entry.files.map((f) => ({
214
+ key: key(f.path),
215
+ rows: f.rows,
216
+ bytes: f.bytes,
217
+ ...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
218
+ ...(f.deletes
219
+ ? { deletes: { key: key(f.deletes.path), rows: f.deletes.rows, bytes: f.deletes.bytes } }
220
+ : {}),
221
+ ...(entry.columns ? { columns: entry.columns } : {}),
222
+ }));
196
223
  };
197
224
 
198
225
  export type RefreshingOptions = {