@queria-io/fdl-js 0.9.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/parquet.d.ts +28 -5
- package/dist/parquet.js +146 -26
- package/dist/parquet.js.map +1 -1
- package/dist/repo.d.ts +23 -2
- package/dist/repo.js +20 -16
- package/dist/repo.js.map +1 -1
- package/package.json +1 -1
- package/src/parquet.ts +180 -28
- package/src/repo.ts +42 -15
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Read DuckLake tables published as [FDL](https://fdl.queria.io) repositories from JavaScript, without DuckDB: resolve a ref to a version, list the data files of its tables from `files.json`, and read the Parquet files by range with [hyparquet](https://github.com/hyparam/hyparquet). It runs in Cloudflare Workers, browsers and Node.
|
|
4
4
|
|
|
5
|
-
Reading never writes to the store; only the [retention](#retention) functions for a repository's host do.
|
|
5
|
+
Reading never writes to the store; only the [retention](#retention) functions for a repository's host do. Rows are read as DuckLake returns them: `openParquet` reads a data file's delete file, and `readRows` and `readWhere` leave out the deleted rows. Columns are found by the `id` that `files.json` lists for each column, which DuckLake writes as the Parquet `field_id`, so a renamed column is read under its new name, a dropped column is left out, and a column added after a file was written holds its default (or null) in that file's rows. Renamed fields inside a `struct` keep their names in the file. A table with renamed columns whose `files.json` has no `columns` (written before fdl-cli 0.24) is refused; read it with DuckDB.
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
package/dist/parquet.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type AsyncBuffer, type Compressors, type FileMetaData } from "hyparquet";
|
|
2
|
-
import type
|
|
2
|
+
import { type DataFile } from "./repo.js";
|
|
3
3
|
import type { Store } from "./store.js";
|
|
4
4
|
/** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
|
|
5
5
|
export declare const compressors: Compressors;
|
|
@@ -16,17 +16,35 @@ export type FooterCache = {
|
|
|
16
16
|
export declare const cacheApiFooterCache: (cache: Cache, base?: string) => FooterCache;
|
|
17
17
|
/** A FooterCache in memory, for one process or isolate */
|
|
18
18
|
export declare const memoryFooterCache: () => FooterCache;
|
|
19
|
+
/**
|
|
20
|
+
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
21
|
+
* written before the column was added), the value rows of such a file hold, and the cast from the
|
|
22
|
+
* file's values to the column's type (an int32 file of a column widened to int64 gives numbers)
|
|
23
|
+
*/
|
|
24
|
+
export type FileColumn = {
|
|
25
|
+
name: string;
|
|
26
|
+
physical: string | undefined;
|
|
27
|
+
fill: () => unknown;
|
|
28
|
+
cast: (value: unknown) => unknown;
|
|
29
|
+
};
|
|
19
30
|
export type ParquetFile = {
|
|
20
31
|
key: string;
|
|
21
32
|
file: AsyncBuffer;
|
|
22
33
|
metadata: FileMetaData;
|
|
34
|
+
/** The table's columns, in files.json's order, when the DataFile gave them */
|
|
35
|
+
columns?: FileColumn[];
|
|
36
|
+
/** Positions of the deleted rows, when the DataFile has a delete file */
|
|
37
|
+
deleted?: Set<number>;
|
|
23
38
|
};
|
|
24
39
|
export type OpenOptions = {
|
|
25
40
|
cache?: FooterCache;
|
|
26
41
|
onRead?: OnRead;
|
|
27
42
|
};
|
|
28
|
-
/**
|
|
29
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Opens a data file: reads its footer, from the cache when it is there, and the positions in its
|
|
45
|
+
* delete file when it has one
|
|
46
|
+
*/
|
|
47
|
+
export declare const openParquet: (store: Store, { key, bytes, footerBytes, deletes, columns }: DataFile, options?: OpenOptions) => Promise<ParquetFile>;
|
|
30
48
|
export type RowRange = {
|
|
31
49
|
rowStart: number;
|
|
32
50
|
rowEnd: number;
|
|
@@ -37,8 +55,13 @@ type Value = string | number | bigint;
|
|
|
37
55
|
* statistics are kept. Adjacent row groups are merged into one range
|
|
38
56
|
*/
|
|
39
57
|
export declare const rowRangesFor: (metadata: FileMetaData, column: string, value: Value) => RowRange[];
|
|
40
|
-
/**
|
|
41
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Reads the rows given (all of them by default), the columns given (all by default), leaving out
|
|
60
|
+
* deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
|
|
61
|
+
* read under its new name, a dropped one is left out, and one added after the file was written
|
|
62
|
+
* holds its default (or null). Without them, columns the file lacks are left out
|
|
63
|
+
*/
|
|
64
|
+
export declare const readRows: <T = Record<string, unknown>>({ file, metadata, columns: table, deleted }: ParquetFile, { columns, ...range }?: Partial<RowRange> & {
|
|
42
65
|
columns?: string[];
|
|
43
66
|
}) => Promise<T[]>;
|
|
44
67
|
/**
|
package/dist/parquet.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// files never change), then only the row groups whose min/max can hold the value asked for
|
|
3
3
|
import { decompress } from "fzstd";
|
|
4
4
|
import { parquetMetadata, parquetReadObjects, } from "hyparquet";
|
|
5
|
+
import { FdlError } from "./repo.js";
|
|
5
6
|
/** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
|
|
6
7
|
export const compressors = { ZSTD: (input) => decompress(input) };
|
|
7
8
|
/** A data file as hyparquet's AsyncBuffer */
|
|
@@ -46,23 +47,115 @@ const readFooter = async (file, footerBytes) => {
|
|
|
46
47
|
throw new Error("Parquet footer is longer than the file");
|
|
47
48
|
return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
|
|
48
49
|
};
|
|
49
|
-
/**
|
|
50
|
-
|
|
50
|
+
/** The top-level columns of a file, with their field_id. schema[0] is the root */
|
|
51
|
+
const topLevel = (metadata) => {
|
|
52
|
+
const out = [];
|
|
53
|
+
let i = 1;
|
|
54
|
+
while (i < metadata.schema.length) {
|
|
55
|
+
const s = metadata.schema[i];
|
|
56
|
+
out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
|
|
57
|
+
// Skip the element's subtree
|
|
58
|
+
let pending = s.num_children ?? 0;
|
|
59
|
+
i++;
|
|
60
|
+
while (pending > 0)
|
|
61
|
+
pending += (metadata.schema[i++].num_children ?? 0) - 1;
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
};
|
|
65
|
+
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
66
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
67
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
68
|
+
const SPECIAL = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
69
|
+
const float = (text) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
70
|
+
/** The value a column's default text stands for, typed as hyparquet returns the column */
|
|
71
|
+
const defaultValue = ({ name, type, default: text }) => {
|
|
72
|
+
if (text === undefined)
|
|
73
|
+
return () => null;
|
|
74
|
+
if (INTEGER.has(type))
|
|
75
|
+
return () => Number(text);
|
|
76
|
+
if (type === "float32" || type === "float64")
|
|
77
|
+
return () => float(text);
|
|
78
|
+
if (BIGINT.has(type))
|
|
79
|
+
return () => BigInt(text);
|
|
80
|
+
if (type === "boolean")
|
|
81
|
+
return () => text === "true";
|
|
82
|
+
if (type === "varchar" || type === "json")
|
|
83
|
+
return () => text;
|
|
84
|
+
return () => {
|
|
85
|
+
throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
|
|
86
|
+
};
|
|
87
|
+
};
|
|
88
|
+
/** Finds each column of the table in the file by field_id (by name when the file has none) */
|
|
89
|
+
const fileColumns = (metadata, columns) => {
|
|
90
|
+
const inFile = topLevel(metadata);
|
|
91
|
+
return columns.map((c) => ({
|
|
92
|
+
name: c.name,
|
|
93
|
+
physical: (inFile.find((f) => f.id === c.id) ??
|
|
94
|
+
inFile.find((f) => f.id === undefined && f.name === c.name))?.name,
|
|
95
|
+
fill: defaultValue(c),
|
|
96
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
97
|
+
cast: BIGINT.has(c.type) ? (v) => (typeof v === "number" ? BigInt(v) : v) : (v) => v,
|
|
98
|
+
}));
|
|
99
|
+
};
|
|
100
|
+
const deletedPositions = async (store, deletes, options) => {
|
|
101
|
+
const parquet = await openParquet(store, deletes, options);
|
|
102
|
+
const rows = await readRows(parquet, { columns: ["pos"] });
|
|
103
|
+
return new Set(rows.map((r) => Number(r.pos)));
|
|
104
|
+
};
|
|
105
|
+
/**
|
|
106
|
+
* Opens a data file: reads its footer, from the cache when it is there, and the positions in its
|
|
107
|
+
* delete file when it has one
|
|
108
|
+
*/
|
|
109
|
+
export const openParquet = async (store, { key, bytes, footerBytes, deletes, columns }, options = {}) => {
|
|
110
|
+
const { cache, onRead } = options;
|
|
51
111
|
const file = asyncBuffer(store, key, bytes, onRead);
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
112
|
+
const footer = async () => {
|
|
113
|
+
// The cache only saves reads: when it fails, the footer is read from the file
|
|
114
|
+
const hit = await cache?.get(key).catch(() => undefined);
|
|
115
|
+
if (hit)
|
|
116
|
+
return hit;
|
|
117
|
+
const read = await readFooter(file, footerBytes);
|
|
118
|
+
await cache?.put(key, read).catch(() => { });
|
|
119
|
+
return read;
|
|
120
|
+
};
|
|
121
|
+
const [metadata, deleted] = await Promise.all([
|
|
122
|
+
footer().then(parquetMetadata),
|
|
123
|
+
deletes ? deletedPositions(store, deletes, options) : undefined,
|
|
124
|
+
]);
|
|
125
|
+
return {
|
|
126
|
+
key,
|
|
127
|
+
file,
|
|
128
|
+
metadata,
|
|
129
|
+
...(columns ? { columns: fileColumns(metadata, columns) } : {}),
|
|
130
|
+
...(deleted ? { deleted } : {}),
|
|
131
|
+
};
|
|
59
132
|
};
|
|
133
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
60
134
|
const columnIndex = (metadata, column) => {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
135
|
+
let leaf = 0;
|
|
136
|
+
let i = 1;
|
|
137
|
+
while (i < metadata.schema.length) {
|
|
138
|
+
const s = metadata.schema[i];
|
|
139
|
+
if (!s.num_children) {
|
|
140
|
+
if (s.name === column)
|
|
141
|
+
return leaf;
|
|
142
|
+
leaf++;
|
|
143
|
+
i++;
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (s.name === column)
|
|
147
|
+
throw new Error(`${column} is not a leaf column`);
|
|
148
|
+
// Skip the group's subtree, counting its leaves
|
|
149
|
+
let pending = s.num_children;
|
|
150
|
+
i++;
|
|
151
|
+
while (pending > 0) {
|
|
152
|
+
const child = metadata.schema[i++].num_children ?? 0;
|
|
153
|
+
if (child === 0)
|
|
154
|
+
leaf++;
|
|
155
|
+
pending += child - 1;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
throw new Error(`no column ${column}`);
|
|
66
159
|
};
|
|
67
160
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
68
161
|
// and above) after U+E000-U+FFFF
|
|
@@ -104,17 +197,36 @@ export const rowRangesFor = (metadata, column, value) => {
|
|
|
104
197
|
};
|
|
105
198
|
}, { start: 0, ranges: [] }).ranges;
|
|
106
199
|
};
|
|
107
|
-
/**
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
200
|
+
/**
|
|
201
|
+
* Reads the rows given (all of them by default), the columns given (all by default), leaving out
|
|
202
|
+
* deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
|
|
203
|
+
* read under its new name, a dropped one is left out, and one added after the file was written
|
|
204
|
+
* holds its default (or null). Without them, columns the file lacks are left out
|
|
205
|
+
*/
|
|
206
|
+
export const readRows = async ({ file, metadata, columns: table, deleted }, { columns, ...range } = {}) => {
|
|
207
|
+
const start = range.rowStart ?? 0;
|
|
208
|
+
const keep = (rows) => (deleted ? rows.filter((_, i) => !deleted.has(start + i)) : rows);
|
|
209
|
+
if (!table) {
|
|
210
|
+
const names = new Set(metadata.schema.slice(1).map((s) => s.name));
|
|
211
|
+
const rows = await parquetReadObjects({
|
|
212
|
+
file,
|
|
213
|
+
metadata,
|
|
214
|
+
compressors,
|
|
215
|
+
...range,
|
|
216
|
+
// Columns the file lacks are left out, so a reader can ask for a column added later
|
|
217
|
+
...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
|
|
218
|
+
});
|
|
219
|
+
return keep(rows);
|
|
220
|
+
}
|
|
221
|
+
// Columns the table lacks are left out, so a reader can ask for a column added later
|
|
222
|
+
const wanted = columns ? columns.flatMap((name) => table.filter((t) => t.name === name)) : table;
|
|
223
|
+
const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
|
|
224
|
+
const rows = physical.length > 0
|
|
225
|
+
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
226
|
+
: Array.from({
|
|
227
|
+
length: Math.max(0, Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start),
|
|
228
|
+
}, () => ({}));
|
|
229
|
+
return keep(rows).map((row) => Object.fromEntries(wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.cast(row[c.physical])])));
|
|
118
230
|
};
|
|
119
231
|
// INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
|
|
120
232
|
// by value, and unlike BigInt() it does not throw for 1.5
|
|
@@ -129,7 +241,15 @@ const same = (a, b) => a === b ||
|
|
|
129
241
|
* the other rows. Works best on files sorted by column
|
|
130
242
|
*/
|
|
131
243
|
export const readWhere = async (parquet, column, value, columns) => {
|
|
132
|
-
const
|
|
244
|
+
const c = parquet.columns?.find((t) => t.name === column);
|
|
245
|
+
if (parquet.columns && !c)
|
|
246
|
+
throw new Error(`no column ${column}`);
|
|
247
|
+
// A column added after the file was written holds the same value in every row
|
|
248
|
+
if (c && c.physical === undefined && !same(c.fill(), value))
|
|
249
|
+
return [];
|
|
250
|
+
const ranges = c && c.physical === undefined
|
|
251
|
+
? [{ rowStart: 0, rowEnd: Number(parquet.metadata.num_rows) }]
|
|
252
|
+
: rowRangesFor(parquet.metadata, c?.physical ?? column, value);
|
|
133
253
|
const wanted = columns && [column, ...columns.filter((c) => c !== column)];
|
|
134
254
|
const rows = await Promise.all(ranges.map((r) => readRows(parquet, { ...r, ...(wanted ? { columns: wanted } : {}) })));
|
|
135
255
|
return rows.flat().filter((row) => same(row[column], value));
|
package/dist/parquet.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AAInB,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AAMF,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAY,EACrC,EAAE,KAAK,EAAE,MAAM,KAAkB,EAAE,EACb,EAAE;IACxB,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,8EAA8E;IAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACzD,IAAI,GAAG;QAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;AAC1D,CAAC,CAAC;AAKF,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,0EAA0E;IAC1E,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF,wFAAwF;AACxF,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,EAAE,IAAI,EAAE,QAAQ,EAAe,EAC/B,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,OAAO,kBAAkB,CAAC;QACxB,IAAI;QACJ,QAAQ;QACR,WAAW;QACX,GAAG,KAAK;QACR,oFAAoF;QACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrE,CAAiB,CAAC;AACrB,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../src/parquet.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,2FAA2F;AAC3F,OAAO,EAAE,UAAU,EAAE,MAAM,OAAO,CAAC;AACnC,OAAO,EAIL,eAAe,EACf,kBAAkB,GACnB,MAAM,WAAW,CAAC;AACnB,OAAO,EAA8B,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGjE,2FAA2F;AAC3F,MAAM,CAAC,MAAM,WAAW,GAAgB,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;AAK/E,6CAA6C;AAC7C,MAAM,CAAC,MAAM,WAAW,GAAG,CACzB,KAAY,EACZ,GAAW,EACX,UAAkB,EAClB,MAAe,EACF,EAAE,CAAC,CAAC;IACjB,UAAU;IACV,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,UAAU,EAAE,EAAE;QACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;QACvC,OAAO,IAAI,CAAC;IACd,CAAC;CACF,CAAC,CAAC;AAQH,gGAAgG;AAChG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CACjC,KAAY,EACZ,IAAI,GAAG,6BAA6B,EACvB,EAAE,CAAC,CAAC;IACjB,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE;IACtF,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACnB,KAAK,CAAC,GAAG,CACP,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,EAC9B,IAAI,QAAQ,CAAC,MAAM,EAAE;QACnB,OAAO,EAAE;YACP,cAAc,EAAE,0BAA0B;YAC1C,eAAe,EAAE,qCAAqC;SACvD;KACF,CAAC,CACH;CACJ,CAAC,CAAC;AAEH,0DAA0D;AAC1D,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAgB,EAAE;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC5C,OAAO;QACL,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QACjC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC;KACvD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,SAAS;AAEnC,8FAA8F;AAC9F,MAAM,UAAU,GAAG,KAAK,EAAE,IAAiB,EAAE,WAAoB,EAAwB,EAAE;IACzF,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,KAAK,SAAS,IAAI,WAAW,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,KAAK;QAC3C,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACnD,IAAI,MAAM,GAAG,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC7E,OAAO,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,EAAE,IAAI,CAAC,CAAC;AACxF,CAAC,CAAC;AA0BF,kFAAkF;AAClF,MAAM,QAAQ,GAAG,CAAC,QAAsB,EAAmC,EAAE;IAC3E,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpF,6BAA6B;QAC7B,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC;QAClC,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE5C,0FAA0F;AAC1F,MAAM,OAAO,GAA2B,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AACvF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC;AAE5E,0FAA0F;AAC1F,MAAM,YAAY,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAU,EAAmB,EAAE;IAC9E,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvE,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAChD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,KAAK,MAAM,CAAC;IACrD,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC;IAC7D,OAAO,GAAG,EAAE;QACV,MAAM,IAAI,QAAQ,CAAC,eAAe,IAAI,uBAAuB,IAAI,EAAE,CAAC,CAAC;IACvE,CAAC,CAAC;AACJ,CAAC,CAAC;AAEF,8FAA8F;AAC9F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,OAAiB,EAAgB,EAAE;IAC9E,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CACR,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,CAC5D,EAAE,IAAI;QACP,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;QACrB,0FAA0F;QAC1F,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;KACrF,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,KAAY,EACZ,OAAyC,EACzC,OAAoB,EACE,EAAE;IACxB,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAA2B,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACrF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAC9B,KAAY,EACZ,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAY,EACvD,UAAuB,EAAE,EACH,EAAE;IACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAClC,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,8EAA8E;QAC9E,MAAM,GAAG,GAAG,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACzD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;QACpB,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QACjD,MAAM,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC5C,MAAM,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC;QAC9B,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KAChE,CAAC,CAAC;IACH,OAAO;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChC,CAAC;AACJ,CAAC,CAAC;AAKF,2FAA2F;AAC3F,MAAM,WAAW,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAU,EAAE;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;gBAAE,OAAO,IAAI,CAAC;YACnC,IAAI,EAAE,CAAC;YACP,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,uBAAuB,CAAC,CAAC;QACzE,gDAAgD;QAChD,IAAI,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC;QAC7B,CAAC,EAAE,CAAC;QACJ,OAAO,OAAO,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAE,CAAC,YAAY,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,KAAK,CAAC;gBAAE,IAAI,EAAE,CAAC;YACxB,OAAO,IAAI,KAAK,GAAG,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;AACzC,CAAC,CAAC;AAEF,6FAA6F;AAC7F,iCAAiC;AACjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE;IACpC,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAChE,CAAC,CAAC;AAEF,yFAAyF;AACzF,mFAAmF;AACnF,MAAM,MAAM,GAAG,CAAC,CAAQ,EAAE,CAAQ,EAAW,EAAE;IAC7C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,GAAY,EAAE,GAAY,EAAE,EAAE,CAC1D,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,GAAY,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,GAAY,CAAC,CAAC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsB,EAAE,MAAc,EAAE,KAAY,EAAc,EAAE;IAC/F,MAAM,CAAC,GAAG,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxC,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAC/B,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE;QAC3B,MAAM,GAAG,GAAG,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,IAAI,KAAK,EAAE,GAAG,CAAC;YAChF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,OAAO;YACL,KAAK,EAAE,GAAG;YACV,MAAM,EACJ,IAAI,EAAE,MAAM,KAAK,KAAK;gBACpB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;SACpD,CAAC;IACJ,CAAC,EACD,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACzB,CAAC,MAAM,CAAC;AACX,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAC3B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAe,EACxD,EAAE,OAAO,EAAE,GAAG,KAAK,KAAiD,EAAE,EACxD,EAAE;IAChB,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,IAAI,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,CAAI,IAAS,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACjG,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC;YACpC,IAAI;YACJ,QAAQ;YACR,WAAW;YACX,GAAG,KAAK;YACR,oFAAoF;YACpF,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAQ,CAAC;IAC3B,CAAC;IACD,qFAAqF;IACrF,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjG,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GACR,QAAQ,CAAC,MAAM,GAAG,CAAC;QACjB,CAAC,CAAC,MAAM,kBAAkB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;QACxF,CAAC,CAAC,KAAK,CAAC,IAAI,CACR;YACE,MAAM,EAAE,IAAI,CAAC,GAAG,CACd,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,GAAG,KAAK,CACtE;SACF,EACD,GAA4B,EAAE,CAAC,CAAC,EAAE,CAAC,CACpC,CAAC;IACR,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC5B,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAC3F,CACK,CAAC;AACX,CAAC,CAAC;AAEF,4FAA4F;AAC5F,0DAA0D;AAC1D,MAAM,IAAI,GAAG,CAAC,CAAU,EAAE,CAAQ,EAAE,EAAE,CACpC,CAAC,KAAK,CAAC;IACP,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAC/C,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC;QAChD,OAAO,CAAC,KAAK,QAAQ;QACrB,kCAAkC;QAClC,CAAC,IAAI,CAAC,CAAC,CAAC;AAEZ;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,OAAoB,EACpB,MAAc,EACd,KAAY,EACZ,OAAkB,EACJ,EAAE;IAChB,MAAM,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;IAC1D,IAAI,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;IAClE,8EAA8E;IAC9E,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvE,MAAM,MAAM,GACV,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS;QAC3B,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,MAAM,EAAE,KAAK,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;IAC3E,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACf,QAAQ,CAA0B,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAC7F,CACF,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAQ,CAAC;AACtE,CAAC,CAAC"}
|
package/dist/repo.d.ts
CHANGED
|
@@ -35,9 +35,21 @@ export type FileEntry = {
|
|
|
35
35
|
bytes: number;
|
|
36
36
|
};
|
|
37
37
|
};
|
|
38
|
+
/** A column of a table in files.json. id is the field_id DuckLake writes in every data file */
|
|
39
|
+
export type Column = {
|
|
40
|
+
id: number;
|
|
41
|
+
name: string;
|
|
42
|
+
type: string;
|
|
43
|
+
nullable: boolean;
|
|
44
|
+
/** The value, as text, that rows of files written before the column was added hold */
|
|
45
|
+
default?: string;
|
|
46
|
+
children?: Column[];
|
|
47
|
+
};
|
|
38
48
|
export type FileList = {
|
|
39
49
|
tables: Record<string, {
|
|
40
50
|
files: FileEntry[];
|
|
51
|
+
/** The table's columns, written from fdl-cli 0.24 */
|
|
52
|
+
columns?: Column[];
|
|
41
53
|
renamed_columns?: boolean;
|
|
42
54
|
/** The table's fdl.primary_key tag, when it has one */
|
|
43
55
|
primary_key?: string[];
|
|
@@ -57,6 +69,14 @@ export type DataFile = {
|
|
|
57
69
|
rows: number;
|
|
58
70
|
bytes: number;
|
|
59
71
|
footerBytes?: number;
|
|
72
|
+
/** The delete file that lists the positions of the file's deleted rows */
|
|
73
|
+
deletes?: {
|
|
74
|
+
key: string;
|
|
75
|
+
rows: number;
|
|
76
|
+
bytes: number;
|
|
77
|
+
};
|
|
78
|
+
/** The table's columns, to find them in the file by field_id and fill those it lacks */
|
|
79
|
+
columns?: Column[];
|
|
60
80
|
};
|
|
61
81
|
/** Whether a value is a version ID: a lower-case UUIDv7 */
|
|
62
82
|
export declare const isVersionId: (value: unknown) => value is string;
|
|
@@ -82,8 +102,9 @@ export declare const open: (store: Store, rev?: string) => Promise<Version>;
|
|
|
82
102
|
export declare const tableNames: (version: Version) => string[];
|
|
83
103
|
/**
|
|
84
104
|
* The data files of a table (schema.table), or undefined when the version has no such table.
|
|
85
|
-
*
|
|
86
|
-
*
|
|
105
|
+
* Each carries its delete file and the table's columns, so openParquet and readRows leave out
|
|
106
|
+
* deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
|
|
107
|
+
* whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
|
|
87
108
|
*/
|
|
88
109
|
export declare const tableFiles: (version: Version, table: string) => DataFile[] | undefined;
|
|
89
110
|
export type RefreshingOptions = {
|
package/dist/repo.js
CHANGED
|
@@ -120,27 +120,31 @@ export const open = async (store, rev = "HEAD") => {
|
|
|
120
120
|
export const tableNames = (version) => Object.keys(version.files.tables);
|
|
121
121
|
/**
|
|
122
122
|
* The data files of a table (schema.table), or undefined when the version has no such table.
|
|
123
|
-
*
|
|
124
|
-
*
|
|
123
|
+
* Each carries its delete file and the table's columns, so openParquet and readRows leave out
|
|
124
|
+
* deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
|
|
125
|
+
* whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
|
|
125
126
|
*/
|
|
126
127
|
export const tableFiles = (version, table) => {
|
|
127
128
|
const entry = version.files.tables[table];
|
|
128
129
|
if (!entry)
|
|
129
130
|
return undefined;
|
|
130
|
-
if (entry.renamed_columns)
|
|
131
|
-
throw new FdlError(`${table} has renamed columns`);
|
|
132
|
-
|
|
133
|
-
if (
|
|
134
|
-
throw new FdlError(`${table}
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
131
|
+
if (entry.renamed_columns && !entry.columns)
|
|
132
|
+
throw new FdlError(`${table} has renamed columns, and files.json does not list its columns`);
|
|
133
|
+
const key = (path) => {
|
|
134
|
+
if (!safe(path))
|
|
135
|
+
throw new FdlError(`${table} names a file outside the repository: ${path}`);
|
|
136
|
+
return version.config.data_dir + path;
|
|
137
|
+
};
|
|
138
|
+
return entry.files.map((f) => ({
|
|
139
|
+
key: key(f.path),
|
|
140
|
+
rows: f.rows,
|
|
141
|
+
bytes: f.bytes,
|
|
142
|
+
...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
|
|
143
|
+
...(f.deletes
|
|
144
|
+
? { deletes: { key: key(f.deletes.path), rows: f.deletes.rows, bytes: f.deletes.bytes } }
|
|
145
|
+
: {}),
|
|
146
|
+
...(entry.columns ? { columns: entry.columns } : {}),
|
|
147
|
+
}));
|
|
144
148
|
};
|
|
145
149
|
/**
|
|
146
150
|
* Wraps load so that its result is kept for ttlMs. When a reload fails, the last result is
|
package/dist/repo.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;
|
|
1
|
+
{"version":3,"file":"repo.js","sourceRoot":"","sources":["../src/repo.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4FAA4F;AAC5F,OAAO,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAEvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,OAAO,QAAS,SAAQ,KAAK;CAAG;AAiEtC,MAAM,UAAU,GAAG,uEAAuE,CAAC;AAE3F,2DAA2D;AAC3D,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC7D,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAEtD,kFAAkF;AAClF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC5D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,KAAK,EAAE;IACZ,KAAK,KAAK,GAAG;IACb,KAAK,KAAK,IAAI;IACd,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACpB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAExB,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAY,EAAE,GAAW,EAAmB,EAAE;IACpE,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,EAAK,KAAY,EAAE,GAAW,EAAc,EAAE,CAC9D,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,CAAM,CAAC;AAE9C,MAAM,UAAU,GAAG,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;IAC/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW;QACnC,MAAM,IAAI,QAAQ,CAAC,GAAG,IAAI,WAAW,GAAG,0BAA0B,CAAC,CAAC;AACxE,CAAC,CAAC;AAEF,2FAA2F;AAC3F,kCAAkC;AAClC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAC5B,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;IACrB,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IACrB,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE5D;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAc,EAAmB,EAAE,CAC3D,OAAO,KAAK,KAAK,QAAQ;IACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;IACnB,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACzE,KAAK;SACF,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SACZ,KAAK,CAAC,GAAG,CAAC;SACV,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;AAElF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAY,EAAmB,EAAE;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAkB,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3D,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAY,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IACtE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QACrB,MAAM,IAAI,QAAQ,CAAC,4CAA4C,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5F,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;AAC7D,CAAC,CAAC;AAEF,qGAAqG;AACrG,MAAM,UAAU,GAAG,CAAC,GAAW,EAAY,EAAE,CAC3C,GAAG,KAAK,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;IACvC,CAAC,CAAC,CAAC,GAAG,CAAC;IACP,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,EAAE,aAAa,GAAG,EAAE,EAAE,cAAc,GAAG,EAAE,CAAC,CAAC;AAE/D;;;;GAIG;AACH,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAmB,EAAE;IAC3E,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,KAAK,EAAE,GAAW,EAAE,KAAa,EAA0B,EAAE;QAC1E,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,6BAA6B,CAAC,CAAC;YACrF,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,0BAA0B,CAAC,CAAC;QACpE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,+BAA+B,MAAM,EAAE,CAAC,CAAC;QACpE,OAAO,MAAM,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,EACnD,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CACtB,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAY,EAAE,EAAU,EAAE,MAAe,EAAoB,EAAE;IAC/F,MAAM,GAAG,GAAG,YAAY,EAAE,GAAG,CAAC;IAC9B,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,KAAK,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,cAAc,CAAC;KACjC,CAAC,CAAC;IACH,iFAAiF;IACjF,IAAI,OAAO,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,CAAC,CAAC;IACzF,IAAI,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,aAAa,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAkB,CAAC;IACjD,IAAI,MAAM,CAAC,EAAE,KAAK,EAAE;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,sBAAsB,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAClF,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,IAAI,YAAY,CAAC;IACrD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QACxB,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;IAChD,IAAI,KAAK,KAAK,IAAI;QAAE,MAAM,IAAI,QAAQ,CAAC,WAAW,EAAE,yCAAyC,CAAC,CAAC;IAC/F,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAa,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,kDAAkD;AAClD,MAAM,CAAC,MAAM,IAAI,GAAG,KAAK,EAAE,KAAY,EAAE,GAAG,GAAG,MAAM,EAAoB,EAAE;IACzE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACjF,OAAO,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;AACxC,CAAC,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAY,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;AAE5F;;;;;GAKG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,KAAa,EAA0B,EAAE;IACpF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1C,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,OAAO;QACzC,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,gEAAgE,CAAC,CAAC;IAC/F,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE;QAC3B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,KAAK,yCAAyC,IAAI,EAAE,CAAC,CAAC;QAC7F,OAAO,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC;IACxC,CAAC,CAAC;IACF,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QAChB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,CAAC,CAAC,OAAO;YACX,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE;YACzF,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACrD,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAQF;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CACxB,IAAsB,EACtB,KAAa,EACb,EAAE,OAAO,EAAE,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,KAAwB,EAAE,EAC3D,EAAE;IACF,IAAI,IAAI,GAA6C,IAAI,CAAC;IAC1D,OAAO,GAAe,EAAE;QACtB,IAAI,IAAI,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACvD,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC;QAC7B,MAAM,KAAK,GAAG,QAAQ;YACpB,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;gBACjB,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YACJ,CAAC,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,KAAK,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC;QACnC,IAAI,GAAG,KAAK,CAAC;QACb,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACf,IAAI,IAAI,KAAK,KAAK;gBAAE,IAAI,GAAG,IAAI,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;AACJ,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@queria-io/fdl-js",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.1",
|
|
4
4
|
"description": "Read DuckLake tables published as FDL repositories, without DuckDB: resolve refs, list a version's files and read Parquet by range",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cloudflare-workers",
|
package/src/parquet.ts
CHANGED
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
parquetMetadata,
|
|
9
9
|
parquetReadObjects,
|
|
10
10
|
} from "hyparquet";
|
|
11
|
-
import type
|
|
11
|
+
import { type Column, type DataFile, FdlError } from "./repo.js";
|
|
12
12
|
import type { Store } from "./store.js";
|
|
13
13
|
|
|
14
14
|
/** DuckDB writes zstd by default. fzstd is plain JS, so it runs in Workers and browsers */
|
|
@@ -81,33 +81,148 @@ const readFooter = async (file: AsyncBuffer, footerBytes?: number): Promise<Arra
|
|
|
81
81
|
return length <= guess ? tail.slice(guess - length) : file.slice(size - length, size);
|
|
82
82
|
};
|
|
83
83
|
|
|
84
|
-
|
|
84
|
+
/**
|
|
85
|
+
* A table's column as one data file holds it: its name in the file (undefined when the file was
|
|
86
|
+
* written before the column was added), the value rows of such a file hold, and the cast from the
|
|
87
|
+
* file's values to the column's type (an int32 file of a column widened to int64 gives numbers)
|
|
88
|
+
*/
|
|
89
|
+
export type FileColumn = {
|
|
90
|
+
name: string;
|
|
91
|
+
physical: string | undefined;
|
|
92
|
+
fill: () => unknown;
|
|
93
|
+
cast: (value: unknown) => unknown;
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
export type ParquetFile = {
|
|
97
|
+
key: string;
|
|
98
|
+
file: AsyncBuffer;
|
|
99
|
+
metadata: FileMetaData;
|
|
100
|
+
/** The table's columns, in files.json's order, when the DataFile gave them */
|
|
101
|
+
columns?: FileColumn[];
|
|
102
|
+
/** Positions of the deleted rows, when the DataFile has a delete file */
|
|
103
|
+
deleted?: Set<number>;
|
|
104
|
+
};
|
|
85
105
|
|
|
86
106
|
export type OpenOptions = { cache?: FooterCache; onRead?: OnRead };
|
|
87
107
|
|
|
88
|
-
/**
|
|
108
|
+
/** The top-level columns of a file, with their field_id. schema[0] is the root */
|
|
109
|
+
const topLevel = (metadata: FileMetaData): { name: string; id?: number }[] => {
|
|
110
|
+
const out: { name: string; id?: number }[] = [];
|
|
111
|
+
let i = 1;
|
|
112
|
+
while (i < metadata.schema.length) {
|
|
113
|
+
const s = metadata.schema[i]!;
|
|
114
|
+
out.push({ name: s.name, ...(s.field_id !== undefined ? { id: s.field_id } : {}) });
|
|
115
|
+
// Skip the element's subtree
|
|
116
|
+
let pending = s.num_children ?? 0;
|
|
117
|
+
i++;
|
|
118
|
+
while (pending > 0) pending += (metadata.schema[i++]!.num_children ?? 0) - 1;
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const INTEGER = new Set(["int8", "int16", "int32", "uint8", "uint16", "uint32"]);
|
|
124
|
+
const BIGINT = new Set(["int64", "uint64"]);
|
|
125
|
+
|
|
126
|
+
/** DuckDB writes infinities and NaN as inf, -inf and nan, which Number() does not read */
|
|
127
|
+
const SPECIAL: Record<string, number> = { inf: Infinity, "-inf": -Infinity, nan: NaN };
|
|
128
|
+
const float = (text: string) => SPECIAL[text.toLowerCase()] ?? Number(text);
|
|
129
|
+
|
|
130
|
+
/** The value a column's default text stands for, typed as hyparquet returns the column */
|
|
131
|
+
const defaultValue = ({ name, type, default: text }: Column): (() => unknown) => {
|
|
132
|
+
if (text === undefined) return () => null;
|
|
133
|
+
if (INTEGER.has(type)) return () => Number(text);
|
|
134
|
+
if (type === "float32" || type === "float64") return () => float(text);
|
|
135
|
+
if (BIGINT.has(type)) return () => BigInt(text);
|
|
136
|
+
if (type === "boolean") return () => text === "true";
|
|
137
|
+
if (type === "varchar" || type === "json") return () => text;
|
|
138
|
+
return () => {
|
|
139
|
+
throw new FdlError(`cannot fill ${name}: a default of type ${type}`);
|
|
140
|
+
};
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
/** Finds each column of the table in the file by field_id (by name when the file has none) */
|
|
144
|
+
const fileColumns = (metadata: FileMetaData, columns: Column[]): FileColumn[] => {
|
|
145
|
+
const inFile = topLevel(metadata);
|
|
146
|
+
return columns.map((c) => ({
|
|
147
|
+
name: c.name,
|
|
148
|
+
physical: (
|
|
149
|
+
inFile.find((f) => f.id === c.id) ??
|
|
150
|
+
inFile.find((f) => f.id === undefined && f.name === c.name)
|
|
151
|
+
)?.name,
|
|
152
|
+
fill: defaultValue(c),
|
|
153
|
+
// A column widened from a narrower integer type keeps numbers in the files written before
|
|
154
|
+
cast: BIGINT.has(c.type) ? (v) => (typeof v === "number" ? BigInt(v) : v) : (v) => v,
|
|
155
|
+
}));
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
const deletedPositions = async (
|
|
159
|
+
store: Store,
|
|
160
|
+
deletes: NonNullable<DataFile["deletes"]>,
|
|
161
|
+
options: OpenOptions,
|
|
162
|
+
): Promise<Set<number>> => {
|
|
163
|
+
const parquet = await openParquet(store, deletes, options);
|
|
164
|
+
const rows = await readRows<{ pos: number | bigint }>(parquet, { columns: ["pos"] });
|
|
165
|
+
return new Set(rows.map((r) => Number(r.pos)));
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Opens a data file: reads its footer, from the cache when it is there, and the positions in its
|
|
170
|
+
* delete file when it has one
|
|
171
|
+
*/
|
|
89
172
|
export const openParquet = async (
|
|
90
173
|
store: Store,
|
|
91
|
-
{ key, bytes, footerBytes }: DataFile,
|
|
92
|
-
|
|
174
|
+
{ key, bytes, footerBytes, deletes, columns }: DataFile,
|
|
175
|
+
options: OpenOptions = {},
|
|
93
176
|
): Promise<ParquetFile> => {
|
|
177
|
+
const { cache, onRead } = options;
|
|
94
178
|
const file = asyncBuffer(store, key, bytes, onRead);
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
179
|
+
const footer = async () => {
|
|
180
|
+
// The cache only saves reads: when it fails, the footer is read from the file
|
|
181
|
+
const hit = await cache?.get(key).catch(() => undefined);
|
|
182
|
+
if (hit) return hit;
|
|
183
|
+
const read = await readFooter(file, footerBytes);
|
|
184
|
+
await cache?.put(key, read).catch(() => {});
|
|
185
|
+
return read;
|
|
186
|
+
};
|
|
187
|
+
const [metadata, deleted] = await Promise.all([
|
|
188
|
+
footer().then(parquetMetadata),
|
|
189
|
+
deletes ? deletedPositions(store, deletes, options) : undefined,
|
|
190
|
+
]);
|
|
191
|
+
return {
|
|
192
|
+
key,
|
|
193
|
+
file,
|
|
194
|
+
metadata,
|
|
195
|
+
...(columns ? { columns: fileColumns(metadata, columns) } : {}),
|
|
196
|
+
...(deleted ? { deleted } : {}),
|
|
197
|
+
};
|
|
101
198
|
};
|
|
102
199
|
|
|
103
200
|
export type RowRange = { rowStart: number; rowEnd: number };
|
|
104
201
|
type Value = string | number | bigint;
|
|
105
202
|
|
|
203
|
+
/** The index in a row group's columns of a top-level leaf column. schema[0] is the root */
|
|
106
204
|
const columnIndex = (metadata: FileMetaData, column: string): number => {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
205
|
+
let leaf = 0;
|
|
206
|
+
let i = 1;
|
|
207
|
+
while (i < metadata.schema.length) {
|
|
208
|
+
const s = metadata.schema[i]!;
|
|
209
|
+
if (!s.num_children) {
|
|
210
|
+
if (s.name === column) return leaf;
|
|
211
|
+
leaf++;
|
|
212
|
+
i++;
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
215
|
+
if (s.name === column) throw new Error(`${column} is not a leaf column`);
|
|
216
|
+
// Skip the group's subtree, counting its leaves
|
|
217
|
+
let pending = s.num_children;
|
|
218
|
+
i++;
|
|
219
|
+
while (pending > 0) {
|
|
220
|
+
const child = metadata.schema[i++]!.num_children ?? 0;
|
|
221
|
+
if (child === 0) leaf++;
|
|
222
|
+
pending += child - 1;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
throw new Error(`no column ${column}`);
|
|
111
226
|
};
|
|
112
227
|
|
|
113
228
|
// A UTF-16 code unit, shifted so that units compare in code point order: surrogates (U+10000
|
|
@@ -157,20 +272,50 @@ export const rowRangesFor = (metadata: FileMetaData, column: string, value: Valu
|
|
|
157
272
|
).ranges;
|
|
158
273
|
};
|
|
159
274
|
|
|
160
|
-
/**
|
|
161
|
-
|
|
162
|
-
|
|
275
|
+
/**
|
|
276
|
+
* Reads the rows given (all of them by default), the columns given (all by default), leaving out
|
|
277
|
+
* deleted rows. With the table's columns, columns are found by field_id, so a renamed column is
|
|
278
|
+
* read under its new name, a dropped one is left out, and one added after the file was written
|
|
279
|
+
* holds its default (or null). Without them, columns the file lacks are left out
|
|
280
|
+
*/
|
|
281
|
+
export const readRows = async <T = Record<string, unknown>>(
|
|
282
|
+
{ file, metadata, columns: table, deleted }: ParquetFile,
|
|
163
283
|
{ columns, ...range }: Partial<RowRange> & { columns?: string[] } = {},
|
|
164
284
|
): Promise<T[]> => {
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
metadata
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
285
|
+
const start = range.rowStart ?? 0;
|
|
286
|
+
const keep = <R>(rows: R[]) => (deleted ? rows.filter((_, i) => !deleted.has(start + i)) : rows);
|
|
287
|
+
if (!table) {
|
|
288
|
+
const names = new Set(metadata.schema.slice(1).map((s) => s.name));
|
|
289
|
+
const rows = await parquetReadObjects({
|
|
290
|
+
file,
|
|
291
|
+
metadata,
|
|
292
|
+
compressors,
|
|
293
|
+
...range,
|
|
294
|
+
// Columns the file lacks are left out, so a reader can ask for a column added later
|
|
295
|
+
...(columns ? { columns: columns.filter((c) => names.has(c)) } : {}),
|
|
296
|
+
});
|
|
297
|
+
return keep(rows) as T[];
|
|
298
|
+
}
|
|
299
|
+
// Columns the table lacks are left out, so a reader can ask for a column added later
|
|
300
|
+
const wanted = columns ? columns.flatMap((name) => table.filter((t) => t.name === name)) : table;
|
|
301
|
+
const physical = wanted.flatMap((c) => (c.physical === undefined ? [] : [c.physical]));
|
|
302
|
+
const rows =
|
|
303
|
+
physical.length > 0
|
|
304
|
+
? await parquetReadObjects({ file, metadata, compressors, ...range, columns: physical })
|
|
305
|
+
: Array.from(
|
|
306
|
+
{
|
|
307
|
+
length: Math.max(
|
|
308
|
+
0,
|
|
309
|
+
Math.min(range.rowEnd ?? Infinity, Number(metadata.num_rows)) - start,
|
|
310
|
+
),
|
|
311
|
+
},
|
|
312
|
+
(): Record<string, unknown> => ({}),
|
|
313
|
+
);
|
|
314
|
+
return keep(rows).map((row) =>
|
|
315
|
+
Object.fromEntries(
|
|
316
|
+
wanted.map((c) => [c.name, c.physical === undefined ? c.fill() : c.cast(row[c.physical])]),
|
|
317
|
+
),
|
|
318
|
+
) as T[];
|
|
174
319
|
};
|
|
175
320
|
|
|
176
321
|
// INT64 columns come back as bigint; let 5 and 5n match. == compares a bigint with a number
|
|
@@ -193,7 +338,14 @@ export const readWhere = async <T = Record<string, unknown>>(
|
|
|
193
338
|
value: Value,
|
|
194
339
|
columns?: string[],
|
|
195
340
|
): Promise<T[]> => {
|
|
196
|
-
const
|
|
341
|
+
const c = parquet.columns?.find((t) => t.name === column);
|
|
342
|
+
if (parquet.columns && !c) throw new Error(`no column ${column}`);
|
|
343
|
+
// A column added after the file was written holds the same value in every row
|
|
344
|
+
if (c && c.physical === undefined && !same(c.fill(), value)) return [];
|
|
345
|
+
const ranges =
|
|
346
|
+
c && c.physical === undefined
|
|
347
|
+
? [{ rowStart: 0, rowEnd: Number(parquet.metadata.num_rows) }]
|
|
348
|
+
: rowRangesFor(parquet.metadata, c?.physical ?? column, value);
|
|
197
349
|
const wanted = columns && [column, ...columns.filter((c) => c !== column)];
|
|
198
350
|
const rows = await Promise.all(
|
|
199
351
|
ranges.map((r) =>
|
package/src/repo.ts
CHANGED
|
@@ -28,11 +28,24 @@ export type FileEntry = {
|
|
|
28
28
|
footer_bytes?: number;
|
|
29
29
|
deletes?: { path: string; rows: number; bytes: number };
|
|
30
30
|
};
|
|
31
|
+
/** A column of a table in files.json. id is the field_id DuckLake writes in every data file */
|
|
32
|
+
export type Column = {
|
|
33
|
+
id: number;
|
|
34
|
+
name: string;
|
|
35
|
+
type: string;
|
|
36
|
+
nullable: boolean;
|
|
37
|
+
/** The value, as text, that rows of files written before the column was added hold */
|
|
38
|
+
default?: string;
|
|
39
|
+
children?: Column[];
|
|
40
|
+
};
|
|
41
|
+
|
|
31
42
|
export type FileList = {
|
|
32
43
|
tables: Record<
|
|
33
44
|
string,
|
|
34
45
|
{
|
|
35
46
|
files: FileEntry[];
|
|
47
|
+
/** The table's columns, written from fdl-cli 0.24 */
|
|
48
|
+
columns?: Column[];
|
|
36
49
|
renamed_columns?: boolean;
|
|
37
50
|
/** The table's fdl.primary_key tag, when it has one */
|
|
38
51
|
primary_key?: string[];
|
|
@@ -46,7 +59,16 @@ export type FileList = {
|
|
|
46
59
|
export type Version = { config: Config; record: VersionRecord; files: FileList };
|
|
47
60
|
|
|
48
61
|
/** A data file to read: its key in the store, rows and sizes from files.json */
|
|
49
|
-
export type DataFile = {
|
|
62
|
+
export type DataFile = {
|
|
63
|
+
key: string;
|
|
64
|
+
rows: number;
|
|
65
|
+
bytes: number;
|
|
66
|
+
footerBytes?: number;
|
|
67
|
+
/** The delete file that lists the positions of the file's deleted rows */
|
|
68
|
+
deletes?: { key: string; rows: number; bytes: number };
|
|
69
|
+
/** The table's columns, to find them in the file by field_id and fill those it lacks */
|
|
70
|
+
columns?: Column[];
|
|
71
|
+
};
|
|
50
72
|
|
|
51
73
|
const VERSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
52
74
|
|
|
@@ -175,24 +197,29 @@ export const tableNames = (version: Version): string[] => Object.keys(version.fi
|
|
|
175
197
|
|
|
176
198
|
/**
|
|
177
199
|
* The data files of a table (schema.table), or undefined when the version has no such table.
|
|
178
|
-
*
|
|
179
|
-
*
|
|
200
|
+
* Each carries its delete file and the table's columns, so openParquet and readRows leave out
|
|
201
|
+
* deleted rows and find renamed columns by field_id. Throws for a table with renamed columns
|
|
202
|
+
* whose files.json has no columns (written before fdl-cli 0.24): its old files carry old names
|
|
180
203
|
*/
|
|
181
204
|
export const tableFiles = (version: Version, table: string): DataFile[] | undefined => {
|
|
182
205
|
const entry = version.files.tables[table];
|
|
183
206
|
if (!entry) return undefined;
|
|
184
|
-
if (entry.renamed_columns
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
if (!safe(
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
}
|
|
195
|
-
|
|
207
|
+
if (entry.renamed_columns && !entry.columns)
|
|
208
|
+
throw new FdlError(`${table} has renamed columns, and files.json does not list its columns`);
|
|
209
|
+
const key = (path: string) => {
|
|
210
|
+
if (!safe(path)) throw new FdlError(`${table} names a file outside the repository: ${path}`);
|
|
211
|
+
return version.config.data_dir + path;
|
|
212
|
+
};
|
|
213
|
+
return entry.files.map((f) => ({
|
|
214
|
+
key: key(f.path),
|
|
215
|
+
rows: f.rows,
|
|
216
|
+
bytes: f.bytes,
|
|
217
|
+
...(f.footer_bytes !== undefined ? { footerBytes: f.footer_bytes } : {}),
|
|
218
|
+
...(f.deletes
|
|
219
|
+
? { deletes: { key: key(f.deletes.path), rows: f.deletes.rows, bytes: f.deletes.bytes } }
|
|
220
|
+
: {}),
|
|
221
|
+
...(entry.columns ? { columns: entry.columns } : {}),
|
|
222
|
+
}));
|
|
196
223
|
};
|
|
197
224
|
|
|
198
225
|
export type RefreshingOptions = {
|