@nteract/sift 0.6.0 → 0.7.0-nightly.42413490472
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/docs/embedding-guide.md +417 -10
- package/lib/THIRD_PARTY_NOTICES.md +29 -4
- package/lib/handoff.css +12 -1
- package/lib/index.js +2847 -2052
- package/lib/protocol-B_bRY_76.js +885 -0
- package/lib/style.css +1 -1
- package/lib/types/packages/sift/src/auto-width.d.ts +9 -3
- package/lib/types/packages/sift/src/components/icons.d.ts +16 -0
- package/lib/types/packages/sift/src/file-data.d.ts +34 -5
- package/lib/types/packages/sift/src/filter-schema.d.ts +11 -0
- package/lib/types/packages/sift/src/index.d.ts +11 -4
- package/lib/types/packages/sift/src/parquet-file.d.ts +52 -5
- package/lib/types/packages/sift/src/permutation.d.ts +24 -0
- package/lib/types/packages/sift/src/predicate.d.ts +56 -3
- package/lib/types/packages/sift/src/query.d.ts +100 -0
- package/lib/types/packages/sift/src/react.d.ts +108 -6
- package/lib/types/packages/sift/src/table.d.ts +168 -2
- package/lib/types/packages/sift/src/wasm-summaries.d.ts +19 -1
- package/lib/types/packages/sift/src/windowed-resident.d.ts +20 -0
- package/lib/types/packages/sift/src/windowed.d.ts +116 -0
- package/lib/types/packages/sift/src/worker/client.d.ts +87 -0
- package/lib/types/packages/sift/src/worker/entry.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/factory.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/host.d.ts +31 -0
- package/lib/types/packages/sift/src/worker/protocol.d.ts +191 -0
- package/lib/wasm/sift_wasm.d.ts +67 -3
- package/lib/wasm/sift_wasm.js +109 -3
- package/lib/wasm/sift_wasm_bg.wasm +0 -0
- package/lib/worker.js +277 -0
- package/package.json +9 -6
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wire protocol between the view (main thread) and the data side running in a
|
|
3
|
+
* worker. Internal: both halves ship in this package and are versioned
|
|
4
|
+
* together. Typed arrays and image bytes cross with transfer lists; strings,
|
|
5
|
+
* filters (including `Set`s), and summaries are structured-cloned.
|
|
6
|
+
*
|
|
7
|
+
* The worker is `createResidentWindowedData` behind a `MessagePort`: it owns the
|
|
8
|
+
* WASM module, the Parquet file, the permutation, and the summaries. The main
|
|
9
|
+
* thread holds a `WindowedTableData` whose methods post these messages. The
|
|
10
|
+
* host's `ByteSource` cannot cross (functions do not clone), so reads are
|
|
11
|
+
* relayed: the worker asks, the main thread answers with transferred bytes.
|
|
12
|
+
*/
|
|
13
|
+
import type { ParquetFileLimits } from "../parquet-file";
|
|
14
|
+
import type { Column, ColumnFileStats, ColumnType, DatasetCoverage } from "../table";
|
|
15
|
+
import type { TableLimitOverflow } from "../file-data";
|
|
16
|
+
import type { DataEvent, QueryResult, ViewQuery, WindowCells } from "../windowed";
|
|
17
|
+
export declare const PROTOCOL_VERSION = 2;
|
|
18
|
+
/**
|
|
19
|
+
* Default worker allocation cap, as a multiple of `limits.maxDecodedBytes`.
|
|
20
|
+
* Decoding a row group transiently needs about twice the retained heap, so a
|
|
21
|
+
* cap at 2× can trap on the last row group a truncating budget admits; 2.5×
|
|
22
|
+
* leaves the budget, not the cap, as what ends a large preview.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DEFAULT_ALLOCATION_CAP_FACTOR = 2.5;
|
|
25
|
+
export type OpenRequest = {
|
|
26
|
+
t: "open";
|
|
27
|
+
id: number;
|
|
28
|
+
protocol: typeof PROTOCOL_VERSION;
|
|
29
|
+
source: {
|
|
30
|
+
kind: "byte-source-relay";
|
|
31
|
+
byteLength: number;
|
|
32
|
+
};
|
|
33
|
+
limits: ParquetFileLimits;
|
|
34
|
+
overflow?: TableLimitOverflow;
|
|
35
|
+
columnOverrides?: Record<string, Partial<Column>>;
|
|
36
|
+
headRows: number;
|
|
37
|
+
wasmUrl?: string;
|
|
38
|
+
wasmFallbackUrl?: string;
|
|
39
|
+
/**
|
|
40
|
+
* Live WASM heap bytes allowed; defaults to `limits.maxDecodedBytes *
|
|
41
|
+
* DEFAULT_ALLOCATION_CAP_FACTOR`.
|
|
42
|
+
*/
|
|
43
|
+
allocationCap?: number;
|
|
44
|
+
/**
|
|
45
|
+
* Soft budget: pause once the committed rows or decoded bytes reach it while
|
|
46
|
+
* row groups remain, and wait for `resume`. Each resume extends it by the
|
|
47
|
+
* same amounts. What triggers a resume is the main thread's concern.
|
|
48
|
+
*/
|
|
49
|
+
autoLoad?: {
|
|
50
|
+
rows?: number;
|
|
51
|
+
decodedBytes?: number;
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
export type ToWorker = OpenRequest | {
|
|
55
|
+
t: "window";
|
|
56
|
+
id: number;
|
|
57
|
+
gen: number;
|
|
58
|
+
viewStart: number;
|
|
59
|
+
viewEnd: number;
|
|
60
|
+
} | {
|
|
61
|
+
t: "query";
|
|
62
|
+
id: number;
|
|
63
|
+
gen: number;
|
|
64
|
+
query: ViewQuery;
|
|
65
|
+
} | {
|
|
66
|
+
t: "image";
|
|
67
|
+
id: number;
|
|
68
|
+
dataRow: number;
|
|
69
|
+
col: number;
|
|
70
|
+
} | {
|
|
71
|
+
t: "cast";
|
|
72
|
+
id: number;
|
|
73
|
+
gen: number;
|
|
74
|
+
col: number;
|
|
75
|
+
to: ColumnType;
|
|
76
|
+
} | {
|
|
77
|
+
t: "undo-cast";
|
|
78
|
+
id: number;
|
|
79
|
+
gen: number;
|
|
80
|
+
col: number;
|
|
81
|
+
}
|
|
82
|
+
/** Answer to a `read` relay; `bytes.buffer` is transferred. */
|
|
83
|
+
| {
|
|
84
|
+
t: "read-result";
|
|
85
|
+
id: number;
|
|
86
|
+
bytes: Uint8Array;
|
|
87
|
+
} | {
|
|
88
|
+
t: "read-error";
|
|
89
|
+
id: number;
|
|
90
|
+
message: string;
|
|
91
|
+
}
|
|
92
|
+
/** Continue a load paused at the `autoLoad` budget. No-op unless paused. */
|
|
93
|
+
| {
|
|
94
|
+
t: "resume";
|
|
95
|
+
} | {
|
|
96
|
+
t: "close";
|
|
97
|
+
};
|
|
98
|
+
export type OpenedReply = {
|
|
99
|
+
t: "opened";
|
|
100
|
+
id: number;
|
|
101
|
+
columns: Column[];
|
|
102
|
+
rowCount: number;
|
|
103
|
+
coverage?: DatasetCoverage;
|
|
104
|
+
datasetStats?: (ColumnFileStats | null)[];
|
|
105
|
+
head: WindowCells;
|
|
106
|
+
rowGroupCount: number;
|
|
107
|
+
};
|
|
108
|
+
/** Load progress after a row group; `complete` is the last one. */
|
|
109
|
+
export type ProgressReply = {
|
|
110
|
+
t: "progress";
|
|
111
|
+
phase: "chunk" | "complete";
|
|
112
|
+
rowGroup: number;
|
|
113
|
+
rowGroupCount: number;
|
|
114
|
+
loadedRows: number;
|
|
115
|
+
totalRows: number;
|
|
116
|
+
truncated: boolean;
|
|
117
|
+
/** Live heap bytes when the allocator reports them, else linear memory size. */
|
|
118
|
+
memoryBytes: number;
|
|
119
|
+
};
|
|
120
|
+
export type FromWorker = OpenedReply
|
|
121
|
+
/** Typed arrays transferred. */
|
|
122
|
+
| {
|
|
123
|
+
t: "window";
|
|
124
|
+
id: number;
|
|
125
|
+
cells: WindowCells;
|
|
126
|
+
} | {
|
|
127
|
+
t: "query";
|
|
128
|
+
id: number;
|
|
129
|
+
result: QueryResult;
|
|
130
|
+
}
|
|
131
|
+
/** Chunk buffers transferred. */
|
|
132
|
+
| {
|
|
133
|
+
t: "image";
|
|
134
|
+
id: number;
|
|
135
|
+
chunks: Uint8Array[] | null;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Cast or undo answer. `head` is recomputed under the new types for the same
|
|
139
|
+
* `headRows` span. Cloned, not transferred: the host keeps its copy.
|
|
140
|
+
*/
|
|
141
|
+
| {
|
|
142
|
+
t: "cast";
|
|
143
|
+
id: number;
|
|
144
|
+
result: QueryResult & {
|
|
145
|
+
columns: Column[];
|
|
146
|
+
};
|
|
147
|
+
head: WindowCells;
|
|
148
|
+
}
|
|
149
|
+
/** Relay request: read `length` bytes at `offset` from the host's `ByteSource`. */
|
|
150
|
+
| {
|
|
151
|
+
t: "read";
|
|
152
|
+
id: number;
|
|
153
|
+
offset: number;
|
|
154
|
+
length: number;
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Push from the data side. A `schema-changed` event carries the refreshed
|
|
158
|
+
* head. A load failure after `opened` is an `error` event with `fatal:
|
|
159
|
+
* false`: the rows so far stay queryable and no `streaming-done` follows.
|
|
160
|
+
*/
|
|
161
|
+
| {
|
|
162
|
+
t: "event";
|
|
163
|
+
event: DataEvent;
|
|
164
|
+
head?: WindowCells;
|
|
165
|
+
} | ProgressReply
|
|
166
|
+
/** `fatal` means the WASM instance is unusable; the worker stops answering. */
|
|
167
|
+
| {
|
|
168
|
+
t: "error";
|
|
169
|
+
id: number | null;
|
|
170
|
+
message: string;
|
|
171
|
+
fatal: boolean;
|
|
172
|
+
}
|
|
173
|
+
/** Ack for `close`; the main thread terminates on it or after a timeout. */
|
|
174
|
+
| {
|
|
175
|
+
t: "closed";
|
|
176
|
+
};
|
|
177
|
+
/** Buffers to transfer for `cells`. Shared buffers are copied first by the sender. */
|
|
178
|
+
export declare function windowCellsTransfer(cells: WindowCells): ArrayBuffer[];
|
|
179
|
+
/**
|
|
180
|
+
* A `Uint8Array` that owns its whole buffer, so transferring the buffer sends
|
|
181
|
+
* exactly these bytes. Copies when the view is a slice or the buffer is shared.
|
|
182
|
+
*/
|
|
183
|
+
export declare function ownedBytes(bytes: Uint8Array): Uint8Array<ArrayBuffer>;
|
|
184
|
+
/**
|
|
185
|
+
* Whether `error` means the WASM instance trapped or refused an allocation.
|
|
186
|
+
* Over-cap allocations requested from JS (wasm-bindgen's malloc for strings and
|
|
187
|
+
* byte arrays) trap as `WebAssembly.RuntimeError` in release builds but throw a
|
|
188
|
+
* plain `Error("invalid malloc request")` in debug builds; over-cap inside Rust
|
|
189
|
+
* always aborts. Both leave the instance unusable.
|
|
190
|
+
*/
|
|
191
|
+
export declare function isFatalWasmError(error: unknown): boolean;
|
package/lib/wasm/sift_wasm.d.ts
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
/* tslint:disable */
|
|
2
2
|
/* eslint-disable */
|
|
3
3
|
|
|
4
|
+
/**
|
|
5
|
+
* Live heap bytes right now.
|
|
6
|
+
*/
|
|
7
|
+
export function allocated_bytes(): number;
|
|
8
|
+
|
|
4
9
|
/**
|
|
5
10
|
* Append one self-contained Arrow IPC stream chunk into an existing store.
|
|
6
11
|
* The first chunk initializes the store schema; later chunks must match it.
|
|
@@ -179,6 +184,8 @@ export function num_rows(handle: number): number;
|
|
|
179
184
|
* ranges in the same order. Ranges may be coalesced supersets of the ones
|
|
180
185
|
* returned by `parquet_row_group_ranges`, but every column chunk of the row
|
|
181
186
|
* group must lie inside a single range. Returns the number of rows appended.
|
|
187
|
+
* A truncating load may append only part of the row group; check
|
|
188
|
+
* `parquet_file_budget` afterwards.
|
|
182
189
|
*/
|
|
183
190
|
export function parquet_append_row_group(file: number, store: number, row_group: number, ranges: Float64Array, data: Uint8Array): number;
|
|
184
191
|
|
|
@@ -199,17 +206,41 @@ export function parquet_column_hints(parquet_bytes: Uint8Array): any;
|
|
|
199
206
|
*/
|
|
200
207
|
export function parquet_create_store(file: number): number;
|
|
201
208
|
|
|
209
|
+
/**
|
|
210
|
+
* Budget state of an open file as `{ exhausted, truncated }`: `exhausted`
|
|
211
|
+
* once a truncating load can take no more rows (stop appending and finish
|
|
212
|
+
* the store), `truncated` once it stopped before the footer's last row. Both
|
|
213
|
+
* stay false under the default error policy.
|
|
214
|
+
*/
|
|
215
|
+
export function parquet_file_budget(file: number): any;
|
|
216
|
+
|
|
202
217
|
/**
|
|
203
218
|
* Canonical column hints (pandas index, HF features) from the open file's
|
|
204
219
|
* footer metadata. Same output as `parquet_column_hints` on the full file.
|
|
205
220
|
*/
|
|
206
221
|
export function parquet_file_column_hints(file: number): any;
|
|
207
222
|
|
|
223
|
+
/**
|
|
224
|
+
* Whole-file column statistics merged from the footer, one entry per
|
|
225
|
+
* top-level column: `{ num_rows, num_row_groups, columns: [{ null_count, min,
|
|
226
|
+
* max, bounded_row_groups, bounds_exact, has_infinite }] }`. Absent values
|
|
227
|
+
* are `null`. Numbers use the store's units (epoch milliseconds for temporal
|
|
228
|
+
* columns, scaled decimals). Computed at open, so it reads no bytes.
|
|
229
|
+
*/
|
|
230
|
+
export function parquet_file_column_stats(file: number): any;
|
|
231
|
+
|
|
208
232
|
/**
|
|
209
233
|
* Footer summary for an open file: row counts and top-level column names.
|
|
210
234
|
*/
|
|
211
235
|
export function parquet_file_info(file: number): any;
|
|
212
236
|
|
|
237
|
+
/**
|
|
238
|
+
* Rows and retained Arrow bytes committed to the store so far, as
|
|
239
|
+
* `{ rows, decoded_bytes }` JS numbers. Lets a host pause a load at a soft
|
|
240
|
+
* budget below the hard `max_decoded_bytes` without guessing at batch sizes.
|
|
241
|
+
*/
|
|
242
|
+
export function parquet_file_usage(file: number): any;
|
|
243
|
+
|
|
213
244
|
/**
|
|
214
245
|
* Metadata length encoded in the last 8 bytes of a Parquet file. Rejects a
|
|
215
246
|
* bad or encrypted magic before the host reads anything else. `tail` is the
|
|
@@ -228,9 +259,12 @@ export function parquet_metadata(parquet_bytes: Uint8Array): Uint32Array;
|
|
|
228
259
|
* contain at least `parquet_footer_length(tail) + 8` bytes. `file_len` is the
|
|
229
260
|
* full file length, used to validate column chunk ranges. Admission limits
|
|
230
261
|
* are checked from the footer here and cumulatively on every appended row
|
|
231
|
-
* group.
|
|
262
|
+
* group. With `truncate` set, rows and decoded bytes are a budget instead:
|
|
263
|
+
* the footer is not rejected for its row count or decoded-size floor, and
|
|
264
|
+
* appends stop once the budget is spent (see `parquet_file_budget`).
|
|
265
|
+
* Returns a file handle; free it with `parquet_close`.
|
|
232
266
|
*/
|
|
233
|
-
export function parquet_open(tail: Uint8Array, file_len: number, max_rows: number, max_columns: number, max_decoded_bytes: number): number;
|
|
267
|
+
export function parquet_open(tail: Uint8Array, file_len: number, max_rows: number, max_columns: number, max_decoded_bytes: number, truncate: boolean): number;
|
|
234
268
|
|
|
235
269
|
/**
|
|
236
270
|
* Byte ranges of every column chunk in a row group, flattened as
|
|
@@ -240,6 +274,20 @@ export function parquet_open(tail: Uint8Array, file_len: number, max_rows: numbe
|
|
|
240
274
|
*/
|
|
241
275
|
export function parquet_row_group_ranges(file: number, row_group: number): Float64Array;
|
|
242
276
|
|
|
277
|
+
/**
|
|
278
|
+
* Highest live heap bytes seen since the instance started.
|
|
279
|
+
*/
|
|
280
|
+
export function peak_allocated_bytes(): number;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Cap on live heap bytes for this instance; 0 removes the cap. A host sets
|
|
284
|
+
* it from its decoded-bytes budget plus headroom before loading anything.
|
|
285
|
+
* Allocations past the cap abort the instance (a `WebAssembly.RuntimeError`
|
|
286
|
+
* in the host). Lowering the cap below what is already live only blocks
|
|
287
|
+
* further growth.
|
|
288
|
+
*/
|
|
289
|
+
export function set_allocation_cap(bytes: number): void;
|
|
290
|
+
|
|
243
291
|
/**
|
|
244
292
|
* Count boolean values in a column: returns [true_count, false_count, null_count].
|
|
245
293
|
*/
|
|
@@ -275,6 +323,15 @@ export function store_filtered_value_counts(handle: number, col: number, mask: U
|
|
|
275
323
|
*/
|
|
276
324
|
export function store_histogram(handle: number, col: number, num_bins: number): any;
|
|
277
325
|
|
|
326
|
+
/**
|
|
327
|
+
* Null count per top-level column over resident rows. O(batches).
|
|
328
|
+
*
|
|
329
|
+
* Counts the current column, so a cast column reports the nulls the user
|
|
330
|
+
* sees after the cast. A `Null`-typed column counts every row, matching
|
|
331
|
+
* `cell_is_null`.
|
|
332
|
+
*/
|
|
333
|
+
export function store_null_counts(handle: number): Uint32Array;
|
|
334
|
+
|
|
278
335
|
/**
|
|
279
336
|
* Sort a column and return sorted row indices.
|
|
280
337
|
* `ascending`: true for asc, false for desc.
|
|
@@ -360,6 +417,7 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
|
|
|
360
417
|
|
|
361
418
|
export interface InitOutput {
|
|
362
419
|
readonly memory: WebAssembly.Memory;
|
|
420
|
+
readonly allocated_bytes: () => number;
|
|
363
421
|
readonly append_arrow_stream_chunk: (a: number, b: number, c: number) => [number, number];
|
|
364
422
|
readonly arrow_ipc_column_hints: (a: number, b: number) => [number, number, number];
|
|
365
423
|
readonly arrow_ipc_column_hints_with_row_count: (a: number, b: number, c: number) => [number, number, number];
|
|
@@ -393,18 +451,24 @@ export interface InitOutput {
|
|
|
393
451
|
readonly parquet_close: (a: number) => void;
|
|
394
452
|
readonly parquet_column_hints: (a: number, b: number) => [number, number, number];
|
|
395
453
|
readonly parquet_create_store: (a: number) => [number, number, number];
|
|
454
|
+
readonly parquet_file_budget: (a: number) => [number, number, number];
|
|
396
455
|
readonly parquet_file_column_hints: (a: number) => [number, number, number];
|
|
456
|
+
readonly parquet_file_column_stats: (a: number) => [number, number, number];
|
|
397
457
|
readonly parquet_file_info: (a: number) => [number, number, number];
|
|
458
|
+
readonly parquet_file_usage: (a: number) => [number, number, number];
|
|
398
459
|
readonly parquet_footer_length: (a: number, b: number) => [number, number, number];
|
|
399
460
|
readonly parquet_metadata: (a: number, b: number) => [number, number, number, number];
|
|
400
|
-
readonly parquet_open: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number, number];
|
|
461
|
+
readonly parquet_open: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number, number];
|
|
401
462
|
readonly parquet_row_group_ranges: (a: number, b: number) => [number, number, number, number];
|
|
463
|
+
readonly peak_allocated_bytes: () => number;
|
|
464
|
+
readonly set_allocation_cap: (a: number) => [number, number];
|
|
402
465
|
readonly store_bool_counts: (a: number, b: number) => [number, number, number, number];
|
|
403
466
|
readonly store_filter_rows: (a: number, b: any) => [number, number, number, number];
|
|
404
467
|
readonly store_filtered_bool_counts: (a: number, b: number, c: number, d: number) => [number, number, number, number];
|
|
405
468
|
readonly store_filtered_histogram: (a: number, b: number, c: number, d: number, e: number) => [number, number, number];
|
|
406
469
|
readonly store_filtered_value_counts: (a: number, b: number, c: number, d: number) => [number, number, number];
|
|
407
470
|
readonly store_histogram: (a: number, b: number, c: number) => [number, number, number];
|
|
471
|
+
readonly store_null_counts: (a: number) => [number, number, number, number];
|
|
408
472
|
readonly store_sort_indices: (a: number, b: number, c: number) => [number, number, number, number];
|
|
409
473
|
readonly store_temporal_histogram: (a: number, b: number) => [number, number, number];
|
|
410
474
|
readonly store_value_counts: (a: number, b: number) => [number, number, number];
|
package/lib/wasm/sift_wasm.js
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
/* @ts-self-types="./sift_wasm.d.ts" */
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Live heap bytes right now.
|
|
5
|
+
* @returns {number}
|
|
6
|
+
*/
|
|
7
|
+
export function allocated_bytes() {
|
|
8
|
+
const ret = wasm.allocated_bytes();
|
|
9
|
+
return ret;
|
|
10
|
+
}
|
|
11
|
+
|
|
3
12
|
/**
|
|
4
13
|
* Append one self-contained Arrow IPC stream chunk into an existing store.
|
|
5
14
|
* The first chunk initializes the store schema; later chunks must match it.
|
|
@@ -486,6 +495,8 @@ export function num_rows(handle) {
|
|
|
486
495
|
* ranges in the same order. Ranges may be coalesced supersets of the ones
|
|
487
496
|
* returned by `parquet_row_group_ranges`, but every column chunk of the row
|
|
488
497
|
* group must lie inside a single range. Returns the number of rows appended.
|
|
498
|
+
* A truncating load may append only part of the row group; check
|
|
499
|
+
* `parquet_file_budget` afterwards.
|
|
489
500
|
* @param {number} file
|
|
490
501
|
* @param {number} store
|
|
491
502
|
* @param {number} row_group
|
|
@@ -543,6 +554,22 @@ export function parquet_create_store(file) {
|
|
|
543
554
|
return ret[0] >>> 0;
|
|
544
555
|
}
|
|
545
556
|
|
|
557
|
+
/**
|
|
558
|
+
* Budget state of an open file as `{ exhausted, truncated }`: `exhausted`
|
|
559
|
+
* once a truncating load can take no more rows (stop appending and finish
|
|
560
|
+
* the store), `truncated` once it stopped before the footer's last row. Both
|
|
561
|
+
* stay false under the default error policy.
|
|
562
|
+
* @param {number} file
|
|
563
|
+
* @returns {any}
|
|
564
|
+
*/
|
|
565
|
+
export function parquet_file_budget(file) {
|
|
566
|
+
const ret = wasm.parquet_file_budget(file);
|
|
567
|
+
if (ret[2]) {
|
|
568
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
569
|
+
}
|
|
570
|
+
return takeFromExternrefTable0(ret[0]);
|
|
571
|
+
}
|
|
572
|
+
|
|
546
573
|
/**
|
|
547
574
|
* Canonical column hints (pandas index, HF features) from the open file's
|
|
548
575
|
* footer metadata. Same output as `parquet_column_hints` on the full file.
|
|
@@ -557,6 +584,23 @@ export function parquet_file_column_hints(file) {
|
|
|
557
584
|
return takeFromExternrefTable0(ret[0]);
|
|
558
585
|
}
|
|
559
586
|
|
|
587
|
+
/**
|
|
588
|
+
* Whole-file column statistics merged from the footer, one entry per
|
|
589
|
+
* top-level column: `{ num_rows, num_row_groups, columns: [{ null_count, min,
|
|
590
|
+
* max, bounded_row_groups, bounds_exact, has_infinite }] }`. Absent values
|
|
591
|
+
* are `null`. Numbers use the store's units (epoch milliseconds for temporal
|
|
592
|
+
* columns, scaled decimals). Computed at open, so it reads no bytes.
|
|
593
|
+
* @param {number} file
|
|
594
|
+
* @returns {any}
|
|
595
|
+
*/
|
|
596
|
+
export function parquet_file_column_stats(file) {
|
|
597
|
+
const ret = wasm.parquet_file_column_stats(file);
|
|
598
|
+
if (ret[2]) {
|
|
599
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
600
|
+
}
|
|
601
|
+
return takeFromExternrefTable0(ret[0]);
|
|
602
|
+
}
|
|
603
|
+
|
|
560
604
|
/**
|
|
561
605
|
* Footer summary for an open file: row counts and top-level column names.
|
|
562
606
|
* @param {number} file
|
|
@@ -570,6 +614,21 @@ export function parquet_file_info(file) {
|
|
|
570
614
|
return takeFromExternrefTable0(ret[0]);
|
|
571
615
|
}
|
|
572
616
|
|
|
617
|
+
/**
|
|
618
|
+
* Rows and retained Arrow bytes committed to the store so far, as
|
|
619
|
+
* `{ rows, decoded_bytes }` JS numbers. Lets a host pause a load at a soft
|
|
620
|
+
* budget below the hard `max_decoded_bytes` without guessing at batch sizes.
|
|
621
|
+
* @param {number} file
|
|
622
|
+
* @returns {any}
|
|
623
|
+
*/
|
|
624
|
+
export function parquet_file_usage(file) {
|
|
625
|
+
const ret = wasm.parquet_file_usage(file);
|
|
626
|
+
if (ret[2]) {
|
|
627
|
+
throw takeFromExternrefTable0(ret[1]);
|
|
628
|
+
}
|
|
629
|
+
return takeFromExternrefTable0(ret[0]);
|
|
630
|
+
}
|
|
631
|
+
|
|
573
632
|
/**
|
|
574
633
|
* Metadata length encoded in the last 8 bytes of a Parquet file. Rejects a
|
|
575
634
|
* bad or encrypted magic before the host reads anything else. `tail` is the
|
|
@@ -610,18 +669,22 @@ export function parquet_metadata(parquet_bytes) {
|
|
|
610
669
|
* contain at least `parquet_footer_length(tail) + 8` bytes. `file_len` is the
|
|
611
670
|
* full file length, used to validate column chunk ranges. Admission limits
|
|
612
671
|
* are checked from the footer here and cumulatively on every appended row
|
|
613
|
-
* group.
|
|
672
|
+
* group. With `truncate` set, rows and decoded bytes are a budget instead:
|
|
673
|
+
* the footer is not rejected for its row count or decoded-size floor, and
|
|
674
|
+
* appends stop once the budget is spent (see `parquet_file_budget`).
|
|
675
|
+
* Returns a file handle; free it with `parquet_close`.
|
|
614
676
|
* @param {Uint8Array} tail
|
|
615
677
|
* @param {number} file_len
|
|
616
678
|
* @param {number} max_rows
|
|
617
679
|
* @param {number} max_columns
|
|
618
680
|
* @param {number} max_decoded_bytes
|
|
681
|
+
* @param {boolean} truncate
|
|
619
682
|
* @returns {number}
|
|
620
683
|
*/
|
|
621
|
-
export function parquet_open(tail, file_len, max_rows, max_columns, max_decoded_bytes) {
|
|
684
|
+
export function parquet_open(tail, file_len, max_rows, max_columns, max_decoded_bytes, truncate) {
|
|
622
685
|
const ptr0 = passArray8ToWasm0(tail, wasm.__wbindgen_malloc);
|
|
623
686
|
const len0 = WASM_VECTOR_LEN;
|
|
624
|
-
const ret = wasm.parquet_open(ptr0, len0, file_len, max_rows, max_columns, max_decoded_bytes);
|
|
687
|
+
const ret = wasm.parquet_open(ptr0, len0, file_len, max_rows, max_columns, max_decoded_bytes, truncate);
|
|
625
688
|
if (ret[2]) {
|
|
626
689
|
throw takeFromExternrefTable0(ret[1]);
|
|
627
690
|
}
|
|
@@ -647,6 +710,30 @@ export function parquet_row_group_ranges(file, row_group) {
|
|
|
647
710
|
return v1;
|
|
648
711
|
}
|
|
649
712
|
|
|
713
|
+
/**
|
|
714
|
+
* Highest live heap bytes seen since the instance started.
|
|
715
|
+
* @returns {number}
|
|
716
|
+
*/
|
|
717
|
+
export function peak_allocated_bytes() {
|
|
718
|
+
const ret = wasm.peak_allocated_bytes();
|
|
719
|
+
return ret;
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* Cap on live heap bytes for this instance; 0 removes the cap. A host sets
|
|
724
|
+
* it from its decoded-bytes budget plus headroom before loading anything.
|
|
725
|
+
* Allocations past the cap abort the instance (a `WebAssembly.RuntimeError`
|
|
726
|
+
* in the host). Lowering the cap below what is already live only blocks
|
|
727
|
+
* further growth.
|
|
728
|
+
* @param {number} bytes
|
|
729
|
+
*/
|
|
730
|
+
export function set_allocation_cap(bytes) {
|
|
731
|
+
const ret = wasm.set_allocation_cap(bytes);
|
|
732
|
+
if (ret[1]) {
|
|
733
|
+
throw takeFromExternrefTable0(ret[0]);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
650
737
|
/**
|
|
651
738
|
* Count boolean values in a column: returns [true_count, false_count, null_count].
|
|
652
739
|
* @param {number} handle
|
|
@@ -753,6 +840,25 @@ export function store_histogram(handle, col, num_bins) {
|
|
|
753
840
|
return takeFromExternrefTable0(ret[0]);
|
|
754
841
|
}
|
|
755
842
|
|
|
843
|
+
/**
|
|
844
|
+
* Null count per top-level column over resident rows. O(batches).
|
|
845
|
+
*
|
|
846
|
+
* Counts the current column, so a cast column reports the nulls the user
|
|
847
|
+
* sees after the cast. A `Null`-typed column counts every row, matching
|
|
848
|
+
* `cell_is_null`.
|
|
849
|
+
* @param {number} handle
|
|
850
|
+
* @returns {Uint32Array}
|
|
851
|
+
*/
|
|
852
|
+
export function store_null_counts(handle) {
|
|
853
|
+
const ret = wasm.store_null_counts(handle);
|
|
854
|
+
if (ret[3]) {
|
|
855
|
+
throw takeFromExternrefTable0(ret[2]);
|
|
856
|
+
}
|
|
857
|
+
var v1 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
|
|
858
|
+
wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
|
|
859
|
+
return v1;
|
|
860
|
+
}
|
|
861
|
+
|
|
756
862
|
/**
|
|
757
863
|
* Sort a column and return sorted row indices.
|
|
758
864
|
* `ascending`: true for asc, false for desc.
|
|
Binary file
|