@nteract/sift 0.6.0 → 0.7.0-nightly.42433965924
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/docs/embedding-guide.md +417 -10
- package/lib/THIRD_PARTY_NOTICES.md +29 -4
- package/lib/handoff.css +12 -1
- package/lib/index.js +2847 -2052
- package/lib/protocol-B_bRY_76.js +885 -0
- package/lib/style.css +1 -1
- package/lib/types/packages/sift/src/auto-width.d.ts +9 -3
- package/lib/types/packages/sift/src/components/icons.d.ts +16 -0
- package/lib/types/packages/sift/src/file-data.d.ts +34 -5
- package/lib/types/packages/sift/src/filter-schema.d.ts +11 -0
- package/lib/types/packages/sift/src/index.d.ts +11 -4
- package/lib/types/packages/sift/src/parquet-file.d.ts +52 -5
- package/lib/types/packages/sift/src/permutation.d.ts +24 -0
- package/lib/types/packages/sift/src/predicate.d.ts +56 -3
- package/lib/types/packages/sift/src/query.d.ts +100 -0
- package/lib/types/packages/sift/src/react.d.ts +108 -6
- package/lib/types/packages/sift/src/table.d.ts +168 -2
- package/lib/types/packages/sift/src/wasm-summaries.d.ts +19 -1
- package/lib/types/packages/sift/src/windowed-resident.d.ts +20 -0
- package/lib/types/packages/sift/src/windowed.d.ts +116 -0
- package/lib/types/packages/sift/src/worker/client.d.ts +87 -0
- package/lib/types/packages/sift/src/worker/entry.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/factory.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/host.d.ts +31 -0
- package/lib/types/packages/sift/src/worker/protocol.d.ts +191 -0
- package/lib/wasm/sift_wasm.d.ts +67 -3
- package/lib/wasm/sift_wasm.js +109 -3
- package/lib/wasm/sift_wasm_bg.wasm +0 -0
- package/lib/worker.js +277 -0
- package/package.json +9 -6
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Byte-range Parquet loading.
|
|
3
|
+
*
|
|
4
|
+
* The host owns the file and answers `read(offset, length)`; Sift reads the
|
|
5
|
+
* footer, then fetches and decodes one row group at a time. The whole file is
|
|
6
|
+
* never buffered in JS or WASM memory. Decoded row groups append to one WASM
|
|
7
|
+
* store, so the table can mount after the first group and grow as the rest
|
|
8
|
+
* arrive.
|
|
9
|
+
*
|
|
10
|
+
* With `overflow: "truncate"`, rows and decoded bytes are a budget: loading
|
|
11
|
+
* stops at the first batch that does not fit, and the store holds a bounded
|
|
12
|
+
* prefix of the file.
|
|
13
|
+
*/
|
|
14
|
+
import type { TableLimitOverflow } from "./file-data";
|
|
1
15
|
import type { Column, TableData } from "./table";
|
|
2
16
|
/**
|
|
3
17
|
* Random-access bytes. `read` resolves with exactly `length` bytes starting at
|
|
@@ -19,6 +33,8 @@ export type ParquetFileLimits = {
|
|
|
19
33
|
};
|
|
20
34
|
export type OpenParquetFileOptions = {
|
|
21
35
|
limits: ParquetFileLimits;
|
|
36
|
+
/** What to do when the rows or decoded-bytes budget runs out. Default `"error"`. */
|
|
37
|
+
overflow?: TableLimitOverflow;
|
|
22
38
|
columnOverrides?: Record<string, Partial<Column>>;
|
|
23
39
|
signal?: AbortSignal;
|
|
24
40
|
};
|
|
@@ -33,6 +49,19 @@ export type ParquetAppendResult = {
|
|
|
33
49
|
*/
|
|
34
50
|
replaced: TableData | null;
|
|
35
51
|
};
|
|
52
|
+
/** Result of appending one row group. */
|
|
53
|
+
export type ParquetRowGroupResult = ParquetAppendResult & {
|
|
54
|
+
/**
|
|
55
|
+
* The budget is spent, possibly partway through this row group. Stop
|
|
56
|
+
* appending and call `finish()`. Always false with `overflow: "error"`.
|
|
57
|
+
*/
|
|
58
|
+
exhausted: boolean;
|
|
59
|
+
};
|
|
60
|
+
/** Rows and retained Arrow bytes an open file has committed to its store. */
|
|
61
|
+
export type ParquetFileUsage = {
|
|
62
|
+
rows: number;
|
|
63
|
+
decodedBytes: number;
|
|
64
|
+
};
|
|
36
65
|
export type OpenParquetFile = {
|
|
37
66
|
/** Row count from the footer. */
|
|
38
67
|
readonly rowCount: number;
|
|
@@ -44,12 +73,23 @@ export type OpenParquetFile = {
|
|
|
44
73
|
readonly columns: Column[];
|
|
45
74
|
/** Rows resident in the store so far. */
|
|
46
75
|
readonly loadedRows: number;
|
|
76
|
+
/** True once the budget stopped loading before the last row in the file. */
|
|
77
|
+
readonly truncated: boolean;
|
|
78
|
+
/** Rows and retained Arrow bytes committed to the store so far. */
|
|
79
|
+
readonly usage: ParquetFileUsage;
|
|
80
|
+
/**
|
|
81
|
+
* Mark the load as paused at a soft budget, with row groups still unread.
|
|
82
|
+
* While paused, `tableData.coverage` reports `truncated` and `resumable`.
|
|
83
|
+
* The next append or `finish()` clears it.
|
|
84
|
+
*/
|
|
85
|
+
setPaused(paused: boolean): void;
|
|
47
86
|
/**
|
|
48
87
|
* Fetch the row group's column chunks and decode them into the store.
|
|
49
88
|
* Append row groups in file order, one at a time. Updates `tableData`'s
|
|
50
|
-
* row count and summaries before resolving.
|
|
89
|
+
* row count and summaries before resolving. Stop once a result reports
|
|
90
|
+
* `exhausted`; later calls reject.
|
|
51
91
|
*/
|
|
52
|
-
appendRowGroup(index: number): Promise<
|
|
92
|
+
appendRowGroup(index: number): Promise<ParquetRowGroupResult>;
|
|
53
93
|
/** Mark the store complete and release footer metadata. Call after the last append. */
|
|
54
94
|
finish(): ParquetAppendResult;
|
|
55
95
|
/** Release footer metadata only. The store and its leases stay valid. */
|
|
@@ -69,12 +109,19 @@ type Range = {
|
|
|
69
109
|
offset: number;
|
|
70
110
|
length: number;
|
|
71
111
|
};
|
|
72
|
-
/**
|
|
73
|
-
|
|
112
|
+
/**
|
|
113
|
+
* Merge column chunk ranges that touch or sit within `gap` bytes. Bridging a
|
|
114
|
+
* gap fetches its bytes too, so a gap is bridged only while the total bytes
|
|
115
|
+
* requested stay within `maxBytes`. Touching and overlapping chunks always
|
|
116
|
+
* merge; they add nothing to the request.
|
|
117
|
+
*/
|
|
118
|
+
export declare function coalesceRanges(flat: ArrayLike<number>, gap?: number, maxBytes?: number): Range[];
|
|
74
119
|
/**
|
|
75
120
|
* Read the footer, validate it, and prepare an empty store with the file's
|
|
76
121
|
* schema. Rejects on bad magic before any other read. Admission limits are
|
|
77
|
-
* checked from the footer here and again as each row group is appended.
|
|
122
|
+
* checked from the footer here and again as each row group is appended. With
|
|
123
|
+
* `overflow: "truncate"` the footer is checked only for columns and footer
|
|
124
|
+
* size, and appends stop when the rows or decoded-bytes budget runs out.
|
|
78
125
|
*/
|
|
79
126
|
export declare function openParquetFile(source: ByteSource, options: OpenParquetFileOptions): Promise<OpenParquetFile>;
|
|
80
127
|
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The view permutation: which data rows pass the column filters, in sort order.
|
|
3
|
+
*
|
|
4
|
+
* Shared by the table engine (resident mode) and the resident windowed adapter,
|
|
5
|
+
* so both compute the same `dataRows` for the same filters and sort. WASM
|
|
6
|
+
* sources supply `filterRows` and `sortColumn`; the JS fallback reads cells.
|
|
7
|
+
*/
|
|
8
|
+
import type { Column, ColumnFilter, TableData } from "./table";
|
|
9
|
+
type CellReader = Pick<TableData, "getCell" | "getCellRaw">;
|
|
10
|
+
export type SortSpec = {
|
|
11
|
+
col: number;
|
|
12
|
+
dir: "asc" | "desc";
|
|
13
|
+
} | null;
|
|
14
|
+
/** Whether `dataRow` passes every filter. `filters` has one entry per column. */
|
|
15
|
+
export declare function rowPassesFilters(data: CellReader, filters: (ColumnFilter | null)[], dataRow: number): boolean;
|
|
16
|
+
/** JS fallback comparator for one column; nulls and non-finite values sort last. */
|
|
17
|
+
export declare function compareDataRows(data: CellReader, columns: Column[], col: number, dir: "asc" | "desc"): (a: number, b: number) => number;
|
|
18
|
+
/**
|
|
19
|
+
* Data rows passing `filters`, in `sort` order. Uses the source's `filterRows`
|
|
20
|
+
* and `sortColumn` when it has them (WASM) and intersects the sorted order
|
|
21
|
+
* with the filter set, else filters and sorts by reading cells.
|
|
22
|
+
*/
|
|
23
|
+
export declare function computePermutation(data: TableData, columns: Column[], filters: (ColumnFilter | null)[], sort: SortSpec, rowCount: number): number[];
|
|
24
|
+
export {};
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* initial page load for users who don't need compute operations.
|
|
6
6
|
*/
|
|
7
7
|
import type { ParquetColumnHint } from "./parquet-features";
|
|
8
|
+
import type { ColumnFileStats } from "./table";
|
|
8
9
|
/** Filter spec passed to WASM store_filter_rows. */
|
|
9
10
|
export type FilterSpecJson = {
|
|
10
11
|
kind: "range";
|
|
@@ -37,7 +38,29 @@ export type ParquetFileInfo = {
|
|
|
37
38
|
row_group_rows: number[];
|
|
38
39
|
column_names: string[];
|
|
39
40
|
};
|
|
40
|
-
|
|
41
|
+
/** One column's footer statistics, as serialized by `parquet_file_column_stats`. */
|
|
42
|
+
export type ColumnFileStatsJson = {
|
|
43
|
+
null_count: number | null;
|
|
44
|
+
min: number | string | boolean | null;
|
|
45
|
+
max: number | string | boolean | null;
|
|
46
|
+
bounded_row_groups: number;
|
|
47
|
+
bounds_exact: boolean;
|
|
48
|
+
has_infinite: boolean;
|
|
49
|
+
};
|
|
50
|
+
/** Footer statistics returned by `parquet_file_column_stats`. */
|
|
51
|
+
export type ParquetFileStatsJson = {
|
|
52
|
+
num_rows: number;
|
|
53
|
+
num_row_groups: number;
|
|
54
|
+
/** One per top-level column, in schema order. */
|
|
55
|
+
columns: ColumnFileStatsJson[];
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Map the snake_case wire shape onto `ColumnFileStats`. A column whose stats
|
|
59
|
+
* are missing or malformed maps to `null` so a bad footer entry degrades to
|
|
60
|
+
* "no footer stats" rather than failing the open.
|
|
61
|
+
*/
|
|
62
|
+
export declare function columnFileStatsFromWasm(stats: unknown, numCols: number): (ColumnFileStats | null)[];
|
|
63
|
+
export type PredicateModule = {
|
|
41
64
|
load_ipc(ipc_bytes: Uint8Array): number;
|
|
42
65
|
load_ipc_bounded(ipc_bytes: Uint8Array, max_rows: number, max_columns: number, max_decoded_bytes: number): number;
|
|
43
66
|
create_arrow_stream_store(): number;
|
|
@@ -51,13 +74,28 @@ type PredicateModule = {
|
|
|
51
74
|
arrow_ipc_column_hints_with_row_count(ipc_bytes: Uint8Array, total_rows: number): ParquetColumnHint[];
|
|
52
75
|
load_parquet_row_group(parquet_bytes: Uint8Array, row_group: number, handle: number): number;
|
|
53
76
|
parquet_footer_length(tail: Uint8Array): number;
|
|
54
|
-
parquet_open(tail: Uint8Array, file_len: number, max_rows: number, max_columns: number, max_decoded_bytes: number): number;
|
|
77
|
+
parquet_open(tail: Uint8Array, file_len: number, max_rows: number, max_columns: number, max_decoded_bytes: number, truncate: boolean): number;
|
|
55
78
|
parquet_file_info(file: number): ParquetFileInfo;
|
|
79
|
+
/** Per-column footer statistics, computed at open. See `ParquetFileStatsJson`. */
|
|
80
|
+
parquet_file_column_stats(file: number): unknown;
|
|
81
|
+
/** Budget state of a truncating load. Both flags stay false in error mode. */
|
|
82
|
+
parquet_file_budget(file: number): {
|
|
83
|
+
exhausted: boolean;
|
|
84
|
+
truncated: boolean;
|
|
85
|
+
};
|
|
86
|
+
/** Rows and retained Arrow bytes committed to the store so far. */
|
|
87
|
+
parquet_file_usage(file: number): {
|
|
88
|
+
rows: number;
|
|
89
|
+
decoded_bytes: number;
|
|
90
|
+
};
|
|
56
91
|
parquet_file_column_hints(file: number): ParquetColumnHint[];
|
|
57
92
|
parquet_create_store(file: number): number;
|
|
58
93
|
/** Flat `[offset, length, ...]` per column chunk of one row group. */
|
|
59
94
|
parquet_row_group_ranges(file: number, row_group: number): Float64Array;
|
|
60
|
-
/**
|
|
95
|
+
/**
|
|
96
|
+
* Decode one row group from `data`, the concatenation of `ranges`. Returns
|
|
97
|
+
* rows appended; a truncating load may append only part of the group.
|
|
98
|
+
*/
|
|
61
99
|
parquet_append_row_group(file: number, store: number, row_group: number, ranges: Float64Array, data: Uint8Array): number;
|
|
62
100
|
parquet_close(file: number): void;
|
|
63
101
|
cast_column(handle: number, col: number, target_type: string): void;
|
|
@@ -86,6 +124,14 @@ type PredicateModule = {
|
|
|
86
124
|
get_cell_image_bytes_at(handle: number, row: number, col: number, idx: number): Uint8Array;
|
|
87
125
|
is_null(handle: number, row: number, col: number): boolean;
|
|
88
126
|
store_viewport_cells(handle: number, rows: Uint32Array): ViewportCells;
|
|
127
|
+
/** Arrow IPC → Parquet with at most `maxRowGroupRows` rows per row group. Demo and tests. */
|
|
128
|
+
write_parquet_ipc_row_groups(ipc: Uint8Array, maxRowGroupRows?: number | null): Uint8Array;
|
|
129
|
+
/** Cap on live heap bytes for this instance; 0 removes it. Over-cap allocations trap. */
|
|
130
|
+
set_allocation_cap?(bytes: number): void;
|
|
131
|
+
/** Live heap bytes right now. */
|
|
132
|
+
allocated_bytes?(): number;
|
|
133
|
+
/** Highest live heap bytes seen since the instance started. */
|
|
134
|
+
peak_allocated_bytes?(): number;
|
|
89
135
|
store_value_counts(handle: number, col: number): {
|
|
90
136
|
label: string;
|
|
91
137
|
count: number;
|
|
@@ -101,6 +147,8 @@ type PredicateModule = {
|
|
|
101
147
|
count: number;
|
|
102
148
|
}[];
|
|
103
149
|
store_bool_counts(handle: number, col: number): Uint32Array;
|
|
150
|
+
/** Null count per top-level column over resident rows, after casts. */
|
|
151
|
+
store_null_counts(handle: number): Uint32Array;
|
|
104
152
|
store_sort_indices(handle: number, col: number, ascending: boolean): Uint32Array;
|
|
105
153
|
store_filtered_value_counts(handle: number, col: number, mask: Uint8Array): {
|
|
106
154
|
label: string;
|
|
@@ -144,6 +192,11 @@ type SiftWasmModuleLoader = () => Promise<{
|
|
|
144
192
|
* loudly at instantiation, which is no worse than no fallback.
|
|
145
193
|
*/
|
|
146
194
|
export declare function setWasmUrl(url: string, fallbackUrl?: string): void;
|
|
195
|
+
/** The URL `setWasmUrl` configured, if any. A worker is told it at `open`. */
|
|
196
|
+
export declare function getWasmUrl(): string | undefined;
|
|
197
|
+
export declare function getWasmFallbackUrl(): string | undefined;
|
|
198
|
+
/** Linear memory of the initialized module, for size reporting between calls. */
|
|
199
|
+
export declare function getWasmMemory(): WebAssembly.Memory | null;
|
|
147
200
|
export declare function ensureModule(): Promise<PredicateModule>;
|
|
148
201
|
/**
|
|
149
202
|
* Swap the wasm module loader (the dynamic import is not interceptable in
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Query seam: the contract between Sift and a host-owned data provider.
|
|
3
|
+
*
|
|
4
|
+
* A provider answers a `TableQuery` with one bounded Arrow IPC stream. Sift
|
|
5
|
+
* loads that stream through its resident path and records that the rows
|
|
6
|
+
* already satisfy the query, so it does not filter them again locally. The
|
|
7
|
+
* host owns the SQL dialect, the base relation, the connection, credentials,
|
|
8
|
+
* and where the query runs; Sift never sees SQL text.
|
|
9
|
+
*/
|
|
10
|
+
import type { ColumnPredicate, FilterPredicate, SortEntry } from "./filter-schema";
|
|
11
|
+
import type { ColumnType } from "./table";
|
|
12
|
+
export type TableQuery = {
|
|
13
|
+
/** Applied before filters. At most one per column. */
|
|
14
|
+
casts: {
|
|
15
|
+
column: string;
|
|
16
|
+
to: ColumnType;
|
|
17
|
+
}[];
|
|
18
|
+
/** Top-level AND. */
|
|
19
|
+
filters: FilterPredicate[];
|
|
20
|
+
sort: SortEntry[];
|
|
21
|
+
};
|
|
22
|
+
export type Count = {
|
|
23
|
+
kind: "exact";
|
|
24
|
+
value: number;
|
|
25
|
+
} | {
|
|
26
|
+
kind: "estimate";
|
|
27
|
+
value: number;
|
|
28
|
+
method?: string;
|
|
29
|
+
} | {
|
|
30
|
+
kind: "lower-bound";
|
|
31
|
+
value: number;
|
|
32
|
+
} | {
|
|
33
|
+
kind: "unknown";
|
|
34
|
+
};
|
|
35
|
+
/** The only window mode this slice supports: the leading rows of the result. */
|
|
36
|
+
export type WindowRequest = {
|
|
37
|
+
mode: "head";
|
|
38
|
+
limit: number;
|
|
39
|
+
};
|
|
40
|
+
export type QueryPage = {
|
|
41
|
+
/** One complete uncompressed Arrow IPC stream; schema-only when empty. */
|
|
42
|
+
ipc: Uint8Array;
|
|
43
|
+
returnedRows: number;
|
|
44
|
+
/** Rows matching the query before the window. Never the base relation count. */
|
|
45
|
+
matchedCount: Count;
|
|
46
|
+
continuation: {
|
|
47
|
+
kind: "end";
|
|
48
|
+
} | {
|
|
49
|
+
kind: "more";
|
|
50
|
+
} | {
|
|
51
|
+
kind: "stopped";
|
|
52
|
+
reason: "budget" | "unsupported-resume";
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
export type QueryCapabilities = {
|
|
56
|
+
/** Leaf predicate ops supported per column name. Columns absent here accept no pushdown. */
|
|
57
|
+
filters: Record<string, readonly ColumnPredicate["op"][]>;
|
|
58
|
+
/** Sortable columns; `[]` means none. */
|
|
59
|
+
sort: readonly string[];
|
|
60
|
+
casts: readonly {
|
|
61
|
+
column: string;
|
|
62
|
+
to: ColumnType;
|
|
63
|
+
}[];
|
|
64
|
+
counts: readonly ("exact" | "estimate")[];
|
|
65
|
+
/**
|
|
66
|
+
* Compound predicates the provider accepts below the top-level AND. Absent
|
|
67
|
+
* means leaves only: Sift's category exclusion (`or(is_null, not_in)`) then
|
|
68
|
+
* cannot be pushed down.
|
|
69
|
+
*/
|
|
70
|
+
compound?: readonly ("and" | "or" | "not")[];
|
|
71
|
+
};
|
|
72
|
+
export type RequestContext = {
|
|
73
|
+
requestId: string;
|
|
74
|
+
signal: AbortSignal;
|
|
75
|
+
budget: {
|
|
76
|
+
maxIpcBytes: number;
|
|
77
|
+
maxDecodedBytes: number;
|
|
78
|
+
deadlineMs?: number;
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
export interface TableQuerySession {
|
|
82
|
+
readonly capabilities: QueryCapabilities;
|
|
83
|
+
query(q: TableQuery & {
|
|
84
|
+
window: WindowRequest;
|
|
85
|
+
}, ctx: RequestContext): Promise<QueryPage>;
|
|
86
|
+
count(q: TableQuery, mode: "cheap" | "exact", ctx: RequestContext): Promise<Count>;
|
|
87
|
+
close(): Promise<void>;
|
|
88
|
+
}
|
|
89
|
+
export type QuerySupport = {
|
|
90
|
+
ok: true;
|
|
91
|
+
} | {
|
|
92
|
+
ok: false;
|
|
93
|
+
reason: string;
|
|
94
|
+
};
|
|
95
|
+
/**
|
|
96
|
+
* Whether `caps` can answer `q` as written: every filter leaf's column and op,
|
|
97
|
+
* every compound node, every sort column, and every cast must be declared.
|
|
98
|
+
* Unsupported queries are refused here rather than narrowed silently.
|
|
99
|
+
*/
|
|
100
|
+
export declare function supportsQuery(caps: QueryCapabilities, q: TableQuery): QuerySupport;
|
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
* cleaning up on unmount.
|
|
17
17
|
*/
|
|
18
18
|
import { type ReactNode } from "react";
|
|
19
|
-
import { type
|
|
19
|
+
import { type TableFileLimits, type TableLimitOverflow } from "./file-data";
|
|
20
20
|
import { type ByteSource } from "./parquet-file";
|
|
21
|
+
import { type Count, type TableQuery, type TableQuerySession } from "./query";
|
|
21
22
|
import { type Column, type ColumnFilter, type ColumnType, type ReplaceDataOptions, type TableData, type TableEngine, type TableEngineState } from "./table";
|
|
22
23
|
export type ArrowStreamManifestChunk = {
|
|
23
24
|
url: string;
|
|
@@ -42,9 +43,22 @@ export type SiftSource = {
|
|
|
42
43
|
| {
|
|
43
44
|
kind: "loader";
|
|
44
45
|
load: () => Promise<TableData>;
|
|
45
|
-
}
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A hosted Arrow IPC or Parquet file; the format is detected from the
|
|
49
|
+
* response. Parquet streams row groups like `byte-source`, with `limits` and
|
|
50
|
+
* `overflow` defaulting to `URL_PARQUET_LIMITS` and `"truncate"`: a large file
|
|
51
|
+
* shows its leading rows and notes the preview in the footer. Arrow IPC is
|
|
52
|
+
* read whole and ignores both.
|
|
53
|
+
*/
|
|
54
|
+
| {
|
|
46
55
|
kind: "url";
|
|
47
56
|
url: string;
|
|
57
|
+
limits?: TableFileLimits;
|
|
58
|
+
overflow?: TableLimitOverflow;
|
|
59
|
+
autoLoad?: SiftAutoLoad;
|
|
60
|
+
/** Experimental: see the `byte-source` option. Parquet only. */
|
|
61
|
+
windowed?: boolean;
|
|
48
62
|
} | {
|
|
49
63
|
kind: "arrow-stream-manifest";
|
|
50
64
|
manifest: ArrowStreamManifest;
|
|
@@ -58,7 +72,71 @@ export type SiftSource = {
|
|
|
58
72
|
| {
|
|
59
73
|
kind: "byte-source";
|
|
60
74
|
source: ByteSource;
|
|
61
|
-
limits?:
|
|
75
|
+
limits?: TableFileLimits;
|
|
76
|
+
/**
|
|
77
|
+
* What to do when the rows or decoded-bytes budget runs out. Default
|
|
78
|
+
* `"error"`. With `"truncate"`, Sift shows the leading rows that fit
|
|
79
|
+
* and notes the preview in its footer.
|
|
80
|
+
*/
|
|
81
|
+
overflow?: TableLimitOverflow;
|
|
82
|
+
/** Pause below the hard budget and let the user load more. Needs `overflow: "truncate"`. */
|
|
83
|
+
autoLoad?: SiftAutoLoad;
|
|
84
|
+
/**
|
|
85
|
+
* Experimental: run the engine over a `WindowedTableData` adapter so cells
|
|
86
|
+
* and the permutation arrive asynchronously. Off by default; a seam for
|
|
87
|
+
* the worker-backed data side.
|
|
88
|
+
*/
|
|
89
|
+
windowed?: boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Experimental: decode and hold the file in the Sift data worker. Needs
|
|
92
|
+
* `setWorkerFactory`; without one the source fails at validation, it
|
|
93
|
+
* never falls back to the main thread. Implies `windowed`. `autoLoad`
|
|
94
|
+
* pauses and resumes the worker's load like the main-thread loader.
|
|
95
|
+
*/
|
|
96
|
+
worker?: boolean;
|
|
97
|
+
/** Live WASM heap bytes allowed in the worker; default `2.5 × maxDecodedBytes`. */
|
|
98
|
+
allocationCap?: number;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* A host-owned query provider. Sift asks it for the head of the result,
|
|
102
|
+
* shows those rows, and keeps brushes local until the user applies them to
|
|
103
|
+
* the whole dataset, which runs one new query. Memoize `session`; a new
|
|
104
|
+
* identity re-queries from `initialQuery`. The session is closed on unmount
|
|
105
|
+
* or source change.
|
|
106
|
+
*/
|
|
107
|
+
| {
|
|
108
|
+
kind: "query-session";
|
|
109
|
+
session: TableQuerySession;
|
|
110
|
+
/** Starting query; missing parts default to no casts, filters, or sort. */
|
|
111
|
+
initialQuery?: Partial<TableQuery>;
|
|
112
|
+
/** Decoded-memory and column limits for each result; `maxRows` is `headRows`. */
|
|
113
|
+
limits?: Partial<TableFileLimits>;
|
|
114
|
+
/** Rows requested per query. Default 10,000. */
|
|
115
|
+
headRows?: number;
|
|
116
|
+
};
|
|
117
|
+
/** What a `query-session` source has committed and whether a query is in flight. */
|
|
118
|
+
export type SiftQueryState = {
|
|
119
|
+
/** Increments per query; a response from an older generation is dropped. */
|
|
120
|
+
generation: number;
|
|
121
|
+
pending: boolean;
|
|
122
|
+
committedQuery: TableQuery;
|
|
123
|
+
resultCount: Count;
|
|
124
|
+
};
|
|
125
|
+
/**
|
|
126
|
+
* Pause a Parquet load at a soft budget below the hard `limits`; the user
|
|
127
|
+
* loads more on demand. Each resume extends the budget by the same amounts.
|
|
128
|
+
*/
|
|
129
|
+
export type SiftAutoLoad = {
|
|
130
|
+
/** Rows to load before pausing. Default: no pause. */
|
|
131
|
+
rows?: number;
|
|
132
|
+
/** Decoded Arrow bytes to load before pausing. Default: no pause. */
|
|
133
|
+
decodedBytes?: number;
|
|
134
|
+
/**
|
|
135
|
+
* What resumes loading. `"manual"`: the footer "Load more" button only.
|
|
136
|
+
* `"scroll"`: also when the viewport scrolls within the overscan of the last
|
|
137
|
+
* loaded row. Default `"manual"`.
|
|
138
|
+
*/
|
|
139
|
+
trigger?: "manual" | "scroll";
|
|
62
140
|
};
|
|
63
141
|
export type SiftLoadMilestone = {
|
|
64
142
|
source: SiftSource["kind"];
|
|
@@ -66,7 +144,17 @@ export type SiftLoadMilestone = {
|
|
|
66
144
|
elapsedMs: number;
|
|
67
145
|
chunkIndex?: number;
|
|
68
146
|
chunkCount?: number;
|
|
147
|
+
/** Rows loaded so far (resident rows, not the file total). */
|
|
69
148
|
rowCount?: number;
|
|
149
|
+
/** Footer row count of a Parquet file. Set on Parquet `streaming-complete`. */
|
|
150
|
+
totalRowCount?: number;
|
|
151
|
+
/** Whether the budget stopped loading early. Set on Parquet `streaming-complete`. */
|
|
152
|
+
truncated?: boolean;
|
|
153
|
+
/**
|
|
154
|
+
* Set on Parquet `streaming-complete`: true when the load paused at an
|
|
155
|
+
* `autoLoad` budget and more rows can be requested, false when it finished.
|
|
156
|
+
*/
|
|
157
|
+
resumable?: boolean;
|
|
70
158
|
byteLength?: number;
|
|
71
159
|
format?: "arrow-ipc" | "parquet";
|
|
72
160
|
};
|
|
@@ -87,10 +175,18 @@ export type SiftTableProps = {
|
|
|
87
175
|
onLoadMilestone?: (milestone: SiftLoadMilestone) => void;
|
|
88
176
|
/**
|
|
89
177
|
* Called when a `loader`, `url`, `byte-source` or manifest source fails to
|
|
90
|
-
* load
|
|
91
|
-
*
|
|
178
|
+
* load; this lets the host log or surface the failure without wrapping the
|
|
179
|
+
* loader. Sift shows its own error state too.
|
|
180
|
+
*
|
|
181
|
+
* A streaming Parquet source (`byte-source`, or a Parquet `url`) can fail
|
|
182
|
+
* after rows have mounted. Sift then keeps those rows, notes the failure in
|
|
183
|
+
* its footer instead of the blocking error box, and still calls this. A host
|
|
184
|
+
* can tell the two cases apart by whether it saw the `engine-mounted`
|
|
185
|
+
* milestone for this load.
|
|
92
186
|
*/
|
|
93
187
|
onLoadError?: (error: Error) => void;
|
|
188
|
+
/** Called as a `query-session` source starts and finishes each query. */
|
|
189
|
+
onQueryStateChange?: (state: SiftQueryState) => void;
|
|
94
190
|
/** Optional control rendered in Sift's footer before built-in buttons. */
|
|
95
191
|
footerControl?: ReactNode;
|
|
96
192
|
/** CSS class name for the container div. */
|
|
@@ -98,7 +194,13 @@ export type SiftTableProps = {
|
|
|
98
194
|
/** Inline styles for the container div. */
|
|
99
195
|
style?: React.CSSProperties;
|
|
100
196
|
};
|
|
101
|
-
|
|
197
|
+
/**
|
|
198
|
+
* Default budget for Parquet behind a `url` source. The response is already
|
|
199
|
+
* buffered, so these bound decoded memory, not the download. Hosts raise them
|
|
200
|
+
* through `limits`; `maxColumns` keeps the `loadTableFile` default.
|
|
201
|
+
*/
|
|
202
|
+
export declare const URL_PARQUET_LIMITS: Readonly<TableFileLimits>;
|
|
203
|
+
export declare function SiftTable({ source, data, url, typeOverrides, columnOverrides, onChange, onLoadMilestone, onLoadError, onQueryStateChange, footerControl, className, style, }: SiftTableProps): import("react").JSX.Element;
|
|
102
204
|
export type SiftTableHandle = {
|
|
103
205
|
engine: TableEngine | null;
|
|
104
206
|
setFilter: (colIndex: number, filter: ColumnFilter) => void;
|