@nteract/sift 0.6.0 → 0.7.0-nightly.42433965924
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/docs/embedding-guide.md +417 -10
- package/lib/THIRD_PARTY_NOTICES.md +29 -4
- package/lib/handoff.css +12 -1
- package/lib/index.js +2847 -2052
- package/lib/protocol-B_bRY_76.js +885 -0
- package/lib/style.css +1 -1
- package/lib/types/packages/sift/src/auto-width.d.ts +9 -3
- package/lib/types/packages/sift/src/components/icons.d.ts +16 -0
- package/lib/types/packages/sift/src/file-data.d.ts +34 -5
- package/lib/types/packages/sift/src/filter-schema.d.ts +11 -0
- package/lib/types/packages/sift/src/index.d.ts +11 -4
- package/lib/types/packages/sift/src/parquet-file.d.ts +52 -5
- package/lib/types/packages/sift/src/permutation.d.ts +24 -0
- package/lib/types/packages/sift/src/predicate.d.ts +56 -3
- package/lib/types/packages/sift/src/query.d.ts +100 -0
- package/lib/types/packages/sift/src/react.d.ts +108 -6
- package/lib/types/packages/sift/src/table.d.ts +168 -2
- package/lib/types/packages/sift/src/wasm-summaries.d.ts +19 -1
- package/lib/types/packages/sift/src/windowed-resident.d.ts +20 -0
- package/lib/types/packages/sift/src/windowed.d.ts +116 -0
- package/lib/types/packages/sift/src/worker/client.d.ts +87 -0
- package/lib/types/packages/sift/src/worker/entry.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/factory.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/host.d.ts +31 -0
- package/lib/types/packages/sift/src/worker/protocol.d.ts +191 -0
- package/lib/wasm/sift_wasm.d.ts +67 -3
- package/lib/wasm/sift_wasm.js +109 -3
- package/lib/wasm/sift_wasm_bg.wasm +0 -0
- package/lib/worker.js +277 -0
- package/package.json +9 -6
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { Count, TableQuery } from "./query";
|
|
2
|
+
import { type WindowedTableData } from "./windowed";
|
|
1
3
|
export type ColumnType = "numeric" | "categorical" | "timestamp" | "boolean" | "image";
|
|
2
4
|
export type Column = {
|
|
3
5
|
key: string;
|
|
@@ -8,6 +10,62 @@ export type Column = {
|
|
|
8
10
|
columnType: ColumnType;
|
|
9
11
|
timezone?: string | null;
|
|
10
12
|
};
|
|
13
|
+
/** Which rows a metric describes: the resident rows, or the whole dataset. */
|
|
14
|
+
export type DataScope = "loaded" | "dataset";
|
|
15
|
+
/** How much to trust a metric's value. */
|
|
16
|
+
export type MetricPrecision = "exact" | "estimated" | "lower-bound";
|
|
17
|
+
/** Where a metric came from: resident rows, file metadata, or a pushed-down query. */
|
|
18
|
+
export type MetricSource = "resident" | "metadata" | "query";
|
|
19
|
+
export type Coverage = {
|
|
20
|
+
scope: DataScope;
|
|
21
|
+
precision: MetricPrecision;
|
|
22
|
+
source: MetricSource;
|
|
23
|
+
};
|
|
24
|
+
export type ScopedValue<T> = Coverage & {
|
|
25
|
+
value: T;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Per-metric coverage of a column summary. Absent means every metric is
|
|
29
|
+
* exact over resident rows, which are the whole dataset.
|
|
30
|
+
*/
|
|
31
|
+
export type SummaryScope = {
|
|
32
|
+
rows: {
|
|
33
|
+
loaded: number;
|
|
34
|
+
dataset?: ScopedValue<number>;
|
|
35
|
+
};
|
|
36
|
+
/** Covers bins / topCategories / allCategories / trueCount / falseCount. */
|
|
37
|
+
distribution: Coverage;
|
|
38
|
+
range?: ScopedValue<{
|
|
39
|
+
min: number;
|
|
40
|
+
max: number;
|
|
41
|
+
}>;
|
|
42
|
+
nullCount?: ScopedValue<number>;
|
|
43
|
+
distinctCount?: ScopedValue<number>;
|
|
44
|
+
};
|
|
45
|
+
/** Whole-dataset coverage of a TableData. Absent means every row is resident. */
|
|
46
|
+
export type DatasetCoverage = {
|
|
47
|
+
loadedRows: number;
|
|
48
|
+
/** Absent when the dataset size is unknown. */
|
|
49
|
+
totalRows?: ScopedValue<number>;
|
|
50
|
+
/** Row groups in the source file, when the source has them. */
|
|
51
|
+
rowGroups?: number;
|
|
52
|
+
truncated: boolean;
|
|
53
|
+
/** True while a load is paused at a soft budget and more rows can be requested. */
|
|
54
|
+
resumable?: boolean;
|
|
55
|
+
};
|
|
56
|
+
/** Per-column statistics read from Parquet footer metadata at open. */
|
|
57
|
+
export type ColumnFileStats = {
|
|
58
|
+
/** Whole-file null count, or null when any row group did not record one. */
|
|
59
|
+
nullCount: number | null;
|
|
60
|
+
min: number | string | boolean | null;
|
|
61
|
+
max: number | string | boolean | null;
|
|
62
|
+
/** Row groups whose bounds were usable. Bounds cover the file only when equal to its row-group count. */
|
|
63
|
+
boundedRowGroups: number;
|
|
64
|
+
/** True when every row group's bounds are exact (not truncated strings). */
|
|
65
|
+
boundsExact: boolean;
|
|
66
|
+
/** min or max is ±Infinity. */
|
|
67
|
+
hasInfinite: boolean;
|
|
68
|
+
};
|
|
11
69
|
export type NumericColumnSummary = {
|
|
12
70
|
kind: "numeric";
|
|
13
71
|
min: number;
|
|
@@ -23,6 +81,7 @@ export type NumericColumnSummary = {
|
|
|
23
81
|
isIndex?: boolean;
|
|
24
82
|
/** Number of null/undefined/NaN values. */
|
|
25
83
|
nullCount?: number;
|
|
84
|
+
scope?: SummaryScope;
|
|
26
85
|
};
|
|
27
86
|
export type CategoryEntry = {
|
|
28
87
|
label: string;
|
|
@@ -39,6 +98,9 @@ export type CategoricalColumnSummary = {
|
|
|
39
98
|
allCategories: CategoryEntry[];
|
|
40
99
|
/** Median string length across all unique values. Used for display heuristics. */
|
|
41
100
|
medianTextLength: number;
|
|
101
|
+
/** Number of null values. */
|
|
102
|
+
nullCount?: number;
|
|
103
|
+
scope?: SummaryScope;
|
|
42
104
|
};
|
|
43
105
|
export type BooleanColumnSummary = {
|
|
44
106
|
kind: "boolean";
|
|
@@ -46,6 +108,7 @@ export type BooleanColumnSummary = {
|
|
|
46
108
|
falseCount: number;
|
|
47
109
|
nullCount: number;
|
|
48
110
|
total: number;
|
|
111
|
+
scope?: SummaryScope;
|
|
49
112
|
};
|
|
50
113
|
export type TimestampColumnSummary = {
|
|
51
114
|
kind: "timestamp";
|
|
@@ -59,6 +122,7 @@ export type TimestampColumnSummary = {
|
|
|
59
122
|
/** Number of null/undefined values. */
|
|
60
123
|
nullCount?: number;
|
|
61
124
|
timezone?: string | null;
|
|
125
|
+
scope?: SummaryScope;
|
|
62
126
|
};
|
|
63
127
|
export type ColumnSummary = NumericColumnSummary | CategoricalColumnSummary | BooleanColumnSummary | TimestampColumnSummary | null;
|
|
64
128
|
export type TableData = {
|
|
@@ -85,6 +149,14 @@ export type TableData = {
|
|
|
85
149
|
filterRows?: (filters: (ColumnFilter | null)[]) => Uint32Array;
|
|
86
150
|
/** Optional: release backing resources when the table engine is destroyed. */
|
|
87
151
|
dispose?: () => void;
|
|
152
|
+
/** Optional: how much of the dataset is resident. Absent means all of it. */
|
|
153
|
+
coverage?: DatasetCoverage;
|
|
154
|
+
/**
|
|
155
|
+
* Optional: footer statistics per column, in column order, fixed at open.
|
|
156
|
+
* Summaries fold them into `scope`; a cast column's entry is ignored since
|
|
157
|
+
* it describes the original type.
|
|
158
|
+
*/
|
|
159
|
+
datasetStats?: readonly (ColumnFileStats | null)[];
|
|
88
160
|
};
|
|
89
161
|
export type RangeFilter = {
|
|
90
162
|
kind: "range";
|
|
@@ -114,13 +186,47 @@ export type TableEngineState = {
|
|
|
114
186
|
filter: ColumnFilter;
|
|
115
187
|
}[];
|
|
116
188
|
filteredCount: number;
|
|
189
|
+
/** Resident rows. The same number as `loadedCount`; kept for compatibility. */
|
|
117
190
|
totalCount: number;
|
|
191
|
+
/** Resident rows; a truncated load holds fewer than the dataset. */
|
|
192
|
+
loadedCount?: number;
|
|
193
|
+
/**
|
|
194
|
+
* The query the resident rows already satisfy, when a provider answered one.
|
|
195
|
+
* `null` means a provider result with no filters; absent means the rows
|
|
196
|
+
* came from a file or stream and every filter is local.
|
|
197
|
+
*/
|
|
198
|
+
committedQuery?: TableQuery | null;
|
|
199
|
+
/** Rows matching `committedQuery` in the whole dataset, when known. */
|
|
200
|
+
resultCount?: Count;
|
|
201
|
+
/**
|
|
202
|
+
* True while a windowed source is answering a filter, sort, or cast;
|
|
203
|
+
* `filteredCount` is then the last known value. Always false for resident data.
|
|
204
|
+
*/
|
|
205
|
+
pending?: boolean;
|
|
206
|
+
/**
|
|
207
|
+
* A windowed data side reported a fatal error. The rows on screen stay;
|
|
208
|
+
* sort, filter, and cast controls are disabled. Always false for resident data.
|
|
209
|
+
*/
|
|
210
|
+
dataFailed?: boolean;
|
|
118
211
|
};
|
|
119
212
|
export type TableEngineOptions = {
|
|
120
213
|
/** Called whenever sort or filter state changes from UI interaction. */
|
|
121
214
|
onChange?: (state: TableEngineState) => void;
|
|
215
|
+
/**
|
|
216
|
+
* Called when the user removes a committed filter pill. The engine never
|
|
217
|
+
* re-runs queries; the host answers `next` and swaps the result in through
|
|
218
|
+
* `replaceData(data, { committedQuery: next })`.
|
|
219
|
+
*/
|
|
220
|
+
onCommittedQueryChange?: (next: TableQuery) => void;
|
|
122
221
|
/** Optional control rendered in the stats/footer bar before built-in buttons. */
|
|
123
222
|
footerControl?: HTMLElement;
|
|
223
|
+
/**
|
|
224
|
+
* Called when the viewport reaches the last loaded rows: the last visible
|
|
225
|
+
* view row is within the overscan of the end and streaming is not active.
|
|
226
|
+
* Fires once per row count, so a paused load gets one call per pause and is
|
|
227
|
+
* re-armed when rows arrive.
|
|
228
|
+
*/
|
|
229
|
+
onScrollNearEnd?: () => void;
|
|
124
230
|
};
|
|
125
231
|
export type ReplaceDataOptions = {
|
|
126
232
|
/** Drop sort state even when the sorted column survives in the new data. */
|
|
@@ -136,12 +242,45 @@ export type ReplaceDataOptions = {
|
|
|
136
242
|
* data settled on. Omit to leave the current state alone.
|
|
137
243
|
*/
|
|
138
244
|
streaming?: boolean;
|
|
245
|
+
/**
|
|
246
|
+
* Footer control to show from now on, as `TableEngineOptions.footerControl`.
|
|
247
|
+
* `null` removes the current one; omit to keep it.
|
|
248
|
+
*/
|
|
249
|
+
footerControl?: HTMLElement | null;
|
|
250
|
+
/**
|
|
251
|
+
* The new rows already satisfy this query (`null`: a provider result with no
|
|
252
|
+
* filters). Local filters and sort are reset instead of being re-applied,
|
|
253
|
+
* and committed filters render as pills the host clears through
|
|
254
|
+
* `onCommittedQueryChange`. Omit to keep the usual preservation, which also
|
|
255
|
+
* clears any committed query and result count.
|
|
256
|
+
*/
|
|
257
|
+
committedQuery?: TableQuery | null;
|
|
258
|
+
};
|
|
259
|
+
/**
|
|
260
|
+
* Options the engine passes itself when a column-shape change rebuilds it.
|
|
261
|
+
* Not public: hosts set a committed query through `replaceData`.
|
|
262
|
+
* @internal
|
|
263
|
+
*/
|
|
264
|
+
export type InternalTableEngineOptions = TableEngineOptions & {
|
|
265
|
+
committedQuery?: TableQuery | null;
|
|
139
266
|
};
|
|
140
267
|
export type TableEngine = {
|
|
141
268
|
onBatchAppended(): void;
|
|
142
269
|
/** Signal that all batches have been loaded and streaming is complete. */
|
|
143
270
|
setStreamingDone(): void;
|
|
144
|
-
|
|
271
|
+
/**
|
|
272
|
+
* Re-arm the streaming guard after `setStreamingDone()`: casts are blocked
|
|
273
|
+
* again and the status indicator shows loading until the next
|
|
274
|
+
* `setStreamingDone()`. Used when a paused load resumes.
|
|
275
|
+
*/
|
|
276
|
+
resumeStreaming(): void;
|
|
277
|
+
replaceData(newData: TableData | WindowedTableData, options?: ReplaceDataOptions): void;
|
|
278
|
+
/**
|
|
279
|
+
* Show `control` in the footer from now on, as
|
|
280
|
+
* `TableEngineOptions.footerControl`; `null` removes the current one. Rows,
|
|
281
|
+
* scroll position, sort, and filters are left alone.
|
|
282
|
+
*/
|
|
283
|
+
setFooterControl(control: HTMLElement | null): void;
|
|
145
284
|
destroy(): void;
|
|
146
285
|
setFilter(colIndex: number, filter: ColumnFilter): void;
|
|
147
286
|
clearFilter(colIndex: number): void;
|
|
@@ -160,6 +299,11 @@ export type TableEngine = {
|
|
|
160
299
|
}[];
|
|
161
300
|
/** Get the full explorer state (sort + filters + counts) in a serializable format. */
|
|
162
301
|
getState(): TableEngineState;
|
|
302
|
+
/**
|
|
303
|
+
* Record how many whole-dataset rows match the committed query. Updates the
|
|
304
|
+
* footer; cleared by `replaceData` without `committedQuery`.
|
|
305
|
+
*/
|
|
306
|
+
setResultCount(count: Count): void;
|
|
163
307
|
/**
|
|
164
308
|
* Currently focused (expanded) data row, or `null` when every row uses the
|
|
165
309
|
* collapsed text-line cap. Tracked by data row, so focus survives sort
|
|
@@ -172,4 +316,26 @@ export type TableEngine = {
|
|
|
172
316
|
*/
|
|
173
317
|
setFocusedDataRow(dataRow: number | null): void;
|
|
174
318
|
};
|
|
175
|
-
|
|
319
|
+
/**
|
|
320
|
+
* A view row and an offset into it, in virtual content pixels (the space
|
|
321
|
+
* `rowPositions` lives in, header excluded). See `captureScrollAnchor`.
|
|
322
|
+
* @internal
|
|
323
|
+
*/
|
|
324
|
+
export type ScrollAnchor = {
|
|
325
|
+
row: number;
|
|
326
|
+
offset: number;
|
|
327
|
+
};
|
|
328
|
+
/**
|
|
329
|
+
* Engine internals for tests. Not exported from `index.ts`.
|
|
330
|
+
* @internal
|
|
331
|
+
*/
|
|
332
|
+
export type TableEngineTestHooks = {
|
|
333
|
+
captureScrollAnchor(): ScrollAnchor | null;
|
|
334
|
+
/** Scroll so `anchor` sits under the header's bottom edge again. */
|
|
335
|
+
restoreScrollAnchor(anchor: ScrollAnchor): void;
|
|
336
|
+
};
|
|
337
|
+
/** @internal */
|
|
338
|
+
export type InternalTableEngine = TableEngine & {
|
|
339
|
+
__test: TableEngineTestHooks;
|
|
340
|
+
};
|
|
341
|
+
export declare function createTable(container: HTMLElement, initialData: TableData | WindowedTableData, options?: TableEngineOptions): TableEngine;
|
|
@@ -1,3 +1,21 @@
|
|
|
1
1
|
import type { getModuleSync } from "./predicate";
|
|
2
|
-
import type { Column, TableData } from "./table";
|
|
2
|
+
import type { Column, ColumnFileStats, DatasetCoverage, SummaryScope, TableData } from "./table";
|
|
3
|
+
type ScopeInputs = {
|
|
4
|
+
numRows: number;
|
|
5
|
+
coverage: DatasetCoverage | undefined;
|
|
6
|
+
/** Footer stats for this column, or null when absent or the column is cast. */
|
|
7
|
+
stats: ColumnFileStats | null;
|
|
8
|
+
/** Null count over resident rows. */
|
|
9
|
+
loadedNullCount: number;
|
|
10
|
+
/** Distinct count over resident rows, where the summary tracks one. */
|
|
11
|
+
uniqueCount?: number;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Assemble per-metric coverage for one column. The distribution (bins,
|
|
15
|
+
* categories, true/false) is always over resident rows; range and null count
|
|
16
|
+
* come from footer metadata when the footer has usable numbers for the whole
|
|
17
|
+
* file, and from resident rows otherwise.
|
|
18
|
+
*/
|
|
19
|
+
export declare function buildSummaryScope({ numRows, coverage, stats, loadedNullCount, uniqueCount, }: ScopeInputs): SummaryScope;
|
|
3
20
|
export declare function updateWasmSummaries(mod: ReturnType<typeof getModuleSync>, handle: number, tableData: TableData, columns: Column[], pandasIndexCols?: Set<string>): void;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A `WindowedTableData` over an in-memory `TableData`, on the main thread.
|
|
3
|
+
*
|
|
4
|
+
* This is how the engine's windowed mode is tested, and the shape a worker
|
|
5
|
+
* implements over a `MessagePort`. The adapter owns the permutation and the
|
|
6
|
+
* summary mask the engine owns in resident mode; it never touches the DOM.
|
|
7
|
+
* Every response resolves on a microtask, so the engine's code path is
|
|
8
|
+
* asynchronous even here.
|
|
9
|
+
*/
|
|
10
|
+
import type { TableData } from "./table";
|
|
11
|
+
import type { WindowedTableData } from "./windowed";
|
|
12
|
+
export type ResidentWindowedData = WindowedTableData & {
|
|
13
|
+
/** The source grew (`TableData.rowCount` already updated): emit `rows-appended`. */
|
|
14
|
+
notifyAppended(): void;
|
|
15
|
+
/** The source finished loading: emit `streaming-done`. */
|
|
16
|
+
notifyStreamingDone(): void;
|
|
17
|
+
};
|
|
18
|
+
export declare function createResidentWindowedData(data: TableData, opts?: {
|
|
19
|
+
headRows?: number;
|
|
20
|
+
}): ResidentWindowedData;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Windowed data: the view↔data protocol for a table whose cells, permutation,
|
|
3
|
+
* filtered summaries, and images arrive asynchronously.
|
|
4
|
+
*
|
|
5
|
+
* The data side owns the permutation (filter + sort) and the summary mask. The
|
|
6
|
+
* view asks for windows in *view* positions and receives the matching data
|
|
7
|
+
* rows alongside their cells, so it never holds the full permutation. Geometry
|
|
8
|
+
* stays dense: `rowCount` is a number within the resident budget.
|
|
9
|
+
*
|
|
10
|
+
* `createResidentWindowedData` adapts today's `TableData` to this shape on the
|
|
11
|
+
* main thread; a worker-backed implementation speaks the same protocol over a
|
|
12
|
+
* `MessagePort`.
|
|
13
|
+
*/
|
|
14
|
+
import type { Column, ColumnFileStats, ColumnFilter, ColumnSummary, ColumnType, DatasetCoverage } from "./table";
|
|
15
|
+
export type WindowCells = {
|
|
16
|
+
/** View position of `dataRows[0]`. */
|
|
17
|
+
viewStart: number;
|
|
18
|
+
/** Data rows in view order: identity for pooled rows, focus, and the detail sheet. */
|
|
19
|
+
dataRows: Uint32Array;
|
|
20
|
+
/** Row-major, `dataRows.length * columns.length`; `""` for null. */
|
|
21
|
+
strings: string[];
|
|
22
|
+
/** 1 = null. */
|
|
23
|
+
nulls: Uint8Array;
|
|
24
|
+
/**
|
|
25
|
+
* Numeric and timestamp values; booleans as 1/0 so the view does not parse
|
|
26
|
+
* display strings. NaN for other columns and for null.
|
|
27
|
+
*/
|
|
28
|
+
numeric: Float64Array;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* The view's filter and sort state, in the engine's own vocabulary. This moves
|
|
32
|
+
* where the permutation is computed, not the filter model; the committed
|
|
33
|
+
* `TableQuery` of a provider result is orthogonal.
|
|
34
|
+
*/
|
|
35
|
+
export type ViewQuery = {
|
|
36
|
+
/** One entry per column; `null` means no filter. */
|
|
37
|
+
filters: (ColumnFilter | null)[];
|
|
38
|
+
sort: {
|
|
39
|
+
col: number;
|
|
40
|
+
dir: "asc" | "desc";
|
|
41
|
+
} | null;
|
|
42
|
+
};
|
|
43
|
+
export type QueryResult = {
|
|
44
|
+
generation: number;
|
|
45
|
+
filteredCount: number;
|
|
46
|
+
/** Filtered summaries, what the header shows. */
|
|
47
|
+
summaries: ColumnSummary[];
|
|
48
|
+
/** Unfiltered snapshot; the engine's "hidden values" labels compare against it. */
|
|
49
|
+
summariesAll: ColumnSummary[];
|
|
50
|
+
};
|
|
51
|
+
export type DataEvent = {
|
|
52
|
+
type: "rows-appended";
|
|
53
|
+
rowCount: number;
|
|
54
|
+
summariesAll: ColumnSummary[];
|
|
55
|
+
} | {
|
|
56
|
+
type: "schema-changed";
|
|
57
|
+
columns: Column[];
|
|
58
|
+
} | {
|
|
59
|
+
type: "streaming-done";
|
|
60
|
+
/**
|
|
61
|
+
* Final counts, when the data side knows them (the worker does): rows
|
|
62
|
+
* resident, the file's row count, and whether the budget stopped early.
|
|
63
|
+
*/
|
|
64
|
+
loadedRows?: number;
|
|
65
|
+
totalRows?: number;
|
|
66
|
+
truncated?: boolean;
|
|
67
|
+
/**
|
|
68
|
+
* True when the load paused at a soft budget with row groups left: the
|
|
69
|
+
* table is at rest, and a `streaming-resumed` follows if more rows are
|
|
70
|
+
* requested. False or absent when loading has finished.
|
|
71
|
+
*/
|
|
72
|
+
resumable?: boolean;
|
|
73
|
+
}
|
|
74
|
+
/** A paused load continues; `rows-appended` events follow. */
|
|
75
|
+
| {
|
|
76
|
+
type: "streaming-resumed";
|
|
77
|
+
rowCount: number;
|
|
78
|
+
} | {
|
|
79
|
+
type: "error";
|
|
80
|
+
message: string;
|
|
81
|
+
fatal: boolean;
|
|
82
|
+
};
|
|
83
|
+
export type WindowedTableData = {
|
|
84
|
+
kind: "windowed";
|
|
85
|
+
columns: Column[];
|
|
86
|
+
/** Resident rows (dense geometry). Grows on `rows-appended`. */
|
|
87
|
+
rowCount: number;
|
|
88
|
+
coverage?: DatasetCoverage;
|
|
89
|
+
datasetStats?: readonly (ColumnFileStats | null)[];
|
|
90
|
+
/**
|
|
91
|
+
* Cells for data rows `[0, headRows)` in identity order, regardless of the
|
|
92
|
+
* current query; the engine uses it only when no sort or filter is active.
|
|
93
|
+
* Available at mount for auto-width and first paint. Refresh it when cell
|
|
94
|
+
* strings change (cast, undo).
|
|
95
|
+
*/
|
|
96
|
+
head: WindowCells;
|
|
97
|
+
/**
|
|
98
|
+
* Cells for view rows `[viewStart, viewEnd)` under the current query. The
|
|
99
|
+
* engine drops a response whose `generation` is no longer current.
|
|
100
|
+
*/
|
|
101
|
+
requestWindow(viewStart: number, viewEnd: number, generation: number): Promise<WindowCells>;
|
|
102
|
+
/** Recompute the permutation and summaries for `query`. */
|
|
103
|
+
setQuery(query: ViewQuery, generation: number): Promise<QueryResult>;
|
|
104
|
+
/** Image bytes for one cell: one entry per image, or `null` when there are none. */
|
|
105
|
+
requestImage(dataRow: number, col: number): Promise<Uint8Array[] | null>;
|
|
106
|
+
castColumn?(col: number, to: ColumnType, generation: number): Promise<QueryResult & {
|
|
107
|
+
columns: Column[];
|
|
108
|
+
}>;
|
|
109
|
+
undoCastColumn?(col: number, generation: number): Promise<QueryResult & {
|
|
110
|
+
columns: Column[];
|
|
111
|
+
}>;
|
|
112
|
+
isColumnCast?(col: number): boolean;
|
|
113
|
+
subscribe(listener: (event: DataEvent) => void): () => void;
|
|
114
|
+
dispose(): void;
|
|
115
|
+
};
|
|
116
|
+
export declare function isWindowedTableData(data: object): data is WindowedTableData;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Main-thread side of the worker protocol: a `WindowedTableData` whose methods
|
|
3
|
+
* post to the worker and resolve on its replies. The engine does not know the
|
|
4
|
+
* difference between this and the in-process adapter.
|
|
5
|
+
*
|
|
6
|
+
* Generations pass straight through; the engine already drops stale answers.
|
|
7
|
+
* The client does reject every pending promise when the worker dies or is
|
|
8
|
+
* closed, so no request waits forever.
|
|
9
|
+
*/
|
|
10
|
+
import type { TableLimitOverflow } from "../file-data";
|
|
11
|
+
import type { ByteSource, ParquetFileLimits } from "../parquet-file";
|
|
12
|
+
import type { SiftLoadMilestone } from "../react";
|
|
13
|
+
import type { Column } from "../table";
|
|
14
|
+
import type { WindowedTableData } from "../windowed";
|
|
15
|
+
/** The subset of `Worker` the client uses; tests substitute a `MessagePort`-backed stub. */
|
|
16
|
+
export type WorkerLike = {
|
|
17
|
+
postMessage(message: unknown): void;
|
|
18
|
+
postMessage(message: unknown, transfer: Transferable[]): void;
|
|
19
|
+
addEventListener(type: "message" | "error" | "messageerror", listener: (event: Event) => void): void;
|
|
20
|
+
terminate(): void;
|
|
21
|
+
};
|
|
22
|
+
export type WorkerWindowedDataOptions = {
|
|
23
|
+
source: ByteSource;
|
|
24
|
+
limits: ParquetFileLimits;
|
|
25
|
+
overflow?: TableLimitOverflow;
|
|
26
|
+
columnOverrides?: Record<string, Partial<Column>>;
|
|
27
|
+
/** Rows in `head`. Default 100. */
|
|
28
|
+
headRows?: number;
|
|
29
|
+
/**
|
|
30
|
+
* Live WASM heap bytes allowed in the worker; default `limits.maxDecodedBytes
|
|
31
|
+
* * DEFAULT_ALLOCATION_CAP_FACTOR` (2.5).
|
|
32
|
+
*/
|
|
33
|
+
allocationCap?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Soft budget: the worker pauses once the committed rows or decoded bytes
|
|
36
|
+
* reach it while row groups remain (a `streaming-done` with `resumable:
|
|
37
|
+
* true`) and continues on `resume()`. Needs `overflow: "truncate"`.
|
|
38
|
+
*/
|
|
39
|
+
autoLoad?: {
|
|
40
|
+
rows?: number;
|
|
41
|
+
decodedBytes?: number;
|
|
42
|
+
};
|
|
43
|
+
signal: AbortSignal;
|
|
44
|
+
/** Load milestones, in the same vocabulary as the main-thread loaders. */
|
|
45
|
+
emit?: (milestone: Omit<SiftLoadMilestone, "elapsedMs" | "source">) => void;
|
|
46
|
+
/** Spawn override; defaults to the registered factory. */
|
|
47
|
+
worker?: WorkerLike;
|
|
48
|
+
};
|
|
49
|
+
export type WorkerWindowedData = WindowedTableData & {
|
|
50
|
+
/** Kill the worker now. `dispose()` asks it to close first. */
|
|
51
|
+
terminate(): void;
|
|
52
|
+
/**
|
|
53
|
+
* Continue a load paused at the `autoLoad` budget. No-op unless paused; a
|
|
54
|
+
* `streaming-resumed` event follows when the worker picks it up.
|
|
55
|
+
*/
|
|
56
|
+
resume(): void;
|
|
57
|
+
readonly rowGroupCount: number;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* Why a worker-backed data side stopped answering:
|
|
61
|
+
*
|
|
62
|
+
* - `factory`: the registered factory is missing or threw while spawning.
|
|
63
|
+
* - `open`: the worker refused the file before the first row group mounted.
|
|
64
|
+
* - `aborted`: the caller's signal aborted before the open finished.
|
|
65
|
+
* - `terminated`: the worker died (a WASM trap, a crash) or was killed.
|
|
66
|
+
* - `closed`: `dispose()` was called.
|
|
67
|
+
*
|
|
68
|
+
* Every request after the stop rejects with the same instance.
|
|
69
|
+
*/
|
|
70
|
+
export type WorkerDataErrorCode = "factory" | "open" | "aborted" | "terminated" | "closed";
|
|
71
|
+
export declare class WorkerDataError extends Error {
|
|
72
|
+
readonly code: WorkerDataErrorCode;
|
|
73
|
+
/** The underlying error, when there is one. Set by hand: `ErrorOptions`
|
|
74
|
+
* and the two-argument `Error` constructor are ES2022 lib types, and
|
|
75
|
+
* consumers type-check this file under older `lib` settings. */
|
|
76
|
+
readonly cause?: unknown;
|
|
77
|
+
constructor(code: WorkerDataErrorCode, message: string, options?: {
|
|
78
|
+
cause?: unknown;
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Spawn the data worker, open `source` in it, and resolve with a
|
|
83
|
+
* `WindowedTableData` once the first row group is loaded. Rejects with a
|
|
84
|
+
* `WorkerDataError` when no factory is registered or it throws, when the open
|
|
85
|
+
* fails, or when `signal` aborts first; the worker is terminated in each case.
|
|
86
|
+
*/
|
|
87
|
+
export declare function createWorkerWindowedData(opts: WorkerWindowedDataOptions): Promise<WorkerWindowedData>;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bundle entry for `@nteract/sift/worker`: the data side in a dedicated worker.
|
|
3
|
+
*
|
|
4
|
+
* Spawn it from the main thread and register the spawn with `setWorkerFactory`:
|
|
5
|
+
*
|
|
6
|
+
* setWorkerFactory(() => new Worker(new URL("@nteract/sift/worker", import.meta.url), { type: "module" }));
|
|
7
|
+
*
|
|
8
|
+
* Importing this module outside a worker (Node, the main thread) is a no-op so
|
|
9
|
+
* resolvability can be checked without a `Worker`.
|
|
10
|
+
*/
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How Sift spawns its data worker. The library ships the worker bundle
|
|
3
|
+
* (`@nteract/sift/worker`) but not the spawn: a same-origin module worker, a
|
|
4
|
+
* blob worker for an opaque-origin frame, or a host-managed pool are the
|
|
5
|
+
* host's choice. With no factory, `worker: true` sources fail at validation
|
|
6
|
+
* rather than falling back to the main thread.
|
|
7
|
+
*/
|
|
8
|
+
export type WorkerFactory = () => Worker;
|
|
9
|
+
/** Register how to spawn the Sift data worker; `null` unregisters. */
|
|
10
|
+
export declare function setWorkerFactory(next: WorkerFactory | null): void;
|
|
11
|
+
export declare function getWorkerFactory(): WorkerFactory | null;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The data side of the worker protocol. Runs inside a dedicated worker (see
|
|
3
|
+
* `entry.ts`) or, in tests, on one end of a `MessageChannel`.
|
|
4
|
+
*
|
|
5
|
+
* It owns the WASM module and the Parquet file, wraps the loaded `TableData` in
|
|
6
|
+
* `createResidentWindowedData`, and answers the view's requests over the port.
|
|
7
|
+
* Bytes come from the host through a read relay. A WASM trap (including the
|
|
8
|
+
* allocation cap) is fatal: the host posts one `fatal` error and stops
|
|
9
|
+
* answering; the main thread terminates the worker and keeps the rows it has.
|
|
10
|
+
*/
|
|
11
|
+
import { type ToWorker } from "./protocol";
|
|
12
|
+
/** What the host needs from `self` or a `MessagePort`. */
|
|
13
|
+
export type HostPort = {
|
|
14
|
+
postMessage(message: unknown, transfer?: Transferable[]): void;
|
|
15
|
+
addEventListener(type: "message", listener: (event: MessageEvent) => void): void;
|
|
16
|
+
removeEventListener(type: "message", listener: (event: MessageEvent) => void): void;
|
|
17
|
+
start?(): void;
|
|
18
|
+
close?(): void;
|
|
19
|
+
};
|
|
20
|
+
export type WorkerHostOptions = {
|
|
21
|
+
/**
|
|
22
|
+
* Test hook: runs before each request is handled. Throwing a
|
|
23
|
+
* `WebAssembly.RuntimeError` here exercises the fatal path without a real trap.
|
|
24
|
+
*/
|
|
25
|
+
beforeRequest?: (message: ToWorker) => void;
|
|
26
|
+
};
|
|
27
|
+
export type WorkerHost = {
|
|
28
|
+
/** Stop handling messages and release the file and store. */
|
|
29
|
+
dispose(): void;
|
|
30
|
+
};
|
|
31
|
+
export declare function createWorkerHost(port: HostPort, options?: WorkerHostOptions): WorkerHost;
|