@nteract/sift 0.6.0 → 0.7.0-nightly.42413490472
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/docs/embedding-guide.md +417 -10
- package/lib/THIRD_PARTY_NOTICES.md +29 -4
- package/lib/handoff.css +12 -1
- package/lib/index.js +2847 -2052
- package/lib/protocol-B_bRY_76.js +885 -0
- package/lib/style.css +1 -1
- package/lib/types/packages/sift/src/auto-width.d.ts +9 -3
- package/lib/types/packages/sift/src/components/icons.d.ts +16 -0
- package/lib/types/packages/sift/src/file-data.d.ts +34 -5
- package/lib/types/packages/sift/src/filter-schema.d.ts +11 -0
- package/lib/types/packages/sift/src/index.d.ts +11 -4
- package/lib/types/packages/sift/src/parquet-file.d.ts +52 -5
- package/lib/types/packages/sift/src/permutation.d.ts +24 -0
- package/lib/types/packages/sift/src/predicate.d.ts +56 -3
- package/lib/types/packages/sift/src/query.d.ts +100 -0
- package/lib/types/packages/sift/src/react.d.ts +108 -6
- package/lib/types/packages/sift/src/table.d.ts +168 -2
- package/lib/types/packages/sift/src/wasm-summaries.d.ts +19 -1
- package/lib/types/packages/sift/src/windowed-resident.d.ts +20 -0
- package/lib/types/packages/sift/src/windowed.d.ts +116 -0
- package/lib/types/packages/sift/src/worker/client.d.ts +87 -0
- package/lib/types/packages/sift/src/worker/entry.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/factory.d.ts +11 -0
- package/lib/types/packages/sift/src/worker/host.d.ts +31 -0
- package/lib/types/packages/sift/src/worker/protocol.d.ts +191 -0
- package/lib/wasm/sift_wasm.d.ts +67 -3
- package/lib/wasm/sift_wasm.js +109 -3
- package/lib/wasm/sift_wasm_bg.wasm +0 -0
- package/lib/worker.js +277 -0
- package/package.json +9 -6
package/lib/worker.js
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import { _ as e, a as t, b as n, c as r, i, m as a, n as o, r as s } from "./protocol-B_bRY_76.js";
|
|
2
|
+
//#region src/worker/host.ts
|
|
3
|
+
function c(c, l = {}) {
|
|
4
|
+
let u = null, d = null, f = null, p = null, m = 0, h = !1, g = !1, _ = 1, v = -1, y = null, b = /* @__PURE__ */ new Map();
|
|
5
|
+
function x(e, t) {
|
|
6
|
+
h || (t && t.length > 0 ? c.postMessage(e, t) : c.postMessage(e));
|
|
7
|
+
}
|
|
8
|
+
function S() {
|
|
9
|
+
return u?.allocated_bytes ? u.allocated_bytes() : e()?.buffer.byteLength ?? 0;
|
|
10
|
+
}
|
|
11
|
+
function C(e, t) {
|
|
12
|
+
let n = t instanceof Error ? t.message : String(t);
|
|
13
|
+
o(t) ? (g = !0, v >= 0 && (n = `row group ${v} exceeded the worker memory limit (${n})`), x({
|
|
14
|
+
t: "error",
|
|
15
|
+
id: e,
|
|
16
|
+
message: n,
|
|
17
|
+
fatal: !0
|
|
18
|
+
}), T()) : x({
|
|
19
|
+
t: "error",
|
|
20
|
+
id: e,
|
|
21
|
+
message: n,
|
|
22
|
+
fatal: !1
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
function w() {
|
|
26
|
+
let e = y;
|
|
27
|
+
y = null, e?.();
|
|
28
|
+
}
|
|
29
|
+
function T() {
|
|
30
|
+
w();
|
|
31
|
+
for (let e of b.values()) e.reject(/* @__PURE__ */ Error("Worker closed"));
|
|
32
|
+
b.clear(), p?.(), p = null;
|
|
33
|
+
try {
|
|
34
|
+
d?.close();
|
|
35
|
+
} catch {}
|
|
36
|
+
d = null;
|
|
37
|
+
try {
|
|
38
|
+
f?.dispose();
|
|
39
|
+
} catch {}
|
|
40
|
+
f = null;
|
|
41
|
+
}
|
|
42
|
+
function E(e) {
|
|
43
|
+
return {
|
|
44
|
+
byteLength: e,
|
|
45
|
+
read(e, t) {
|
|
46
|
+
return new Promise((n, r) => {
|
|
47
|
+
if (h || g) {
|
|
48
|
+
r(/* @__PURE__ */ Error("Worker closed"));
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
let i = _++;
|
|
52
|
+
b.set(i, {
|
|
53
|
+
resolve: n,
|
|
54
|
+
reject: r
|
|
55
|
+
}), x({
|
|
56
|
+
t: "read",
|
|
57
|
+
id: i,
|
|
58
|
+
offset: e,
|
|
59
|
+
length: t
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
function D(e) {
|
|
66
|
+
p?.(), f = e, p = e.subscribe((e) => {
|
|
67
|
+
e.type === "streaming-done" && d && (e = {
|
|
68
|
+
...e,
|
|
69
|
+
loadedRows: d.loadedRows,
|
|
70
|
+
totalRows: d.rowCount,
|
|
71
|
+
truncated: d.truncated,
|
|
72
|
+
resumable: !1
|
|
73
|
+
}), x({
|
|
74
|
+
t: "event",
|
|
75
|
+
event: e
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
function O(e, n) {
|
|
80
|
+
let r = f;
|
|
81
|
+
D(t(e, { headRows: n })), r?.dispose(), x({
|
|
82
|
+
t: "event",
|
|
83
|
+
event: {
|
|
84
|
+
type: "schema-changed",
|
|
85
|
+
columns: e.columns
|
|
86
|
+
},
|
|
87
|
+
head: f.head
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
function k(e, t) {
|
|
91
|
+
d && x({
|
|
92
|
+
t: "progress",
|
|
93
|
+
phase: e,
|
|
94
|
+
rowGroup: t,
|
|
95
|
+
rowGroupCount: d.rowGroupCount,
|
|
96
|
+
loadedRows: d.loadedRows,
|
|
97
|
+
totalRows: d.rowCount,
|
|
98
|
+
truncated: d.truncated,
|
|
99
|
+
memoryBytes: S()
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
function A(e) {
|
|
103
|
+
if (m <= 0) return;
|
|
104
|
+
let t = S();
|
|
105
|
+
if (t > m) throw new WebAssembly.RuntimeError(`${t.toLocaleString()} live bytes after row group ${e} > cap ${m.toLocaleString()}`);
|
|
106
|
+
}
|
|
107
|
+
async function j(e) {
|
|
108
|
+
if (e.protocol !== 2) throw Error(`Sift worker protocol ${e.protocol} does not match 2`);
|
|
109
|
+
if (e.wasmUrl && n(e.wasmUrl, e.wasmFallbackUrl), u = await a(), m = e.allocationCap ?? e.limits.maxDecodedBytes * 2.5, typeof u.set_allocation_cap == "function" && u.set_allocation_cap(m), d = await r(E(e.source.byteLength), {
|
|
110
|
+
limits: e.limits,
|
|
111
|
+
overflow: e.overflow,
|
|
112
|
+
columnOverrides: e.columnOverrides
|
|
113
|
+
}), h) return;
|
|
114
|
+
let i = !1;
|
|
115
|
+
if (d.rowGroupCount > 0 && (v = 0, i = (await d.appendRowGroup(0)).exhausted, v = -1), h) return;
|
|
116
|
+
D(t(d.tableData, { headRows: e.headRows }));
|
|
117
|
+
let s = f;
|
|
118
|
+
x({
|
|
119
|
+
t: "opened",
|
|
120
|
+
id: e.id,
|
|
121
|
+
columns: s.columns,
|
|
122
|
+
rowCount: s.rowCount,
|
|
123
|
+
coverage: s.coverage,
|
|
124
|
+
datasetStats: s.datasetStats ? [...s.datasetStats] : void 0,
|
|
125
|
+
head: s.head,
|
|
126
|
+
rowGroupCount: d.rowGroupCount
|
|
127
|
+
});
|
|
128
|
+
try {
|
|
129
|
+
await M(d, i, e.headRows, e.autoLoad);
|
|
130
|
+
} catch (e) {
|
|
131
|
+
if (h || g) return;
|
|
132
|
+
if (o(e)) throw e;
|
|
133
|
+
v = -1, x({
|
|
134
|
+
t: "event",
|
|
135
|
+
event: {
|
|
136
|
+
type: "error",
|
|
137
|
+
message: e instanceof Error ? e.message : String(e),
|
|
138
|
+
fatal: !1
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
try {
|
|
142
|
+
d.close();
|
|
143
|
+
} catch {}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
async function M(e, t, n, r) {
|
|
147
|
+
let i = r?.rows ?? Infinity, a = r?.decodedBytes ?? Infinity;
|
|
148
|
+
async function o(t, n) {
|
|
149
|
+
if (n || t >= e.rowGroupCount) return;
|
|
150
|
+
let { usage: o } = e;
|
|
151
|
+
o.rows < i && o.decodedBytes < a || (e.setPaused(!0), x({
|
|
152
|
+
t: "event",
|
|
153
|
+
event: {
|
|
154
|
+
type: "streaming-done",
|
|
155
|
+
loadedRows: e.loadedRows,
|
|
156
|
+
totalRows: e.rowCount,
|
|
157
|
+
truncated: !0,
|
|
158
|
+
resumable: !0
|
|
159
|
+
}
|
|
160
|
+
}), await new Promise((e) => {
|
|
161
|
+
y = e;
|
|
162
|
+
}), !(h || g) && (i = o.rows + (r?.rows ?? Infinity), a = o.decodedBytes + (r?.decodedBytes ?? Infinity), e.setPaused(!1), x({
|
|
163
|
+
t: "event",
|
|
164
|
+
event: {
|
|
165
|
+
type: "streaming-resumed",
|
|
166
|
+
rowCount: e.loadedRows
|
|
167
|
+
}
|
|
168
|
+
})));
|
|
169
|
+
}
|
|
170
|
+
if (k("chunk", 0), A(0), await o(1, t), h || g) return;
|
|
171
|
+
for (let r = 1; r < e.rowGroupCount && !t; r++) {
|
|
172
|
+
if (await new Promise((e) => setTimeout(e, 0)), h || g) return;
|
|
173
|
+
v = r;
|
|
174
|
+
let i = await e.appendRowGroup(r);
|
|
175
|
+
if (h || g || (i.replaced ? O(i.replaced, n) : f.notifyAppended(), t = i.exhausted, k("chunk", r), A(r), v = -1, await o(r + 1, t), h || g)) return;
|
|
176
|
+
}
|
|
177
|
+
let { replaced: s } = e.finish();
|
|
178
|
+
s && O(s, n), f.notifyStreamingDone(), k("complete", e.rowGroupCount - 1), e.close();
|
|
179
|
+
}
|
|
180
|
+
async function N(e) {
|
|
181
|
+
if (!(h || g)) {
|
|
182
|
+
switch (l.beforeRequest?.(e), e.t) {
|
|
183
|
+
case "open":
|
|
184
|
+
await j(e);
|
|
185
|
+
return;
|
|
186
|
+
case "read-result": {
|
|
187
|
+
let t = b.get(e.id);
|
|
188
|
+
b.delete(e.id), t?.resolve(e.bytes);
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
case "read-error": {
|
|
192
|
+
let t = b.get(e.id);
|
|
193
|
+
b.delete(e.id), t?.reject(Error(e.message));
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
case "resume":
|
|
197
|
+
w();
|
|
198
|
+
return;
|
|
199
|
+
case "close":
|
|
200
|
+
h = !0, T(), c.postMessage({ t: "closed" }), c.close?.();
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
if (!f) throw Error(`Request "${e.t}" before open`);
|
|
204
|
+
switch (e.t) {
|
|
205
|
+
case "window": {
|
|
206
|
+
let t = await f.requestWindow(e.viewStart, e.viewEnd, e.gen);
|
|
207
|
+
x({
|
|
208
|
+
t: "window",
|
|
209
|
+
id: e.id,
|
|
210
|
+
cells: t
|
|
211
|
+
}, i(t));
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
case "query": {
|
|
215
|
+
let t = await f.setQuery(e.query, e.gen);
|
|
216
|
+
x({
|
|
217
|
+
t: "query",
|
|
218
|
+
id: e.id,
|
|
219
|
+
result: t
|
|
220
|
+
});
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
case "image": {
|
|
224
|
+
let t = await f.requestImage(e.dataRow, e.col), n = t ? t.map(s) : null;
|
|
225
|
+
x({
|
|
226
|
+
t: "image",
|
|
227
|
+
id: e.id,
|
|
228
|
+
chunks: n
|
|
229
|
+
}, n ? [...new Set(n.map((e) => e.buffer))] : void 0);
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
case "cast": {
|
|
233
|
+
if (!f.castColumn) throw Error("This source cannot cast columns");
|
|
234
|
+
let t = await f.castColumn(e.col, e.to, e.gen);
|
|
235
|
+
x({
|
|
236
|
+
t: "cast",
|
|
237
|
+
id: e.id,
|
|
238
|
+
result: {
|
|
239
|
+
...t,
|
|
240
|
+
columns: [...t.columns]
|
|
241
|
+
},
|
|
242
|
+
head: f.head
|
|
243
|
+
});
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
case "undo-cast": {
|
|
247
|
+
if (!f.undoCastColumn) throw Error("This source cannot undo casts");
|
|
248
|
+
let t = await f.undoCastColumn(e.col, e.gen);
|
|
249
|
+
x({
|
|
250
|
+
t: "cast",
|
|
251
|
+
id: e.id,
|
|
252
|
+
result: {
|
|
253
|
+
...t,
|
|
254
|
+
columns: [...t.columns]
|
|
255
|
+
},
|
|
256
|
+
head: f.head
|
|
257
|
+
});
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
let P = (e) => {
|
|
264
|
+
let t = e.data;
|
|
265
|
+
!t || typeof t != "object" || typeof t.t != "string" || N(t).catch((e) => {
|
|
266
|
+
h || C("id" in t ? t.id : null, e);
|
|
267
|
+
});
|
|
268
|
+
};
|
|
269
|
+
return c.addEventListener("message", P), c.start?.(), { dispose() {
|
|
270
|
+
h || (h = !0, c.removeEventListener("message", P), T());
|
|
271
|
+
} };
|
|
272
|
+
}
|
|
273
|
+
//#endregion
|
|
274
|
+
//#region src/worker/entry.ts
|
|
275
|
+
var l = globalThis;
|
|
276
|
+
typeof l.postMessage == "function" && typeof l.addEventListener == "function" && l.document === void 0 && c(l);
|
|
277
|
+
//#endregion
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nteract/sift",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0-nightly.42413490472",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "git+https://github.com/nteract/nteract.git",
|
|
@@ -18,6 +18,10 @@
|
|
|
18
18
|
"types": "./lib/types/packages/sift/src/handoff.d.ts",
|
|
19
19
|
"import": "./lib/handoff.js"
|
|
20
20
|
},
|
|
21
|
+
"./worker": {
|
|
22
|
+
"types": "./lib/types/packages/sift/src/worker/entry.d.ts",
|
|
23
|
+
"import": "./lib/worker.js"
|
|
24
|
+
},
|
|
21
25
|
"./style.css": "./lib/style.css",
|
|
22
26
|
"./handoff.css": "./lib/handoff.css",
|
|
23
27
|
"./sift.wasm": "./lib/wasm/sift_wasm_bg.wasm"
|
|
@@ -48,12 +52,11 @@
|
|
|
48
52
|
"@nteract/odometer": "0.1.0"
|
|
49
53
|
},
|
|
50
54
|
"dependencies": {
|
|
51
|
-
"@chenglou/pretext": "
|
|
55
|
+
"@chenglou/pretext": "^1.0.0",
|
|
52
56
|
"@radix-ui/react-popover": "^1.1.15",
|
|
53
57
|
"@radix-ui/react-slot": "^1.2.4",
|
|
54
58
|
"class-variance-authority": "^0.7.1",
|
|
55
59
|
"clsx": "^2.1.1",
|
|
56
|
-
"lucide-react": "^0.513.0",
|
|
57
60
|
"rxjs": "^7.8.2",
|
|
58
61
|
"tailwind-merge": "^3.5.0"
|
|
59
62
|
},
|
|
@@ -62,9 +65,9 @@
|
|
|
62
65
|
],
|
|
63
66
|
"license": "BSD-3-Clause",
|
|
64
67
|
"nteractRelease": {
|
|
65
|
-
"channel": "
|
|
66
|
-
"sourceSha": "
|
|
67
|
-
"runId": "
|
|
68
|
+
"channel": "nightly",
|
|
69
|
+
"sourceSha": "6fcf623be3e4ee1ffc1abe9328e5a7aab0956c3a",
|
|
70
|
+
"runId": "42413490472"
|
|
68
71
|
},
|
|
69
72
|
"scripts": {
|
|
70
73
|
"generate": "npx tsx src/generate-data.ts",
|