@perspective-dev/client 4.5.1 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cdn/perspective-server.worker.js +1 -1
- package/dist/cdn/perspective-server.worker.js.map +3 -3
- package/dist/cdn/perspective.js +2 -2
- package/dist/cdn/perspective.js.map +4 -4
- package/dist/esm/perspective.browser.d.ts +50 -1
- package/dist/esm/perspective.inline.js +2 -2
- package/dist/esm/perspective.inline.js.map +4 -4
- package/dist/esm/perspective.js +2 -2
- package/dist/esm/perspective.js.map +4 -4
- package/dist/esm/perspective.node.d.ts +3 -0
- package/dist/esm/perspective.node.js +328 -25
- package/dist/esm/perspective.node.js.map +4 -4
- package/dist/esm/ts-rs/TableInitOptions.d.ts +6 -0
- package/dist/esm/virtual_servers/clickhouse.js +1 -1
- package/dist/esm/virtual_servers/clickhouse.js.map +2 -2
- package/dist/esm/virtual_servers/duckdb.js +1 -1
- package/dist/esm/virtual_servers/duckdb.js.map +2 -2
- package/dist/esm/wasm/emscripten_api.d.ts +14 -1
- package/dist/esm/wasm/engine.d.ts +11 -1
- package/dist/esm/wasm/features.d.ts +4 -0
- package/dist/wasm/perspective-js.d.ts +5 -5
- package/dist/wasm/perspective-js.js +9 -9
- package/dist/wasm/perspective-js.wasm +0 -0
- package/dist/wasm/perspective-js.wasm.d.ts +5 -5
- package/package.json +1 -1
- package/src/rust/typed_array.rs +88 -54
- package/src/ts/perspective.browser.ts +176 -20
- package/src/ts/perspective.cdn.ts +11 -5
- package/src/ts/perspective.node.ts +157 -5
- package/src/ts/ts-rs/TableInitOptions.ts +7 -1
- package/src/ts/virtual_servers/clickhouse.ts +1 -0
- package/src/ts/virtual_servers/duckdb.ts +3 -1
- package/src/ts/wasm/cdn.ts +32 -0
- package/src/ts/wasm/emscripten_api.ts +25 -0
- package/src/ts/wasm/engine.ts +59 -12
- package/src/ts/wasm/features.ts +31 -0
- package/src/ts/wasm/perspective-server.poly.ts +173 -3
package/src/ts/wasm/engine.ts
CHANGED
|
@@ -22,6 +22,33 @@ export interface PerspectiveServerOptions {
|
|
|
22
22
|
on_poll_request?: (x: PerspectiveServer) => Promise<void>;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* A server-wide FIFO async mutex.
|
|
27
|
+
*/
|
|
28
|
+
class AsyncLock {
|
|
29
|
+
private tail: Promise<unknown> = Promise.resolve();
|
|
30
|
+
run<T>(fn: () => Promise<T> | T): Promise<T> {
|
|
31
|
+
const result = this.tail.then(fn, fn);
|
|
32
|
+
this.tail = result.then(
|
|
33
|
+
() => undefined,
|
|
34
|
+
() => undefined,
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
return result;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function residency_safepoint(
|
|
42
|
+
mod: MainModule,
|
|
43
|
+
lock: AsyncLock,
|
|
44
|
+
server: EmscriptenServer,
|
|
45
|
+
) {
|
|
46
|
+
const safepoint = (mod as any).psp_residency_safepoint;
|
|
47
|
+
if (safepoint) {
|
|
48
|
+
await lock.run(() => safepoint(server));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
25
52
|
export class PerspectivePollThread {
|
|
26
53
|
private poll_handle?: Promise<void>;
|
|
27
54
|
private server: PerspectiveServer;
|
|
@@ -63,10 +90,13 @@ export class PerspectiveServer {
|
|
|
63
90
|
server: EmscriptenServer;
|
|
64
91
|
module: MainModule;
|
|
65
92
|
on_poll_request?: (x: PerspectiveServer) => Promise<void>;
|
|
93
|
+
lock: AsyncLock;
|
|
94
|
+
|
|
66
95
|
constructor(module: MainModule, options?: PerspectiveServerOptions) {
|
|
67
96
|
this.clients = new Map();
|
|
68
97
|
this.module = module;
|
|
69
98
|
this.on_poll_request = options?.on_poll_request;
|
|
99
|
+
this.lock = new AsyncLock();
|
|
70
100
|
this.server = module._psp_new_server(
|
|
71
101
|
!!options?.on_poll_request ? 1 : 0,
|
|
72
102
|
) as EmscriptenServer;
|
|
@@ -85,12 +115,16 @@ export class PerspectiveServer {
|
|
|
85
115
|
this.server,
|
|
86
116
|
client_id,
|
|
87
117
|
this.clients,
|
|
118
|
+
this.lock,
|
|
88
119
|
this.on_poll_request && (() => this.on_poll_request!(this)),
|
|
89
120
|
);
|
|
90
121
|
}
|
|
91
122
|
|
|
92
123
|
async poll() {
|
|
93
|
-
const polled = this.
|
|
124
|
+
const polled = await this.lock.run(() =>
|
|
125
|
+
this.module._psp_poll(this.server as any),
|
|
126
|
+
);
|
|
127
|
+
|
|
94
128
|
await decode_api_responses(
|
|
95
129
|
this.module,
|
|
96
130
|
polled,
|
|
@@ -98,6 +132,8 @@ export class PerspectiveServer {
|
|
|
98
132
|
await this.clients.get(msg.client_id)!(msg.data);
|
|
99
133
|
},
|
|
100
134
|
);
|
|
135
|
+
|
|
136
|
+
await residency_safepoint(this.module, this.lock, this.server);
|
|
101
137
|
}
|
|
102
138
|
|
|
103
139
|
delete() {
|
|
@@ -111,6 +147,7 @@ export class PerspectiveSession {
|
|
|
111
147
|
private server: EmscriptenServer,
|
|
112
148
|
private client_id: number,
|
|
113
149
|
private client_map: Map<number, (buffer: Uint8Array) => Promise<void>>,
|
|
150
|
+
private lock: AsyncLock,
|
|
114
151
|
private on_poll_request?: () => Promise<void>,
|
|
115
152
|
) {}
|
|
116
153
|
|
|
@@ -119,13 +156,17 @@ export class PerspectiveSession {
|
|
|
119
156
|
this.mod,
|
|
120
157
|
view,
|
|
121
158
|
async (viewPtr) => {
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
159
|
+
const len = this.mod._psp_is_memory64()
|
|
160
|
+
? (BigInt(view.byteLength) as any as number)
|
|
161
|
+
: (view.byteLength as any);
|
|
162
|
+
|
|
163
|
+
return this.lock.run(() =>
|
|
164
|
+
this.mod._psp_handle_request(
|
|
165
|
+
this.server as any,
|
|
166
|
+
this.client_id,
|
|
167
|
+
viewPtr as any,
|
|
168
|
+
len,
|
|
169
|
+
),
|
|
129
170
|
);
|
|
130
171
|
},
|
|
131
172
|
);
|
|
@@ -139,10 +180,14 @@ export class PerspectiveSession {
|
|
|
139
180
|
} else {
|
|
140
181
|
await this.poll();
|
|
141
182
|
}
|
|
183
|
+
|
|
184
|
+
await residency_safepoint(this.mod, this.lock, this.server);
|
|
142
185
|
}
|
|
143
186
|
|
|
144
187
|
private async poll() {
|
|
145
|
-
const polled = this.
|
|
188
|
+
const polled = await this.lock.run(() =>
|
|
189
|
+
this.mod._psp_poll(this.server as any),
|
|
190
|
+
);
|
|
146
191
|
await decode_api_responses(
|
|
147
192
|
this.mod,
|
|
148
193
|
polled,
|
|
@@ -166,13 +211,15 @@ async function convert_typed_array_to_pointer(
|
|
|
166
211
|
array: Uint8Array,
|
|
167
212
|
callback: (_: PspPtr) => Promise<PspPtr>,
|
|
168
213
|
): Promise<PspPtr> {
|
|
214
|
+
const is_64 = core._psp_is_memory64();
|
|
169
215
|
const ptr = core._psp_alloc(
|
|
170
|
-
|
|
216
|
+
is_64
|
|
171
217
|
? (BigInt(array.byteLength) as any as number)
|
|
172
218
|
: (array.byteLength as any),
|
|
173
219
|
);
|
|
174
220
|
|
|
175
|
-
|
|
221
|
+
const addr = is_64 ? Number(ptr) : Number(ptr) >>> 0;
|
|
222
|
+
core.HEAPU8.set(array, addr);
|
|
176
223
|
const msg = await callback(ptr);
|
|
177
224
|
core._psp_free(ptr);
|
|
178
225
|
return msg;
|
|
@@ -210,7 +257,7 @@ async function decode_api_responses(
|
|
|
210
257
|
const is_64 = core._psp_is_memory64();
|
|
211
258
|
const response = new DataView(
|
|
212
259
|
core.HEAPU8.buffer,
|
|
213
|
-
Number(ptr),
|
|
260
|
+
is_64 ? Number(ptr) : Number(ptr) >>> 0,
|
|
214
261
|
is_64 ? 12 : 8,
|
|
215
262
|
);
|
|
216
263
|
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
|
2
|
+
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
|
3
|
+
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
|
4
|
+
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
|
5
|
+
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
|
6
|
+
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
|
7
|
+
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
|
8
|
+
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
|
9
|
+
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
|
10
|
+
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
|
11
|
+
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
|
12
|
+
|
|
13
|
+
// The encoding of `(module (memory i64 1))` — validates only on hosts with
|
|
14
|
+
// WebAssembly Memory64 support (the same probe `wasm-feature-detect` uses).
|
|
15
|
+
const MEMORY64_PROBE = new Uint8Array([
|
|
16
|
+
0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x05, 0x03, 0x01, 0x04,
|
|
17
|
+
0x01,
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
let MEMORY64_SUPPORTED: boolean | undefined;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Does this JavaScript host support WebAssembly Memory64 (wasm64)?
|
|
24
|
+
*/
|
|
25
|
+
export function host_supports_memory64(): boolean {
|
|
26
|
+
if (MEMORY64_SUPPORTED === undefined) {
|
|
27
|
+
MEMORY64_SUPPORTED = WebAssembly.validate(MEMORY64_PROBE);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return MEMORY64_SUPPORTED;
|
|
31
|
+
}
|
|
@@ -12,9 +12,6 @@
|
|
|
12
12
|
|
|
13
13
|
import type * as perspective_server from "@perspective-dev/server/dist/wasm/perspective-server.js";
|
|
14
14
|
|
|
15
|
-
type PspPtr = bigint | number;
|
|
16
|
-
type EmscriptenServer = number;
|
|
17
|
-
|
|
18
15
|
var out = console.log.bind(console);
|
|
19
16
|
var err = console.error.bind(console);
|
|
20
17
|
|
|
@@ -50,6 +47,151 @@ export default async function (obj: any) {
|
|
|
50
47
|
let psp_module: perspective_server.MainModule;
|
|
51
48
|
let is_memory64 = false;
|
|
52
49
|
let wasm_memory: WebAssembly.Memory;
|
|
50
|
+
|
|
51
|
+
// A `BACKING_STORE_DISK` column's bytes live in wasm linear memory; the
|
|
52
|
+
// residency manager flushes/loads them to/from OPFS through these imports.
|
|
53
|
+
const heap = () => new Uint8Array(wasm_memory.buffer);
|
|
54
|
+
const toAddr = (p: number | bigint) =>
|
|
55
|
+
is_memory64 ? Number(p as bigint) : (p as number) >>> 0;
|
|
56
|
+
|
|
57
|
+
function readCString(p: number | bigint) {
|
|
58
|
+
const addr = toAddr(p);
|
|
59
|
+
const h = heap();
|
|
60
|
+
let end = addr;
|
|
61
|
+
while (h[end]) ++end;
|
|
62
|
+
return UTF8Decoder.decode(h.subarray(addr, end));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// A `DiskBridge` backs `BACKING_STORE_DISK` columns: `store`/`load` flush and
|
|
66
|
+
// re-read a column buffer to/from disk synchronously (the only async step,
|
|
67
|
+
// `ensureOpen`, is hoisted into the JS safepoint between engine calls). The
|
|
68
|
+
// browser default is OPFS; a host can inject an alternative (e.g. `node:fs`
|
|
69
|
+
// in `perspective.node.ts`) via `obj.make_disk_bridge`. `store`/`load`/
|
|
70
|
+
// `remove` receive raw wasm pointers and use the heap helpers; `ensureOpen`
|
|
71
|
+
// receives a resolved file name string (from `victim_fname`).
|
|
72
|
+
const disk_helpers = { heap, toAddr, readCString };
|
|
73
|
+
|
|
74
|
+
function makeOpfsBridge() {
|
|
75
|
+
async function opfsOpenFile(name: string, create: boolean) {
|
|
76
|
+
const parts = name.split("/").filter((s) => s.length > 0);
|
|
77
|
+
// @ts-ignore — navigator.storage.getDirectory is OPFS, Worker-only.
|
|
78
|
+
let dir = await navigator.storage.getDirectory();
|
|
79
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
80
|
+
dir = await dir.getDirectoryHandle(parts[i], { create });
|
|
81
|
+
}
|
|
82
|
+
return await dir.getFileHandle(parts[parts.length - 1], { create });
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// One OPFS `FileSystemSyncAccessHandle` is kept open per disk-backed file
|
|
86
|
+
// for its lifetime.
|
|
87
|
+
const handles = new Map<string, any>();
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
async ensureOpen(name: string) {
|
|
91
|
+
if (handles.has(name)) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const fh = await opfsOpenFile(name, true);
|
|
95
|
+
// @ts-ignore — createSyncAccessHandle is Worker-only OPFS.
|
|
96
|
+
const ah = await fh.createSyncAccessHandle();
|
|
97
|
+
handles.set(name, ah);
|
|
98
|
+
},
|
|
99
|
+
store(
|
|
100
|
+
namePtr: number | bigint,
|
|
101
|
+
dataPtr: number | bigint,
|
|
102
|
+
len: number,
|
|
103
|
+
): number {
|
|
104
|
+
try {
|
|
105
|
+
const ah = handles.get(readCString(namePtr));
|
|
106
|
+
if (!ah) {
|
|
107
|
+
// `commit` only evicts victims whose handle the safepoint
|
|
108
|
+
// just opened, so this should be unreachable.
|
|
109
|
+
return -1;
|
|
110
|
+
}
|
|
111
|
+
ah.truncate(len);
|
|
112
|
+
if (len > 0) {
|
|
113
|
+
const addr = toAddr(dataPtr);
|
|
114
|
+
ah.write(heap().subarray(addr, addr + len), { at: 0 });
|
|
115
|
+
}
|
|
116
|
+
ah.flush();
|
|
117
|
+
return len;
|
|
118
|
+
} catch (e) {
|
|
119
|
+
err("psp_opfs_store failed", e);
|
|
120
|
+
return -1;
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
load(
|
|
124
|
+
namePtr: number | bigint,
|
|
125
|
+
dataPtr: number | bigint,
|
|
126
|
+
len: number,
|
|
127
|
+
): number {
|
|
128
|
+
try {
|
|
129
|
+
if (len <= 0) {
|
|
130
|
+
return 0;
|
|
131
|
+
}
|
|
132
|
+
const ah = handles.get(readCString(namePtr));
|
|
133
|
+
// A restore always follows an evict that left the handle open;
|
|
134
|
+
// a miss (or never-flushed file) reads as zeros.
|
|
135
|
+
if (!ah) {
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
const addr = toAddr(dataPtr);
|
|
139
|
+
return ah.read(heap().subarray(addr, addr + len), {
|
|
140
|
+
at: 0,
|
|
141
|
+
});
|
|
142
|
+
} catch (e) {
|
|
143
|
+
return 0;
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
remove(namePtr: number | bigint) {
|
|
147
|
+
try {
|
|
148
|
+
const name = readCString(namePtr);
|
|
149
|
+
const ah = handles.get(name);
|
|
150
|
+
if (ah) {
|
|
151
|
+
try {
|
|
152
|
+
ah.close();
|
|
153
|
+
} catch (e) {
|
|
154
|
+
/* already closed */
|
|
155
|
+
}
|
|
156
|
+
handles.delete(name);
|
|
157
|
+
}
|
|
158
|
+
// Fire-and-forget the (async) file deletion; the handle is
|
|
159
|
+
// closed, so the bytes are no longer referenced.
|
|
160
|
+
void (async () => {
|
|
161
|
+
try {
|
|
162
|
+
const parts = name
|
|
163
|
+
.split("/")
|
|
164
|
+
.filter((s) => s.length > 0);
|
|
165
|
+
// @ts-ignore
|
|
166
|
+
let dir = await navigator.storage.getDirectory();
|
|
167
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
168
|
+
dir = await dir.getDirectoryHandle(parts[i], {
|
|
169
|
+
create: false,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
await dir.removeEntry(parts[parts.length - 1]);
|
|
173
|
+
} catch (e) {
|
|
174
|
+
/* already gone */
|
|
175
|
+
}
|
|
176
|
+
})();
|
|
177
|
+
} catch (e) {
|
|
178
|
+
/* already gone */
|
|
179
|
+
}
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// The host may inject a disk bridge (e.g. `node:fs`); otherwise default to
|
|
185
|
+
// OPFS. Synchronous `store`/`load`/`remove` env imports — no JSPI.
|
|
186
|
+
const disk = obj.make_disk_bridge
|
|
187
|
+
? obj.make_disk_bridge(disk_helpers)
|
|
188
|
+
: makeOpfsBridge();
|
|
189
|
+
|
|
190
|
+
const opfs_imports: Record<string, any> = {
|
|
191
|
+
psp_opfs_store: disk.store,
|
|
192
|
+
psp_opfs_load: disk.load,
|
|
193
|
+
psp_opfs_remove: disk.remove,
|
|
194
|
+
};
|
|
53
195
|
const module = {
|
|
54
196
|
HaveOffsetConverter() {
|
|
55
197
|
console.error("HaveOffsetConverter");
|
|
@@ -211,6 +353,7 @@ export default async function (obj: any) {
|
|
|
211
353
|
console.error("proc_exit", e);
|
|
212
354
|
return 0;
|
|
213
355
|
},
|
|
356
|
+
...opfs_imports,
|
|
214
357
|
};
|
|
215
358
|
|
|
216
359
|
const x = await obj.instantiateWasm(
|
|
@@ -228,14 +371,41 @@ export default async function (obj: any) {
|
|
|
228
371
|
},
|
|
229
372
|
);
|
|
230
373
|
|
|
374
|
+
// No JSPI: every export is exposed verbatim (emscripten `_`-prefixed alias).
|
|
375
|
+
// Restore is a synchronous import, so `psp_handle_request`/`psp_poll` stay
|
|
376
|
+
// synchronous WASM calls.
|
|
231
377
|
const extensions: Record<string, any> = {};
|
|
232
378
|
for (const [name, func] of Object.entries(x)) {
|
|
233
379
|
extensions[`_${name}`] = func;
|
|
234
380
|
}
|
|
235
381
|
|
|
382
|
+
// JS-driven residency safepoint. The only async OPFS step — opening each
|
|
383
|
+
// victim's sync access handle — happens HERE, between the two synchronous C++
|
|
384
|
+
// phases: `prepare` picks the cold victims (no freeing), we open their
|
|
385
|
+
// handles, then `commit` flushes + frees them (now a synchronous write). The
|
|
386
|
+
// `engine.ts` server-wide lock serializes this against engine calls. A no-op
|
|
387
|
+
// when nothing is over budget.
|
|
388
|
+
const psp_residency_safepoint = async (server: number | bigint) => {
|
|
389
|
+
const mod = psp_module as any;
|
|
390
|
+
if (!mod.psp_residency_prepare) {
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
const n = Number(mod.psp_residency_prepare(server));
|
|
394
|
+
for (let i = 0; i < n; i++) {
|
|
395
|
+
const fname = readCString(
|
|
396
|
+
mod.psp_residency_victim_fname(server, i),
|
|
397
|
+
);
|
|
398
|
+
if (fname) {
|
|
399
|
+
await disk.ensureOpen(fname);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
mod.psp_residency_commit(server);
|
|
403
|
+
};
|
|
404
|
+
|
|
236
405
|
return {
|
|
237
406
|
...x,
|
|
238
407
|
...extensions,
|
|
408
|
+
psp_residency_safepoint,
|
|
239
409
|
get HEAPU8() {
|
|
240
410
|
// @ts-ignore
|
|
241
411
|
return new Uint8Array(wasm_memory.buffer);
|