@perspective-dev/client 4.5.1 → 4.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,4 +19,10 @@ export type TableInitOptions = {
19
19
  * relative to insertion order).
20
20
  */
21
21
  limit?: number;
22
+ /**
23
+ * Back this [`Table`]'s canonical data with the on-disk storage backend
24
+ * instead of memory. On native targets this is a memory-mapped file; on
25
+ * WASM it is OPFS (Worker only). Defaults to in-memory.
26
+ */
27
+ page_to_disk?: boolean;
22
28
  };
@@ -2,4 +2,17 @@ export type * from "../../../dist/wasm/perspective-js.js";
2
2
  import type * as perspective_server_t from "@perspective-dev/server/dist/wasm/perspective-server.js";
3
3
  export type PspPtr = BigInt | number;
4
4
  export type EmscriptenServer = bigint | number;
5
- export declare function compile_perspective(wasmBinary: ArrayBuffer): Promise<perspective_server_t.MainModule>;
5
+ export interface DiskBridgeHelpers {
6
+ heap: () => Uint8Array;
7
+ toAddr: (p: number | bigint) => number;
8
+ readCString: (p: number | bigint) => string;
9
+ }
10
+ export interface CompileOptions {
11
+ make_disk_bridge?: (helpers: DiskBridgeHelpers) => {
12
+ store(namePtr: number | bigint, dataPtr: number | bigint, len: number): number;
13
+ load(namePtr: number | bigint, dataPtr: number | bigint, len: number): number;
14
+ remove(namePtr: number | bigint): void;
15
+ ensureOpen(name: string): Promise<void>;
16
+ };
17
+ }
18
+ export declare function compile_perspective(wasmBinary: ArrayBuffer, opts?: CompileOptions): Promise<perspective_server_t.MainModule>;
@@ -7,6 +7,13 @@ export type ApiResponse = {
7
7
  export interface PerspectiveServerOptions {
8
8
  on_poll_request?: (x: PerspectiveServer) => Promise<void>;
9
9
  }
10
+ /**
11
+ * A server-wide FIFO async mutex.
12
+ */
13
+ declare class AsyncLock {
14
+ private tail;
15
+ run<T>(fn: () => Promise<T> | T): Promise<T>;
16
+ }
10
17
  export declare class PerspectivePollThread {
11
18
  private poll_handle?;
12
19
  private server;
@@ -19,6 +26,7 @@ export declare class PerspectiveServer {
19
26
  server: EmscriptenServer;
20
27
  module: MainModule;
21
28
  on_poll_request?: (x: PerspectiveServer) => Promise<void>;
29
+ lock: AsyncLock;
22
30
  constructor(module: MainModule, options?: PerspectiveServerOptions);
23
31
  /**
24
32
  * Helper function to create server emitter/receiver pairs
@@ -32,9 +40,11 @@ export declare class PerspectiveSession {
32
40
  private server;
33
41
  private client_id;
34
42
  private client_map;
43
+ private lock;
35
44
  private on_poll_request?;
36
- constructor(mod: MainModule, server: EmscriptenServer, client_id: number, client_map: Map<number, (buffer: Uint8Array) => Promise<void>>, on_poll_request?: (() => Promise<void>) | undefined);
45
+ constructor(mod: MainModule, server: EmscriptenServer, client_id: number, client_map: Map<number, (buffer: Uint8Array) => Promise<void>>, lock: AsyncLock, on_poll_request?: (() => Promise<void>) | undefined);
37
46
  handle_request(view: Uint8Array): Promise<void>;
38
47
  private poll;
39
48
  close(): void;
40
49
  }
50
+ export {};
@@ -874,11 +874,11 @@ export interface InitOutput {
874
874
  readonly virtualdataslice_setStringCol: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
875
875
  readonly virtualserver_handleRequest: (a: number, b: number, c: number) => number;
876
876
  readonly virtualserver_new: (a: number, b: number) => void;
877
- readonly __wasm_bindgen_func_elem_860: (a: number, b: number) => void;
878
- readonly __wasm_bindgen_func_elem_4015: (a: number, b: number) => void;
879
- readonly __wasm_bindgen_func_elem_11006: (a: number, b: number, c: number, d: number) => void;
880
- readonly __wasm_bindgen_func_elem_4032: (a: number, b: number, c: number) => void;
881
- readonly __wasm_bindgen_func_elem_1768: (a: number, b: number) => number;
877
+ readonly __wasm_bindgen_func_elem_1085: (a: number, b: number) => void;
878
+ readonly __wasm_bindgen_func_elem_4003: (a: number, b: number) => void;
879
+ readonly __wasm_bindgen_func_elem_10943: (a: number, b: number, c: number, d: number) => void;
880
+ readonly __wasm_bindgen_func_elem_4020: (a: number, b: number, c: number) => void;
881
+ readonly __wasm_bindgen_func_elem_1757: (a: number, b: number) => number;
882
882
  readonly __wbindgen_export: (a: number, b: number) => number;
883
883
  readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
884
884
  readonly __wbindgen_export3: (a: number) => void;
@@ -2016,7 +2016,7 @@ function __wbg_get_imports() {
2016
2016
  const a = state0.a;
2017
2017
  state0.a = 0;
2018
2018
  try {
2019
- return __wasm_bindgen_func_elem_11006(a, state0.b, arg0, arg1);
2019
+ return __wasm_bindgen_func_elem_10943(a, state0.b, arg0, arg1);
2020
2020
  } finally {
2021
2021
  state0.a = a;
2022
2022
  }
@@ -2175,12 +2175,12 @@ function __wbg_get_imports() {
2175
2175
  },
2176
2176
  __wbindgen_cast_0000000000000001: function(arg0, arg1) {
2177
2177
  // Cast intrinsic for `Closure(Closure { dtor_idx: 12, function: Function { arguments: [], shim_idx: 31, ret: NamedExternref("Promise<any>"), inner_ret: Some(NamedExternref("Promise<any>")) }, mutable: false }) -> Externref`.
2178
- const ret = makeClosure(arg0, arg1, wasm.__wasm_bindgen_func_elem_860, __wasm_bindgen_func_elem_1768);
2178
+ const ret = makeClosure(arg0, arg1, wasm.__wasm_bindgen_func_elem_1085, __wasm_bindgen_func_elem_1757);
2179
2179
  return addHeapObject(ret);
2180
2180
  },
2181
2181
  __wbindgen_cast_0000000000000002: function(arg0, arg1) {
2182
2182
  // Cast intrinsic for `Closure(Closure { dtor_idx: 568, function: Function { arguments: [Externref], shim_idx: 569, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`.
2183
- const ret = makeMutClosure(arg0, arg1, wasm.__wasm_bindgen_func_elem_4015, __wasm_bindgen_func_elem_4032);
2183
+ const ret = makeMutClosure(arg0, arg1, wasm.__wasm_bindgen_func_elem_4003, __wasm_bindgen_func_elem_4020);
2184
2184
  return addHeapObject(ret);
2185
2185
  },
2186
2186
  __wbindgen_cast_0000000000000003: function(arg0) {
@@ -2257,17 +2257,17 @@ function __wbg_get_imports() {
2257
2257
  };
2258
2258
  }
2259
2259
 
2260
- function __wasm_bindgen_func_elem_1768(arg0, arg1) {
2261
- const ret = wasm.__wasm_bindgen_func_elem_1768(arg0, arg1);
2260
+ function __wasm_bindgen_func_elem_1757(arg0, arg1) {
2261
+ const ret = wasm.__wasm_bindgen_func_elem_1757(arg0, arg1);
2262
2262
  return takeObject(ret);
2263
2263
  }
2264
2264
 
2265
- function __wasm_bindgen_func_elem_4032(arg0, arg1, arg2) {
2266
- wasm.__wasm_bindgen_func_elem_4032(arg0, arg1, addHeapObject(arg2));
2265
+ function __wasm_bindgen_func_elem_4020(arg0, arg1, arg2) {
2266
+ wasm.__wasm_bindgen_func_elem_4020(arg0, arg1, addHeapObject(arg2));
2267
2267
  }
2268
2268
 
2269
- function __wasm_bindgen_func_elem_11006(arg0, arg1, arg2, arg3) {
2270
- wasm.__wasm_bindgen_func_elem_11006(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3));
2269
+ function __wasm_bindgen_func_elem_10943(arg0, arg1, arg2, arg3) {
2270
+ wasm.__wasm_bindgen_func_elem_10943(arg0, arg1, addHeapObject(arg2), addHeapObject(arg3));
2271
2271
  }
2272
2272
 
2273
2273
  const ClientFinalization = (typeof FinalizationRegistry === 'undefined')
Binary file
@@ -93,11 +93,11 @@ export const virtualdataslice_setIntegerCol: (a: number, b: number, c: number, d
93
93
  export const virtualdataslice_setStringCol: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
94
94
  export const virtualserver_handleRequest: (a: number, b: number, c: number) => number;
95
95
  export const virtualserver_new: (a: number, b: number) => void;
96
- export const __wasm_bindgen_func_elem_860: (a: number, b: number) => void;
97
- export const __wasm_bindgen_func_elem_4015: (a: number, b: number) => void;
98
- export const __wasm_bindgen_func_elem_11006: (a: number, b: number, c: number, d: number) => void;
99
- export const __wasm_bindgen_func_elem_4032: (a: number, b: number, c: number) => void;
100
- export const __wasm_bindgen_func_elem_1768: (a: number, b: number) => number;
96
+ export const __wasm_bindgen_func_elem_1085: (a: number, b: number) => void;
97
+ export const __wasm_bindgen_func_elem_4003: (a: number, b: number) => void;
98
+ export const __wasm_bindgen_func_elem_10943: (a: number, b: number, c: number, d: number) => void;
99
+ export const __wasm_bindgen_func_elem_4020: (a: number, b: number, c: number) => void;
100
+ export const __wasm_bindgen_func_elem_1757: (a: number, b: number) => number;
101
101
  export const __wbindgen_export: (a: number, b: number) => number;
102
102
  export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
103
103
  export const __wbindgen_export3: (a: number) => void;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@perspective-dev/client",
3
- "version": "4.5.1",
3
+ "version": "4.5.2",
4
4
  "description": "Client for Perspective, a high-performance WASM engine with reactive Tables, Views, and joins.",
5
5
  "keywords": [
6
6
  "perspective",
@@ -16,6 +16,16 @@ export type * from "./virtual_server.ts";
16
16
  import WebSocket, { WebSocketServer as HttpWebSocketServer } from "ws";
17
17
  import stoppable from "stoppable";
18
18
  import { promises as fs } from "node:fs";
19
+ import {
20
+ openSync,
21
+ readSync,
22
+ writeSync,
23
+ closeSync,
24
+ mkdirSync,
25
+ unlinkSync,
26
+ rmSync,
27
+ } from "node:fs";
28
+ import os from "node:os";
19
29
  import http from "node:http";
20
30
  import path from "node:path";
21
31
  import { webcrypto } from "node:crypto";
@@ -56,12 +66,123 @@ const uncompressed_client_wasm = await fs
56
66
  .then((buffer) => load_wasm_stage_0(buffer.buffer as ArrayBuffer));
57
67
 
58
68
  await perspective_client.default({ module_or_path: uncompressed_client_wasm });
69
+
70
+ function make_node_disk_bridge({
71
+ heap,
72
+ toAddr,
73
+ readCString,
74
+ }: {
75
+ heap: () => Uint8Array;
76
+ toAddr: (p: number | bigint) => number;
77
+ readCString: (p: number | bigint) => string;
78
+ }) {
79
+ const root = path.join(os.tmpdir(), `perspective-${process.pid}`);
80
+ const resolve_path = (name: string) => {
81
+ const p = path.join(root, name);
82
+ if (!p.startsWith(root)) {
83
+ throw new Error(`refusing disk path outside root: ${name}`);
84
+ }
85
+ return p;
86
+ };
87
+
88
+ let cleaned = false;
89
+ const cleanup = () => {
90
+ if (cleaned) return;
91
+ cleaned = true;
92
+ try {
93
+ rmSync(root, { recursive: true, force: true });
94
+ } catch (e) {
95
+ /* best effort */
96
+ }
97
+ };
98
+ process.on("exit", cleanup);
99
+ process.on("SIGINT", () => {
100
+ cleanup();
101
+ process.exit(130);
102
+ });
103
+
104
+ return {
105
+ // node:fs is synchronous — nothing to pre-open between safepoint phases.
106
+ async ensureOpen(_name: string) {},
107
+ store(
108
+ namePtr: number | bigint,
109
+ dataPtr: number | bigint,
110
+ len: number,
111
+ ): number {
112
+ try {
113
+ const p = resolve_path(readCString(namePtr));
114
+ mkdirSync(path.dirname(p), { recursive: true });
115
+ const fd = openSync(p, "w");
116
+ try {
117
+ if (len > 0) {
118
+ const addr = toAddr(dataPtr);
119
+ const view = heap().subarray(addr, addr + len);
120
+ let off = 0;
121
+ while (off < len) {
122
+ off += writeSync(fd, view, off, len - off, off);
123
+ }
124
+ }
125
+ } finally {
126
+ closeSync(fd);
127
+ }
128
+ return len;
129
+ } catch (e) {
130
+ console.error("node disk store failed", e);
131
+ return -1;
132
+ }
133
+ },
134
+ load(
135
+ namePtr: number | bigint,
136
+ dataPtr: number | bigint,
137
+ len: number,
138
+ ): number {
139
+ try {
140
+ if (len <= 0) return 0;
141
+ let fd: number;
142
+ try {
143
+ fd = openSync(resolve_path(readCString(namePtr)), "r");
144
+ } catch (e) {
145
+ return 0; // never-flushed file reads as zeros
146
+ }
147
+ try {
148
+ const addr = toAddr(dataPtr);
149
+ const view = heap().subarray(addr, addr + len);
150
+ let off = 0;
151
+ let n: number;
152
+ while (
153
+ off < len &&
154
+ (n = readSync(fd, view, off, len - off, off)) > 0
155
+ ) {
156
+ off += n;
157
+ }
158
+ return off;
159
+ } finally {
160
+ closeSync(fd);
161
+ }
162
+ } catch (e) {
163
+ return 0;
164
+ }
165
+ },
166
+ remove(namePtr: number | bigint) {
167
+ try {
168
+ unlinkSync(resolve_path(readCString(namePtr)));
169
+ } catch (e) {
170
+ /* already gone */
171
+ }
172
+ },
173
+ };
174
+ }
175
+
59
176
  const SYNC_MODULE = await fs
60
177
  .readFile(
61
178
  resolve("@perspective-dev/server/dist/wasm/perspective-server.wasm"),
62
179
  )
63
180
  .then((buffer) => load_wasm_stage_0(buffer.buffer as ArrayBuffer))
64
- .then((buffer) => compile_perspective(buffer.buffer as ArrayBuffer));
181
+ .then((buffer) =>
182
+ compile_perspective(buffer.buffer as ArrayBuffer, {
183
+ make_disk_bridge: make_node_disk_bridge,
184
+ }),
185
+ );
65
186
 
66
187
  let SYNC_CLIENT: perspective_client.Client;
67
188
 
@@ -18,4 +18,10 @@ index?: string,
18
18
  * _earliest_ rows will be overwritten (where _earliest_ is defined as
19
19
  * relative to insertion order).
20
20
  */
21
- limit?: number, };
21
+ limit?: number,
22
+ /**
23
+ * Back this [`Table`]'s canonical data with the on-disk storage backend
24
+ * instead of memory. On native targets this is a memory-mapped file; on
25
+ * WASM it is OPFS (Worker only). Defaults to in-memory.
26
+ */
27
+ page_to_disk?: boolean, };
@@ -17,13 +17,38 @@ import type * as perspective_server_t from "@perspective-dev/server/dist/wasm/pe
17
17
  export type PspPtr = BigInt | number;
18
18
  export type EmscriptenServer = bigint | number;
19
19
 
20
+ export interface DiskBridgeHelpers {
21
+ heap: () => Uint8Array;
22
+ toAddr: (p: number | bigint) => number;
23
+ readCString: (p: number | bigint) => string;
24
+ }
25
+
26
+ export interface CompileOptions {
27
+ make_disk_bridge?: (helpers: DiskBridgeHelpers) => {
28
+ store(
29
+ namePtr: number | bigint,
30
+ dataPtr: number | bigint,
31
+ len: number,
32
+ ): number;
33
+ load(
34
+ namePtr: number | bigint,
35
+ dataPtr: number | bigint,
36
+ len: number,
37
+ ): number;
38
+ remove(namePtr: number | bigint): void;
39
+ ensureOpen(name: string): Promise<void>;
40
+ };
41
+ }
42
+
20
43
  export async function compile_perspective(
21
44
  wasmBinary: ArrayBuffer,
45
+ opts?: CompileOptions,
22
46
  ): Promise<perspective_server_t.MainModule> {
23
47
  const module = await perspective_server.default({
24
48
  locateFile(x: any) {
25
49
  return x;
26
50
  },
51
+ make_disk_bridge: opts?.make_disk_bridge,
27
52
  instantiateWasm: async (
28
53
  imports: any,
29
54
  receive: (_: WebAssembly.Instance) => void,
@@ -22,6 +22,33 @@ export interface PerspectiveServerOptions {
22
22
  on_poll_request?: (x: PerspectiveServer) => Promise<void>;
23
23
  }
24
24
 
25
+ /**
26
+ * A server-wide FIFO async mutex.
27
+ */
28
+ class AsyncLock {
29
+ private tail: Promise<unknown> = Promise.resolve();
30
+ run<T>(fn: () => Promise<T> | T): Promise<T> {
31
+ const result = this.tail.then(fn, fn);
32
+ this.tail = result.then(
33
+ () => undefined,
34
+ () => undefined,
35
+ );
36
+
37
+ return result;
38
+ }
39
+ }
40
+
41
+ async function residency_safepoint(
42
+ mod: MainModule,
43
+ lock: AsyncLock,
44
+ server: EmscriptenServer,
45
+ ) {
46
+ const safepoint = (mod as any).psp_residency_safepoint;
47
+ if (safepoint) {
48
+ await lock.run(() => safepoint(server));
49
+ }
50
+ }
51
+
25
52
  export class PerspectivePollThread {
26
53
  private poll_handle?: Promise<void>;
27
54
  private server: PerspectiveServer;
@@ -63,10 +90,13 @@ export class PerspectiveServer {
63
90
  server: EmscriptenServer;
64
91
  module: MainModule;
65
92
  on_poll_request?: (x: PerspectiveServer) => Promise<void>;
93
+ lock: AsyncLock;
94
+
66
95
  constructor(module: MainModule, options?: PerspectiveServerOptions) {
67
96
  this.clients = new Map();
68
97
  this.module = module;
69
98
  this.on_poll_request = options?.on_poll_request;
99
+ this.lock = new AsyncLock();
70
100
  this.server = module._psp_new_server(
71
101
  !!options?.on_poll_request ? 1 : 0,
72
102
  ) as EmscriptenServer;
@@ -85,12 +115,16 @@ export class PerspectiveServer {
85
115
  this.server,
86
116
  client_id,
87
117
  this.clients,
118
+ this.lock,
88
119
  this.on_poll_request && (() => this.on_poll_request!(this)),
89
120
  );
90
121
  }
91
122
 
92
123
  async poll() {
93
- const polled = this.module._psp_poll(this.server as any);
124
+ const polled = await this.lock.run(() =>
125
+ this.module._psp_poll(this.server as any),
126
+ );
127
+
94
128
  await decode_api_responses(
95
129
  this.module,
96
130
  polled,
@@ -98,6 +132,8 @@ export class PerspectiveServer {
98
132
  await this.clients.get(msg.client_id)!(msg.data);
99
133
  },
100
134
  );
135
+
136
+ await residency_safepoint(this.module, this.lock, this.server);
101
137
  }
102
138
 
103
139
  delete() {
@@ -111,6 +147,7 @@ export class PerspectiveSession {
111
147
  private server: EmscriptenServer,
112
148
  private client_id: number,
113
149
  private client_map: Map<number, (buffer: Uint8Array) => Promise<void>>,
150
+ private lock: AsyncLock,
114
151
  private on_poll_request?: () => Promise<void>,
115
152
  ) {}
116
153
 
@@ -119,13 +156,17 @@ export class PerspectiveSession {
119
156
  this.mod,
120
157
  view,
121
158
  async (viewPtr) => {
122
- return this.mod._psp_handle_request(
123
- this.server as any,
124
- this.client_id,
125
- viewPtr as any,
126
- this.mod._psp_is_memory64()
127
- ? (BigInt(view.byteLength) as any as number)
128
- : (view.byteLength as any),
159
+ const len = this.mod._psp_is_memory64()
160
+ ? (BigInt(view.byteLength) as any as number)
161
+ : (view.byteLength as any);
162
+
163
+ return this.lock.run(() =>
164
+ this.mod._psp_handle_request(
165
+ this.server as any,
166
+ this.client_id,
167
+ viewPtr as any,
168
+ len,
169
+ ),
129
170
  );
130
171
  },
131
172
  );
@@ -139,10 +180,14 @@ export class PerspectiveSession {
139
180
  } else {
140
181
  await this.poll();
141
182
  }
183
+
184
+ await residency_safepoint(this.mod, this.lock, this.server);
142
185
  }
143
186
 
144
187
  private async poll() {
145
- const polled = this.mod._psp_poll(this.server as any);
188
+ const polled = await this.lock.run(() =>
189
+ this.mod._psp_poll(this.server as any),
190
+ );
146
191
  await decode_api_responses(
147
192
  this.mod,
148
193
  polled,
@@ -172,9 +217,9 @@ async function convert_typed_array_to_pointer(
172
217
  : (array.byteLength as any),
173
218
  );
174
219
 
175
- core.HEAPU8.set(array, Number(ptr));
176
- const msg = await callback(ptr);
177
- core._psp_free(ptr);
220
+ core.HEAPU8.set(array, Number(ptr) >>> 0);
221
+ const msg = await callback(Number(ptr) >>> 0);
222
+ core._psp_free(Number(ptr) >>> 0);
178
223
  return msg;
179
224
  }
180
225
 
@@ -210,7 +255,7 @@ async function decode_api_responses(
210
255
  const is_64 = core._psp_is_memory64();
211
256
  const response = new DataView(
212
257
  core.HEAPU8.buffer,
213
- Number(ptr),
258
+ Number(ptr) >>> 0,
214
259
  is_64 ? 12 : 8,
215
260
  );
216
261