@spooky-sync/core 0.0.1-canary.155 → 0.0.1-canary.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +31 -0
- package/dist/index.js +154 -24
- package/dist/types.d.ts +30 -0
- package/package.json +3 -3
- package/src/modules/devtools/index.ts +51 -14
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/modules/devtools/storage-info.ts +27 -0
- package/src/services/stream-processor/index.ts +146 -20
- package/src/services/stream-processor/stream-processor.reset.test.ts +114 -2
- package/src/services/stream-processor/wasm-types.ts +5 -0
- package/src/sp00ky.ts +80 -40
- package/src/types.ts +23 -0
|
@@ -31,10 +31,37 @@ export interface EngineStorageDiagnostics {
|
|
|
31
31
|
error?: string;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Shared-tabs coordination state. Reported ONLY when `sharedTabs: true` was
|
|
36
|
+
* configured (apps that never asked for it get `null`, so the panel shows
|
|
37
|
+
* nothing). `active: false` with a `reason` is itself the useful signal: the
|
|
38
|
+
* app asked to share one store and this tab is not, so it owns or contends for
|
|
39
|
+
* the OPFS pool alone.
|
|
40
|
+
*/
|
|
41
|
+
export interface SharedTabsInfo {
|
|
42
|
+
active: boolean;
|
|
43
|
+
/** Why inactive: a capability gate reason, or 'fell-back' when the broker
|
|
44
|
+
* was reachable but no role landed (election timeout, rejected tab). */
|
|
45
|
+
reason?: string;
|
|
46
|
+
role?: 'solo' | 'leader' | 'follower';
|
|
47
|
+
/** This tab's id (also the suffix of every mutation id it mints). */
|
|
48
|
+
tabId?: string;
|
|
49
|
+
/** Monotonic id of the current leadership term; embedded in the worker lock. */
|
|
50
|
+
leadershipId?: number;
|
|
51
|
+
/** Who owns the store: this tab when leader, else the leader's tab id. */
|
|
52
|
+
leaderTabId?: string | null;
|
|
53
|
+
/** Leader only: attached follower tabs. */
|
|
54
|
+
followers?: number;
|
|
55
|
+
/** Leader only: ingest batches relayed to followers so far. */
|
|
56
|
+
relayedBatches?: number;
|
|
57
|
+
}
|
|
58
|
+
|
|
34
59
|
export interface StorageInfo {
|
|
35
60
|
at: number;
|
|
36
61
|
engine: { kind: 'surrealdb' | 'sqlite' | 'custom'; store: string; bucketId: string };
|
|
37
62
|
health: { status: 'unknown' | 'persistent' | 'memory'; fallback: boolean; error?: string };
|
|
63
|
+
/** Shared-tabs state, or null when the feature was never requested. */
|
|
64
|
+
tabs?: SharedTabsInfo | null;
|
|
38
65
|
browser: {
|
|
39
66
|
/** `navigator.storage.persisted()` — whether the origin's storage is
|
|
40
67
|
* exempt from eviction (unrelated to the OPFS pool lock). */
|
|
@@ -57,6 +57,21 @@ export interface StreamUpdateReceiver {
|
|
|
57
57
|
onStreamUpdate(update: StreamUpdate): void;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Read a circuit snapshot out of the pre-`persistCircuit` persisted shape
|
|
62
|
+
* (`[[{ state }]]`, a raw SurrealDB result). Returns null for anything else.
|
|
63
|
+
*/
|
|
64
|
+
function extractLegacyState(result: unknown): string | null {
|
|
65
|
+
if (
|
|
66
|
+
Array.isArray(result) &&
|
|
67
|
+
Array.isArray(result[0]) &&
|
|
68
|
+
typeof result[0][0]?.state === 'string'
|
|
69
|
+
) {
|
|
70
|
+
return result[0][0].state;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
60
75
|
export class StreamProcessorService {
|
|
61
76
|
private logger: Logger;
|
|
62
77
|
private processor: WasmProcessor | undefined;
|
|
@@ -89,6 +104,32 @@ export class StreamProcessorService {
|
|
|
89
104
|
// stomp the leader's snapshot under the same key (the pre-existing cross-tab
|
|
90
105
|
// localStorage hazard); a promoted follower flips this back on.
|
|
91
106
|
private persistState = true;
|
|
107
|
+
// Snapshot persistence is OPT-IN and off by default (`persistCircuit`).
|
|
108
|
+
//
|
|
109
|
+
// `Circuit::save` deep-clones the WHOLE store (every row of every ingested
|
|
110
|
+
// table, full bodies) plus every view cache and JSON-encodes the result. It
|
|
111
|
+
// used to run from `ingest`, `flushCoalescing`, `registerQueryPlan` and
|
|
112
|
+
// `unregisterQueryPlan`, i.e. once per sync batch AND once per query
|
|
113
|
+
// register/unregister. On a 3.7k-game collection whose list registers and
|
|
114
|
+
// drops one windowed query per 50 rows scrolled, that measured 220 whole-store
|
|
115
|
+
// serializations and ~1 GB of transient JSON for a single scroll, 5.5x the
|
|
116
|
+
// wall time of the same ingests without it, and it doubled the wasm heap
|
|
117
|
+
// high-water mark (wasm32 dlmalloc never returns pages, so the peak is
|
|
118
|
+
// permanent). It also ran the encode a second time in
|
|
119
|
+
// `LocalStoragePersistenceClient.set` and wrote it synchronously on the main
|
|
120
|
+
// thread. All of it was dead weight: `loadState`'s shape check could never
|
|
121
|
+
// match what either shipped persistence client returns, so the browser never
|
|
122
|
+
// restored a snapshot, and first paint comes from the local SQLite store
|
|
123
|
+
// anyway.
|
|
124
|
+
//
|
|
125
|
+
// When enabled, we mark the circuit dirty and let a checkpoint timer (plus a
|
|
126
|
+
// `pagehide` flush) do at most one snapshot per interval, mirroring
|
|
127
|
+
// `ssp-node`'s "NEVER per-ingest" rule.
|
|
128
|
+
private persistCircuit = false;
|
|
129
|
+
private checkpointMs = 30_000;
|
|
130
|
+
private checkpointTimer: ReturnType<typeof setInterval> | null = null;
|
|
131
|
+
private snapshotDirty = false;
|
|
132
|
+
private pagehideHandler: (() => void) | null = null;
|
|
92
133
|
|
|
93
134
|
constructor(
|
|
94
135
|
public events: EventSystem<StreamProcessorEvents>,
|
|
@@ -201,10 +242,7 @@ export class StreamProcessorService {
|
|
|
201
242
|
if (buffered.length > 0) {
|
|
202
243
|
this.dispatchUpdates(buffered);
|
|
203
244
|
}
|
|
204
|
-
|
|
205
|
-
// snapshot covers the whole batch. Kept fire-and-forget like the per-ingest
|
|
206
|
-
// call it replaces.
|
|
207
|
-
this.saveState();
|
|
245
|
+
this.markSnapshotDirty();
|
|
208
246
|
}
|
|
209
247
|
|
|
210
248
|
/**
|
|
@@ -265,32 +303,120 @@ export class StreamProcessorService {
|
|
|
265
303
|
this.stateGeneration++;
|
|
266
304
|
this.batching = false;
|
|
267
305
|
this.batchBuffer.clear();
|
|
306
|
+
this.snapshotDirty = false;
|
|
307
|
+
const previous = this.processor;
|
|
268
308
|
this.processor = new Sp00kyProcessor() as unknown as WasmProcessor;
|
|
309
|
+
this.freeProcessor(previous);
|
|
269
310
|
this.logger.info(
|
|
270
311
|
{ Category: 'sp00ky-client::StreamProcessorService::reset' },
|
|
271
312
|
'Stream processor reset (fresh circuit)'
|
|
272
313
|
);
|
|
273
314
|
}
|
|
274
315
|
|
|
316
|
+
/**
|
|
317
|
+
* Release the wasm circuit and stop checkpointing. Call when the client is
|
|
318
|
+
* torn down; a recreated client (provider remount, HMR) would otherwise stack
|
|
319
|
+
* one full circuit per instance.
|
|
320
|
+
*/
|
|
321
|
+
dispose(): void {
|
|
322
|
+
this.stopCheckpoints();
|
|
323
|
+
const previous = this.processor;
|
|
324
|
+
this.processor = undefined;
|
|
325
|
+
this.isInitialized = false;
|
|
326
|
+
this.batching = false;
|
|
327
|
+
this.batchBuffer.clear();
|
|
328
|
+
this.receivers = [];
|
|
329
|
+
this.freeProcessor(previous);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Explicitly run the wasm-bindgen destructor. Guarded: stale wasm builds may
|
|
334
|
+
* not expose `free`, and a double free must not take the app down.
|
|
335
|
+
*/
|
|
336
|
+
private freeProcessor(processor: WasmProcessor | undefined): void {
|
|
337
|
+
if (!processor || typeof processor.free !== 'function') return;
|
|
338
|
+
try {
|
|
339
|
+
processor.free();
|
|
340
|
+
} catch (e) {
|
|
341
|
+
this.logger.debug(
|
|
342
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::freeProcessor' },
|
|
343
|
+
'Failed to free previous wasm circuit'
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
275
348
|
/** Toggle circuit-state persistence (shared-tabs follower/leader role). */
|
|
276
349
|
setPersistenceEnabled(enabled: boolean): void {
|
|
277
350
|
this.persistState = enabled;
|
|
351
|
+
if (!enabled) this.stopCheckpoints();
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Opt into snapshot persistence (`persistCircuit`). Off by default: see the
|
|
356
|
+
* `persistCircuit` field comment for why per-ingest snapshots were removed.
|
|
357
|
+
* Must be called before `init()` for a snapshot to be restored at boot.
|
|
358
|
+
*/
|
|
359
|
+
configureCircuitPersistence(enabled: boolean, checkpointMs?: number): void {
|
|
360
|
+
this.persistCircuit = enabled;
|
|
361
|
+
if (checkpointMs && checkpointMs > 0) this.checkpointMs = checkpointMs;
|
|
362
|
+
if (!enabled) this.stopCheckpoints();
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Record that the circuit changed. Cheap and O(1), the expensive snapshot is
|
|
367
|
+
* deferred to the checkpoint timer, and skipped entirely when
|
|
368
|
+
* `persistCircuit` is off (the default).
|
|
369
|
+
*/
|
|
370
|
+
private markSnapshotDirty(): void {
|
|
371
|
+
if (!this.persistCircuit || !this.persistState) return;
|
|
372
|
+
this.snapshotDirty = true;
|
|
373
|
+
this.startCheckpoints();
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
private startCheckpoints(): void {
|
|
377
|
+
if (this.checkpointTimer) return;
|
|
378
|
+
this.checkpointTimer = setInterval(() => {
|
|
379
|
+
if (!this.snapshotDirty) return;
|
|
380
|
+
this.snapshotDirty = false;
|
|
381
|
+
void this.saveState();
|
|
382
|
+
}, this.checkpointMs);
|
|
383
|
+
// Node/test environments have no `window`; the interval alone is enough there.
|
|
384
|
+
if (typeof window !== 'undefined' && !this.pagehideHandler) {
|
|
385
|
+
this.pagehideHandler = () => {
|
|
386
|
+
if (!this.snapshotDirty) return;
|
|
387
|
+
this.snapshotDirty = false;
|
|
388
|
+
void this.saveState();
|
|
389
|
+
};
|
|
390
|
+
window.addEventListener('pagehide', this.pagehideHandler);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Stop checkpointing and drop the `pagehide` listener. */
|
|
395
|
+
stopCheckpoints(): void {
|
|
396
|
+
if (this.checkpointTimer) {
|
|
397
|
+
clearInterval(this.checkpointTimer);
|
|
398
|
+
this.checkpointTimer = null;
|
|
399
|
+
}
|
|
400
|
+
if (this.pagehideHandler && typeof window !== 'undefined') {
|
|
401
|
+
window.removeEventListener('pagehide', this.pagehideHandler);
|
|
402
|
+
}
|
|
403
|
+
this.pagehideHandler = null;
|
|
404
|
+
this.snapshotDirty = false;
|
|
278
405
|
}
|
|
279
406
|
|
|
280
407
|
async loadState() {
|
|
281
|
-
if (!this.processor || !this.persistState) return;
|
|
408
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
282
409
|
try {
|
|
283
410
|
const result = await this.persistenceClient.get(this.stateKey());
|
|
284
411
|
|
|
285
|
-
//
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
) {
|
|
293
|
-
const state = result[0][0].state;
|
|
412
|
+
// `save_state` returns a JSON string and every PersistenceClient round-trips
|
|
413
|
+
// it as one: localStorage via JSON.parse(getItem(...)), surrealdb via
|
|
414
|
+
// `_00_kv:<key>.val`. This used to test for a raw SurrealDB result shape
|
|
415
|
+
// (`result[0][0].state`), which no shipped client can ever produce, so the
|
|
416
|
+
// browser wrote a snapshot on every ingest and never restored one. The legacy
|
|
417
|
+
// shape is still accepted so an old persisted row still loads.
|
|
418
|
+
const state = typeof result === 'string' ? result : extractLegacyState(result);
|
|
419
|
+
if (state) {
|
|
294
420
|
this.logger.info(
|
|
295
421
|
{
|
|
296
422
|
stateLength: state.length,
|
|
@@ -369,7 +495,7 @@ export class StreamProcessorService {
|
|
|
369
495
|
}
|
|
370
496
|
|
|
371
497
|
async saveState() {
|
|
372
|
-
if (!this.processor || !this.persistState) return;
|
|
498
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
373
499
|
const generation = this.stateGeneration;
|
|
374
500
|
try {
|
|
375
501
|
// Assuming processor has a save_state method that returns the state string/bytes
|
|
@@ -454,10 +580,10 @@ export class StreamProcessorService {
|
|
|
454
580
|
// Direct handler call instead of event
|
|
455
581
|
this.notifyUpdates(updates);
|
|
456
582
|
}
|
|
457
|
-
// While batching (inside `ingestMany`), `flushCoalescing`
|
|
458
|
-
// for the whole batch
|
|
583
|
+
// While batching (inside `ingestMany`), `flushCoalescing` marks dirty once
|
|
584
|
+
// for the whole batch, skip the redundant per-record mark here.
|
|
459
585
|
if (!this.batching) {
|
|
460
|
-
this.
|
|
586
|
+
this.markSnapshotDirty();
|
|
461
587
|
}
|
|
462
588
|
return rawUpdates;
|
|
463
589
|
} catch (e) {
|
|
@@ -534,7 +660,7 @@ export class StreamProcessorService {
|
|
|
534
660
|
snapshotMs: initialUpdate.timing_snapshot_ms ?? 0,
|
|
535
661
|
},
|
|
536
662
|
};
|
|
537
|
-
this.
|
|
663
|
+
this.markSnapshotDirty();
|
|
538
664
|
this.logger.debug(
|
|
539
665
|
{
|
|
540
666
|
queryHash: queryPlan.queryHash,
|
|
@@ -561,7 +687,7 @@ export class StreamProcessorService {
|
|
|
561
687
|
if (!this.processor) return;
|
|
562
688
|
try {
|
|
563
689
|
this.processor.unregister_view(queryHash);
|
|
564
|
-
this.
|
|
690
|
+
this.markSnapshotDirty();
|
|
565
691
|
} catch (e) {
|
|
566
692
|
this.logger.error(
|
|
567
693
|
{ error: e, Category: 'sp00ky-client::StreamProcessorService::unregisterQueryPlan' },
|
|
@@ -18,6 +18,7 @@ vi.mock('@spooky-sync/ssp-wasm', () => ({
|
|
|
18
18
|
set_permissions: vi.fn(),
|
|
19
19
|
save_state: vi.fn(() => 'state-bytes'),
|
|
20
20
|
load_state: vi.fn(),
|
|
21
|
+
free: vi.fn(),
|
|
21
22
|
};
|
|
22
23
|
processorInstances.push(instance);
|
|
23
24
|
return instance;
|
|
@@ -44,7 +45,12 @@ function makeService() {
|
|
|
44
45
|
get: vi.fn(async () => null),
|
|
45
46
|
remove: vi.fn(async () => {}),
|
|
46
47
|
};
|
|
47
|
-
const service = new StreamProcessorService(
|
|
48
|
+
const service = new StreamProcessorService(
|
|
49
|
+
{} as any,
|
|
50
|
+
{} as any,
|
|
51
|
+
persistence as any,
|
|
52
|
+
silentLogger
|
|
53
|
+
);
|
|
48
54
|
return { service, persistence, persisted };
|
|
49
55
|
}
|
|
50
56
|
|
|
@@ -83,6 +89,7 @@ describe('StreamProcessorService.reset', () => {
|
|
|
83
89
|
|
|
84
90
|
it('routes persisted state to the per-bucket key', async () => {
|
|
85
91
|
const { service, persistence } = makeService();
|
|
92
|
+
service.configureCircuitPersistence(true);
|
|
86
93
|
await service.init();
|
|
87
94
|
service.setStateKeySuffix('u1');
|
|
88
95
|
await service.saveState();
|
|
@@ -91,8 +98,9 @@ describe('StreamProcessorService.reset', () => {
|
|
|
91
98
|
|
|
92
99
|
it('drops a saveState that raced a reset (old circuit never persists into the new key)', async () => {
|
|
93
100
|
const { service, persistence } = makeService();
|
|
101
|
+
service.configureCircuitPersistence(true);
|
|
94
102
|
await service.init();
|
|
95
|
-
// The snapshot is taken, then a reset lands before the persist step
|
|
103
|
+
// The snapshot is taken, then a reset lands before the persist step,
|
|
96
104
|
// simulated by bumping the generation from inside save_state().
|
|
97
105
|
processorInstances[0].save_state.mockImplementation(() => {
|
|
98
106
|
void service.reset();
|
|
@@ -101,4 +109,108 @@ describe('StreamProcessorService.reset', () => {
|
|
|
101
109
|
await service.saveState();
|
|
102
110
|
expect(persistence.set).not.toHaveBeenCalled();
|
|
103
111
|
});
|
|
112
|
+
|
|
113
|
+
it('frees the replaced wasm circuit on reset', async () => {
|
|
114
|
+
const { service } = makeService();
|
|
115
|
+
await service.init();
|
|
116
|
+
const first = processorInstances[0];
|
|
117
|
+
await service.reset();
|
|
118
|
+
// V8 cannot see wasm-internal bytes, so the FinalizationRegistry may never
|
|
119
|
+
// run. The old circuit must be freed explicitly or its whole store stays
|
|
120
|
+
// resident for the rest of the session.
|
|
121
|
+
expect(first.free).toHaveBeenCalledTimes(1);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// The renderer-OOM guardrail. `Circuit::save` deep-clones the entire store (every
|
|
126
|
+
// row of every ingested table) and JSON-encodes it. It used to run once per
|
|
127
|
+
// ingest batch and once per query register/unregister, which on a large windowed
|
|
128
|
+
// list meant hundreds of whole-store serializations per scroll, for a snapshot
|
|
129
|
+
// the browser could never read back. Snapshots are now opt-in and checkpointed.
|
|
130
|
+
describe('StreamProcessorService circuit snapshots', () => {
|
|
131
|
+
const plan = {
|
|
132
|
+
queryHash: 'q1',
|
|
133
|
+
surql: 'SELECT * FROM thing;',
|
|
134
|
+
params: {},
|
|
135
|
+
ttl: '10m',
|
|
136
|
+
lastActiveAt: new Date(),
|
|
137
|
+
localArray: [],
|
|
138
|
+
remoteArray: [],
|
|
139
|
+
meta: { tableName: 'thing' },
|
|
140
|
+
} as any;
|
|
141
|
+
|
|
142
|
+
it('never snapshots on ingest or register/unregister by default', async () => {
|
|
143
|
+
const { service, persistence } = makeService();
|
|
144
|
+
await service.init();
|
|
145
|
+
const processor = processorInstances[0];
|
|
146
|
+
|
|
147
|
+
service.registerQueryPlan(plan);
|
|
148
|
+
service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
|
|
149
|
+
service.ingestMany([
|
|
150
|
+
{ table: 'thing', op: 'CREATE', id: 'thing:b', record: { id: 'thing:b' } },
|
|
151
|
+
{ table: 'thing', op: 'CREATE', id: 'thing:c', record: { id: 'thing:c' } },
|
|
152
|
+
]);
|
|
153
|
+
service.unregisterQueryPlan('q1');
|
|
154
|
+
|
|
155
|
+
expect(processor.save_state).not.toHaveBeenCalled();
|
|
156
|
+
expect(persistence.set).not.toHaveBeenCalled();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('still never snapshots inline when persistence is opted in', async () => {
|
|
160
|
+
const { service, persistence } = makeService();
|
|
161
|
+
service.configureCircuitPersistence(true, 30_000);
|
|
162
|
+
await service.init();
|
|
163
|
+
const processor = processorInstances[0];
|
|
164
|
+
|
|
165
|
+
service.registerQueryPlan(plan);
|
|
166
|
+
service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
|
|
167
|
+
service.unregisterQueryPlan('q1');
|
|
168
|
+
|
|
169
|
+
// Marked dirty, but the write waits for the checkpoint interval.
|
|
170
|
+
expect(processor.save_state).not.toHaveBeenCalled();
|
|
171
|
+
expect(persistence.set).not.toHaveBeenCalled();
|
|
172
|
+
service.stopCheckpoints();
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('writes at most one snapshot per checkpoint tick', async () => {
|
|
176
|
+
vi.useFakeTimers();
|
|
177
|
+
try {
|
|
178
|
+
const { service } = makeService();
|
|
179
|
+
service.configureCircuitPersistence(true, 1000);
|
|
180
|
+
await service.init();
|
|
181
|
+
const processor = processorInstances[0];
|
|
182
|
+
|
|
183
|
+
for (let i = 0; i < 50; i++) {
|
|
184
|
+
service.ingest('thing', 'CREATE', `thing:${i}`, { id: `thing:${i}` });
|
|
185
|
+
}
|
|
186
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
187
|
+
expect(processor.save_state).toHaveBeenCalledTimes(1);
|
|
188
|
+
|
|
189
|
+
// Idle interval: nothing changed, so nothing is serialized.
|
|
190
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
191
|
+
expect(processor.save_state).toHaveBeenCalledTimes(1);
|
|
192
|
+
|
|
193
|
+
service.stopCheckpoints();
|
|
194
|
+
} finally {
|
|
195
|
+
vi.useRealTimers();
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it('restores a snapshot the shipped persistence clients actually return', async () => {
|
|
200
|
+
const { service, persistence } = makeService();
|
|
201
|
+
// Both clients round-trip `save_state`'s output as a bare string. The old
|
|
202
|
+
// shape check looked for a raw SurrealDB result, so it never matched and the
|
|
203
|
+
// browser never restored anything it wrote.
|
|
204
|
+
persistence.get.mockResolvedValue('state-bytes' as any);
|
|
205
|
+
service.configureCircuitPersistence(true);
|
|
206
|
+
await service.init();
|
|
207
|
+
expect(processorInstances[0].load_state).toHaveBeenCalledWith('state-bytes');
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it('does not restore a snapshot when persistence is off', async () => {
|
|
211
|
+
const { service, persistence } = makeService();
|
|
212
|
+
persistence.get.mockResolvedValue('state-bytes' as any);
|
|
213
|
+
await service.init();
|
|
214
|
+
expect(processorInstances[0].load_state).not.toHaveBeenCalled();
|
|
215
|
+
});
|
|
104
216
|
});
|
|
@@ -44,4 +44,9 @@ export interface WasmProcessor {
|
|
|
44
44
|
// build degrades gracefully instead of throwing.
|
|
45
45
|
load_state?(state: string): void;
|
|
46
46
|
save_state?(): string;
|
|
47
|
+
// wasm-bindgen destructor. Releases the circuit (store + every view cache)
|
|
48
|
+
// inside wasm linear memory. Without it the bytes only come back when V8
|
|
49
|
+
// happens to GC the JS wrapper, which it has no reason to hurry since it
|
|
50
|
+
// cannot see how much wasm memory the wrapper is holding.
|
|
51
|
+
free?(): void;
|
|
47
52
|
}
|
package/src/sp00ky.ts
CHANGED
|
@@ -9,12 +9,9 @@ import type {
|
|
|
9
9
|
UpdateOptions,
|
|
10
10
|
RunOptions,
|
|
11
11
|
SyncHealth,
|
|
12
|
-
StorageHealth
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
RemoteDatabaseService,
|
|
16
|
-
createLocalEngine,
|
|
17
|
-
} from './services/database/index';
|
|
12
|
+
StorageHealth,
|
|
13
|
+
} from './types';
|
|
14
|
+
import { LocalMigrator, RemoteDatabaseService, createLocalEngine } from './services/database/index';
|
|
18
15
|
import type { LocalStore } from './services/database/index';
|
|
19
16
|
import { StaleEpochError } from './services/database/index';
|
|
20
17
|
import type { UpEvent } from './modules/sync/index';
|
|
@@ -30,10 +27,9 @@ import type {
|
|
|
30
27
|
BucketNames,
|
|
31
28
|
BackendNames,
|
|
32
29
|
BackendRoutes,
|
|
33
|
-
RoutePayload
|
|
34
|
-
import {
|
|
35
|
-
QueryBuilder
|
|
30
|
+
RoutePayload,
|
|
36
31
|
} from '@spooky-sync/query-builder';
|
|
32
|
+
import { QueryBuilder } from '@spooky-sync/query-builder';
|
|
37
33
|
|
|
38
34
|
import { DevToolsService } from './modules/devtools/index';
|
|
39
35
|
import { createLogger } from './services/logger/index';
|
|
@@ -60,14 +56,19 @@ import { computeTabsFingerprint, hash53, type TabRole } from './services/tabs/pr
|
|
|
60
56
|
import type { SqliteCacheEngine } from './services/database/sqlite-cache-engine';
|
|
61
57
|
|
|
62
58
|
export class BucketHandle {
|
|
63
|
-
constructor(
|
|
59
|
+
constructor(
|
|
60
|
+
private bucketName: string,
|
|
61
|
+
private remote: RemoteDatabaseService
|
|
62
|
+
) {}
|
|
64
63
|
|
|
65
64
|
async put(path: string, content: string | Uint8Array | Blob): Promise<void> {
|
|
66
65
|
await this.remote.query(`RETURN f"${this.bucketName}:/${path}".put($content);`, { content });
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
async get(path: string): Promise<unknown> {
|
|
70
|
-
const [result] = await this.remote.query<[unknown]>(
|
|
69
|
+
const [result] = await this.remote.query<[unknown]>(
|
|
70
|
+
`RETURN f"${this.bucketName}:/${path}".get();`
|
|
71
|
+
);
|
|
71
72
|
return result;
|
|
72
73
|
}
|
|
73
74
|
|
|
@@ -76,26 +77,36 @@ export class BucketHandle {
|
|
|
76
77
|
}
|
|
77
78
|
|
|
78
79
|
async exists(path: string): Promise<boolean> {
|
|
79
|
-
const [result] = await this.remote.query<[boolean]>(
|
|
80
|
+
const [result] = await this.remote.query<[boolean]>(
|
|
81
|
+
`RETURN f"${this.bucketName}:/${path}".exists();`
|
|
82
|
+
);
|
|
80
83
|
return result;
|
|
81
84
|
}
|
|
82
85
|
|
|
83
86
|
async head(path: string): Promise<Record<string, unknown>> {
|
|
84
|
-
const [result] = await this.remote.query<[Record<string, unknown>]>(
|
|
87
|
+
const [result] = await this.remote.query<[Record<string, unknown>]>(
|
|
88
|
+
`RETURN f"${this.bucketName}:/${path}".head();`
|
|
89
|
+
);
|
|
85
90
|
return result;
|
|
86
91
|
}
|
|
87
92
|
|
|
88
93
|
async copy(sourcePath: string, targetPath: string): Promise<void> {
|
|
89
|
-
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, {
|
|
94
|
+
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, {
|
|
95
|
+
target: targetPath,
|
|
96
|
+
});
|
|
90
97
|
}
|
|
91
98
|
|
|
92
99
|
async rename(sourcePath: string, targetPath: string): Promise<void> {
|
|
93
|
-
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, {
|
|
100
|
+
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, {
|
|
101
|
+
target: targetPath,
|
|
102
|
+
});
|
|
94
103
|
}
|
|
95
104
|
|
|
96
105
|
async list(prefix?: string): Promise<string[]> {
|
|
97
106
|
const p = prefix ?? '';
|
|
98
|
-
const [result] = await this.remote.query<[string[]]>(
|
|
107
|
+
const [result] = await this.remote.query<[string[]]>(
|
|
108
|
+
`RETURN f"${this.bucketName}:/${p}".list();`
|
|
109
|
+
);
|
|
99
110
|
return result;
|
|
100
111
|
}
|
|
101
112
|
}
|
|
@@ -273,6 +284,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
273
284
|
this.persistenceClient,
|
|
274
285
|
logger
|
|
275
286
|
);
|
|
287
|
+
// Circuit snapshots are opt-in and checkpointed, never per-ingest. See
|
|
288
|
+
// `persistCircuit` in types.ts for the measurements behind that default.
|
|
289
|
+
this.streamProcessor.configureCircuitPersistence(
|
|
290
|
+
config.persistCircuit ?? false,
|
|
291
|
+
config.circuitCheckpointMs
|
|
292
|
+
);
|
|
276
293
|
this.migrator = new LocalMigrator(this.local, logger);
|
|
277
294
|
|
|
278
295
|
this.cache = new CacheModule(
|
|
@@ -295,7 +312,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
295
312
|
this.local,
|
|
296
313
|
this.remote,
|
|
297
314
|
logger,
|
|
298
|
-
config.crdtDebounceMs ?? 500
|
|
315
|
+
config.crdtDebounceMs ?? 500
|
|
299
316
|
);
|
|
300
317
|
|
|
301
318
|
this.dataModule = new DataModule(
|
|
@@ -325,7 +342,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
325
342
|
degradeAfterConsecutiveFailures:
|
|
326
343
|
this.config.syncHealth === false
|
|
327
344
|
? 0
|
|
328
|
-
: this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
|
|
345
|
+
: (this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3),
|
|
329
346
|
}
|
|
330
347
|
);
|
|
331
348
|
|
|
@@ -368,23 +385,38 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
368
385
|
// every module. Nothing starts until init() calls coordinator.start().
|
|
369
386
|
if (tabsSupport.supported) {
|
|
370
387
|
this.tabsCoordinator = this.buildTabsCoordinator();
|
|
388
|
+
} else if (this.config.sharedTabs) {
|
|
389
|
+
this.logger.info(
|
|
390
|
+
{
|
|
391
|
+
reason: (tabsSupport as { reason: string }).reason,
|
|
392
|
+
Category: 'sp00ky-client::Sp00kyClient::tabs',
|
|
393
|
+
},
|
|
394
|
+
'sharedTabs requested but unsupported here; running solo'
|
|
395
|
+
);
|
|
396
|
+
}
|
|
397
|
+
// Report shared-tabs state whenever the feature was REQUESTED, active or
|
|
398
|
+
// not: "asked to share one store, running alone because X" is exactly the
|
|
399
|
+
// thing you need to see in the panel. Apps that never set the flag report
|
|
400
|
+
// null and the panel shows no section at all.
|
|
401
|
+
if (this.config.sharedTabs) {
|
|
402
|
+
const unsupportedReason = tabsSupport.supported
|
|
403
|
+
? undefined
|
|
404
|
+
: (tabsSupport as { reason: string }).reason;
|
|
371
405
|
this.devTools.setTabsInfoProvider(() => {
|
|
372
|
-
if (!this.sharedActive || !this.tabsCoordinator) return null;
|
|
373
406
|
const c = this.tabsCoordinator;
|
|
407
|
+
if (!this.sharedActive || !c) {
|
|
408
|
+
return { active: false, reason: unsupportedReason ?? 'fell-back' };
|
|
409
|
+
}
|
|
410
|
+
const hub = c.syncHub;
|
|
374
411
|
return {
|
|
412
|
+
active: true,
|
|
375
413
|
role: c.role,
|
|
376
414
|
tabId: c.tabId,
|
|
377
415
|
leadershipId: c.leadershipId,
|
|
378
|
-
leaderTabId: c.leaderTabId,
|
|
379
|
-
followers:
|
|
380
|
-
relayedBatches: c.syncHub?.relayedBatches ?? null,
|
|
416
|
+
leaderTabId: c.role === 'leader' ? c.tabId : c.leaderTabId,
|
|
417
|
+
...(hub ? { followers: hub.followerCount, relayedBatches: hub.relayedBatches } : {}),
|
|
381
418
|
};
|
|
382
419
|
});
|
|
383
|
-
} else if (this.config.sharedTabs) {
|
|
384
|
-
this.logger.info(
|
|
385
|
-
{ reason: (tabsSupport as { reason: string }).reason, Category: 'sp00ky-client::Sp00kyClient::tabs' },
|
|
386
|
-
'sharedTabs requested but unsupported here; running solo'
|
|
387
|
-
);
|
|
388
420
|
}
|
|
389
421
|
}
|
|
390
422
|
|
|
@@ -437,7 +469,8 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
437
469
|
this.cache.setIngestRelay(null);
|
|
438
470
|
this.sync.setTabContext('solo', tabId);
|
|
439
471
|
},
|
|
440
|
-
currentStorageHealth: () =>
|
|
472
|
+
currentStorageHealth: () =>
|
|
473
|
+
this.local.storageHealth ?? { status: 'unknown', fallback: false },
|
|
441
474
|
};
|
|
442
475
|
return new TabsCoordinator({
|
|
443
476
|
tabId,
|
|
@@ -515,9 +548,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
515
548
|
for (const row of records) {
|
|
516
549
|
const id = row?.id;
|
|
517
550
|
const table =
|
|
518
|
-
id && typeof id === 'object' && id.table !== undefined
|
|
519
|
-
? String(id.table)
|
|
520
|
-
: undefined;
|
|
551
|
+
id && typeof id === 'object' && id.table !== undefined ? String(id.table) : undefined;
|
|
521
552
|
if (!table) continue;
|
|
522
553
|
this.crdtManager.applyRow(table, row);
|
|
523
554
|
}
|
|
@@ -594,9 +625,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
594
625
|
await this.streamProcessor.init();
|
|
595
626
|
// Seed table `select` permissions from the schema before any query is
|
|
596
627
|
// registered — otherwise the SSP default-denies every non-`_00_` table.
|
|
597
|
-
this.streamProcessor.setPermissions(
|
|
598
|
-
extractSelectPermissions(this.config.schemaSurql)
|
|
599
|
-
);
|
|
628
|
+
this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
|
|
600
629
|
this.logger.debug(
|
|
601
630
|
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
602
631
|
'StreamProcessor initialized'
|
|
@@ -758,7 +787,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
758
787
|
*/
|
|
759
788
|
private async doSwitchBucket(target: string, gateRelease?: (() => void) | null): Promise<void> {
|
|
760
789
|
this.logger.info(
|
|
761
|
-
{
|
|
790
|
+
{
|
|
791
|
+
target,
|
|
792
|
+
from: this.local.currentBucketId,
|
|
793
|
+
Category: 'sp00ky-client::Sp00kyClient::doSwitchBucket',
|
|
794
|
+
},
|
|
762
795
|
'Switching local bucket'
|
|
763
796
|
);
|
|
764
797
|
|
|
@@ -843,6 +876,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
843
876
|
if (this.tabsCoordinator) await this.tabsCoordinator.stop();
|
|
844
877
|
await this.local.close();
|
|
845
878
|
await this.remote.close();
|
|
879
|
+
// Free the wasm circuit explicitly. V8 cannot see wasm-internal bytes, so
|
|
880
|
+
// relying on the wasm-bindgen FinalizationRegistry leaves the whole store
|
|
881
|
+
// resident until a GC that may never come, and a client that is recreated
|
|
882
|
+
// (provider remount, HMR) would stack circuits.
|
|
883
|
+
this.streamProcessor.dispose();
|
|
846
884
|
}
|
|
847
885
|
|
|
848
886
|
/**
|
|
@@ -883,7 +921,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
883
921
|
table: string,
|
|
884
922
|
recordId: string,
|
|
885
923
|
field: string,
|
|
886
|
-
fallbackText?: string
|
|
924
|
+
fallbackText?: string
|
|
887
925
|
): Promise<CrdtField> {
|
|
888
926
|
return this.crdtManager.open(table, recordId, field, fallbackText);
|
|
889
927
|
}
|
|
@@ -1137,10 +1175,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
1137
1175
|
this.dataModule.recordFrontendTiming(queryHash, ms);
|
|
1138
1176
|
}
|
|
1139
1177
|
|
|
1140
|
-
run<
|
|
1141
|
-
B
|
|
1142
|
-
R
|
|
1143
|
-
|
|
1178
|
+
run<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(
|
|
1179
|
+
backend: B,
|
|
1180
|
+
path: R,
|
|
1181
|
+
payload: RoutePayload<S, B, R>,
|
|
1182
|
+
options?: RunOptions
|
|
1183
|
+
) {
|
|
1144
1184
|
return this.dataModule.run(backend, path, payload, options);
|
|
1145
1185
|
}
|
|
1146
1186
|
|