@spooky-sync/core 0.0.1-canary.156 → 0.0.1-canary.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +31 -0
- package/dist/index.js +132 -13
- package/dist/types.d.ts +23 -0
- package/package.json +3 -3
- package/src/modules/devtools/index.ts +44 -9
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/services/stream-processor/index.ts +146 -20
- package/src/services/stream-processor/stream-processor.reset.test.ts +114 -2
- package/src/services/stream-processor/wasm-types.ts +5 -0
- package/src/sp00ky.ts +60 -32
- package/src/types.ts +23 -0
package/dist/index.d.ts
CHANGED
|
@@ -140,6 +140,11 @@ declare class StreamProcessorService {
|
|
|
140
140
|
private stateKeySuffix;
|
|
141
141
|
private stateGeneration;
|
|
142
142
|
private persistState;
|
|
143
|
+
private persistCircuit;
|
|
144
|
+
private checkpointMs;
|
|
145
|
+
private checkpointTimer;
|
|
146
|
+
private snapshotDirty;
|
|
147
|
+
private pagehideHandler;
|
|
143
148
|
constructor(events: EventSystem<StreamProcessorEvents>, db: LocalStore, persistenceClient: PersistenceClient, logger: Logger);
|
|
144
149
|
/**
|
|
145
150
|
* Add a receiver for stream updates.
|
|
@@ -200,8 +205,34 @@ declare class StreamProcessorService {
|
|
|
200
205
|
* afterwards (a fresh circuit default-denies every table).
|
|
201
206
|
*/
|
|
202
207
|
reset(): Promise<void>;
|
|
208
|
+
/**
|
|
209
|
+
* Release the wasm circuit and stop checkpointing. Call when the client is
|
|
210
|
+
* torn down; a recreated client (provider remount, HMR) would otherwise stack
|
|
211
|
+
* one full circuit per instance.
|
|
212
|
+
*/
|
|
213
|
+
dispose(): void;
|
|
214
|
+
/**
|
|
215
|
+
* Explicitly run the wasm-bindgen destructor. Guarded: stale wasm builds may
|
|
216
|
+
* not expose `free`, and a double free must not take the app down.
|
|
217
|
+
*/
|
|
218
|
+
private freeProcessor;
|
|
203
219
|
/** Toggle circuit-state persistence (shared-tabs follower/leader role). */
|
|
204
220
|
setPersistenceEnabled(enabled: boolean): void;
|
|
221
|
+
/**
|
|
222
|
+
* Opt into snapshot persistence (`persistCircuit`). Off by default: see the
|
|
223
|
+
* `persistCircuit` field comment for why per-ingest snapshots were removed.
|
|
224
|
+
* Must be called before `init()` for a snapshot to be restored at boot.
|
|
225
|
+
*/
|
|
226
|
+
configureCircuitPersistence(enabled: boolean, checkpointMs?: number): void;
|
|
227
|
+
/**
|
|
228
|
+
* Record that the circuit changed. Cheap and O(1), the expensive snapshot is
|
|
229
|
+
* deferred to the checkpoint timer, and skipped entirely when
|
|
230
|
+
* `persistCircuit` is off (the default).
|
|
231
|
+
*/
|
|
232
|
+
private markSnapshotDirty;
|
|
233
|
+
private startCheckpoints;
|
|
234
|
+
/** Stop checkpointing and drop the `pagehide` listener. */
|
|
235
|
+
stopCheckpoints(): void;
|
|
205
236
|
loadState(): Promise<void>;
|
|
206
237
|
/**
|
|
207
238
|
* Seed per-table `select` permission predicates ({ [table]: whereText }).
|
package/dist/index.js
CHANGED
|
@@ -5694,15 +5694,18 @@ async function walkOpfs(maxEntries = 2e3, maxDepth = 8) {
|
|
|
5694
5694
|
|
|
5695
5695
|
//#endregion
|
|
5696
5696
|
//#region src/modules/devtools/index.ts
|
|
5697
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
5698
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
5697
|
+
const CORE_VERSION = "0.0.1-canary.157";
|
|
5698
|
+
const WASM_VERSION = "0.0.1-canary.157";
|
|
5699
5699
|
const SURREAL_VERSION = "3.0.3";
|
|
5700
|
-
var DevToolsService = class {
|
|
5700
|
+
var DevToolsService = class DevToolsService {
|
|
5701
5701
|
eventsHistory = [];
|
|
5702
5702
|
eventIdCounter = 0;
|
|
5703
5703
|
version = CORE_VERSION;
|
|
5704
5704
|
backendInfo = emptyBackendInfo();
|
|
5705
5705
|
enabled = false;
|
|
5706
|
+
static NOTIFY_MIN_INTERVAL_MS = 250;
|
|
5707
|
+
notifyTimer = null;
|
|
5708
|
+
lastNotifyAt = 0;
|
|
5706
5709
|
/** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
|
|
5707
5710
|
* feature was REQUESTED (so an inactive/degraded tab still reports why). */
|
|
5708
5711
|
tabsInfoProvider = null;
|
|
@@ -5971,9 +5974,30 @@ var DevToolsService = class {
|
|
|
5971
5974
|
return { granted: false };
|
|
5972
5975
|
}
|
|
5973
5976
|
}
|
|
5977
|
+
/**
|
|
5978
|
+
* Request a state push. Coalesced (see {@link NOTIFY_MIN_INTERVAL_MS}): the
|
|
5979
|
+
* first call after an idle period pushes straight away so the panel stays
|
|
5980
|
+
* responsive, and any calls during the window collapse into ONE trailing push
|
|
5981
|
+
* that serializes the state as of the flush, not as of the request. Callers
|
|
5982
|
+
* stay fire-and-forget.
|
|
5983
|
+
*/
|
|
5974
5984
|
notifyDevTools() {
|
|
5975
5985
|
if (!this.enabled) return;
|
|
5976
|
-
if (typeof window
|
|
5986
|
+
if (typeof window === "undefined") return;
|
|
5987
|
+
if (this.notifyTimer !== null) return;
|
|
5988
|
+
const waited = Date.now() - this.lastNotifyAt;
|
|
5989
|
+
if (waited >= DevToolsService.NOTIFY_MIN_INTERVAL_MS) {
|
|
5990
|
+
this.flushNotify();
|
|
5991
|
+
return;
|
|
5992
|
+
}
|
|
5993
|
+
this.notifyTimer = setTimeout(() => {
|
|
5994
|
+
this.notifyTimer = null;
|
|
5995
|
+
if (this.enabled) this.flushNotify();
|
|
5996
|
+
}, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
|
|
5997
|
+
}
|
|
5998
|
+
flushNotify() {
|
|
5999
|
+
this.lastNotifyAt = Date.now();
|
|
6000
|
+
window.postMessage({
|
|
5977
6001
|
type: "SP00KY_STATE_CHANGED",
|
|
5978
6002
|
source: "sp00ky-devtools-page",
|
|
5979
6003
|
state: this.getState()
|
|
@@ -6298,6 +6322,14 @@ var AuthService = class {
|
|
|
6298
6322
|
|
|
6299
6323
|
//#endregion
|
|
6300
6324
|
//#region src/services/stream-processor/index.ts
|
|
6325
|
+
/**
|
|
6326
|
+
* Read a circuit snapshot out of the pre-`persistCircuit` persisted shape
|
|
6327
|
+
* (`[[{ state }]]`, a raw SurrealDB result). Returns null for anything else.
|
|
6328
|
+
*/
|
|
6329
|
+
function extractLegacyState(result) {
|
|
6330
|
+
if (Array.isArray(result) && Array.isArray(result[0]) && typeof result[0][0]?.state === "string") return result[0][0].state;
|
|
6331
|
+
return null;
|
|
6332
|
+
}
|
|
6301
6333
|
var StreamProcessorService = class {
|
|
6302
6334
|
logger;
|
|
6303
6335
|
processor;
|
|
@@ -6312,6 +6344,11 @@ var StreamProcessorService = class {
|
|
|
6312
6344
|
stateKeySuffix = "";
|
|
6313
6345
|
stateGeneration = 0;
|
|
6314
6346
|
persistState = true;
|
|
6347
|
+
persistCircuit = false;
|
|
6348
|
+
checkpointMs = 3e4;
|
|
6349
|
+
checkpointTimer = null;
|
|
6350
|
+
snapshotDirty = false;
|
|
6351
|
+
pagehideHandler = null;
|
|
6315
6352
|
constructor(events, db, persistenceClient, logger) {
|
|
6316
6353
|
this.events = events;
|
|
6317
6354
|
this.db = db;
|
|
@@ -6392,7 +6429,7 @@ var StreamProcessorService = class {
|
|
|
6392
6429
|
const buffered = Array.from(this.batchBuffer.values());
|
|
6393
6430
|
this.batchBuffer.clear();
|
|
6394
6431
|
if (buffered.length > 0) this.dispatchUpdates(buffered);
|
|
6395
|
-
this.
|
|
6432
|
+
this.markSnapshotDirty();
|
|
6396
6433
|
}
|
|
6397
6434
|
/**
|
|
6398
6435
|
* Initialize the WASM module and processor.
|
|
@@ -6436,19 +6473,99 @@ var StreamProcessorService = class {
|
|
|
6436
6473
|
this.stateGeneration++;
|
|
6437
6474
|
this.batching = false;
|
|
6438
6475
|
this.batchBuffer.clear();
|
|
6476
|
+
this.snapshotDirty = false;
|
|
6477
|
+
const previous = this.processor;
|
|
6439
6478
|
this.processor = new Sp00kyProcessor();
|
|
6479
|
+
this.freeProcessor(previous);
|
|
6440
6480
|
this.logger.info({ Category: "sp00ky-client::StreamProcessorService::reset" }, "Stream processor reset (fresh circuit)");
|
|
6441
6481
|
}
|
|
6482
|
+
/**
|
|
6483
|
+
* Release the wasm circuit and stop checkpointing. Call when the client is
|
|
6484
|
+
* torn down; a recreated client (provider remount, HMR) would otherwise stack
|
|
6485
|
+
* one full circuit per instance.
|
|
6486
|
+
*/
|
|
6487
|
+
dispose() {
|
|
6488
|
+
this.stopCheckpoints();
|
|
6489
|
+
const previous = this.processor;
|
|
6490
|
+
this.processor = void 0;
|
|
6491
|
+
this.isInitialized = false;
|
|
6492
|
+
this.batching = false;
|
|
6493
|
+
this.batchBuffer.clear();
|
|
6494
|
+
this.receivers = [];
|
|
6495
|
+
this.freeProcessor(previous);
|
|
6496
|
+
}
|
|
6497
|
+
/**
|
|
6498
|
+
* Explicitly run the wasm-bindgen destructor. Guarded: stale wasm builds may
|
|
6499
|
+
* not expose `free`, and a double free must not take the app down.
|
|
6500
|
+
*/
|
|
6501
|
+
freeProcessor(processor) {
|
|
6502
|
+
if (!processor || typeof processor.free !== "function") return;
|
|
6503
|
+
try {
|
|
6504
|
+
processor.free();
|
|
6505
|
+
} catch (e) {
|
|
6506
|
+
this.logger.debug({
|
|
6507
|
+
error: e,
|
|
6508
|
+
Category: "sp00ky-client::StreamProcessorService::freeProcessor"
|
|
6509
|
+
}, "Failed to free previous wasm circuit");
|
|
6510
|
+
}
|
|
6511
|
+
}
|
|
6442
6512
|
/** Toggle circuit-state persistence (shared-tabs follower/leader role). */
|
|
6443
6513
|
setPersistenceEnabled(enabled) {
|
|
6444
6514
|
this.persistState = enabled;
|
|
6515
|
+
if (!enabled) this.stopCheckpoints();
|
|
6516
|
+
}
|
|
6517
|
+
/**
|
|
6518
|
+
* Opt into snapshot persistence (`persistCircuit`). Off by default: see the
|
|
6519
|
+
* `persistCircuit` field comment for why per-ingest snapshots were removed.
|
|
6520
|
+
* Must be called before `init()` for a snapshot to be restored at boot.
|
|
6521
|
+
*/
|
|
6522
|
+
configureCircuitPersistence(enabled, checkpointMs) {
|
|
6523
|
+
this.persistCircuit = enabled;
|
|
6524
|
+
if (checkpointMs && checkpointMs > 0) this.checkpointMs = checkpointMs;
|
|
6525
|
+
if (!enabled) this.stopCheckpoints();
|
|
6526
|
+
}
|
|
6527
|
+
/**
|
|
6528
|
+
* Record that the circuit changed. Cheap and O(1), the expensive snapshot is
|
|
6529
|
+
* deferred to the checkpoint timer, and skipped entirely when
|
|
6530
|
+
* `persistCircuit` is off (the default).
|
|
6531
|
+
*/
|
|
6532
|
+
markSnapshotDirty() {
|
|
6533
|
+
if (!this.persistCircuit || !this.persistState) return;
|
|
6534
|
+
this.snapshotDirty = true;
|
|
6535
|
+
this.startCheckpoints();
|
|
6536
|
+
}
|
|
6537
|
+
startCheckpoints() {
|
|
6538
|
+
if (this.checkpointTimer) return;
|
|
6539
|
+
this.checkpointTimer = setInterval(() => {
|
|
6540
|
+
if (!this.snapshotDirty) return;
|
|
6541
|
+
this.snapshotDirty = false;
|
|
6542
|
+
this.saveState();
|
|
6543
|
+
}, this.checkpointMs);
|
|
6544
|
+
if (typeof window !== "undefined" && !this.pagehideHandler) {
|
|
6545
|
+
this.pagehideHandler = () => {
|
|
6546
|
+
if (!this.snapshotDirty) return;
|
|
6547
|
+
this.snapshotDirty = false;
|
|
6548
|
+
this.saveState();
|
|
6549
|
+
};
|
|
6550
|
+
window.addEventListener("pagehide", this.pagehideHandler);
|
|
6551
|
+
}
|
|
6552
|
+
}
|
|
6553
|
+
/** Stop checkpointing and drop the `pagehide` listener. */
|
|
6554
|
+
stopCheckpoints() {
|
|
6555
|
+
if (this.checkpointTimer) {
|
|
6556
|
+
clearInterval(this.checkpointTimer);
|
|
6557
|
+
this.checkpointTimer = null;
|
|
6558
|
+
}
|
|
6559
|
+
if (this.pagehideHandler && typeof window !== "undefined") window.removeEventListener("pagehide", this.pagehideHandler);
|
|
6560
|
+
this.pagehideHandler = null;
|
|
6561
|
+
this.snapshotDirty = false;
|
|
6445
6562
|
}
|
|
6446
6563
|
async loadState() {
|
|
6447
|
-
if (!this.processor || !this.persistState) return;
|
|
6564
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
6448
6565
|
try {
|
|
6449
6566
|
const result = await this.persistenceClient.get(this.stateKey());
|
|
6450
|
-
|
|
6451
|
-
|
|
6567
|
+
const state = typeof result === "string" ? result : extractLegacyState(result);
|
|
6568
|
+
if (state) {
|
|
6452
6569
|
this.logger.info({
|
|
6453
6570
|
stateLength: state.length,
|
|
6454
6571
|
Category: "sp00ky-client::StreamProcessorService::loadState"
|
|
@@ -6502,7 +6619,7 @@ var StreamProcessorService = class {
|
|
|
6502
6619
|
}, "Session auth context updated");
|
|
6503
6620
|
}
|
|
6504
6621
|
async saveState() {
|
|
6505
|
-
if (!this.processor || !this.persistState) return;
|
|
6622
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
6506
6623
|
const generation = this.stateGeneration;
|
|
6507
6624
|
try {
|
|
6508
6625
|
if (typeof this.processor.save_state === "function") {
|
|
@@ -6561,7 +6678,7 @@ var StreamProcessorService = class {
|
|
|
6561
6678
|
}));
|
|
6562
6679
|
this.notifyUpdates(updates);
|
|
6563
6680
|
}
|
|
6564
|
-
if (!this.batching) this.
|
|
6681
|
+
if (!this.batching) this.markSnapshotDirty();
|
|
6565
6682
|
return rawUpdates;
|
|
6566
6683
|
} catch (e) {
|
|
6567
6684
|
this.logger.error({
|
|
@@ -6614,7 +6731,7 @@ var StreamProcessorService = class {
|
|
|
6614
6731
|
snapshotMs: initialUpdate.timing_snapshot_ms ?? 0
|
|
6615
6732
|
}
|
|
6616
6733
|
};
|
|
6617
|
-
this.
|
|
6734
|
+
this.markSnapshotDirty();
|
|
6618
6735
|
this.logger.debug({
|
|
6619
6736
|
queryHash: queryPlan.queryHash,
|
|
6620
6737
|
surql: queryPlan.surql,
|
|
@@ -6637,7 +6754,7 @@ var StreamProcessorService = class {
|
|
|
6637
6754
|
if (!this.processor) return;
|
|
6638
6755
|
try {
|
|
6639
6756
|
this.processor.unregister_view(queryHash);
|
|
6640
|
-
this.
|
|
6757
|
+
this.markSnapshotDirty();
|
|
6641
6758
|
} catch (e) {
|
|
6642
6759
|
this.logger.error({
|
|
6643
6760
|
error: e,
|
|
@@ -8759,6 +8876,7 @@ var Sp00kyClient = class {
|
|
|
8759
8876
|
else this.persistenceClient = config.persistenceClient;
|
|
8760
8877
|
this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
|
|
8761
8878
|
this.streamProcessor = new StreamProcessorService(new EventSystem(["stream_update"]), this.local, this.persistenceClient, logger);
|
|
8879
|
+
this.streamProcessor.configureCircuitPersistence(config.persistCircuit ?? false, config.circuitCheckpointMs);
|
|
8762
8880
|
this.migrator = new LocalMigrator(this.local, logger);
|
|
8763
8881
|
this.cache = new CacheModule(this.local, this.streamProcessor, (update) => {
|
|
8764
8882
|
this.dataModule.onStreamUpdate(update);
|
|
@@ -8821,7 +8939,7 @@ var Sp00kyClient = class {
|
|
|
8821
8939
|
return new TabsCoordinator({
|
|
8822
8940
|
tabId,
|
|
8823
8941
|
fingerprint: computeTabsFingerprint({
|
|
8824
|
-
coreVersion: "0.0.1-canary.
|
|
8942
|
+
coreVersion: "0.0.1-canary.157",
|
|
8825
8943
|
schemaHash: hash53(this.config.schemaSurql),
|
|
8826
8944
|
endpoint: this.config.database.endpoint ?? "",
|
|
8827
8945
|
namespace: this.config.database.namespace,
|
|
@@ -9117,6 +9235,7 @@ var Sp00kyClient = class {
|
|
|
9117
9235
|
if (this.tabsCoordinator) await this.tabsCoordinator.stop();
|
|
9118
9236
|
await this.local.close();
|
|
9119
9237
|
await this.remote.close();
|
|
9238
|
+
this.streamProcessor.dispose();
|
|
9120
9239
|
}
|
|
9121
9240
|
/**
|
|
9122
9241
|
* Subscribe to a feature flag for the current user. Returns a
|
package/dist/types.d.ts
CHANGED
|
@@ -495,6 +495,29 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
495
495
|
* Inspect via `window.__00__.getState().database.tabs` and `__sqliteStats`.
|
|
496
496
|
*/
|
|
497
497
|
sharedTabs?: boolean;
|
|
498
|
+
/**
|
|
499
|
+
* Persist the in-browser SSP circuit (store + view caches) as a snapshot so a
|
|
500
|
+
* reload can restore it instead of re-materializing. Default `false`, and
|
|
501
|
+
* that default is deliberate.
|
|
502
|
+
*
|
|
503
|
+
* The circuit is DERIVED state: the durable local store (OPFS SQLite) is the
|
|
504
|
+
* source of truth, and every first paint already reads row bodies from it
|
|
505
|
+
* (`DataManager.createNewQuery` / `materializeRecords`) using the circuit only
|
|
506
|
+
* for row identity and ordering. A snapshot buys nothing on reload while
|
|
507
|
+
* costing a full deep clone of every row of every ingested table plus a JSON
|
|
508
|
+
* encode of the result, `Circuit::save` in the Rust core, mirroring the
|
|
509
|
+
* server's rule in `ssp-node`: *never per-ingest*.
|
|
510
|
+
*
|
|
511
|
+
* When enabled, snapshots are written on a checkpoint interval
|
|
512
|
+
* ({@link circuitCheckpointMs}) and on `pagehide`, never per ingest or per
|
|
513
|
+
* query registration. Enable only for a workload that has measured a win.
|
|
514
|
+
*/
|
|
515
|
+
persistCircuit?: boolean;
|
|
516
|
+
/**
|
|
517
|
+
* Checkpoint interval in milliseconds for {@link persistCircuit}. Defaults to
|
|
518
|
+
* 30000. Ignored when `persistCircuit` is off.
|
|
519
|
+
*/
|
|
520
|
+
circuitCheckpointMs?: number;
|
|
498
521
|
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
499
522
|
otelTransmit?: PinoTransmit;
|
|
500
523
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.157",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.157",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.157",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"fast-json-patch": "^3.1.1",
|
|
@@ -51,6 +51,18 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
51
51
|
// (the on-demand GET_STATE pull) still works before the push channel turns on.
|
|
52
52
|
private enabled = false;
|
|
53
53
|
|
|
54
|
+
// A state push serializes EVERY active query's full record set (see
|
|
55
|
+
// `getActiveQueries`) and postMessage clones it again, so its cost scales with
|
|
56
|
+
// the whole client dataset — and it is triggered per event, including one per
|
|
57
|
+
// local DB query (`DATABASE_LOCAL_QUERY` → `logEvent`). Unthrottled, a single
|
|
58
|
+
// page load's few hundred local queries turn a handful of MB of rows into GBs
|
|
59
|
+
// of short-lived large-object garbage and OOM the renderer (V8
|
|
60
|
+
// "young object promotion failed"). Coalesce instead: push immediately when
|
|
61
|
+
// idle, then at most once per window, always serializing the LATEST state.
|
|
62
|
+
private static readonly NOTIFY_MIN_INTERVAL_MS = 250;
|
|
63
|
+
private notifyTimer: ReturnType<typeof setTimeout> | null = null;
|
|
64
|
+
private lastNotifyAt = 0;
|
|
65
|
+
|
|
54
66
|
/** Shared-tabs snapshot for the panel, wired by Sp00kyClient whenever the
|
|
55
67
|
* feature was REQUESTED (so an inactive/degraded tab still reports why). */
|
|
56
68
|
private tabsInfoProvider: (() => SharedTabsInfo | null) | null = null;
|
|
@@ -408,19 +420,42 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
408
420
|
}
|
|
409
421
|
}
|
|
410
422
|
|
|
423
|
+
/**
|
|
424
|
+
* Request a state push. Coalesced (see {@link NOTIFY_MIN_INTERVAL_MS}): the
|
|
425
|
+
* first call after an idle period pushes straight away so the panel stays
|
|
426
|
+
* responsive, and any calls during the window collapse into ONE trailing push
|
|
427
|
+
* that serializes the state as of the flush, not as of the request. Callers
|
|
428
|
+
* stay fire-and-forget.
|
|
429
|
+
*/
|
|
411
430
|
private notifyDevTools() {
|
|
412
431
|
// No consumer attached → no getState() serialization, no postMessage broadcast.
|
|
413
432
|
if (!this.enabled) return;
|
|
414
|
-
if (typeof window
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
);
|
|
433
|
+
if (typeof window === 'undefined') return;
|
|
434
|
+
// A trailing push is already queued; it will carry this change too.
|
|
435
|
+
if (this.notifyTimer !== null) return;
|
|
436
|
+
|
|
437
|
+
const waited = Date.now() - this.lastNotifyAt;
|
|
438
|
+
if (waited >= DevToolsService.NOTIFY_MIN_INTERVAL_MS) {
|
|
439
|
+
this.flushNotify();
|
|
440
|
+
return;
|
|
423
441
|
}
|
|
442
|
+
this.notifyTimer = setTimeout(() => {
|
|
443
|
+
this.notifyTimer = null;
|
|
444
|
+
// Still gated on `enabled`: the panel may have disconnected while queued.
|
|
445
|
+
if (this.enabled) this.flushNotify();
|
|
446
|
+
}, DevToolsService.NOTIFY_MIN_INTERVAL_MS - waited);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
private flushNotify() {
|
|
450
|
+
this.lastNotifyAt = Date.now();
|
|
451
|
+
window.postMessage(
|
|
452
|
+
{
|
|
453
|
+
type: 'SP00KY_STATE_CHANGED',
|
|
454
|
+
source: 'sp00ky-devtools-page',
|
|
455
|
+
state: this.getState(),
|
|
456
|
+
},
|
|
457
|
+
'*'
|
|
458
|
+
);
|
|
424
459
|
}
|
|
425
460
|
|
|
426
461
|
private serializeForDevTools(data: any, seen = new WeakSet<object>()): any {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import { describe, it, expect, vi, afterEach } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { DevToolsService } from './index';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A state push serializes EVERY active query's full record set and postMessage
|
|
7
|
+
* clones it again, so its cost scales with the whole client dataset. It is
|
|
8
|
+
* requested per event — including one per local DB query — so without
|
|
9
|
+
* coalescing a page load's few hundred local queries turn a few MB of rows into
|
|
10
|
+
* GBs of short-lived large-object garbage and OOM the renderer. These tests pin
|
|
11
|
+
* the coalescing, not the payload.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
function harness(recordCount = 500) {
|
|
15
|
+
const posted: any[] = [];
|
|
16
|
+
const listeners: ((e: any) => void)[] = [];
|
|
17
|
+
const fakeWindow: any = {
|
|
18
|
+
postMessage: (msg: any) => posted.push(msg),
|
|
19
|
+
addEventListener: (_type: string, cb: (e: any) => void) => listeners.push(cb),
|
|
20
|
+
dispatchEvent: () => true,
|
|
21
|
+
};
|
|
22
|
+
fakeWindow.self = fakeWindow;
|
|
23
|
+
vi.stubGlobal('window', fakeWindow);
|
|
24
|
+
vi.stubGlobal('CustomEvent', class {
|
|
25
|
+
type: string;
|
|
26
|
+
constructor(type: string) {
|
|
27
|
+
this.type = type;
|
|
28
|
+
}
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const noop = () => {};
|
|
32
|
+
const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
33
|
+
logger.child = () => logger;
|
|
34
|
+
|
|
35
|
+
const local: any = {
|
|
36
|
+
query: async () => [],
|
|
37
|
+
getConfig: () => ({ store: 'memory' }),
|
|
38
|
+
currentBucketId: 'anon',
|
|
39
|
+
storageHealth: { status: 'memory', fallback: false },
|
|
40
|
+
};
|
|
41
|
+
const remote: any = { query: async () => [] };
|
|
42
|
+
const auth: any = {
|
|
43
|
+
isAuthenticated: false,
|
|
44
|
+
currentUser: undefined,
|
|
45
|
+
eventSystem: { subscribe: noop },
|
|
46
|
+
};
|
|
47
|
+
// One query holding a lot of rows: the thing whose repeated serialization is
|
|
48
|
+
// what actually blows the heap.
|
|
49
|
+
const records = Array.from({ length: recordCount }, (_, i) => ({ id: `game:${i}`, pgn: 'x' }));
|
|
50
|
+
const dataManager: any = {
|
|
51
|
+
getActiveQueries: () => [
|
|
52
|
+
{
|
|
53
|
+
config: { id: new RecordId('_00_query', 'q1'), params: {} },
|
|
54
|
+
status: 'idle',
|
|
55
|
+
records,
|
|
56
|
+
updateCount: 1,
|
|
57
|
+
},
|
|
58
|
+
],
|
|
59
|
+
phaseTimings: () => ({}),
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const service = new DevToolsService(local, remote, logger, { tables: [] } as any, auth, dataManager);
|
|
63
|
+
// Announce a consumer, exactly like the extension's page-script does.
|
|
64
|
+
for (const cb of listeners) {
|
|
65
|
+
cb({ source: fakeWindow, data: { type: 'SP00KY_DEVTOOLS_CONNECT' } });
|
|
66
|
+
}
|
|
67
|
+
const statePushes = () => posted.filter((m) => m.type === 'SP00KY_STATE_CHANGED');
|
|
68
|
+
return { service, statePushes, posted };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
afterEach(() => {
|
|
72
|
+
vi.useRealTimers();
|
|
73
|
+
vi.unstubAllGlobals();
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
describe('DevToolsService state-push coalescing', () => {
|
|
77
|
+
it('pushes once immediately on connect', () => {
|
|
78
|
+
vi.useFakeTimers();
|
|
79
|
+
const { statePushes } = harness();
|
|
80
|
+
expect(statePushes().length).toBe(1);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('collapses a burst of per-query events into ONE trailing push', () => {
|
|
84
|
+
vi.useFakeTimers();
|
|
85
|
+
const { service, statePushes } = harness();
|
|
86
|
+
const before = statePushes().length;
|
|
87
|
+
|
|
88
|
+
// What a page load looks like: hundreds of LOCAL_QUERY events, each of which
|
|
89
|
+
// used to serialize the entire query state.
|
|
90
|
+
for (let i = 0; i < 400; i++) {
|
|
91
|
+
(service as any).logEvent('LOCAL_QUERY', { query: 'SELECT * FROM game', vars: {} });
|
|
92
|
+
}
|
|
93
|
+
// Nothing extra yet — the burst is queued, not serialized 400 times.
|
|
94
|
+
expect(statePushes().length).toBe(before);
|
|
95
|
+
|
|
96
|
+
vi.advanceTimersByTime(300);
|
|
97
|
+
expect(statePushes().length).toBe(before + 1);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('still pushes again once the window has passed', () => {
|
|
101
|
+
vi.useFakeTimers();
|
|
102
|
+
const { service, statePushes } = harness();
|
|
103
|
+
const before = statePushes().length;
|
|
104
|
+
|
|
105
|
+
(service as any).logEvent('A', {});
|
|
106
|
+
vi.advanceTimersByTime(300);
|
|
107
|
+
expect(statePushes().length).toBe(before + 1);
|
|
108
|
+
|
|
109
|
+
// An event arriving right after that flush is still inside the window, so it
|
|
110
|
+
// queues rather than pushing again...
|
|
111
|
+
(service as any).logEvent('B', {});
|
|
112
|
+
expect(statePushes().length).toBe(before + 1);
|
|
113
|
+
vi.advanceTimersByTime(300);
|
|
114
|
+
expect(statePushes().length).toBe(before + 2);
|
|
115
|
+
|
|
116
|
+
// ...but once the tab has been idle past the window, the next event pushes
|
|
117
|
+
// straight away, so the panel never waits on a quiet app.
|
|
118
|
+
vi.advanceTimersByTime(1000);
|
|
119
|
+
(service as any).logEvent('C', {});
|
|
120
|
+
expect(statePushes().length).toBe(before + 3);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('drops a queued push when the consumer disconnects mid-window', () => {
|
|
124
|
+
vi.useFakeTimers();
|
|
125
|
+
const { service, statePushes, posted } = harness();
|
|
126
|
+
const before = statePushes().length;
|
|
127
|
+
|
|
128
|
+
(service as any).logEvent('A', {});
|
|
129
|
+
(service as any).enabled = false; // panel closed while the push was queued
|
|
130
|
+
vi.advanceTimersByTime(300);
|
|
131
|
+
expect(statePushes().length).toBe(before);
|
|
132
|
+
expect(posted.some((m) => m.type === 'SP00KY_STATE_CHANGED' && m.state === undefined)).toBe(false);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('serializes the LATEST state at flush time, not at request time', () => {
|
|
136
|
+
vi.useFakeTimers();
|
|
137
|
+
const { service, statePushes } = harness();
|
|
138
|
+
|
|
139
|
+
(service as any).logEvent('FIRST', {});
|
|
140
|
+
(service as any).logEvent('SECOND', {});
|
|
141
|
+
vi.advanceTimersByTime(300);
|
|
142
|
+
|
|
143
|
+
const last = statePushes().at(-1);
|
|
144
|
+
const types = last.state.eventsHistory.map((e: any) => e.eventType);
|
|
145
|
+
// Both events of the coalesced window are present in the single push.
|
|
146
|
+
expect(types).toContain('FIRST');
|
|
147
|
+
expect(types).toContain('SECOND');
|
|
148
|
+
});
|
|
149
|
+
});
|
|
@@ -57,6 +57,21 @@ export interface StreamUpdateReceiver {
|
|
|
57
57
|
onStreamUpdate(update: StreamUpdate): void;
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Read a circuit snapshot out of the pre-`persistCircuit` persisted shape
|
|
62
|
+
* (`[[{ state }]]`, a raw SurrealDB result). Returns null for anything else.
|
|
63
|
+
*/
|
|
64
|
+
function extractLegacyState(result: unknown): string | null {
|
|
65
|
+
if (
|
|
66
|
+
Array.isArray(result) &&
|
|
67
|
+
Array.isArray(result[0]) &&
|
|
68
|
+
typeof result[0][0]?.state === 'string'
|
|
69
|
+
) {
|
|
70
|
+
return result[0][0].state;
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
60
75
|
export class StreamProcessorService {
|
|
61
76
|
private logger: Logger;
|
|
62
77
|
private processor: WasmProcessor | undefined;
|
|
@@ -89,6 +104,32 @@ export class StreamProcessorService {
|
|
|
89
104
|
// stomp the leader's snapshot under the same key (the pre-existing cross-tab
|
|
90
105
|
// localStorage hazard); a promoted follower flips this back on.
|
|
91
106
|
private persistState = true;
|
|
107
|
+
// Snapshot persistence is OPT-IN and off by default (`persistCircuit`).
|
|
108
|
+
//
|
|
109
|
+
// `Circuit::save` deep-clones the WHOLE store (every row of every ingested
|
|
110
|
+
// table, full bodies) plus every view cache and JSON-encodes the result. It
|
|
111
|
+
// used to run from `ingest`, `flushCoalescing`, `registerQueryPlan` and
|
|
112
|
+
// `unregisterQueryPlan`, i.e. once per sync batch AND once per query
|
|
113
|
+
// register/unregister. On a 3.7k-game collection whose list registers and
|
|
114
|
+
// drops one windowed query per 50 rows scrolled, that measured 220 whole-store
|
|
115
|
+
// serializations and ~1 GB of transient JSON for a single scroll, 5.5x the
|
|
116
|
+
// wall time of the same ingests without it, and it doubled the wasm heap
|
|
117
|
+
// high-water mark (wasm32 dlmalloc never returns pages, so the peak is
|
|
118
|
+
// permanent). It also ran the encode a second time in
|
|
119
|
+
// `LocalStoragePersistenceClient.set` and wrote it synchronously on the main
|
|
120
|
+
// thread. All of it was dead weight: `loadState`'s shape check could never
|
|
121
|
+
// match what either shipped persistence client returns, so the browser never
|
|
122
|
+
// restored a snapshot, and first paint comes from the local SQLite store
|
|
123
|
+
// anyway.
|
|
124
|
+
//
|
|
125
|
+
// When enabled, we mark the circuit dirty and let a checkpoint timer (plus a
|
|
126
|
+
// `pagehide` flush) do at most one snapshot per interval, mirroring
|
|
127
|
+
// `ssp-node`'s "NEVER per-ingest" rule.
|
|
128
|
+
private persistCircuit = false;
|
|
129
|
+
private checkpointMs = 30_000;
|
|
130
|
+
private checkpointTimer: ReturnType<typeof setInterval> | null = null;
|
|
131
|
+
private snapshotDirty = false;
|
|
132
|
+
private pagehideHandler: (() => void) | null = null;
|
|
92
133
|
|
|
93
134
|
constructor(
|
|
94
135
|
public events: EventSystem<StreamProcessorEvents>,
|
|
@@ -201,10 +242,7 @@ export class StreamProcessorService {
|
|
|
201
242
|
if (buffered.length > 0) {
|
|
202
243
|
this.dispatchUpdates(buffered);
|
|
203
244
|
}
|
|
204
|
-
|
|
205
|
-
// snapshot covers the whole batch. Kept fire-and-forget like the per-ingest
|
|
206
|
-
// call it replaces.
|
|
207
|
-
this.saveState();
|
|
245
|
+
this.markSnapshotDirty();
|
|
208
246
|
}
|
|
209
247
|
|
|
210
248
|
/**
|
|
@@ -265,32 +303,120 @@ export class StreamProcessorService {
|
|
|
265
303
|
this.stateGeneration++;
|
|
266
304
|
this.batching = false;
|
|
267
305
|
this.batchBuffer.clear();
|
|
306
|
+
this.snapshotDirty = false;
|
|
307
|
+
const previous = this.processor;
|
|
268
308
|
this.processor = new Sp00kyProcessor() as unknown as WasmProcessor;
|
|
309
|
+
this.freeProcessor(previous);
|
|
269
310
|
this.logger.info(
|
|
270
311
|
{ Category: 'sp00ky-client::StreamProcessorService::reset' },
|
|
271
312
|
'Stream processor reset (fresh circuit)'
|
|
272
313
|
);
|
|
273
314
|
}
|
|
274
315
|
|
|
316
|
+
/**
|
|
317
|
+
* Release the wasm circuit and stop checkpointing. Call when the client is
|
|
318
|
+
* torn down; a recreated client (provider remount, HMR) would otherwise stack
|
|
319
|
+
* one full circuit per instance.
|
|
320
|
+
*/
|
|
321
|
+
dispose(): void {
|
|
322
|
+
this.stopCheckpoints();
|
|
323
|
+
const previous = this.processor;
|
|
324
|
+
this.processor = undefined;
|
|
325
|
+
this.isInitialized = false;
|
|
326
|
+
this.batching = false;
|
|
327
|
+
this.batchBuffer.clear();
|
|
328
|
+
this.receivers = [];
|
|
329
|
+
this.freeProcessor(previous);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Explicitly run the wasm-bindgen destructor. Guarded: stale wasm builds may
|
|
334
|
+
* not expose `free`, and a double free must not take the app down.
|
|
335
|
+
*/
|
|
336
|
+
private freeProcessor(processor: WasmProcessor | undefined): void {
|
|
337
|
+
if (!processor || typeof processor.free !== 'function') return;
|
|
338
|
+
try {
|
|
339
|
+
processor.free();
|
|
340
|
+
} catch (e) {
|
|
341
|
+
this.logger.debug(
|
|
342
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::freeProcessor' },
|
|
343
|
+
'Failed to free previous wasm circuit'
|
|
344
|
+
);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
275
348
|
/** Toggle circuit-state persistence (shared-tabs follower/leader role). */
|
|
276
349
|
setPersistenceEnabled(enabled: boolean): void {
|
|
277
350
|
this.persistState = enabled;
|
|
351
|
+
if (!enabled) this.stopCheckpoints();
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Opt into snapshot persistence (`persistCircuit`). Off by default: see the
|
|
356
|
+
* `persistCircuit` field comment for why per-ingest snapshots were removed.
|
|
357
|
+
* Must be called before `init()` for a snapshot to be restored at boot.
|
|
358
|
+
*/
|
|
359
|
+
configureCircuitPersistence(enabled: boolean, checkpointMs?: number): void {
|
|
360
|
+
this.persistCircuit = enabled;
|
|
361
|
+
if (checkpointMs && checkpointMs > 0) this.checkpointMs = checkpointMs;
|
|
362
|
+
if (!enabled) this.stopCheckpoints();
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Record that the circuit changed. Cheap and O(1), the expensive snapshot is
|
|
367
|
+
* deferred to the checkpoint timer, and skipped entirely when
|
|
368
|
+
* `persistCircuit` is off (the default).
|
|
369
|
+
*/
|
|
370
|
+
private markSnapshotDirty(): void {
|
|
371
|
+
if (!this.persistCircuit || !this.persistState) return;
|
|
372
|
+
this.snapshotDirty = true;
|
|
373
|
+
this.startCheckpoints();
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
private startCheckpoints(): void {
|
|
377
|
+
if (this.checkpointTimer) return;
|
|
378
|
+
this.checkpointTimer = setInterval(() => {
|
|
379
|
+
if (!this.snapshotDirty) return;
|
|
380
|
+
this.snapshotDirty = false;
|
|
381
|
+
void this.saveState();
|
|
382
|
+
}, this.checkpointMs);
|
|
383
|
+
// Node/test environments have no `window`; the interval alone is enough there.
|
|
384
|
+
if (typeof window !== 'undefined' && !this.pagehideHandler) {
|
|
385
|
+
this.pagehideHandler = () => {
|
|
386
|
+
if (!this.snapshotDirty) return;
|
|
387
|
+
this.snapshotDirty = false;
|
|
388
|
+
void this.saveState();
|
|
389
|
+
};
|
|
390
|
+
window.addEventListener('pagehide', this.pagehideHandler);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Stop checkpointing and drop the `pagehide` listener. */
|
|
395
|
+
stopCheckpoints(): void {
|
|
396
|
+
if (this.checkpointTimer) {
|
|
397
|
+
clearInterval(this.checkpointTimer);
|
|
398
|
+
this.checkpointTimer = null;
|
|
399
|
+
}
|
|
400
|
+
if (this.pagehideHandler && typeof window !== 'undefined') {
|
|
401
|
+
window.removeEventListener('pagehide', this.pagehideHandler);
|
|
402
|
+
}
|
|
403
|
+
this.pagehideHandler = null;
|
|
404
|
+
this.snapshotDirty = false;
|
|
278
405
|
}
|
|
279
406
|
|
|
280
407
|
async loadState() {
|
|
281
|
-
if (!this.processor || !this.persistState) return;
|
|
408
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
282
409
|
try {
|
|
283
410
|
const result = await this.persistenceClient.get(this.stateKey());
|
|
284
411
|
|
|
285
|
-
//
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
) {
|
|
293
|
-
const state = result[0][0].state;
|
|
412
|
+
// `save_state` returns a JSON string and every PersistenceClient round-trips
|
|
413
|
+
// it as one: localStorage via JSON.parse(getItem(...)), surrealdb via
|
|
414
|
+
// `_00_kv:<key>.val`. This used to test for a raw SurrealDB result shape
|
|
415
|
+
// (`result[0][0].state`), which no shipped client can ever produce, so the
|
|
416
|
+
// browser wrote a snapshot on every ingest and never restored one. The legacy
|
|
417
|
+
// shape is still accepted so an old persisted row still loads.
|
|
418
|
+
const state = typeof result === 'string' ? result : extractLegacyState(result);
|
|
419
|
+
if (state) {
|
|
294
420
|
this.logger.info(
|
|
295
421
|
{
|
|
296
422
|
stateLength: state.length,
|
|
@@ -369,7 +495,7 @@ export class StreamProcessorService {
|
|
|
369
495
|
}
|
|
370
496
|
|
|
371
497
|
async saveState() {
|
|
372
|
-
if (!this.processor || !this.persistState) return;
|
|
498
|
+
if (!this.processor || !this.persistState || !this.persistCircuit) return;
|
|
373
499
|
const generation = this.stateGeneration;
|
|
374
500
|
try {
|
|
375
501
|
// Assuming processor has a save_state method that returns the state string/bytes
|
|
@@ -454,10 +580,10 @@ export class StreamProcessorService {
|
|
|
454
580
|
// Direct handler call instead of event
|
|
455
581
|
this.notifyUpdates(updates);
|
|
456
582
|
}
|
|
457
|
-
// While batching (inside `ingestMany`), `flushCoalescing`
|
|
458
|
-
// for the whole batch
|
|
583
|
+
// While batching (inside `ingestMany`), `flushCoalescing` marks dirty once
|
|
584
|
+
// for the whole batch, skip the redundant per-record mark here.
|
|
459
585
|
if (!this.batching) {
|
|
460
|
-
this.
|
|
586
|
+
this.markSnapshotDirty();
|
|
461
587
|
}
|
|
462
588
|
return rawUpdates;
|
|
463
589
|
} catch (e) {
|
|
@@ -534,7 +660,7 @@ export class StreamProcessorService {
|
|
|
534
660
|
snapshotMs: initialUpdate.timing_snapshot_ms ?? 0,
|
|
535
661
|
},
|
|
536
662
|
};
|
|
537
|
-
this.
|
|
663
|
+
this.markSnapshotDirty();
|
|
538
664
|
this.logger.debug(
|
|
539
665
|
{
|
|
540
666
|
queryHash: queryPlan.queryHash,
|
|
@@ -561,7 +687,7 @@ export class StreamProcessorService {
|
|
|
561
687
|
if (!this.processor) return;
|
|
562
688
|
try {
|
|
563
689
|
this.processor.unregister_view(queryHash);
|
|
564
|
-
this.
|
|
690
|
+
this.markSnapshotDirty();
|
|
565
691
|
} catch (e) {
|
|
566
692
|
this.logger.error(
|
|
567
693
|
{ error: e, Category: 'sp00ky-client::StreamProcessorService::unregisterQueryPlan' },
|
|
@@ -18,6 +18,7 @@ vi.mock('@spooky-sync/ssp-wasm', () => ({
|
|
|
18
18
|
set_permissions: vi.fn(),
|
|
19
19
|
save_state: vi.fn(() => 'state-bytes'),
|
|
20
20
|
load_state: vi.fn(),
|
|
21
|
+
free: vi.fn(),
|
|
21
22
|
};
|
|
22
23
|
processorInstances.push(instance);
|
|
23
24
|
return instance;
|
|
@@ -44,7 +45,12 @@ function makeService() {
|
|
|
44
45
|
get: vi.fn(async () => null),
|
|
45
46
|
remove: vi.fn(async () => {}),
|
|
46
47
|
};
|
|
47
|
-
const service = new StreamProcessorService(
|
|
48
|
+
const service = new StreamProcessorService(
|
|
49
|
+
{} as any,
|
|
50
|
+
{} as any,
|
|
51
|
+
persistence as any,
|
|
52
|
+
silentLogger
|
|
53
|
+
);
|
|
48
54
|
return { service, persistence, persisted };
|
|
49
55
|
}
|
|
50
56
|
|
|
@@ -83,6 +89,7 @@ describe('StreamProcessorService.reset', () => {
|
|
|
83
89
|
|
|
84
90
|
it('routes persisted state to the per-bucket key', async () => {
|
|
85
91
|
const { service, persistence } = makeService();
|
|
92
|
+
service.configureCircuitPersistence(true);
|
|
86
93
|
await service.init();
|
|
87
94
|
service.setStateKeySuffix('u1');
|
|
88
95
|
await service.saveState();
|
|
@@ -91,8 +98,9 @@ describe('StreamProcessorService.reset', () => {
|
|
|
91
98
|
|
|
92
99
|
it('drops a saveState that raced a reset (old circuit never persists into the new key)', async () => {
|
|
93
100
|
const { service, persistence } = makeService();
|
|
101
|
+
service.configureCircuitPersistence(true);
|
|
94
102
|
await service.init();
|
|
95
|
-
// The snapshot is taken, then a reset lands before the persist step
|
|
103
|
+
// The snapshot is taken, then a reset lands before the persist step,
|
|
96
104
|
// simulated by bumping the generation from inside save_state().
|
|
97
105
|
processorInstances[0].save_state.mockImplementation(() => {
|
|
98
106
|
void service.reset();
|
|
@@ -101,4 +109,108 @@ describe('StreamProcessorService.reset', () => {
|
|
|
101
109
|
await service.saveState();
|
|
102
110
|
expect(persistence.set).not.toHaveBeenCalled();
|
|
103
111
|
});
|
|
112
|
+
|
|
113
|
+
it('frees the replaced wasm circuit on reset', async () => {
|
|
114
|
+
const { service } = makeService();
|
|
115
|
+
await service.init();
|
|
116
|
+
const first = processorInstances[0];
|
|
117
|
+
await service.reset();
|
|
118
|
+
// V8 cannot see wasm-internal bytes, so the FinalizationRegistry may never
|
|
119
|
+
// run. The old circuit must be freed explicitly or its whole store stays
|
|
120
|
+
// resident for the rest of the session.
|
|
121
|
+
expect(first.free).toHaveBeenCalledTimes(1);
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
// The renderer-OOM guardrail. `Circuit::save` deep-clones the entire store (every
|
|
126
|
+
// row of every ingested table) and JSON-encodes it. It used to run once per
|
|
127
|
+
// ingest batch and once per query register/unregister, which on a large windowed
|
|
128
|
+
// list meant hundreds of whole-store serializations per scroll, for a snapshot
|
|
129
|
+
// the browser could never read back. Snapshots are now opt-in and checkpointed.
|
|
130
|
+
describe('StreamProcessorService circuit snapshots', () => {
|
|
131
|
+
const plan = {
|
|
132
|
+
queryHash: 'q1',
|
|
133
|
+
surql: 'SELECT * FROM thing;',
|
|
134
|
+
params: {},
|
|
135
|
+
ttl: '10m',
|
|
136
|
+
lastActiveAt: new Date(),
|
|
137
|
+
localArray: [],
|
|
138
|
+
remoteArray: [],
|
|
139
|
+
meta: { tableName: 'thing' },
|
|
140
|
+
} as any;
|
|
141
|
+
|
|
142
|
+
it('never snapshots on ingest or register/unregister by default', async () => {
|
|
143
|
+
const { service, persistence } = makeService();
|
|
144
|
+
await service.init();
|
|
145
|
+
const processor = processorInstances[0];
|
|
146
|
+
|
|
147
|
+
service.registerQueryPlan(plan);
|
|
148
|
+
service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
|
|
149
|
+
service.ingestMany([
|
|
150
|
+
{ table: 'thing', op: 'CREATE', id: 'thing:b', record: { id: 'thing:b' } },
|
|
151
|
+
{ table: 'thing', op: 'CREATE', id: 'thing:c', record: { id: 'thing:c' } },
|
|
152
|
+
]);
|
|
153
|
+
service.unregisterQueryPlan('q1');
|
|
154
|
+
|
|
155
|
+
expect(processor.save_state).not.toHaveBeenCalled();
|
|
156
|
+
expect(persistence.set).not.toHaveBeenCalled();
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('still never snapshots inline when persistence is opted in', async () => {
|
|
160
|
+
const { service, persistence } = makeService();
|
|
161
|
+
service.configureCircuitPersistence(true, 30_000);
|
|
162
|
+
await service.init();
|
|
163
|
+
const processor = processorInstances[0];
|
|
164
|
+
|
|
165
|
+
service.registerQueryPlan(plan);
|
|
166
|
+
service.ingest('thing', 'CREATE', 'thing:a', { id: 'thing:a' });
|
|
167
|
+
service.unregisterQueryPlan('q1');
|
|
168
|
+
|
|
169
|
+
// Marked dirty, but the write waits for the checkpoint interval.
|
|
170
|
+
expect(processor.save_state).not.toHaveBeenCalled();
|
|
171
|
+
expect(persistence.set).not.toHaveBeenCalled();
|
|
172
|
+
service.stopCheckpoints();
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
it('writes at most one snapshot per checkpoint tick', async () => {
|
|
176
|
+
vi.useFakeTimers();
|
|
177
|
+
try {
|
|
178
|
+
const { service } = makeService();
|
|
179
|
+
service.configureCircuitPersistence(true, 1000);
|
|
180
|
+
await service.init();
|
|
181
|
+
const processor = processorInstances[0];
|
|
182
|
+
|
|
183
|
+
for (let i = 0; i < 50; i++) {
|
|
184
|
+
service.ingest('thing', 'CREATE', `thing:${i}`, { id: `thing:${i}` });
|
|
185
|
+
}
|
|
186
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
187
|
+
expect(processor.save_state).toHaveBeenCalledTimes(1);
|
|
188
|
+
|
|
189
|
+
// Idle interval: nothing changed, so nothing is serialized.
|
|
190
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
191
|
+
expect(processor.save_state).toHaveBeenCalledTimes(1);
|
|
192
|
+
|
|
193
|
+
service.stopCheckpoints();
|
|
194
|
+
} finally {
|
|
195
|
+
vi.useRealTimers();
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
it('restores a snapshot the shipped persistence clients actually return', async () => {
|
|
200
|
+
const { service, persistence } = makeService();
|
|
201
|
+
// Both clients round-trip `save_state`'s output as a bare string. The old
|
|
202
|
+
// shape check looked for a raw SurrealDB result, so it never matched and the
|
|
203
|
+
// browser never restored anything it wrote.
|
|
204
|
+
persistence.get.mockResolvedValue('state-bytes' as any);
|
|
205
|
+
service.configureCircuitPersistence(true);
|
|
206
|
+
await service.init();
|
|
207
|
+
expect(processorInstances[0].load_state).toHaveBeenCalledWith('state-bytes');
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it('does not restore a snapshot when persistence is off', async () => {
|
|
211
|
+
const { service, persistence } = makeService();
|
|
212
|
+
persistence.get.mockResolvedValue('state-bytes' as any);
|
|
213
|
+
await service.init();
|
|
214
|
+
expect(processorInstances[0].load_state).not.toHaveBeenCalled();
|
|
215
|
+
});
|
|
104
216
|
});
|
|
@@ -44,4 +44,9 @@ export interface WasmProcessor {
|
|
|
44
44
|
// build degrades gracefully instead of throwing.
|
|
45
45
|
load_state?(state: string): void;
|
|
46
46
|
save_state?(): string;
|
|
47
|
+
// wasm-bindgen destructor. Releases the circuit (store + every view cache)
|
|
48
|
+
// inside wasm linear memory. Without it the bytes only come back when V8
|
|
49
|
+
// happens to GC the JS wrapper, which it has no reason to hurry since it
|
|
50
|
+
// cannot see how much wasm memory the wrapper is holding.
|
|
51
|
+
free?(): void;
|
|
47
52
|
}
|
package/src/sp00ky.ts
CHANGED
|
@@ -9,12 +9,9 @@ import type {
|
|
|
9
9
|
UpdateOptions,
|
|
10
10
|
RunOptions,
|
|
11
11
|
SyncHealth,
|
|
12
|
-
StorageHealth
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
RemoteDatabaseService,
|
|
16
|
-
createLocalEngine,
|
|
17
|
-
} from './services/database/index';
|
|
12
|
+
StorageHealth,
|
|
13
|
+
} from './types';
|
|
14
|
+
import { LocalMigrator, RemoteDatabaseService, createLocalEngine } from './services/database/index';
|
|
18
15
|
import type { LocalStore } from './services/database/index';
|
|
19
16
|
import { StaleEpochError } from './services/database/index';
|
|
20
17
|
import type { UpEvent } from './modules/sync/index';
|
|
@@ -30,10 +27,9 @@ import type {
|
|
|
30
27
|
BucketNames,
|
|
31
28
|
BackendNames,
|
|
32
29
|
BackendRoutes,
|
|
33
|
-
RoutePayload
|
|
34
|
-
import {
|
|
35
|
-
QueryBuilder
|
|
30
|
+
RoutePayload,
|
|
36
31
|
} from '@spooky-sync/query-builder';
|
|
32
|
+
import { QueryBuilder } from '@spooky-sync/query-builder';
|
|
37
33
|
|
|
38
34
|
import { DevToolsService } from './modules/devtools/index';
|
|
39
35
|
import { createLogger } from './services/logger/index';
|
|
@@ -60,14 +56,19 @@ import { computeTabsFingerprint, hash53, type TabRole } from './services/tabs/pr
|
|
|
60
56
|
import type { SqliteCacheEngine } from './services/database/sqlite-cache-engine';
|
|
61
57
|
|
|
62
58
|
export class BucketHandle {
|
|
63
|
-
constructor(
|
|
59
|
+
constructor(
|
|
60
|
+
private bucketName: string,
|
|
61
|
+
private remote: RemoteDatabaseService
|
|
62
|
+
) {}
|
|
64
63
|
|
|
65
64
|
async put(path: string, content: string | Uint8Array | Blob): Promise<void> {
|
|
66
65
|
await this.remote.query(`RETURN f"${this.bucketName}:/${path}".put($content);`, { content });
|
|
67
66
|
}
|
|
68
67
|
|
|
69
68
|
async get(path: string): Promise<unknown> {
|
|
70
|
-
const [result] = await this.remote.query<[unknown]>(
|
|
69
|
+
const [result] = await this.remote.query<[unknown]>(
|
|
70
|
+
`RETURN f"${this.bucketName}:/${path}".get();`
|
|
71
|
+
);
|
|
71
72
|
return result;
|
|
72
73
|
}
|
|
73
74
|
|
|
@@ -76,26 +77,36 @@ export class BucketHandle {
|
|
|
76
77
|
}
|
|
77
78
|
|
|
78
79
|
async exists(path: string): Promise<boolean> {
|
|
79
|
-
const [result] = await this.remote.query<[boolean]>(
|
|
80
|
+
const [result] = await this.remote.query<[boolean]>(
|
|
81
|
+
`RETURN f"${this.bucketName}:/${path}".exists();`
|
|
82
|
+
);
|
|
80
83
|
return result;
|
|
81
84
|
}
|
|
82
85
|
|
|
83
86
|
async head(path: string): Promise<Record<string, unknown>> {
|
|
84
|
-
const [result] = await this.remote.query<[Record<string, unknown>]>(
|
|
87
|
+
const [result] = await this.remote.query<[Record<string, unknown>]>(
|
|
88
|
+
`RETURN f"${this.bucketName}:/${path}".head();`
|
|
89
|
+
);
|
|
85
90
|
return result;
|
|
86
91
|
}
|
|
87
92
|
|
|
88
93
|
async copy(sourcePath: string, targetPath: string): Promise<void> {
|
|
89
|
-
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, {
|
|
94
|
+
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, {
|
|
95
|
+
target: targetPath,
|
|
96
|
+
});
|
|
90
97
|
}
|
|
91
98
|
|
|
92
99
|
async rename(sourcePath: string, targetPath: string): Promise<void> {
|
|
93
|
-
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, {
|
|
100
|
+
await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, {
|
|
101
|
+
target: targetPath,
|
|
102
|
+
});
|
|
94
103
|
}
|
|
95
104
|
|
|
96
105
|
async list(prefix?: string): Promise<string[]> {
|
|
97
106
|
const p = prefix ?? '';
|
|
98
|
-
const [result] = await this.remote.query<[string[]]>(
|
|
107
|
+
const [result] = await this.remote.query<[string[]]>(
|
|
108
|
+
`RETURN f"${this.bucketName}:/${p}".list();`
|
|
109
|
+
);
|
|
99
110
|
return result;
|
|
100
111
|
}
|
|
101
112
|
}
|
|
@@ -273,6 +284,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
273
284
|
this.persistenceClient,
|
|
274
285
|
logger
|
|
275
286
|
);
|
|
287
|
+
// Circuit snapshots are opt-in and checkpointed, never per-ingest. See
|
|
288
|
+
// `persistCircuit` in types.ts for the measurements behind that default.
|
|
289
|
+
this.streamProcessor.configureCircuitPersistence(
|
|
290
|
+
config.persistCircuit ?? false,
|
|
291
|
+
config.circuitCheckpointMs
|
|
292
|
+
);
|
|
276
293
|
this.migrator = new LocalMigrator(this.local, logger);
|
|
277
294
|
|
|
278
295
|
this.cache = new CacheModule(
|
|
@@ -295,7 +312,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
295
312
|
this.local,
|
|
296
313
|
this.remote,
|
|
297
314
|
logger,
|
|
298
|
-
config.crdtDebounceMs ?? 500
|
|
315
|
+
config.crdtDebounceMs ?? 500
|
|
299
316
|
);
|
|
300
317
|
|
|
301
318
|
this.dataModule = new DataModule(
|
|
@@ -325,7 +342,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
325
342
|
degradeAfterConsecutiveFailures:
|
|
326
343
|
this.config.syncHealth === false
|
|
327
344
|
? 0
|
|
328
|
-
: this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
|
|
345
|
+
: (this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3),
|
|
329
346
|
}
|
|
330
347
|
);
|
|
331
348
|
|
|
@@ -370,7 +387,10 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
370
387
|
this.tabsCoordinator = this.buildTabsCoordinator();
|
|
371
388
|
} else if (this.config.sharedTabs) {
|
|
372
389
|
this.logger.info(
|
|
373
|
-
{
|
|
390
|
+
{
|
|
391
|
+
reason: (tabsSupport as { reason: string }).reason,
|
|
392
|
+
Category: 'sp00ky-client::Sp00kyClient::tabs',
|
|
393
|
+
},
|
|
374
394
|
'sharedTabs requested but unsupported here; running solo'
|
|
375
395
|
);
|
|
376
396
|
}
|
|
@@ -449,7 +469,8 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
449
469
|
this.cache.setIngestRelay(null);
|
|
450
470
|
this.sync.setTabContext('solo', tabId);
|
|
451
471
|
},
|
|
452
|
-
currentStorageHealth: () =>
|
|
472
|
+
currentStorageHealth: () =>
|
|
473
|
+
this.local.storageHealth ?? { status: 'unknown', fallback: false },
|
|
453
474
|
};
|
|
454
475
|
return new TabsCoordinator({
|
|
455
476
|
tabId,
|
|
@@ -527,9 +548,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
527
548
|
for (const row of records) {
|
|
528
549
|
const id = row?.id;
|
|
529
550
|
const table =
|
|
530
|
-
id && typeof id === 'object' && id.table !== undefined
|
|
531
|
-
? String(id.table)
|
|
532
|
-
: undefined;
|
|
551
|
+
id && typeof id === 'object' && id.table !== undefined ? String(id.table) : undefined;
|
|
533
552
|
if (!table) continue;
|
|
534
553
|
this.crdtManager.applyRow(table, row);
|
|
535
554
|
}
|
|
@@ -606,9 +625,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
606
625
|
await this.streamProcessor.init();
|
|
607
626
|
// Seed table `select` permissions from the schema before any query is
|
|
608
627
|
// registered — otherwise the SSP default-denies every non-`_00_` table.
|
|
609
|
-
this.streamProcessor.setPermissions(
|
|
610
|
-
extractSelectPermissions(this.config.schemaSurql)
|
|
611
|
-
);
|
|
628
|
+
this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
|
|
612
629
|
this.logger.debug(
|
|
613
630
|
{ Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
614
631
|
'StreamProcessor initialized'
|
|
@@ -770,7 +787,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
770
787
|
*/
|
|
771
788
|
private async doSwitchBucket(target: string, gateRelease?: (() => void) | null): Promise<void> {
|
|
772
789
|
this.logger.info(
|
|
773
|
-
{
|
|
790
|
+
{
|
|
791
|
+
target,
|
|
792
|
+
from: this.local.currentBucketId,
|
|
793
|
+
Category: 'sp00ky-client::Sp00kyClient::doSwitchBucket',
|
|
794
|
+
},
|
|
774
795
|
'Switching local bucket'
|
|
775
796
|
);
|
|
776
797
|
|
|
@@ -855,6 +876,11 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
855
876
|
if (this.tabsCoordinator) await this.tabsCoordinator.stop();
|
|
856
877
|
await this.local.close();
|
|
857
878
|
await this.remote.close();
|
|
879
|
+
// Free the wasm circuit explicitly. V8 cannot see wasm-internal bytes, so
|
|
880
|
+
// relying on the wasm-bindgen FinalizationRegistry leaves the whole store
|
|
881
|
+
// resident until a GC that may never come, and a client that is recreated
|
|
882
|
+
// (provider remount, HMR) would stack circuits.
|
|
883
|
+
this.streamProcessor.dispose();
|
|
858
884
|
}
|
|
859
885
|
|
|
860
886
|
/**
|
|
@@ -895,7 +921,7 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
895
921
|
table: string,
|
|
896
922
|
recordId: string,
|
|
897
923
|
field: string,
|
|
898
|
-
fallbackText?: string
|
|
924
|
+
fallbackText?: string
|
|
899
925
|
): Promise<CrdtField> {
|
|
900
926
|
return this.crdtManager.open(table, recordId, field, fallbackText);
|
|
901
927
|
}
|
|
@@ -1149,10 +1175,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
1149
1175
|
this.dataModule.recordFrontendTiming(queryHash, ms);
|
|
1150
1176
|
}
|
|
1151
1177
|
|
|
1152
|
-
run<
|
|
1153
|
-
B
|
|
1154
|
-
R
|
|
1155
|
-
|
|
1178
|
+
run<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(
|
|
1179
|
+
backend: B,
|
|
1180
|
+
path: R,
|
|
1181
|
+
payload: RoutePayload<S, B, R>,
|
|
1182
|
+
options?: RunOptions
|
|
1183
|
+
) {
|
|
1156
1184
|
return this.dataModule.run(backend, path, payload, options);
|
|
1157
1185
|
}
|
|
1158
1186
|
|
package/src/types.ts
CHANGED
|
@@ -162,6 +162,29 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
162
162
|
* Inspect via `window.__00__.getState().database.tabs` and `__sqliteStats`.
|
|
163
163
|
*/
|
|
164
164
|
sharedTabs?: boolean;
|
|
165
|
+
/**
|
|
166
|
+
* Persist the in-browser SSP circuit (store + view caches) as a snapshot so a
|
|
167
|
+
* reload can restore it instead of re-materializing. Default `false`, and
|
|
168
|
+
* that default is deliberate.
|
|
169
|
+
*
|
|
170
|
+
* The circuit is DERIVED state: the durable local store (OPFS SQLite) is the
|
|
171
|
+
* source of truth, and every first paint already reads row bodies from it
|
|
172
|
+
* (`DataManager.createNewQuery` / `materializeRecords`) using the circuit only
|
|
173
|
+
* for row identity and ordering. A snapshot buys nothing on reload while
|
|
174
|
+
* costing a full deep clone of every row of every ingested table plus a JSON
|
|
175
|
+
* encode of the result, `Circuit::save` in the Rust core, mirroring the
|
|
176
|
+
* server's rule in `ssp-node`: *never per-ingest*.
|
|
177
|
+
*
|
|
178
|
+
* When enabled, snapshots are written on a checkpoint interval
|
|
179
|
+
* ({@link circuitCheckpointMs}) and on `pagehide`, never per ingest or per
|
|
180
|
+
* query registration. Enable only for a workload that has measured a win.
|
|
181
|
+
*/
|
|
182
|
+
persistCircuit?: boolean;
|
|
183
|
+
/**
|
|
184
|
+
* Checkpoint interval in milliseconds for {@link persistCircuit}. Defaults to
|
|
185
|
+
* 30000. Ignored when `persistCircuit` is off.
|
|
186
|
+
*/
|
|
187
|
+
circuitCheckpointMs?: number;
|
|
165
188
|
/** A pino browser transmit object for forwarding logs (e.g. via @spooky-sync/core/otel). */
|
|
166
189
|
otelTransmit?: PinoTransmit;
|
|
167
190
|
/**
|