@spooky-sync/core 0.0.1-canary.103 → 0.0.1-canary.105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +180 -3
- package/dist/index.js +2212 -1620
- package/dist/types.d.ts +15 -4
- package/package.json +3 -3
- package/src/modules/cache/index.ts +35 -2
- package/src/modules/crdt/crdt-field.ts +9 -2
- package/src/modules/crdt/index.ts +7 -2
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.status.test.ts +143 -2
- package/src/modules/data/index.ts +229 -13
- package/src/modules/devtools/index.ts +8 -5
- package/src/modules/ref-tables.test.ts +26 -1
- package/src/modules/ref-tables.ts +19 -0
- package/src/modules/sync/queue/queue-down.ts +6 -0
- package/src/modules/sync/queue/queue-up.ts +14 -0
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +33 -4
- package/src/modules/sync/sync.ts +76 -5
- package/src/services/database/local-migrator.ts +6 -6
- package/src/services/database/local.test.ts +31 -0
- package/src/services/database/local.ts +298 -72
- package/src/services/stream-processor/index.ts +48 -2
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/sp00ky.auth-order.test.ts +27 -0
- package/src/sp00ky.ts +132 -2
- package/src/types.ts +15 -4
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
RoutePayload,
|
|
8
8
|
} from '@spooky-sync/query-builder';
|
|
9
9
|
import type { LocalDatabaseService } from '../../services/database/index';
|
|
10
|
+
import { StaleEpochError } from '../../services/database/index';
|
|
10
11
|
import type { CacheModule, RecordWithId, CacheRecord } from '../cache/index';
|
|
11
12
|
import type { Logger } from '../../services/logger/index';
|
|
12
13
|
import type { StreamUpdate } from '../../services/stream-processor/index';
|
|
@@ -72,6 +73,16 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
72
73
|
private statusSubscriptions: Map<QueryHash, Set<QueryStatusCallback>> = new Map();
|
|
73
74
|
private mutationCallbacks: Set<MutationCallback> = new Set();
|
|
74
75
|
private debounceTimers: Map<QueryHash, NodeJS.Timeout> = new Map();
|
|
76
|
+
// The update each debounce timer would process on its trailing edge. Kept in a
|
|
77
|
+
// map (not just the timer closure) so `flushPendingStreamUpdate` can process
|
|
78
|
+
// it early — the sync engine flushes before flipping a query to `idle`, so
|
|
79
|
+
// subscribers never observe idle status with stale (partial-window) rows.
|
|
80
|
+
private pendingStreamUpdates: Map<QueryHash, StreamUpdate> = new Map();
|
|
81
|
+
// Refcount of in-flight fetch cycles per query (registration + concurrent
|
|
82
|
+
// poll/LIVE sync rounds can overlap). Status flips to `fetching` on 0→1 and
|
|
83
|
+
// back to `idle` only on 1→0, so an inner cycle finishing can't emit a
|
|
84
|
+
// premature idle mid-registration.
|
|
85
|
+
private fetchDepth: Map<QueryHash, number> = new Map();
|
|
75
86
|
private logger: Logger;
|
|
76
87
|
/**
|
|
77
88
|
* Optional observer notified whenever a query's fetch status changes.
|
|
@@ -308,6 +319,31 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
308
319
|
}
|
|
309
320
|
}
|
|
310
321
|
|
|
322
|
+
/**
|
|
323
|
+
* Enter a fetch cycle for a query. Refcounted: registration and concurrent
|
|
324
|
+
* poll/LIVE sync rounds can overlap on the same hash, and only the OUTERMOST
|
|
325
|
+
* cycle may flip the status — 0→1 emits `fetching`, and `endFetching`'s 1→0
|
|
326
|
+
* emits `idle`. Always pair with `endFetching` in a `finally`.
|
|
327
|
+
*/
|
|
328
|
+
beginFetching(queryHash: string): void {
|
|
329
|
+
const depth = this.fetchDepth.get(queryHash) ?? 0;
|
|
330
|
+
this.fetchDepth.set(queryHash, depth + 1);
|
|
331
|
+
if (depth === 0) {
|
|
332
|
+
this.setQueryStatus(queryHash, 'fetching');
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** Leave a fetch cycle started with {@link beginFetching}; emits `idle` on the last exit. */
|
|
337
|
+
endFetching(queryHash: string): void {
|
|
338
|
+
const depth = this.fetchDepth.get(queryHash) ?? 0;
|
|
339
|
+
if (depth <= 1) {
|
|
340
|
+
this.fetchDepth.delete(queryHash);
|
|
341
|
+
this.setQueryStatus(queryHash, 'idle');
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
this.fetchDepth.set(queryHash, depth - 1);
|
|
345
|
+
}
|
|
346
|
+
|
|
311
347
|
/**
|
|
312
348
|
* Subscribe to mutations (for sync)
|
|
313
349
|
*/
|
|
@@ -342,6 +378,9 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
342
378
|
clearTimeout(existing);
|
|
343
379
|
this.debounceTimers.delete(queryHash);
|
|
344
380
|
}
|
|
381
|
+
// The DELETE update carries the full latest localArray, so the coalesced
|
|
382
|
+
// CREATE/UPDATE it supersedes is already reflected — drop it.
|
|
383
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
345
384
|
await this.processStreamUpdate(update);
|
|
346
385
|
return;
|
|
347
386
|
}
|
|
@@ -353,14 +392,35 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
353
392
|
}
|
|
354
393
|
|
|
355
394
|
// Set new timer
|
|
395
|
+
this.pendingStreamUpdates.set(queryHash, update);
|
|
356
396
|
const timer = setTimeout(async () => {
|
|
357
397
|
this.debounceTimers.delete(queryHash);
|
|
398
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
358
399
|
await this.processStreamUpdate(update);
|
|
359
400
|
}, this.streamDebounceTime);
|
|
360
401
|
|
|
361
402
|
this.debounceTimers.set(queryHash, timer);
|
|
362
403
|
}
|
|
363
404
|
|
|
405
|
+
/**
|
|
406
|
+
* Process a query's pending (debounced) stream update NOW instead of on the
|
|
407
|
+
* trailing edge. Called by the sync engine before it flips a query back to
|
|
408
|
+
* `idle`, so the status change never races ahead of the rows it fetched.
|
|
409
|
+
* No-op when nothing is pending. The pending entry is removed before the
|
|
410
|
+
* await so a concurrently-firing timer can't process it twice.
|
|
411
|
+
*/
|
|
412
|
+
async flushPendingStreamUpdate(queryHash: string): Promise<void> {
|
|
413
|
+
const timer = this.debounceTimers.get(queryHash);
|
|
414
|
+
if (timer) {
|
|
415
|
+
clearTimeout(timer);
|
|
416
|
+
this.debounceTimers.delete(queryHash);
|
|
417
|
+
}
|
|
418
|
+
const pending = this.pendingStreamUpdates.get(queryHash);
|
|
419
|
+
if (!pending) return;
|
|
420
|
+
this.pendingStreamUpdates.delete(queryHash);
|
|
421
|
+
await this.processStreamUpdate(pending);
|
|
422
|
+
}
|
|
423
|
+
|
|
364
424
|
// Materialize a query's result rows from the local DB. For a windowed query
|
|
365
425
|
// (`LIMIT n START m`, m>0) the original surql is NOT re-run — re-applying
|
|
366
426
|
// `START m` against the shared local store would skip the window's own rows
|
|
@@ -430,6 +490,11 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
430
490
|
this.recordPhase(queryState, 'sspTransform', update.transformMs);
|
|
431
491
|
const percentiles = this.computeMaterializationPercentiles(queryState.materializationSamples);
|
|
432
492
|
|
|
493
|
+
// Fence against bucket switches: this update's `localArray` came from the
|
|
494
|
+
// pre-switch SSP circuit; applying it after a switch would show (and
|
|
495
|
+
// persist) the previous user's ids in the new bucket.
|
|
496
|
+
const epoch = this.local.epoch;
|
|
497
|
+
|
|
433
498
|
try {
|
|
434
499
|
// Materialize the query's rows. For a windowed (offset) query, re-running
|
|
435
500
|
// the original surql would re-apply `START n` against the shared local DB
|
|
@@ -438,6 +503,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
438
503
|
// original ORDER BY for stable display order. Non-offset queries keep the
|
|
439
504
|
// normal re-query path.
|
|
440
505
|
const newRecords = await this.materializeRecords(queryState, localArray);
|
|
506
|
+
if (epoch !== this.local.epoch) return;
|
|
441
507
|
queryState.config.localArray = localArray;
|
|
442
508
|
|
|
443
509
|
const prevJson = JSON.stringify(queryState.records);
|
|
@@ -450,6 +516,7 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
450
516
|
// every observed engine step.
|
|
451
517
|
if (recordsChanged) {
|
|
452
518
|
queryState.updateCount++;
|
|
519
|
+
queryState.lastUpdatedAt = Date.now();
|
|
453
520
|
}
|
|
454
521
|
|
|
455
522
|
await this.local.query(
|
|
@@ -473,7 +540,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
473
540
|
materializationP55: percentiles.p55,
|
|
474
541
|
materializationP90: percentiles.p90,
|
|
475
542
|
materializationP99: percentiles.p99,
|
|
476
|
-
}
|
|
543
|
+
},
|
|
544
|
+
{ epoch }
|
|
477
545
|
);
|
|
478
546
|
|
|
479
547
|
if (!recordsChanged) {
|
|
@@ -501,6 +569,13 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
501
569
|
'Query updated from stream'
|
|
502
570
|
);
|
|
503
571
|
} catch (err) {
|
|
572
|
+
if (err instanceof StaleEpochError) {
|
|
573
|
+
this.logger.debug(
|
|
574
|
+
{ queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
575
|
+
'Dropped stream update from before a bucket switch'
|
|
576
|
+
);
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
504
579
|
queryState.errorCount++;
|
|
505
580
|
this.logger.error(
|
|
506
581
|
{ err, queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
|
|
@@ -750,6 +825,8 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
750
825
|
clearTimeout(debounce);
|
|
751
826
|
this.debounceTimers.delete(hash);
|
|
752
827
|
}
|
|
828
|
+
this.pendingStreamUpdates.delete(hash);
|
|
829
|
+
this.fetchDepth.delete(hash);
|
|
753
830
|
this.cache.unregisterQuery(hash);
|
|
754
831
|
this.activeQueries.delete(hash);
|
|
755
832
|
this.subscriptions.delete(hash);
|
|
@@ -782,11 +859,21 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
782
859
|
);
|
|
783
860
|
return;
|
|
784
861
|
}
|
|
862
|
+
const epoch = this.local.epoch;
|
|
785
863
|
queryState.config.localArray = localArray;
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
864
|
+
try {
|
|
865
|
+
await this.local.query(
|
|
866
|
+
surql.seal(surql.updateSet('id', ['localArray'])),
|
|
867
|
+
{
|
|
868
|
+
id: queryState.config.id,
|
|
869
|
+
localArray,
|
|
870
|
+
},
|
|
871
|
+
{ epoch }
|
|
872
|
+
);
|
|
873
|
+
} catch (err) {
|
|
874
|
+
if (err instanceof StaleEpochError) return;
|
|
875
|
+
throw err;
|
|
876
|
+
}
|
|
790
877
|
}
|
|
791
878
|
|
|
792
879
|
async updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray): Promise<void> {
|
|
@@ -798,11 +885,124 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
798
885
|
);
|
|
799
886
|
return;
|
|
800
887
|
}
|
|
888
|
+
const epoch = this.local.epoch;
|
|
801
889
|
queryState.config.remoteArray = remoteArray;
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
890
|
+
try {
|
|
891
|
+
await this.local.query(
|
|
892
|
+
surql.seal(surql.updateSet('id', ['remoteArray'])),
|
|
893
|
+
{
|
|
894
|
+
id: queryState.config.id,
|
|
895
|
+
remoteArray,
|
|
896
|
+
},
|
|
897
|
+
{ epoch }
|
|
898
|
+
);
|
|
899
|
+
} catch (err) {
|
|
900
|
+
if (err instanceof StaleEpochError) return;
|
|
901
|
+
throw err;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Cancel every armed timer ahead of a local-bucket switch: stream-update
|
|
907
|
+
* debounce timers (their pending updates carry the OLD bucket's id-sets) and
|
|
908
|
+
* per-query TTL heartbeats (they'd refresh the previous user's remote
|
|
909
|
+
* `_00_query` rows under the new session). The rebind re-arms heartbeats.
|
|
910
|
+
*/
|
|
911
|
+
quiesce(): void {
|
|
912
|
+
for (const timer of this.debounceTimers.values()) {
|
|
913
|
+
clearTimeout(timer);
|
|
914
|
+
}
|
|
915
|
+
this.debounceTimers.clear();
|
|
916
|
+
this.pendingStreamUpdates.clear();
|
|
917
|
+
this.fetchDepth.clear();
|
|
918
|
+
for (const queryState of this.activeQueries.values()) {
|
|
919
|
+
if (queryState.ttlTimer) {
|
|
920
|
+
clearTimeout(queryState.ttlTimer);
|
|
921
|
+
queryState.ttlTimer = null;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
/**
|
|
927
|
+
* Re-home every active query in a freshly-opened bucket, KEEPING its hash —
|
|
928
|
+
* `useQuery` subscriptions are keyed by hash and don't re-register on auth
|
|
929
|
+
* changes, so the hooks must stay attached. Per query:
|
|
930
|
+
* 1. reset the sync arrays + hydration flag and drop the previous user's
|
|
931
|
+
* records, notifying subscribers with the new-bucket materialization
|
|
932
|
+
* (usually empty) so their rows leave the UI immediately;
|
|
933
|
+
* 2. recreate the `_00_query` row in the new bucket;
|
|
934
|
+
* 3. re-register the SSP view on the (fresh, post-reset) processor — this
|
|
935
|
+
* also rebinds the view to the NEW `$auth` context;
|
|
936
|
+
* 4. restart the TTL heartbeat.
|
|
937
|
+
* Returns the hashes so the caller can enqueue remote re-registration, which
|
|
938
|
+
* refills records from the server via the normal register→sync→notify path.
|
|
939
|
+
*/
|
|
940
|
+
async rebindAfterBucketSwitch(): Promise<QueryHash[]> {
|
|
941
|
+
const hashes: QueryHash[] = [];
|
|
942
|
+
for (const [hash, queryState] of this.activeQueries.entries()) {
|
|
943
|
+
const config = queryState.config;
|
|
944
|
+
config.localArray = [];
|
|
945
|
+
config.remoteArray = [];
|
|
946
|
+
config.subqueryRemoteArray = undefined;
|
|
947
|
+
queryState.hydrated = false;
|
|
948
|
+
queryState.syncNotified = false;
|
|
949
|
+
queryState.records = [];
|
|
950
|
+
// Via setQueryStatus (not a bare assignment) so status observers see the
|
|
951
|
+
// flip back to a loading state.
|
|
952
|
+
this.setQueryStatus(hash, 'fetching');
|
|
953
|
+
|
|
954
|
+
try {
|
|
955
|
+
await withRetry(this.logger, () =>
|
|
956
|
+
this.local.query<[QueryConfigRecord]>(surql.seal(surql.create('id', 'data')), {
|
|
957
|
+
id: config.id,
|
|
958
|
+
data: {
|
|
959
|
+
surql: config.surql,
|
|
960
|
+
params: config.params,
|
|
961
|
+
localArray: [],
|
|
962
|
+
remoteArray: [],
|
|
963
|
+
lastActiveAt: new Date(),
|
|
964
|
+
createdAt: new Date(),
|
|
965
|
+
ttl: config.ttl,
|
|
966
|
+
tableName: config.tableName,
|
|
967
|
+
updateCount: queryState.updateCount,
|
|
968
|
+
rowCount: 0,
|
|
969
|
+
errorCount: queryState.errorCount,
|
|
970
|
+
},
|
|
971
|
+
})
|
|
972
|
+
);
|
|
973
|
+
|
|
974
|
+
const { localArray } = this.cache.registerQuery({
|
|
975
|
+
queryHash: hash,
|
|
976
|
+
surql: config.surql,
|
|
977
|
+
params: config.params,
|
|
978
|
+
ttl: new Duration(config.ttl),
|
|
979
|
+
lastActiveAt: new Date(),
|
|
980
|
+
});
|
|
981
|
+
config.localArray = localArray;
|
|
982
|
+
await this.local.query(surql.seal(surql.updateSet('id', ['localArray', 'rowCount'])), {
|
|
983
|
+
id: config.id,
|
|
984
|
+
localArray,
|
|
985
|
+
rowCount: localArray.length,
|
|
986
|
+
});
|
|
987
|
+
} catch (err) {
|
|
988
|
+
this.logger.error(
|
|
989
|
+
{ err, hash, Category: 'sp00ky-client::DataModule::rebindAfterBucketSwitch' },
|
|
990
|
+
'Failed to rebind query after bucket switch; remote re-registration will retry'
|
|
991
|
+
);
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
// Notify AFTER the SSP re-registration so a subscriber that re-reads
|
|
995
|
+
// synchronously sees consistent (empty) state.
|
|
996
|
+
const subscribers = this.subscriptions.get(hash);
|
|
997
|
+
if (subscribers) {
|
|
998
|
+
for (const callback of subscribers) {
|
|
999
|
+
callback(queryState.records);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
this.startTTLHeartbeat(queryState, hash);
|
|
1003
|
+
hashes.push(hash);
|
|
1004
|
+
}
|
|
1005
|
+
return hashes;
|
|
806
1006
|
}
|
|
807
1007
|
|
|
808
1008
|
/**
|
|
@@ -812,18 +1012,29 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
812
1012
|
async notifyQuerySynced(queryHash: string): Promise<void> {
|
|
813
1013
|
const queryState = this.activeQueries.get(queryHash);
|
|
814
1014
|
if (!queryState) return;
|
|
1015
|
+
const epoch = this.local.epoch;
|
|
815
1016
|
|
|
816
1017
|
// Re-query local DB for latest data (windowed queries materialize from the
|
|
817
1018
|
// list_ref window so they resolve even if the in-browser SSP never emits —
|
|
818
1019
|
// it can't compute a high offset whose preceding rows aren't resident).
|
|
819
1020
|
const newRecords = await this.materializeRecords(queryState);
|
|
1021
|
+
// Bucket switched while we materialized: these rows mix old-bucket reads
|
|
1022
|
+
// with new-bucket state — drop them; the rebind/re-registration re-emits.
|
|
1023
|
+
if (epoch !== this.local.epoch) return;
|
|
820
1024
|
const changed = JSON.stringify(queryState.records) !== JSON.stringify(newRecords);
|
|
821
1025
|
queryState.records = newRecords;
|
|
822
1026
|
|
|
823
|
-
// Notify if data changed OR if this
|
|
824
|
-
// The latter handles "query truly has no
|
|
825
|
-
|
|
1027
|
+
// Notify if data changed OR if this registration lifetime hasn't emitted a
|
|
1028
|
+
// post-sync notification yet. The latter handles "query truly has no
|
|
1029
|
+
// results" so the UI can stop loading — gated on the in-memory
|
|
1030
|
+
// `syncNotified` flag rather than `updateCount === 0`, because updateCount
|
|
1031
|
+
// is PERSISTED across deregister/re-register: a re-registered empty window
|
|
1032
|
+
// (updateCount > 0, records unchanged) would otherwise never emit and its
|
|
1033
|
+
// subscribers would show a loading state forever.
|
|
1034
|
+
if (changed || !queryState.syncNotified) {
|
|
1035
|
+
queryState.syncNotified = true;
|
|
826
1036
|
queryState.updateCount++;
|
|
1037
|
+
queryState.lastUpdatedAt = Date.now();
|
|
827
1038
|
const subscribers = this.subscriptions.get(queryHash);
|
|
828
1039
|
if (subscribers) {
|
|
829
1040
|
for (const callback of subscribers) {
|
|
@@ -1391,10 +1602,15 @@ export class DataModule<S extends SchemaStructure> {
|
|
|
1391
1602
|
ttlTimer: null,
|
|
1392
1603
|
ttlDurationMs: parseDuration(ttl),
|
|
1393
1604
|
updateCount: persistedUpdateCount,
|
|
1605
|
+
lastUpdatedAt: null,
|
|
1394
1606
|
materializationSamples: [],
|
|
1395
1607
|
lastIngestLatencyMs: null,
|
|
1396
1608
|
errorCount: persistedErrorCount,
|
|
1397
|
-
|
|
1609
|
+
// Born `fetching`, not `idle`: every cold registration is followed by a
|
|
1610
|
+
// `register` down-event whose lifecycle (Sp00kySync.registerQuery) resolves
|
|
1611
|
+
// the status to `idle` once the initial sync completed. Starting idle left
|
|
1612
|
+
// a gap where a fresh windowed query looked settled while still empty.
|
|
1613
|
+
status: 'fetching',
|
|
1398
1614
|
phaseSamples: {},
|
|
1399
1615
|
phaseLast: {},
|
|
1400
1616
|
registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
|
|
@@ -124,6 +124,10 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
124
124
|
const queries = this.dataManager.getActiveQueries();
|
|
125
125
|
queries.forEach((q) => {
|
|
126
126
|
const queryHash = this.hashString(encodeRecordId(q.config.id));
|
|
127
|
+
const createdAt =
|
|
128
|
+
q.config.lastActiveAt instanceof Date
|
|
129
|
+
? q.config.lastActiveAt.getTime()
|
|
130
|
+
: new Date(q.config.lastActiveAt || Date.now()).getTime();
|
|
127
131
|
result.set(queryHash, {
|
|
128
132
|
queryHash,
|
|
129
133
|
status: 'active',
|
|
@@ -131,11 +135,10 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
131
135
|
// registration flag above. `fetchStatus` is 'idle' | 'fetching'.
|
|
132
136
|
fetchStatus: q.status,
|
|
133
137
|
isFetching: q.status === 'fetching',
|
|
134
|
-
createdAt
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
lastUpdate: Date.now(),
|
|
138
|
+
createdAt,
|
|
139
|
+
// Real last-update time; before the first update it equals createdAt.
|
|
140
|
+
// (Previously Date.now(), which reset the column on every state push.)
|
|
141
|
+
lastUpdate: q.lastUpdatedAt ?? createdAt,
|
|
139
142
|
updateCount: q.updateCount,
|
|
140
143
|
ttl: q.config.ttl,
|
|
141
144
|
query: q.config.surql,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { RecordId } from 'surrealdb';
|
|
3
|
-
import { ANON_USER_ID, listRefTableFor, sanitizeUserId } from './ref-tables';
|
|
3
|
+
import { ANON_USER_ID, bucketIdForUser, listRefTableFor, sanitizeUserId } from './ref-tables';
|
|
4
4
|
|
|
5
5
|
describe('listRefTableFor', () => {
|
|
6
6
|
it('returns global table in single mode regardless of user', () => {
|
|
@@ -64,3 +64,28 @@ describe('sanitizeUserId', () => {
|
|
|
64
64
|
expect(sanitizeUserId('user:has-dash')).toBeNull();
|
|
65
65
|
});
|
|
66
66
|
});
|
|
67
|
+
|
|
68
|
+
describe('bucketIdForUser', () => {
|
|
69
|
+
it('routes signed-out sessions to the anon bucket', () => {
|
|
70
|
+
expect(bucketIdForUser(null)).toBe(ANON_USER_ID);
|
|
71
|
+
expect(bucketIdForUser(undefined)).toBe(ANON_USER_ID);
|
|
72
|
+
expect(bucketIdForUser(ANON_USER_ID)).toBe(ANON_USER_ID);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('uses the sanitized id for valid users', () => {
|
|
76
|
+
expect(bucketIdForUser('user:abc123')).toBe('abc123');
|
|
77
|
+
expect(bucketIdForUser(new RecordId('user', 'xyz'))).toBe('xyz');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('gives unsanitizable ids a deterministic per-user bucket, never anon', () => {
|
|
81
|
+
const a = bucketIdForUser('user:has-dash');
|
|
82
|
+
const b = bucketIdForUser('user:has-dash');
|
|
83
|
+
const c = bucketIdForUser('user:other-dash');
|
|
84
|
+
// Falling back to the shared anon bucket here would recreate the
|
|
85
|
+
// cross-user local-cache leak this helper exists to prevent.
|
|
86
|
+
expect(a).not.toBe(ANON_USER_ID);
|
|
87
|
+
expect(a).toBe(b);
|
|
88
|
+
expect(a).not.toBe(c);
|
|
89
|
+
expect(a).toMatch(/^u[0-9a-f]+$/);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
// already reads it from `SPKY_SSP_REF_MODE`; add a matching codegen
|
|
11
11
|
// export then.
|
|
12
12
|
|
|
13
|
+
import { cyrb53 } from '@spooky-sync/query-builder';
|
|
14
|
+
|
|
13
15
|
export type RefMode = 'single' | 'dedicated';
|
|
14
16
|
|
|
15
17
|
/**
|
|
@@ -53,6 +55,23 @@ export function sanitizeUserId(userId: unknown): string | null {
|
|
|
53
55
|
return raw;
|
|
54
56
|
}
|
|
55
57
|
|
|
58
|
+
/**
|
|
59
|
+
* Resolve the LOCAL storage bucket id for a user. Every user gets their own
|
|
60
|
+
* IndexedDB-backed local store (`indxdb://sp00ky-<bucketId>`) so cached rows,
|
|
61
|
+
* query state, and the mutation outbox never leak across accounts on a shared
|
|
62
|
+
* device. Signed-out sessions share the `anon` bucket.
|
|
63
|
+
*
|
|
64
|
+
* An id that fails sanitization still gets a DETERMINISTIC per-user bucket
|
|
65
|
+
* (cyrb53 hex of the raw id) — falling back to `anon` here would put an
|
|
66
|
+
* authenticated user in the shared bucket and recreate the cross-user leak.
|
|
67
|
+
*/
|
|
68
|
+
export function bucketIdForUser(userId: unknown): string {
|
|
69
|
+
if (userId === null || userId === undefined || userId === ANON_USER_ID) return ANON_USER_ID;
|
|
70
|
+
const uid = sanitizeUserId(userId);
|
|
71
|
+
if (uid) return uid;
|
|
72
|
+
return `u${cyrb53(String(userId)).toString(16)}`;
|
|
73
|
+
}
|
|
74
|
+
|
|
56
75
|
/**
|
|
57
76
|
* Returns the `_00_list_ref` table name for `(mode, userId)`. Falls
|
|
58
77
|
* back to the global `_00_list_ref` when sanitization fails or in
|
|
@@ -72,6 +72,12 @@ export class DownQueue {
|
|
|
72
72
|
});
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
/** Drop all queued events (bucket switch: old-bucket register/sync work is
|
|
76
|
+
* re-derived after the switch; replaying it would target dead query rows). */
|
|
77
|
+
clear(): void {
|
|
78
|
+
this.queue = [];
|
|
79
|
+
}
|
|
80
|
+
|
|
75
81
|
async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
|
|
76
82
|
const event = this.queue.shift();
|
|
77
83
|
if (event) {
|
|
@@ -105,6 +105,20 @@ export class UpQueue {
|
|
|
105
105
|
this.debouncedMutations.set(key, { timer, firstBeforeRecord });
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
+
/**
|
|
109
|
+
* Cancel all pending debounce timers WITHOUT enqueueing their events. Used
|
|
110
|
+
* on local-bucket switches: the mutation's `_00_pending_mutations` row was
|
|
111
|
+
* already persisted at mutation time, so dropping the in-memory push only
|
|
112
|
+
* defers it to that bucket's next `loadFromDatabase` — it must NOT be pushed
|
|
113
|
+
* now, the remote session already belongs to the next user.
|
|
114
|
+
*/
|
|
115
|
+
clearDebounceTimers(): void {
|
|
116
|
+
for (const { timer } of this.debouncedMutations.values()) {
|
|
117
|
+
clearTimeout(timer);
|
|
118
|
+
}
|
|
119
|
+
this.debouncedMutations.clear();
|
|
120
|
+
}
|
|
121
|
+
|
|
108
122
|
async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
|
|
109
123
|
const event = this.queue.shift();
|
|
110
124
|
if (event) {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { SyncScheduler } from './scheduler';
|
|
3
|
+
import type { UpQueue, DownQueue, UpEvent } from './queue/index';
|
|
4
|
+
|
|
5
|
+
// `SyncScheduler.pause()` is the drain step of a local-bucket switch: it must
|
|
6
|
+
// (a) let an IN-FLIGHT queue item finish — including its outbox-row delete,
|
|
7
|
+
// which has to land in the old bucket — and only then resolve, and (b) refuse
|
|
8
|
+
// to start new rounds until `resume()`. The pause point is between items.
|
|
9
|
+
|
|
10
|
+
const silentLogger = {
|
|
11
|
+
child: () => silentLogger,
|
|
12
|
+
debug: () => {},
|
|
13
|
+
info: () => {},
|
|
14
|
+
warn: () => {},
|
|
15
|
+
error: () => {},
|
|
16
|
+
} as any;
|
|
17
|
+
|
|
18
|
+
function makeQueues(items: UpEvent[]) {
|
|
19
|
+
const upQueue = {
|
|
20
|
+
queue: [...items],
|
|
21
|
+
get size() {
|
|
22
|
+
return this.queue.length;
|
|
23
|
+
},
|
|
24
|
+
events: { subscribe: () => {} },
|
|
25
|
+
loadFromDatabase: async () => {},
|
|
26
|
+
async next(fn: (event: UpEvent) => Promise<void>) {
|
|
27
|
+
const event = this.queue.shift();
|
|
28
|
+
if (event) await fn(event);
|
|
29
|
+
},
|
|
30
|
+
} as unknown as UpQueue & { queue: UpEvent[] };
|
|
31
|
+
|
|
32
|
+
const downQueue = {
|
|
33
|
+
queue: [] as unknown[],
|
|
34
|
+
get size() {
|
|
35
|
+
return this.queue.length;
|
|
36
|
+
},
|
|
37
|
+
events: { subscribe: () => {} },
|
|
38
|
+
async next() {},
|
|
39
|
+
clear() {
|
|
40
|
+
this.queue = [];
|
|
41
|
+
},
|
|
42
|
+
} as unknown as DownQueue;
|
|
43
|
+
|
|
44
|
+
return { upQueue, downQueue };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
|
|
48
|
+
|
|
49
|
+
describe('SyncScheduler.pause', () => {
|
|
50
|
+
it('waits for the in-flight item (push + outbox delete) before resolving', async () => {
|
|
51
|
+
const { upQueue, downQueue } = makeQueues([upEvent(1), upEvent(2)]);
|
|
52
|
+
let release!: () => void;
|
|
53
|
+
const gate = new Promise<void>((r) => (release = r));
|
|
54
|
+
const processed: unknown[] = [];
|
|
55
|
+
const scheduler = new SyncScheduler(
|
|
56
|
+
upQueue,
|
|
57
|
+
downQueue,
|
|
58
|
+
async (event) => {
|
|
59
|
+
await gate; // simulate a slow remote push
|
|
60
|
+
processed.push(event);
|
|
61
|
+
},
|
|
62
|
+
async () => {},
|
|
63
|
+
silentLogger
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
const round = scheduler.syncUp();
|
|
67
|
+
const pauseDone = vi.fn();
|
|
68
|
+
const pause = scheduler.pause().then(pauseDone);
|
|
69
|
+
|
|
70
|
+
// The in-flight item hasn't finished — pause must not have resolved.
|
|
71
|
+
await Promise.resolve();
|
|
72
|
+
expect(pauseDone).not.toHaveBeenCalled();
|
|
73
|
+
|
|
74
|
+
release();
|
|
75
|
+
await pause;
|
|
76
|
+
await round;
|
|
77
|
+
// Exactly the in-flight item completed; the second stayed queued.
|
|
78
|
+
expect(processed).toHaveLength(1);
|
|
79
|
+
expect(upQueue.size).toBe(1);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('refuses new rounds while paused and drains them on resume', async () => {
|
|
83
|
+
const { upQueue, downQueue } = makeQueues([upEvent(1)]);
|
|
84
|
+
const processed: unknown[] = [];
|
|
85
|
+
const scheduler = new SyncScheduler(
|
|
86
|
+
upQueue,
|
|
87
|
+
downQueue,
|
|
88
|
+
async (event) => {
|
|
89
|
+
processed.push(event);
|
|
90
|
+
},
|
|
91
|
+
async () => {},
|
|
92
|
+
silentLogger
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
await scheduler.pause();
|
|
96
|
+
await scheduler.syncUp();
|
|
97
|
+
expect(processed).toHaveLength(0);
|
|
98
|
+
|
|
99
|
+
scheduler.resume();
|
|
100
|
+
await vi.waitFor(() => expect(processed).toHaveLength(1));
|
|
101
|
+
expect(upQueue.size).toBe(0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('resolves immediately when nothing is in flight', async () => {
|
|
105
|
+
const { upQueue, downQueue } = makeQueues([]);
|
|
106
|
+
const scheduler = new SyncScheduler(upQueue, downQueue, async () => {}, async () => {}, silentLogger);
|
|
107
|
+
await expect(scheduler.pause()).resolves.toBeUndefined();
|
|
108
|
+
});
|
|
109
|
+
});
|
|
@@ -9,6 +9,8 @@ import { SyncQueueEventTypes } from './events/index';
|
|
|
9
9
|
export class SyncScheduler {
|
|
10
10
|
private isSyncingUp: boolean = false;
|
|
11
11
|
private isSyncingDown: boolean = false;
|
|
12
|
+
private paused: boolean = false;
|
|
13
|
+
private pauseWaiters: Array<() => void> = [];
|
|
12
14
|
|
|
13
15
|
constructor(
|
|
14
16
|
private upQueue: UpQueue,
|
|
@@ -49,15 +51,40 @@ export class SyncScheduler {
|
|
|
49
51
|
this.downQueue.push(event);
|
|
50
52
|
}
|
|
51
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Suspend syncing for a local-bucket switch. Refuses new rounds and resolves
|
|
56
|
+
* once any in-flight round has finished — the pause point is BETWEEN queue
|
|
57
|
+
* items, never between an item's remote push and its outbox-row delete, so a
|
|
58
|
+
* processed mutation's `DELETE _00_pending_mutations` always lands in the
|
|
59
|
+
* store it was read from.
|
|
60
|
+
*/
|
|
61
|
+
pause(): Promise<void> {
|
|
62
|
+
this.paused = true;
|
|
63
|
+
if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
|
|
64
|
+
return new Promise<void>((resolve) => this.pauseWaiters.push(resolve));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
resume(): void {
|
|
68
|
+
this.paused = false;
|
|
69
|
+
void this.syncUp();
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private maybeResolvePause() {
|
|
73
|
+
if (!this.paused || this.isSyncingUp || this.isSyncingDown) return;
|
|
74
|
+
const waiters = this.pauseWaiters;
|
|
75
|
+
this.pauseWaiters = [];
|
|
76
|
+
for (const resolve of waiters) resolve();
|
|
77
|
+
}
|
|
78
|
+
|
|
52
79
|
/**
|
|
53
80
|
* Process upload queue
|
|
54
81
|
*/
|
|
55
82
|
async syncUp() {
|
|
56
|
-
if (this.isSyncingUp) return;
|
|
83
|
+
if (this.isSyncingUp || this.paused) return;
|
|
57
84
|
this.isSyncingUp = true;
|
|
58
85
|
let processedAny = false;
|
|
59
86
|
try {
|
|
60
|
-
while (this.upQueue.size > 0) {
|
|
87
|
+
while (this.upQueue.size > 0 && !this.paused) {
|
|
61
88
|
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
62
89
|
processedAny = true;
|
|
63
90
|
}
|
|
@@ -77,6 +104,7 @@ export class SyncScheduler {
|
|
|
77
104
|
);
|
|
78
105
|
} finally {
|
|
79
106
|
this.isSyncingUp = false;
|
|
107
|
+
this.maybeResolvePause();
|
|
80
108
|
void this.syncDown();
|
|
81
109
|
}
|
|
82
110
|
}
|
|
@@ -85,13 +113,13 @@ export class SyncScheduler {
|
|
|
85
113
|
* Process download queue
|
|
86
114
|
*/
|
|
87
115
|
async syncDown() {
|
|
88
|
-
if (this.isSyncingDown) return;
|
|
116
|
+
if (this.isSyncingDown || this.paused) return;
|
|
89
117
|
if (this.upQueue.size > 0) return;
|
|
90
118
|
|
|
91
119
|
this.isSyncingDown = true;
|
|
92
120
|
let processedAny = false;
|
|
93
121
|
try {
|
|
94
|
-
while (this.downQueue.size > 0) {
|
|
122
|
+
while (this.downQueue.size > 0 && !this.paused) {
|
|
95
123
|
if (this.upQueue.size > 0) break;
|
|
96
124
|
await this.downQueue.next(this.onProcessDown);
|
|
97
125
|
processedAny = true;
|
|
@@ -112,6 +140,7 @@ export class SyncScheduler {
|
|
|
112
140
|
);
|
|
113
141
|
} finally {
|
|
114
142
|
this.isSyncingDown = false;
|
|
143
|
+
this.maybeResolvePause();
|
|
115
144
|
}
|
|
116
145
|
}
|
|
117
146
|
|