@spooky-sync/core 0.0.1-canary.161 → 0.0.1-canary.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +294 -4
- package/dist/index.js +1033 -293
- package/dist/otel/index.d.ts +1 -1
- package/dist/types.d.ts +98 -1
- package/package.json +3 -3
- package/src/modules/crdt/crdt-hydration.test.ts +4 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +109 -7
- package/src/modules/data/data.membership.test.ts +297 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +291 -39
- package/src/modules/data/window-query.ts +53 -13
- package/src/modules/sync/sync.live-removal.test.ts +134 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.ts +143 -43
- package/src/services/database/connection-supervisor.test.ts +255 -0
- package/src/services/database/connection-supervisor.ts +351 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +21 -1
- package/src/services/database/index.ts +1 -0
- package/src/services/database/local-migrator.ts +1 -0
- package/src/services/database/remote.ts +99 -3
- package/src/services/database/sqlite-cache-engine.ts +1 -0
- package/src/sp00ky.ts +32 -1
- package/src/types.ts +99 -0
- package/src/utils/index.ts +38 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { Sp00kySync } from './sync';
|
|
3
|
+
|
|
4
|
+
// The reconnect handler re-registers active queries and re-issues the ref LIVE.
|
|
5
|
+
// It must fire on BOTH drop paths, and the distinction is the whole bug:
|
|
6
|
+
//
|
|
7
|
+
// - recovered drop: error -> reconnecting -> connected (no `disconnected`!)
|
|
8
|
+
// - gave-up drop: error -> disconnected -> (supervisor) -> connected
|
|
9
|
+
//
|
|
10
|
+
// Watching only `disconnected` therefore misses every *successful* reconnect —
|
|
11
|
+
// the common case — and leaves the dead server-side LIVE in place with the
|
|
12
|
+
// list_ref poll silently the only sync path.
|
|
13
|
+
|
|
14
|
+
/** Minimal stand-in for the SDK's event publisher. */
|
|
15
|
+
function makeClient() {
|
|
16
|
+
const listeners = new Map<string, Array<(...a: any[]) => void>>();
|
|
17
|
+
return {
|
|
18
|
+
subscribe(event: string, cb: (...a: any[]) => void) {
|
|
19
|
+
const arr = listeners.get(event) ?? [];
|
|
20
|
+
arr.push(cb);
|
|
21
|
+
listeners.set(event, arr);
|
|
22
|
+
return () => {
|
|
23
|
+
/* noop */
|
|
24
|
+
};
|
|
25
|
+
},
|
|
26
|
+
emit(event: string, ...args: any[]) {
|
|
27
|
+
for (const cb of listeners.get(event) ?? []) cb(...args);
|
|
28
|
+
},
|
|
29
|
+
liveOf: async () => ({ subscribe: () => () => {} }),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function makeSync(hashes: string[], userId: string | null = 'user:alice') {
|
|
34
|
+
const logger: any = {
|
|
35
|
+
child: () => logger,
|
|
36
|
+
debug: () => {},
|
|
37
|
+
info: () => {},
|
|
38
|
+
warn: () => {},
|
|
39
|
+
error: () => {},
|
|
40
|
+
trace: () => {},
|
|
41
|
+
};
|
|
42
|
+
const client = makeClient();
|
|
43
|
+
const status = { value: 'connected' as string };
|
|
44
|
+
const remote: any = {
|
|
45
|
+
query: vi.fn().mockResolvedValue(['live-uuid']),
|
|
46
|
+
getClient: () => client,
|
|
47
|
+
getStatus: () => status.value,
|
|
48
|
+
};
|
|
49
|
+
const dataModule: any = {
|
|
50
|
+
getActiveQueryHashes: () => hashes,
|
|
51
|
+
getCurrentUserId: () => userId,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const sync = new Sp00kySync({} as any, remote, {} as any, dataModule, {} as any, logger);
|
|
55
|
+
|
|
56
|
+
// `subscribeToReconnect` is normally wired from init(), which would also start
|
|
57
|
+
// timers and load the outbox. Call it directly instead.
|
|
58
|
+
(sync as any).currentUserId = userId;
|
|
59
|
+
(sync as any).subscribeToReconnect();
|
|
60
|
+
|
|
61
|
+
const enqueued: any[] = [];
|
|
62
|
+
(sync as any).scheduler = { enqueueDownEvent: (e: any) => enqueued.push(e) };
|
|
63
|
+
|
|
64
|
+
return { sync, client, remote, enqueued, status };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Let the `connected` handler's un-awaited `restartRefLiveQuery()` chain run. */
|
|
68
|
+
const flush = () => new Promise((r) => setTimeout(r, 0));
|
|
69
|
+
|
|
70
|
+
describe('sync reconnect re-subscription', () => {
|
|
71
|
+
beforeEach(() => vi.clearAllMocks());
|
|
72
|
+
|
|
73
|
+
it('re-registers queries and restarts LIVE on a recovered drop (reconnecting -> connected)', async () => {
|
|
74
|
+
const { client, remote, enqueued } = makeSync(['h1', 'h2']);
|
|
75
|
+
|
|
76
|
+
// The SDK's own reconnect: no `disconnected` is ever published.
|
|
77
|
+
client.emit('reconnecting');
|
|
78
|
+
client.emit('connected');
|
|
79
|
+
await flush();
|
|
80
|
+
|
|
81
|
+
expect(enqueued).toEqual([
|
|
82
|
+
{ type: 'register', payload: { hash: 'h1' } },
|
|
83
|
+
{ type: 'register', payload: { hash: 'h2' } },
|
|
84
|
+
]);
|
|
85
|
+
const sqls = (remote.query as any).mock.calls.map((c: any[]) => c[0]);
|
|
86
|
+
expect(sqls.some((s: string) => s.startsWith('LIVE SELECT'))).toBe(true);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('re-registers queries and restarts LIVE after the SDK gives up (disconnected -> connected)', async () => {
|
|
90
|
+
const { client, remote, enqueued } = makeSync(['h1']);
|
|
91
|
+
|
|
92
|
+
client.emit('disconnected');
|
|
93
|
+
client.emit('connected');
|
|
94
|
+
await flush();
|
|
95
|
+
|
|
96
|
+
expect(enqueued).toEqual([{ type: 'register', payload: { hash: 'h1' } }]);
|
|
97
|
+
const sqls = (remote.query as any).mock.calls.map((c: any[]) => c[0]);
|
|
98
|
+
expect(sqls.some((s: string) => s.startsWith('LIVE SELECT'))).toBe(true);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('does nothing on the initial connect', async () => {
|
|
102
|
+
const { client, remote, enqueued } = makeSync(['h1']);
|
|
103
|
+
|
|
104
|
+
client.emit('connected');
|
|
105
|
+
await flush();
|
|
106
|
+
|
|
107
|
+
expect(enqueued).toEqual([]);
|
|
108
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('only reacts once per drop', async () => {
|
|
112
|
+
const { client, enqueued } = makeSync(['h1']);
|
|
113
|
+
|
|
114
|
+
client.emit('reconnecting');
|
|
115
|
+
client.emit('connected');
|
|
116
|
+
await flush();
|
|
117
|
+
// A spurious second `connected` (e.g. a re-published event) must not
|
|
118
|
+
// trigger a second refetch storm.
|
|
119
|
+
client.emit('connected');
|
|
120
|
+
await flush();
|
|
121
|
+
|
|
122
|
+
expect(enqueued).toHaveLength(1);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("does not KILL the dead session's LIVE uuid on reconnect", async () => {
|
|
126
|
+
const { sync, client, remote } = makeSync(['h1']);
|
|
127
|
+
// A LIVE was running before the drop. Its uuid belongs to the old
|
|
128
|
+
// WebSocket session, so KILLing it on the new socket is pointless work
|
|
129
|
+
// queued ahead of the restart that actually matters.
|
|
130
|
+
const unsub = vi.fn();
|
|
131
|
+
(sync as any).currentLiveQueryUuid = 'stale-uuid';
|
|
132
|
+
(sync as any).liveQueryUnsubscribe = unsub;
|
|
133
|
+
|
|
134
|
+
client.emit('reconnecting');
|
|
135
|
+
expect(unsub).toHaveBeenCalled();
|
|
136
|
+
expect((sync as any).currentLiveQueryUuid).toBeNull();
|
|
137
|
+
|
|
138
|
+
client.emit('connected');
|
|
139
|
+
await flush();
|
|
140
|
+
|
|
141
|
+
const sqls = (remote.query as any).mock.calls.map((c: any[]) => c[0]);
|
|
142
|
+
expect(sqls.some((q: string) => q.includes('KILL'))).toBe(false);
|
|
143
|
+
expect(sqls.some((q: string) => q.startsWith('LIVE SELECT'))).toBe(true);
|
|
144
|
+
});
|
|
145
|
+
});
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
-
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
2
1
|
import type {
|
|
2
|
+
ConnectionSupervisor,
|
|
3
|
+
LocalStore,
|
|
4
|
+
RemoteDatabaseService,
|
|
5
|
+
} from '../../services/database/index';
|
|
6
|
+
import type {
|
|
7
|
+
ConnectionState,
|
|
3
8
|
RecordVersionArray,
|
|
4
9
|
RecordVersionDiff,
|
|
5
10
|
SyncHealth,
|
|
@@ -11,6 +16,7 @@ import type { DownEvent, UpEvent } from './queue/index';
|
|
|
11
16
|
import { DownQueue, UpQueue } from './queue/index';
|
|
12
17
|
import type { RecordId, Uuid } from 'surrealdb';
|
|
13
18
|
import {
|
|
19
|
+
applyRecordVersionDiff,
|
|
14
20
|
ArraySyncer,
|
|
15
21
|
buildListRefSelect,
|
|
16
22
|
buildSubqueryListRefSelect,
|
|
@@ -31,6 +37,7 @@ import {
|
|
|
31
37
|
extractIdPart,
|
|
32
38
|
extractTablePart,
|
|
33
39
|
surql,
|
|
40
|
+
withTimeout,
|
|
34
41
|
} from '../../utils/index';
|
|
35
42
|
import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
36
43
|
import { mutationOwnerTabId } from '../data/mutation-id';
|
|
@@ -66,6 +73,13 @@ export interface Sp00kySyncOptions {
|
|
|
66
73
|
* up-queue for the session. Defaults to 30000; `0` disables the timeout.
|
|
67
74
|
*/
|
|
68
75
|
pushTimeoutMs?: number;
|
|
76
|
+
/**
|
|
77
|
+
* Transport supervisor. Sync reads its state to report `connection` in
|
|
78
|
+
* {@link SyncHealth} so a UI can show "reconnecting…" the instant the socket
|
|
79
|
+
* drops, without waiting for the degrade threshold. Optional: omitted in
|
|
80
|
+
* tests, where `connection` then reports `connected`.
|
|
81
|
+
*/
|
|
82
|
+
connectionSupervisor?: ConnectionSupervisor;
|
|
69
83
|
}
|
|
70
84
|
|
|
71
85
|
/**
|
|
@@ -87,7 +101,12 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
87
101
|
return this.syncEngine.events;
|
|
88
102
|
}
|
|
89
103
|
private scheduler: SyncScheduler;
|
|
90
|
-
|
|
104
|
+
/**
|
|
105
|
+
* Set by any event that means the socket we registered on is gone, so the
|
|
106
|
+
* next `connected` knows it must re-subscribe rather than treat itself as the
|
|
107
|
+
* initial connect. See {@link subscribeToReconnect}.
|
|
108
|
+
*/
|
|
109
|
+
private needsResubscribe: boolean = false;
|
|
91
110
|
public events = createSyncEventSystem();
|
|
92
111
|
|
|
93
112
|
// Auth identity that drives per-user `_00_list_ref_user_<id>` routing
|
|
@@ -216,6 +235,18 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
216
235
|
private static readonly SELF_HEAL_BASE_MS = 2_000;
|
|
217
236
|
private static readonly SELF_HEAL_MAX_MS = 30_000;
|
|
218
237
|
|
|
238
|
+
/**
|
|
239
|
+
* Transport supervisor, when one was supplied. Sync only reads state from it;
|
|
240
|
+
* it never drives reconnects itself.
|
|
241
|
+
*/
|
|
242
|
+
private readonly connectionSupervisor?: ConnectionSupervisor;
|
|
243
|
+
/**
|
|
244
|
+
* Mirror of the supervisor's state. Defaults to `connected` so a client
|
|
245
|
+
* constructed without a supervisor (tests, embedders) reports the same health
|
|
246
|
+
* shape it always has rather than a permanent false "disconnected".
|
|
247
|
+
*/
|
|
248
|
+
private connectionState: ConnectionState = 'connected';
|
|
249
|
+
|
|
219
250
|
/** Current sync-health snapshot. */
|
|
220
251
|
get syncHealth(): SyncHealth {
|
|
221
252
|
return {
|
|
@@ -224,6 +255,7 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
224
255
|
kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
|
|
225
256
|
error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
|
|
226
257
|
everConnected: this.hasSyncedOnce,
|
|
258
|
+
connection: this.connectionState,
|
|
227
259
|
};
|
|
228
260
|
}
|
|
229
261
|
|
|
@@ -244,6 +276,25 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
244
276
|
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
245
277
|
}
|
|
246
278
|
|
|
279
|
+
/**
|
|
280
|
+
* Mirror the supervisor's transport state into {@link SyncHealth} and emit on
|
|
281
|
+
* every change, so a UI can react to a dropped socket immediately instead of
|
|
282
|
+
* waiting for `degradeAfterFailures` failed rounds. `status` is untouched:
|
|
283
|
+
* a brief reconnect is not a degradation.
|
|
284
|
+
*
|
|
285
|
+
* No explicit unsubscribe: the supervisor is owned by the same client and
|
|
286
|
+
* drops all subscribers in its own `dispose()`, which `Sp00kyClient.close()`
|
|
287
|
+
* calls first.
|
|
288
|
+
*/
|
|
289
|
+
private subscribeToConnectionState(): void {
|
|
290
|
+
if (!this.connectionSupervisor) return;
|
|
291
|
+
this.connectionSupervisor.subscribe((state) => {
|
|
292
|
+
if (this.connectionState === state) return;
|
|
293
|
+
this.connectionState = state;
|
|
294
|
+
this.emitSyncHealth();
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
|
|
247
298
|
/**
|
|
248
299
|
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
249
300
|
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
@@ -392,6 +443,7 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
392
443
|
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
393
444
|
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
394
445
|
this.pushTimeoutMs = Math.max(0, options?.pushTimeoutMs ?? 30_000);
|
|
446
|
+
this.connectionSupervisor = options?.connectionSupervisor;
|
|
395
447
|
}
|
|
396
448
|
|
|
397
449
|
/**
|
|
@@ -404,6 +456,7 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
404
456
|
this.isInit = true;
|
|
405
457
|
await this.scheduler.init({ loadOutbox: this.tabRole !== 'follower' });
|
|
406
458
|
this.subscribeToReconnect();
|
|
459
|
+
this.subscribeToConnectionState();
|
|
407
460
|
void this.scheduler.syncUp();
|
|
408
461
|
void this.scheduler.syncDown();
|
|
409
462
|
// No initial LIVE subscription — wait for `setCurrentUserId` to fire
|
|
@@ -905,13 +958,20 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
905
958
|
this.liveQueryUnsubscribe = null;
|
|
906
959
|
}
|
|
907
960
|
if (this.currentLiveQueryUuid !== null) {
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
961
|
+
// A LIVE subscription is scoped to its WebSocket session, so after a
|
|
962
|
+
// reconnect the server-side one is already gone and there is nothing to
|
|
963
|
+
// KILL. Sending it anyway either fails (no connection) or races the fresh
|
|
964
|
+
// socket's readiness while holding up the restart behind it. Local
|
|
965
|
+
// bookkeeping is cleared either way.
|
|
966
|
+
if (this.remote.getStatus() === 'connected') {
|
|
967
|
+
try {
|
|
968
|
+
await this.remote.query('KILL $u', { u: this.currentLiveQueryUuid });
|
|
969
|
+
} catch (err) {
|
|
970
|
+
this.logger.debug(
|
|
971
|
+
{ err, Category: 'sp00ky-client::Sp00kySync::killRefLiveQuery' },
|
|
972
|
+
'Prior LIVE KILL failed; continuing'
|
|
973
|
+
);
|
|
974
|
+
}
|
|
915
975
|
}
|
|
916
976
|
this.currentLiveQueryUuid = null;
|
|
917
977
|
}
|
|
@@ -922,21 +982,58 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
922
982
|
await this.startRefLiveQueries();
|
|
923
983
|
}
|
|
924
984
|
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
985
|
+
/**
|
|
986
|
+
* Drop local LIVE bookkeeping without issuing a `KILL`.
|
|
987
|
+
*
|
|
988
|
+
* Called when the socket dies. The server-side subscription is scoped to that
|
|
989
|
+
* WebSocket session and died with it, so there is nothing left to kill — and
|
|
990
|
+
* by the time the reconnect handler runs, the client reports `connected`
|
|
991
|
+
* again, which would otherwise send a `KILL` for a stale uuid on the *new*
|
|
992
|
+
* session and hold up the restart queued behind it.
|
|
993
|
+
*/
|
|
994
|
+
private invalidateRefLiveQuery(): void {
|
|
995
|
+
if (this.liveQueryUnsubscribe) {
|
|
996
|
+
try {
|
|
997
|
+
this.liveQueryUnsubscribe();
|
|
998
|
+
} catch {
|
|
999
|
+
/* ignore */
|
|
1000
|
+
}
|
|
1001
|
+
this.liveQueryUnsubscribe = null;
|
|
1002
|
+
}
|
|
1003
|
+
this.currentLiveQueryUuid = null;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
// Only the connect that follows a prior drop counts as a reconnect; the
|
|
1007
|
+
// initial connect after init() must not trigger a refetch storm.
|
|
1008
|
+
//
|
|
1009
|
+
// Both drop events have to be watched. The SDK publishes `disconnected` ONLY
|
|
1010
|
+
// when it has given up entirely (attempts exhausted, or the engine
|
|
1011
|
+
// terminated); an ordinary recovered drop goes `error` -> `reconnecting` ->
|
|
1012
|
+
// `connected` and never touches `disconnected`. Listening for `disconnected`
|
|
1013
|
+
// alone therefore misses every successful reconnect — the exact case this
|
|
1014
|
+
// handler exists for — and leaves the dead server-side LIVE in place with the
|
|
1015
|
+
// poll as the only sync path.
|
|
928
1016
|
private subscribeToReconnect() {
|
|
929
1017
|
const client = this.remote.getClient();
|
|
930
1018
|
client.subscribe('disconnected', () => {
|
|
931
|
-
this.
|
|
1019
|
+
this.needsResubscribe = true;
|
|
1020
|
+
this.invalidateRefLiveQuery();
|
|
932
1021
|
this.logger.info(
|
|
933
1022
|
{ Category: 'sp00ky-client::Sp00kySync::onDisconnect' },
|
|
934
1023
|
'Remote disconnected'
|
|
935
1024
|
);
|
|
936
1025
|
});
|
|
1026
|
+
client.subscribe('reconnecting', () => {
|
|
1027
|
+
this.needsResubscribe = true;
|
|
1028
|
+
this.invalidateRefLiveQuery();
|
|
1029
|
+
this.logger.info(
|
|
1030
|
+
{ Category: 'sp00ky-client::Sp00kySync::onReconnecting' },
|
|
1031
|
+
'Remote socket dropped; awaiting reconnect'
|
|
1032
|
+
);
|
|
1033
|
+
});
|
|
937
1034
|
client.subscribe('connected', () => {
|
|
938
|
-
if (!this.
|
|
939
|
-
this.
|
|
1035
|
+
if (!this.needsResubscribe) return;
|
|
1036
|
+
this.needsResubscribe = false;
|
|
940
1037
|
this.logger.info(
|
|
941
1038
|
{ Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
942
1039
|
'Remote reconnected, refetching active queries'
|
|
@@ -1082,7 +1179,29 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1082
1179
|
// `config.id` is `_00_query:<hash>`, so its id-part IS the query hash
|
|
1083
1180
|
// (a SHA-256 over query content + sessionId) — the key DataModule uses.
|
|
1084
1181
|
const hash = extractIdPart(existing.config.id);
|
|
1182
|
+
|
|
1183
|
+
// Apply the event to `remoteArray` — the authoritative membership rows are
|
|
1184
|
+
// now rendered FROM. Only registration and the poll used to write it, so a
|
|
1185
|
+
// LIVE removal left the departed id in the list (in memory and persisted)
|
|
1186
|
+
// until the next poll tick, which is up to 5s of showing a deleted row. This
|
|
1187
|
+
// also persists the durable `_00_window` mirror, so the removal survives a
|
|
1188
|
+
// reload with no network.
|
|
1189
|
+
if (existing.config.membershipKnown && (diff.removed.length || diff.added.length)) {
|
|
1190
|
+
const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], diff);
|
|
1191
|
+
if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) {
|
|
1192
|
+
await this.dataModule.updateQueryRemoteArray(hash, next);
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1085
1196
|
await this.runSyncForQuery(hash, diff);
|
|
1197
|
+
|
|
1198
|
+
// A removal-only diff sets `fetching` false in `runSyncForQuery`, so it gets
|
|
1199
|
+
// no `flushPendingStreamUpdate`/`endFetching` re-render — and a removal needs
|
|
1200
|
+
// no record fetch to trigger one either. Force it, mirroring what the poll
|
|
1201
|
+
// path already does for its own removals (`refetchListRefForQuery`).
|
|
1202
|
+
if (diff.removed.length > 0 && diff.added.length === 0 && diff.updated.length === 0) {
|
|
1203
|
+
await this.dataModule.notifyQuerySynced(hash);
|
|
1204
|
+
}
|
|
1086
1205
|
}
|
|
1087
1206
|
|
|
1088
1207
|
/**
|
|
@@ -1156,22 +1275,11 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1156
1275
|
* retries rather than rolling the mutation back.
|
|
1157
1276
|
*/
|
|
1158
1277
|
private withPushTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
promise.then(
|
|
1165
|
-
(value) => {
|
|
1166
|
-
clearTimeout(timer);
|
|
1167
|
-
resolve(value);
|
|
1168
|
-
},
|
|
1169
|
-
(err) => {
|
|
1170
|
-
clearTimeout(timer);
|
|
1171
|
-
reject(err);
|
|
1172
|
-
}
|
|
1173
|
-
);
|
|
1174
|
-
});
|
|
1278
|
+
return withTimeout(
|
|
1279
|
+
promise,
|
|
1280
|
+
this.pushTimeoutMs,
|
|
1281
|
+
`Mutation push timed out after ${this.pushTimeoutMs}ms (${label})`
|
|
1282
|
+
);
|
|
1175
1283
|
}
|
|
1176
1284
|
|
|
1177
1285
|
private async processUpEvent(event: UpEvent) {
|
|
@@ -1424,18 +1532,10 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1424
1532
|
* would otherwise resurrect a just-deleted record.
|
|
1425
1533
|
*/
|
|
1426
1534
|
private async getPendingDeleteIds(): Promise<Set<string>> {
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
return new Set((rows ?? []).map((r) => encodeRecordId(r.recordId)));
|
|
1432
|
-
} catch (err) {
|
|
1433
|
-
this.logger.warn(
|
|
1434
|
-
{ err, Category: 'sp00ky-client::Sp00kySync::getPendingDeleteIds' },
|
|
1435
|
-
'Failed to read pending deletes; sync may briefly resurrect a just-deleted record'
|
|
1436
|
-
);
|
|
1437
|
-
return new Set();
|
|
1438
|
-
}
|
|
1535
|
+
// Single implementation, shared with the render path: `materializeRecords`
|
|
1536
|
+
// subtracts the same set so a row whose DELETE is still in the outbox is
|
|
1537
|
+
// neither re-fetched here nor rendered there.
|
|
1538
|
+
return (await this.dataModule.getPendingRecordIds()).deletes;
|
|
1439
1539
|
}
|
|
1440
1540
|
|
|
1441
1541
|
/**
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { ConnectionSupervisor } from './connection-supervisor';
|
|
3
|
+
import type { ReconnectConfig } from '../../types';
|
|
4
|
+
|
|
5
|
+
// The supervisor covers the two failure modes the SDK's own reconnect cannot:
|
|
6
|
+
// - it stopped trying (attempts exhausted, or its post-reconnect handshake
|
|
7
|
+
// threw and it terminated the engine) — nothing else would ever reconnect
|
|
8
|
+
// - the socket never closed at all (half-open: peer gone, readyState OPEN,
|
|
9
|
+
// no `close` event, so no reconnect is ever triggered)
|
|
10
|
+
|
|
11
|
+
const CONFIG: Required<ReconnectConfig> = {
|
|
12
|
+
attempts: -1,
|
|
13
|
+
retryDelayMax: 8_000,
|
|
14
|
+
heartbeatIntervalMs: 1_000,
|
|
15
|
+
heartbeatTimeoutMs: 500,
|
|
16
|
+
superviseRetryDelayMaxMs: 4_000,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function makeRemote() {
|
|
20
|
+
const handlers = new Map<string, Array<(...a: any[]) => void>>();
|
|
21
|
+
const state = { status: 'connected' as string };
|
|
22
|
+
const remote: any = {
|
|
23
|
+
getStatus: () => state.status,
|
|
24
|
+
getReconnectConfig: () => CONFIG,
|
|
25
|
+
subscribeConnection: (event: string, cb: (...a: any[]) => void) => {
|
|
26
|
+
const arr = handlers.get(event) ?? [];
|
|
27
|
+
arr.push(cb);
|
|
28
|
+
handlers.set(event, arr);
|
|
29
|
+
return () => {
|
|
30
|
+
handlers.set(
|
|
31
|
+
event,
|
|
32
|
+
(handlers.get(event) ?? []).filter((h) => h !== cb)
|
|
33
|
+
);
|
|
34
|
+
};
|
|
35
|
+
},
|
|
36
|
+
connect: vi.fn().mockImplementation(async () => {
|
|
37
|
+
state.status = 'connected';
|
|
38
|
+
}),
|
|
39
|
+
forceClose: vi.fn().mockImplementation(async () => {
|
|
40
|
+
state.status = 'disconnected';
|
|
41
|
+
emit('disconnected');
|
|
42
|
+
}),
|
|
43
|
+
query: vi.fn().mockResolvedValue([true]),
|
|
44
|
+
};
|
|
45
|
+
function emit(event: string, ...args: any[]) {
|
|
46
|
+
for (const cb of Array.from(handlers.get(event) ?? [])) cb(...args);
|
|
47
|
+
}
|
|
48
|
+
return { remote, emit, state, handlerCount: () => handlers.size };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const silentLogger: any = (() => {
|
|
52
|
+
const l: any = {
|
|
53
|
+
child: () => l,
|
|
54
|
+
debug: () => {},
|
|
55
|
+
info: () => {},
|
|
56
|
+
warn: () => {},
|
|
57
|
+
error: () => {},
|
|
58
|
+
trace: () => {},
|
|
59
|
+
};
|
|
60
|
+
return l;
|
|
61
|
+
})();
|
|
62
|
+
|
|
63
|
+
function makeSupervisor(remote: any) {
|
|
64
|
+
return new ConnectionSupervisor(remote, silentLogger, CONFIG);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe('ConnectionSupervisor', () => {
|
|
68
|
+
beforeEach(() => {
|
|
69
|
+
vi.useFakeTimers();
|
|
70
|
+
vi.clearAllMocks();
|
|
71
|
+
});
|
|
72
|
+
afterEach(() => {
|
|
73
|
+
vi.useRealTimers();
|
|
74
|
+
vi.unstubAllGlobals();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('reconnects after the SDK gives up, retrying on backoff until it succeeds', async () => {
|
|
78
|
+
const { remote, emit, state } = makeRemote();
|
|
79
|
+
// Two failures, then success — proves the loop keeps going rather than
|
|
80
|
+
// giving up like the SDK does.
|
|
81
|
+
remote.connect
|
|
82
|
+
.mockRejectedValueOnce(new Error('refused'))
|
|
83
|
+
.mockRejectedValueOnce(new Error('refused'))
|
|
84
|
+
.mockImplementationOnce(async () => {
|
|
85
|
+
state.status = 'connected';
|
|
86
|
+
emit('connected');
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
const sup = makeSupervisor(remote);
|
|
90
|
+
sup.start();
|
|
91
|
+
|
|
92
|
+
state.status = 'disconnected';
|
|
93
|
+
emit('disconnected');
|
|
94
|
+
expect(sup.connection).toBe('disconnected');
|
|
95
|
+
|
|
96
|
+
// 1s, then 2s, then 4s (capped at superviseRetryDelayMaxMs).
|
|
97
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
98
|
+
expect(remote.connect).toHaveBeenCalledTimes(1);
|
|
99
|
+
expect(sup.connection).toBe('reconnecting');
|
|
100
|
+
|
|
101
|
+
await vi.advanceTimersByTimeAsync(2_000);
|
|
102
|
+
expect(remote.connect).toHaveBeenCalledTimes(2);
|
|
103
|
+
|
|
104
|
+
await vi.advanceTimersByTimeAsync(4_000);
|
|
105
|
+
expect(remote.connect).toHaveBeenCalledTimes(3);
|
|
106
|
+
expect(sup.connection).toBe('connected');
|
|
107
|
+
|
|
108
|
+
// Recovered: no further attempts.
|
|
109
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
110
|
+
expect(remote.connect).toHaveBeenCalledTimes(3);
|
|
111
|
+
|
|
112
|
+
sup.dispose();
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('does not reconnect while the SDK is still retrying', async () => {
|
|
116
|
+
const { remote, emit, state } = makeRemote();
|
|
117
|
+
const sup = makeSupervisor(remote);
|
|
118
|
+
sup.start();
|
|
119
|
+
|
|
120
|
+
state.status = 'reconnecting';
|
|
121
|
+
emit('reconnecting');
|
|
122
|
+
expect(sup.connection).toBe('reconnecting');
|
|
123
|
+
|
|
124
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
125
|
+
// The SDK owns the socket during its own retry loop; racing it would open
|
|
126
|
+
// a second connection.
|
|
127
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
128
|
+
|
|
129
|
+
sup.dispose();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('forces a teardown when a heartbeat never answers (half-open socket)', async () => {
|
|
133
|
+
const { remote, emit } = makeRemote();
|
|
134
|
+
// The defining symptom: the RPC never settles and no `close` ever fires.
|
|
135
|
+
remote.query.mockImplementation(() => new Promise(() => {}));
|
|
136
|
+
|
|
137
|
+
const sup = makeSupervisor(remote);
|
|
138
|
+
emit('connected');
|
|
139
|
+
sup.start();
|
|
140
|
+
|
|
141
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatIntervalMs);
|
|
142
|
+
expect(remote.query).toHaveBeenCalledWith('RETURN true');
|
|
143
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
144
|
+
|
|
145
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatTimeoutMs);
|
|
146
|
+
expect(remote.forceClose).toHaveBeenCalledTimes(1);
|
|
147
|
+
|
|
148
|
+
// The forced close published `disconnected`, so the revive loop takes over.
|
|
149
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
150
|
+
expect(remote.connect).toHaveBeenCalled();
|
|
151
|
+
|
|
152
|
+
sup.dispose();
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it('keeps heartbeating while the connection is healthy', async () => {
|
|
156
|
+
const { remote, emit } = makeRemote();
|
|
157
|
+
const sup = makeSupervisor(remote);
|
|
158
|
+
emit('connected');
|
|
159
|
+
sup.start();
|
|
160
|
+
|
|
161
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatIntervalMs * 3 + 10);
|
|
162
|
+
expect(remote.query.mock.calls.length).toBeGreaterThanOrEqual(3);
|
|
163
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
164
|
+
expect(sup.connection).toBe('connected');
|
|
165
|
+
|
|
166
|
+
sup.dispose();
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it('parks reconnects while the browser reports offline, and resumes on online', async () => {
|
|
170
|
+
const listeners = new Map<string, Array<() => void>>();
|
|
171
|
+
vi.stubGlobal('window', {
|
|
172
|
+
addEventListener: (e: string, cb: () => void) => {
|
|
173
|
+
listeners.set(e, [...(listeners.get(e) ?? []), cb]);
|
|
174
|
+
},
|
|
175
|
+
removeEventListener: () => {},
|
|
176
|
+
});
|
|
177
|
+
const { remote, emit, state } = makeRemote();
|
|
178
|
+
const sup = makeSupervisor(remote);
|
|
179
|
+
sup.start();
|
|
180
|
+
|
|
181
|
+
state.status = 'disconnected';
|
|
182
|
+
emit('disconnected');
|
|
183
|
+
listeners.get('offline')?.forEach((cb) => cb());
|
|
184
|
+
|
|
185
|
+
// Retrying a socket against a down interface only burns backoff.
|
|
186
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
187
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
188
|
+
|
|
189
|
+
// `online` is the strongest available hint a reconnect will now work, so it
|
|
190
|
+
// probes immediately rather than waiting out a backoff.
|
|
191
|
+
listeners.get('online')?.forEach((cb) => cb());
|
|
192
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
193
|
+
expect(remote.connect).toHaveBeenCalledTimes(1);
|
|
194
|
+
|
|
195
|
+
sup.dispose();
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('probes immediately when a hidden tab becomes visible', async () => {
|
|
199
|
+
const listeners = new Map<string, Array<() => void>>();
|
|
200
|
+
vi.stubGlobal('document', {
|
|
201
|
+
visibilityState: 'visible',
|
|
202
|
+
addEventListener: (e: string, cb: () => void) => {
|
|
203
|
+
listeners.set(e, [...(listeners.get(e) ?? []), cb]);
|
|
204
|
+
},
|
|
205
|
+
removeEventListener: () => {},
|
|
206
|
+
});
|
|
207
|
+
const { remote, emit } = makeRemote();
|
|
208
|
+
const sup = makeSupervisor(remote);
|
|
209
|
+
emit('connected');
|
|
210
|
+
sup.start();
|
|
211
|
+
|
|
212
|
+
listeners.get('visibilitychange')?.forEach((cb) => cb());
|
|
213
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
214
|
+
// Connected-looking socket: probe it rather than reconnect. A sleep/wake
|
|
215
|
+
// cycle is exactly how you get a socket that looks fine and isn't.
|
|
216
|
+
expect(remote.query).toHaveBeenCalledWith('RETURN true');
|
|
217
|
+
|
|
218
|
+
sup.dispose();
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('dispose stops every timer and unsubscribes', async () => {
|
|
222
|
+
const { remote, emit, state } = makeRemote();
|
|
223
|
+
const sup = makeSupervisor(remote);
|
|
224
|
+
emit('connected');
|
|
225
|
+
sup.start();
|
|
226
|
+
|
|
227
|
+
sup.dispose();
|
|
228
|
+
state.status = 'disconnected';
|
|
229
|
+
emit('disconnected');
|
|
230
|
+
|
|
231
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
232
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
233
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
234
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
it('reports state changes to subscribers', () => {
|
|
238
|
+
const { remote, emit, state } = makeRemote();
|
|
239
|
+
const sup = makeSupervisor(remote);
|
|
240
|
+
sup.start();
|
|
241
|
+
|
|
242
|
+
const seen: string[] = [];
|
|
243
|
+
sup.subscribe((s) => seen.push(s));
|
|
244
|
+
// Fires immediately with the current value.
|
|
245
|
+
expect(seen).toEqual(['connected']);
|
|
246
|
+
|
|
247
|
+
state.status = 'reconnecting';
|
|
248
|
+
emit('reconnecting');
|
|
249
|
+
state.status = 'connected';
|
|
250
|
+
emit('connected');
|
|
251
|
+
expect(seen).toEqual(['connected', 'reconnecting', 'connected']);
|
|
252
|
+
|
|
253
|
+
sup.dispose();
|
|
254
|
+
});
|
|
255
|
+
});
|