@spooky-sync/core 0.0.1-canary.160 → 0.0.1-canary.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +294 -4
- package/dist/index.js +1053 -295
- package/dist/otel/index.d.ts +1 -1
- package/dist/types.d.ts +98 -1
- package/package.json +3 -3
- package/src/modules/crdt/crdt-hydration.test.ts +4 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +109 -7
- package/src/modules/data/data.membership.test.ts +297 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +291 -39
- package/src/modules/data/window-query.ts +53 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +27 -0
- package/src/modules/sync/queue/queue-up.ts +24 -5
- package/src/modules/sync/sync.live-removal.test.ts +134 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.ts +143 -43
- package/src/services/database/connection-supervisor.test.ts +255 -0
- package/src/services/database/connection-supervisor.ts +351 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +21 -1
- package/src/services/database/index.ts +1 -0
- package/src/services/database/local-migrator.ts +1 -0
- package/src/services/database/remote.ts +99 -3
- package/src/services/database/sqlite-cache-engine.ts +1 -0
- package/src/sp00ky.ts +32 -1
- package/src/types.ts +99 -0
- package/src/utils/index.ts +38 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { ConnectionSupervisor } from './connection-supervisor';
|
|
3
|
+
import type { ReconnectConfig } from '../../types';
|
|
4
|
+
|
|
5
|
+
// The supervisor covers the two failure modes the SDK's own reconnect cannot:
|
|
6
|
+
// - it stopped trying (attempts exhausted, or its post-reconnect handshake
|
|
7
|
+
// threw and it terminated the engine) — nothing else would ever reconnect
|
|
8
|
+
// - the socket never closed at all (half-open: peer gone, readyState OPEN,
|
|
9
|
+
// no `close` event, so no reconnect is ever triggered)
|
|
10
|
+
|
|
11
|
+
const CONFIG: Required<ReconnectConfig> = {
|
|
12
|
+
attempts: -1,
|
|
13
|
+
retryDelayMax: 8_000,
|
|
14
|
+
heartbeatIntervalMs: 1_000,
|
|
15
|
+
heartbeatTimeoutMs: 500,
|
|
16
|
+
superviseRetryDelayMaxMs: 4_000,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function makeRemote() {
|
|
20
|
+
const handlers = new Map<string, Array<(...a: any[]) => void>>();
|
|
21
|
+
const state = { status: 'connected' as string };
|
|
22
|
+
const remote: any = {
|
|
23
|
+
getStatus: () => state.status,
|
|
24
|
+
getReconnectConfig: () => CONFIG,
|
|
25
|
+
subscribeConnection: (event: string, cb: (...a: any[]) => void) => {
|
|
26
|
+
const arr = handlers.get(event) ?? [];
|
|
27
|
+
arr.push(cb);
|
|
28
|
+
handlers.set(event, arr);
|
|
29
|
+
return () => {
|
|
30
|
+
handlers.set(
|
|
31
|
+
event,
|
|
32
|
+
(handlers.get(event) ?? []).filter((h) => h !== cb)
|
|
33
|
+
);
|
|
34
|
+
};
|
|
35
|
+
},
|
|
36
|
+
connect: vi.fn().mockImplementation(async () => {
|
|
37
|
+
state.status = 'connected';
|
|
38
|
+
}),
|
|
39
|
+
forceClose: vi.fn().mockImplementation(async () => {
|
|
40
|
+
state.status = 'disconnected';
|
|
41
|
+
emit('disconnected');
|
|
42
|
+
}),
|
|
43
|
+
query: vi.fn().mockResolvedValue([true]),
|
|
44
|
+
};
|
|
45
|
+
function emit(event: string, ...args: any[]) {
|
|
46
|
+
for (const cb of Array.from(handlers.get(event) ?? [])) cb(...args);
|
|
47
|
+
}
|
|
48
|
+
return { remote, emit, state, handlerCount: () => handlers.size };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const silentLogger: any = (() => {
|
|
52
|
+
const l: any = {
|
|
53
|
+
child: () => l,
|
|
54
|
+
debug: () => {},
|
|
55
|
+
info: () => {},
|
|
56
|
+
warn: () => {},
|
|
57
|
+
error: () => {},
|
|
58
|
+
trace: () => {},
|
|
59
|
+
};
|
|
60
|
+
return l;
|
|
61
|
+
})();
|
|
62
|
+
|
|
63
|
+
function makeSupervisor(remote: any) {
|
|
64
|
+
return new ConnectionSupervisor(remote, silentLogger, CONFIG);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe('ConnectionSupervisor', () => {
|
|
68
|
+
beforeEach(() => {
|
|
69
|
+
vi.useFakeTimers();
|
|
70
|
+
vi.clearAllMocks();
|
|
71
|
+
});
|
|
72
|
+
afterEach(() => {
|
|
73
|
+
vi.useRealTimers();
|
|
74
|
+
vi.unstubAllGlobals();
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('reconnects after the SDK gives up, retrying on backoff until it succeeds', async () => {
|
|
78
|
+
const { remote, emit, state } = makeRemote();
|
|
79
|
+
// Two failures, then success — proves the loop keeps going rather than
|
|
80
|
+
// giving up like the SDK does.
|
|
81
|
+
remote.connect
|
|
82
|
+
.mockRejectedValueOnce(new Error('refused'))
|
|
83
|
+
.mockRejectedValueOnce(new Error('refused'))
|
|
84
|
+
.mockImplementationOnce(async () => {
|
|
85
|
+
state.status = 'connected';
|
|
86
|
+
emit('connected');
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
const sup = makeSupervisor(remote);
|
|
90
|
+
sup.start();
|
|
91
|
+
|
|
92
|
+
state.status = 'disconnected';
|
|
93
|
+
emit('disconnected');
|
|
94
|
+
expect(sup.connection).toBe('disconnected');
|
|
95
|
+
|
|
96
|
+
// 1s, then 2s, then 4s (capped at superviseRetryDelayMaxMs).
|
|
97
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
98
|
+
expect(remote.connect).toHaveBeenCalledTimes(1);
|
|
99
|
+
expect(sup.connection).toBe('reconnecting');
|
|
100
|
+
|
|
101
|
+
await vi.advanceTimersByTimeAsync(2_000);
|
|
102
|
+
expect(remote.connect).toHaveBeenCalledTimes(2);
|
|
103
|
+
|
|
104
|
+
await vi.advanceTimersByTimeAsync(4_000);
|
|
105
|
+
expect(remote.connect).toHaveBeenCalledTimes(3);
|
|
106
|
+
expect(sup.connection).toBe('connected');
|
|
107
|
+
|
|
108
|
+
// Recovered: no further attempts.
|
|
109
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
110
|
+
expect(remote.connect).toHaveBeenCalledTimes(3);
|
|
111
|
+
|
|
112
|
+
sup.dispose();
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('does not reconnect while the SDK is still retrying', async () => {
|
|
116
|
+
const { remote, emit, state } = makeRemote();
|
|
117
|
+
const sup = makeSupervisor(remote);
|
|
118
|
+
sup.start();
|
|
119
|
+
|
|
120
|
+
state.status = 'reconnecting';
|
|
121
|
+
emit('reconnecting');
|
|
122
|
+
expect(sup.connection).toBe('reconnecting');
|
|
123
|
+
|
|
124
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
125
|
+
// The SDK owns the socket during its own retry loop; racing it would open
|
|
126
|
+
// a second connection.
|
|
127
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
128
|
+
|
|
129
|
+
sup.dispose();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('forces a teardown when a heartbeat never answers (half-open socket)', async () => {
|
|
133
|
+
const { remote, emit } = makeRemote();
|
|
134
|
+
// The defining symptom: the RPC never settles and no `close` ever fires.
|
|
135
|
+
remote.query.mockImplementation(() => new Promise(() => {}));
|
|
136
|
+
|
|
137
|
+
const sup = makeSupervisor(remote);
|
|
138
|
+
emit('connected');
|
|
139
|
+
sup.start();
|
|
140
|
+
|
|
141
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatIntervalMs);
|
|
142
|
+
expect(remote.query).toHaveBeenCalledWith('RETURN true');
|
|
143
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
144
|
+
|
|
145
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatTimeoutMs);
|
|
146
|
+
expect(remote.forceClose).toHaveBeenCalledTimes(1);
|
|
147
|
+
|
|
148
|
+
// The forced close published `disconnected`, so the revive loop takes over.
|
|
149
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
150
|
+
expect(remote.connect).toHaveBeenCalled();
|
|
151
|
+
|
|
152
|
+
sup.dispose();
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it('keeps heartbeating while the connection is healthy', async () => {
|
|
156
|
+
const { remote, emit } = makeRemote();
|
|
157
|
+
const sup = makeSupervisor(remote);
|
|
158
|
+
emit('connected');
|
|
159
|
+
sup.start();
|
|
160
|
+
|
|
161
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatIntervalMs * 3 + 10);
|
|
162
|
+
expect(remote.query.mock.calls.length).toBeGreaterThanOrEqual(3);
|
|
163
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
164
|
+
expect(sup.connection).toBe('connected');
|
|
165
|
+
|
|
166
|
+
sup.dispose();
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
it('parks reconnects while the browser reports offline, and resumes on online', async () => {
|
|
170
|
+
const listeners = new Map<string, Array<() => void>>();
|
|
171
|
+
vi.stubGlobal('window', {
|
|
172
|
+
addEventListener: (e: string, cb: () => void) => {
|
|
173
|
+
listeners.set(e, [...(listeners.get(e) ?? []), cb]);
|
|
174
|
+
},
|
|
175
|
+
removeEventListener: () => {},
|
|
176
|
+
});
|
|
177
|
+
const { remote, emit, state } = makeRemote();
|
|
178
|
+
const sup = makeSupervisor(remote);
|
|
179
|
+
sup.start();
|
|
180
|
+
|
|
181
|
+
state.status = 'disconnected';
|
|
182
|
+
emit('disconnected');
|
|
183
|
+
listeners.get('offline')?.forEach((cb) => cb());
|
|
184
|
+
|
|
185
|
+
// Retrying a socket against a down interface only burns backoff.
|
|
186
|
+
await vi.advanceTimersByTimeAsync(30_000);
|
|
187
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
188
|
+
|
|
189
|
+
// `online` is the strongest available hint a reconnect will now work, so it
|
|
190
|
+
// probes immediately rather than waiting out a backoff.
|
|
191
|
+
listeners.get('online')?.forEach((cb) => cb());
|
|
192
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
193
|
+
expect(remote.connect).toHaveBeenCalledTimes(1);
|
|
194
|
+
|
|
195
|
+
sup.dispose();
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('probes immediately when a hidden tab becomes visible', async () => {
|
|
199
|
+
const listeners = new Map<string, Array<() => void>>();
|
|
200
|
+
vi.stubGlobal('document', {
|
|
201
|
+
visibilityState: 'visible',
|
|
202
|
+
addEventListener: (e: string, cb: () => void) => {
|
|
203
|
+
listeners.set(e, [...(listeners.get(e) ?? []), cb]);
|
|
204
|
+
},
|
|
205
|
+
removeEventListener: () => {},
|
|
206
|
+
});
|
|
207
|
+
const { remote, emit } = makeRemote();
|
|
208
|
+
const sup = makeSupervisor(remote);
|
|
209
|
+
emit('connected');
|
|
210
|
+
sup.start();
|
|
211
|
+
|
|
212
|
+
listeners.get('visibilitychange')?.forEach((cb) => cb());
|
|
213
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
214
|
+
// Connected-looking socket: probe it rather than reconnect. A sleep/wake
|
|
215
|
+
// cycle is exactly how you get a socket that looks fine and isn't.
|
|
216
|
+
expect(remote.query).toHaveBeenCalledWith('RETURN true');
|
|
217
|
+
|
|
218
|
+
sup.dispose();
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('dispose stops every timer and unsubscribes', async () => {
|
|
222
|
+
const { remote, emit, state } = makeRemote();
|
|
223
|
+
const sup = makeSupervisor(remote);
|
|
224
|
+
emit('connected');
|
|
225
|
+
sup.start();
|
|
226
|
+
|
|
227
|
+
sup.dispose();
|
|
228
|
+
state.status = 'disconnected';
|
|
229
|
+
emit('disconnected');
|
|
230
|
+
|
|
231
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
232
|
+
expect(remote.connect).not.toHaveBeenCalled();
|
|
233
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
234
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
it('reports state changes to subscribers', () => {
|
|
238
|
+
const { remote, emit, state } = makeRemote();
|
|
239
|
+
const sup = makeSupervisor(remote);
|
|
240
|
+
sup.start();
|
|
241
|
+
|
|
242
|
+
const seen: string[] = [];
|
|
243
|
+
sup.subscribe((s) => seen.push(s));
|
|
244
|
+
// Fires immediately with the current value.
|
|
245
|
+
expect(seen).toEqual(['connected']);
|
|
246
|
+
|
|
247
|
+
state.status = 'reconnecting';
|
|
248
|
+
emit('reconnecting');
|
|
249
|
+
state.status = 'connected';
|
|
250
|
+
emit('connected');
|
|
251
|
+
expect(seen).toEqual(['connected', 'reconnecting', 'connected']);
|
|
252
|
+
|
|
253
|
+
sup.dispose();
|
|
254
|
+
});
|
|
255
|
+
});
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import type { ConnectionState, ReconnectConfig } from '../../types';
|
|
2
|
+
import { withTimeout } from '../../utils/index';
|
|
3
|
+
import type { Logger } from '../logger/index';
|
|
4
|
+
import type { RemoteDatabaseService } from './remote';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Keeps the remote WebSocket alive for the whole life of the page.
|
|
8
|
+
*
|
|
9
|
+
* The SurrealDB SDK reconnects on its own after a socket `close`, but that
|
|
10
|
+
* covers only one of three ways the connection dies:
|
|
11
|
+
*
|
|
12
|
+
* 1. **Socket closes, SDK recovers.** Handled entirely by the SDK. This
|
|
13
|
+
* supervisor only observes it (to report `reconnecting` upward).
|
|
14
|
+
* 2. **Socket closes, SDK gives up.** With `attempts: -1` this shouldn't happen
|
|
15
|
+
* from exhaustion — but the SDK also terminates the engine permanently when
|
|
16
|
+
* its post-reconnect handshake throws (it re-runs `version()`, `use()`,
|
|
17
|
+
* `authenticate()` on every reconnect and closes the engine on any error).
|
|
18
|
+
* One transient hiccup there would otherwise kill the page's connection for
|
|
19
|
+
* good. The revive loop re-opens from scratch.
|
|
20
|
+
* 3. **Socket never closes at all.** A half-open connection: the peer is gone
|
|
21
|
+
* (NAT timeout, wifi switch, laptop sleep) but no FIN ever arrives, so
|
|
22
|
+
* `readyState` stays OPEN and the SDK's own 30s ping — fire-and-forget, no
|
|
23
|
+
* response deadline — never notices. Nothing ever fires a `close` event, so
|
|
24
|
+
* nothing ever triggers a reconnect. The heartbeat detects this and forces
|
|
25
|
+
* the teardown that case 2's loop then repairs.
|
|
26
|
+
*
|
|
27
|
+
* Plus wake triggers: coming back `online` or un-hiding the tab probes
|
|
28
|
+
* immediately rather than waiting out a backoff that was scheduled while the
|
|
29
|
+
* network was known-down.
|
|
30
|
+
*/
|
|
31
|
+
export class ConnectionSupervisor {
|
|
32
|
+
private readonly logger: Logger;
|
|
33
|
+
private readonly config: Required<ReconnectConfig>;
|
|
34
|
+
|
|
35
|
+
private state: ConnectionState = 'disconnected';
|
|
36
|
+
private subscribers = new Set<(state: ConnectionState) => void>();
|
|
37
|
+
|
|
38
|
+
private started = false;
|
|
39
|
+
private disposed = false;
|
|
40
|
+
|
|
41
|
+
private heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
|
|
42
|
+
private heartbeatInFlight = false;
|
|
43
|
+
|
|
44
|
+
private reviveTimer: ReturnType<typeof setTimeout> | null = null;
|
|
45
|
+
private reviveAttempts = 0;
|
|
46
|
+
private reviving = false;
|
|
47
|
+
/**
|
|
48
|
+
* Set while the browser reports itself offline. Retrying a socket against a
|
|
49
|
+
* down interface only burns backoff, so the loop parks until `online` fires.
|
|
50
|
+
*/
|
|
51
|
+
private suspended = false;
|
|
52
|
+
|
|
53
|
+
private teardown: Array<() => void> = [];
|
|
54
|
+
|
|
55
|
+
private static readonly REVIVE_BASE_MS = 1_000;
|
|
56
|
+
|
|
57
|
+
constructor(
|
|
58
|
+
private readonly remote: RemoteDatabaseService,
|
|
59
|
+
logger: Logger,
|
|
60
|
+
config?: Required<ReconnectConfig>
|
|
61
|
+
) {
|
|
62
|
+
this.logger = logger.child({ service: 'ConnectionSupervisor' });
|
|
63
|
+
this.config = config ?? remote.getReconnectConfig();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Latest observed transport state. */
|
|
67
|
+
get connection(): ConnectionState {
|
|
68
|
+
return this.state;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Observe transport state. Fires immediately with the current value and again
|
|
73
|
+
* on every change. Returns an unsubscribe.
|
|
74
|
+
*/
|
|
75
|
+
subscribe(cb: (state: ConnectionState) => void): () => void {
|
|
76
|
+
cb(this.state);
|
|
77
|
+
this.subscribers.add(cb);
|
|
78
|
+
return () => {
|
|
79
|
+
this.subscribers.delete(cb);
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Begin supervising. Call once, after the initial {@link
|
|
85
|
+
* RemoteDatabaseService.connect}. Idempotent.
|
|
86
|
+
*/
|
|
87
|
+
start(): void {
|
|
88
|
+
if (this.started || this.disposed) return;
|
|
89
|
+
this.started = true;
|
|
90
|
+
|
|
91
|
+
this.setState(this.remote.getStatus());
|
|
92
|
+
|
|
93
|
+
this.teardown.push(
|
|
94
|
+
this.remote.subscribeConnection('connecting', () => this.setState('connecting')),
|
|
95
|
+
this.remote.subscribeConnection('reconnecting', () => {
|
|
96
|
+
this.setState('reconnecting');
|
|
97
|
+
// The SDK owns the retry from here; ours would fight it for the socket.
|
|
98
|
+
this.stopHeartbeat();
|
|
99
|
+
}),
|
|
100
|
+
this.remote.subscribeConnection('connected', () => {
|
|
101
|
+
this.reviveAttempts = 0;
|
|
102
|
+
this.clearReviveTimer();
|
|
103
|
+
this.setState('connected');
|
|
104
|
+
this.startHeartbeat();
|
|
105
|
+
}),
|
|
106
|
+
this.remote.subscribeConnection('disconnected', () => {
|
|
107
|
+
this.setState('disconnected');
|
|
108
|
+
this.stopHeartbeat();
|
|
109
|
+
// The SDK has stopped trying (exhausted, terminated, or handshake
|
|
110
|
+
// failure). From here on, reconnecting is entirely our job.
|
|
111
|
+
this.scheduleRevive();
|
|
112
|
+
}),
|
|
113
|
+
this.remote.subscribeConnection('error', (err) => {
|
|
114
|
+
this.logger.debug(
|
|
115
|
+
{ err, Category: 'sp00ky-client::ConnectionSupervisor::error' },
|
|
116
|
+
'Transport error'
|
|
117
|
+
);
|
|
118
|
+
})
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
this.installWakeTriggers();
|
|
122
|
+
|
|
123
|
+
if (this.state === 'connected') this.startHeartbeat();
|
|
124
|
+
else this.scheduleRevive();
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Stop all timers and listeners. Safe to call more than once. */
|
|
128
|
+
dispose(): void {
|
|
129
|
+
this.disposed = true;
|
|
130
|
+
this.started = false;
|
|
131
|
+
this.stopHeartbeat();
|
|
132
|
+
this.clearReviveTimer();
|
|
133
|
+
for (const off of this.teardown) {
|
|
134
|
+
try {
|
|
135
|
+
off();
|
|
136
|
+
} catch {
|
|
137
|
+
/* ignore */
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
this.teardown = [];
|
|
141
|
+
this.subscribers.clear();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private setState(next: ConnectionState): void {
|
|
145
|
+
if (this.state === next) return;
|
|
146
|
+
this.state = next;
|
|
147
|
+
this.logger.info(
|
|
148
|
+
{ state: next, Category: 'sp00ky-client::ConnectionSupervisor::state' },
|
|
149
|
+
'Connection state changed'
|
|
150
|
+
);
|
|
151
|
+
for (const cb of this.subscribers) {
|
|
152
|
+
try {
|
|
153
|
+
cb(next);
|
|
154
|
+
} catch (err) {
|
|
155
|
+
this.logger.debug(
|
|
156
|
+
{ err, Category: 'sp00ky-client::ConnectionSupervisor::state' },
|
|
157
|
+
'Connection subscriber threw'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ---- Revive loop -------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
private clearReviveTimer(): void {
|
|
166
|
+
if (this.reviveTimer !== null) {
|
|
167
|
+
clearTimeout(this.reviveTimer);
|
|
168
|
+
this.reviveTimer = null;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* Queue the next `connect()` attempt on exponential backoff, capped at
|
|
174
|
+
* `superviseRetryDelayMaxMs`. Never gives up — the page is expected to
|
|
175
|
+
* outlive any outage.
|
|
176
|
+
*/
|
|
177
|
+
private scheduleRevive(): void {
|
|
178
|
+
if (this.disposed || this.suspended) return;
|
|
179
|
+
if (this.reviveTimer !== null || this.reviving) return;
|
|
180
|
+
const delay = Math.min(
|
|
181
|
+
this.config.superviseRetryDelayMaxMs,
|
|
182
|
+
ConnectionSupervisor.REVIVE_BASE_MS * 2 ** this.reviveAttempts
|
|
183
|
+
);
|
|
184
|
+
this.reviveTimer = setTimeout(() => {
|
|
185
|
+
this.reviveTimer = null;
|
|
186
|
+
void this.revive();
|
|
187
|
+
}, delay);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private async revive(): Promise<void> {
|
|
191
|
+
if (this.disposed || this.suspended || this.reviving) return;
|
|
192
|
+
// The SDK may have recovered on its own between scheduling and firing.
|
|
193
|
+
if (this.remote.getStatus() === 'connected') {
|
|
194
|
+
this.reviveAttempts = 0;
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
this.reviving = true;
|
|
198
|
+
this.reviveAttempts++;
|
|
199
|
+
this.setState('reconnecting');
|
|
200
|
+
this.logger.info(
|
|
201
|
+
{
|
|
202
|
+
attempt: this.reviveAttempts,
|
|
203
|
+
Category: 'sp00ky-client::ConnectionSupervisor::revive',
|
|
204
|
+
},
|
|
205
|
+
'Re-opening the remote connection'
|
|
206
|
+
);
|
|
207
|
+
try {
|
|
208
|
+
await this.remote.connect();
|
|
209
|
+
// Don't reset `reviveAttempts` or start the heartbeat here — the
|
|
210
|
+
// `connected` handler does both, and it's the only signal that the
|
|
211
|
+
// handshake (version/use/authenticate) actually completed.
|
|
212
|
+
} catch (err) {
|
|
213
|
+
this.logger.warn(
|
|
214
|
+
{
|
|
215
|
+
err,
|
|
216
|
+
attempt: this.reviveAttempts,
|
|
217
|
+
Category: 'sp00ky-client::ConnectionSupervisor::revive',
|
|
218
|
+
},
|
|
219
|
+
'Reconnect attempt failed; will retry'
|
|
220
|
+
);
|
|
221
|
+
} finally {
|
|
222
|
+
this.reviving = false;
|
|
223
|
+
}
|
|
224
|
+
if (this.remote.getStatus() !== 'connected') this.scheduleRevive();
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ---- Heartbeat watchdog ------------------------------------------------
|
|
228
|
+
|
|
229
|
+
private stopHeartbeat(): void {
|
|
230
|
+
if (this.heartbeatTimer !== null) {
|
|
231
|
+
clearTimeout(this.heartbeatTimer);
|
|
232
|
+
this.heartbeatTimer = null;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
private startHeartbeat(): void {
|
|
237
|
+
this.stopHeartbeat();
|
|
238
|
+
if (this.disposed || this.suspended) return;
|
|
239
|
+
if (!(this.config.heartbeatIntervalMs > 0)) return;
|
|
240
|
+
this.heartbeatTimer = setTimeout(
|
|
241
|
+
() => void this.beat(),
|
|
242
|
+
this.config.heartbeatIntervalMs
|
|
243
|
+
);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Probe the server end-to-end. Deliberately goes through
|
|
248
|
+
* `remote.query` — the same serialized queue every other remote call uses —
|
|
249
|
+
* so a queue wedged behind a stuck RPC also fails the heartbeat instead of
|
|
250
|
+
* being invisible to it.
|
|
251
|
+
*/
|
|
252
|
+
private async beat(): Promise<void> {
|
|
253
|
+
this.heartbeatTimer = null;
|
|
254
|
+
if (this.disposed || this.suspended) return;
|
|
255
|
+
if (this.remote.getStatus() !== 'connected') return;
|
|
256
|
+
if (this.heartbeatInFlight) {
|
|
257
|
+
this.startHeartbeat();
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
this.heartbeatInFlight = true;
|
|
261
|
+
try {
|
|
262
|
+
await withTimeout(
|
|
263
|
+
this.remote.query('RETURN true'),
|
|
264
|
+
this.config.heartbeatTimeoutMs,
|
|
265
|
+
`Heartbeat timed out after ${this.config.heartbeatTimeoutMs}ms`
|
|
266
|
+
);
|
|
267
|
+
this.startHeartbeat();
|
|
268
|
+
} catch (err) {
|
|
269
|
+
this.logger.warn(
|
|
270
|
+
{ err, Category: 'sp00ky-client::ConnectionSupervisor::heartbeat' },
|
|
271
|
+
'Heartbeat failed; tearing the socket down to force a reconnect'
|
|
272
|
+
);
|
|
273
|
+
// Force the `close` the transport never delivered. The resulting
|
|
274
|
+
// `disconnected` event drives the revive loop.
|
|
275
|
+
await this.remote.forceClose();
|
|
276
|
+
if (this.remote.getStatus() !== 'connected') this.scheduleRevive();
|
|
277
|
+
} finally {
|
|
278
|
+
this.heartbeatInFlight = false;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ---- Wake triggers -----------------------------------------------------
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* A restored network or an un-hidden tab is the strongest available hint that
|
|
286
|
+
* a reconnect will now succeed, so probe immediately instead of waiting out a
|
|
287
|
+
* backoff scheduled under worse conditions.
|
|
288
|
+
*/
|
|
289
|
+
private installWakeTriggers(): void {
|
|
290
|
+
if (typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
|
|
291
|
+
const onOnline = () => {
|
|
292
|
+
this.suspended = false;
|
|
293
|
+
this.wake('online');
|
|
294
|
+
};
|
|
295
|
+
const onOffline = () => {
|
|
296
|
+
this.logger.info(
|
|
297
|
+
{ Category: 'sp00ky-client::ConnectionSupervisor::offline' },
|
|
298
|
+
'Browser reports offline; parking reconnects until online'
|
|
299
|
+
);
|
|
300
|
+
this.suspended = true;
|
|
301
|
+
this.stopHeartbeat();
|
|
302
|
+
this.clearReviveTimer();
|
|
303
|
+
this.setState('disconnected');
|
|
304
|
+
};
|
|
305
|
+
window.addEventListener('online', onOnline);
|
|
306
|
+
window.addEventListener('offline', onOffline);
|
|
307
|
+
this.teardown.push(
|
|
308
|
+
() => window.removeEventListener('online', onOnline),
|
|
309
|
+
() => window.removeEventListener('offline', onOffline)
|
|
310
|
+
);
|
|
311
|
+
// A tab restored from bfcache keeps its dead socket; `pageshow` is the
|
|
312
|
+
// only event that fires in that path.
|
|
313
|
+
const onPageShow = () => this.wake('pageshow');
|
|
314
|
+
window.addEventListener('pageshow', onPageShow);
|
|
315
|
+
this.teardown.push(() => window.removeEventListener('pageshow', onPageShow));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
if (typeof document !== 'undefined' && typeof document.addEventListener === 'function') {
|
|
319
|
+
const onVisibility = () => {
|
|
320
|
+
if (document.visibilityState !== 'visible') return;
|
|
321
|
+
this.wake('visibilitychange');
|
|
322
|
+
};
|
|
323
|
+
document.addEventListener('visibilitychange', onVisibility);
|
|
324
|
+
this.teardown.push(() => document.removeEventListener('visibilitychange', onVisibility));
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Reset the backoff and act on whichever problem is present: reconnect if the
|
|
330
|
+
* socket is gone, otherwise probe it (it may be half-open — which is exactly
|
|
331
|
+
* what a sleep/wake cycle produces).
|
|
332
|
+
*/
|
|
333
|
+
private wake(reason: string): void {
|
|
334
|
+
if (this.disposed || this.suspended) return;
|
|
335
|
+
this.logger.debug(
|
|
336
|
+
{ reason, state: this.state, Category: 'sp00ky-client::ConnectionSupervisor::wake' },
|
|
337
|
+
'Wake trigger; probing the connection'
|
|
338
|
+
);
|
|
339
|
+
this.reviveAttempts = 0;
|
|
340
|
+
if (this.remote.getStatus() === 'connected') {
|
|
341
|
+
this.stopHeartbeat();
|
|
342
|
+
void this.beat();
|
|
343
|
+
return;
|
|
344
|
+
}
|
|
345
|
+
// 'reconnecting' means the SDK's own loop owns the socket; don't race it.
|
|
346
|
+
if (this.remote.getStatus() === 'disconnected') {
|
|
347
|
+
this.clearReviveTimer();
|
|
348
|
+
void this.revive();
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { AbstractDatabaseService } from './database';
|
|
3
|
+
import { DatabaseEventTypes, createDatabaseEventSystem } from './events/index';
|
|
4
|
+
import { classifySyncError } from '../../utils/index';
|
|
5
|
+
|
|
6
|
+
// Every remote query is serialized through one promise chain (to keep the WASM
|
|
7
|
+
// engine's transactions safe), so a call that never settles doesn't just fail —
|
|
8
|
+
// it blocks every later query behind it, including the sync poll's own health
|
|
9
|
+
// probe. That's how a half-open socket wedges the client while health still
|
|
10
|
+
// reports `healthy`: nothing ever throws, so nothing ever degrades.
|
|
11
|
+
|
|
12
|
+
const silentLogger: any = (() => {
|
|
13
|
+
const l: any = {
|
|
14
|
+
child: () => l,
|
|
15
|
+
debug: () => {},
|
|
16
|
+
info: () => {},
|
|
17
|
+
warn: () => {},
|
|
18
|
+
error: () => {},
|
|
19
|
+
trace: () => {},
|
|
20
|
+
};
|
|
21
|
+
return l;
|
|
22
|
+
})();
|
|
23
|
+
|
|
24
|
+
class TestService extends AbstractDatabaseService {
|
|
25
|
+
protected eventType = DatabaseEventTypes.RemoteQuery;
|
|
26
|
+
constructor(client: any, timeoutMs: number) {
|
|
27
|
+
super(client, silentLogger, createDatabaseEventSystem());
|
|
28
|
+
this.queryTimeoutMs = timeoutMs;
|
|
29
|
+
}
|
|
30
|
+
async connect(): Promise<void> {}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe('AbstractDatabaseService query timeout', () => {
|
|
34
|
+
beforeEach(() => vi.useFakeTimers());
|
|
35
|
+
afterEach(() => vi.useRealTimers());
|
|
36
|
+
|
|
37
|
+
it('rejects a never-settling query and classifies it as a network failure', async () => {
|
|
38
|
+
const client = { query: vi.fn().mockImplementation(() => new Promise(() => {})) };
|
|
39
|
+
const svc = new TestService(client, 1_000);
|
|
40
|
+
|
|
41
|
+
const pending = svc.query('SELECT * FROM thread');
|
|
42
|
+
const assertion = expect(pending).rejects.toThrow(/timed out/);
|
|
43
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
44
|
+
await assertion;
|
|
45
|
+
|
|
46
|
+
// "timed out" in the message is load-bearing: it routes the failure to
|
|
47
|
+
// `network`, which makes the sync queues retry instead of rolling the
|
|
48
|
+
// mutation back as a permanent application error.
|
|
49
|
+
const err = await pending.catch((e) => e);
|
|
50
|
+
expect(classifySyncError(err)).toBe('network');
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('unblocks the serialized queue so later queries still run', async () => {
|
|
54
|
+
let calls = 0;
|
|
55
|
+
const client = {
|
|
56
|
+
query: vi.fn().mockImplementation(() => {
|
|
57
|
+
calls++;
|
|
58
|
+
// First call hangs forever; the second would never be reached at all if
|
|
59
|
+
// the timeout didn't settle the chain link.
|
|
60
|
+
if (calls === 1) return new Promise(() => {});
|
|
61
|
+
return Promise.resolve([['ok']]);
|
|
62
|
+
}),
|
|
63
|
+
};
|
|
64
|
+
const svc = new TestService(client, 1_000);
|
|
65
|
+
|
|
66
|
+
const stuck = svc.query('SELECT 1');
|
|
67
|
+
const next = svc.query('SELECT 2');
|
|
68
|
+
const stuckAssertion = expect(stuck).rejects.toThrow(/timed out/);
|
|
69
|
+
|
|
70
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
71
|
+
await stuckAssertion;
|
|
72
|
+
await expect(next).resolves.toEqual([['ok']]);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it('leaves queries unbounded when the timeout is disabled', async () => {
|
|
76
|
+
const client = { query: vi.fn().mockResolvedValue([['ok']]) };
|
|
77
|
+
const svc = new TestService(client, 0);
|
|
78
|
+
|
|
79
|
+
await expect(svc.query('SELECT 1')).resolves.toEqual([['ok']]);
|
|
80
|
+
// No timer was armed, so a slow-but-alive query is never cut off.
|
|
81
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
82
|
+
});
|
|
83
|
+
});
|