@spooky-sync/core 0.0.1-canary.161 → 0.0.1-canary.162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +294 -4
- package/dist/index.js +1033 -293
- package/dist/otel/index.d.ts +1 -1
- package/dist/types.d.ts +98 -1
- package/package.json +3 -3
- package/src/modules/crdt/crdt-hydration.test.ts +4 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +109 -7
- package/src/modules/data/data.membership.test.ts +297 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +291 -39
- package/src/modules/data/window-query.ts +53 -13
- package/src/modules/sync/sync.live-removal.test.ts +134 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.ts +143 -43
- package/src/services/database/connection-supervisor.test.ts +255 -0
- package/src/services/database/connection-supervisor.ts +351 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +21 -1
- package/src/services/database/index.ts +1 -0
- package/src/services/database/local-migrator.ts +1 -0
- package/src/services/database/remote.ts +99 -3
- package/src/services/database/sqlite-cache-engine.ts +1 -0
- package/src/sp00ky.ts +32 -1
- package/src/types.ts +99 -0
- package/src/utils/index.ts +38 -0
package/dist/otel/index.d.ts
CHANGED
package/dist/types.d.ts
CHANGED
|
@@ -457,6 +457,22 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
457
457
|
* multi-hop path (escape hatch while the worker-side path beds in).
|
|
458
458
|
*/
|
|
459
459
|
workerSelect?: boolean;
|
|
460
|
+
/**
|
|
461
|
+
* WebSocket reconnect + liveness tuning. All fields optional; the defaults
|
|
462
|
+
* keep the connection alive indefinitely without configuration. See
|
|
463
|
+
* {@link ReconnectConfig}.
|
|
464
|
+
*/
|
|
465
|
+
reconnect?: ReconnectConfig;
|
|
466
|
+
/**
|
|
467
|
+
* Deadline (ms) for every remote RPC. Remote queries are serialized through
|
|
468
|
+
* a single promise chain, so one call that never settles (half-open socket:
|
|
469
|
+
* the WebSocket looks open, the peer is gone, no `close` event fires) would
|
|
470
|
+
* otherwise wedge ALL later remote traffic behind it — including the sync
|
|
471
|
+
* poll's own health probe, leaving health pinned at `healthy` with no
|
|
472
|
+
* banner and no self-heal. The deadline turns that into an ordinary network
|
|
473
|
+
* failure the queue retries. `0` disables. Defaults to `60_000`.
|
|
474
|
+
*/
|
|
475
|
+
queryTimeoutMs?: number;
|
|
460
476
|
};
|
|
461
477
|
/** The schema definition. */
|
|
462
478
|
schema: S;
|
|
@@ -587,6 +603,15 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
587
603
|
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
588
604
|
*/
|
|
589
605
|
syncHealth?: SyncHealthConfig | false;
|
|
606
|
+
/**
|
|
607
|
+
* Deadline (ms) for a single outgoing mutation push. Tighter than
|
|
608
|
+
* {@link Sp00kyConfig.database.queryTimeoutMs} because the up-queue drains
|
|
609
|
+
* one mutation at a time behind an `isSyncingUp` flag: a push that never
|
|
610
|
+
* settles stops every later mutation for the session, with no retry and no
|
|
611
|
+
* error. On expiry the push is treated as a network failure and re-queued.
|
|
612
|
+
* `0` disables. Defaults to `30_000`.
|
|
613
|
+
*/
|
|
614
|
+
pushTimeoutMs?: number;
|
|
590
615
|
}
|
|
591
616
|
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
592
617
|
interface SyncHealthConfig {
|
|
@@ -598,6 +623,53 @@ interface SyncHealthConfig {
|
|
|
598
623
|
*/
|
|
599
624
|
degradeAfterConsecutiveFailures?: number;
|
|
600
625
|
}
|
|
626
|
+
/**
|
|
627
|
+
* Tunables for WebSocket reconnect and liveness detection. See
|
|
628
|
+
* {@link Sp00kyConfig.database.reconnect}.
|
|
629
|
+
*
|
|
630
|
+
* Two independent mechanisms cooperate here. The SurrealDB SDK reconnects on
|
|
631
|
+
* its own after a socket `close` (`attempts` / `retryDelayMax`), and a
|
|
632
|
+
* supervisor above it re-opens the connection from scratch whenever the SDK
|
|
633
|
+
* gives up or its post-reconnect handshake fails — the SDK terminates the
|
|
634
|
+
* engine permanently in that case, so a supervisor is required, not optional.
|
|
635
|
+
* The heartbeat covers the third case: a socket that never closes at all.
|
|
636
|
+
*/
|
|
637
|
+
interface ReconnectConfig {
|
|
638
|
+
/**
|
|
639
|
+
* SDK reconnect attempts after a socket close. `-1` retries forever.
|
|
640
|
+
* Defaults to `-1` (the SDK's own default is `5`, which caps recovery at a
|
|
641
|
+
* ~62s outage and then gives up for the life of the page).
|
|
642
|
+
*/
|
|
643
|
+
attempts?: number;
|
|
644
|
+
/** Cap on the SDK's exponential backoff delay. Defaults to `15_000`. */
|
|
645
|
+
retryDelayMax?: number;
|
|
646
|
+
/**
|
|
647
|
+
* Cadence of the application-level liveness probe (`RETURN true`) that
|
|
648
|
+
* detects a half-open socket the transport never reports as closed.
|
|
649
|
+
* `0` disables the heartbeat. Defaults to `20_000`.
|
|
650
|
+
*/
|
|
651
|
+
heartbeatIntervalMs?: number;
|
|
652
|
+
/**
|
|
653
|
+
* Deadline for a heartbeat response. Exceeding it means the socket is dead
|
|
654
|
+
* regardless of what its `readyState` claims, so the connection is torn down
|
|
655
|
+
* and rebuilt. Defaults to `10_000`.
|
|
656
|
+
*/
|
|
657
|
+
heartbeatTimeoutMs?: number;
|
|
658
|
+
/**
|
|
659
|
+
* Cap on the supervisor's own backoff between `connect()` retries once the
|
|
660
|
+
* SDK has given up. Defaults to `15_000`.
|
|
661
|
+
*/
|
|
662
|
+
superviseRetryDelayMaxMs?: number;
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* Transport-level connection state, independent of {@link SyncHealthStatus}.
|
|
666
|
+
*
|
|
667
|
+
* These answer different questions: `connection` is about the socket,
|
|
668
|
+
* `status` is about whether sync rounds are succeeding. A `connected` socket
|
|
669
|
+
* can still be `degraded` (server erroring), and a `reconnecting` socket is
|
|
670
|
+
* usually still `healthy` for the first few seconds.
|
|
671
|
+
*/
|
|
672
|
+
type ConnectionState = 'connecting' | 'connected' | 'reconnecting' | 'disconnected';
|
|
601
673
|
type SyncHealthStatus = 'healthy' | 'degraded';
|
|
602
674
|
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
603
675
|
interface SyncHealth {
|
|
@@ -616,6 +688,13 @@ interface SyncHealth {
|
|
|
616
688
|
* a working session. Never resets back to `false` once set.
|
|
617
689
|
*/
|
|
618
690
|
everConnected: boolean;
|
|
691
|
+
/**
|
|
692
|
+
* Live transport state of the remote WebSocket. Distinct from `status`: this
|
|
693
|
+
* one flips the instant the socket drops, whereas `status` only degrades
|
|
694
|
+
* after a sustained run of failed sync rounds. Use it to show "reconnecting…"
|
|
695
|
+
* immediately without waiting for the degrade threshold.
|
|
696
|
+
*/
|
|
697
|
+
connection: ConnectionState;
|
|
619
698
|
}
|
|
620
699
|
type StorageHealthStatus = 'unknown' | 'persistent' | 'memory';
|
|
621
700
|
/**
|
|
@@ -699,6 +778,24 @@ interface QueryConfig {
|
|
|
699
778
|
* `rowCount` / `localArray`.
|
|
700
779
|
*/
|
|
701
780
|
subqueryRemoteArray?: RecordVersionArray;
|
|
781
|
+
/**
|
|
782
|
+
* Whether authoritative membership (`remoteArray`) has ever been established
|
|
783
|
+
* for this query — either fetched from `_00_list_ref` this session, or read
|
|
784
|
+
* back from the durable `_00_window` row on a cold start.
|
|
785
|
+
*
|
|
786
|
+
* Tri-state matters: "known and empty" must render an empty list, while
|
|
787
|
+
* "never established" has to fall back to a predicate scan of the local store
|
|
788
|
+
* so a query first run on this device still paints offline. A
|
|
789
|
+
* `remoteArray.length === 0` check cannot tell those apart.
|
|
790
|
+
*/
|
|
791
|
+
membershipKnown?: boolean;
|
|
792
|
+
/**
|
|
793
|
+
* Key of this query's durable `_00_window` membership row: a hash of
|
|
794
|
+
* `{surql, params}` WITHOUT the `session::id()` salt that `id` carries, so it
|
|
795
|
+
* survives a reload (which mints a new session id) and a bucket switch.
|
|
796
|
+
* In-memory only.
|
|
797
|
+
*/
|
|
798
|
+
membershipKey?: string;
|
|
702
799
|
/** Time-To-Live for this query. */
|
|
703
800
|
ttl: QueryTimeToLive;
|
|
704
801
|
/** Timestamp when the query was last accessed/active. */
|
|
@@ -871,4 +968,4 @@ interface DebounceOptions {
|
|
|
871
968
|
delay?: number;
|
|
872
969
|
}
|
|
873
970
|
//#endregion
|
|
874
|
-
export {
|
|
971
|
+
export { Sp00kyQueryResultPromise as A, LocalStore as B, ReconnectConfig as C, RunOptions as D, RegistrationTimings as E, SyncHealthConfig as F, SyncEventSystem as G, DatabaseEventSystem as H, SyncHealthStatus as I, EventDefinition as K, TimingPhase as L, StorageHealthStatus as M, StoreType as N, Sp00kyConfig as O, SyncHealth as P, UpdateOptions as R, QueryUpdateCallback as S, RecordVersionDiff as T, DatabaseEventTypes as U, SealedQuery as V, Logger$1 as W, QueryState as _, MATERIALIZATION_SAMPLE_WINDOW as a, QueryTimeToLive as b, MutationEventType as c, PinoTransmit as d, PreloadOptions as f, QueryHash as g, QueryConfigRecord as h, Level$1 as i, StorageHealth as j, Sp00kyQueryResult as k, PersistenceClient as l, QueryConfig as m, DebounceOptions as n, MutationCallback as o, PreloadRefresh as p, EventSystem as q, EventSubscriptionOptions as r, MutationEvent as s, ConnectionState as t, PhaseStat as u, QueryStatus as v, RecordVersionArray as w, QueryTimings as x, QueryStatusCallback as y, UpEvent as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.162",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.162",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.162",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"fast-json-patch": "^3.1.1",
|
|
@@ -76,6 +76,10 @@ function makeRemote(handlers: {
|
|
|
76
76
|
subscribe: () => () => {},
|
|
77
77
|
}),
|
|
78
78
|
}) as any,
|
|
79
|
+
// CrdtManager watches transport events so it can re-issue table LIVEs after
|
|
80
|
+
// a reconnect. These tests never drop the socket, so a no-op is enough.
|
|
81
|
+
getStatus: () => 'connected' as const,
|
|
82
|
+
subscribeConnection: vi.fn().mockReturnValue(() => {}),
|
|
79
83
|
} satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
|
|
80
84
|
return remote;
|
|
81
85
|
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
3
|
+
import { CrdtManager } from './index';
|
|
4
|
+
import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
|
|
5
|
+
import type { Logger } from '../../services/logger/index';
|
|
6
|
+
|
|
7
|
+
// A LIVE subscription lives and dies with its WebSocket session, and
|
|
8
|
+
// `ensureTableSubscription` is memoized on `liveByTable`. Without a reconnect
|
|
9
|
+
// hook, the first dropped socket kills CRDT realtime for good: the map still
|
|
10
|
+
// holds a uuid for a subscription the server has forgotten, so every later
|
|
11
|
+
// `open()` short-circuits and no LIVE is ever re-issued.
|
|
12
|
+
|
|
13
|
+
const noop = () => {};
|
|
14
|
+
|
|
15
|
+
/** A promise plus its resolver, so a test can hold an RPC open on demand. */
|
|
16
|
+
function deferred<T>(): { promise: Promise<T>; resolve: (v: T) => void } {
|
|
17
|
+
let resolve!: (v: T) => void;
|
|
18
|
+
const promise = new Promise<T>((r) => {
|
|
19
|
+
resolve = r;
|
|
20
|
+
});
|
|
21
|
+
return { promise, resolve };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function silentLogger(): Logger {
|
|
25
|
+
const fake: any = {};
|
|
26
|
+
fake.info = noop;
|
|
27
|
+
fake.warn = noop;
|
|
28
|
+
fake.debug = noop;
|
|
29
|
+
fake.error = noop;
|
|
30
|
+
fake.trace = noop;
|
|
31
|
+
fake.child = () => fake;
|
|
32
|
+
return fake as Logger;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function singleTableSchema(): SchemaStructure {
|
|
36
|
+
return {
|
|
37
|
+
tables: [
|
|
38
|
+
{
|
|
39
|
+
name: 'thread',
|
|
40
|
+
columns: { body: { type: 'string', optional: false, crdt: 'text' } },
|
|
41
|
+
primaryKey: ['id'],
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
relationships: [],
|
|
45
|
+
backends: {},
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function makeRemote() {
|
|
50
|
+
const handlers = new Map<string, Array<(...a: any[]) => void>>();
|
|
51
|
+
const remote = {
|
|
52
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
53
|
+
if (sql.includes('LIVE SELECT')) return ['live-uuid'];
|
|
54
|
+
if (sql.includes('SELECT * FROM ONLY')) return [null];
|
|
55
|
+
return [];
|
|
56
|
+
}),
|
|
57
|
+
getClient: () =>
|
|
58
|
+
({ liveOf: async () => ({ subscribe: () => () => {} }) }) as any,
|
|
59
|
+
getStatus: () => 'connected' as const,
|
|
60
|
+
subscribeConnection: (event: string, cb: (...a: any[]) => void) => {
|
|
61
|
+
handlers.set(event, [...(handlers.get(event) ?? []), cb]);
|
|
62
|
+
return () => {
|
|
63
|
+
handlers.set(event, (handlers.get(event) ?? []).filter((h) => h !== cb));
|
|
64
|
+
};
|
|
65
|
+
},
|
|
66
|
+
} satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
|
|
67
|
+
const emit = (event: string) => {
|
|
68
|
+
for (const cb of Array.from(handlers.get(event) ?? [])) cb();
|
|
69
|
+
};
|
|
70
|
+
return { remote, emit };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function makeLocal(): LocalStore {
|
|
74
|
+
return {
|
|
75
|
+
query: vi.fn().mockResolvedValue([null]),
|
|
76
|
+
} satisfies Partial<LocalStore> as unknown as LocalStore;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const liveSelects = (remote: RemoteDatabaseService) =>
|
|
80
|
+
(remote.query as ReturnType<typeof vi.fn>).mock.calls
|
|
81
|
+
.map((c) => c[0] as string)
|
|
82
|
+
.filter((s) => s.includes('LIVE SELECT'));
|
|
83
|
+
|
|
84
|
+
describe('CrdtManager reconnect', () => {
|
|
85
|
+
it('re-issues the table LIVE after a recovered drop', async () => {
|
|
86
|
+
const { remote, emit } = makeRemote();
|
|
87
|
+
const manager = new CrdtManager(singleTableSchema(), makeLocal(), remote, silentLogger());
|
|
88
|
+
manager.setSessionId('s1');
|
|
89
|
+
|
|
90
|
+
await manager.open('thread', 'thread:abc', 'body');
|
|
91
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(1));
|
|
92
|
+
|
|
93
|
+
// The SDK's own reconnect path: `reconnecting` then `connected`, never
|
|
94
|
+
// `disconnected`.
|
|
95
|
+
emit('reconnecting');
|
|
96
|
+
emit('connected');
|
|
97
|
+
|
|
98
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(2));
|
|
99
|
+
// No KILL: that uuid belonged to a session that no longer exists.
|
|
100
|
+
const sqls = (remote.query as ReturnType<typeof vi.fn>).mock.calls.map((c) => c[0] as string);
|
|
101
|
+
expect(sqls.some((s) => s.includes('KILL'))).toBe(false);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('re-issues the table LIVE after the SDK gives up', async () => {
|
|
105
|
+
const { remote, emit } = makeRemote();
|
|
106
|
+
const manager = new CrdtManager(singleTableSchema(), makeLocal(), remote, silentLogger());
|
|
107
|
+
manager.setSessionId('s1');
|
|
108
|
+
|
|
109
|
+
await manager.open('thread', 'thread:abc', 'body');
|
|
110
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(1));
|
|
111
|
+
|
|
112
|
+
emit('disconnected');
|
|
113
|
+
emit('connected');
|
|
114
|
+
|
|
115
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(2));
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it('does not resurrect a LIVE for a table whose fields were all closed', async () => {
|
|
119
|
+
const { remote, emit } = makeRemote();
|
|
120
|
+
const manager = new CrdtManager(singleTableSchema(), makeLocal(), remote, silentLogger());
|
|
121
|
+
manager.setSessionId('s1');
|
|
122
|
+
|
|
123
|
+
await manager.open('thread', 'thread:abc', 'body');
|
|
124
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(1));
|
|
125
|
+
|
|
126
|
+
// A drop can outlive the editor that needed the feed.
|
|
127
|
+
emit('reconnecting');
|
|
128
|
+
manager.close('thread', 'thread:abc', 'body');
|
|
129
|
+
emit('connected');
|
|
130
|
+
|
|
131
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
132
|
+
expect(liveSelects(remote)).toHaveLength(1);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('discards and re-registers a LIVE that was mid-flight when the socket dropped', async () => {
|
|
136
|
+
const handlers = new Map<string, Array<() => void>>();
|
|
137
|
+
const firstLive = deferred<void>();
|
|
138
|
+
let liveCalls = 0;
|
|
139
|
+
const remote = {
|
|
140
|
+
query: vi.fn().mockImplementation(async (sql: string) => {
|
|
141
|
+
if (sql.includes('LIVE SELECT')) {
|
|
142
|
+
liveCalls++;
|
|
143
|
+
// Hold the FIRST registration open so the drop lands mid-flight.
|
|
144
|
+
if (liveCalls === 1) {
|
|
145
|
+
await firstLive.promise;
|
|
146
|
+
return ['dead-session-uuid'];
|
|
147
|
+
}
|
|
148
|
+
return ['fresh-uuid'];
|
|
149
|
+
}
|
|
150
|
+
return [null];
|
|
151
|
+
}),
|
|
152
|
+
getClient: () => ({ liveOf: async () => ({ subscribe: () => () => {} }) }) as any,
|
|
153
|
+
getStatus: () => 'connected' as const,
|
|
154
|
+
subscribeConnection: (event: string, cb: () => void) => {
|
|
155
|
+
handlers.set(event, [...(handlers.get(event) ?? []), cb]);
|
|
156
|
+
return () => {};
|
|
157
|
+
},
|
|
158
|
+
} satisfies Partial<RemoteDatabaseService> as unknown as RemoteDatabaseService;
|
|
159
|
+
const emit = (event: string) => {
|
|
160
|
+
for (const cb of Array.from(handlers.get(event) ?? [])) cb();
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
const manager = new CrdtManager(singleTableSchema(), makeLocal(), remote, silentLogger());
|
|
164
|
+
manager.setSessionId('s1');
|
|
165
|
+
await manager.open('thread', 'thread:abc', 'body');
|
|
166
|
+
await vi.waitFor(() => expect(liveCalls).toBe(1));
|
|
167
|
+
|
|
168
|
+
// Drop + recover while the first registration is still awaiting its uuid.
|
|
169
|
+
emit('reconnecting');
|
|
170
|
+
emit('connected');
|
|
171
|
+
firstLive.resolve();
|
|
172
|
+
|
|
173
|
+
// The doomed uuid must never be recorded — if it were, `liveByTable` would
|
|
174
|
+
// short-circuit every future restart and CRDT realtime would be dead for
|
|
175
|
+
// good. A fresh registration must take its place.
|
|
176
|
+
await vi.waitFor(() => expect(liveCalls).toBe(2));
|
|
177
|
+
expect((manager as any).liveByTable.get('thread')).toBe('fresh-uuid');
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it('stops reacting to transport events after dispose', async () => {
|
|
181
|
+
const { remote, emit } = makeRemote();
|
|
182
|
+
const manager = new CrdtManager(singleTableSchema(), makeLocal(), remote, silentLogger());
|
|
183
|
+
manager.setSessionId('s1');
|
|
184
|
+
|
|
185
|
+
await manager.open('thread', 'thread:abc', 'body');
|
|
186
|
+
await vi.waitFor(() => expect(liveSelects(remote)).toHaveLength(1));
|
|
187
|
+
|
|
188
|
+
manager.dispose();
|
|
189
|
+
emit('reconnecting');
|
|
190
|
+
emit('connected');
|
|
191
|
+
|
|
192
|
+
await new Promise((r) => setTimeout(r, 10));
|
|
193
|
+
expect(liveSelects(remote)).toHaveLength(1);
|
|
194
|
+
});
|
|
195
|
+
});
|
|
@@ -33,8 +33,19 @@ export class CrdtManager {
|
|
|
33
33
|
private fields = new Map<string, CrdtField>();
|
|
34
34
|
// One LIVE subscription per parent table (e.g. "thread" → uuid).
|
|
35
35
|
private liveByTable = new Map<string, Uuid>();
|
|
36
|
-
// Coalesces concurrent first-time subscribes for the same table.
|
|
37
|
-
|
|
36
|
+
// Coalesces concurrent first-time subscribes for the same table. Stamped with
|
|
37
|
+
// the connection generation it started under, so a restart after a drop can
|
|
38
|
+
// tell a reusable in-flight attempt from a doomed one.
|
|
39
|
+
private pendingLive = new Map<string, { promise: Promise<void>; generation: number }>();
|
|
40
|
+
// Tables whose LIVE died with a dropped socket, awaiting the next `connected`
|
|
41
|
+
// to be re-subscribed. Held separately because `liveByTable` is cleared on the
|
|
42
|
+
// drop (those uuids belong to a session that no longer exists).
|
|
43
|
+
private staleTables = new Set<string>();
|
|
44
|
+
// Bumped on every socket drop. A LIVE registration that started under an
|
|
45
|
+
// older generation is discarded rather than recorded, so a drop landing
|
|
46
|
+
// mid-registration can't leave a uuid from a dead session in `liveByTable`.
|
|
47
|
+
private connectionGeneration = 0;
|
|
48
|
+
private connectionUnsubscribes: Array<() => void> = [];
|
|
38
49
|
private logger: Logger;
|
|
39
50
|
// SurrealDB session id, used as the per-session key inside `_00_cursor`.
|
|
40
51
|
private sessionId: string = '';
|
|
@@ -47,6 +58,66 @@ export class CrdtManager {
|
|
|
47
58
|
private debounceMs: number = 500,
|
|
48
59
|
) {
|
|
49
60
|
this.logger = logger.child({ service: 'CrdtManager' });
|
|
61
|
+
this.subscribeToReconnect();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Re-establish table LIVEs after a socket drop.
|
|
66
|
+
*
|
|
67
|
+
* A LIVE subscription lives and dies with its WebSocket session, and
|
|
68
|
+
* `ensureTableSubscription` is memoized on `liveByTable` — so without this,
|
|
69
|
+
* the first reconnect leaves CRDT realtime permanently dead: the map still
|
|
70
|
+
* holds a uuid for a subscription the server has forgotten, so every later
|
|
71
|
+
* `open()` short-circuits and no LIVE is ever re-issued.
|
|
72
|
+
*
|
|
73
|
+
* Both drop events matter: the SDK publishes `reconnecting` (not
|
|
74
|
+
* `disconnected`) when it intends to recover on its own, and `disconnected`
|
|
75
|
+
* only once it has given up.
|
|
76
|
+
*/
|
|
77
|
+
private subscribeToReconnect(): void {
|
|
78
|
+
const onDrop = () => {
|
|
79
|
+
this.connectionGeneration++;
|
|
80
|
+
for (const table of this.liveByTable.keys()) this.staleTables.add(table);
|
|
81
|
+
// In-flight registrations count too: they'd otherwise complete against
|
|
82
|
+
// the dead session and be silently dropped by the generation guard.
|
|
83
|
+
for (const table of this.pendingLive.keys()) this.staleTables.add(table);
|
|
84
|
+
// No KILL: the server-side subscriptions are already gone, and these
|
|
85
|
+
// uuids would resolve against the new session.
|
|
86
|
+
this.liveByTable.clear();
|
|
87
|
+
};
|
|
88
|
+
this.connectionUnsubscribes.push(
|
|
89
|
+
this.remote.subscribeConnection('reconnecting', onDrop),
|
|
90
|
+
this.remote.subscribeConnection('disconnected', onDrop),
|
|
91
|
+
this.remote.subscribeConnection('connected', () => {
|
|
92
|
+
const tables = Array.from(this.staleTables);
|
|
93
|
+
this.staleTables.clear();
|
|
94
|
+
for (const table of tables) {
|
|
95
|
+
// Only resurrect tables that still have an open field — a drop can
|
|
96
|
+
// outlive the editor that needed the feed.
|
|
97
|
+
if (!this.hasOpenFieldFor(table)) continue;
|
|
98
|
+
void this.ensureTableSubscription(table);
|
|
99
|
+
}
|
|
100
|
+
})
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Stop observing transport events. Separate from {@link closeAll}, which also
|
|
105
|
+
* runs on a bucket switch where the manager keeps being used. */
|
|
106
|
+
dispose(): void {
|
|
107
|
+
for (const off of this.connectionUnsubscribes) {
|
|
108
|
+
try {
|
|
109
|
+
off();
|
|
110
|
+
} catch {
|
|
111
|
+
/* ignore */
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
this.connectionUnsubscribes = [];
|
|
115
|
+
this.staleTables.clear();
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private hasOpenFieldFor(table: string): boolean {
|
|
119
|
+
const prefix = `${table}:`;
|
|
120
|
+
return Array.from(this.fields.keys()).some((k) => k.startsWith(prefix));
|
|
50
121
|
}
|
|
51
122
|
|
|
52
123
|
/** Set the session id that scopes this client's cursor entries. Must be
|
|
@@ -139,10 +210,10 @@ export class CrdtManager {
|
|
|
139
210
|
}
|
|
140
211
|
|
|
141
212
|
// If no fields on this table remain open, tear down the table-wide LIVE.
|
|
142
|
-
|
|
143
|
-
const stillOpen = Array.from(this.fields.keys()).some((k) => k.startsWith(tablePrefix));
|
|
144
|
-
if (!stillOpen) {
|
|
213
|
+
if (!this.hasOpenFieldFor(table)) {
|
|
145
214
|
this.killTableSubscription(table);
|
|
215
|
+
// Also drop any pending post-reconnect restart for it.
|
|
216
|
+
this.staleTables.delete(table);
|
|
146
217
|
}
|
|
147
218
|
|
|
148
219
|
this.logger.debug(
|
|
@@ -164,6 +235,9 @@ export class CrdtManager {
|
|
|
164
235
|
for (const table of Array.from(this.liveByTable.keys())) {
|
|
165
236
|
this.killTableSubscription(table);
|
|
166
237
|
}
|
|
238
|
+
// Nothing is open any more, so nothing should be resurrected on the next
|
|
239
|
+
// reconnect; `open()` will re-subscribe on demand.
|
|
240
|
+
this.staleTables.clear();
|
|
167
241
|
}
|
|
168
242
|
|
|
169
243
|
/** Ensure a single `LIVE SELECT * FROM <table>` is running, shared across
|
|
@@ -171,8 +245,26 @@ export class CrdtManager {
|
|
|
171
245
|
private async ensureTableSubscription(table: string): Promise<void> {
|
|
172
246
|
if (this.liveByTable.has(table)) return;
|
|
173
247
|
|
|
248
|
+
// Registering a LIVE takes two round-trips, so a socket drop can land
|
|
249
|
+
// mid-flight. The generation stamp is what keeps that honest: a registration
|
|
250
|
+
// that started before the drop must never record its uuid, because this
|
|
251
|
+
// method short-circuits on `liveByTable` and that stale entry (pointing at a
|
|
252
|
+
// session the server has forgotten) would block every future restart.
|
|
253
|
+
const generation = this.connectionGeneration;
|
|
254
|
+
|
|
174
255
|
const pending = this.pendingLive.get(table);
|
|
175
|
-
if (pending)
|
|
256
|
+
if (pending) {
|
|
257
|
+
if (pending.generation === generation) return pending.promise;
|
|
258
|
+
// A registration from before the drop is still settling. It will discard
|
|
259
|
+
// itself; wait it out, then register fresh on the new socket.
|
|
260
|
+
await pending.promise.catch(() => {});
|
|
261
|
+
if (this.liveByTable.has(table)) return;
|
|
262
|
+
if (generation !== this.connectionGeneration) return;
|
|
263
|
+
// Another caller may have won the race to re-register during that await;
|
|
264
|
+
// join it rather than opening a second LIVE on the same table.
|
|
265
|
+
const successor = this.pendingLive.get(table);
|
|
266
|
+
if (successor && successor.generation === generation) return successor.promise;
|
|
267
|
+
}
|
|
176
268
|
|
|
177
269
|
const start = (async () => {
|
|
178
270
|
try {
|
|
@@ -180,6 +272,14 @@ export class CrdtManager {
|
|
|
180
272
|
`LIVE SELECT * FROM ${table}`,
|
|
181
273
|
);
|
|
182
274
|
|
|
275
|
+
if (generation !== this.connectionGeneration) {
|
|
276
|
+
this.logger.debug(
|
|
277
|
+
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
278
|
+
'Socket dropped while registering LIVE; discarding it'
|
|
279
|
+
);
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
|
|
183
283
|
const subscription = await this.remote.getClient().liveOf(uuid);
|
|
184
284
|
subscription.subscribe((message) => {
|
|
185
285
|
if (message.action === 'KILLED') return;
|
|
@@ -187,6 +287,8 @@ export class CrdtManager {
|
|
|
187
287
|
this.dispatchRow(table, message.value as Record<string, unknown>);
|
|
188
288
|
});
|
|
189
289
|
|
|
290
|
+
if (generation !== this.connectionGeneration) return;
|
|
291
|
+
|
|
190
292
|
this.liveByTable.set(table, uuid);
|
|
191
293
|
this.logger.info(
|
|
192
294
|
{ table, Category: 'sp00ky-client::CrdtManager::ensureTableSubscription' },
|
|
@@ -200,7 +302,7 @@ export class CrdtManager {
|
|
|
200
302
|
}
|
|
201
303
|
})();
|
|
202
304
|
|
|
203
|
-
this.pendingLive.set(table, start);
|
|
305
|
+
this.pendingLive.set(table, { promise: start, generation });
|
|
204
306
|
try {
|
|
205
307
|
await start;
|
|
206
308
|
} finally {
|