@spooky-sync/core 0.0.1-canary.205 → 0.0.1-canary.207
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +78 -6
- package/dist/index.js +151 -44
- package/dist/types.d.ts +8 -0
- package/package.json +3 -3
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +26 -4
- package/src/modules/data/data.membership.test.ts +62 -1
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.rebind.test.ts +23 -0
- package/src/modules/data/index.ts +82 -31
- package/src/modules/devtools/index.ts +6 -0
- package/src/modules/sync/sync.live-removal.test.ts +40 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +71 -6
- package/src/services/tabs/coordinator.test.ts +118 -2
- package/src/services/tabs/coordinator.ts +25 -0
- package/src/services/tabs/protocol.ts +17 -1
- package/src/sp00ky.ts +9 -11
- package/src/types.ts +8 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { Sp00kySync } from './sync';
|
|
4
|
+
import type { IngestTuple } from '../../services/tabs/protocol';
|
|
5
|
+
|
|
6
|
+
// Shared-tabs sync roles. A follower's optimistic write used to enter only
|
|
7
|
+
// its own circuit: the leader and every other follower learned of it after the
|
|
8
|
+
// server round-trip, through a LIVE event SurrealDB v3 sometimes drops. Now the
|
|
9
|
+
// follower posts the ingested tuples to the leader, which feeds its own circuit
|
|
10
|
+
// and fans them out (excluding the origin), and the leader tells every follower
|
|
11
|
+
// when a push settled so the row does not blink out of the render set.
|
|
12
|
+
|
|
13
|
+
function makeLogger(): any {
|
|
14
|
+
const logger: any = {
|
|
15
|
+
child: () => logger,
|
|
16
|
+
debug: () => {},
|
|
17
|
+
info: () => {},
|
|
18
|
+
warn: () => {},
|
|
19
|
+
error: () => {},
|
|
20
|
+
trace: () => {},
|
|
21
|
+
};
|
|
22
|
+
return logger;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const tuple = (op: IngestTuple['op'], id = 'thread:c'): IngestTuple => ({
|
|
26
|
+
table: 'thread',
|
|
27
|
+
op,
|
|
28
|
+
id,
|
|
29
|
+
record: { id, _00_rv: 1 },
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
function makeSync() {
|
|
33
|
+
const queryId = new RecordId('_00_query', 'h1');
|
|
34
|
+
const queryState: any = {
|
|
35
|
+
config: {
|
|
36
|
+
id: queryId,
|
|
37
|
+
localArray: [['thread:a', 1]],
|
|
38
|
+
remoteArray: [['thread:a', 1]],
|
|
39
|
+
membershipKnown: true,
|
|
40
|
+
membershipKey: 'stable-key',
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
const updateQueryRemoteArray = vi.fn(async (_h: string, next: any) => {
|
|
44
|
+
queryState.config.remoteArray = next;
|
|
45
|
+
});
|
|
46
|
+
const dataModule: any = {
|
|
47
|
+
getQueryById: vi.fn((id: RecordId) => (String(id.id) === 'h1' ? queryState : undefined)),
|
|
48
|
+
getQueryByHash: vi.fn().mockReturnValue(queryState),
|
|
49
|
+
updateQueryRemoteArray,
|
|
50
|
+
notifyQuerySynced: vi.fn().mockResolvedValue(undefined),
|
|
51
|
+
notifyTableQueries: vi.fn().mockResolvedValue(undefined),
|
|
52
|
+
noteWriteSettled: vi.fn(),
|
|
53
|
+
getActiveQueryHashes: () => ['h1'],
|
|
54
|
+
getPendingRecordIds: async () => ({ writes: new Set(), deletes: new Set() }),
|
|
55
|
+
};
|
|
56
|
+
const cache: any = { applyRelayedIngest: vi.fn() };
|
|
57
|
+
const sync = new Sp00kySync(
|
|
58
|
+
{} as any,
|
|
59
|
+
{ query: vi.fn() } as any,
|
|
60
|
+
cache,
|
|
61
|
+
dataModule,
|
|
62
|
+
{} as any,
|
|
63
|
+
makeLogger()
|
|
64
|
+
);
|
|
65
|
+
(sync as any).runSyncForQuery = vi.fn().mockResolvedValue(undefined);
|
|
66
|
+
(sync as any).upQueue.enqueueFromDatabase = vi.fn().mockResolvedValue(undefined);
|
|
67
|
+
(sync as any).scheduler.enqueueMutation = vi.fn();
|
|
68
|
+
|
|
69
|
+
const hub: any = {
|
|
70
|
+
onFollowerMessage: null,
|
|
71
|
+
relayIngest: vi.fn(),
|
|
72
|
+
broadcast: vi.fn(),
|
|
73
|
+
sendTo: vi.fn(),
|
|
74
|
+
};
|
|
75
|
+
const forwarder: any = {
|
|
76
|
+
onLeaderMessage: null,
|
|
77
|
+
mutationEnqueued: vi.fn(),
|
|
78
|
+
ingest: vi.fn(),
|
|
79
|
+
};
|
|
80
|
+
return { sync, cache, dataModule, hub, forwarder, queryState, updateQueryRemoteArray };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
describe('shared-tabs leader', () => {
|
|
84
|
+
beforeEach(() => vi.clearAllMocks());
|
|
85
|
+
|
|
86
|
+
it('applies a follower ingest to its own circuit and relays it to the other followers', () => {
|
|
87
|
+
const { sync, cache, hub } = makeSync();
|
|
88
|
+
sync.promoteToLeader(hub);
|
|
89
|
+
const tuples = [tuple('CREATE')];
|
|
90
|
+
|
|
91
|
+
hub.onFollowerMessage('tab-2', { type: 'ingest', tuples });
|
|
92
|
+
|
|
93
|
+
expect(cache.applyRelayedIngest).toHaveBeenCalledWith(tuples);
|
|
94
|
+
expect(hub.relayIngest).toHaveBeenCalledWith(tuples, 'tab-2');
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it('re-materializes the table queries for a relayed DELETE', async () => {
|
|
98
|
+
const { sync, dataModule, hub } = makeSync();
|
|
99
|
+
sync.promoteToLeader(hub);
|
|
100
|
+
|
|
101
|
+
hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('DELETE')] });
|
|
102
|
+
await Promise.resolve();
|
|
103
|
+
|
|
104
|
+
expect(dataModule.notifyTableQueries).toHaveBeenCalledWith('thread');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('does not re-materialize for a relayed CREATE/UPDATE (the stream update covers it)', async () => {
|
|
108
|
+
const { sync, dataModule, hub } = makeSync();
|
|
109
|
+
sync.promoteToLeader(hub);
|
|
110
|
+
|
|
111
|
+
hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('UPDATE')] });
|
|
112
|
+
await Promise.resolve();
|
|
113
|
+
|
|
114
|
+
expect(dataModule.notifyTableQueries).not.toHaveBeenCalled();
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('treats a follower write as activity for the poll backoff', () => {
|
|
118
|
+
const { sync, hub } = makeSync();
|
|
119
|
+
sync.promoteToLeader(hub);
|
|
120
|
+
|
|
121
|
+
(sync as any).listRefIdleStreak = 7;
|
|
122
|
+
hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('CREATE')] });
|
|
123
|
+
expect((sync as any).listRefIdleStreak).toBe(0);
|
|
124
|
+
|
|
125
|
+
(sync as any).listRefIdleStreak = 7;
|
|
126
|
+
hub.onFollowerMessage('tab-2', {
|
|
127
|
+
type: 'mutation-enqueued',
|
|
128
|
+
mutationId: '_00_pending_mutations:1_0001_tab-2',
|
|
129
|
+
});
|
|
130
|
+
expect((sync as any).listRefIdleStreak).toBe(0);
|
|
131
|
+
expect((sync as any).upQueue.enqueueFromDatabase).toHaveBeenCalledWith(
|
|
132
|
+
'_00_pending_mutations:1_0001_tab-2'
|
|
133
|
+
);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it('notes a settled mutation locally and broadcasts it to every follower', () => {
|
|
137
|
+
const { sync, dataModule, hub } = makeSync();
|
|
138
|
+
sync.promoteToLeader(hub);
|
|
139
|
+
|
|
140
|
+
(sync as any).handleMutationSettled({
|
|
141
|
+
type: 'create',
|
|
142
|
+
mutation_id: new RecordId('_00_pending_mutations', '1_0001_tab-2'),
|
|
143
|
+
record_id: new RecordId('thread', 'c'),
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'create');
|
|
147
|
+
expect(hub.broadcast).toHaveBeenCalledWith({
|
|
148
|
+
type: 'mutation-settled',
|
|
149
|
+
mutationId: '_00_pending_mutations:1_0001_tab-2',
|
|
150
|
+
recordId: 'thread:c',
|
|
151
|
+
eventType: 'create',
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it('solo: a settled mutation is noted locally with no hub to broadcast on', () => {
|
|
156
|
+
const { sync, dataModule } = makeSync();
|
|
157
|
+
(sync as any).handleMutationSettled({
|
|
158
|
+
type: 'update',
|
|
159
|
+
mutation_id: new RecordId('_00_pending_mutations', '1_0001_solo'),
|
|
160
|
+
record_id: new RecordId('thread', 'c'),
|
|
161
|
+
});
|
|
162
|
+
expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'update');
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
describe('shared-tabs follower', () => {
|
|
167
|
+
beforeEach(() => vi.clearAllMocks());
|
|
168
|
+
|
|
169
|
+
it('starts the local settled-write grace on mutation-settled', () => {
|
|
170
|
+
const { sync, dataModule, forwarder } = makeSync();
|
|
171
|
+
sync.demoteToFollower(forwarder);
|
|
172
|
+
|
|
173
|
+
forwarder.onLeaderMessage({
|
|
174
|
+
type: 'mutation-settled',
|
|
175
|
+
mutationId: '_00_pending_mutations:1_0001_tab-1',
|
|
176
|
+
recordId: 'thread:c',
|
|
177
|
+
eventType: 'delete',
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'delete');
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it('feeds a relayed ingest to the local circuit', async () => {
|
|
184
|
+
const { sync, cache, dataModule, forwarder } = makeSync();
|
|
185
|
+
sync.demoteToFollower(forwarder);
|
|
186
|
+
const tuples = [tuple('DELETE')];
|
|
187
|
+
|
|
188
|
+
forwarder.onLeaderMessage({ type: 'ingest-relay', tuples, leadershipId: 1, seq: 1 });
|
|
189
|
+
await Promise.resolve();
|
|
190
|
+
|
|
191
|
+
expect(cache.applyRelayedIngest).toHaveBeenCalledWith(tuples);
|
|
192
|
+
expect(dataModule.notifyTableQueries).toHaveBeenCalledWith('thread');
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it('applies a relayed list_ref change for one of its own queries', async () => {
|
|
196
|
+
const { sync, forwarder, updateQueryRemoteArray } = makeSync();
|
|
197
|
+
sync.demoteToFollower(forwarder);
|
|
198
|
+
|
|
199
|
+
forwarder.onLeaderMessage({
|
|
200
|
+
type: 'list-ref-change',
|
|
201
|
+
action: 'CREATE',
|
|
202
|
+
queryId: '_00_query:h1',
|
|
203
|
+
recordId: 'thread:c',
|
|
204
|
+
version: 1,
|
|
205
|
+
parent: false,
|
|
206
|
+
});
|
|
207
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
208
|
+
|
|
209
|
+
expect(updateQueryRemoteArray).toHaveBeenCalledWith('h1', [
|
|
210
|
+
['thread:a', 1],
|
|
211
|
+
['thread:c', 1],
|
|
212
|
+
]);
|
|
213
|
+
expect((sync as any).runSyncForQuery).toHaveBeenCalled();
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
it('ignores a relayed list_ref change for another tab query', async () => {
|
|
217
|
+
const { sync, forwarder, updateQueryRemoteArray } = makeSync();
|
|
218
|
+
sync.demoteToFollower(forwarder);
|
|
219
|
+
|
|
220
|
+
forwarder.onLeaderMessage({
|
|
221
|
+
type: 'list-ref-change',
|
|
222
|
+
action: 'CREATE',
|
|
223
|
+
queryId: '_00_query:foreign',
|
|
224
|
+
recordId: 'thread:c',
|
|
225
|
+
version: 1,
|
|
226
|
+
parent: false,
|
|
227
|
+
});
|
|
228
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
229
|
+
|
|
230
|
+
expect(updateQueryRemoteArray).not.toHaveBeenCalled();
|
|
231
|
+
expect((sync as any).runSyncForQuery).not.toHaveBeenCalled();
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it('forwards mutation ids to the leader instead of queueing locally', async () => {
|
|
235
|
+
const { sync, forwarder } = makeSync();
|
|
236
|
+
sync.demoteToFollower(forwarder);
|
|
237
|
+
|
|
238
|
+
await sync.enqueueMutation([
|
|
239
|
+
{
|
|
240
|
+
type: 'create',
|
|
241
|
+
mutation_id: new RecordId('_00_pending_mutations', '1_0001_tab-2'),
|
|
242
|
+
record_id: new RecordId('thread', 'c'),
|
|
243
|
+
} as any,
|
|
244
|
+
]);
|
|
245
|
+
|
|
246
|
+
expect(forwarder.mutationEnqueued).toHaveBeenCalledWith('_00_pending_mutations:1_0001_tab-2');
|
|
247
|
+
expect((sync as any).scheduler.enqueueMutation).not.toHaveBeenCalled();
|
|
248
|
+
});
|
|
249
|
+
});
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -43,6 +43,7 @@ import {
|
|
|
43
43
|
import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
44
44
|
import { mutationOwnerTabId } from '../data/mutation-id';
|
|
45
45
|
import type { LeaderSyncHub, SyncForwarder } from '../../services/tabs/coordinator';
|
|
46
|
+
import type { IngestTuple } from '../../services/tabs/protocol';
|
|
46
47
|
import { parseRecordIdString } from '../../utils/index';
|
|
47
48
|
|
|
48
49
|
/**
|
|
@@ -562,13 +563,24 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
562
563
|
this.forwarder = null;
|
|
563
564
|
this.leaderDutiesInFlight = null;
|
|
564
565
|
hub.onFollowerMessage = (tabId, msg) => {
|
|
565
|
-
void tabId;
|
|
566
566
|
switch (msg.type) {
|
|
567
567
|
case 'sync-hello':
|
|
568
568
|
break;
|
|
569
569
|
case 'mutation-enqueued':
|
|
570
|
+
// A write is activity: snap the poll back to its base cadence so the
|
|
571
|
+
// membership for it lands fast even if LIVE drops the event.
|
|
572
|
+
this.listRefIdleStreak = 0;
|
|
570
573
|
void this.enqueueForwardedMutation(msg.mutationId);
|
|
571
574
|
break;
|
|
575
|
+
case 'ingest':
|
|
576
|
+
// A follower's optimistic write. The row is already in the shared
|
|
577
|
+
// store; feed this tab's circuit and fan it out to every OTHER
|
|
578
|
+
// follower, so the write shows up everywhere in one hop instead of
|
|
579
|
+
// after the server round-trip (which also depends on LIVE delivery).
|
|
580
|
+
this.applyRelayedIngest(msg.tuples);
|
|
581
|
+
hub.relayIngest(msg.tuples, tabId);
|
|
582
|
+
this.listRefIdleStreak = 0;
|
|
583
|
+
break;
|
|
572
584
|
case 'request-poll':
|
|
573
585
|
this.listRefIdleStreak = 0;
|
|
574
586
|
break;
|
|
@@ -617,6 +629,17 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
617
629
|
void this.killRefLiveQuery();
|
|
618
630
|
forwarder.onLeaderMessage = (msg) => {
|
|
619
631
|
switch (msg.type) {
|
|
632
|
+
case 'ingest-relay':
|
|
633
|
+
this.applyRelayedIngest(msg.tuples);
|
|
634
|
+
break;
|
|
635
|
+
case 'mutation-settled':
|
|
636
|
+
// The leader pushed a write and deleted its outbox row from the
|
|
637
|
+
// shared store. Without this the row would leave this tab's render
|
|
638
|
+
// set (it is in neither membership nor pending writes) until the
|
|
639
|
+
// relayed `_00_list_ref` event lands: the blink the leader itself
|
|
640
|
+
// is already protected from by `handleMutationSettled`.
|
|
641
|
+
this.dataModule.noteWriteSettled(msg.recordId, msg.eventType);
|
|
642
|
+
break;
|
|
620
643
|
case 'list-ref-change':
|
|
621
644
|
void this.applyRelayedListRefChange(msg).catch((err) => {
|
|
622
645
|
this.logger.error(
|
|
@@ -633,8 +656,8 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
633
656
|
});
|
|
634
657
|
break;
|
|
635
658
|
default:
|
|
636
|
-
// db-ready
|
|
637
|
-
//
|
|
659
|
+
// db-ready is consumed by the coordinator's attach handshake before
|
|
660
|
+
// the sync handler is installed.
|
|
638
661
|
break;
|
|
639
662
|
}
|
|
640
663
|
};
|
|
@@ -673,6 +696,26 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
673
696
|
await this.upQueue.enqueueFromDatabase(mutationId);
|
|
674
697
|
}
|
|
675
698
|
|
|
699
|
+
/**
|
|
700
|
+
* Tuples another tab already committed to the shared store: feed them to
|
|
701
|
+
* THIS tab's circuit (no local write). A DELETE additionally forces a
|
|
702
|
+
* re-materialize of the table's queries, exactly as the writing tab does
|
|
703
|
+
* for itself, because the SSP may not emit a view update for it.
|
|
704
|
+
*/
|
|
705
|
+
private applyRelayedIngest(tuples: IngestTuple[]): void {
|
|
706
|
+
this.cache.applyRelayedIngest(tuples);
|
|
707
|
+
const deletedTables = new Set<string>();
|
|
708
|
+
for (const t of tuples) if (t.op === 'DELETE') deletedTables.add(t.table);
|
|
709
|
+
for (const table of deletedTables) {
|
|
710
|
+
void this.dataModule.notifyTableQueries(table).catch((err) => {
|
|
711
|
+
this.logger.warn(
|
|
712
|
+
{ err, table, Category: 'sp00ky-client::Sp00kySync::applyRelayedIngest' },
|
|
713
|
+
'Re-materialize after relayed delete failed'
|
|
714
|
+
);
|
|
715
|
+
});
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
676
719
|
/** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
|
|
677
720
|
* and run the exact same handling the LIVE subscription would have. */
|
|
678
721
|
private async applyRelayedListRefChange(msg: {
|
|
@@ -1282,8 +1325,19 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1282
1325
|
// until the next poll tick, which is up to 5s of showing a deleted row. This
|
|
1283
1326
|
// also persists the durable `_00_window` mirror, so the removal survives a
|
|
1284
1327
|
// reload with no network.
|
|
1285
|
-
|
|
1286
|
-
|
|
1328
|
+
//
|
|
1329
|
+
// Derived from the raw action, NOT from `diff`: `createDiffFromDbOp` is
|
|
1330
|
+
// empty when the circuit already holds the row at this version, which is
|
|
1331
|
+
// every tab that ingested the write optimistically (its own, or one
|
|
1332
|
+
// relayed from another tab). The fetch is rightly skipped then, but the
|
|
1333
|
+
// membership still has to be recorded, or the row lives on the
|
|
1334
|
+
// settled-write grace alone until the poll catches it.
|
|
1335
|
+
if (existing.config.membershipKnown) {
|
|
1336
|
+
const membershipDiff: RecordVersionDiff =
|
|
1337
|
+
action === 'DELETE'
|
|
1338
|
+
? { added: [], updated: [], removed: [recordId] }
|
|
1339
|
+
: { added: [{ id: recordId, version }], updated: [], removed: [] };
|
|
1340
|
+
const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], membershipDiff);
|
|
1287
1341
|
if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) {
|
|
1288
1342
|
await this.dataModule.updateQueryRemoteArray(hash, next);
|
|
1289
1343
|
}
|
|
@@ -1437,7 +1491,18 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1437
1491
|
* vanish, and return, while every other client showed it throughout.
|
|
1438
1492
|
*/
|
|
1439
1493
|
private handleMutationSettled(event: UpEvent): void {
|
|
1440
|
-
|
|
1494
|
+
const recordId = encodeRecordId(event.record_id);
|
|
1495
|
+
this.dataModule.noteWriteSettled(recordId, event.type);
|
|
1496
|
+
// Shared-tabs: the outbox row just left the SHARED store, so every
|
|
1497
|
+
// follower rendering the row as a pending write has the same gap. All of
|
|
1498
|
+
// them, not just the owner: any tab whose query matched the row was
|
|
1499
|
+
// showing it through `pendingWrites`.
|
|
1500
|
+
this.hub?.broadcast({
|
|
1501
|
+
type: 'mutation-settled',
|
|
1502
|
+
mutationId: encodeRecordId(event.mutation_id),
|
|
1503
|
+
recordId,
|
|
1504
|
+
eventType: event.type,
|
|
1505
|
+
});
|
|
1441
1506
|
}
|
|
1442
1507
|
|
|
1443
1508
|
private async handleRollback(event: UpEvent, error: Error): Promise<void> {
|
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
2
|
import { handleConnect, __resetBrokerForTests } from './tabs-broker-worker';
|
|
3
|
-
import { installBrokerGlobals, installFakeLocks } from './fake-ports.fixture';
|
|
4
|
-
import {
|
|
3
|
+
import { fakeChannel, flush, installBrokerGlobals, installFakeLocks } from './fake-ports.fixture';
|
|
4
|
+
import {
|
|
5
|
+
TabsCoordinator,
|
|
6
|
+
SyncForwarder,
|
|
7
|
+
type CoordinatorHooks,
|
|
8
|
+
type LeaderSyncHub,
|
|
9
|
+
} from './coordinator';
|
|
5
10
|
import type { StorageHealth } from '../../types';
|
|
6
11
|
import type { LeaderToFollowerMessage } from './protocol';
|
|
7
12
|
|
|
@@ -156,6 +161,117 @@ describe('TabsCoordinator integration', () => {
|
|
|
156
161
|
await a.coordinator.stop();
|
|
157
162
|
});
|
|
158
163
|
|
|
164
|
+
// A follower's optimistic write must reach the leader (which ingests it and
|
|
165
|
+
// fans it out) in one hop. Before this, only the mutation id was forwarded
|
|
166
|
+
// and every other tab waited on the server round-trip + LIVE delivery.
|
|
167
|
+
it('forwards a follower ingest to the leader hub tagged with the origin tab', async () => {
|
|
168
|
+
const a = makeCoordinator('tab-1');
|
|
169
|
+
await a.coordinator.start('anon');
|
|
170
|
+
const b = makeCoordinator('tab-2');
|
|
171
|
+
await b.coordinator.start('anon');
|
|
172
|
+
|
|
173
|
+
const seen: { tabId: string; msg: unknown }[] = [];
|
|
174
|
+
a.log.hub!.onFollowerMessage = (tabId, msg) => seen.push({ tabId, msg });
|
|
175
|
+
const tuples = [{ table: 'game', op: 'CREATE' as const, id: 'game:9', record: { id: 'game:9' } }];
|
|
176
|
+
b.log.forwarder!.ingest(tuples);
|
|
177
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
178
|
+
|
|
179
|
+
const ingest = seen.filter((s) => (s.msg as { type: string }).type === 'ingest');
|
|
180
|
+
expect(ingest).toHaveLength(1);
|
|
181
|
+
expect(ingest[0].tabId).toBe('tab-2');
|
|
182
|
+
expect((ingest[0].msg as { tuples: unknown }).tuples).toEqual(tuples);
|
|
183
|
+
await b.coordinator.stop();
|
|
184
|
+
await a.coordinator.stop();
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('fan-out with exceptTabId skips the origin follower and reaches the others', async () => {
|
|
188
|
+
const a = makeCoordinator('tab-1');
|
|
189
|
+
await a.coordinator.start('anon');
|
|
190
|
+
const b = makeCoordinator('tab-2');
|
|
191
|
+
await b.coordinator.start('anon');
|
|
192
|
+
const c = makeCoordinator('tab-3');
|
|
193
|
+
await c.coordinator.start('anon');
|
|
194
|
+
|
|
195
|
+
const seenB: LeaderToFollowerMessage[] = [];
|
|
196
|
+
const seenC: LeaderToFollowerMessage[] = [];
|
|
197
|
+
b.log.forwarder!.onLeaderMessage = (msg) => seenB.push(msg);
|
|
198
|
+
c.log.forwarder!.onLeaderMessage = (msg) => seenC.push(msg);
|
|
199
|
+
a.log.hub!.relayIngest(
|
|
200
|
+
[{ table: 'game', op: 'CREATE', id: 'game:9', record: { id: 'game:9' } }],
|
|
201
|
+
'tab-2'
|
|
202
|
+
);
|
|
203
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
204
|
+
|
|
205
|
+
expect(seenB.filter((m) => m.type === 'ingest-relay')).toHaveLength(0);
|
|
206
|
+
expect(seenC.filter((m) => m.type === 'ingest-relay')).toHaveLength(1);
|
|
207
|
+
await c.coordinator.stop();
|
|
208
|
+
await b.coordinator.stop();
|
|
209
|
+
await a.coordinator.stop();
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
// Unlike a mutation notify, an ingest is NOT queued across a leaderless
|
|
213
|
+
// window: the next leader primes its circuit from the shared store, which
|
|
214
|
+
// already holds the row, and a stale replay would put an older `_00_rv` in
|
|
215
|
+
// its version memo.
|
|
216
|
+
it('drops an ingest while detached instead of queueing it', async () => {
|
|
217
|
+
const forwarder = new SyncForwarder('tab-x');
|
|
218
|
+
forwarder.ingest([{ table: 'game', op: 'CREATE', id: 'game:1', record: { id: 'game:1' } }]);
|
|
219
|
+
|
|
220
|
+
const { port1, port2 } = fakeChannel();
|
|
221
|
+
const seen: { type: string }[] = [];
|
|
222
|
+
port2.onmessage = (ev) => {
|
|
223
|
+
seen.push(ev.data as { type: string });
|
|
224
|
+
};
|
|
225
|
+
forwarder.rebind(port1 as unknown as MessagePort);
|
|
226
|
+
await flush();
|
|
227
|
+
|
|
228
|
+
expect(seen.map((m) => m.type)).toEqual(['sync-hello']);
|
|
229
|
+
forwarder.unbind();
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
// Relay traffic that lands between `db-ready` and the sync hooks being
|
|
233
|
+
// installed (the store adopt is async) used to hit a null handler on a
|
|
234
|
+
// first attach and vanish.
|
|
235
|
+
it('buffers leader messages that land before the follower hooks are installed', async () => {
|
|
236
|
+
const a = makeCoordinator('tab-1');
|
|
237
|
+
await a.coordinator.start('anon');
|
|
238
|
+
|
|
239
|
+
let adoptStartedResolve!: () => void;
|
|
240
|
+
const adoptStartedP = new Promise<void>((r) => {
|
|
241
|
+
adoptStartedResolve = r;
|
|
242
|
+
});
|
|
243
|
+
let releaseAdopt!: () => void;
|
|
244
|
+
const adoptGate = new Promise<void>((r) => {
|
|
245
|
+
releaseAdopt = r;
|
|
246
|
+
});
|
|
247
|
+
const seen: LeaderToFollowerMessage[] = [];
|
|
248
|
+
const b = makeCoordinator('tab-2', {
|
|
249
|
+
async adoptAttached() {
|
|
250
|
+
adoptStartedResolve();
|
|
251
|
+
await adoptGate;
|
|
252
|
+
},
|
|
253
|
+
becomeSyncFollower(forwarder) {
|
|
254
|
+
forwarder.onLeaderMessage = (msg) => seen.push(msg);
|
|
255
|
+
},
|
|
256
|
+
});
|
|
257
|
+
const started = b.coordinator.start('anon');
|
|
258
|
+
await adoptStartedP;
|
|
259
|
+
a.log.hub!.broadcast({
|
|
260
|
+
type: 'mutation-settled',
|
|
261
|
+
mutationId: '_00_pending_mutations:1_0001_tab-1',
|
|
262
|
+
recordId: 'game:1',
|
|
263
|
+
eventType: 'create',
|
|
264
|
+
});
|
|
265
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
266
|
+
expect(seen).toHaveLength(0);
|
|
267
|
+
|
|
268
|
+
releaseAdopt();
|
|
269
|
+
await started;
|
|
270
|
+
expect(seen.map((m) => m.type)).toEqual(['mutation-settled']);
|
|
271
|
+
await b.coordinator.stop();
|
|
272
|
+
await a.coordinator.stop();
|
|
273
|
+
});
|
|
274
|
+
|
|
159
275
|
it('promotes the follower when the leader stops (failover)', async () => {
|
|
160
276
|
const a = makeCoordinator('tab-1');
|
|
161
277
|
await a.coordinator.start('anon');
|
|
@@ -201,6 +201,18 @@ export class SyncForwarder {
|
|
|
201
201
|
mutationEnqueued(mutationId: string): void {
|
|
202
202
|
this.post({ type: 'mutation-enqueued', mutationId });
|
|
203
203
|
}
|
|
204
|
+
/** An optimistic write this tab just ingested. Deliberately NOT queued while
|
|
205
|
+
* detached: a new leader primes its circuit from the shared store, which
|
|
206
|
+
* already holds the row, and replaying a stale tuple at it later would put
|
|
207
|
+
* an older `_00_rv` in its version memo. */
|
|
208
|
+
ingest(tuples: IngestTuple[]): void {
|
|
209
|
+
if (!this.port) return;
|
|
210
|
+
try {
|
|
211
|
+
this.port.postMessage({ type: 'ingest', tuples });
|
|
212
|
+
} catch {
|
|
213
|
+
/* dead port; broker re-mints */
|
|
214
|
+
}
|
|
215
|
+
}
|
|
204
216
|
requestPoll(): void {
|
|
205
217
|
this.post({ type: 'request-poll' });
|
|
206
218
|
}
|
|
@@ -545,6 +557,12 @@ export class TabsCoordinator {
|
|
|
545
557
|
const forwarder = this.forwarder;
|
|
546
558
|
await new Promise<void>((resolve) => {
|
|
547
559
|
let adopted = false;
|
|
560
|
+
let attached = false;
|
|
561
|
+
// Relay traffic that lands between `db-ready` and the sync hooks being
|
|
562
|
+
// installed (the store adopt is async). Held and replayed, not dropped:
|
|
563
|
+
// on a first attach there is no previous handler, so an ingest-relay or
|
|
564
|
+
// settled notice in that window used to vanish.
|
|
565
|
+
const pending: LeaderToFollowerMessage[] = [];
|
|
548
566
|
const previousHandler = forwarder.onLeaderMessage;
|
|
549
567
|
forwarder.onLeaderMessage = (msg) => {
|
|
550
568
|
if (msg.type === 'db-ready' && !adopted) {
|
|
@@ -557,11 +575,18 @@ export class TabsCoordinator {
|
|
|
557
575
|
})
|
|
558
576
|
.then(() => {
|
|
559
577
|
this.deps.hooks.becomeSyncFollower(forwarder);
|
|
578
|
+
attached = true;
|
|
579
|
+
const backlog = pending.splice(0);
|
|
580
|
+
for (const m of backlog) forwarder.onLeaderMessage?.(m);
|
|
560
581
|
this.setRole('follower');
|
|
561
582
|
resolve();
|
|
562
583
|
});
|
|
563
584
|
return;
|
|
564
585
|
}
|
|
586
|
+
if (!attached) {
|
|
587
|
+
pending.push(msg);
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
565
590
|
previousHandler?.(msg);
|
|
566
591
|
};
|
|
567
592
|
forwarder.rebind(syncPort);
|
|
@@ -215,7 +215,13 @@ export type FollowerToLeaderMessage =
|
|
|
215
215
|
* leader should drain it. Idempotent; a new leader's loadFromDatabase is
|
|
216
216
|
* the backstop for a notify lost in a failover window. */
|
|
217
217
|
| { type: 'mutation-enqueued'; mutationId: string }
|
|
218
|
-
| { type: 'request-poll' }
|
|
218
|
+
| { type: 'request-poll' }
|
|
219
|
+
/** An optimistic write this follower committed to the SHARED store and
|
|
220
|
+
* ingested into its own circuit. The leader ingests it (no DB write, the
|
|
221
|
+
* row is already there) and fans it out to every OTHER follower as
|
|
222
|
+
* `ingest-relay`, so a follower's write lands in every tab in one hop
|
|
223
|
+
* instead of after the server round-trip. */
|
|
224
|
+
| { type: 'ingest'; tuples: IngestTuple[] };
|
|
219
225
|
|
|
220
226
|
export type LeaderToFollowerMessage =
|
|
221
227
|
| { type: 'db-ready'; leadershipId: number; bucketId: string; storageHealth: StorageHealth }
|
|
@@ -239,4 +245,14 @@ export type LeaderToFollowerMessage =
|
|
|
239
245
|
recordId: string;
|
|
240
246
|
eventType: 'create' | 'update' | 'delete';
|
|
241
247
|
error: string;
|
|
248
|
+
}
|
|
249
|
+
/** The leader's drain pushed a mutation and deleted its outbox row from the
|
|
250
|
+
* SHARED store. Every follower starts its settled-write grace so a row it
|
|
251
|
+
* was rendering as a pending write does not blink out before its
|
|
252
|
+
* `_00_list_ref` membership arrives. */
|
|
253
|
+
| {
|
|
254
|
+
type: 'mutation-settled';
|
|
255
|
+
mutationId: string;
|
|
256
|
+
recordId: string;
|
|
257
|
+
eventType: 'create' | 'update' | 'delete';
|
|
242
258
|
};
|
package/src/sp00ky.ts
CHANGED
|
@@ -673,20 +673,18 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
673
673
|
resumeSyncLeaderDuties: () => this.sync.resumeLeaderDuties(),
|
|
674
674
|
becomeSyncFollower: (forwarder) => {
|
|
675
675
|
this.streamProcessor.setPersistenceEnabled(false);
|
|
676
|
-
|
|
676
|
+
// A follower's own mutations go to the leader, which ingests them and
|
|
677
|
+
// fans them out to the other followers: one hop, no server round-trip.
|
|
678
|
+
// Only the mutation path relays; this tab's sync fetches are the
|
|
679
|
+
// leader's data coming back and must not be re-broadcast.
|
|
680
|
+
this.cache.setIngestRelay((tuples) => forwarder.ingest(tuples), {
|
|
681
|
+
localWritesOnly: true,
|
|
682
|
+
});
|
|
677
683
|
this.sync.setTabContext('follower', tabId);
|
|
678
684
|
this.dataModule.setTabId(tabId);
|
|
685
|
+
// Installs the whole syncPort handler: ingest-relay, list_ref relay,
|
|
686
|
+
// settled writes, rollbacks.
|
|
679
687
|
this.sync.demoteToFollower(forwarder);
|
|
680
|
-
// demoteToFollower installed the sync-level handler (list_ref relay,
|
|
681
|
-
// rollbacks); layer the cache-level ingest relay in front of it.
|
|
682
|
-
const inner = forwarder.onLeaderMessage;
|
|
683
|
-
forwarder.onLeaderMessage = (msg) => {
|
|
684
|
-
if (msg.type === 'ingest-relay') {
|
|
685
|
-
this.cache.applyRelayedIngest(msg.tuples);
|
|
686
|
-
return;
|
|
687
|
-
}
|
|
688
|
-
inner?.(msg);
|
|
689
|
-
};
|
|
690
688
|
},
|
|
691
689
|
becomeSyncSolo: () => {
|
|
692
690
|
this.streamProcessor.setPersistenceEnabled(true);
|
package/src/types.ts
CHANGED
|
@@ -513,6 +513,11 @@ export interface QueryConfig {
|
|
|
513
513
|
* "never established" has to fall back to a predicate scan of the local store
|
|
514
514
|
* so a query first run on this device still paints offline. A
|
|
515
515
|
* `remoteArray.length === 0` check cannot tell those apart.
|
|
516
|
+
*
|
|
517
|
+
* On a cold start it is seeded from the durable `_00_window` row when that
|
|
518
|
+
* row is non-empty, or empty but `confirmed` (the server reported zero rows
|
|
519
|
+
* for the query). An unconfirmed empty row is ignored, so a device poisoned
|
|
520
|
+
* by an old client that mirrored unflushed reads still self-heals.
|
|
516
521
|
*/
|
|
517
522
|
membershipKnown?: boolean;
|
|
518
523
|
/**
|
|
@@ -528,6 +533,9 @@ export interface QueryConfig {
|
|
|
528
533
|
* genuine transition and must be honoured, or removed rows resurrect.
|
|
529
534
|
*
|
|
530
535
|
* In-memory only: a fresh session must re-earn the right to believe empties.
|
|
536
|
+
* What does persist is the `confirmed` marker on the `_00_window` row, which
|
|
537
|
+
* an empty set earns when it arrives with a server row count of zero or after
|
|
538
|
+
* a non-empty set in the same session.
|
|
531
539
|
*/
|
|
532
540
|
remoteSeen?: boolean;
|
|
533
541
|
/**
|