@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2514 -58
- package/dist/index.js +12561 -2449
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +303 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +751 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +101 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +127 -24
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +523 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1815 -151
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +450 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +164 -82
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +180 -0
- package/src/modules/sync/queue/queue-down.ts +80 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +175 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +1726 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +41 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1693 -0
- package/src/types.ts +528 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { SyncScheduler } from './scheduler';
|
|
3
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent } from './queue/index';
|
|
4
|
+
|
|
5
|
+
// A queue whose drain throws re-queues the failing item at the HEAD and stops
|
|
6
|
+
// the pass (see DownQueue.next). Nothing used to re-arm it: the queues only
|
|
7
|
+
// moved on a fresh enqueue, so one transient failure — canonically the SSP
|
|
8
|
+
// answering 503 NOT_READY for the whole of its bootstrap window — parked every
|
|
9
|
+
// pending `register` forever, and the `useQuery` waiting on it never left its
|
|
10
|
+
// loading state. These cover the backoff that makes that self-heal.
|
|
11
|
+
|
|
12
|
+
const silentLogger = {
|
|
13
|
+
child: () => silentLogger,
|
|
14
|
+
debug: () => {},
|
|
15
|
+
info: () => {},
|
|
16
|
+
warn: () => {},
|
|
17
|
+
error: () => {},
|
|
18
|
+
} as any;
|
|
19
|
+
|
|
20
|
+
/** A queue that mirrors the real ones: a throwing handler re-heads the item. */
|
|
21
|
+
function makeQueue<E>(items: E[]) {
|
|
22
|
+
return {
|
|
23
|
+
queue: [...items],
|
|
24
|
+
get size() {
|
|
25
|
+
return this.queue.length;
|
|
26
|
+
},
|
|
27
|
+
events: { subscribe: () => {} },
|
|
28
|
+
loadFromDatabase: async () => {},
|
|
29
|
+
clear() {
|
|
30
|
+
this.queue = [];
|
|
31
|
+
},
|
|
32
|
+
async next(fn: (event: E) => Promise<void>) {
|
|
33
|
+
const event = this.queue.shift();
|
|
34
|
+
if (!event) return;
|
|
35
|
+
const err = await this.run(event, fn);
|
|
36
|
+
if (err !== undefined) throw err;
|
|
37
|
+
},
|
|
38
|
+
// The concurrent down drain takes events itself and runs them, so the fake
|
|
39
|
+
// has to mirror that half of the real DownQueue too: skip events whose hash
|
|
40
|
+
// is already in flight, and hand the error back rather than throwing.
|
|
41
|
+
takeNext(busy: ReadonlySet<string>) {
|
|
42
|
+
for (let i = 0; i < this.queue.length; i++) {
|
|
43
|
+
const event = this.queue[i] as unknown as { payload?: { hash?: string } };
|
|
44
|
+
const hash = event?.payload?.hash;
|
|
45
|
+
if (hash !== undefined && busy.has(hash)) continue;
|
|
46
|
+
return this.queue.splice(i, 1)[0];
|
|
47
|
+
}
|
|
48
|
+
return undefined;
|
|
49
|
+
},
|
|
50
|
+
async run(event: E, fn: (event: E) => Promise<void>) {
|
|
51
|
+
try {
|
|
52
|
+
await fn(event);
|
|
53
|
+
return undefined;
|
|
54
|
+
} catch (err) {
|
|
55
|
+
this.queue.unshift(event);
|
|
56
|
+
return err;
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const downEvent = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
|
|
63
|
+
const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
|
|
64
|
+
|
|
65
|
+
describe('SyncScheduler retry', () => {
|
|
66
|
+
beforeEach(() => vi.useFakeTimers());
|
|
67
|
+
afterEach(() => vi.useRealTimers());
|
|
68
|
+
|
|
69
|
+
it('re-drains a failed down event instead of parking it forever', async () => {
|
|
70
|
+
const downQueue = makeQueue([downEvent('q1')]);
|
|
71
|
+
const upQueue = makeQueue<UpEvent>([]);
|
|
72
|
+
let attempts = 0;
|
|
73
|
+
const scheduler = new SyncScheduler(
|
|
74
|
+
upQueue as unknown as UpQueue,
|
|
75
|
+
downQueue as unknown as DownQueue,
|
|
76
|
+
async () => {},
|
|
77
|
+
async () => {
|
|
78
|
+
attempts++;
|
|
79
|
+
// Fail the way a bootstrapping SSP does, then succeed.
|
|
80
|
+
if (attempts < 3) throw new Error('503 NOT_READY');
|
|
81
|
+
},
|
|
82
|
+
silentLogger
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
await scheduler.syncDown();
|
|
86
|
+
expect(attempts).toBe(1);
|
|
87
|
+
expect(downQueue.size).toBe(1); // re-headed, not dropped
|
|
88
|
+
|
|
89
|
+
// Backoff: 500ms, then 1000ms.
|
|
90
|
+
await vi.advanceTimersByTimeAsync(500);
|
|
91
|
+
expect(attempts).toBe(2);
|
|
92
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
93
|
+
expect(attempts).toBe(3);
|
|
94
|
+
|
|
95
|
+
expect(downQueue.size).toBe(0);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('stops retrying once the queue drains', async () => {
|
|
99
|
+
const downQueue = makeQueue([downEvent('q1')]);
|
|
100
|
+
const upQueue = makeQueue<UpEvent>([]);
|
|
101
|
+
let attempts = 0;
|
|
102
|
+
const scheduler = new SyncScheduler(
|
|
103
|
+
upQueue as unknown as UpQueue,
|
|
104
|
+
downQueue as unknown as DownQueue,
|
|
105
|
+
async () => {},
|
|
106
|
+
async () => {
|
|
107
|
+
attempts++;
|
|
108
|
+
throw new Error('boom');
|
|
109
|
+
},
|
|
110
|
+
silentLogger
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
await scheduler.syncDown();
|
|
114
|
+
await vi.advanceTimersByTimeAsync(500);
|
|
115
|
+
expect(attempts).toBe(2);
|
|
116
|
+
|
|
117
|
+
// Drain it out from under the scheduler; the next retry finds nothing and
|
|
118
|
+
// schedules no further work.
|
|
119
|
+
downQueue.clear();
|
|
120
|
+
await vi.advanceTimersByTimeAsync(1000);
|
|
121
|
+
expect(attempts).toBe(2);
|
|
122
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
123
|
+
expect(attempts).toBe(2);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('comes back for the down queue while the up queue holds the floor', async () => {
|
|
127
|
+
const downQueue = makeQueue([downEvent('q1')]);
|
|
128
|
+
const upQueue = makeQueue<UpEvent>([upEvent(1)]);
|
|
129
|
+
let downAttempts = 0;
|
|
130
|
+
const scheduler = new SyncScheduler(
|
|
131
|
+
upQueue as unknown as UpQueue,
|
|
132
|
+
downQueue as unknown as DownQueue,
|
|
133
|
+
async () => {},
|
|
134
|
+
async () => {
|
|
135
|
+
downAttempts++;
|
|
136
|
+
},
|
|
137
|
+
silentLogger
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
// Yields to the non-empty up queue.
|
|
141
|
+
await scheduler.syncDown();
|
|
142
|
+
expect(downAttempts).toBe(0);
|
|
143
|
+
|
|
144
|
+
// Once the up queue empties, the re-armed pass picks the down event up
|
|
145
|
+
// without needing a fresh enqueue.
|
|
146
|
+
upQueue.clear();
|
|
147
|
+
await vi.advanceTimersByTimeAsync(500);
|
|
148
|
+
expect(downAttempts).toBe(1);
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
it('pause cancels pending retries', async () => {
|
|
152
|
+
const downQueue = makeQueue([downEvent('q1')]);
|
|
153
|
+
const upQueue = makeQueue<UpEvent>([]);
|
|
154
|
+
let attempts = 0;
|
|
155
|
+
const scheduler = new SyncScheduler(
|
|
156
|
+
upQueue as unknown as UpQueue,
|
|
157
|
+
downQueue as unknown as DownQueue,
|
|
158
|
+
async () => {},
|
|
159
|
+
async () => {
|
|
160
|
+
attempts++;
|
|
161
|
+
throw new Error('boom');
|
|
162
|
+
},
|
|
163
|
+
silentLogger
|
|
164
|
+
);
|
|
165
|
+
|
|
166
|
+
await scheduler.syncDown();
|
|
167
|
+
expect(attempts).toBe(1);
|
|
168
|
+
|
|
169
|
+
await scheduler.pause();
|
|
170
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
171
|
+
expect(attempts).toBe(1);
|
|
172
|
+
});
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
describe('SyncScheduler down concurrency', () => {
|
|
176
|
+
beforeEach(() => vi.useFakeTimers());
|
|
177
|
+
afterEach(() => vi.useRealTimers());
|
|
178
|
+
|
|
179
|
+
/** Scheduler over a down queue of `hashes`, whose handler blocks until the
|
|
180
|
+
* returned `release` is called for that hash. Lets a test observe exactly
|
|
181
|
+
* which events are in flight at the same moment. */
|
|
182
|
+
function makeConcurrent(hashes: string[]) {
|
|
183
|
+
const downQueue = makeQueue(hashes.map(downEvent));
|
|
184
|
+
const upQueue = makeQueue<UpEvent>([]);
|
|
185
|
+
const started: string[] = [];
|
|
186
|
+
const gates = new Map<string, () => void>();
|
|
187
|
+
const scheduler = new SyncScheduler(
|
|
188
|
+
upQueue as unknown as UpQueue,
|
|
189
|
+
downQueue as unknown as DownQueue,
|
|
190
|
+
async () => {},
|
|
191
|
+
async (event) => {
|
|
192
|
+
const hash = (event as { payload: { hash: string } }).payload.hash;
|
|
193
|
+
started.push(hash);
|
|
194
|
+
await new Promise<void>((resolve) => gates.set(hash, resolve));
|
|
195
|
+
},
|
|
196
|
+
silentLogger
|
|
197
|
+
);
|
|
198
|
+
return { scheduler, started, release: (h: string) => gates.get(h)?.() };
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
it('runs several independent hashes at once instead of one at a time', async () => {
|
|
202
|
+
// The bug this whole change exists for: strictly serial draining meant one
|
|
203
|
+
// registration RPC at a time for the entire client.
|
|
204
|
+
const { scheduler, started, release } = makeConcurrent(['a', 'b', 'c', 'd', 'e']);
|
|
205
|
+
const pass = scheduler.syncDown();
|
|
206
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
207
|
+
|
|
208
|
+
// Bounded by MAX_CONCURRENT_DOWN (4), not 1.
|
|
209
|
+
expect(started).toEqual(['a', 'b', 'c', 'd']);
|
|
210
|
+
|
|
211
|
+
// Freeing one slot immediately admits the next event.
|
|
212
|
+
release('a');
|
|
213
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
214
|
+
expect(started).toEqual(['a', 'b', 'c', 'd', 'e']);
|
|
215
|
+
|
|
216
|
+
for (const h of ['b', 'c', 'd', 'e']) release(h);
|
|
217
|
+
await pass;
|
|
218
|
+
});
|
|
219
|
+
|
|
220
|
+
it('never overlaps two events for the SAME hash', async () => {
|
|
221
|
+
// Per-hash ordering is the one invariant the serial queue was providing.
|
|
222
|
+
const { scheduler, started, release } = makeConcurrent(['a', 'a', 'b']);
|
|
223
|
+
const pass = scheduler.syncDown();
|
|
224
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
225
|
+
|
|
226
|
+
// The second 'a' is skipped while the first is in flight; 'b' goes ahead.
|
|
227
|
+
expect(started).toEqual(['a', 'b']);
|
|
228
|
+
|
|
229
|
+
release('a');
|
|
230
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
231
|
+
expect(started).toEqual(['a', 'b', 'a']);
|
|
232
|
+
|
|
233
|
+
release('b');
|
|
234
|
+
release('a');
|
|
235
|
+
await pass;
|
|
236
|
+
});
|
|
237
|
+
});
|
|
@@ -1,14 +1,41 @@
|
|
|
1
|
-
import { Logger } from '../../services/logger/index';
|
|
2
|
-
import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
1
|
+
import type { Logger } from '../../services/logger/index';
|
|
2
|
+
import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
|
|
3
3
|
import { SyncQueueEventTypes } from './events/index';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* SyncScheduler manages when to sync: queue management and orchestration.
|
|
7
7
|
* Decides the order and timing of sync operations.
|
|
8
8
|
*/
|
|
9
|
+
/** Backoff for re-draining a queue that halted on an error. */
|
|
10
|
+
const RETRY_BASE_MS = 500;
|
|
11
|
+
/**
|
|
12
|
+
* How many down events may be in flight at once.
|
|
13
|
+
*
|
|
14
|
+
* The down queue was strictly serial: one `register`/`sync`/`cleanup` RPC at a
|
|
15
|
+
* time for the WHOLE client. Measured in production that drained at roughly one
|
|
16
|
+
* event per 8.8s, so a list that registers a query per scrolled-to window took
|
|
17
|
+
* minutes to fill — the rows were already cached, only the registration lagged.
|
|
18
|
+
* Ordering is a per-hash requirement, not a global one (see `takeNext`), so
|
|
19
|
+
* independent hashes can go in parallel. Bounded to stay polite to the SSP.
|
|
20
|
+
*/
|
|
21
|
+
const MAX_CONCURRENT_DOWN = 4;
|
|
22
|
+
const RETRY_MAX_MS = 15_000;
|
|
23
|
+
|
|
9
24
|
export class SyncScheduler {
|
|
10
25
|
private isSyncingUp: boolean = false;
|
|
11
26
|
private isSyncingDown: boolean = false;
|
|
27
|
+
private paused: boolean = false;
|
|
28
|
+
private pauseWaiters: Array<() => void> = [];
|
|
29
|
+
// A failed drain re-queues its item at the HEAD (see DownQueue.next) and
|
|
30
|
+
// stops the pass. Without a timer nothing ever drains it again: the queues
|
|
31
|
+
// only move on a fresh enqueue, so a transient failure — canonically the
|
|
32
|
+
// SSP answering 503 NOT_READY for the whole of its bootstrap window — left
|
|
33
|
+
// every pending `register` parked forever and its `useQuery` loading forever.
|
|
34
|
+
// Retry on a backoff so that heals itself instead of needing a reload.
|
|
35
|
+
private upRetryTimer?: ReturnType<typeof setTimeout>;
|
|
36
|
+
private downRetryTimer?: ReturnType<typeof setTimeout>;
|
|
37
|
+
private upRetryAttempt = 0;
|
|
38
|
+
private downRetryAttempt = 0;
|
|
12
39
|
|
|
13
40
|
constructor(
|
|
14
41
|
private upQueue: UpQueue,
|
|
@@ -16,11 +43,21 @@ export class SyncScheduler {
|
|
|
16
43
|
private onProcessUp: (event: UpEvent) => Promise<void>,
|
|
17
44
|
private onProcessDown: (event: DownEvent) => Promise<void>,
|
|
18
45
|
private logger: Logger,
|
|
19
|
-
private onRollback?: RollbackCallback
|
|
46
|
+
private onRollback?: RollbackCallback,
|
|
47
|
+
// Reports the outcome of each drained sync round (one syncUp/syncDown pass
|
|
48
|
+
// that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
|
|
49
|
+
// with the error when the round halted on a failure. Drives the consumer's
|
|
50
|
+
// sync-health tracking; empty/no-op rounds report nothing.
|
|
51
|
+
private onSyncOutcome?: (ok: boolean, error?: unknown) => void,
|
|
52
|
+
// Reports each mutation the server accepted, once its outbox row is gone.
|
|
53
|
+
// Lets the consumer keep the row rendered until its membership arrives.
|
|
54
|
+
private onSettled?: (event: UpEvent) => void
|
|
20
55
|
) {}
|
|
21
56
|
|
|
22
|
-
async init() {
|
|
23
|
-
|
|
57
|
+
async init(opts: { loadOutbox?: boolean } = {}) {
|
|
58
|
+
// Shared-tabs followers skip the outbox load: the shared store's
|
|
59
|
+
// `_00_pending_mutations` is drained by exactly one tab (the leader).
|
|
60
|
+
if (opts.loadOutbox !== false) await this.upQueue.loadFromDatabase();
|
|
24
61
|
this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, this.syncUp.bind(this));
|
|
25
62
|
this.downQueue.events.subscribe(
|
|
26
63
|
SyncQueueEventTypes.QueryItemEnqueued,
|
|
@@ -44,18 +81,112 @@ export class SyncScheduler {
|
|
|
44
81
|
this.downQueue.push(event);
|
|
45
82
|
}
|
|
46
83
|
|
|
84
|
+
/**
|
|
85
|
+
* Suspend syncing for a local-bucket switch. Refuses new rounds and resolves
|
|
86
|
+
* once any in-flight round has finished — the pause point is BETWEEN queue
|
|
87
|
+
* items, never between an item's remote push and its outbox-row delete, so a
|
|
88
|
+
* processed mutation's `DELETE _00_pending_mutations` always lands in the
|
|
89
|
+
* store it was read from.
|
|
90
|
+
*/
|
|
91
|
+
pause(): Promise<void> {
|
|
92
|
+
this.paused = true;
|
|
93
|
+
this.clearRetryTimers();
|
|
94
|
+
if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
|
|
95
|
+
return new Promise<void>((resolve) => this.pauseWaiters.push(resolve));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
resume(): void {
|
|
99
|
+
this.paused = false;
|
|
100
|
+
// A resume is a fresh start, not a continuation of the failing streak.
|
|
101
|
+
this.upRetryAttempt = 0;
|
|
102
|
+
this.downRetryAttempt = 0;
|
|
103
|
+
void this.syncUp();
|
|
104
|
+
void this.syncDown();
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private maybeResolvePause() {
|
|
108
|
+
if (!this.paused || this.isSyncingUp || this.isSyncingDown) return;
|
|
109
|
+
const waiters = this.pauseWaiters;
|
|
110
|
+
this.pauseWaiters = [];
|
|
111
|
+
for (const resolve of waiters) resolve();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Exponential backoff, capped. Attempt 0 is the first retry. */
|
|
115
|
+
private retryDelay(attempt: number): number {
|
|
116
|
+
return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
private scheduleUpRetry() {
|
|
120
|
+
if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
|
|
121
|
+
const delay = this.retryDelay(this.upRetryAttempt++);
|
|
122
|
+
this.upRetryTimer = setTimeout(() => {
|
|
123
|
+
this.upRetryTimer = undefined;
|
|
124
|
+
void this.syncUp();
|
|
125
|
+
}, delay);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Re-arm the down pass. With no argument this is failure backoff and the
|
|
130
|
+
* streak grows; with an explicit delay it is a yield (the up-queue holds the
|
|
131
|
+
* floor), which is not a failure and must not push the backoff out.
|
|
132
|
+
*/
|
|
133
|
+
private scheduleDownRetry(delayMs?: number) {
|
|
134
|
+
if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
|
|
135
|
+
const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
|
|
136
|
+
this.downRetryTimer = setTimeout(() => {
|
|
137
|
+
this.downRetryTimer = undefined;
|
|
138
|
+
void this.syncDown();
|
|
139
|
+
}, delay);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
private clearRetryTimers() {
|
|
143
|
+
if (this.upRetryTimer) {
|
|
144
|
+
clearTimeout(this.upRetryTimer);
|
|
145
|
+
this.upRetryTimer = undefined;
|
|
146
|
+
}
|
|
147
|
+
if (this.downRetryTimer) {
|
|
148
|
+
clearTimeout(this.downRetryTimer);
|
|
149
|
+
this.downRetryTimer = undefined;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Stop all pending retries. Call when tearing the client down. */
|
|
154
|
+
dispose(): void {
|
|
155
|
+
this.paused = true;
|
|
156
|
+
this.clearRetryTimers();
|
|
157
|
+
}
|
|
158
|
+
|
|
47
159
|
/**
|
|
48
160
|
* Process upload queue
|
|
49
161
|
*/
|
|
50
162
|
async syncUp() {
|
|
51
|
-
if (this.isSyncingUp) return;
|
|
163
|
+
if (this.isSyncingUp || this.paused) return;
|
|
52
164
|
this.isSyncingUp = true;
|
|
165
|
+
let processedAny = false;
|
|
53
166
|
try {
|
|
54
|
-
while (this.upQueue.size > 0) {
|
|
55
|
-
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
167
|
+
while (this.upQueue.size > 0 && !this.paused) {
|
|
168
|
+
await this.upQueue.next(this.onProcessUp, this.onRollback, this.onSettled);
|
|
169
|
+
processedAny = true;
|
|
56
170
|
}
|
|
171
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
172
|
+
this.upRetryAttempt = 0;
|
|
173
|
+
} catch (error) {
|
|
174
|
+
this.onSyncOutcome?.(false, error);
|
|
175
|
+
this.scheduleUpRetry();
|
|
176
|
+
// syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
|
|
177
|
+
// (broadcast synchronously, return value dropped) and is also kicked off
|
|
178
|
+
// via `void this.syncDown()` below. A rejection escaping here therefore
|
|
179
|
+
// surfaces as an *unhandled promise rejection* in the console rather than
|
|
180
|
+
// anything a caller can catch. UpQueue.next already logs the failing item
|
|
181
|
+
// (and re-queues it for retry on the next trigger), so swallow here to
|
|
182
|
+
// keep the failure contained instead of leaking it globally.
|
|
183
|
+
this.logger.debug(
|
|
184
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
|
|
185
|
+
'syncUp halted on a queue error; item re-queued, will retry on next trigger'
|
|
186
|
+
);
|
|
57
187
|
} finally {
|
|
58
188
|
this.isSyncingUp = false;
|
|
189
|
+
this.maybeResolvePause();
|
|
59
190
|
void this.syncDown();
|
|
60
191
|
}
|
|
61
192
|
}
|
|
@@ -64,17 +195,88 @@ export class SyncScheduler {
|
|
|
64
195
|
* Process download queue
|
|
65
196
|
*/
|
|
66
197
|
async syncDown() {
|
|
67
|
-
if (this.isSyncingDown) return;
|
|
68
|
-
|
|
198
|
+
if (this.isSyncingDown || this.paused) return;
|
|
199
|
+
// Down-sync yields to a non-empty up-queue so a register never races ahead
|
|
200
|
+
// of the mutation it should observe. That yield used to be permanent: if
|
|
201
|
+
// the up-queue never drained, nothing re-armed the down pass. Come back on
|
|
202
|
+
// the backoff instead, so a wedged push delays reads rather than killing
|
|
203
|
+
// them.
|
|
204
|
+
if (this.upQueue.size > 0) {
|
|
205
|
+
this.scheduleDownRetry(RETRY_BASE_MS);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
69
208
|
|
|
70
209
|
this.isSyncingDown = true;
|
|
210
|
+
let processedAny = false;
|
|
211
|
+
// Hashes currently in flight. `takeNext` skips their events, so per-hash
|
|
212
|
+
// ordering holds while independent hashes run concurrently.
|
|
213
|
+
const busy = new Set<string>();
|
|
214
|
+
const inFlight = new Set<Promise<void>>();
|
|
215
|
+
// First failure of this pass. Once set we stop TAKING work (mirroring the
|
|
216
|
+
// old "halt the pass and retry on a backoff") but still let what is already
|
|
217
|
+
// in flight finish, rather than abandoning RPCs mid-round.
|
|
218
|
+
let failure: unknown | undefined;
|
|
71
219
|
try {
|
|
72
|
-
|
|
73
|
-
if (this.
|
|
74
|
-
|
|
220
|
+
for (;;) {
|
|
221
|
+
if (this.paused) break;
|
|
222
|
+
// Yield to the up queue: stop taking new down work so a pending
|
|
223
|
+
// mutation gets its turn. The old code `break`-ed out and came back
|
|
224
|
+
// only on a backoff, so a steady trickle of local writes could starve
|
|
225
|
+
// registration indefinitely; draining what is in flight and re-arming
|
|
226
|
+
// immediately interleaves the two instead.
|
|
227
|
+
const yieldToUp = this.upQueue.size > 0;
|
|
228
|
+
while (
|
|
229
|
+
failure === undefined &&
|
|
230
|
+
!yieldToUp &&
|
|
231
|
+
inFlight.size < MAX_CONCURRENT_DOWN
|
|
232
|
+
) {
|
|
233
|
+
const event = this.downQueue.takeNext(busy);
|
|
234
|
+
if (!event) break;
|
|
235
|
+
const hash = event.payload.hash;
|
|
236
|
+
busy.add(hash);
|
|
237
|
+
processedAny = true;
|
|
238
|
+
// `run` never rejects — it hands the error back — so one failing
|
|
239
|
+
// event cannot reject the `Promise.race` below and lose the rest.
|
|
240
|
+
const task: Promise<void> = this.downQueue
|
|
241
|
+
.run(event, this.onProcessDown)
|
|
242
|
+
.then((error) => {
|
|
243
|
+
if (error !== undefined && failure === undefined) failure = error;
|
|
244
|
+
})
|
|
245
|
+
.finally(() => {
|
|
246
|
+
busy.delete(hash);
|
|
247
|
+
inFlight.delete(task);
|
|
248
|
+
});
|
|
249
|
+
inFlight.add(task);
|
|
250
|
+
}
|
|
251
|
+
if (inFlight.size === 0) break;
|
|
252
|
+
// Wake as soon as ANY slot frees, so the pool refills continuously
|
|
253
|
+
// instead of in barriers.
|
|
254
|
+
await Promise.race(inFlight);
|
|
255
|
+
if (yieldToUp && inFlight.size === 0) break;
|
|
75
256
|
}
|
|
257
|
+
if (failure !== undefined) throw failure;
|
|
258
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
259
|
+
this.downRetryAttempt = 0;
|
|
260
|
+
// Re-arm if we stopped early for the up queue (or hit the pool cap with
|
|
261
|
+
// work still queued) — otherwise that work waits for a fresh enqueue.
|
|
262
|
+
if (this.downQueue.size > 0) this.scheduleDownRetry(RETRY_BASE_MS);
|
|
263
|
+
} catch (error) {
|
|
264
|
+
this.onSyncOutcome?.(false, error);
|
|
265
|
+
this.scheduleDownRetry();
|
|
266
|
+
// Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
|
|
267
|
+
// subscriber (and is also called via `void this.syncDown()`), so a thrown
|
|
268
|
+
// error here becomes an unhandled rejection. The canonical case is a
|
|
269
|
+
// transient remote 500 on `fn::query::register` — DownQueue.next logs it
|
|
270
|
+
// and re-queues the event at the head; we just stop draining this pass and
|
|
271
|
+
// let the next enqueue retry, without spamming the console with an
|
|
272
|
+
// "Uncaught (in promise) ... 500 Internal Server Error".
|
|
273
|
+
this.logger.debug(
|
|
274
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
|
|
275
|
+
'syncDown halted on a queue error; item re-queued, will retry on next trigger'
|
|
276
|
+
);
|
|
76
277
|
} finally {
|
|
77
278
|
this.isSyncingDown = false;
|
|
279
|
+
this.maybeResolvePause();
|
|
78
280
|
}
|
|
79
281
|
}
|
|
80
282
|
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { Sp00kySync } from './sync';
|
|
4
|
+
|
|
5
|
+
// Guards the teardown half of shared views. A `_00_query` row can be shared by
|
|
6
|
+
// several sessions of the same user, so releasing it MUST go through
|
|
7
|
+
// `fn::query::unsubscribe` — which drops only this session from `subscribers`
|
|
8
|
+
// and deletes the row when it was the last one. A bare `DELETE $id` here would
|
|
9
|
+
// destroy the view and every `_00_list_ref` edge hanging off it for every other
|
|
10
|
+
// live tab, which reads to the user as a list that silently goes blank.
|
|
11
|
+
//
|
|
12
|
+
// This was previously unenforceable: `_00_query` granted no delete permission,
|
|
13
|
+
// so the old `DELETE $id` affected zero rows and did nothing at all. Now that
|
|
14
|
+
// the table grants delete, a regression here is destructive rather than inert.
|
|
15
|
+
|
|
16
|
+
function makeSync(opts: { hasSubscribers?: boolean[] } = {}) {
|
|
17
|
+
const logger: any = {
|
|
18
|
+
child: () => logger,
|
|
19
|
+
debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {},
|
|
20
|
+
};
|
|
21
|
+
const remote: any = { query: vi.fn().mockResolvedValue([{ released: true, remaining: 0 }]) };
|
|
22
|
+
const queryId = new RecordId('_00_query', 'h1');
|
|
23
|
+
const queryState: any = { config: { id: queryId } };
|
|
24
|
+
|
|
25
|
+
// Successive hasSubscribers() answers: [before the release, after it].
|
|
26
|
+
const answers = opts.hasSubscribers ?? [false, false];
|
|
27
|
+
let call = 0;
|
|
28
|
+
const hasSubscribers = vi.fn(() => answers[Math.min(call++, answers.length - 1)]);
|
|
29
|
+
|
|
30
|
+
const finalizeDeregister = vi.fn();
|
|
31
|
+
const dataModule: any = {
|
|
32
|
+
getQueryByHash: vi.fn().mockReturnValue(queryState),
|
|
33
|
+
hasSubscribers,
|
|
34
|
+
finalizeDeregister,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const sync = new Sp00kySync(
|
|
38
|
+
{} as any, remote, {} as any, dataModule, {} as any, logger,
|
|
39
|
+
);
|
|
40
|
+
const enqueueDownEvent = vi.fn();
|
|
41
|
+
(sync as any).enqueueDownEvent = enqueueDownEvent;
|
|
42
|
+
|
|
43
|
+
const run = (hash: string) => (sync as any).cleanupQuery(hash) as Promise<void>;
|
|
44
|
+
return { remote, dataModule, queryId, finalizeDeregister, enqueueDownEvent, run };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
describe('cleanupQuery — releasing a possibly-shared view', () => {
|
|
48
|
+
beforeEach(() => vi.clearAllMocks());
|
|
49
|
+
|
|
50
|
+
it('does not touch the remote view at all; the TTL sweep reclaims it', async () => {
|
|
51
|
+
// Eager release is deliberately disabled. It was inert for months (the
|
|
52
|
+
// table granted no delete permission), and making it real turned every
|
|
53
|
+
// best-effort guard misfire into a live delete of the row and all its
|
|
54
|
+
// edges. TTL remains the only reclamation that has actually run.
|
|
55
|
+
const { remote, run } = makeSync();
|
|
56
|
+
|
|
57
|
+
await run('h1');
|
|
58
|
+
|
|
59
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('never issues a bare DELETE', async () => {
|
|
63
|
+
// Belt and braces: if the eager path is ever re-enabled, it must go through
|
|
64
|
+
// the refcounted `fn::query::unsubscribe`, never a raw DELETE, which would
|
|
65
|
+
// tear the view out from under other sessions sharing the row.
|
|
66
|
+
const { remote, run } = makeSync();
|
|
67
|
+
|
|
68
|
+
await run('h1');
|
|
69
|
+
|
|
70
|
+
const sql = remote.query.mock.calls.map((c: any[]) => c[0]).join('\n');
|
|
71
|
+
expect(sql).not.toMatch(/\bDELETE\b/i);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('frees local state once released', async () => {
|
|
75
|
+
const { finalizeDeregister, enqueueDownEvent, run } = makeSync();
|
|
76
|
+
|
|
77
|
+
await run('h1');
|
|
78
|
+
|
|
79
|
+
expect(finalizeDeregister).toHaveBeenCalledWith('h1');
|
|
80
|
+
expect(enqueueDownEvent).not.toHaveBeenCalled();
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('does not touch the remote row if a subscriber reappeared before the release', async () => {
|
|
84
|
+
const { remote, finalizeDeregister, run } = makeSync({
|
|
85
|
+
hasSubscribers: [true],
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
await run('h1');
|
|
89
|
+
|
|
90
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
91
|
+
expect(finalizeDeregister).not.toHaveBeenCalled();
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('still frees local state when a subscriber reappears mid-cleanup', async () => {
|
|
95
|
+
// With no remote round trip there is no window to lose a re-subscribe in,
|
|
96
|
+
// so this collapses to the ordinary local free. The remote view survives
|
|
97
|
+
// regardless (TTL owns it), which is precisely why the reappearing
|
|
98
|
+
// subscriber is safe: a re-register finds the row still there.
|
|
99
|
+
const { remote, finalizeDeregister, run } = makeSync({
|
|
100
|
+
hasSubscribers: [false, true],
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
await run('h1');
|
|
104
|
+
|
|
105
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
106
|
+
expect(finalizeDeregister).toHaveBeenCalledWith('h1');
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
it('is tolerant of an already torn-down query', async () => {
|
|
110
|
+
const { remote, dataModule, run } = makeSync();
|
|
111
|
+
dataModule.getQueryByHash.mockReturnValue(undefined);
|
|
112
|
+
|
|
113
|
+
await expect(run('gone')).resolves.toBeUndefined();
|
|
114
|
+
expect(remote.query).not.toHaveBeenCalled();
|
|
115
|
+
});
|
|
116
|
+
});
|