@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2514 -58
  3. package/dist/index.js +12561 -2449
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +101 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +127 -24
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +164 -82
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +180 -0
  65. package/src/modules/sync/queue/queue-down.ts +80 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1726 -99
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -17
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +110 -14
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1693 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -0,0 +1,237 @@
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
2
+ import { SyncScheduler } from './scheduler';
3
+ import type { UpQueue, DownQueue, DownEvent, UpEvent } from './queue/index';
4
+
5
+ // A queue whose drain throws re-queues the failing item at the HEAD and stops
6
+ // the pass (see DownQueue.next). Nothing used to re-arm it: the queues only
7
+ // moved on a fresh enqueue, so one transient failure — canonically the SSP
8
+ // answering 503 NOT_READY for the whole of its bootstrap window — parked every
9
+ // pending `register` forever, and the `useQuery` waiting on it never left its
10
+ // loading state. These cover the backoff that makes that self-heal.
11
+
12
+ const silentLogger = {
13
+ child: () => silentLogger,
14
+ debug: () => {},
15
+ info: () => {},
16
+ warn: () => {},
17
+ error: () => {},
18
+ } as any;
19
+
20
+ /** A queue that mirrors the real ones: a throwing handler re-heads the item. */
21
+ function makeQueue<E>(items: E[]) {
22
+ return {
23
+ queue: [...items],
24
+ get size() {
25
+ return this.queue.length;
26
+ },
27
+ events: { subscribe: () => {} },
28
+ loadFromDatabase: async () => {},
29
+ clear() {
30
+ this.queue = [];
31
+ },
32
+ async next(fn: (event: E) => Promise<void>) {
33
+ const event = this.queue.shift();
34
+ if (!event) return;
35
+ const err = await this.run(event, fn);
36
+ if (err !== undefined) throw err;
37
+ },
38
+ // The concurrent down drain takes events itself and runs them, so the fake
39
+ // has to mirror that half of the real DownQueue too: skip events whose hash
40
+ // is already in flight, and hand the error back rather than throwing.
41
+ takeNext(busy: ReadonlySet<string>) {
42
+ for (let i = 0; i < this.queue.length; i++) {
43
+ const event = this.queue[i] as unknown as { payload?: { hash?: string } };
44
+ const hash = event?.payload?.hash;
45
+ if (hash !== undefined && busy.has(hash)) continue;
46
+ return this.queue.splice(i, 1)[0];
47
+ }
48
+ return undefined;
49
+ },
50
+ async run(event: E, fn: (event: E) => Promise<void>) {
51
+ try {
52
+ await fn(event);
53
+ return undefined;
54
+ } catch (err) {
55
+ this.queue.unshift(event);
56
+ return err;
57
+ }
58
+ },
59
+ };
60
+ }
61
+
62
+ const downEvent = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
63
+ const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
64
+
65
+ describe('SyncScheduler retry', () => {
66
+ beforeEach(() => vi.useFakeTimers());
67
+ afterEach(() => vi.useRealTimers());
68
+
69
+ it('re-drains a failed down event instead of parking it forever', async () => {
70
+ const downQueue = makeQueue([downEvent('q1')]);
71
+ const upQueue = makeQueue<UpEvent>([]);
72
+ let attempts = 0;
73
+ const scheduler = new SyncScheduler(
74
+ upQueue as unknown as UpQueue,
75
+ downQueue as unknown as DownQueue,
76
+ async () => {},
77
+ async () => {
78
+ attempts++;
79
+ // Fail the way a bootstrapping SSP does, then succeed.
80
+ if (attempts < 3) throw new Error('503 NOT_READY');
81
+ },
82
+ silentLogger
83
+ );
84
+
85
+ await scheduler.syncDown();
86
+ expect(attempts).toBe(1);
87
+ expect(downQueue.size).toBe(1); // re-headed, not dropped
88
+
89
+ // Backoff: 500ms, then 1000ms.
90
+ await vi.advanceTimersByTimeAsync(500);
91
+ expect(attempts).toBe(2);
92
+ await vi.advanceTimersByTimeAsync(1000);
93
+ expect(attempts).toBe(3);
94
+
95
+ expect(downQueue.size).toBe(0);
96
+ });
97
+
98
+ it('stops retrying once the queue drains', async () => {
99
+ const downQueue = makeQueue([downEvent('q1')]);
100
+ const upQueue = makeQueue<UpEvent>([]);
101
+ let attempts = 0;
102
+ const scheduler = new SyncScheduler(
103
+ upQueue as unknown as UpQueue,
104
+ downQueue as unknown as DownQueue,
105
+ async () => {},
106
+ async () => {
107
+ attempts++;
108
+ throw new Error('boom');
109
+ },
110
+ silentLogger
111
+ );
112
+
113
+ await scheduler.syncDown();
114
+ await vi.advanceTimersByTimeAsync(500);
115
+ expect(attempts).toBe(2);
116
+
117
+ // Drain it out from under the scheduler; the next retry finds nothing and
118
+ // schedules no further work.
119
+ downQueue.clear();
120
+ await vi.advanceTimersByTimeAsync(1000);
121
+ expect(attempts).toBe(2);
122
+ await vi.advanceTimersByTimeAsync(60_000);
123
+ expect(attempts).toBe(2);
124
+ });
125
+
126
+ it('comes back for the down queue while the up queue holds the floor', async () => {
127
+ const downQueue = makeQueue([downEvent('q1')]);
128
+ const upQueue = makeQueue<UpEvent>([upEvent(1)]);
129
+ let downAttempts = 0;
130
+ const scheduler = new SyncScheduler(
131
+ upQueue as unknown as UpQueue,
132
+ downQueue as unknown as DownQueue,
133
+ async () => {},
134
+ async () => {
135
+ downAttempts++;
136
+ },
137
+ silentLogger
138
+ );
139
+
140
+ // Yields to the non-empty up queue.
141
+ await scheduler.syncDown();
142
+ expect(downAttempts).toBe(0);
143
+
144
+ // Once the up queue empties, the re-armed pass picks the down event up
145
+ // without needing a fresh enqueue.
146
+ upQueue.clear();
147
+ await vi.advanceTimersByTimeAsync(500);
148
+ expect(downAttempts).toBe(1);
149
+ });
150
+
151
+ it('pause cancels pending retries', async () => {
152
+ const downQueue = makeQueue([downEvent('q1')]);
153
+ const upQueue = makeQueue<UpEvent>([]);
154
+ let attempts = 0;
155
+ const scheduler = new SyncScheduler(
156
+ upQueue as unknown as UpQueue,
157
+ downQueue as unknown as DownQueue,
158
+ async () => {},
159
+ async () => {
160
+ attempts++;
161
+ throw new Error('boom');
162
+ },
163
+ silentLogger
164
+ );
165
+
166
+ await scheduler.syncDown();
167
+ expect(attempts).toBe(1);
168
+
169
+ await scheduler.pause();
170
+ await vi.advanceTimersByTimeAsync(60_000);
171
+ expect(attempts).toBe(1);
172
+ });
173
+ });
174
+
175
+ describe('SyncScheduler down concurrency', () => {
176
+ beforeEach(() => vi.useFakeTimers());
177
+ afterEach(() => vi.useRealTimers());
178
+
179
+ /** Scheduler over a down queue of `hashes`, whose handler blocks until the
180
+ * returned `release` is called for that hash. Lets a test observe exactly
181
+ * which events are in flight at the same moment. */
182
+ function makeConcurrent(hashes: string[]) {
183
+ const downQueue = makeQueue(hashes.map(downEvent));
184
+ const upQueue = makeQueue<UpEvent>([]);
185
+ const started: string[] = [];
186
+ const gates = new Map<string, () => void>();
187
+ const scheduler = new SyncScheduler(
188
+ upQueue as unknown as UpQueue,
189
+ downQueue as unknown as DownQueue,
190
+ async () => {},
191
+ async (event) => {
192
+ const hash = (event as { payload: { hash: string } }).payload.hash;
193
+ started.push(hash);
194
+ await new Promise<void>((resolve) => gates.set(hash, resolve));
195
+ },
196
+ silentLogger
197
+ );
198
+ return { scheduler, started, release: (h: string) => gates.get(h)?.() };
199
+ }
200
+
201
+ it('runs several independent hashes at once instead of one at a time', async () => {
202
+ // The bug this whole change exists for: strictly serial draining meant one
203
+ // registration RPC at a time for the entire client.
204
+ const { scheduler, started, release } = makeConcurrent(['a', 'b', 'c', 'd', 'e']);
205
+ const pass = scheduler.syncDown();
206
+ await vi.advanceTimersByTimeAsync(0);
207
+
208
+ // Bounded by MAX_CONCURRENT_DOWN (4), not 1.
209
+ expect(started).toEqual(['a', 'b', 'c', 'd']);
210
+
211
+ // Freeing one slot immediately admits the next event.
212
+ release('a');
213
+ await vi.advanceTimersByTimeAsync(0);
214
+ expect(started).toEqual(['a', 'b', 'c', 'd', 'e']);
215
+
216
+ for (const h of ['b', 'c', 'd', 'e']) release(h);
217
+ await pass;
218
+ });
219
+
220
+ it('never overlaps two events for the SAME hash', async () => {
221
+ // Per-hash ordering is the one invariant the serial queue was providing.
222
+ const { scheduler, started, release } = makeConcurrent(['a', 'a', 'b']);
223
+ const pass = scheduler.syncDown();
224
+ await vi.advanceTimersByTimeAsync(0);
225
+
226
+ // The second 'a' is skipped while the first is in flight; 'b' goes ahead.
227
+ expect(started).toEqual(['a', 'b']);
228
+
229
+ release('a');
230
+ await vi.advanceTimersByTimeAsync(0);
231
+ expect(started).toEqual(['a', 'b', 'a']);
232
+
233
+ release('b');
234
+ release('a');
235
+ await pass;
236
+ });
237
+ });
@@ -1,14 +1,41 @@
1
- import { Logger } from '../../services/logger/index';
2
- import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
1
+ import type { Logger } from '../../services/logger/index';
2
+ import type { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
3
3
  import { SyncQueueEventTypes } from './events/index';
4
4
 
5
5
  /**
6
6
  * SyncScheduler manages when to sync: queue management and orchestration.
7
7
  * Decides the order and timing of sync operations.
8
8
  */
9
+ /** Backoff for re-draining a queue that halted on an error. */
10
+ const RETRY_BASE_MS = 500;
11
+ /**
12
+ * How many down events may be in flight at once.
13
+ *
14
+ * The down queue was strictly serial: one `register`/`sync`/`cleanup` RPC at a
15
+ * time for the WHOLE client. Measured in production that drained at roughly one
16
+ * event per 8.8s, so a list that registers a query per scrolled-to window took
17
+ * minutes to fill — the rows were already cached, only the registration lagged.
18
+ * Ordering is a per-hash requirement, not a global one (see `takeNext`), so
19
+ * independent hashes can go in parallel. Bounded to stay polite to the SSP.
20
+ */
21
+ const MAX_CONCURRENT_DOWN = 4;
22
+ const RETRY_MAX_MS = 15_000;
23
+
9
24
  export class SyncScheduler {
10
25
  private isSyncingUp: boolean = false;
11
26
  private isSyncingDown: boolean = false;
27
+ private paused: boolean = false;
28
+ private pauseWaiters: Array<() => void> = [];
29
+ // A failed drain re-queues its item at the HEAD (see DownQueue.next) and
30
+ // stops the pass. Without a timer nothing ever drains it again: the queues
31
+ // only move on a fresh enqueue, so a transient failure — canonically the
32
+ // SSP answering 503 NOT_READY for the whole of its bootstrap window — left
33
+ // every pending `register` parked forever and its `useQuery` loading forever.
34
+ // Retry on a backoff so that heals itself instead of needing a reload.
35
+ private upRetryTimer?: ReturnType<typeof setTimeout>;
36
+ private downRetryTimer?: ReturnType<typeof setTimeout>;
37
+ private upRetryAttempt = 0;
38
+ private downRetryAttempt = 0;
12
39
 
13
40
  constructor(
14
41
  private upQueue: UpQueue,
@@ -16,11 +43,21 @@ export class SyncScheduler {
16
43
  private onProcessUp: (event: UpEvent) => Promise<void>,
17
44
  private onProcessDown: (event: DownEvent) => Promise<void>,
18
45
  private logger: Logger,
19
- private onRollback?: RollbackCallback
46
+ private onRollback?: RollbackCallback,
47
+ // Reports the outcome of each drained sync round (one syncUp/syncDown pass
48
+ // that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
49
+ // with the error when the round halted on a failure. Drives the consumer's
50
+ // sync-health tracking; empty/no-op rounds report nothing.
51
+ private onSyncOutcome?: (ok: boolean, error?: unknown) => void,
52
+ // Reports each mutation the server accepted, once its outbox row is gone.
53
+ // Lets the consumer keep the row rendered until its membership arrives.
54
+ private onSettled?: (event: UpEvent) => void
20
55
  ) {}
21
56
 
22
- async init() {
23
- await this.upQueue.loadFromDatabase();
57
+ async init(opts: { loadOutbox?: boolean } = {}) {
58
+ // Shared-tabs followers skip the outbox load: the shared store's
59
+ // `_00_pending_mutations` is drained by exactly one tab (the leader).
60
+ if (opts.loadOutbox !== false) await this.upQueue.loadFromDatabase();
24
61
  this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, this.syncUp.bind(this));
25
62
  this.downQueue.events.subscribe(
26
63
  SyncQueueEventTypes.QueryItemEnqueued,
@@ -44,18 +81,112 @@ export class SyncScheduler {
44
81
  this.downQueue.push(event);
45
82
  }
46
83
 
84
+ /**
85
+ * Suspend syncing for a local-bucket switch. Refuses new rounds and resolves
86
+ * once any in-flight round has finished — the pause point is BETWEEN queue
87
+ * items, never between an item's remote push and its outbox-row delete, so a
88
+ * processed mutation's `DELETE _00_pending_mutations` always lands in the
89
+ * store it was read from.
90
+ */
91
+ pause(): Promise<void> {
92
+ this.paused = true;
93
+ this.clearRetryTimers();
94
+ if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
95
+ return new Promise<void>((resolve) => this.pauseWaiters.push(resolve));
96
+ }
97
+
98
+ resume(): void {
99
+ this.paused = false;
100
+ // A resume is a fresh start, not a continuation of the failing streak.
101
+ this.upRetryAttempt = 0;
102
+ this.downRetryAttempt = 0;
103
+ void this.syncUp();
104
+ void this.syncDown();
105
+ }
106
+
107
+ private maybeResolvePause() {
108
+ if (!this.paused || this.isSyncingUp || this.isSyncingDown) return;
109
+ const waiters = this.pauseWaiters;
110
+ this.pauseWaiters = [];
111
+ for (const resolve of waiters) resolve();
112
+ }
113
+
114
+ /** Exponential backoff, capped. Attempt 0 is the first retry. */
115
+ private retryDelay(attempt: number): number {
116
+ return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
117
+ }
118
+
119
+ private scheduleUpRetry() {
120
+ if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
121
+ const delay = this.retryDelay(this.upRetryAttempt++);
122
+ this.upRetryTimer = setTimeout(() => {
123
+ this.upRetryTimer = undefined;
124
+ void this.syncUp();
125
+ }, delay);
126
+ }
127
+
128
+ /**
129
+ * Re-arm the down pass. With no argument this is failure backoff and the
130
+ * streak grows; with an explicit delay it is a yield (the up-queue holds the
131
+ * floor), which is not a failure and must not push the backoff out.
132
+ */
133
+ private scheduleDownRetry(delayMs?: number) {
134
+ if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
135
+ const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
136
+ this.downRetryTimer = setTimeout(() => {
137
+ this.downRetryTimer = undefined;
138
+ void this.syncDown();
139
+ }, delay);
140
+ }
141
+
142
+ private clearRetryTimers() {
143
+ if (this.upRetryTimer) {
144
+ clearTimeout(this.upRetryTimer);
145
+ this.upRetryTimer = undefined;
146
+ }
147
+ if (this.downRetryTimer) {
148
+ clearTimeout(this.downRetryTimer);
149
+ this.downRetryTimer = undefined;
150
+ }
151
+ }
152
+
153
+ /** Stop all pending retries. Call when tearing the client down. */
154
+ dispose(): void {
155
+ this.paused = true;
156
+ this.clearRetryTimers();
157
+ }
158
+
47
159
  /**
48
160
  * Process upload queue
49
161
  */
50
162
  async syncUp() {
51
- if (this.isSyncingUp) return;
163
+ if (this.isSyncingUp || this.paused) return;
52
164
  this.isSyncingUp = true;
165
+ let processedAny = false;
53
166
  try {
54
- while (this.upQueue.size > 0) {
55
- await this.upQueue.next(this.onProcessUp, this.onRollback);
167
+ while (this.upQueue.size > 0 && !this.paused) {
168
+ await this.upQueue.next(this.onProcessUp, this.onRollback, this.onSettled);
169
+ processedAny = true;
56
170
  }
171
+ if (processedAny) this.onSyncOutcome?.(true);
172
+ this.upRetryAttempt = 0;
173
+ } catch (error) {
174
+ this.onSyncOutcome?.(false, error);
175
+ this.scheduleUpRetry();
176
+ // syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
177
+ // (broadcast synchronously, return value dropped) and is also kicked off
178
+ // via `void this.syncDown()` below. A rejection escaping here therefore
179
+ // surfaces as an *unhandled promise rejection* in the console rather than
180
+ // anything a caller can catch. UpQueue.next already logs the failing item
181
+ // (and re-queues it for retry on the next trigger), so swallow here to
182
+ // keep the failure contained instead of leaking it globally.
183
+ this.logger.debug(
184
+ { error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
185
+ 'syncUp halted on a queue error; item re-queued, will retry on next trigger'
186
+ );
57
187
  } finally {
58
188
  this.isSyncingUp = false;
189
+ this.maybeResolvePause();
59
190
  void this.syncDown();
60
191
  }
61
192
  }
@@ -64,17 +195,88 @@ export class SyncScheduler {
64
195
  * Process download queue
65
196
  */
66
197
  async syncDown() {
67
- if (this.isSyncingDown) return;
68
- if (this.upQueue.size > 0) return;
198
+ if (this.isSyncingDown || this.paused) return;
199
+ // Down-sync yields to a non-empty up-queue so a register never races ahead
200
+ // of the mutation it should observe. That yield used to be permanent: if
201
+ // the up-queue never drained, nothing re-armed the down pass. Come back on
202
+ // the backoff instead, so a wedged push delays reads rather than killing
203
+ // them.
204
+ if (this.upQueue.size > 0) {
205
+ this.scheduleDownRetry(RETRY_BASE_MS);
206
+ return;
207
+ }
69
208
 
70
209
  this.isSyncingDown = true;
210
+ let processedAny = false;
211
+ // Hashes currently in flight. `takeNext` skips their events, so per-hash
212
+ // ordering holds while independent hashes run concurrently.
213
+ const busy = new Set<string>();
214
+ const inFlight = new Set<Promise<void>>();
215
+ // First failure of this pass. Once set we stop TAKING work (mirroring the
216
+ // old "halt the pass and retry on a backoff") but still let what is already
217
+ // in flight finish, rather than abandoning RPCs mid-round.
218
+ let failure: unknown | undefined;
71
219
  try {
72
- while (this.downQueue.size > 0) {
73
- if (this.upQueue.size > 0) break;
74
- await this.downQueue.next(this.onProcessDown);
220
+ for (;;) {
221
+ if (this.paused) break;
222
+ // Yield to the up queue: stop taking new down work so a pending
223
+ // mutation gets its turn. The old code `break`-ed out and came back
224
+ // only on a backoff, so a steady trickle of local writes could starve
225
+ // registration indefinitely; draining what is in flight and re-arming
226
+ // immediately interleaves the two instead.
227
+ const yieldToUp = this.upQueue.size > 0;
228
+ while (
229
+ failure === undefined &&
230
+ !yieldToUp &&
231
+ inFlight.size < MAX_CONCURRENT_DOWN
232
+ ) {
233
+ const event = this.downQueue.takeNext(busy);
234
+ if (!event) break;
235
+ const hash = event.payload.hash;
236
+ busy.add(hash);
237
+ processedAny = true;
238
+ // `run` never rejects — it hands the error back — so one failing
239
+ // event cannot reject the `Promise.race` below and lose the rest.
240
+ const task: Promise<void> = this.downQueue
241
+ .run(event, this.onProcessDown)
242
+ .then((error) => {
243
+ if (error !== undefined && failure === undefined) failure = error;
244
+ })
245
+ .finally(() => {
246
+ busy.delete(hash);
247
+ inFlight.delete(task);
248
+ });
249
+ inFlight.add(task);
250
+ }
251
+ if (inFlight.size === 0) break;
252
+ // Wake as soon as ANY slot frees, so the pool refills continuously
253
+ // instead of in barriers.
254
+ await Promise.race(inFlight);
255
+ if (yieldToUp && inFlight.size === 0) break;
75
256
  }
257
+ if (failure !== undefined) throw failure;
258
+ if (processedAny) this.onSyncOutcome?.(true);
259
+ this.downRetryAttempt = 0;
260
+ // Re-arm if we stopped early for the up queue (or hit the pool cap with
261
+ // work still queued) — otherwise that work waits for a fresh enqueue.
262
+ if (this.downQueue.size > 0) this.scheduleDownRetry(RETRY_BASE_MS);
263
+ } catch (error) {
264
+ this.onSyncOutcome?.(false, error);
265
+ this.scheduleDownRetry();
266
+ // Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
267
+ // subscriber (and is also called via `void this.syncDown()`), so a thrown
268
+ // error here becomes an unhandled rejection. The canonical case is a
269
+ // transient remote 500 on `fn::query::register` — DownQueue.next logs it
270
+ // and re-queues the event at the head; we just stop draining this pass and
271
+ // let the next enqueue retry, without spamming the console with an
272
+ // "Uncaught (in promise) ... 500 Internal Server Error".
273
+ this.logger.debug(
274
+ { error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
275
+ 'syncDown halted on a queue error; item re-queued, will retry on next trigger'
276
+ );
76
277
  } finally {
77
278
  this.isSyncingDown = false;
279
+ this.maybeResolvePause();
78
280
  }
79
281
  }
80
282
 
@@ -0,0 +1,116 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { Sp00kySync } from './sync';
4
+
5
+ // Guards the teardown half of shared views. A `_00_query` row can be shared by
6
+ // several sessions of the same user, so releasing it MUST go through
7
+ // `fn::query::unsubscribe` — which drops only this session from `subscribers`
8
+ // and deletes the row when it was the last one. A bare `DELETE $id` here would
9
+ // destroy the view and every `_00_list_ref` edge hanging off it for every other
10
+ // live tab, which reads to the user as a list that silently goes blank.
11
+ //
12
+ // This was previously unenforceable: `_00_query` granted no delete permission,
13
+ // so the old `DELETE $id` affected zero rows and did nothing at all. Now that
14
+ // the table grants delete, a regression here is destructive rather than inert.
15
+
16
+ function makeSync(opts: { hasSubscribers?: boolean[] } = {}) {
17
+ const logger: any = {
18
+ child: () => logger,
19
+ debug: () => {}, info: () => {}, warn: () => {}, error: () => {}, trace: () => {},
20
+ };
21
+ const remote: any = { query: vi.fn().mockResolvedValue([{ released: true, remaining: 0 }]) };
22
+ const queryId = new RecordId('_00_query', 'h1');
23
+ const queryState: any = { config: { id: queryId } };
24
+
25
+ // Successive hasSubscribers() answers: [before the release, after it].
26
+ const answers = opts.hasSubscribers ?? [false, false];
27
+ let call = 0;
28
+ const hasSubscribers = vi.fn(() => answers[Math.min(call++, answers.length - 1)]);
29
+
30
+ const finalizeDeregister = vi.fn();
31
+ const dataModule: any = {
32
+ getQueryByHash: vi.fn().mockReturnValue(queryState),
33
+ hasSubscribers,
34
+ finalizeDeregister,
35
+ };
36
+
37
+ const sync = new Sp00kySync(
38
+ {} as any, remote, {} as any, dataModule, {} as any, logger,
39
+ );
40
+ const enqueueDownEvent = vi.fn();
41
+ (sync as any).enqueueDownEvent = enqueueDownEvent;
42
+
43
+ const run = (hash: string) => (sync as any).cleanupQuery(hash) as Promise<void>;
44
+ return { remote, dataModule, queryId, finalizeDeregister, enqueueDownEvent, run };
45
+ }
46
+
47
+ describe('cleanupQuery — releasing a possibly-shared view', () => {
48
+ beforeEach(() => vi.clearAllMocks());
49
+
50
+ it('does not touch the remote view at all; the TTL sweep reclaims it', async () => {
51
+ // Eager release is deliberately disabled. It was inert for months (the
52
+ // table granted no delete permission), and making it real turned every
53
+ // best-effort guard misfire into a live delete of the row and all its
54
+ // edges. TTL remains the only reclamation that has actually run.
55
+ const { remote, run } = makeSync();
56
+
57
+ await run('h1');
58
+
59
+ expect(remote.query).not.toHaveBeenCalled();
60
+ });
61
+
62
+ it('never issues a bare DELETE', async () => {
63
+ // Belt and braces: if the eager path is ever re-enabled, it must go through
64
+ // the refcounted `fn::query::unsubscribe`, never a raw DELETE, which would
65
+ // tear the view out from under other sessions sharing the row.
66
+ const { remote, run } = makeSync();
67
+
68
+ await run('h1');
69
+
70
+ const sql = remote.query.mock.calls.map((c: any[]) => c[0]).join('\n');
71
+ expect(sql).not.toMatch(/\bDELETE\b/i);
72
+ });
73
+
74
+ it('frees local state once released', async () => {
75
+ const { finalizeDeregister, enqueueDownEvent, run } = makeSync();
76
+
77
+ await run('h1');
78
+
79
+ expect(finalizeDeregister).toHaveBeenCalledWith('h1');
80
+ expect(enqueueDownEvent).not.toHaveBeenCalled();
81
+ });
82
+
83
+ it('does not touch the remote row if a subscriber reappeared before the release', async () => {
84
+ const { remote, finalizeDeregister, run } = makeSync({
85
+ hasSubscribers: [true],
86
+ });
87
+
88
+ await run('h1');
89
+
90
+ expect(remote.query).not.toHaveBeenCalled();
91
+ expect(finalizeDeregister).not.toHaveBeenCalled();
92
+ });
93
+
94
+ it('still frees local state when a subscriber reappears mid-cleanup', async () => {
95
+ // With no remote round trip there is no window to lose a re-subscribe in,
96
+ // so this collapses to the ordinary local free. The remote view survives
97
+ // regardless (TTL owns it), which is precisely why the reappearing
98
+ // subscriber is safe: a re-register finds the row still there.
99
+ const { remote, finalizeDeregister, run } = makeSync({
100
+ hasSubscribers: [false, true],
101
+ });
102
+
103
+ await run('h1');
104
+
105
+ expect(remote.query).not.toHaveBeenCalled();
106
+ expect(finalizeDeregister).toHaveBeenCalledWith('h1');
107
+ });
108
+
109
+ it('is tolerant of an already torn-down query', async () => {
110
+ const { remote, dataModule, run } = makeSync();
111
+ dataModule.getQueryByHash.mockReturnValue(undefined);
112
+
113
+ await expect(run('gone')).resolves.toBeUndefined();
114
+ expect(remote.query).not.toHaveBeenCalled();
115
+ });
116
+ });