@spooky-sync/core 0.0.1-canary.17 → 0.0.1-canary.170

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2046 -365
  3. package/dist/index.js +11016 -2476
  4. package/dist/otel/index.d.ts +21 -0
  5. package/dist/otel/index.js +86 -0
  6. package/dist/sqlite-open.js +276 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +420 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +434 -0
  11. package/dist/types.d.ts +997 -0
  12. package/package.json +14 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +94 -18
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +4 -4
  17. package/src/build-globals.d.ts +12 -0
  18. package/src/events/events.test.ts +2 -1
  19. package/src/events/index.ts +3 -0
  20. package/src/index.ts +24 -2
  21. package/src/modules/app-release/index.test.ts +125 -0
  22. package/src/modules/app-release/index.ts +201 -0
  23. package/src/modules/auth/events/index.ts +2 -1
  24. package/src/modules/auth/index.ts +59 -20
  25. package/src/modules/cache/index.ts +112 -32
  26. package/src/modules/cache/types.ts +2 -2
  27. package/src/modules/crdt/crdt-field.ts +294 -0
  28. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  29. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  30. package/src/modules/crdt/index.ts +463 -0
  31. package/src/modules/crdt/loro-loader.ts +25 -0
  32. package/src/modules/data/data.hydration.test.ts +142 -0
  33. package/src/modules/data/data.membership.test.ts +297 -0
  34. package/src/modules/data/data.rebind.test.ts +147 -0
  35. package/src/modules/data/data.run.test.ts +113 -0
  36. package/src/modules/data/data.status.test.ts +249 -0
  37. package/src/modules/data/id-set-plan.test.ts +122 -0
  38. package/src/modules/data/index.ts +1403 -129
  39. package/src/modules/data/mutation-id.test.ts +25 -0
  40. package/src/modules/data/mutation-id.ts +35 -0
  41. package/src/modules/data/window-query.test.ts +52 -0
  42. package/src/modules/data/window-query.ts +194 -0
  43. package/src/modules/devtools/flags.ts +349 -0
  44. package/src/modules/devtools/index.ts +382 -37
  45. package/src/modules/devtools/notify-throttle.test.ts +149 -0
  46. package/src/modules/devtools/storage-info.test.ts +79 -0
  47. package/src/modules/devtools/storage-info.ts +168 -0
  48. package/src/modules/devtools/versions.test.ts +74 -0
  49. package/src/modules/devtools/versions.ts +81 -0
  50. package/src/modules/feature-flag/index.test.ts +251 -0
  51. package/src/modules/feature-flag/index.ts +308 -0
  52. package/src/modules/ref-tables.test.ts +91 -0
  53. package/src/modules/ref-tables.ts +88 -0
  54. package/src/modules/sync/engine.ts +101 -37
  55. package/src/modules/sync/events/index.ts +9 -2
  56. package/src/modules/sync/queue/queue-down.ts +12 -5
  57. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  58. package/src/modules/sync/queue/queue-up.ts +216 -56
  59. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  60. package/src/modules/sync/scheduler.ts +77 -9
  61. package/src/modules/sync/sync.health.test.ts +149 -0
  62. package/src/modules/sync/sync.live-removal.test.ts +134 -0
  63. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  64. package/src/modules/sync/sync.subquery.test.ts +82 -0
  65. package/src/modules/sync/sync.ts +1456 -92
  66. package/src/modules/sync/utils.test.ts +269 -2
  67. package/src/modules/sync/utils.ts +182 -17
  68. package/src/otel/index.ts +127 -0
  69. package/src/services/blobs/blob-cache.test.ts +359 -0
  70. package/src/services/blobs/blob-cache.ts +603 -0
  71. package/src/services/blobs/blob-manifest.ts +227 -0
  72. package/src/services/blobs/blob-store.test.ts +77 -0
  73. package/src/services/blobs/blob-store.ts +359 -0
  74. package/src/services/blobs/blob.fixture.ts +90 -0
  75. package/src/services/blobs/index.ts +70 -0
  76. package/src/services/database/cache-engine.ts +160 -0
  77. package/src/services/database/connection-supervisor.test.ts +289 -0
  78. package/src/services/database/connection-supervisor.ts +415 -0
  79. package/src/services/database/database.query-timeout.test.ts +83 -0
  80. package/src/services/database/database.ts +32 -12
  81. package/src/services/database/engine-factory.ts +33 -0
  82. package/src/services/database/events/index.ts +2 -1
  83. package/src/services/database/index.ts +7 -0
  84. package/src/services/database/local-migrator.ts +30 -27
  85. package/src/services/database/local.test.ts +64 -0
  86. package/src/services/database/local.ts +478 -67
  87. package/src/services/database/plan-render.test.ts +159 -0
  88. package/src/services/database/plan-render.ts +108 -0
  89. package/src/services/database/relation-resolver.test.ts +413 -0
  90. package/src/services/database/relation-resolver.ts +0 -0
  91. package/src/services/database/remote.ts +110 -14
  92. package/src/services/database/sqlite-cache-engine.test.ts +464 -0
  93. package/src/services/database/sqlite-cache-engine.ts +1159 -0
  94. package/src/services/database/sqlite-open.test.ts +150 -0
  95. package/src/services/database/sqlite-open.ts +164 -0
  96. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  97. package/src/services/database/sqlite-plan-sql.ts +106 -0
  98. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  99. package/src/services/database/sqlite-select.test.ts +246 -0
  100. package/src/services/database/sqlite-select.ts +113 -0
  101. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  102. package/src/services/database/sqlite-transport.ts +219 -0
  103. package/src/services/database/sqlite-worker.ts +437 -0
  104. package/src/services/database/surql-translate.ts +291 -0
  105. package/src/services/database/surreal-cache-engine.ts +141 -0
  106. package/src/services/logger/index.ts +6 -109
  107. package/src/services/persistence/localstorage.ts +2 -2
  108. package/src/services/persistence/resilient.ts +11 -4
  109. package/src/services/persistence/surrealdb.ts +10 -10
  110. package/src/services/stream-processor/index.ts +444 -52
  111. package/src/services/stream-processor/permissions.test.ts +47 -0
  112. package/src/services/stream-processor/permissions.ts +53 -0
  113. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  114. package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
  115. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  116. package/src/services/stream-processor/wasm-types.ts +23 -2
  117. package/src/services/tabs/broker-client.ts +283 -0
  118. package/src/services/tabs/broker.test.ts +278 -0
  119. package/src/services/tabs/coordinator.test.ts +244 -0
  120. package/src/services/tabs/coordinator.ts +576 -0
  121. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  122. package/src/services/tabs/leader-locks.ts +75 -0
  123. package/src/services/tabs/protocol.ts +242 -0
  124. package/src/services/tabs/support.ts +36 -0
  125. package/src/services/tabs/tabs-broker-worker.ts +586 -0
  126. package/src/sp00ky.auth-order.test.ts +92 -0
  127. package/src/sp00ky.init-query.test.ts +183 -0
  128. package/src/sp00ky.ts +1424 -0
  129. package/src/types.ts +467 -15
  130. package/src/utils/error-classification.test.ts +44 -0
  131. package/src/utils/error-classification.ts +7 -0
  132. package/src/utils/index.ts +73 -13
  133. package/src/utils/parser.ts +3 -2
  134. package/src/utils/semver.test.ts +32 -0
  135. package/src/utils/semver.ts +30 -0
  136. package/src/utils/surql.ts +30 -18
  137. package/src/utils/withRetry.test.ts +1 -1
  138. package/tsdown.config.ts +86 -1
  139. package/src/spooky.ts +0 -395
@@ -1,33 +1,202 @@
1
- import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
- import { MutationEvent, RecordVersionArray } from '../../types';
1
+ import type {
2
+ ConnectionSupervisor,
3
+ LocalStore,
4
+ RemoteDatabaseService,
5
+ } from '../../services/database/index';
6
+ import type {
7
+ ConnectionState,
8
+ RecordVersionArray,
9
+ RecordVersionDiff,
10
+ SyncHealth,
11
+ SyncHealthStatus,
12
+ } from '../../types';
3
13
  import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
4
- import { Logger } from '../../services/logger/index';
5
- import { DownEvent, DownQueue, UpEvent, UpQueue } from './queue/index';
6
- import { RecordId, Uuid } from 'surrealdb';
7
- import { ArraySyncer, createDiffFromDbOp } from './utils';
14
+ import type { Logger } from '../../services/logger/index';
15
+ import type { DownEvent, UpEvent } from './queue/index';
16
+ import { DownQueue, UpQueue } from './queue/index';
17
+ import type { RecordId, Uuid } from 'surrealdb';
18
+ import {
19
+ applyRecordVersionDiff,
20
+ ArraySyncer,
21
+ buildListRefSelect,
22
+ buildSubqueryListRefSelect,
23
+ createDiffFromDbOp,
24
+ diffRecordVersionArray,
25
+ listRefPollDelayMs,
26
+ recordVersionArraysEqual,
27
+ resolveListRefPollInterval,
28
+ } from './utils';
8
29
  import { SyncEngine } from './engine';
9
30
  import { SyncScheduler } from './scheduler';
10
- import { SchemaStructure } from '@spooky-sync/query-builder';
11
- import { CacheModule } from '../cache/index';
12
- import { DataModule } from '../data/index';
13
- import { encodeRecordId, extractTablePart, parseDuration, surql } from '../../utils/index';
31
+ import type { SchemaStructure } from '@spooky-sync/query-builder';
32
+ import type { CacheModule } from '../cache/index';
33
+ import type { DataModule } from '../data/index';
34
+ import {
35
+ classifySyncError,
36
+ encodeRecordId,
37
+ extractIdPart,
38
+ extractTablePart,
39
+ surql,
40
+ withTimeout,
41
+ } from '../../utils/index';
42
+ import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
43
+ import { mutationOwnerTabId } from '../data/mutation-id';
44
+ import type { LeaderSyncHub, SyncForwarder } from '../../services/tabs/coordinator';
45
+ import { parseRecordIdString } from '../../utils/index';
14
46
 
15
47
  /**
16
- * The main synchronization engine for Spooky.
48
+ * Tunables for `Sp00kySync` construction.
49
+ */
50
+ export interface Sp00kySyncOptions {
51
+ /**
52
+ * Cadence (ms) for the `_00_list_ref` poll fallback that catches
53
+ * cross-session UPDATEs the LIVE-permission gap drops. Non-positive
54
+ * values fall back to the default; see
55
+ * {@link resolveListRefPollInterval}.
56
+ */
57
+ refSyncIntervalMs?: number;
58
+ /**
59
+ * Enable realtime sync for unauthenticated clients against the shared
60
+ * `_00_list_ref_anon` table. See {@link Sp00kyConfig.enableAnonymousLiveQueries}.
61
+ * Defaults to `false`.
62
+ */
63
+ anonymousLiveQueries?: boolean;
64
+ /**
65
+ * Consecutive failed sync rounds before sync health flips to `degraded`.
66
+ * `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
67
+ * Defaults to `3`.
68
+ */
69
+ degradeAfterConsecutiveFailures?: number;
70
+ /**
71
+ * Max time a single mutation push may take before it is treated as a network
72
+ * failure and retried. Guards against an RPC that never settles wedging the
73
+ * up-queue for the session. Defaults to 30000; `0` disables the timeout.
74
+ */
75
+ pushTimeoutMs?: number;
76
+ /**
77
+ * Transport supervisor. Sync reads its state to report `connection` in
78
+ * {@link SyncHealth} so a UI can show "reconnecting…" the instant the socket
79
+ * drops, without waiting for the degrade threshold. Optional: omitted in
80
+ * tests, where `connection` then reports `connected`.
81
+ */
82
+ connectionSupervisor?: ConnectionSupervisor;
83
+ }
84
+
85
+ /**
86
+ * The main synchronization engine for Sp00ky.
17
87
  * Handles the bidirectional synchronization between the local database and the remote backend.
18
88
  * Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
19
89
  * @template S The schema structure type.
20
90
  */
21
- export class SpookySync<S extends SchemaStructure> {
22
- private clientId: string = '';
91
+ export class Sp00kySync<S extends SchemaStructure> {
23
92
  private upQueue: UpQueue;
24
93
  private downQueue: DownQueue;
25
94
  private isInit: boolean = false;
26
95
  private logger: Logger;
27
96
  private syncEngine: SyncEngine;
97
+ /** Engine-level events (e.g. `SYNC_REMOTE_DATA_INGESTED`). Distinct
98
+ * from `this.events`, which carries Sp00kySync-level events like
99
+ * `SYNC_QUERY_UPDATED` and `SYNC_MUTATION_ROLLED_BACK`. */
100
+ public get engineEvents() {
101
+ return this.syncEngine.events;
102
+ }
28
103
  private scheduler: SyncScheduler;
104
+ /**
105
+ * Set by any event that means the socket we registered on is gone, so the
106
+ * next `connected` knows it must re-subscribe rather than treat itself as the
107
+ * initial connect. See {@link subscribeToReconnect}.
108
+ */
109
+ private needsResubscribe: boolean = false;
110
+ /** When the last reconnect-driven full refetch ran, for burst coalescing. */
111
+ private lastReconnectRefetchAt = 0;
112
+ /**
113
+ * Minimum gap between reconnect-driven full refetches. Long enough to absorb
114
+ * a flapping socket (the SDK reconnect ladder starts at 1s), short enough
115
+ * that a genuine drop minutes later still refetches.
116
+ */
117
+ private static readonly RECONNECT_REFETCH_COOLDOWN_MS = 10_000;
29
118
  public events = createSyncEventSystem();
30
119
 
120
+ // Auth identity that drives per-user `_00_list_ref_user_<id>` routing
121
+ // in `RefMode.Dedicated`. Updated by `setCurrentUserId` from the auth
122
+ // subscription in `Sp00kyClient`; null when unauthenticated.
123
+ private currentUserId: string | null = null;
124
+
125
+ // ---- shared-tabs role state ----
126
+ // Followers keep their own remote WS (registration, per-query sync, poll)
127
+ // but must never drain the shared outbox or hold a second list_ref LIVE.
128
+ // The leader relays its LIVE events and routes rollbacks by mutation owner.
129
+ private tabRole: 'solo' | 'leader' | 'follower' = 'solo';
130
+ private tabId: string | null = null;
131
+ private hub: LeaderSyncHub | null = null;
132
+ private forwarder: SyncForwarder | null = null;
133
+
134
+ private refMode: RefMode = DEFAULT_REF_MODE;
135
+
136
+ // When true, an unauthenticated client still runs the `_00_list_ref` poll
137
+ // and LIVE subscription, routed to the shared `_00_list_ref_anon` table, so
138
+ // a logged-out page gets realtime `useQuery` updates. Off by default.
139
+ private readonly anonLiveEnabled: boolean;
140
+
141
+ // Bookkeeping for the LIVE subscription on `_00_list_ref[_user_*]`.
142
+ // SurrealDB binds the permission context at LIVE-registration time and
143
+ // the table name in dedicated mode depends on the authenticated user,
144
+ // so we have to re-register whenever auth state flips.
145
+ private currentLiveQueryUuid: Uuid | null = null;
146
+ private liveQueryUnsubscribe: (() => void) | null = null;
147
+
148
+ // Periodic re-poll of `_00_list_ref` as a safety net for missed LIVE
149
+ // notifications. SurrealDB v3 occasionally drops LIVE deliveries
150
+ // across sessions even when the row matches the permission rule;
151
+ // this catches those without requiring users to reload. The
152
+ // interval is configurable via the constructor; see
153
+ // `resolveListRefPollInterval` for fallback semantics.
154
+ //
155
+ // Self-rescheduling rather than setInterval so each tick can pick
156
+ // its own delay via `nextPollDelayMs` — slows the poll down when
157
+ // LIVE is delivering events and speeds it back up when LIVE quiets.
158
+ private listRefPollTimer: ReturnType<typeof setTimeout> | null = null;
159
+ private listRefPollRunning: boolean = false;
160
+ // The currently-executing poll tick, if any. `stopListRefPoll` only stops
161
+ // future ticks; a bucket switch must also AWAIT the in-flight one so its
162
+ // local writes land in the store it started against.
163
+ private listRefPollInFlight: Promise<void> | null = null;
164
+ public readonly refSyncIntervalMs: number;
165
+
166
+ // Consecutive poll cycles that observed NO list_ref change. Drives the
167
+ // adaptive backoff in `startListRefPoll` via `listRefPollDelayMs`: an idle
168
+ // page coasts from the fast base cadence toward the 5s cap, and any activity
169
+ // (a poll-detected change or a LIVE event) resets it to 0 so the poll snaps
170
+ // back to responsive. Replaces the old LIVE-liveness backoff, which kept the
171
+ // poll pinned at 500ms forever whenever LIVE wasn't firing (the common case
172
+ // on a quiet page, thanks to the cross-session LIVE-permission gap).
173
+ private listRefIdleStreak: number = 0;
174
+
175
+ // `${queryHash}:${recordId}` -> consecutive rounds the id has been "still
176
+ // remote" (left the query's list_ref but still exists upstream). Used to
177
+ // distinguish a PERSISTENT view-membership disagreement (the `job:` churn,
178
+ // converged once it crosses the threshold) from a record that's merely
179
+ // mid-deletion (still-remote for ~one round, then gone) — which must NOT be
180
+ // converged, or it gets stranded in this window before its delete is observed.
181
+ private stillRemoteStreaks: Map<string, number> = new Map();
182
+
183
+ // Wall-clock timestamp (ms) of the most recent LIVE event delivered
184
+ // through `handleRemoteListRefChange`. Kept as a diagnostic / liveness
185
+ // signal; the poll cadence is now driven by `listRefIdleStreak`.
186
+ private lastLiveEventAt: number | null = null;
187
+
188
+ // Number of times the initial `_00_list_ref[_user_*]` LIVE subscription
189
+ // had to retry on `setCurrentUserId`. Stays at 0 when the SSP has
190
+ // pre-emptively created the user's dedicated tables; otherwise
191
+ // increments on each retry attempt until LIVE succeeds or attempts
192
+ // are exhausted. Surfaced as a diagnostic so the e2e suite can prove
193
+ // the pre-emptive table-creation path is keeping the first sign-in
194
+ // off the lazy-creation race.
195
+ private _liveRetryCount: number = 0;
196
+ public get liveRetryCount(): number {
197
+ return this._liveRetryCount;
198
+ }
199
+
31
200
  get isSyncing() {
32
201
  return this.scheduler.isSyncing;
33
202
  }
@@ -37,13 +206,11 @@ export class SpookySync<S extends SchemaStructure> {
37
206
  }
38
207
 
39
208
  subscribeToPendingMutations(cb: (count: number) => void): () => void {
40
- const id1 = this.upQueue.events.subscribe(
41
- SyncQueueEventTypes.MutationEnqueued,
42
- (event) => cb(event.payload.queueSize)
209
+ const id1 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, (event) =>
210
+ cb(event.payload.queueSize)
43
211
  );
44
- const id2 = this.upQueue.events.subscribe(
45
- SyncQueueEventTypes.MutationDequeued,
46
- (event) => cb(event.payload.queueSize)
212
+ const id2 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationDequeued, (event) =>
213
+ cb(event.payload.queueSize)
47
214
  );
48
215
  return () => {
49
216
  this.upQueue.events.unsubscribe(id1);
@@ -51,16 +218,224 @@ export class SpookySync<S extends SchemaStructure> {
51
218
  };
52
219
  }
53
220
 
221
+ // ---- Sync health -------------------------------------------------------
222
+ // `0` disables degraded reporting (config `syncHealth: false`). Resolved
223
+ // from config in Sp00kyClient and passed through the constructor options.
224
+ private readonly degradeAfterFailures: number;
225
+ /** Per-push RPC deadline; see {@link withPushTimeout}. */
226
+ private readonly pushTimeoutMs: number;
227
+ private consecutiveSyncFailures = 0;
228
+ private syncHealthStatus: SyncHealthStatus = 'healthy';
229
+ private lastSyncErrorKind: 'network' | 'application' | undefined;
230
+ private lastSyncErrorMessage: string | undefined;
231
+ // Latched `true` on the first successful sync round; never reset. Lets a UI
232
+ // tell a cold-start "connecting" phase (never reached the server) apart from
233
+ // a real lost connection after a working session.
234
+ private hasSyncedOnce = false;
235
+
236
+ // Self-heal: while degraded, re-drive sync on an exponential backoff so the
237
+ // app recovers on its own — even when the socket never actually dropped (in
238
+ // that case no `connected` event fires, so this re-registration is the ONLY
239
+ // thing that re-probes the server). Started on the degrade transition,
240
+ // cleared on recovery. Capped cadence so a long outage doesn't busy-loop.
241
+ private selfHealTimer: ReturnType<typeof setTimeout> | null = null;
242
+ private selfHealAttempts = 0;
243
+ private static readonly SELF_HEAL_BASE_MS = 2_000;
244
+ private static readonly SELF_HEAL_MAX_MS = 30_000;
245
+
246
+ /**
247
+ * Transport supervisor, when one was supplied. Sync only reads state from it;
248
+ * it never drives reconnects itself.
249
+ */
250
+ private readonly connectionSupervisor?: ConnectionSupervisor;
251
+ /**
252
+ * Mirror of the supervisor's state. Defaults to `connected` so a client
253
+ * constructed without a supervisor (tests, embedders) reports the same health
254
+ * shape it always has rather than a permanent false "disconnected".
255
+ */
256
+ private connectionState: ConnectionState = 'connected';
257
+
258
+ /** Current sync-health snapshot. */
259
+ get syncHealth(): SyncHealth {
260
+ return {
261
+ status: this.syncHealthStatus,
262
+ consecutiveFailures: this.consecutiveSyncFailures,
263
+ kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
264
+ error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
265
+ everConnected: this.hasSyncedOnce,
266
+ connection: this.connectionState,
267
+ };
268
+ }
269
+
270
+ /**
271
+ * Observe sync health. The callback fires immediately with the current
272
+ * status and again on every healthy↔degraded transition. Returns an
273
+ * unsubscribe. Mirrors {@link subscribeToPendingMutations}.
274
+ */
275
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
276
+ cb(this.syncHealth);
277
+ const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
278
+ cb(event.payload)
279
+ );
280
+ return () => this.events.unsubscribe(id);
281
+ }
282
+
283
+ private emitSyncHealth(): void {
284
+ this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
285
+ }
286
+
287
+ /**
288
+ * Mirror the supervisor's transport state into {@link SyncHealth} and emit on
289
+ * every change, so a UI can react to a dropped socket immediately instead of
290
+ * waiting for `degradeAfterFailures` failed rounds. `status` is untouched:
291
+ * a brief reconnect is not a degradation.
292
+ *
293
+ * No explicit unsubscribe: the supervisor is owned by the same client and
294
+ * drops all subscribers in its own `dispose()`, which `Sp00kyClient.close()`
295
+ * calls first.
296
+ */
297
+ private subscribeToConnectionState(): void {
298
+ if (!this.connectionSupervisor) return;
299
+ this.connectionSupervisor.subscribe((state) => {
300
+ if (this.connectionState === state) return;
301
+ this.connectionState = state;
302
+ this.emitSyncHealth();
303
+ });
304
+ }
305
+
306
+ /**
307
+ * Fed by the scheduler once per drained sync round. Individual failures are
308
+ * absorbed by the queue's retry; only a run of `degradeAfterFailures`
309
+ * consecutive failures flips the status to `degraded`, and the next clean
310
+ * round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
311
+ * is 0).
312
+ */
313
+ private recordSyncOutcome(ok: boolean, error?: unknown): void {
314
+ if (this.degradeAfterFailures <= 0) return;
315
+ if (ok) {
316
+ // Latch first-ever success so a UI can drop the connecting phase. Set
317
+ // before the early return so a clean cold start (0 prior failures) counts.
318
+ this.hasSyncedOnce = true;
319
+ if (this.consecutiveSyncFailures === 0) return;
320
+ this.consecutiveSyncFailures = 0;
321
+ if (this.syncHealthStatus !== 'healthy') {
322
+ this.syncHealthStatus = 'healthy';
323
+ this.lastSyncErrorKind = undefined;
324
+ this.lastSyncErrorMessage = undefined;
325
+ this.stopSelfHeal();
326
+ this.logger.info(
327
+ { Category: 'sp00ky-client::Sp00kySync::syncHealth' },
328
+ 'Sync recovered; health back to healthy'
329
+ );
330
+ this.emitSyncHealth();
331
+ }
332
+ return;
333
+ }
334
+ this.consecutiveSyncFailures++;
335
+ this.lastSyncErrorKind = classifySyncError(error);
336
+ this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
337
+ if (
338
+ this.syncHealthStatus !== 'degraded' &&
339
+ this.consecutiveSyncFailures >= this.degradeAfterFailures
340
+ ) {
341
+ this.syncHealthStatus = 'degraded';
342
+ this.logger.warn(
343
+ {
344
+ consecutiveFailures: this.consecutiveSyncFailures,
345
+ kind: this.lastSyncErrorKind,
346
+ error,
347
+ Category: 'sp00ky-client::Sp00kySync::syncHealth',
348
+ },
349
+ 'Sync degraded after sustained failures'
350
+ );
351
+ this.emitSyncHealth();
352
+ this.startSelfHeal();
353
+ }
354
+ }
355
+
356
+ /**
357
+ * Begin self-heal retries (no-op if already running). Started on the
358
+ * healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
359
+ */
360
+ private startSelfHeal(): void {
361
+ if (this.selfHealTimer !== null) return;
362
+ this.selfHealAttempts = 0;
363
+ this.scheduleSelfHeal();
364
+ }
365
+
366
+ private scheduleSelfHeal(): void {
367
+ const delay = Math.min(
368
+ Sp00kySync.SELF_HEAL_MAX_MS,
369
+ Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts
370
+ );
371
+ this.selfHealTimer = setTimeout(async () => {
372
+ this.selfHealTimer = null;
373
+ if (this.syncHealthStatus !== 'degraded') return;
374
+ this.selfHealAttempts++;
375
+ this.logger.debug(
376
+ {
377
+ attempt: this.selfHealAttempts,
378
+ delayMs: delay,
379
+ Category: 'sp00ky-client::Sp00kySync::selfHeal',
380
+ },
381
+ 'Self-heal: re-driving sync while degraded'
382
+ );
383
+ try {
384
+ // Retry whatever is still queued first; the failing op (register or
385
+ // mutation) was re-queued by the queue, so this re-probes the server
386
+ // and reports the outcome through the scheduler → recordSyncOutcome.
387
+ if (this.upQueue.size > 0) {
388
+ await this.scheduler.syncUp();
389
+ } else if (this.downQueue.size > 0) {
390
+ await this.scheduler.syncDown();
391
+ } else {
392
+ // Nothing queued (e.g. the failing op was rolled back + dropped):
393
+ // re-register active queries — mirroring the reconnect handler — so
394
+ // there's a concrete op whose success flips health. If there are no
395
+ // active queries either, probe connectivity directly.
396
+ const hashes = this.dataModule.getActiveQueryHashes();
397
+ if (hashes.length > 0) {
398
+ for (const hash of hashes) {
399
+ this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
400
+ }
401
+ await this.scheduler.syncDown();
402
+ } else {
403
+ await this.remote.query('RETURN true');
404
+ this.recordSyncOutcome(true);
405
+ }
406
+ }
407
+ } catch (err) {
408
+ // Only the direct connectivity probe can throw here (syncUp/syncDown
409
+ // swallow + self-report); treat a probe failure as another failed round.
410
+ this.recordSyncOutcome(false, err);
411
+ }
412
+ // Keep retrying until recovery. recordSyncOutcome(true) calls stopSelfHeal
413
+ // (clearing any pending timer), so only continue while still degraded.
414
+ if (this.syncHealthStatus === 'degraded') this.scheduleSelfHeal();
415
+ }, delay);
416
+ }
417
+
418
+ private stopSelfHeal(): void {
419
+ if (this.selfHealTimer !== null) {
420
+ clearTimeout(this.selfHealTimer);
421
+ this.selfHealTimer = null;
422
+ }
423
+ this.selfHealAttempts = 0;
424
+ }
425
+
54
426
  constructor(
55
- private local: LocalDatabaseService,
427
+ private local: LocalStore,
56
428
  private remote: RemoteDatabaseService,
57
429
  private cache: CacheModule,
58
430
  private dataModule: DataModule<S>,
59
431
  private schema: S,
60
- logger: Logger
432
+ logger: Logger,
433
+ options?: Sp00kySyncOptions
61
434
  ) {
62
- this.logger = logger.child({ service: 'SpookySync' });
63
- this.upQueue = new UpQueue(this.local, this.logger);
435
+ this.logger = logger.child({ service: 'Sp00kySync' });
436
+ this.upQueue = new UpQueue(this.local, this.logger, (dropped) =>
437
+ this.onMutationDropped(dropped)
438
+ );
64
439
  this.downQueue = new DownQueue(this.local, this.logger);
65
440
  this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
66
441
  this.scheduler = new SyncScheduler(
@@ -69,43 +444,683 @@ export class SpookySync<S extends SchemaStructure> {
69
444
  this.processUpEvent.bind(this),
70
445
  this.processDownEvent.bind(this),
71
446
  this.logger,
72
- this.handleRollback.bind(this)
447
+ this.handleRollback.bind(this),
448
+ this.recordSyncOutcome.bind(this)
73
449
  );
450
+ this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
451
+ this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
452
+ this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
453
+ this.pushTimeoutMs = Math.max(0, options?.pushTimeoutMs ?? 30_000);
454
+ this.connectionSupervisor = options?.connectionSupervisor;
74
455
  }
75
456
 
76
457
  /**
77
458
  * Initializes the synchronization system.
78
459
  * Starts the scheduler and initiates the initial sync cycles.
79
- * @param clientId The unique identifier for this client instance.
80
460
  * @throws Error if already initialized.
81
461
  */
82
- public async init(clientId: string) {
83
- if (this.isInit) throw new Error('SpookySync is already initialized');
84
- this.clientId = clientId;
462
+ public async init() {
463
+ if (this.isInit) throw new Error('Sp00kySync is already initialized');
85
464
  this.isInit = true;
86
- await this.scheduler.init();
87
- void this.scheduler.syncUp();
465
+ await this.scheduler.init({ loadOutbox: this.tabRole !== 'follower' });
466
+ this.subscribeToReconnect();
467
+ this.subscribeToConnectionState();
88
468
  void this.scheduler.syncUp();
89
469
  void this.scheduler.syncDown();
90
- void this.startRefLiveQueries();
470
+ // No initial LIVE subscription — wait for `setCurrentUserId` to fire
471
+ // from the auth subscription. In dedicated mode the table name
472
+ // depends on the authenticated user, and an unauthenticated
473
+ // subscription wouldn't match any of the per-user tables anyway.
474
+ //
475
+ // Exception: when anonymous live queries are enabled, start realtime now
476
+ // against the shared `_00_list_ref_anon` table so a logged-out client
477
+ // syncs immediately. `setCurrentUserId` re-points LIVE to the per-user
478
+ // table on sign-in. Guard on `currentUserId` because the auth callback can
479
+ // fire (and authenticate) before `init()` runs — don't clobber that back
480
+ // to the anon table. `setCurrentUserId(null)` is a no-op on first load
481
+ // (it's already null), so this is the only place anon realtime starts.
482
+ if (this.anonLiveEnabled && !this.currentUserId) {
483
+ this.startListRefPoll();
484
+ this.restartRefLiveQuery().catch((err) => {
485
+ this.logger.debug(
486
+ { err, Category: 'sp00ky-client::Sp00kySync::init' },
487
+ 'Anonymous ref LIVE start failed; relying on periodic poll fallback'
488
+ );
489
+ });
490
+ }
491
+ }
492
+
493
+ // ---- shared-tabs roles ------------------------------------------------------
494
+
495
+ /** Set BEFORE init(): shapes what init boots (a follower loads no outbox and
496
+ * never starts LIVE; its own registration/poll paths stay untouched). */
497
+ public setTabContext(role: 'solo' | 'leader' | 'follower', tabId: string | null): void {
498
+ this.tabRole = role;
499
+ this.tabId = tabId;
500
+ }
501
+
502
+ /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
503
+ * relay LIVE events and rollbacks to followers via `hub`. Idempotent for a
504
+ * boot-time leader; a runtime promotion (failover) reloads the outbox,
505
+ * which now holds EVERY tab's rows, and restarts LIVE under this session. */
506
+ public async promoteToLeader(hub: LeaderSyncHub): Promise<void> {
507
+ this.tabRole = 'leader';
508
+ this.hub = hub;
509
+ this.forwarder = null;
510
+ hub.onFollowerMessage = (tabId, msg) => {
511
+ void tabId;
512
+ switch (msg.type) {
513
+ case 'sync-hello':
514
+ break;
515
+ case 'mutation-enqueued':
516
+ void this.enqueueForwardedMutation(msg.mutationId);
517
+ break;
518
+ case 'request-poll':
519
+ this.listRefIdleStreak = 0;
520
+ break;
521
+ }
522
+ };
523
+ if (this.isInit) {
524
+ await this.upQueue.loadFromDatabase();
525
+ void this.scheduler.syncUp();
526
+ if (this.currentUserId || this.anonLiveEnabled) {
527
+ this.startListRefPoll();
528
+ await this.restartRefLiveQuery().catch((err) => {
529
+ this.logger.warn(
530
+ { err, Category: 'sp00ky-client::Sp00kySync::promoteToLeader' },
531
+ 'LIVE restart failed on promotion; poll fallback covers it'
532
+ );
533
+ });
534
+ }
535
+ }
536
+ }
537
+
538
+ /** Follower duties: no outbox drain, no LIVE. Mutations forward to the
539
+ * leader; everything else (registration, per-query sync, poll) runs
540
+ * against this tab's own remote session as usual. */
541
+ public demoteToFollower(forwarder: SyncForwarder): void {
542
+ this.tabRole = 'follower';
543
+ this.hub = null;
544
+ this.forwarder = forwarder;
545
+ void this.killRefLiveQuery();
546
+ forwarder.onLeaderMessage = (msg) => {
547
+ switch (msg.type) {
548
+ case 'list-ref-change':
549
+ void this.applyRelayedListRefChange(msg).catch((err) => {
550
+ this.logger.error(
551
+ { err, Category: 'sp00ky-client::Sp00kySync::relay' },
552
+ 'Relayed list_ref change failed'
553
+ );
554
+ });
555
+ break;
556
+ case 'mutation-rolled-back':
557
+ this.events.emit(SyncEventTypes.MutationRolledBack, {
558
+ eventType: msg.eventType,
559
+ recordId: msg.recordId,
560
+ error: msg.error,
561
+ });
562
+ break;
563
+ default:
564
+ // db-ready and ingest-relay are handled by the engine/cache wiring
565
+ // in Sp00kyClient before messages reach this handler.
566
+ break;
567
+ }
568
+ };
569
+ }
570
+
571
+ /**
572
+ * A pending mutation was discarded because it can never be sent.
573
+ *
574
+ * This is a lost write, so it must not stay invisible. Every failure in this
575
+ * chain used to be a `logger.error` an app running `logLevel: 'fatal'` never
576
+ * shows, which is how an outbox could sit undrained for hours with the UI
577
+ * reporting nothing. Surfaces as a rollback event (the mutation will never
578
+ * apply, which is what a subscriber needs to know) and degrades sync health.
579
+ */
580
+ private onMutationDropped(dropped: {
581
+ mutationId: string;
582
+ recordId?: string;
583
+ mutationType?: string;
584
+ reason: string;
585
+ }): void {
586
+ this.logger.error(
587
+ { ...dropped, Category: 'sp00ky-client::Sp00kySync::onMutationDropped' },
588
+ 'Dropped a pending mutation that can never be sent'
589
+ );
590
+ this.recordSyncOutcome(false, new Error(`dropped mutation: ${dropped.reason}`));
591
+ this.events.emit(SyncEventTypes.MutationRolledBack, {
592
+ eventType: (dropped.mutationType as 'create' | 'update' | 'delete') ?? 'update',
593
+ recordId: dropped.recordId ?? dropped.mutationId,
594
+ error: `dropped: ${dropped.reason}`,
595
+ });
596
+ }
597
+
598
+ /** A forwarded outbox row from a follower: load + drain it. Idempotent. */
599
+ public async enqueueForwardedMutation(mutationId: string): Promise<void> {
600
+ if (this.tabRole !== 'leader') return;
601
+ await this.upQueue.enqueueFromDatabase(mutationId);
602
+ }
603
+
604
+ /** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
605
+ * and run the exact same handling the LIVE subscription would have. */
606
+ private async applyRelayedListRefChange(msg: {
607
+ action: 'CREATE' | 'UPDATE' | 'DELETE';
608
+ queryId: string;
609
+ recordId: string;
610
+ version: number;
611
+ parent: boolean;
612
+ }): Promise<void> {
613
+ const queryId = parseRecordIdString(msg.queryId);
614
+ // Foreign query (another tab's session-salted hash): not ours, ignore.
615
+ if (!this.dataModule.getQueryById(queryId)) return;
616
+ const recordId = parseRecordIdString(msg.recordId);
617
+ if (msg.parent) {
618
+ await this.handleRemoteSubqueryChange(msg.action, queryId, recordId, msg.version);
619
+ } else {
620
+ await this.handleRemoteListRefChange(msg.action, queryId, recordId, msg.version);
621
+ }
622
+ }
623
+
624
+ /** One immediate poll cycle (failover convergence). */
625
+ public async forcePollRound(): Promise<void> {
626
+ this.listRefIdleStreak = 0;
627
+ await this.pollListRefForActiveQueries().catch(() => false);
628
+ }
629
+
630
+ /**
631
+ * Quiesce all sync activity ahead of a local-bucket switch. After this
632
+ * resolves, nothing in the sync module writes to the local store: the poll
633
+ * loop is stopped AND its in-flight tick awaited, LIVE is killed, debounce
634
+ * timers are cancelled (their outbox rows are already persisted), and the
635
+ * scheduler has drained its in-flight queue item — including that item's
636
+ * outbox-row delete, which must land in the OLD bucket. Queued down-events
637
+ * are dropped (they reference old-bucket query rows; the post-switch rebind
638
+ * re-enqueues registrations). The old user's un-pushed outbox is deliberately
639
+ * NOT drained: the remote session already belongs to the next user.
640
+ */
641
+ public async prepareBucketSwitch(): Promise<void> {
642
+ this.stopSelfHeal();
643
+ this.stopListRefPoll();
644
+ if (this.listRefPollInFlight) await this.listRefPollInFlight;
645
+ await this.killRefLiveQuery();
646
+ this.upQueue.clearDebounceTimers();
647
+ await this.scheduler.pause();
648
+ this.downQueue.clear();
649
+ this.stillRemoteStreaks.clear();
650
+ this.logger.info(
651
+ { Category: 'sp00ky-client::Sp00kySync::prepareBucketSwitch' },
652
+ 'Sync quiesced for bucket switch'
653
+ );
654
+ }
655
+
656
+ /**
657
+ * Resume syncing against the freshly-opened bucket: reload the mutation
658
+ * outbox from ITS `_00_pending_mutations` (the new user's own un-pushed
659
+ * offline work) and restart the scheduler. LIVE + the list_ref poll restart
660
+ * via the `setCurrentUserId` call that follows in the auth listener.
661
+ */
662
+ public async completeBucketSwitch(): Promise<void> {
663
+ await this.upQueue.loadFromDatabase();
664
+ this.scheduler.resume();
665
+ this.logger.info(
666
+ { Category: 'sp00ky-client::Sp00kySync::completeBucketSwitch' },
667
+ 'Sync resumed after bucket switch'
668
+ );
669
+ }
670
+
671
+ /**
672
+ * Push the authenticated user's record id from the parent client's
673
+ * auth subscription. Tears down the existing `_00_list_ref` LIVE (if
674
+ * any) and re-registers it under the new user's dedicated table so
675
+ * SurrealDB binds the permission rule under the post-flip auth
676
+ * context. Pass `null` on sign-out.
677
+ *
678
+ * The dedicated `_00_list_ref_user_<id>` table is created lazily by
679
+ * the SSP when the first query registration arrives, which may be
680
+ * concurrent with this call. We retry the LIVE registration with a
681
+ * short backoff so a "table not found" race resolves without
682
+ * surfacing as a permanent auth-loading hang.
683
+ */
684
+ public async setCurrentUserId(userId: string | null): Promise<void> {
685
+ if (this.currentUserId === userId) return;
686
+ this.currentUserId = userId;
687
+ if (!userId) {
688
+ if (this.anonLiveEnabled) {
689
+ // Signed out but anonymous realtime is on: keep the poll running and
690
+ // re-point LIVE from the (now stale) per-user table to the shared
691
+ // `_00_list_ref_anon`. `startListRefPoll` is idempotent; the poll
692
+ // re-resolves `listRefTable()` each tick so it follows automatically.
693
+ this.startListRefPoll();
694
+ await this.restartRefLiveQuery().catch((err) => {
695
+ this.logger.debug(
696
+ { err, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
697
+ 'Anonymous ref LIVE restart failed; relying on periodic poll fallback'
698
+ );
699
+ });
700
+ return;
701
+ }
702
+ await this.killRefLiveQuery();
703
+ this.stopListRefPoll();
704
+ return;
705
+ }
706
+ // Start periodic polling FIRST so we have a deterministic fallback
707
+ // even when LIVE registration fails or SurrealDB drops a delivery.
708
+ this.startListRefPoll();
709
+ // Try to start LIVE with backoff for low-latency delivery on the
710
+ // happy path; the poll handles the rest.
711
+ const attemptDelays = [0, 250, 500, 1000, 2000];
712
+ for (let i = 0; i < attemptDelays.length; i++) {
713
+ if (attemptDelays[i] > 0) {
714
+ this._liveRetryCount++;
715
+ await new Promise((r) => setTimeout(r, attemptDelays[i]));
716
+ }
717
+ try {
718
+ await this.restartRefLiveQuery();
719
+ return;
720
+ } catch (err) {
721
+ this.logger.debug(
722
+ { err, attempt: i + 1, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
723
+ 'Ref LIVE start failed; relying on periodic poll fallback'
724
+ );
725
+ }
726
+ }
727
+ }
728
+
729
+ private startListRefPoll(): void {
730
+ if (this.listRefPollRunning) return;
731
+ this.listRefPollRunning = true;
732
+ this.logger.debug(
733
+ {
734
+ intervalMs: this.refSyncIntervalMs,
735
+ Category: 'sp00ky-client::Sp00kySync::startListRefPoll',
736
+ },
737
+ 'list_ref poll loop started'
738
+ );
739
+ const schedule = (delayMs: number) => {
740
+ this.listRefPollTimer = setTimeout(async () => {
741
+ if (!this.listRefPollRunning) return;
742
+ let changed = false;
743
+ const tick = (async () => {
744
+ changed = await this.pollListRefForActiveQueries();
745
+ })();
746
+ this.listRefPollInFlight = tick.catch(() => {});
747
+ try {
748
+ await tick;
749
+ } finally {
750
+ this.listRefPollInFlight = null;
751
+ if (!this.listRefPollRunning) return;
752
+ // Reset the idle streak on any observed change so the poll snaps
753
+ // back to the fast base cadence; otherwise grow it so a quiet page
754
+ // backs off toward the cap. (`handleRemoteListRefChange` also resets
755
+ // it when a LIVE event lands.)
756
+ this.listRefIdleStreak = changed ? 0 : this.listRefIdleStreak + 1;
757
+ const next = listRefPollDelayMs({
758
+ idleStreak: this.listRefIdleStreak,
759
+ baseIntervalMs: this.refSyncIntervalMs,
760
+ });
761
+ schedule(next);
762
+ }
763
+ }, delayMs);
764
+ };
765
+ schedule(this.refSyncIntervalMs);
766
+ }
767
+
768
+ private stopListRefPoll(): void {
769
+ this.listRefPollRunning = false;
770
+ if (this.listRefPollTimer !== null) {
771
+ clearTimeout(this.listRefPollTimer);
772
+ this.listRefPollTimer = null;
773
+ }
774
+ }
775
+
776
+ /**
777
+ * One poll cycle: refetch `_00_list_ref` for every active query. Returns
778
+ * whether ANY query's remoteArray actually changed — the scheduler uses this
779
+ * to drive the adaptive idle backoff.
780
+ *
781
+ * Also the ONLY health signal that runs while the page is idle. Sync health is
782
+ * otherwise activity-driven (mutations/registrations via the scheduler,
783
+ * reconnect re-registration, self-heal), so on a quiet page a stale `degraded`
784
+ * would linger until the next mutation and a genuine idle drop would be
785
+ * invisible. We fold the cycle's aggregate reachability into `recordSyncOutcome`
786
+ * so idle health self-recovers (and self-degrades) with no user action. A clean
787
+ * cycle is idempotent when already healthy (`recordSyncOutcome` early-returns at
788
+ * `consecutiveSyncFailures === 0`), so a healthy idle page pays nothing.
789
+ */
790
+ private async pollListRefForActiveQueries(): Promise<boolean> {
791
+ const hashes = this.dataModule.getActiveQueryHashes();
792
+ if (hashes.length === 0) {
793
+ // No active queries to piggyback on, but health still needs a heartbeat —
794
+ // probe connectivity directly so an idle page with no live queries doesn't
795
+ // go blind. Cheap, and gated by the same adaptive backoff (≤5s idle cap).
796
+ try {
797
+ await this.remote.query('RETURN true');
798
+ this.recordSyncOutcome(true);
799
+ } catch (err) {
800
+ this.recordSyncOutcome(false, err);
801
+ }
802
+ return false;
803
+ }
804
+ let anyChanged = false;
805
+ // `reached` = the server answered at least once this cycle (a success, or an
806
+ // *application* error, which still proves reachability). `firstNetworkErr`
807
+ // holds the first network-classified failure. A cycle that only produced
808
+ // network errors reports the outcome as a down round; a mixed/app cycle counts
809
+ // as reached; an all-application cycle reports nothing (that's a query-shape
810
+ // fault owned by the registration path, not a reachability signal).
811
+ let reached = false;
812
+ let firstNetworkErr: unknown;
813
+ for (const hash of hashes) {
814
+ try {
815
+ if (await this.refetchListRefForQuery(hash)) anyChanged = true;
816
+ reached = true;
817
+ } catch (err) {
818
+ if (classifySyncError(err) === 'network') {
819
+ if (firstNetworkErr === undefined) firstNetworkErr = err;
820
+ } else {
821
+ reached = true;
822
+ }
823
+ this.logger.debug(
824
+ {
825
+ err: (err as Error)?.message ?? err,
826
+ hash,
827
+ Category: 'sp00ky-client::Sp00kySync::pollListRefForActiveQueries',
828
+ },
829
+ 'Per-query list_ref poll failed'
830
+ );
831
+ }
832
+ }
833
+ // Call the private outcome recorder directly rather than routing through the
834
+ // scheduler — the scheduler only reports on rounds that drained ≥1 queue item
835
+ // (`processedAny`), and this isn't a queue round.
836
+ if (reached) {
837
+ this.recordSyncOutcome(true);
838
+ } else if (firstNetworkErr !== undefined) {
839
+ this.recordSyncOutcome(false, firstNetworkErr);
840
+ }
841
+ return anyChanged;
842
+ }
843
+
844
+ /**
845
+ * Pull the upstream list_ref entries for `queryHash`, diff them
846
+ * against the local `remoteArray` cache, sync any added/updated rows
847
+ * through the SyncEngine, then persist the new remoteArray. This is
848
+ * the same shape `createRemoteQuery` does for its initial fetch and
849
+ * what `handleRemoteListRefChange` does per-LIVE-event — we reuse
850
+ * it on a timer as a fallback for missed LIVE notifications.
851
+ */
852
+ private async refetchListRefForQuery(queryHash: string): Promise<boolean> {
853
+ const queryState = this.dataModule.getQueryByHash(queryHash);
854
+ if (!queryState) return false;
855
+ const listRefTbl = this.listRefTable();
856
+ const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
857
+ buildListRefSelect(listRefTbl),
858
+ { in: queryState.config.id }
859
+ );
860
+ if (!Array.isArray(items)) return false;
861
+ const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
862
+ // Capture which ids LEFT the query's window (present in the cached
863
+ // remoteArray, absent from `fresh`) BEFORE we overwrite remoteArray — these
864
+ // are cross-window deletes (or rows that scrolled out). They drive the
865
+ // forced re-render below.
866
+ const prevRemote = queryState.config.remoteArray ?? [];
867
+ const freshIds = new Set(fresh.map(([id]) => id));
868
+ const removedIds = prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id);
869
+ // Idempotent poll: only persist the remoteArray when it actually changed.
870
+ // The poll runs continuously as a LIVE fallback, so on a quiet page `fresh`
871
+ // equals the cached array every tick — re-writing it (an `UPDATE _00_query`
872
+ // each cycle, per active query) was pure churn and the bulk of the idle
873
+ // traffic. `recordVersionArraysEqual` is order-insensitive because the
874
+ // list_ref SELECT has no `ORDER BY`.
875
+ const changed = !recordVersionArraysEqual(fresh, queryState.config.remoteArray);
876
+ if (changed) {
877
+ // Update the cached remoteArray so the next diff/sync sees the new state.
878
+ // `syncQuery` (below) then writes through `cache.saveBatch`, which UPSERTs
879
+ // the local DB row and ingests it into the in-browser SSP — the SSP's
880
+ // stream updates run `processStreamUpdate`, which re-queries the local DB
881
+ // and notifies subscribers. We skip an explicit `notifyQuerySynced`
882
+ // because that path races the stream-update path (can notify with stale
883
+ // records).
884
+ await this.dataModule.updateQueryRemoteArray(queryHash, fresh);
885
+ }
886
+ // Run `syncQuery` every tick regardless: it's a no-op when localArray has
887
+ // caught up to remoteArray (`if (!diff) return`, issues no query), but it
888
+ // covers the rare case where remoteArray is stable yet localArray is behind
889
+ // (a prior record fetch failed) — so a missed row still gets retried.
890
+ // For REMOVALS it runs the ids through `handleRemovedRecords`, which deletes
891
+ // confirmed-gone records from the local DB.
892
+ try {
893
+ await this.syncQuery(queryHash);
894
+ } catch (err) {
895
+ this.logger.info(
896
+ {
897
+ err: (err as Error)?.message ?? err,
898
+ queryHash,
899
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
900
+ },
901
+ 'syncQuery failed during poll'
902
+ );
903
+ }
904
+ // Cross-session fallback for `.related()` child rows: the LIVE-permission
905
+ // gap can drop child-edge notifications, so converge their bodies on the
906
+ // poll too (idempotent — no-op when nothing changed).
907
+ await this.syncSubqueryChildren(queryHash).catch((err) => {
908
+ this.logger.info(
909
+ {
910
+ err: (err as Error)?.message ?? err,
911
+ queryHash,
912
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
913
+ },
914
+ 'Subquery child sync failed during poll'
915
+ );
916
+ });
917
+ // A REMOVAL needs no record fetch, so unlike the added-row path it doesn't
918
+ // get a re-render from the SSP stream on this code path reliably (and the
919
+ // non-windowed window-0 query re-queries the local DB rather than the id-set).
920
+ // Force a re-materialize + notify so the deleted row drops from the list in
921
+ // this (second) window — the reliable, LIVE-independent cross-window path.
922
+ if (removedIds.length > 0) {
923
+ try {
924
+ await this.dataModule.notifyQuerySynced(queryHash);
925
+ } catch (err) {
926
+ this.logger.info(
927
+ {
928
+ err: (err as Error)?.message ?? err,
929
+ queryHash,
930
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
931
+ },
932
+ 'notifyQuerySynced failed during poll-removal re-render'
933
+ );
934
+ }
935
+ }
936
+ return changed;
937
+ }
938
+
939
+ /**
940
+ * Resolve the current `_00_list_ref` table name for the active auth
941
+ * context. Public so the `createRemoteQuery` initial-fetch path can
942
+ * read from the right per-user table.
943
+ *
944
+ * Reads the user id from `DataModule` rather than the local mirror,
945
+ * because `DataModule.setCurrentUserId` runs synchronously from the
946
+ * auth callback (before any `await`), whereas `sync.setCurrentUserId`
947
+ * is async — the userQuery's initial fetch can fire between those
948
+ * two points and we need the correct table name immediately.
949
+ */
950
+ public listRefTable(): string {
951
+ const userId = this.dataModule.getCurrentUserId();
952
+ // Unauthenticated with the flag on → the shared `_00_list_ref_anon` table.
953
+ if (userId == null && this.anonLiveEnabled) {
954
+ return listRefTableFor(this.refMode, ANON_USER_ID);
955
+ }
956
+ return listRefTableFor(this.refMode, userId);
957
+ }
958
+
959
+ private async killRefLiveQuery(): Promise<void> {
960
+ if (this.liveQueryUnsubscribe) {
961
+ try {
962
+ this.liveQueryUnsubscribe();
963
+ } catch {
964
+ /* ignore */
965
+ }
966
+ this.liveQueryUnsubscribe = null;
967
+ }
968
+ if (this.currentLiveQueryUuid !== null) {
969
+ // A LIVE subscription is scoped to its WebSocket session, so after a
970
+ // reconnect the server-side one is already gone and there is nothing to
971
+ // KILL. Sending it anyway either fails (no connection) or races the fresh
972
+ // socket's readiness while holding up the restart behind it. Local
973
+ // bookkeeping is cleared either way.
974
+ if (this.remote.getStatus() === 'connected') {
975
+ try {
976
+ await this.remote.query('KILL $u', { u: this.currentLiveQueryUuid });
977
+ } catch (err) {
978
+ this.logger.debug(
979
+ { err, Category: 'sp00ky-client::Sp00kySync::killRefLiveQuery' },
980
+ 'Prior LIVE KILL failed; continuing'
981
+ );
982
+ }
983
+ }
984
+ this.currentLiveQueryUuid = null;
985
+ }
986
+ }
987
+
988
+ private async restartRefLiveQuery(): Promise<void> {
989
+ await this.killRefLiveQuery();
990
+ await this.startRefLiveQueries();
991
+ }
992
+
993
+ /**
994
+ * Drop local LIVE bookkeeping without issuing a `KILL`.
995
+ *
996
+ * Called when the socket dies. The server-side subscription is scoped to that
997
+ * WebSocket session and died with it, so there is nothing left to kill — and
998
+ * by the time the reconnect handler runs, the client reports `connected`
999
+ * again, which would otherwise send a `KILL` for a stale uuid on the *new*
1000
+ * session and hold up the restart queued behind it.
1001
+ */
1002
+ private invalidateRefLiveQuery(): void {
1003
+ if (this.liveQueryUnsubscribe) {
1004
+ try {
1005
+ this.liveQueryUnsubscribe();
1006
+ } catch {
1007
+ /* ignore */
1008
+ }
1009
+ this.liveQueryUnsubscribe = null;
1010
+ }
1011
+ this.currentLiveQueryUuid = null;
1012
+ }
1013
+
1014
+ // Only the connect that follows a prior drop counts as a reconnect; the
1015
+ // initial connect after init() must not trigger a refetch storm.
1016
+ //
1017
+ // Both drop events have to be watched. The SDK publishes `disconnected` ONLY
1018
+ // when it has given up entirely (attempts exhausted, or the engine
1019
+ // terminated); an ordinary recovered drop goes `error` -> `reconnecting` ->
1020
+ // `connected` and never touches `disconnected`. Listening for `disconnected`
1021
+ // alone therefore misses every successful reconnect — the exact case this
1022
+ // handler exists for — and leaves the dead server-side LIVE in place with the
1023
+ // poll as the only sync path.
1024
+ private subscribeToReconnect() {
1025
+ const client = this.remote.getClient();
1026
+ client.subscribe('disconnected', () => {
1027
+ this.needsResubscribe = true;
1028
+ this.invalidateRefLiveQuery();
1029
+ this.logger.info(
1030
+ { Category: 'sp00ky-client::Sp00kySync::onDisconnect' },
1031
+ 'Remote disconnected'
1032
+ );
1033
+ });
1034
+ client.subscribe('reconnecting', () => {
1035
+ this.needsResubscribe = true;
1036
+ this.invalidateRefLiveQuery();
1037
+ this.logger.info(
1038
+ { Category: 'sp00ky-client::Sp00kySync::onReconnecting' },
1039
+ 'Remote socket dropped; awaiting reconnect'
1040
+ );
1041
+ });
1042
+ client.subscribe('connected', () => {
1043
+ if (!this.needsResubscribe) return;
1044
+ this.needsResubscribe = false;
1045
+ // A flapping socket produces reconnecting -> connected repeatedly, and
1046
+ // each cycle used to re-register EVERY active query (a busy app has
1047
+ // dozens). That is the "everything reloads about a second after a blip"
1048
+ // symptom: the SDK's retryDelay is 1s, so the refetch lands right after
1049
+ // the drop the user never saw. Collapse bursts into one refetch.
1050
+ const sinceLast = Date.now() - this.lastReconnectRefetchAt;
1051
+ if (sinceLast < Sp00kySync.RECONNECT_REFETCH_COOLDOWN_MS) {
1052
+ this.logger.debug(
1053
+ { sinceLast, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
1054
+ 'Reconnected again within the cooldown; skipping duplicate refetch'
1055
+ );
1056
+ return;
1057
+ }
1058
+ this.lastReconnectRefetchAt = Date.now();
1059
+ const hashes = this.dataModule.getActiveQueryHashes();
1060
+ this.logger.info(
1061
+ { queries: hashes.length, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
1062
+ 'Remote reconnected, refetching active queries'
1063
+ );
1064
+ for (const hash of hashes) {
1065
+ this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
1066
+ }
1067
+ // The WS reconnect leaves the server-side LIVE subscription dead — the
1068
+ // re-enqueued `register` events only re-fetch initial state, they don't
1069
+ // re-subscribe. Without this, LIVE never recovers after a reconnect and
1070
+ // the poll silently becomes the sole sync path (and never backs off).
1071
+ // Authenticated → per-user table; signed-out with anon live enabled →
1072
+ // the shared `_00_list_ref_anon`. Otherwise there's no table to re-bind.
1073
+ if (this.currentUserId || this.anonLiveEnabled) {
1074
+ this.restartRefLiveQuery().catch((err) => {
1075
+ this.logger.debug(
1076
+ { err, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
1077
+ 'LIVE restart after reconnect failed; relying on poll fallback'
1078
+ );
1079
+ });
1080
+ }
1081
+ });
91
1082
  }
92
1083
 
93
1084
  private async startRefLiveQueries() {
1085
+ // Shared-tabs follower: exactly one LIVE per user exists, on the leader;
1086
+ // its events reach this tab through the relay.
1087
+ if (this.tabRole === 'follower') return;
1088
+ const tableName = this.listRefTable();
94
1089
  this.logger.debug(
95
- { clientId: this.clientId, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
1090
+ { tableName, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
96
1091
  'Starting ref live queries'
97
1092
  );
98
1093
 
99
- const [queryUuid] = await this.remote.query<[Uuid]>(
100
- 'LIVE SELECT * FROM _spooky_list_ref'
101
- );
1094
+ const [queryUuid] = await this.remote.query<[Uuid]>(`LIVE SELECT * FROM ${tableName}`);
1095
+ this.currentLiveQueryUuid = queryUuid;
102
1096
 
103
- (await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
1097
+ const live = await this.remote.getClient().liveOf(queryUuid);
1098
+ this.liveQueryUnsubscribe = live.subscribe((message) => {
104
1099
  this.logger.debug(
105
- { message, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
1100
+ { message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
106
1101
  'Live update received'
107
1102
  );
108
1103
  if (message.action === 'KILLED') return;
1104
+ // Subquery child edges (rows with `parent` set) are NOT primary window
1105
+ // rows — the client's `RecordVersionArray` only tracks primary rows, so
1106
+ // routing them through `handleRemoteListRefChange` would surface them as
1107
+ // spurious "added" diffs and pollute the window. Instead route them to
1108
+ // the dedicated child-body sync so `.related()` data stays realtime
1109
+ // cross-session (the LIVE-permission gap otherwise leaves it to the poll).
1110
+ if ((message.value as { parent?: unknown }).parent != null) {
1111
+ this.handleRemoteSubqueryChange(
1112
+ message.action,
1113
+ message.value.in as RecordId<string>,
1114
+ message.value.out as RecordId<string>,
1115
+ message.value.version as number
1116
+ ).catch((err) => {
1117
+ this.logger.error(
1118
+ { err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
1119
+ 'Error handling remote subquery change'
1120
+ );
1121
+ });
1122
+ return;
1123
+ }
109
1124
  this.handleRemoteListRefChange(
110
1125
  message.action,
111
1126
  message.value.in as RecordId<string>,
@@ -113,7 +1128,7 @@ export class SpookySync<S extends SchemaStructure> {
113
1128
  message.value.version as number
114
1129
  ).catch((err) => {
115
1130
  this.logger.error(
116
- { err, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
1131
+ { err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
117
1132
  'Error handling remote list ref change'
118
1133
  );
119
1134
  });
@@ -126,16 +1141,47 @@ export class SpookySync<S extends SchemaStructure> {
126
1141
  recordId: RecordId,
127
1142
  version: number
128
1143
  ) {
1144
+ // Any LIVE delivery is evidence of activity — a CREATE/UPDATE/DELETE on a
1145
+ // query's window, or a notification for an unknown local query. Reset the
1146
+ // poll's idle streak so it snaps back to the fast base cadence (the page
1147
+ // is clearly not idle), and record the timestamp as a liveness diagnostic.
1148
+ this.lastLiveEventAt = Date.now();
1149
+ this.listRefIdleStreak = 0;
1150
+
1151
+ // Shared-tabs leader: the list_ref table is USER-scoped, so this LIVE also
1152
+ // carries events for FOLLOWER tabs' queries (their own session-salted
1153
+ // hashes). Relay every primary event; each follower resolves the queryId
1154
+ // against its own DataModule and ignores foreign ones. Then continue with
1155
+ // this tab's own handling below.
1156
+ this.hub?.broadcast({
1157
+ type: 'list-ref-change',
1158
+ action,
1159
+ queryId: encodeRecordId(queryId),
1160
+ recordId: encodeRecordId(recordId),
1161
+ version,
1162
+ parent: false,
1163
+ });
1164
+
1165
+ // NOTE: DELETE is handled like CREATE/UPDATE below. When another window (or
1166
+ // this one) deletes a record, the server's SSP removes it from `_00_list_ref`
1167
+ // and the LIVE subscription delivers a DELETE here — `createDiffFromDbOp`
1168
+ // turns it into a `removed: [recordId]` diff so the row drops from the window
1169
+ // in realtime. (It was previously ignored, so other windows only caught up on
1170
+ // reload / the slow poll.)
129
1171
  const existing = this.dataModule.getQueryById(queryId);
130
1172
 
131
1173
  if (!existing) {
132
- this.logger.warn(
133
- {
134
- queryId: queryId.toString(),
135
- Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
136
- },
137
- 'Received remote update for unknown local query'
138
- );
1174
+ // With a hub attached, an unknown query is the NORMAL case (it belongs
1175
+ // to a follower tab); without one it still warrants the warning.
1176
+ if (!this.hub) {
1177
+ this.logger.warn(
1178
+ {
1179
+ queryId: queryId.toString(),
1180
+ Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
1181
+ },
1182
+ 'Received remote update for unknown local query'
1183
+ );
1184
+ }
139
1185
  return;
140
1186
  }
141
1187
 
@@ -148,12 +1194,87 @@ export class SpookySync<S extends SchemaStructure> {
148
1194
  recordId,
149
1195
  version,
150
1196
  localArray,
151
- Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
1197
+ Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
152
1198
  },
153
1199
  'Live update is being processed'
154
1200
  );
155
1201
  const diff = createDiffFromDbOp(action, recordId, version, localArray);
156
- await this.syncEngine.syncRecords(diff);
1202
+ // `config.id` is `_00_query:<hash>`, so its id-part IS the query hash
1203
+ // (a SHA-256 over query content + sessionId) — the key DataModule uses.
1204
+ const hash = extractIdPart(existing.config.id);
1205
+
1206
+ // Apply the event to `remoteArray` — the authoritative membership rows are
1207
+ // now rendered FROM. Only registration and the poll used to write it, so a
1208
+ // LIVE removal left the departed id in the list (in memory and persisted)
1209
+ // until the next poll tick, which is up to 5s of showing a deleted row. This
1210
+ // also persists the durable `_00_window` mirror, so the removal survives a
1211
+ // reload with no network.
1212
+ if (existing.config.membershipKnown && (diff.removed.length || diff.added.length)) {
1213
+ const next = applyRecordVersionDiff(existing.config.remoteArray ?? [], diff);
1214
+ if (!recordVersionArraysEqual(next, existing.config.remoteArray ?? [])) {
1215
+ await this.dataModule.updateQueryRemoteArray(hash, next);
1216
+ }
1217
+ }
1218
+
1219
+ await this.runSyncForQuery(hash, diff);
1220
+
1221
+ // A removal-only diff sets `fetching` false in `runSyncForQuery`, so it gets
1222
+ // no `flushPendingStreamUpdate`/`endFetching` re-render — and a removal needs
1223
+ // no record fetch to trigger one either. Force it, mirroring what the poll
1224
+ // path already does for its own removals (`refetchListRefForQuery`).
1225
+ if (diff.removed.length > 0 && diff.added.length === 0 && diff.updated.length === 0) {
1226
+ await this.dataModule.notifyQuerySynced(hash);
1227
+ }
1228
+ }
1229
+
1230
+ /**
1231
+ * Handle a LIVE change to a SUBQUERY child edge (a `_00_list_ref` row with
1232
+ * `parent` set) for a `.related()` query. Unlike primary rows, child rows
1233
+ * must NOT touch the query's `localArray`/`remoteArray`/`rowCount`; we only
1234
+ * keep the child BODY fresh in the local cache so the in-browser SSP's
1235
+ * subquery-table dependency re-materializes the parent view.
1236
+ *
1237
+ * CREATE/UPDATE fetch+upsert the child body. DELETE is intentionally a
1238
+ * no-op: a child leaving this query's set must not delete a body another
1239
+ * query may still show (see `syncSubqueryChildren` deletion-safety note);
1240
+ * a genuine record delete propagates via the normal delete path.
1241
+ */
1242
+ private async handleRemoteSubqueryChange(
1243
+ action: 'CREATE' | 'UPDATE' | 'DELETE',
1244
+ queryId: RecordId,
1245
+ childId: RecordId,
1246
+ version: number
1247
+ ) {
1248
+ this.lastLiveEventAt = Date.now();
1249
+ this.listRefIdleStreak = 0;
1250
+
1251
+ if (action === 'DELETE') return;
1252
+
1253
+ // Relay child-edge events too (see handleRemoteListRefChange).
1254
+ this.hub?.broadcast({
1255
+ type: 'list-ref-change',
1256
+ action,
1257
+ queryId: encodeRecordId(queryId),
1258
+ recordId: encodeRecordId(childId),
1259
+ version,
1260
+ parent: true,
1261
+ });
1262
+
1263
+ const existing = this.dataModule.getQueryById(queryId);
1264
+ if (!existing) return;
1265
+
1266
+ const item = { id: childId, version };
1267
+ await this.syncEngine.syncRecords(
1268
+ action === 'CREATE'
1269
+ ? { added: [item], updated: [], removed: [] }
1270
+ : { added: [], updated: [item], removed: [] }
1271
+ );
1272
+
1273
+ // Keep the in-memory child array in step so the poll's idempotent diff
1274
+ // doesn't re-fetch this body on the next tick.
1275
+ const key = encodeRecordId(childId);
1276
+ const prev = existing.config.subqueryRemoteArray ?? [];
1277
+ existing.config.subqueryRemoteArray = [...prev.filter(([id]) => id !== key), [key, version]];
157
1278
  }
158
1279
 
159
1280
  /**
@@ -164,38 +1285,67 @@ export class SpookySync<S extends SchemaStructure> {
164
1285
  this.scheduler.enqueueDownEvent(event);
165
1286
  }
166
1287
 
1288
+ /**
1289
+ * Bound a mutation push so it always settles.
1290
+ *
1291
+ * `SyncScheduler.syncUp` early-returns while `isSyncingUp` is true, and that
1292
+ * flag only clears in the `finally` of the drain loop. A push whose RPC never
1293
+ * settles (socket dropped mid-flight, response lost) therefore wedges the
1294
+ * up-queue for the rest of the session: no retry, no error, no further
1295
+ * mutation ever sent. A timeout turns that into an ordinary network failure,
1296
+ * which `UpQueue.next` re-queues for the next trigger. The message deliberately
1297
+ * contains "timed out" so `classifySyncError` treats it as `network` and
1298
+ * retries rather than rolling the mutation back.
1299
+ */
1300
+ private withPushTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
1301
+ return withTimeout(
1302
+ promise,
1303
+ this.pushTimeoutMs,
1304
+ `Mutation push timed out after ${this.pushTimeoutMs}ms (${label})`
1305
+ );
1306
+ }
1307
+
167
1308
  private async processUpEvent(event: UpEvent) {
168
1309
  this.logger.debug(
169
- { event, Category: 'spooky-client::SpookySync::processUpEvent' },
1310
+ { event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
170
1311
  'Processing up event'
171
1312
  );
172
- console.log('xx1', event);
173
1313
  switch (event.type) {
174
- case 'create':
1314
+ case 'create': {
175
1315
  const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
176
1316
  const prefixedParams = Object.fromEntries(
177
1317
  dataKeys.map(({ key, variable }) => [variable, event.data[key]])
178
1318
  );
179
1319
  const query = surql.seal(surql.createSet('id', dataKeys));
180
- await this.remote.query(query, {
181
- id: event.record_id,
182
- ...prefixedParams,
183
- });
1320
+ await this.withPushTimeout(
1321
+ this.remote.query(query, {
1322
+ id: event.record_id,
1323
+ ...prefixedParams,
1324
+ }),
1325
+ 'create'
1326
+ );
184
1327
  break;
1328
+ }
185
1329
  case 'update':
186
- await this.remote.query(`UPDATE $id MERGE $data`, {
187
- id: event.record_id,
188
- data: event.data,
189
- });
1330
+ await this.withPushTimeout(
1331
+ this.remote.query(`UPDATE $id MERGE $data`, {
1332
+ id: event.record_id,
1333
+ data: event.data,
1334
+ }),
1335
+ 'update'
1336
+ );
190
1337
  break;
191
1338
  case 'delete':
192
- await this.remote.query(`DELETE $id`, {
193
- id: event.record_id,
194
- });
1339
+ await this.withPushTimeout(
1340
+ this.remote.query(`DELETE $id`, {
1341
+ id: event.record_id,
1342
+ }),
1343
+ 'delete'
1344
+ );
195
1345
  break;
196
1346
  default:
197
1347
  this.logger.error(
198
- { event, Category: 'spooky-client::SpookySync::processUpEvent' },
1348
+ { event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
199
1349
  'processUpEvent unknown event type'
200
1350
  );
201
1351
  return;
@@ -205,9 +1355,7 @@ export class SpookySync<S extends SchemaStructure> {
205
1355
  private async handleRollback(event: UpEvent, error: Error): Promise<void> {
206
1356
  const recordId = encodeRecordId(event.record_id);
207
1357
  const tableName =
208
- event.type === 'create' && event.tableName
209
- ? event.tableName
210
- : extractTablePart(recordId);
1358
+ event.type === 'create' && event.tableName ? event.tableName : extractTablePart(recordId);
211
1359
 
212
1360
  this.logger.warn(
213
1361
  {
@@ -215,7 +1363,7 @@ export class SpookySync<S extends SchemaStructure> {
215
1363
  recordId,
216
1364
  tableName,
217
1365
  error: error.message,
218
- Category: 'spooky-client::SpookySync::handleRollback',
1366
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
219
1367
  },
220
1368
  'Rolling back failed mutation'
221
1369
  );
@@ -231,7 +1379,7 @@ export class SpookySync<S extends SchemaStructure> {
231
1379
  this.logger.warn(
232
1380
  {
233
1381
  recordId,
234
- Category: 'spooky-client::SpookySync::handleRollback',
1382
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
235
1383
  },
236
1384
  'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
237
1385
  );
@@ -241,7 +1389,7 @@ export class SpookySync<S extends SchemaStructure> {
241
1389
  this.logger.warn(
242
1390
  {
243
1391
  recordId,
244
- Category: 'spooky-client::SpookySync::handleRollback',
1392
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
245
1393
  },
246
1394
  'Delete rollback not implemented. Down-sync will reconcile.'
247
1395
  );
@@ -253,11 +1401,26 @@ export class SpookySync<S extends SchemaStructure> {
253
1401
  recordId,
254
1402
  error: error.message,
255
1403
  });
1404
+
1405
+ // Shared-tabs: the store rollback above already propagated to every tab
1406
+ // via the ingest relay; additionally deliver the EVENT to the tab that
1407
+ // owns the mutation so its UI (toasts, subscribeToRollbacks) fires there.
1408
+ const mutationId = encodeRecordId(event.mutation_id);
1409
+ const owner = mutationOwnerTabId(mutationId);
1410
+ if (this.hub && owner && this.tabId && owner !== this.tabId) {
1411
+ this.hub.sendTo(owner, {
1412
+ type: 'mutation-rolled-back',
1413
+ mutationId,
1414
+ recordId,
1415
+ eventType: event.type,
1416
+ error: error.message,
1417
+ });
1418
+ }
256
1419
  }
257
1420
 
258
1421
  private async processDownEvent(event: DownEvent) {
259
1422
  this.logger.debug(
260
- { event, Category: 'spooky-client::SpookySync::processDownEvent' },
1423
+ { event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
261
1424
  'Processing down event'
262
1425
  );
263
1426
  switch (event.type) {
@@ -281,7 +1444,7 @@ export class SpookySync<S extends SchemaStructure> {
281
1444
  const queryState = this.dataModule.getQueryByHash(hash);
282
1445
  if (!queryState) {
283
1446
  this.logger.warn(
284
- { hash, Category: 'spooky-client::SpookySync::syncQuery' },
1447
+ { hash, Category: 'sp00ky-client::Sp00kySync::syncQuery' },
285
1448
  'Query not found'
286
1449
  );
287
1450
  return;
@@ -295,7 +1458,107 @@ export class SpookySync<S extends SchemaStructure> {
295
1458
  if (!diff) {
296
1459
  return;
297
1460
  }
298
- return this.syncEngine.syncRecords(diff);
1461
+ return this.runSyncForQuery(hash, diff);
1462
+ }
1463
+
1464
+ /**
1465
+ * Run a sync for a single query while reflecting its fetch status. Marks the
1466
+ * query `fetching` for the duration when the diff actually pulls records
1467
+ * (added/updated), then resets to `idle` in a `finally` so a failed sync
1468
+ * never leaves a query stuck `fetching`. Part A's notification coalescing
1469
+ * means the single resulting UI update lands after this completes.
1470
+ */
1471
+ private async runSyncForQuery(hash: string, diff: RecordVersionDiff): Promise<void> {
1472
+ // Don't let sync re-add a record the user just deleted locally. The remote
1473
+ // delete is queued in the outbox, so until it's processed the server's
1474
+ // `_00_list_ref` still lists the record — the diff then classifies it as
1475
+ // `added` (present remotely, absent locally) and `syncRecords` re-fetches +
1476
+ // re-inserts it, so a deleted database reappears a few seconds later. Drop
1477
+ // any id with a pending local DELETE from the re-add paths. Once the remote
1478
+ // delete lands, the pending row clears and the server drops it from
1479
+ // `_00_list_ref`, so this guard naturally stops applying.
1480
+ if (diff.added.length > 0 || diff.updated.length > 0) {
1481
+ const pendingDeletes = await this.getPendingDeleteIds();
1482
+ if (pendingDeletes.size > 0) {
1483
+ diff = {
1484
+ added: diff.added.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
1485
+ updated: diff.updated.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
1486
+ removed: diff.removed,
1487
+ };
1488
+ }
1489
+ }
1490
+
1491
+ const fetching = diff.added.length + diff.updated.length > 0;
1492
+ if (fetching) {
1493
+ this.dataModule.beginFetching(hash);
1494
+ }
1495
+ try {
1496
+ const { remoteFetchMs, stillRemoteIds } = await this.syncEngine.syncRecords(diff);
1497
+ if (fetching) {
1498
+ this.dataModule.recordRemoteFetch(hash, remoteFetchMs);
1499
+ }
1500
+ // Converge localArray to the authoritative remoteArray for ids that left
1501
+ // the server's list_ref but still exist — a view-membership change, not a
1502
+ // delete — so the poll's diff stops re-flagging them every tick (the `job:`
1503
+ // churn). CRUCIAL: only converge after the id has been still-remote for
1504
+ // several CONSECUTIVE rounds. A record that's merely mid-deletion is
1505
+ // still-remote for ~one round (its delete hasn't committed when our
1506
+ // existence check races it) and is gone the next round → it never reaches
1507
+ // the threshold, so it's deleted normally instead of being stranded here.
1508
+ if (stillRemoteIds.length > 0) {
1509
+ const CONVERGE_AFTER = 3;
1510
+ const toConverge: string[] = [];
1511
+ for (const id of stillRemoteIds) {
1512
+ const key = `${hash}:${id}`;
1513
+ const n = (this.stillRemoteStreaks.get(key) ?? 0) + 1;
1514
+ if (n >= CONVERGE_AFTER) {
1515
+ this.stillRemoteStreaks.delete(key);
1516
+ toConverge.push(id);
1517
+ } else {
1518
+ this.stillRemoteStreaks.set(key, n);
1519
+ }
1520
+ }
1521
+ if (toConverge.length > 0) {
1522
+ const qs = this.dataModule.getQueryByHash(hash);
1523
+ const local = qs?.config.localArray;
1524
+ if (local && local.length > 0) {
1525
+ const drop = new Set(toConverge);
1526
+ const next = local.filter(([id]) => !drop.has(id));
1527
+ if (next.length !== local.length) {
1528
+ await this.dataModule.updateQueryLocalArray(hash, next);
1529
+ }
1530
+ }
1531
+ }
1532
+ }
1533
+ } finally {
1534
+ if (fetching) {
1535
+ // Land the coalesced result BEFORE flipping to idle: the final stream
1536
+ // update sits on a debounce timer, and an `idle` that races ahead of it
1537
+ // would let consumers treat a partially-filled window as authoritative.
1538
+ try {
1539
+ await this.dataModule.flushPendingStreamUpdate(hash);
1540
+ } catch (err) {
1541
+ this.logger.warn(
1542
+ { err, hash, Category: 'sp00ky-client::Sp00kySync::runSyncForQuery' },
1543
+ 'Failed to flush pending stream update before idle'
1544
+ );
1545
+ }
1546
+ this.dataModule.endFetching(hash);
1547
+ }
1548
+ }
1549
+ }
1550
+
1551
+ /**
1552
+ * Record ids with a pending local DELETE in the outbox (`_00_pending_mutations`).
1553
+ * Sync must not re-fetch/re-insert these — the remote delete is async, so the
1554
+ * server's `_00_list_ref` still lists them until it's processed, and the diff
1555
+ * would otherwise resurrect a just-deleted record.
1556
+ */
1557
+ private async getPendingDeleteIds(): Promise<Set<string>> {
1558
+ // Single implementation, shared with the render path: `materializeRecords`
1559
+ // subtracts the same set so a row whose DELETE is still in the outbox is
1560
+ // neither re-fetched here nor rendered there.
1561
+ return (await this.dataModule.getPendingRecordIds()).deletes;
299
1562
  }
300
1563
 
301
1564
  /**
@@ -303,26 +1566,46 @@ export class SpookySync<S extends SchemaStructure> {
303
1566
  * @param mutations Array of UpEvents (create/update/delete) to enqueue.
304
1567
  */
305
1568
  public async enqueueMutation(mutations: UpEvent[]) {
1569
+ // Follower: the outbox rows are already committed in the SHARED store (the
1570
+ // mutation tx went through the leader's worker); only the leader drains,
1571
+ // so hand over the ids instead of queueing locally. A notify lost in a
1572
+ // failover window is covered by the new leader's loadFromDatabase.
1573
+ if (this.tabRole === 'follower') {
1574
+ for (const m of mutations) {
1575
+ this.forwarder?.mutationEnqueued(encodeRecordId(m.mutation_id));
1576
+ }
1577
+ return;
1578
+ }
306
1579
  this.scheduler.enqueueMutation(mutations);
307
1580
  }
308
1581
 
309
1582
  private async registerQuery(queryHash: string) {
1583
+ // Hold `fetching` across the WHOLE registration (remote view creation +
1584
+ // initial sync + post-sync notify). A query is born `fetching` in
1585
+ // createNewQuery; this refcounted cycle is what resolves it to `idle` — so
1586
+ // consumers (e.g. useQuery's `isSettled`) never see an idle query whose
1587
+ // window is still empty/partially materialized.
1588
+ this.dataModule.beginFetching(queryHash);
310
1589
  try {
311
1590
  this.logger.debug(
312
- { queryHash, Category: 'spooky-client::SpookySync::registerQuery' },
1591
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
313
1592
  'Register Query state'
314
1593
  );
315
1594
  await this.createRemoteQuery(queryHash);
316
1595
  await this.syncQuery(queryHash);
317
- // Always notify after sync completes — handles empty result sets
318
- // where no stream updates fire but the UI needs to stop loading
1596
+ // Land any still-debounced stream result, then always notify — handles
1597
+ // empty result sets where no stream updates fire but the UI needs to
1598
+ // stop loading.
1599
+ await this.dataModule.flushPendingStreamUpdate(queryHash);
319
1600
  await this.dataModule.notifyQuerySynced(queryHash);
320
1601
  } catch (e) {
321
1602
  this.logger.error(
322
- { err: e, Category: 'spooky-client::SpookySync::registerQuery' },
1603
+ { err: e, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
323
1604
  'registerQuery error'
324
1605
  );
325
1606
  throw e;
1607
+ } finally {
1608
+ this.dataModule.endFetching(queryHash);
326
1609
  }
327
1610
  }
328
1611
 
@@ -331,15 +1614,15 @@ export class SpookySync<S extends SchemaStructure> {
331
1614
 
332
1615
  if (!queryState) {
333
1616
  this.logger.warn(
334
- { queryHash, Category: 'spooky-client::SpookySync::createRemoteQuery' },
1617
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::createRemoteQuery' },
335
1618
  'Query to register not found'
336
1619
  );
337
1620
  throw new Error('Query to register not found');
338
1621
  }
339
- // Delegate to remote function which handles DBSP registration & persistence
1622
+ // Delegate to remote function which handles DBSP registration & persistence.
1623
+ // clientId is set server-side from session::id() — see fn::query::register.
340
1624
  await this.remote.query('fn::query::register($config)', {
341
1625
  config: {
342
- clientId: this.clientId,
343
1626
  id: queryState.config.id,
344
1627
  surql: queryState.config.surql,
345
1628
  params: queryState.config.params,
@@ -347,8 +1630,14 @@ export class SpookySync<S extends SchemaStructure> {
347
1630
  },
348
1631
  });
349
1632
 
1633
+ // Initial materialized-view fetch — pull from the same per-user
1634
+ // `_00_list_ref_user_<id>` (or global `_00_list_ref` in single
1635
+ // mode) that the LIVE subscription listens on, so the two stay in
1636
+ // sync. `parent IS NONE` excludes subquery entries; the
1637
+ // `localArray` cache only tracks primary records.
1638
+ const listRefTbl = this.listRefTable();
350
1639
  const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
351
- surql.selectByFieldsAnd('_spooky_list_ref', ['in'], ['out', 'version']),
1640
+ buildListRefSelect(listRefTbl),
352
1641
  {
353
1642
  in: queryState.config.id,
354
1643
  }
@@ -358,7 +1647,7 @@ export class SpookySync<S extends SchemaStructure> {
358
1647
  {
359
1648
  queryId: encodeRecordId(queryState.config.id),
360
1649
  items,
361
- Category: 'spooky-client::SpookySync::createRemoteQuery',
1650
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
362
1651
  },
363
1652
  'Got query record version array from remote'
364
1653
  );
@@ -369,7 +1658,7 @@ export class SpookySync<S extends SchemaStructure> {
369
1658
  {
370
1659
  queryId: encodeRecordId(queryState.config.id),
371
1660
  array,
372
- Category: 'spooky-client::SpookySync::createRemoteQuery',
1661
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
373
1662
  },
374
1663
  'createdRemoteQuery'
375
1664
  );
@@ -378,13 +1667,75 @@ export class SpookySync<S extends SchemaStructure> {
378
1667
  /// Incantation existed already
379
1668
  await this.dataModule.updateQueryRemoteArray(queryHash, array);
380
1669
  }
1670
+
1671
+ // Pull the bodies of any `.related()` subquery children into the local
1672
+ // cache. The primary fetch above (`parent IS NONE`) tracks only window
1673
+ // rows, so without this a cold-reload re-materialization of the
1674
+ // correlated surql finds no child rows and related fields come back
1675
+ // empty. Best-effort: never fail registration over it.
1676
+ await this.syncSubqueryChildren(queryHash).catch((err) => {
1677
+ this.logger.info(
1678
+ {
1679
+ err: (err as Error)?.message ?? err,
1680
+ queryHash,
1681
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
1682
+ },
1683
+ 'Subquery child sync failed during registration; poll will retry'
1684
+ );
1685
+ });
1686
+ }
1687
+
1688
+ /**
1689
+ * Sync the BODIES of a `.related()` query's subquery child rows into the
1690
+ * local cache, separately from the primary window array. The SSP writes
1691
+ * each matched child as a `_00_list_ref` edge tagged `parent`/`parent_rel`;
1692
+ * `buildSubqueryListRefSelect` pulls those `out`+`version` pairs (any
1693
+ * nesting depth). We diff against the in-memory `subqueryRemoteArray` and
1694
+ * fetch added/updated bodies through the SyncEngine — which `saveBatch`s
1695
+ * them into the local DB AND the in-browser SSP, whose subquery-table
1696
+ * dependency then re-materializes the parent view (no explicit notify).
1697
+ *
1698
+ * Deletion safety: we pass `removed: []` deliberately. A child body can be
1699
+ * shared by other queries; letting `handleRemovedRecords` delete one that
1700
+ * merely left THIS query's child set would clobber data another query still
1701
+ * shows. Genuine record deletes flow through the normal delete path; a
1702
+ * lingering orphan body is invisible (the correlated WHERE stops matching).
1703
+ *
1704
+ * Kept off `runSyncForQuery` on purpose so child fetches never flip the
1705
+ * query to `fetching` or skew its DevTools timings.
1706
+ */
1707
+ private async syncSubqueryChildren(queryHash: string): Promise<void> {
1708
+ const queryState = this.dataModule.getQueryByHash(queryHash);
1709
+ if (!queryState) return;
1710
+
1711
+ const listRefTbl = this.listRefTable();
1712
+ const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
1713
+ buildSubqueryListRefSelect(listRefTbl),
1714
+ { in: queryState.config.id }
1715
+ );
1716
+ if (!Array.isArray(items)) return;
1717
+
1718
+ const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
1719
+ const prev = queryState.config.subqueryRemoteArray ?? [];
1720
+ if (recordVersionArraysEqual(fresh, prev)) return; // idempotent: nothing new
1721
+
1722
+ const diff = diffRecordVersionArray(prev, fresh);
1723
+ if (diff.added.length > 0 || diff.updated.length > 0) {
1724
+ await this.syncEngine.syncRecords({
1725
+ added: diff.added,
1726
+ updated: diff.updated,
1727
+ removed: [], // never delete child bodies here — see method doc
1728
+ });
1729
+ }
1730
+ // In-memory only — child rows must never enter the persisted primary array.
1731
+ queryState.config.subqueryRemoteArray = fresh;
381
1732
  }
382
1733
 
383
- private async heartbeatQuery(queryHash: string) {
1734
+ public async heartbeatQuery(queryHash: string) {
384
1735
  const queryState = this.dataModule.getQueryByHash(queryHash);
385
1736
  if (!queryState) {
386
1737
  this.logger.warn(
387
- { queryHash, Category: 'spooky-client::SpookySync::heartbeatQuery' },
1738
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::heartbeatQuery' },
388
1739
  'Query to register not found'
389
1740
  );
390
1741
  throw new Error('Query to register not found');
@@ -394,17 +1745,30 @@ export class SpookySync<S extends SchemaStructure> {
394
1745
  });
395
1746
  }
396
1747
 
1748
+ // Eager teardown of a deregistered query's remote `_00_query` view (opt-in,
1749
+ // e.g. a viewport-windowed list cancelling an off-screen window). Query ids
1750
+ // are a deterministic hash of (surql+params), so a DELETE racing a scroll-back
1751
+ // re-register (same id) could nuke a freshly-recreated view — hence two
1752
+ // guards: abort if a subscriber reappeared BEFORE the delete; re-register if
1753
+ // one reappears DURING the delete's network await. Tolerant of a
1754
+ // missing/already-gone query (no throw).
397
1755
  private async cleanupQuery(queryHash: string) {
398
1756
  const queryState = this.dataModule.getQueryByHash(queryHash);
399
- if (!queryState) {
400
- this.logger.warn(
401
- { queryHash, Category: 'spooky-client::SpookySync::cleanupQuery' },
402
- 'Query to register not found'
403
- );
404
- throw new Error('Query to register not found');
1757
+ if (!queryState) return; // already torn down / never registered
1758
+
1759
+ // Re-subscribed before the queued cleanup ran → keep everything as-is.
1760
+ if (this.dataModule.hasSubscribers(queryHash)) return;
1761
+
1762
+ await this.remote.query(`DELETE $id`, { id: queryState.config.id });
1763
+
1764
+ // Re-subscribed while we awaited the DELETE → the remote view is now gone
1765
+ // but a subscriber needs it; recreate it instead of leaving a zombie.
1766
+ if (this.dataModule.hasSubscribers(queryHash)) {
1767
+ this.enqueueDownEvent({ type: 'register', payload: { hash: queryHash } });
1768
+ return;
405
1769
  }
406
- await this.remote.query(`DELETE $id`, {
407
- id: queryState.config.id,
408
- });
1770
+
1771
+ // No subscribers throughout → safe to free the local view + state.
1772
+ this.dataModule.finalizeDeregister(queryHash);
409
1773
  }
410
1774
  }