@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.160

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/AGENTS.md +56 -0
  2. package/dist/index.d.ts +1406 -364
  3. package/dist/index.js +7754 -1390
  4. package/dist/otel/index.d.ts +21 -0
  5. package/dist/otel/index.js +86 -0
  6. package/dist/sqlite-open.js +276 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +420 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +434 -0
  11. package/dist/types.d.ts +874 -0
  12. package/package.json +20 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/sp00ky-core/SKILL.md +258 -0
  15. package/skills/sp00ky-core/references/auth.md +98 -0
  16. package/skills/sp00ky-core/references/config.md +76 -0
  17. package/src/build-globals.d.ts +12 -0
  18. package/src/events/events.test.ts +2 -1
  19. package/src/events/index.ts +3 -0
  20. package/src/index.ts +16 -2
  21. package/src/modules/app-release/index.test.ts +125 -0
  22. package/src/modules/app-release/index.ts +201 -0
  23. package/src/modules/auth/events/index.ts +2 -1
  24. package/src/modules/auth/index.ts +59 -20
  25. package/src/modules/cache/index.ts +112 -32
  26. package/src/modules/cache/types.ts +2 -2
  27. package/src/modules/crdt/crdt-field.ts +294 -0
  28. package/src/modules/crdt/crdt-hydration.test.ts +206 -0
  29. package/src/modules/crdt/index.ts +361 -0
  30. package/src/modules/crdt/loro-loader.ts +25 -0
  31. package/src/modules/data/data.hydration.test.ts +142 -0
  32. package/src/modules/data/data.rebind.test.ts +147 -0
  33. package/src/modules/data/data.run.test.ts +113 -0
  34. package/src/modules/data/data.status.test.ts +249 -0
  35. package/src/modules/data/index.ts +1151 -129
  36. package/src/modules/data/mutation-id.test.ts +25 -0
  37. package/src/modules/data/mutation-id.ts +35 -0
  38. package/src/modules/data/window-query.test.ts +52 -0
  39. package/src/modules/data/window-query.ts +154 -0
  40. package/src/modules/devtools/index.ts +325 -37
  41. package/src/modules/devtools/notify-throttle.test.ts +149 -0
  42. package/src/modules/devtools/storage-info.test.ts +79 -0
  43. package/src/modules/devtools/storage-info.ts +143 -0
  44. package/src/modules/devtools/versions.test.ts +74 -0
  45. package/src/modules/devtools/versions.ts +81 -0
  46. package/src/modules/feature-flag/index.test.ts +120 -0
  47. package/src/modules/feature-flag/index.ts +209 -0
  48. package/src/modules/ref-tables.test.ts +91 -0
  49. package/src/modules/ref-tables.ts +88 -0
  50. package/src/modules/sync/engine.ts +101 -37
  51. package/src/modules/sync/events/index.ts +9 -2
  52. package/src/modules/sync/queue/queue-down.ts +12 -5
  53. package/src/modules/sync/queue/queue-up.forwarded.test.ts +137 -0
  54. package/src/modules/sync/queue/queue-up.ts +198 -57
  55. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  56. package/src/modules/sync/scheduler.ts +77 -9
  57. package/src/modules/sync/sync.health.test.ts +149 -0
  58. package/src/modules/sync/sync.subquery.test.ts +82 -0
  59. package/src/modules/sync/sync.ts +1333 -92
  60. package/src/modules/sync/utils.test.ts +269 -2
  61. package/src/modules/sync/utils.ts +182 -17
  62. package/src/otel/index.ts +127 -0
  63. package/src/services/database/cache-engine.ts +160 -0
  64. package/src/services/database/database.ts +11 -11
  65. package/src/services/database/engine-factory.ts +33 -0
  66. package/src/services/database/events/index.ts +2 -1
  67. package/src/services/database/index.ts +6 -0
  68. package/src/services/database/local-migrator.ts +28 -27
  69. package/src/services/database/local.test.ts +64 -0
  70. package/src/services/database/local.ts +478 -67
  71. package/src/services/database/plan-render.test.ts +159 -0
  72. package/src/services/database/plan-render.ts +108 -0
  73. package/src/services/database/relation-resolver.test.ts +413 -0
  74. package/src/services/database/relation-resolver.ts +0 -0
  75. package/src/services/database/remote.ts +13 -13
  76. package/src/services/database/sqlite-cache-engine.test.ts +464 -0
  77. package/src/services/database/sqlite-cache-engine.ts +1154 -0
  78. package/src/services/database/sqlite-open.test.ts +150 -0
  79. package/src/services/database/sqlite-open.ts +164 -0
  80. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  81. package/src/services/database/sqlite-plan-sql.ts +106 -0
  82. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  83. package/src/services/database/sqlite-select.test.ts +246 -0
  84. package/src/services/database/sqlite-select.ts +113 -0
  85. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  86. package/src/services/database/sqlite-transport.ts +219 -0
  87. package/src/services/database/sqlite-worker.ts +437 -0
  88. package/src/services/database/surql-translate.ts +291 -0
  89. package/src/services/database/surreal-cache-engine.ts +141 -0
  90. package/src/services/logger/index.ts +6 -109
  91. package/src/services/persistence/localstorage.ts +2 -2
  92. package/src/services/persistence/resilient.ts +11 -4
  93. package/src/services/persistence/surrealdb.ts +10 -10
  94. package/src/services/stream-processor/index.ts +444 -52
  95. package/src/services/stream-processor/permissions.test.ts +47 -0
  96. package/src/services/stream-processor/permissions.ts +53 -0
  97. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  98. package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
  99. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  100. package/src/services/stream-processor/wasm-types.ts +23 -2
  101. package/src/services/tabs/broker-client.ts +283 -0
  102. package/src/services/tabs/broker.test.ts +278 -0
  103. package/src/services/tabs/coordinator.test.ts +192 -0
  104. package/src/services/tabs/coordinator.ts +567 -0
  105. package/src/services/tabs/fake-ports.fixture.ts +70 -0
  106. package/src/services/tabs/leader-locks.ts +75 -0
  107. package/src/services/tabs/protocol.ts +239 -0
  108. package/src/services/tabs/support.ts +36 -0
  109. package/src/services/tabs/tabs-broker-worker.ts +581 -0
  110. package/src/sp00ky.auth-order.test.ts +92 -0
  111. package/src/sp00ky.init-query.test.ts +183 -0
  112. package/src/sp00ky.ts +1225 -0
  113. package/src/types.ts +342 -15
  114. package/src/utils/error-classification.test.ts +44 -0
  115. package/src/utils/error-classification.ts +7 -0
  116. package/src/utils/index.ts +35 -13
  117. package/src/utils/parser.ts +3 -2
  118. package/src/utils/semver.test.ts +32 -0
  119. package/src/utils/semver.ts +30 -0
  120. package/src/utils/surql.ts +30 -18
  121. package/src/utils/withRetry.test.ts +1 -1
  122. package/tsdown.config.ts +86 -1
  123. package/src/spooky.ts +0 -395
@@ -1,33 +1,175 @@
1
- import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
- import { MutationEvent, RecordVersionArray } from '../../types';
1
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
2
+ import type {
3
+ RecordVersionArray,
4
+ RecordVersionDiff,
5
+ SyncHealth,
6
+ SyncHealthStatus,
7
+ } from '../../types';
3
8
  import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
4
- import { Logger } from '../../services/logger/index';
5
- import { DownEvent, DownQueue, UpEvent, UpQueue } from './queue/index';
6
- import { RecordId, Uuid } from 'surrealdb';
7
- import { ArraySyncer, createDiffFromDbOp } from './utils';
9
+ import type { Logger } from '../../services/logger/index';
10
+ import type { DownEvent, UpEvent } from './queue/index';
11
+ import { DownQueue, UpQueue } from './queue/index';
12
+ import type { RecordId, Uuid } from 'surrealdb';
13
+ import {
14
+ ArraySyncer,
15
+ buildListRefSelect,
16
+ buildSubqueryListRefSelect,
17
+ createDiffFromDbOp,
18
+ diffRecordVersionArray,
19
+ listRefPollDelayMs,
20
+ recordVersionArraysEqual,
21
+ resolveListRefPollInterval,
22
+ } from './utils';
8
23
  import { SyncEngine } from './engine';
9
24
  import { SyncScheduler } from './scheduler';
10
- import { SchemaStructure } from '@spooky-sync/query-builder';
11
- import { CacheModule } from '../cache/index';
12
- import { DataModule } from '../data/index';
13
- import { encodeRecordId, extractTablePart, parseDuration, surql } from '../../utils/index';
25
+ import type { SchemaStructure } from '@spooky-sync/query-builder';
26
+ import type { CacheModule } from '../cache/index';
27
+ import type { DataModule } from '../data/index';
28
+ import {
29
+ classifySyncError,
30
+ encodeRecordId,
31
+ extractIdPart,
32
+ extractTablePart,
33
+ surql,
34
+ } from '../../utils/index';
35
+ import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
36
+ import { mutationOwnerTabId } from '../data/mutation-id';
37
+ import type { LeaderSyncHub, SyncForwarder } from '../../services/tabs/coordinator';
38
+ import { parseRecordIdString } from '../../utils/index';
14
39
 
15
40
  /**
16
- * The main synchronization engine for Spooky.
41
+ * Tunables for `Sp00kySync` construction.
42
+ */
43
+ export interface Sp00kySyncOptions {
44
+ /**
45
+ * Cadence (ms) for the `_00_list_ref` poll fallback that catches
46
+ * cross-session UPDATEs the LIVE-permission gap drops. Non-positive
47
+ * values fall back to the default; see
48
+ * {@link resolveListRefPollInterval}.
49
+ */
50
+ refSyncIntervalMs?: number;
51
+ /**
52
+ * Enable realtime sync for unauthenticated clients against the shared
53
+ * `_00_list_ref_anon` table. See {@link Sp00kyConfig.enableAnonymousLiveQueries}.
54
+ * Defaults to `false`.
55
+ */
56
+ anonymousLiveQueries?: boolean;
57
+ /**
58
+ * Consecutive failed sync rounds before sync health flips to `degraded`.
59
+ * `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
60
+ * Defaults to `3`.
61
+ */
62
+ degradeAfterConsecutiveFailures?: number;
63
+ /**
64
+ * Max time a single mutation push may take before it is treated as a network
65
+ * failure and retried. Guards against an RPC that never settles wedging the
66
+ * up-queue for the session. Defaults to 30000; `0` disables the timeout.
67
+ */
68
+ pushTimeoutMs?: number;
69
+ }
70
+
71
+ /**
72
+ * The main synchronization engine for Sp00ky.
17
73
  * Handles the bidirectional synchronization between the local database and the remote backend.
18
74
  * Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
19
75
  * @template S The schema structure type.
20
76
  */
21
- export class SpookySync<S extends SchemaStructure> {
22
- private clientId: string = '';
77
+ export class Sp00kySync<S extends SchemaStructure> {
23
78
  private upQueue: UpQueue;
24
79
  private downQueue: DownQueue;
25
80
  private isInit: boolean = false;
26
81
  private logger: Logger;
27
82
  private syncEngine: SyncEngine;
83
+ /** Engine-level events (e.g. `SYNC_REMOTE_DATA_INGESTED`). Distinct
84
+ * from `this.events`, which carries Sp00kySync-level events like
85
+ * `SYNC_QUERY_UPDATED` and `SYNC_MUTATION_ROLLED_BACK`. */
86
+ public get engineEvents() {
87
+ return this.syncEngine.events;
88
+ }
28
89
  private scheduler: SyncScheduler;
90
+ private wasDisconnected: boolean = false;
29
91
  public events = createSyncEventSystem();
30
92
 
93
+ // Auth identity that drives per-user `_00_list_ref_user_<id>` routing
94
+ // in `RefMode.Dedicated`. Updated by `setCurrentUserId` from the auth
95
+ // subscription in `Sp00kyClient`; null when unauthenticated.
96
+ private currentUserId: string | null = null;
97
+
98
+ // ---- shared-tabs role state ----
99
+ // Followers keep their own remote WS (registration, per-query sync, poll)
100
+ // but must never drain the shared outbox or hold a second list_ref LIVE.
101
+ // The leader relays its LIVE events and routes rollbacks by mutation owner.
102
+ private tabRole: 'solo' | 'leader' | 'follower' = 'solo';
103
+ private tabId: string | null = null;
104
+ private hub: LeaderSyncHub | null = null;
105
+ private forwarder: SyncForwarder | null = null;
106
+
107
+ private refMode: RefMode = DEFAULT_REF_MODE;
108
+
109
+ // When true, an unauthenticated client still runs the `_00_list_ref` poll
110
+ // and LIVE subscription, routed to the shared `_00_list_ref_anon` table, so
111
+ // a logged-out page gets realtime `useQuery` updates. Off by default.
112
+ private readonly anonLiveEnabled: boolean;
113
+
114
+ // Bookkeeping for the LIVE subscription on `_00_list_ref[_user_*]`.
115
+ // SurrealDB binds the permission context at LIVE-registration time and
116
+ // the table name in dedicated mode depends on the authenticated user,
117
+ // so we have to re-register whenever auth state flips.
118
+ private currentLiveQueryUuid: Uuid | null = null;
119
+ private liveQueryUnsubscribe: (() => void) | null = null;
120
+
121
+ // Periodic re-poll of `_00_list_ref` as a safety net for missed LIVE
122
+ // notifications. SurrealDB v3 occasionally drops LIVE deliveries
123
+ // across sessions even when the row matches the permission rule;
124
+ // this catches those without requiring users to reload. The
125
+ // interval is configurable via the constructor; see
126
+ // `resolveListRefPollInterval` for fallback semantics.
127
+ //
128
+ // Self-rescheduling rather than setInterval so each tick can pick
129
+ // its own delay via `nextPollDelayMs` — slows the poll down when
130
+ // LIVE is delivering events and speeds it back up when LIVE quiets.
131
+ private listRefPollTimer: ReturnType<typeof setTimeout> | null = null;
132
+ private listRefPollRunning: boolean = false;
133
+ // The currently-executing poll tick, if any. `stopListRefPoll` only stops
134
+ // future ticks; a bucket switch must also AWAIT the in-flight one so its
135
+ // local writes land in the store it started against.
136
+ private listRefPollInFlight: Promise<void> | null = null;
137
+ public readonly refSyncIntervalMs: number;
138
+
139
+ // Consecutive poll cycles that observed NO list_ref change. Drives the
140
+ // adaptive backoff in `startListRefPoll` via `listRefPollDelayMs`: an idle
141
+ // page coasts from the fast base cadence toward the 5s cap, and any activity
142
+ // (a poll-detected change or a LIVE event) resets it to 0 so the poll snaps
143
+ // back to responsive. Replaces the old LIVE-liveness backoff, which kept the
144
+ // poll pinned at 500ms forever whenever LIVE wasn't firing (the common case
145
+ // on a quiet page, thanks to the cross-session LIVE-permission gap).
146
+ private listRefIdleStreak: number = 0;
147
+
148
+ // `${queryHash}:${recordId}` -> consecutive rounds the id has been "still
149
+ // remote" (left the query's list_ref but still exists upstream). Used to
150
+ // distinguish a PERSISTENT view-membership disagreement (the `job:` churn,
151
+ // converged once it crosses the threshold) from a record that's merely
152
+ // mid-deletion (still-remote for ~one round, then gone) — which must NOT be
153
+ // converged, or it gets stranded in this window before its delete is observed.
154
+ private stillRemoteStreaks: Map<string, number> = new Map();
155
+
156
+ // Wall-clock timestamp (ms) of the most recent LIVE event delivered
157
+ // through `handleRemoteListRefChange`. Kept as a diagnostic / liveness
158
+ // signal; the poll cadence is now driven by `listRefIdleStreak`.
159
+ private lastLiveEventAt: number | null = null;
160
+
161
+ // Number of times the initial `_00_list_ref[_user_*]` LIVE subscription
162
+ // had to retry on `setCurrentUserId`. Stays at 0 when the SSP has
163
+ // pre-emptively created the user's dedicated tables; otherwise
164
+ // increments on each retry attempt until LIVE succeeds or attempts
165
+ // are exhausted. Surfaced as a diagnostic so the e2e suite can prove
166
+ // the pre-emptive table-creation path is keeping the first sign-in
167
+ // off the lazy-creation race.
168
+ private _liveRetryCount: number = 0;
169
+ public get liveRetryCount(): number {
170
+ return this._liveRetryCount;
171
+ }
172
+
31
173
  get isSyncing() {
32
174
  return this.scheduler.isSyncing;
33
175
  }
@@ -37,13 +179,11 @@ export class SpookySync<S extends SchemaStructure> {
37
179
  }
38
180
 
39
181
  subscribeToPendingMutations(cb: (count: number) => void): () => void {
40
- const id1 = this.upQueue.events.subscribe(
41
- SyncQueueEventTypes.MutationEnqueued,
42
- (event) => cb(event.payload.queueSize)
182
+ const id1 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, (event) =>
183
+ cb(event.payload.queueSize)
43
184
  );
44
- const id2 = this.upQueue.events.subscribe(
45
- SyncQueueEventTypes.MutationDequeued,
46
- (event) => cb(event.payload.queueSize)
185
+ const id2 = this.upQueue.events.subscribe(SyncQueueEventTypes.MutationDequeued, (event) =>
186
+ cb(event.payload.queueSize)
47
187
  );
48
188
  return () => {
49
189
  this.upQueue.events.unsubscribe(id1);
@@ -51,16 +191,192 @@ export class SpookySync<S extends SchemaStructure> {
51
191
  };
52
192
  }
53
193
 
194
+ // ---- Sync health -------------------------------------------------------
195
+ // `0` disables degraded reporting (config `syncHealth: false`). Resolved
196
+ // from config in Sp00kyClient and passed through the constructor options.
197
+ private readonly degradeAfterFailures: number;
198
+ /** Per-push RPC deadline; see {@link withPushTimeout}. */
199
+ private readonly pushTimeoutMs: number;
200
+ private consecutiveSyncFailures = 0;
201
+ private syncHealthStatus: SyncHealthStatus = 'healthy';
202
+ private lastSyncErrorKind: 'network' | 'application' | undefined;
203
+ private lastSyncErrorMessage: string | undefined;
204
+ // Latched `true` on the first successful sync round; never reset. Lets a UI
205
+ // tell a cold-start "connecting" phase (never reached the server) apart from
206
+ // a real lost connection after a working session.
207
+ private hasSyncedOnce = false;
208
+
209
+ // Self-heal: while degraded, re-drive sync on an exponential backoff so the
210
+ // app recovers on its own — even when the socket never actually dropped (in
211
+ // that case no `connected` event fires, so this re-registration is the ONLY
212
+ // thing that re-probes the server). Started on the degrade transition,
213
+ // cleared on recovery. Capped cadence so a long outage doesn't busy-loop.
214
+ private selfHealTimer: ReturnType<typeof setTimeout> | null = null;
215
+ private selfHealAttempts = 0;
216
+ private static readonly SELF_HEAL_BASE_MS = 2_000;
217
+ private static readonly SELF_HEAL_MAX_MS = 30_000;
218
+
219
+ /** Current sync-health snapshot. */
220
+ get syncHealth(): SyncHealth {
221
+ return {
222
+ status: this.syncHealthStatus,
223
+ consecutiveFailures: this.consecutiveSyncFailures,
224
+ kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
225
+ error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
226
+ everConnected: this.hasSyncedOnce,
227
+ };
228
+ }
229
+
230
+ /**
231
+ * Observe sync health. The callback fires immediately with the current
232
+ * status and again on every healthy↔degraded transition. Returns an
233
+ * unsubscribe. Mirrors {@link subscribeToPendingMutations}.
234
+ */
235
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
236
+ cb(this.syncHealth);
237
+ const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
238
+ cb(event.payload)
239
+ );
240
+ return () => this.events.unsubscribe(id);
241
+ }
242
+
243
+ private emitSyncHealth(): void {
244
+ this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
245
+ }
246
+
247
+ /**
248
+ * Fed by the scheduler once per drained sync round. Individual failures are
249
+ * absorbed by the queue's retry; only a run of `degradeAfterFailures`
250
+ * consecutive failures flips the status to `degraded`, and the next clean
251
+ * round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
252
+ * is 0).
253
+ */
254
+ private recordSyncOutcome(ok: boolean, error?: unknown): void {
255
+ if (this.degradeAfterFailures <= 0) return;
256
+ if (ok) {
257
+ // Latch first-ever success so a UI can drop the connecting phase. Set
258
+ // before the early return so a clean cold start (0 prior failures) counts.
259
+ this.hasSyncedOnce = true;
260
+ if (this.consecutiveSyncFailures === 0) return;
261
+ this.consecutiveSyncFailures = 0;
262
+ if (this.syncHealthStatus !== 'healthy') {
263
+ this.syncHealthStatus = 'healthy';
264
+ this.lastSyncErrorKind = undefined;
265
+ this.lastSyncErrorMessage = undefined;
266
+ this.stopSelfHeal();
267
+ this.logger.info(
268
+ { Category: 'sp00ky-client::Sp00kySync::syncHealth' },
269
+ 'Sync recovered; health back to healthy'
270
+ );
271
+ this.emitSyncHealth();
272
+ }
273
+ return;
274
+ }
275
+ this.consecutiveSyncFailures++;
276
+ this.lastSyncErrorKind = classifySyncError(error);
277
+ this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
278
+ if (
279
+ this.syncHealthStatus !== 'degraded' &&
280
+ this.consecutiveSyncFailures >= this.degradeAfterFailures
281
+ ) {
282
+ this.syncHealthStatus = 'degraded';
283
+ this.logger.warn(
284
+ {
285
+ consecutiveFailures: this.consecutiveSyncFailures,
286
+ kind: this.lastSyncErrorKind,
287
+ error,
288
+ Category: 'sp00ky-client::Sp00kySync::syncHealth',
289
+ },
290
+ 'Sync degraded after sustained failures'
291
+ );
292
+ this.emitSyncHealth();
293
+ this.startSelfHeal();
294
+ }
295
+ }
296
+
297
+ /**
298
+ * Begin self-heal retries (no-op if already running). Started on the
299
+ * healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
300
+ */
301
+ private startSelfHeal(): void {
302
+ if (this.selfHealTimer !== null) return;
303
+ this.selfHealAttempts = 0;
304
+ this.scheduleSelfHeal();
305
+ }
306
+
307
+ private scheduleSelfHeal(): void {
308
+ const delay = Math.min(
309
+ Sp00kySync.SELF_HEAL_MAX_MS,
310
+ Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts
311
+ );
312
+ this.selfHealTimer = setTimeout(async () => {
313
+ this.selfHealTimer = null;
314
+ if (this.syncHealthStatus !== 'degraded') return;
315
+ this.selfHealAttempts++;
316
+ this.logger.debug(
317
+ {
318
+ attempt: this.selfHealAttempts,
319
+ delayMs: delay,
320
+ Category: 'sp00ky-client::Sp00kySync::selfHeal',
321
+ },
322
+ 'Self-heal: re-driving sync while degraded'
323
+ );
324
+ try {
325
+ // Retry whatever is still queued first; the failing op (register or
326
+ // mutation) was re-queued by the queue, so this re-probes the server
327
+ // and reports the outcome through the scheduler → recordSyncOutcome.
328
+ if (this.upQueue.size > 0) {
329
+ await this.scheduler.syncUp();
330
+ } else if (this.downQueue.size > 0) {
331
+ await this.scheduler.syncDown();
332
+ } else {
333
+ // Nothing queued (e.g. the failing op was rolled back + dropped):
334
+ // re-register active queries — mirroring the reconnect handler — so
335
+ // there's a concrete op whose success flips health. If there are no
336
+ // active queries either, probe connectivity directly.
337
+ const hashes = this.dataModule.getActiveQueryHashes();
338
+ if (hashes.length > 0) {
339
+ for (const hash of hashes) {
340
+ this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
341
+ }
342
+ await this.scheduler.syncDown();
343
+ } else {
344
+ await this.remote.query('RETURN true');
345
+ this.recordSyncOutcome(true);
346
+ }
347
+ }
348
+ } catch (err) {
349
+ // Only the direct connectivity probe can throw here (syncUp/syncDown
350
+ // swallow + self-report); treat a probe failure as another failed round.
351
+ this.recordSyncOutcome(false, err);
352
+ }
353
+ // Keep retrying until recovery. recordSyncOutcome(true) calls stopSelfHeal
354
+ // (clearing any pending timer), so only continue while still degraded.
355
+ if (this.syncHealthStatus === 'degraded') this.scheduleSelfHeal();
356
+ }, delay);
357
+ }
358
+
359
+ private stopSelfHeal(): void {
360
+ if (this.selfHealTimer !== null) {
361
+ clearTimeout(this.selfHealTimer);
362
+ this.selfHealTimer = null;
363
+ }
364
+ this.selfHealAttempts = 0;
365
+ }
366
+
54
367
  constructor(
55
- private local: LocalDatabaseService,
368
+ private local: LocalStore,
56
369
  private remote: RemoteDatabaseService,
57
370
  private cache: CacheModule,
58
371
  private dataModule: DataModule<S>,
59
372
  private schema: S,
60
- logger: Logger
373
+ logger: Logger,
374
+ options?: Sp00kySyncOptions
61
375
  ) {
62
- this.logger = logger.child({ service: 'SpookySync' });
63
- this.upQueue = new UpQueue(this.local, this.logger);
376
+ this.logger = logger.child({ service: 'Sp00kySync' });
377
+ this.upQueue = new UpQueue(this.local, this.logger, (dropped) =>
378
+ this.onMutationDropped(dropped)
379
+ );
64
380
  this.downQueue = new DownQueue(this.local, this.logger);
65
381
  this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
66
382
  this.scheduler = new SyncScheduler(
@@ -69,43 +385,622 @@ export class SpookySync<S extends SchemaStructure> {
69
385
  this.processUpEvent.bind(this),
70
386
  this.processDownEvent.bind(this),
71
387
  this.logger,
72
- this.handleRollback.bind(this)
388
+ this.handleRollback.bind(this),
389
+ this.recordSyncOutcome.bind(this)
73
390
  );
391
+ this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
392
+ this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
393
+ this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
394
+ this.pushTimeoutMs = Math.max(0, options?.pushTimeoutMs ?? 30_000);
74
395
  }
75
396
 
76
397
  /**
77
398
  * Initializes the synchronization system.
78
399
  * Starts the scheduler and initiates the initial sync cycles.
79
- * @param clientId The unique identifier for this client instance.
80
400
  * @throws Error if already initialized.
81
401
  */
82
- public async init(clientId: string) {
83
- if (this.isInit) throw new Error('SpookySync is already initialized');
84
- this.clientId = clientId;
402
+ public async init() {
403
+ if (this.isInit) throw new Error('Sp00kySync is already initialized');
85
404
  this.isInit = true;
86
- await this.scheduler.init();
87
- void this.scheduler.syncUp();
405
+ await this.scheduler.init({ loadOutbox: this.tabRole !== 'follower' });
406
+ this.subscribeToReconnect();
88
407
  void this.scheduler.syncUp();
89
408
  void this.scheduler.syncDown();
90
- void this.startRefLiveQueries();
409
+ // No initial LIVE subscription — wait for `setCurrentUserId` to fire
410
+ // from the auth subscription. In dedicated mode the table name
411
+ // depends on the authenticated user, and an unauthenticated
412
+ // subscription wouldn't match any of the per-user tables anyway.
413
+ //
414
+ // Exception: when anonymous live queries are enabled, start realtime now
415
+ // against the shared `_00_list_ref_anon` table so a logged-out client
416
+ // syncs immediately. `setCurrentUserId` re-points LIVE to the per-user
417
+ // table on sign-in. Guard on `currentUserId` because the auth callback can
418
+ // fire (and authenticate) before `init()` runs — don't clobber that back
419
+ // to the anon table. `setCurrentUserId(null)` is a no-op on first load
420
+ // (it's already null), so this is the only place anon realtime starts.
421
+ if (this.anonLiveEnabled && !this.currentUserId) {
422
+ this.startListRefPoll();
423
+ this.restartRefLiveQuery().catch((err) => {
424
+ this.logger.debug(
425
+ { err, Category: 'sp00ky-client::Sp00kySync::init' },
426
+ 'Anonymous ref LIVE start failed; relying on periodic poll fallback'
427
+ );
428
+ });
429
+ }
430
+ }
431
+
432
+ // ---- shared-tabs roles ------------------------------------------------------
433
+
434
+ /** Set BEFORE init(): shapes what init boots (a follower loads no outbox and
435
+ * never starts LIVE; its own registration/poll paths stay untouched). */
436
+ public setTabContext(role: 'solo' | 'leader' | 'follower', tabId: string | null): void {
437
+ this.tabRole = role;
438
+ this.tabId = tabId;
439
+ }
440
+
441
+ /** Leader duties: drain the shared outbox, own the single list_ref LIVE,
442
+ * relay LIVE events and rollbacks to followers via `hub`. Idempotent for a
443
+ * boot-time leader; a runtime promotion (failover) reloads the outbox,
444
+ * which now holds EVERY tab's rows, and restarts LIVE under this session. */
445
+ public async promoteToLeader(hub: LeaderSyncHub): Promise<void> {
446
+ this.tabRole = 'leader';
447
+ this.hub = hub;
448
+ this.forwarder = null;
449
+ hub.onFollowerMessage = (tabId, msg) => {
450
+ void tabId;
451
+ switch (msg.type) {
452
+ case 'sync-hello':
453
+ break;
454
+ case 'mutation-enqueued':
455
+ void this.enqueueForwardedMutation(msg.mutationId);
456
+ break;
457
+ case 'request-poll':
458
+ this.listRefIdleStreak = 0;
459
+ break;
460
+ }
461
+ };
462
+ if (this.isInit) {
463
+ await this.upQueue.loadFromDatabase();
464
+ void this.scheduler.syncUp();
465
+ if (this.currentUserId || this.anonLiveEnabled) {
466
+ this.startListRefPoll();
467
+ await this.restartRefLiveQuery().catch((err) => {
468
+ this.logger.warn(
469
+ { err, Category: 'sp00ky-client::Sp00kySync::promoteToLeader' },
470
+ 'LIVE restart failed on promotion; poll fallback covers it'
471
+ );
472
+ });
473
+ }
474
+ }
475
+ }
476
+
477
+ /** Follower duties: no outbox drain, no LIVE. Mutations forward to the
478
+ * leader; everything else (registration, per-query sync, poll) runs
479
+ * against this tab's own remote session as usual. */
480
+ public demoteToFollower(forwarder: SyncForwarder): void {
481
+ this.tabRole = 'follower';
482
+ this.hub = null;
483
+ this.forwarder = forwarder;
484
+ void this.killRefLiveQuery();
485
+ forwarder.onLeaderMessage = (msg) => {
486
+ switch (msg.type) {
487
+ case 'list-ref-change':
488
+ void this.applyRelayedListRefChange(msg).catch((err) => {
489
+ this.logger.error(
490
+ { err, Category: 'sp00ky-client::Sp00kySync::relay' },
491
+ 'Relayed list_ref change failed'
492
+ );
493
+ });
494
+ break;
495
+ case 'mutation-rolled-back':
496
+ this.events.emit(SyncEventTypes.MutationRolledBack, {
497
+ eventType: msg.eventType,
498
+ recordId: msg.recordId,
499
+ error: msg.error,
500
+ });
501
+ break;
502
+ default:
503
+ // db-ready and ingest-relay are handled by the engine/cache wiring
504
+ // in Sp00kyClient before messages reach this handler.
505
+ break;
506
+ }
507
+ };
508
+ }
509
+
510
+ /**
511
+ * A pending mutation was discarded because it can never be sent.
512
+ *
513
+ * This is a lost write, so it must not stay invisible. Every failure in this
514
+ * chain used to be a `logger.error` an app running `logLevel: 'fatal'` never
515
+ * shows, which is how an outbox could sit undrained for hours with the UI
516
+ * reporting nothing. Surfaces as a rollback event (the mutation will never
517
+ * apply, which is what a subscriber needs to know) and degrades sync health.
518
+ */
519
+ private onMutationDropped(dropped: {
520
+ mutationId: string;
521
+ recordId?: string;
522
+ mutationType?: string;
523
+ reason: string;
524
+ }): void {
525
+ this.logger.error(
526
+ { ...dropped, Category: 'sp00ky-client::Sp00kySync::onMutationDropped' },
527
+ 'Dropped a pending mutation that can never be sent'
528
+ );
529
+ this.recordSyncOutcome(false, new Error(`dropped mutation: ${dropped.reason}`));
530
+ this.events.emit(SyncEventTypes.MutationRolledBack, {
531
+ eventType: (dropped.mutationType as 'create' | 'update' | 'delete') ?? 'update',
532
+ recordId: dropped.recordId ?? dropped.mutationId,
533
+ error: `dropped: ${dropped.reason}`,
534
+ });
535
+ }
536
+
537
+ /** A forwarded outbox row from a follower: load + drain it. Idempotent. */
538
+ public async enqueueForwardedMutation(mutationId: string): Promise<void> {
539
+ if (this.tabRole !== 'leader') return;
540
+ await this.upQueue.enqueueFromDatabase(mutationId);
541
+ }
542
+
543
+ /** A relayed `_00_list_ref` LIVE event: resolve against THIS tab's queries
544
+ * and run the exact same handling the LIVE subscription would have. */
545
+ private async applyRelayedListRefChange(msg: {
546
+ action: 'CREATE' | 'UPDATE' | 'DELETE';
547
+ queryId: string;
548
+ recordId: string;
549
+ version: number;
550
+ parent: boolean;
551
+ }): Promise<void> {
552
+ const queryId = parseRecordIdString(msg.queryId);
553
+ // Foreign query (another tab's session-salted hash): not ours, ignore.
554
+ if (!this.dataModule.getQueryById(queryId)) return;
555
+ const recordId = parseRecordIdString(msg.recordId);
556
+ if (msg.parent) {
557
+ await this.handleRemoteSubqueryChange(msg.action, queryId, recordId, msg.version);
558
+ } else {
559
+ await this.handleRemoteListRefChange(msg.action, queryId, recordId, msg.version);
560
+ }
561
+ }
562
+
563
+ /** One immediate poll cycle (failover convergence). */
564
+ public async forcePollRound(): Promise<void> {
565
+ this.listRefIdleStreak = 0;
566
+ await this.pollListRefForActiveQueries().catch(() => false);
567
+ }
568
+
569
+ /**
570
+ * Quiesce all sync activity ahead of a local-bucket switch. After this
571
+ * resolves, nothing in the sync module writes to the local store: the poll
572
+ * loop is stopped AND its in-flight tick awaited, LIVE is killed, debounce
573
+ * timers are cancelled (their outbox rows are already persisted), and the
574
+ * scheduler has drained its in-flight queue item — including that item's
575
+ * outbox-row delete, which must land in the OLD bucket. Queued down-events
576
+ * are dropped (they reference old-bucket query rows; the post-switch rebind
577
+ * re-enqueues registrations). The old user's un-pushed outbox is deliberately
578
+ * NOT drained: the remote session already belongs to the next user.
579
+ */
580
+ public async prepareBucketSwitch(): Promise<void> {
581
+ this.stopSelfHeal();
582
+ this.stopListRefPoll();
583
+ if (this.listRefPollInFlight) await this.listRefPollInFlight;
584
+ await this.killRefLiveQuery();
585
+ this.upQueue.clearDebounceTimers();
586
+ await this.scheduler.pause();
587
+ this.downQueue.clear();
588
+ this.stillRemoteStreaks.clear();
589
+ this.logger.info(
590
+ { Category: 'sp00ky-client::Sp00kySync::prepareBucketSwitch' },
591
+ 'Sync quiesced for bucket switch'
592
+ );
593
+ }
594
+
595
+ /**
596
+ * Resume syncing against the freshly-opened bucket: reload the mutation
597
+ * outbox from ITS `_00_pending_mutations` (the new user's own un-pushed
598
+ * offline work) and restart the scheduler. LIVE + the list_ref poll restart
599
+ * via the `setCurrentUserId` call that follows in the auth listener.
600
+ */
601
+ public async completeBucketSwitch(): Promise<void> {
602
+ await this.upQueue.loadFromDatabase();
603
+ this.scheduler.resume();
604
+ this.logger.info(
605
+ { Category: 'sp00ky-client::Sp00kySync::completeBucketSwitch' },
606
+ 'Sync resumed after bucket switch'
607
+ );
608
+ }
609
+
610
+ /**
611
+ * Push the authenticated user's record id from the parent client's
612
+ * auth subscription. Tears down the existing `_00_list_ref` LIVE (if
613
+ * any) and re-registers it under the new user's dedicated table so
614
+ * SurrealDB binds the permission rule under the post-flip auth
615
+ * context. Pass `null` on sign-out.
616
+ *
617
+ * The dedicated `_00_list_ref_user_<id>` table is created lazily by
618
+ * the SSP when the first query registration arrives, which may be
619
+ * concurrent with this call. We retry the LIVE registration with a
620
+ * short backoff so a "table not found" race resolves without
621
+ * surfacing as a permanent auth-loading hang.
622
+ */
623
+ public async setCurrentUserId(userId: string | null): Promise<void> {
624
+ if (this.currentUserId === userId) return;
625
+ this.currentUserId = userId;
626
+ if (!userId) {
627
+ if (this.anonLiveEnabled) {
628
+ // Signed out but anonymous realtime is on: keep the poll running and
629
+ // re-point LIVE from the (now stale) per-user table to the shared
630
+ // `_00_list_ref_anon`. `startListRefPoll` is idempotent; the poll
631
+ // re-resolves `listRefTable()` each tick so it follows automatically.
632
+ this.startListRefPoll();
633
+ await this.restartRefLiveQuery().catch((err) => {
634
+ this.logger.debug(
635
+ { err, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
636
+ 'Anonymous ref LIVE restart failed; relying on periodic poll fallback'
637
+ );
638
+ });
639
+ return;
640
+ }
641
+ await this.killRefLiveQuery();
642
+ this.stopListRefPoll();
643
+ return;
644
+ }
645
+ // Start periodic polling FIRST so we have a deterministic fallback
646
+ // even when LIVE registration fails or SurrealDB drops a delivery.
647
+ this.startListRefPoll();
648
+ // Try to start LIVE with backoff for low-latency delivery on the
649
+ // happy path; the poll handles the rest.
650
+ const attemptDelays = [0, 250, 500, 1000, 2000];
651
+ for (let i = 0; i < attemptDelays.length; i++) {
652
+ if (attemptDelays[i] > 0) {
653
+ this._liveRetryCount++;
654
+ await new Promise((r) => setTimeout(r, attemptDelays[i]));
655
+ }
656
+ try {
657
+ await this.restartRefLiveQuery();
658
+ return;
659
+ } catch (err) {
660
+ this.logger.debug(
661
+ { err, attempt: i + 1, Category: 'sp00ky-client::Sp00kySync::setCurrentUserId' },
662
+ 'Ref LIVE start failed; relying on periodic poll fallback'
663
+ );
664
+ }
665
+ }
666
+ }
667
+
668
+ private startListRefPoll(): void {
669
+ if (this.listRefPollRunning) return;
670
+ this.listRefPollRunning = true;
671
+ this.logger.debug(
672
+ {
673
+ intervalMs: this.refSyncIntervalMs,
674
+ Category: 'sp00ky-client::Sp00kySync::startListRefPoll',
675
+ },
676
+ 'list_ref poll loop started'
677
+ );
678
+ const schedule = (delayMs: number) => {
679
+ this.listRefPollTimer = setTimeout(async () => {
680
+ if (!this.listRefPollRunning) return;
681
+ let changed = false;
682
+ const tick = (async () => {
683
+ changed = await this.pollListRefForActiveQueries();
684
+ })();
685
+ this.listRefPollInFlight = tick.catch(() => {});
686
+ try {
687
+ await tick;
688
+ } finally {
689
+ this.listRefPollInFlight = null;
690
+ if (!this.listRefPollRunning) return;
691
+ // Reset the idle streak on any observed change so the poll snaps
692
+ // back to the fast base cadence; otherwise grow it so a quiet page
693
+ // backs off toward the cap. (`handleRemoteListRefChange` also resets
694
+ // it when a LIVE event lands.)
695
+ this.listRefIdleStreak = changed ? 0 : this.listRefIdleStreak + 1;
696
+ const next = listRefPollDelayMs({
697
+ idleStreak: this.listRefIdleStreak,
698
+ baseIntervalMs: this.refSyncIntervalMs,
699
+ });
700
+ schedule(next);
701
+ }
702
+ }, delayMs);
703
+ };
704
+ schedule(this.refSyncIntervalMs);
705
+ }
706
+
707
+ private stopListRefPoll(): void {
708
+ this.listRefPollRunning = false;
709
+ if (this.listRefPollTimer !== null) {
710
+ clearTimeout(this.listRefPollTimer);
711
+ this.listRefPollTimer = null;
712
+ }
713
+ }
714
+
715
+ /**
716
+ * One poll cycle: refetch `_00_list_ref` for every active query. Returns
717
+ * whether ANY query's remoteArray actually changed — the scheduler uses this
718
+ * to drive the adaptive idle backoff.
719
+ *
720
+ * Also the ONLY health signal that runs while the page is idle. Sync health is
721
+ * otherwise activity-driven (mutations/registrations via the scheduler,
722
+ * reconnect re-registration, self-heal), so on a quiet page a stale `degraded`
723
+ * would linger until the next mutation and a genuine idle drop would be
724
+ * invisible. We fold the cycle's aggregate reachability into `recordSyncOutcome`
725
+ * so idle health self-recovers (and self-degrades) with no user action. A clean
726
+ * cycle is idempotent when already healthy (`recordSyncOutcome` early-returns at
727
+ * `consecutiveSyncFailures === 0`), so a healthy idle page pays nothing.
728
+ */
729
+ private async pollListRefForActiveQueries(): Promise<boolean> {
730
+ const hashes = this.dataModule.getActiveQueryHashes();
731
+ if (hashes.length === 0) {
732
+ // No active queries to piggyback on, but health still needs a heartbeat —
733
+ // probe connectivity directly so an idle page with no live queries doesn't
734
+ // go blind. Cheap, and gated by the same adaptive backoff (≤5s idle cap).
735
+ try {
736
+ await this.remote.query('RETURN true');
737
+ this.recordSyncOutcome(true);
738
+ } catch (err) {
739
+ this.recordSyncOutcome(false, err);
740
+ }
741
+ return false;
742
+ }
743
+ let anyChanged = false;
744
+ // `reached` = the server answered at least once this cycle (a success, or an
745
+ // *application* error, which still proves reachability). `firstNetworkErr`
746
+ // holds the first network-classified failure. A cycle that only produced
747
+ // network errors reports the outcome as a down round; a mixed/app cycle counts
748
+ // as reached; an all-application cycle reports nothing (that's a query-shape
749
+ // fault owned by the registration path, not a reachability signal).
750
+ let reached = false;
751
+ let firstNetworkErr: unknown;
752
+ for (const hash of hashes) {
753
+ try {
754
+ if (await this.refetchListRefForQuery(hash)) anyChanged = true;
755
+ reached = true;
756
+ } catch (err) {
757
+ if (classifySyncError(err) === 'network') {
758
+ if (firstNetworkErr === undefined) firstNetworkErr = err;
759
+ } else {
760
+ reached = true;
761
+ }
762
+ this.logger.debug(
763
+ {
764
+ err: (err as Error)?.message ?? err,
765
+ hash,
766
+ Category: 'sp00ky-client::Sp00kySync::pollListRefForActiveQueries',
767
+ },
768
+ 'Per-query list_ref poll failed'
769
+ );
770
+ }
771
+ }
772
+ // Call the private outcome recorder directly rather than routing through the
773
+ // scheduler — the scheduler only reports on rounds that drained ≥1 queue item
774
+ // (`processedAny`), and this isn't a queue round.
775
+ if (reached) {
776
+ this.recordSyncOutcome(true);
777
+ } else if (firstNetworkErr !== undefined) {
778
+ this.recordSyncOutcome(false, firstNetworkErr);
779
+ }
780
+ return anyChanged;
781
+ }
782
+
783
+ /**
784
+ * Pull the upstream list_ref entries for `queryHash`, diff them
785
+ * against the local `remoteArray` cache, sync any added/updated rows
786
+ * through the SyncEngine, then persist the new remoteArray. This is
787
+ * the same shape `createRemoteQuery` does for its initial fetch and
788
+ * what `handleRemoteListRefChange` does per-LIVE-event — we reuse
789
+ * it on a timer as a fallback for missed LIVE notifications.
790
+ */
791
+ private async refetchListRefForQuery(queryHash: string): Promise<boolean> {
792
+ const queryState = this.dataModule.getQueryByHash(queryHash);
793
+ if (!queryState) return false;
794
+ const listRefTbl = this.listRefTable();
795
+ const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
796
+ buildListRefSelect(listRefTbl),
797
+ { in: queryState.config.id }
798
+ );
799
+ if (!Array.isArray(items)) return false;
800
+ const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
801
+ // Capture which ids LEFT the query's window (present in the cached
802
+ // remoteArray, absent from `fresh`) BEFORE we overwrite remoteArray — these
803
+ // are cross-window deletes (or rows that scrolled out). They drive the
804
+ // forced re-render below.
805
+ const prevRemote = queryState.config.remoteArray ?? [];
806
+ const freshIds = new Set(fresh.map(([id]) => id));
807
+ const removedIds = prevRemote.filter(([id]) => !freshIds.has(id)).map(([id]) => id);
808
+ // Idempotent poll: only persist the remoteArray when it actually changed.
809
+ // The poll runs continuously as a LIVE fallback, so on a quiet page `fresh`
810
+ // equals the cached array every tick — re-writing it (an `UPDATE _00_query`
811
+ // each cycle, per active query) was pure churn and the bulk of the idle
812
+ // traffic. `recordVersionArraysEqual` is order-insensitive because the
813
+ // list_ref SELECT has no `ORDER BY`.
814
+ const changed = !recordVersionArraysEqual(fresh, queryState.config.remoteArray);
815
+ if (changed) {
816
+ // Update the cached remoteArray so the next diff/sync sees the new state.
817
+ // `syncQuery` (below) then writes through `cache.saveBatch`, which UPSERTs
818
+ // the local DB row and ingests it into the in-browser SSP — the SSP's
819
+ // stream updates run `processStreamUpdate`, which re-queries the local DB
820
+ // and notifies subscribers. We skip an explicit `notifyQuerySynced`
821
+ // because that path races the stream-update path (can notify with stale
822
+ // records).
823
+ await this.dataModule.updateQueryRemoteArray(queryHash, fresh);
824
+ }
825
+ // Run `syncQuery` every tick regardless: it's a no-op when localArray has
826
+ // caught up to remoteArray (`if (!diff) return`, issues no query), but it
827
+ // covers the rare case where remoteArray is stable yet localArray is behind
828
+ // (a prior record fetch failed) — so a missed row still gets retried.
829
+ // For REMOVALS it runs the ids through `handleRemovedRecords`, which deletes
830
+ // confirmed-gone records from the local DB.
831
+ try {
832
+ await this.syncQuery(queryHash);
833
+ } catch (err) {
834
+ this.logger.info(
835
+ {
836
+ err: (err as Error)?.message ?? err,
837
+ queryHash,
838
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
839
+ },
840
+ 'syncQuery failed during poll'
841
+ );
842
+ }
843
+ // Cross-session fallback for `.related()` child rows: the LIVE-permission
844
+ // gap can drop child-edge notifications, so converge their bodies on the
845
+ // poll too (idempotent — no-op when nothing changed).
846
+ await this.syncSubqueryChildren(queryHash).catch((err) => {
847
+ this.logger.info(
848
+ {
849
+ err: (err as Error)?.message ?? err,
850
+ queryHash,
851
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
852
+ },
853
+ 'Subquery child sync failed during poll'
854
+ );
855
+ });
856
+ // A REMOVAL needs no record fetch, so unlike the added-row path it doesn't
857
+ // get a re-render from the SSP stream on this code path reliably (and the
858
+ // non-windowed window-0 query re-queries the local DB rather than the id-set).
859
+ // Force a re-materialize + notify so the deleted row drops from the list in
860
+ // this (second) window — the reliable, LIVE-independent cross-window path.
861
+ if (removedIds.length > 0) {
862
+ try {
863
+ await this.dataModule.notifyQuerySynced(queryHash);
864
+ } catch (err) {
865
+ this.logger.info(
866
+ {
867
+ err: (err as Error)?.message ?? err,
868
+ queryHash,
869
+ Category: 'sp00ky-client::Sp00kySync::refetchListRefForQuery',
870
+ },
871
+ 'notifyQuerySynced failed during poll-removal re-render'
872
+ );
873
+ }
874
+ }
875
+ return changed;
876
+ }
877
+
878
+ /**
879
+ * Resolve the current `_00_list_ref` table name for the active auth
880
+ * context. Public so the `createRemoteQuery` initial-fetch path can
881
+ * read from the right per-user table.
882
+ *
883
+ * Reads the user id from `DataModule` rather than the local mirror,
884
+ * because `DataModule.setCurrentUserId` runs synchronously from the
885
+ * auth callback (before any `await`), whereas `sync.setCurrentUserId`
886
+ * is async — the userQuery's initial fetch can fire between those
887
+ * two points and we need the correct table name immediately.
888
+ */
889
+ public listRefTable(): string {
890
+ const userId = this.dataModule.getCurrentUserId();
891
+ // Unauthenticated with the flag on → the shared `_00_list_ref_anon` table.
892
+ if (userId == null && this.anonLiveEnabled) {
893
+ return listRefTableFor(this.refMode, ANON_USER_ID);
894
+ }
895
+ return listRefTableFor(this.refMode, userId);
896
+ }
897
+
898
+ private async killRefLiveQuery(): Promise<void> {
899
+ if (this.liveQueryUnsubscribe) {
900
+ try {
901
+ this.liveQueryUnsubscribe();
902
+ } catch {
903
+ /* ignore */
904
+ }
905
+ this.liveQueryUnsubscribe = null;
906
+ }
907
+ if (this.currentLiveQueryUuid !== null) {
908
+ try {
909
+ await this.remote.query('KILL $u', { u: this.currentLiveQueryUuid });
910
+ } catch (err) {
911
+ this.logger.debug(
912
+ { err, Category: 'sp00ky-client::Sp00kySync::killRefLiveQuery' },
913
+ 'Prior LIVE KILL failed; continuing'
914
+ );
915
+ }
916
+ this.currentLiveQueryUuid = null;
917
+ }
918
+ }
919
+
920
+ private async restartRefLiveQuery(): Promise<void> {
921
+ await this.killRefLiveQuery();
922
+ await this.startRefLiveQueries();
923
+ }
924
+
925
+ // Only the connect that follows a prior disconnect counts as a
926
+ // reconnect; the initial connect after init() must not trigger a
927
+ // refetch storm.
928
+ private subscribeToReconnect() {
929
+ const client = this.remote.getClient();
930
+ client.subscribe('disconnected', () => {
931
+ this.wasDisconnected = true;
932
+ this.logger.info(
933
+ { Category: 'sp00ky-client::Sp00kySync::onDisconnect' },
934
+ 'Remote disconnected'
935
+ );
936
+ });
937
+ client.subscribe('connected', () => {
938
+ if (!this.wasDisconnected) return;
939
+ this.wasDisconnected = false;
940
+ this.logger.info(
941
+ { Category: 'sp00ky-client::Sp00kySync::onReconnect' },
942
+ 'Remote reconnected, refetching active queries'
943
+ );
944
+ for (const hash of this.dataModule.getActiveQueryHashes()) {
945
+ this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
946
+ }
947
+ // The WS reconnect leaves the server-side LIVE subscription dead — the
948
+ // re-enqueued `register` events only re-fetch initial state, they don't
949
+ // re-subscribe. Without this, LIVE never recovers after a reconnect and
950
+ // the poll silently becomes the sole sync path (and never backs off).
951
+ // Authenticated → per-user table; signed-out with anon live enabled →
952
+ // the shared `_00_list_ref_anon`. Otherwise there's no table to re-bind.
953
+ if (this.currentUserId || this.anonLiveEnabled) {
954
+ this.restartRefLiveQuery().catch((err) => {
955
+ this.logger.debug(
956
+ { err, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
957
+ 'LIVE restart after reconnect failed; relying on poll fallback'
958
+ );
959
+ });
960
+ }
961
+ });
91
962
  }
92
963
 
93
964
  private async startRefLiveQueries() {
965
+ // Shared-tabs follower: exactly one LIVE per user exists, on the leader;
966
+ // its events reach this tab through the relay.
967
+ if (this.tabRole === 'follower') return;
968
+ const tableName = this.listRefTable();
94
969
  this.logger.debug(
95
- { clientId: this.clientId, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
970
+ { tableName, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
96
971
  'Starting ref live queries'
97
972
  );
98
973
 
99
- const [queryUuid] = await this.remote.query<[Uuid]>(
100
- 'LIVE SELECT * FROM _spooky_list_ref'
101
- );
974
+ const [queryUuid] = await this.remote.query<[Uuid]>(`LIVE SELECT * FROM ${tableName}`);
975
+ this.currentLiveQueryUuid = queryUuid;
102
976
 
103
- (await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
977
+ const live = await this.remote.getClient().liveOf(queryUuid);
978
+ this.liveQueryUnsubscribe = live.subscribe((message) => {
104
979
  this.logger.debug(
105
- { message, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
980
+ { message, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
106
981
  'Live update received'
107
982
  );
108
983
  if (message.action === 'KILLED') return;
984
+ // Subquery child edges (rows with `parent` set) are NOT primary window
985
+ // rows — the client's `RecordVersionArray` only tracks primary rows, so
986
+ // routing them through `handleRemoteListRefChange` would surface them as
987
+ // spurious "added" diffs and pollute the window. Instead route them to
988
+ // the dedicated child-body sync so `.related()` data stays realtime
989
+ // cross-session (the LIVE-permission gap otherwise leaves it to the poll).
990
+ if ((message.value as { parent?: unknown }).parent != null) {
991
+ this.handleRemoteSubqueryChange(
992
+ message.action,
993
+ message.value.in as RecordId<string>,
994
+ message.value.out as RecordId<string>,
995
+ message.value.version as number
996
+ ).catch((err) => {
997
+ this.logger.error(
998
+ { err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
999
+ 'Error handling remote subquery change'
1000
+ );
1001
+ });
1002
+ return;
1003
+ }
109
1004
  this.handleRemoteListRefChange(
110
1005
  message.action,
111
1006
  message.value.in as RecordId<string>,
@@ -113,7 +1008,7 @@ export class SpookySync<S extends SchemaStructure> {
113
1008
  message.value.version as number
114
1009
  ).catch((err) => {
115
1010
  this.logger.error(
116
- { err, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
1011
+ { err, Category: 'sp00ky-client::Sp00kySync::startRefLiveQueries' },
117
1012
  'Error handling remote list ref change'
118
1013
  );
119
1014
  });
@@ -126,16 +1021,47 @@ export class SpookySync<S extends SchemaStructure> {
126
1021
  recordId: RecordId,
127
1022
  version: number
128
1023
  ) {
1024
+ // Any LIVE delivery is evidence of activity — a CREATE/UPDATE/DELETE on a
1025
+ // query's window, or a notification for an unknown local query. Reset the
1026
+ // poll's idle streak so it snaps back to the fast base cadence (the page
1027
+ // is clearly not idle), and record the timestamp as a liveness diagnostic.
1028
+ this.lastLiveEventAt = Date.now();
1029
+ this.listRefIdleStreak = 0;
1030
+
1031
+ // Shared-tabs leader: the list_ref table is USER-scoped, so this LIVE also
1032
+ // carries events for FOLLOWER tabs' queries (their own session-salted
1033
+ // hashes). Relay every primary event; each follower resolves the queryId
1034
+ // against its own DataModule and ignores foreign ones. Then continue with
1035
+ // this tab's own handling below.
1036
+ this.hub?.broadcast({
1037
+ type: 'list-ref-change',
1038
+ action,
1039
+ queryId: encodeRecordId(queryId),
1040
+ recordId: encodeRecordId(recordId),
1041
+ version,
1042
+ parent: false,
1043
+ });
1044
+
1045
+ // NOTE: DELETE is handled like CREATE/UPDATE below. When another window (or
1046
+ // this one) deletes a record, the server's SSP removes it from `_00_list_ref`
1047
+ // and the LIVE subscription delivers a DELETE here — `createDiffFromDbOp`
1048
+ // turns it into a `removed: [recordId]` diff so the row drops from the window
1049
+ // in realtime. (It was previously ignored, so other windows only caught up on
1050
+ // reload / the slow poll.)
129
1051
  const existing = this.dataModule.getQueryById(queryId);
130
1052
 
131
1053
  if (!existing) {
132
- this.logger.warn(
133
- {
134
- queryId: queryId.toString(),
135
- Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
136
- },
137
- 'Received remote update for unknown local query'
138
- );
1054
+ // With a hub attached, an unknown query is the NORMAL case (it belongs
1055
+ // to a follower tab); without one it still warrants the warning.
1056
+ if (!this.hub) {
1057
+ this.logger.warn(
1058
+ {
1059
+ queryId: queryId.toString(),
1060
+ Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
1061
+ },
1062
+ 'Received remote update for unknown local query'
1063
+ );
1064
+ }
139
1065
  return;
140
1066
  }
141
1067
 
@@ -148,12 +1074,65 @@ export class SpookySync<S extends SchemaStructure> {
148
1074
  recordId,
149
1075
  version,
150
1076
  localArray,
151
- Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
1077
+ Category: 'sp00ky-client::Sp00kySync::handleRemoteListRefChange',
152
1078
  },
153
1079
  'Live update is being processed'
154
1080
  );
155
1081
  const diff = createDiffFromDbOp(action, recordId, version, localArray);
156
- await this.syncEngine.syncRecords(diff);
1082
+ // `config.id` is `_00_query:<hash>`, so its id-part IS the query hash
1083
+ // (a SHA-256 over query content + sessionId) — the key DataModule uses.
1084
+ const hash = extractIdPart(existing.config.id);
1085
+ await this.runSyncForQuery(hash, diff);
1086
+ }
1087
+
1088
+ /**
1089
+ * Handle a LIVE change to a SUBQUERY child edge (a `_00_list_ref` row with
1090
+ * `parent` set) for a `.related()` query. Unlike primary rows, child rows
1091
+ * must NOT touch the query's `localArray`/`remoteArray`/`rowCount`; we only
1092
+ * keep the child BODY fresh in the local cache so the in-browser SSP's
1093
+ * subquery-table dependency re-materializes the parent view.
1094
+ *
1095
+ * CREATE/UPDATE fetch+upsert the child body. DELETE is intentionally a
1096
+ * no-op: a child leaving this query's set must not delete a body another
1097
+ * query may still show (see `syncSubqueryChildren` deletion-safety note);
1098
+ * a genuine record delete propagates via the normal delete path.
1099
+ */
1100
+ private async handleRemoteSubqueryChange(
1101
+ action: 'CREATE' | 'UPDATE' | 'DELETE',
1102
+ queryId: RecordId,
1103
+ childId: RecordId,
1104
+ version: number
1105
+ ) {
1106
+ this.lastLiveEventAt = Date.now();
1107
+ this.listRefIdleStreak = 0;
1108
+
1109
+ if (action === 'DELETE') return;
1110
+
1111
+ // Relay child-edge events too (see handleRemoteListRefChange).
1112
+ this.hub?.broadcast({
1113
+ type: 'list-ref-change',
1114
+ action,
1115
+ queryId: encodeRecordId(queryId),
1116
+ recordId: encodeRecordId(childId),
1117
+ version,
1118
+ parent: true,
1119
+ });
1120
+
1121
+ const existing = this.dataModule.getQueryById(queryId);
1122
+ if (!existing) return;
1123
+
1124
+ const item = { id: childId, version };
1125
+ await this.syncEngine.syncRecords(
1126
+ action === 'CREATE'
1127
+ ? { added: [item], updated: [], removed: [] }
1128
+ : { added: [], updated: [item], removed: [] }
1129
+ );
1130
+
1131
+ // Keep the in-memory child array in step so the poll's idempotent diff
1132
+ // doesn't re-fetch this body on the next tick.
1133
+ const key = encodeRecordId(childId);
1134
+ const prev = existing.config.subqueryRemoteArray ?? [];
1135
+ existing.config.subqueryRemoteArray = [...prev.filter(([id]) => id !== key), [key, version]];
157
1136
  }
158
1137
 
159
1138
  /**
@@ -164,38 +1143,78 @@ export class SpookySync<S extends SchemaStructure> {
164
1143
  this.scheduler.enqueueDownEvent(event);
165
1144
  }
166
1145
 
1146
+ /**
1147
+ * Bound a mutation push so it always settles.
1148
+ *
1149
+ * `SyncScheduler.syncUp` early-returns while `isSyncingUp` is true, and that
1150
+ * flag only clears in the `finally` of the drain loop. A push whose RPC never
1151
+ * settles (socket dropped mid-flight, response lost) therefore wedges the
1152
+ * up-queue for the rest of the session: no retry, no error, no further
1153
+ * mutation ever sent. A timeout turns that into an ordinary network failure,
1154
+ * which `UpQueue.next` re-queues for the next trigger. The message deliberately
1155
+ * contains "timed out" so `classifySyncError` treats it as `network` and
1156
+ * retries rather than rolling the mutation back.
1157
+ */
1158
+ private withPushTimeout<T>(promise: Promise<T>, label: string): Promise<T> {
1159
+ if (!(this.pushTimeoutMs > 0)) return promise;
1160
+ return new Promise<T>((resolve, reject) => {
1161
+ const timer = setTimeout(() => {
1162
+ reject(new Error(`Mutation push timed out after ${this.pushTimeoutMs}ms (${label})`));
1163
+ }, this.pushTimeoutMs);
1164
+ promise.then(
1165
+ (value) => {
1166
+ clearTimeout(timer);
1167
+ resolve(value);
1168
+ },
1169
+ (err) => {
1170
+ clearTimeout(timer);
1171
+ reject(err);
1172
+ }
1173
+ );
1174
+ });
1175
+ }
1176
+
167
1177
  private async processUpEvent(event: UpEvent) {
168
1178
  this.logger.debug(
169
- { event, Category: 'spooky-client::SpookySync::processUpEvent' },
1179
+ { event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
170
1180
  'Processing up event'
171
1181
  );
172
- console.log('xx1', event);
173
1182
  switch (event.type) {
174
- case 'create':
1183
+ case 'create': {
175
1184
  const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
176
1185
  const prefixedParams = Object.fromEntries(
177
1186
  dataKeys.map(({ key, variable }) => [variable, event.data[key]])
178
1187
  );
179
1188
  const query = surql.seal(surql.createSet('id', dataKeys));
180
- await this.remote.query(query, {
181
- id: event.record_id,
182
- ...prefixedParams,
183
- });
1189
+ await this.withPushTimeout(
1190
+ this.remote.query(query, {
1191
+ id: event.record_id,
1192
+ ...prefixedParams,
1193
+ }),
1194
+ 'create'
1195
+ );
184
1196
  break;
1197
+ }
185
1198
  case 'update':
186
- await this.remote.query(`UPDATE $id MERGE $data`, {
187
- id: event.record_id,
188
- data: event.data,
189
- });
1199
+ await this.withPushTimeout(
1200
+ this.remote.query(`UPDATE $id MERGE $data`, {
1201
+ id: event.record_id,
1202
+ data: event.data,
1203
+ }),
1204
+ 'update'
1205
+ );
190
1206
  break;
191
1207
  case 'delete':
192
- await this.remote.query(`DELETE $id`, {
193
- id: event.record_id,
194
- });
1208
+ await this.withPushTimeout(
1209
+ this.remote.query(`DELETE $id`, {
1210
+ id: event.record_id,
1211
+ }),
1212
+ 'delete'
1213
+ );
195
1214
  break;
196
1215
  default:
197
1216
  this.logger.error(
198
- { event, Category: 'spooky-client::SpookySync::processUpEvent' },
1217
+ { event, Category: 'sp00ky-client::Sp00kySync::processUpEvent' },
199
1218
  'processUpEvent unknown event type'
200
1219
  );
201
1220
  return;
@@ -205,9 +1224,7 @@ export class SpookySync<S extends SchemaStructure> {
205
1224
  private async handleRollback(event: UpEvent, error: Error): Promise<void> {
206
1225
  const recordId = encodeRecordId(event.record_id);
207
1226
  const tableName =
208
- event.type === 'create' && event.tableName
209
- ? event.tableName
210
- : extractTablePart(recordId);
1227
+ event.type === 'create' && event.tableName ? event.tableName : extractTablePart(recordId);
211
1228
 
212
1229
  this.logger.warn(
213
1230
  {
@@ -215,7 +1232,7 @@ export class SpookySync<S extends SchemaStructure> {
215
1232
  recordId,
216
1233
  tableName,
217
1234
  error: error.message,
218
- Category: 'spooky-client::SpookySync::handleRollback',
1235
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
219
1236
  },
220
1237
  'Rolling back failed mutation'
221
1238
  );
@@ -231,7 +1248,7 @@ export class SpookySync<S extends SchemaStructure> {
231
1248
  this.logger.warn(
232
1249
  {
233
1250
  recordId,
234
- Category: 'spooky-client::SpookySync::handleRollback',
1251
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
235
1252
  },
236
1253
  'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
237
1254
  );
@@ -241,7 +1258,7 @@ export class SpookySync<S extends SchemaStructure> {
241
1258
  this.logger.warn(
242
1259
  {
243
1260
  recordId,
244
- Category: 'spooky-client::SpookySync::handleRollback',
1261
+ Category: 'sp00ky-client::Sp00kySync::handleRollback',
245
1262
  },
246
1263
  'Delete rollback not implemented. Down-sync will reconcile.'
247
1264
  );
@@ -253,11 +1270,26 @@ export class SpookySync<S extends SchemaStructure> {
253
1270
  recordId,
254
1271
  error: error.message,
255
1272
  });
1273
+
1274
+ // Shared-tabs: the store rollback above already propagated to every tab
1275
+ // via the ingest relay; additionally deliver the EVENT to the tab that
1276
+ // owns the mutation so its UI (toasts, subscribeToRollbacks) fires there.
1277
+ const mutationId = encodeRecordId(event.mutation_id);
1278
+ const owner = mutationOwnerTabId(mutationId);
1279
+ if (this.hub && owner && this.tabId && owner !== this.tabId) {
1280
+ this.hub.sendTo(owner, {
1281
+ type: 'mutation-rolled-back',
1282
+ mutationId,
1283
+ recordId,
1284
+ eventType: event.type,
1285
+ error: error.message,
1286
+ });
1287
+ }
256
1288
  }
257
1289
 
258
1290
  private async processDownEvent(event: DownEvent) {
259
1291
  this.logger.debug(
260
- { event, Category: 'spooky-client::SpookySync::processDownEvent' },
1292
+ { event, Category: 'sp00ky-client::Sp00kySync::processDownEvent' },
261
1293
  'Processing down event'
262
1294
  );
263
1295
  switch (event.type) {
@@ -281,7 +1313,7 @@ export class SpookySync<S extends SchemaStructure> {
281
1313
  const queryState = this.dataModule.getQueryByHash(hash);
282
1314
  if (!queryState) {
283
1315
  this.logger.warn(
284
- { hash, Category: 'spooky-client::SpookySync::syncQuery' },
1316
+ { hash, Category: 'sp00ky-client::Sp00kySync::syncQuery' },
285
1317
  'Query not found'
286
1318
  );
287
1319
  return;
@@ -295,7 +1327,115 @@ export class SpookySync<S extends SchemaStructure> {
295
1327
  if (!diff) {
296
1328
  return;
297
1329
  }
298
- return this.syncEngine.syncRecords(diff);
1330
+ return this.runSyncForQuery(hash, diff);
1331
+ }
1332
+
1333
+ /**
1334
+ * Run a sync for a single query while reflecting its fetch status. Marks the
1335
+ * query `fetching` for the duration when the diff actually pulls records
1336
+ * (added/updated), then resets to `idle` in a `finally` so a failed sync
1337
+ * never leaves a query stuck `fetching`. Part A's notification coalescing
1338
+ * means the single resulting UI update lands after this completes.
1339
+ */
1340
+ private async runSyncForQuery(hash: string, diff: RecordVersionDiff): Promise<void> {
1341
+ // Don't let sync re-add a record the user just deleted locally. The remote
1342
+ // delete is queued in the outbox, so until it's processed the server's
1343
+ // `_00_list_ref` still lists the record — the diff then classifies it as
1344
+ // `added` (present remotely, absent locally) and `syncRecords` re-fetches +
1345
+ // re-inserts it, so a deleted database reappears a few seconds later. Drop
1346
+ // any id with a pending local DELETE from the re-add paths. Once the remote
1347
+ // delete lands, the pending row clears and the server drops it from
1348
+ // `_00_list_ref`, so this guard naturally stops applying.
1349
+ if (diff.added.length > 0 || diff.updated.length > 0) {
1350
+ const pendingDeletes = await this.getPendingDeleteIds();
1351
+ if (pendingDeletes.size > 0) {
1352
+ diff = {
1353
+ added: diff.added.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
1354
+ updated: diff.updated.filter((r) => !pendingDeletes.has(encodeRecordId(r.id))),
1355
+ removed: diff.removed,
1356
+ };
1357
+ }
1358
+ }
1359
+
1360
+ const fetching = diff.added.length + diff.updated.length > 0;
1361
+ if (fetching) {
1362
+ this.dataModule.beginFetching(hash);
1363
+ }
1364
+ try {
1365
+ const { remoteFetchMs, stillRemoteIds } = await this.syncEngine.syncRecords(diff);
1366
+ if (fetching) {
1367
+ this.dataModule.recordRemoteFetch(hash, remoteFetchMs);
1368
+ }
1369
+ // Converge localArray to the authoritative remoteArray for ids that left
1370
+ // the server's list_ref but still exist — a view-membership change, not a
1371
+ // delete — so the poll's diff stops re-flagging them every tick (the `job:`
1372
+ // churn). CRUCIAL: only converge after the id has been still-remote for
1373
+ // several CONSECUTIVE rounds. A record that's merely mid-deletion is
1374
+ // still-remote for ~one round (its delete hasn't committed when our
1375
+ // existence check races it) and is gone the next round → it never reaches
1376
+ // the threshold, so it's deleted normally instead of being stranded here.
1377
+ if (stillRemoteIds.length > 0) {
1378
+ const CONVERGE_AFTER = 3;
1379
+ const toConverge: string[] = [];
1380
+ for (const id of stillRemoteIds) {
1381
+ const key = `${hash}:${id}`;
1382
+ const n = (this.stillRemoteStreaks.get(key) ?? 0) + 1;
1383
+ if (n >= CONVERGE_AFTER) {
1384
+ this.stillRemoteStreaks.delete(key);
1385
+ toConverge.push(id);
1386
+ } else {
1387
+ this.stillRemoteStreaks.set(key, n);
1388
+ }
1389
+ }
1390
+ if (toConverge.length > 0) {
1391
+ const qs = this.dataModule.getQueryByHash(hash);
1392
+ const local = qs?.config.localArray;
1393
+ if (local && local.length > 0) {
1394
+ const drop = new Set(toConverge);
1395
+ const next = local.filter(([id]) => !drop.has(id));
1396
+ if (next.length !== local.length) {
1397
+ await this.dataModule.updateQueryLocalArray(hash, next);
1398
+ }
1399
+ }
1400
+ }
1401
+ }
1402
+ } finally {
1403
+ if (fetching) {
1404
+ // Land the coalesced result BEFORE flipping to idle: the final stream
1405
+ // update sits on a debounce timer, and an `idle` that races ahead of it
1406
+ // would let consumers treat a partially-filled window as authoritative.
1407
+ try {
1408
+ await this.dataModule.flushPendingStreamUpdate(hash);
1409
+ } catch (err) {
1410
+ this.logger.warn(
1411
+ { err, hash, Category: 'sp00ky-client::Sp00kySync::runSyncForQuery' },
1412
+ 'Failed to flush pending stream update before idle'
1413
+ );
1414
+ }
1415
+ this.dataModule.endFetching(hash);
1416
+ }
1417
+ }
1418
+ }
1419
+
1420
+ /**
1421
+ * Record ids with a pending local DELETE in the outbox (`_00_pending_mutations`).
1422
+ * Sync must not re-fetch/re-insert these — the remote delete is async, so the
1423
+ * server's `_00_list_ref` still lists them until it's processed, and the diff
1424
+ * would otherwise resurrect a just-deleted record.
1425
+ */
1426
+ private async getPendingDeleteIds(): Promise<Set<string>> {
1427
+ try {
1428
+ const [rows] = await this.local.query<[{ recordId: RecordId<string> }[]]>(
1429
+ "SELECT recordId FROM _00_pending_mutations WHERE mutationType = 'delete'"
1430
+ );
1431
+ return new Set((rows ?? []).map((r) => encodeRecordId(r.recordId)));
1432
+ } catch (err) {
1433
+ this.logger.warn(
1434
+ { err, Category: 'sp00ky-client::Sp00kySync::getPendingDeleteIds' },
1435
+ 'Failed to read pending deletes; sync may briefly resurrect a just-deleted record'
1436
+ );
1437
+ return new Set();
1438
+ }
299
1439
  }
300
1440
 
301
1441
  /**
@@ -303,26 +1443,46 @@ export class SpookySync<S extends SchemaStructure> {
303
1443
  * @param mutations Array of UpEvents (create/update/delete) to enqueue.
304
1444
  */
305
1445
  public async enqueueMutation(mutations: UpEvent[]) {
1446
+ // Follower: the outbox rows are already committed in the SHARED store (the
1447
+ // mutation tx went through the leader's worker); only the leader drains,
1448
+ // so hand over the ids instead of queueing locally. A notify lost in a
1449
+ // failover window is covered by the new leader's loadFromDatabase.
1450
+ if (this.tabRole === 'follower') {
1451
+ for (const m of mutations) {
1452
+ this.forwarder?.mutationEnqueued(encodeRecordId(m.mutation_id));
1453
+ }
1454
+ return;
1455
+ }
306
1456
  this.scheduler.enqueueMutation(mutations);
307
1457
  }
308
1458
 
309
1459
  private async registerQuery(queryHash: string) {
1460
+ // Hold `fetching` across the WHOLE registration (remote view creation +
1461
+ // initial sync + post-sync notify). A query is born `fetching` in
1462
+ // createNewQuery; this refcounted cycle is what resolves it to `idle` — so
1463
+ // consumers (e.g. useQuery's `isSettled`) never see an idle query whose
1464
+ // window is still empty/partially materialized.
1465
+ this.dataModule.beginFetching(queryHash);
310
1466
  try {
311
1467
  this.logger.debug(
312
- { queryHash, Category: 'spooky-client::SpookySync::registerQuery' },
1468
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
313
1469
  'Register Query state'
314
1470
  );
315
1471
  await this.createRemoteQuery(queryHash);
316
1472
  await this.syncQuery(queryHash);
317
- // Always notify after sync completes — handles empty result sets
318
- // where no stream updates fire but the UI needs to stop loading
1473
+ // Land any still-debounced stream result, then always notify — handles
1474
+ // empty result sets where no stream updates fire but the UI needs to
1475
+ // stop loading.
1476
+ await this.dataModule.flushPendingStreamUpdate(queryHash);
319
1477
  await this.dataModule.notifyQuerySynced(queryHash);
320
1478
  } catch (e) {
321
1479
  this.logger.error(
322
- { err: e, Category: 'spooky-client::SpookySync::registerQuery' },
1480
+ { err: e, Category: 'sp00ky-client::Sp00kySync::registerQuery' },
323
1481
  'registerQuery error'
324
1482
  );
325
1483
  throw e;
1484
+ } finally {
1485
+ this.dataModule.endFetching(queryHash);
326
1486
  }
327
1487
  }
328
1488
 
@@ -331,15 +1491,15 @@ export class SpookySync<S extends SchemaStructure> {
331
1491
 
332
1492
  if (!queryState) {
333
1493
  this.logger.warn(
334
- { queryHash, Category: 'spooky-client::SpookySync::createRemoteQuery' },
1494
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::createRemoteQuery' },
335
1495
  'Query to register not found'
336
1496
  );
337
1497
  throw new Error('Query to register not found');
338
1498
  }
339
- // Delegate to remote function which handles DBSP registration & persistence
1499
+ // Delegate to remote function which handles DBSP registration & persistence.
1500
+ // clientId is set server-side from session::id() — see fn::query::register.
340
1501
  await this.remote.query('fn::query::register($config)', {
341
1502
  config: {
342
- clientId: this.clientId,
343
1503
  id: queryState.config.id,
344
1504
  surql: queryState.config.surql,
345
1505
  params: queryState.config.params,
@@ -347,8 +1507,14 @@ export class SpookySync<S extends SchemaStructure> {
347
1507
  },
348
1508
  });
349
1509
 
1510
+ // Initial materialized-view fetch — pull from the same per-user
1511
+ // `_00_list_ref_user_<id>` (or global `_00_list_ref` in single
1512
+ // mode) that the LIVE subscription listens on, so the two stay in
1513
+ // sync. `parent IS NONE` excludes subquery entries; the
1514
+ // `localArray` cache only tracks primary records.
1515
+ const listRefTbl = this.listRefTable();
350
1516
  const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
351
- surql.selectByFieldsAnd('_spooky_list_ref', ['in'], ['out', 'version']),
1517
+ buildListRefSelect(listRefTbl),
352
1518
  {
353
1519
  in: queryState.config.id,
354
1520
  }
@@ -358,7 +1524,7 @@ export class SpookySync<S extends SchemaStructure> {
358
1524
  {
359
1525
  queryId: encodeRecordId(queryState.config.id),
360
1526
  items,
361
- Category: 'spooky-client::SpookySync::createRemoteQuery',
1527
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
362
1528
  },
363
1529
  'Got query record version array from remote'
364
1530
  );
@@ -369,7 +1535,7 @@ export class SpookySync<S extends SchemaStructure> {
369
1535
  {
370
1536
  queryId: encodeRecordId(queryState.config.id),
371
1537
  array,
372
- Category: 'spooky-client::SpookySync::createRemoteQuery',
1538
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
373
1539
  },
374
1540
  'createdRemoteQuery'
375
1541
  );
@@ -378,13 +1544,75 @@ export class SpookySync<S extends SchemaStructure> {
378
1544
  /// Incantation existed already
379
1545
  await this.dataModule.updateQueryRemoteArray(queryHash, array);
380
1546
  }
1547
+
1548
+ // Pull the bodies of any `.related()` subquery children into the local
1549
+ // cache. The primary fetch above (`parent IS NONE`) tracks only window
1550
+ // rows, so without this a cold-reload re-materialization of the
1551
+ // correlated surql finds no child rows and related fields come back
1552
+ // empty. Best-effort: never fail registration over it.
1553
+ await this.syncSubqueryChildren(queryHash).catch((err) => {
1554
+ this.logger.info(
1555
+ {
1556
+ err: (err as Error)?.message ?? err,
1557
+ queryHash,
1558
+ Category: 'sp00ky-client::Sp00kySync::createRemoteQuery',
1559
+ },
1560
+ 'Subquery child sync failed during registration; poll will retry'
1561
+ );
1562
+ });
1563
+ }
1564
+
1565
+ /**
1566
+ * Sync the BODIES of a `.related()` query's subquery child rows into the
1567
+ * local cache, separately from the primary window array. The SSP writes
1568
+ * each matched child as a `_00_list_ref` edge tagged `parent`/`parent_rel`;
1569
+ * `buildSubqueryListRefSelect` pulls those `out`+`version` pairs (any
1570
+ * nesting depth). We diff against the in-memory `subqueryRemoteArray` and
1571
+ * fetch added/updated bodies through the SyncEngine — which `saveBatch`s
1572
+ * them into the local DB AND the in-browser SSP, whose subquery-table
1573
+ * dependency then re-materializes the parent view (no explicit notify).
1574
+ *
1575
+ * Deletion safety: we pass `removed: []` deliberately. A child body can be
1576
+ * shared by other queries; letting `handleRemovedRecords` delete one that
1577
+ * merely left THIS query's child set would clobber data another query still
1578
+ * shows. Genuine record deletes flow through the normal delete path; a
1579
+ * lingering orphan body is invisible (the correlated WHERE stops matching).
1580
+ *
1581
+ * Kept off `runSyncForQuery` on purpose so child fetches never flip the
1582
+ * query to `fetching` or skew its DevTools timings.
1583
+ */
1584
+ private async syncSubqueryChildren(queryHash: string): Promise<void> {
1585
+ const queryState = this.dataModule.getQueryByHash(queryHash);
1586
+ if (!queryState) return;
1587
+
1588
+ const listRefTbl = this.listRefTable();
1589
+ const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
1590
+ buildSubqueryListRefSelect(listRefTbl),
1591
+ { in: queryState.config.id }
1592
+ );
1593
+ if (!Array.isArray(items)) return;
1594
+
1595
+ const fresh: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
1596
+ const prev = queryState.config.subqueryRemoteArray ?? [];
1597
+ if (recordVersionArraysEqual(fresh, prev)) return; // idempotent: nothing new
1598
+
1599
+ const diff = diffRecordVersionArray(prev, fresh);
1600
+ if (diff.added.length > 0 || diff.updated.length > 0) {
1601
+ await this.syncEngine.syncRecords({
1602
+ added: diff.added,
1603
+ updated: diff.updated,
1604
+ removed: [], // never delete child bodies here — see method doc
1605
+ });
1606
+ }
1607
+ // In-memory only — child rows must never enter the persisted primary array.
1608
+ queryState.config.subqueryRemoteArray = fresh;
381
1609
  }
382
1610
 
383
- private async heartbeatQuery(queryHash: string) {
1611
+ public async heartbeatQuery(queryHash: string) {
384
1612
  const queryState = this.dataModule.getQueryByHash(queryHash);
385
1613
  if (!queryState) {
386
1614
  this.logger.warn(
387
- { queryHash, Category: 'spooky-client::SpookySync::heartbeatQuery' },
1615
+ { queryHash, Category: 'sp00ky-client::Sp00kySync::heartbeatQuery' },
388
1616
  'Query to register not found'
389
1617
  );
390
1618
  throw new Error('Query to register not found');
@@ -394,17 +1622,30 @@ export class SpookySync<S extends SchemaStructure> {
394
1622
  });
395
1623
  }
396
1624
 
1625
+ // Eager teardown of a deregistered query's remote `_00_query` view (opt-in,
1626
+ // e.g. a viewport-windowed list cancelling an off-screen window). Query ids
1627
+ // are a deterministic hash of (surql+params), so a DELETE racing a scroll-back
1628
+ // re-register (same id) could nuke a freshly-recreated view — hence two
1629
+ // guards: abort if a subscriber reappeared BEFORE the delete; re-register if
1630
+ // one reappears DURING the delete's network await. Tolerant of a
1631
+ // missing/already-gone query (no throw).
397
1632
  private async cleanupQuery(queryHash: string) {
398
1633
  const queryState = this.dataModule.getQueryByHash(queryHash);
399
- if (!queryState) {
400
- this.logger.warn(
401
- { queryHash, Category: 'spooky-client::SpookySync::cleanupQuery' },
402
- 'Query to register not found'
403
- );
404
- throw new Error('Query to register not found');
1634
+ if (!queryState) return; // already torn down / never registered
1635
+
1636
+ // Re-subscribed before the queued cleanup ran → keep everything as-is.
1637
+ if (this.dataModule.hasSubscribers(queryHash)) return;
1638
+
1639
+ await this.remote.query(`DELETE $id`, { id: queryState.config.id });
1640
+
1641
+ // Re-subscribed while we awaited the DELETE → the remote view is now gone
1642
+ // but a subscriber needs it; recreate it instead of leaving a zombie.
1643
+ if (this.dataModule.hasSubscribers(queryHash)) {
1644
+ this.enqueueDownEvent({ type: 'register', payload: { hash: queryHash } });
1645
+ return;
405
1646
  }
406
- await this.remote.query(`DELETE $id`, {
407
- id: queryState.config.id,
408
- });
1647
+
1648
+ // No subscribers throughout → safe to free the local view + state.
1649
+ this.dataModule.finalizeDeregister(queryHash);
409
1650
  }
410
1651
  }