@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.160

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/AGENTS.md +56 -0
  2. package/dist/index.d.ts +1406 -364
  3. package/dist/index.js +7754 -1390
  4. package/dist/otel/index.d.ts +21 -0
  5. package/dist/otel/index.js +86 -0
  6. package/dist/sqlite-open.js +276 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +420 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +434 -0
  11. package/dist/types.d.ts +874 -0
  12. package/package.json +20 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/sp00ky-core/SKILL.md +258 -0
  15. package/skills/sp00ky-core/references/auth.md +98 -0
  16. package/skills/sp00ky-core/references/config.md +76 -0
  17. package/src/build-globals.d.ts +12 -0
  18. package/src/events/events.test.ts +2 -1
  19. package/src/events/index.ts +3 -0
  20. package/src/index.ts +16 -2
  21. package/src/modules/app-release/index.test.ts +125 -0
  22. package/src/modules/app-release/index.ts +201 -0
  23. package/src/modules/auth/events/index.ts +2 -1
  24. package/src/modules/auth/index.ts +59 -20
  25. package/src/modules/cache/index.ts +112 -32
  26. package/src/modules/cache/types.ts +2 -2
  27. package/src/modules/crdt/crdt-field.ts +294 -0
  28. package/src/modules/crdt/crdt-hydration.test.ts +206 -0
  29. package/src/modules/crdt/index.ts +361 -0
  30. package/src/modules/crdt/loro-loader.ts +25 -0
  31. package/src/modules/data/data.hydration.test.ts +142 -0
  32. package/src/modules/data/data.rebind.test.ts +147 -0
  33. package/src/modules/data/data.run.test.ts +113 -0
  34. package/src/modules/data/data.status.test.ts +249 -0
  35. package/src/modules/data/index.ts +1151 -129
  36. package/src/modules/data/mutation-id.test.ts +25 -0
  37. package/src/modules/data/mutation-id.ts +35 -0
  38. package/src/modules/data/window-query.test.ts +52 -0
  39. package/src/modules/data/window-query.ts +154 -0
  40. package/src/modules/devtools/index.ts +325 -37
  41. package/src/modules/devtools/notify-throttle.test.ts +149 -0
  42. package/src/modules/devtools/storage-info.test.ts +79 -0
  43. package/src/modules/devtools/storage-info.ts +143 -0
  44. package/src/modules/devtools/versions.test.ts +74 -0
  45. package/src/modules/devtools/versions.ts +81 -0
  46. package/src/modules/feature-flag/index.test.ts +120 -0
  47. package/src/modules/feature-flag/index.ts +209 -0
  48. package/src/modules/ref-tables.test.ts +91 -0
  49. package/src/modules/ref-tables.ts +88 -0
  50. package/src/modules/sync/engine.ts +101 -37
  51. package/src/modules/sync/events/index.ts +9 -2
  52. package/src/modules/sync/queue/queue-down.ts +12 -5
  53. package/src/modules/sync/queue/queue-up.forwarded.test.ts +137 -0
  54. package/src/modules/sync/queue/queue-up.ts +198 -57
  55. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  56. package/src/modules/sync/scheduler.ts +77 -9
  57. package/src/modules/sync/sync.health.test.ts +149 -0
  58. package/src/modules/sync/sync.subquery.test.ts +82 -0
  59. package/src/modules/sync/sync.ts +1333 -92
  60. package/src/modules/sync/utils.test.ts +269 -2
  61. package/src/modules/sync/utils.ts +182 -17
  62. package/src/otel/index.ts +127 -0
  63. package/src/services/database/cache-engine.ts +160 -0
  64. package/src/services/database/database.ts +11 -11
  65. package/src/services/database/engine-factory.ts +33 -0
  66. package/src/services/database/events/index.ts +2 -1
  67. package/src/services/database/index.ts +6 -0
  68. package/src/services/database/local-migrator.ts +28 -27
  69. package/src/services/database/local.test.ts +64 -0
  70. package/src/services/database/local.ts +478 -67
  71. package/src/services/database/plan-render.test.ts +159 -0
  72. package/src/services/database/plan-render.ts +108 -0
  73. package/src/services/database/relation-resolver.test.ts +413 -0
  74. package/src/services/database/relation-resolver.ts +0 -0
  75. package/src/services/database/remote.ts +13 -13
  76. package/src/services/database/sqlite-cache-engine.test.ts +464 -0
  77. package/src/services/database/sqlite-cache-engine.ts +1154 -0
  78. package/src/services/database/sqlite-open.test.ts +150 -0
  79. package/src/services/database/sqlite-open.ts +164 -0
  80. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  81. package/src/services/database/sqlite-plan-sql.ts +106 -0
  82. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  83. package/src/services/database/sqlite-select.test.ts +246 -0
  84. package/src/services/database/sqlite-select.ts +113 -0
  85. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  86. package/src/services/database/sqlite-transport.ts +219 -0
  87. package/src/services/database/sqlite-worker.ts +437 -0
  88. package/src/services/database/surql-translate.ts +291 -0
  89. package/src/services/database/surreal-cache-engine.ts +141 -0
  90. package/src/services/logger/index.ts +6 -109
  91. package/src/services/persistence/localstorage.ts +2 -2
  92. package/src/services/persistence/resilient.ts +11 -4
  93. package/src/services/persistence/surrealdb.ts +10 -10
  94. package/src/services/stream-processor/index.ts +444 -52
  95. package/src/services/stream-processor/permissions.test.ts +47 -0
  96. package/src/services/stream-processor/permissions.ts +53 -0
  97. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  98. package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
  99. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  100. package/src/services/stream-processor/wasm-types.ts +23 -2
  101. package/src/services/tabs/broker-client.ts +283 -0
  102. package/src/services/tabs/broker.test.ts +278 -0
  103. package/src/services/tabs/coordinator.test.ts +192 -0
  104. package/src/services/tabs/coordinator.ts +567 -0
  105. package/src/services/tabs/fake-ports.fixture.ts +70 -0
  106. package/src/services/tabs/leader-locks.ts +75 -0
  107. package/src/services/tabs/protocol.ts +239 -0
  108. package/src/services/tabs/support.ts +36 -0
  109. package/src/services/tabs/tabs-broker-worker.ts +581 -0
  110. package/src/sp00ky.auth-order.test.ts +92 -0
  111. package/src/sp00ky.init-query.test.ts +183 -0
  112. package/src/sp00ky.ts +1225 -0
  113. package/src/types.ts +342 -15
  114. package/src/utils/error-classification.test.ts +44 -0
  115. package/src/utils/error-classification.ts +7 -0
  116. package/src/utils/index.ts +35 -13
  117. package/src/utils/parser.ts +3 -2
  118. package/src/utils/semver.test.ts +32 -0
  119. package/src/utils/semver.ts +30 -0
  120. package/src/utils/surql.ts +30 -18
  121. package/src/utils/withRetry.test.ts +1 -1
  122. package/tsdown.config.ts +86 -1
  123. package/src/spooky.ts +0 -395
@@ -1,28 +1,35 @@
1
1
  import { RecordId, Duration } from 'surrealdb';
2
- import {
2
+ import type {
3
3
  SchemaStructure,
4
4
  TableNames,
5
5
  BackendNames,
6
6
  BackendRoutes,
7
7
  RoutePayload,
8
+ QueryPlan,
8
9
  } from '@spooky-sync/query-builder';
9
- import { LocalDatabaseService } from '../../services/database/index';
10
- import { CacheModule, RecordWithId } from '../cache/index';
11
- import { Logger } from '../../services/logger/index';
12
- import { StreamUpdate } from '../../services/stream-processor/index';
13
- import {
14
- MutationEvent,
10
+ import type { LocalStore } from '../../services/database/index';
11
+ import { StaleEpochError } from '../../services/database/index';
12
+ import type { CacheModule, RecordWithId, CacheRecord } from '../cache/index';
13
+ import type { Logger } from '../../services/logger/index';
14
+ import type { StreamUpdate } from '../../services/stream-processor/index';
15
+ import type {
15
16
  QueryConfig,
16
17
  QueryHash,
17
18
  QueryState,
19
+ QueryStatus,
20
+ QueryStatusCallback,
18
21
  QueryTimeToLive,
19
22
  QueryUpdateCallback,
20
23
  MutationCallback,
21
24
  RecordVersionArray,
22
25
  QueryConfigRecord,
23
26
  UpdateOptions,
27
+ QueryTimings,
28
+ PhaseStat,
29
+ RegistrationTimings,
24
30
  RunOptions,
25
31
  } from '../../types';
32
+ import { MATERIALIZATION_SAMPLE_WINDOW } from '../../types';
26
33
  import {
27
34
  parseRecordIdString,
28
35
  extractIdPart,
@@ -31,11 +38,30 @@ import {
31
38
  withRetry,
32
39
  surql,
33
40
  parseParams,
41
+ cleanRecord,
34
42
  extractTablePart,
35
43
  generateId,
36
44
  } from '../../utils/index';
37
- import { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
38
- import { PushEventOptions } from '../../events/index';
45
+ import type { CreateEvent, DeleteEvent, UpdateEvent } from '../sync/index';
46
+ import type { PushEventOptions } from '../../events/index';
47
+ import { buildWindowMaterialization, buildWindowMaterializationPlan } from './window-query';
48
+ import { mintMutationId } from './mutation-id';
49
+
50
+ /** Push a timing sample (ms) into a rolling window, capped at the sample window. */
51
+ function pushSample(samples: number[], ms: number): void {
52
+ samples.push(ms);
53
+ if (samples.length > MATERIALIZATION_SAMPLE_WINDOW) samples.shift();
54
+ }
55
+
56
+ /** Build a {lastMs,p50,p90,p99,count} summary from a rolling sample window. */
57
+ function phaseStatOf(samples: number[], lastMs: number | null): PhaseStat {
58
+ if (samples.length === 0) {
59
+ return { lastMs, p50: null, p90: null, p99: null, count: 0 };
60
+ }
61
+ const sorted = [...samples].sort((a, b) => a - b);
62
+ const pick = (q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!;
63
+ return { lastMs, p50: pick(0.5), p90: pick(0.9), p99: pick(0.99), count: samples.length };
64
+ }
39
65
 
40
66
  /**
41
67
  * DataModule - Unified query and mutation management
@@ -44,25 +70,117 @@ import { PushEventOptions } from '../../events/index';
44
70
  * Uses CacheModule for all storage operations.
45
71
  */
46
72
  export class DataModule<S extends SchemaStructure> {
73
+ /** Tab identity baked into mutation ids (shared-tabs rollback routing);
74
+ * undefined in solo mode, where mutation-id falls back to a session id. */
75
+ private tabId: string | undefined;
47
76
  private activeQueries: Map<QueryHash, QueryState> = new Map();
48
77
  private pendingQueries: Map<QueryHash, Promise<QueryHash>> = new Map();
49
78
  private subscriptions: Map<QueryHash, Set<QueryUpdateCallback>> = new Map();
79
+ private statusSubscriptions: Map<QueryHash, Set<QueryStatusCallback>> = new Map();
50
80
  private mutationCallbacks: Set<MutationCallback> = new Set();
51
81
  private debounceTimers: Map<QueryHash, NodeJS.Timeout> = new Map();
82
+ // The update each debounce timer would process on its trailing edge. Kept in a
83
+ // map (not just the timer closure) so `flushPendingStreamUpdate` can process
84
+ // it early — the sync engine flushes before flipping a query to `idle`, so
85
+ // subscribers never observe idle status with stale (partial-window) rows.
86
+ private pendingStreamUpdates: Map<QueryHash, StreamUpdate> = new Map();
87
+ // Refcount of in-flight fetch cycles per query (registration + concurrent
88
+ // poll/LIVE sync rounds can overlap). Status flips to `fetching` on 0→1 and
89
+ // back to `idle` only on 1→0, so an inner cycle finishing can't emit a
90
+ // premature idle mid-registration.
91
+ private fetchDepth: Map<QueryHash, number> = new Map();
52
92
  private logger: Logger;
93
+ /**
94
+ * Optional observer notified whenever a query's fetch status changes.
95
+ * Wired by Sp00kyClient to push status changes into DevTools. Kept as a
96
+ * settable field (rather than a constructor arg) because DevTools is
97
+ * constructed after DataModule.
98
+ */
99
+ public onQueryStatusChange?: (hash: QueryHash, status: QueryStatus) => void;
100
+ /**
101
+ * Optional observer invoked when a still-subscribed query's TTL heartbeat
102
+ * fires (~90% of the TTL). Wired by Sp00kyClient to
103
+ * `Sp00kySync.heartbeatQuery`, which refreshes the remote `_00_query`
104
+ * row's `lastActiveAt` so an actively-watched query never expires. Settable
105
+ * field (not a constructor arg) because the sync engine is wired after
106
+ * DataModule is constructed — mirrors `onQueryStatusChange`.
107
+ */
108
+ public onHeartbeat?: (hash: QueryHash) => void;
109
+ /**
110
+ * Optional hook fired by {@link deregisterQuery} when an opt-in query (e.g. a
111
+ * viewport-windowed list cancelling an off-screen window) loses its last
112
+ * subscriber. Wired by Sp00kyClient to enqueue a `cleanup` down-event, which
113
+ * tears the remote `_00_query` view down (releasing its `_00_list_ref` edges)
114
+ * instead of leaving it for the TTL sweep. The local view + state are freed in
115
+ * {@link finalizeDeregister} only after that remote delete, so a fast
116
+ * re-subscribe (scroll back) can abort/heal the teardown — see `cleanupQuery`.
117
+ */
118
+ public onDeregister?: (hash: QueryHash) => void;
119
+ // Salt for query-id hashing. Set from SurrealDB's session::id() so two
120
+ // browser sessions registering the same logical query (same surql + params)
121
+ // don't collide on the same `_00_query` row — each session gets its own.
122
+ // Empty string until init(sessionId) is called.
123
+ private sessionId: string = '';
124
+ // Authenticated user record id (e.g. `"user:abc"`). Updated by
125
+ // `setCurrentUserId` from the auth subscription. null when
126
+ // unauthenticated. Consulted by `Sp00kySync.listRefTable()` so the
127
+ // poll and LIVE subscription target the same per-user
128
+ // `_00_list_ref_user_<id>` table the sync engine writes to.
129
+ private currentUserId: string | null = null;
53
130
 
54
131
  constructor(
55
132
  private cache: CacheModule,
56
- private local: LocalDatabaseService,
133
+ private local: LocalStore,
57
134
  private schema: S,
58
135
  logger: Logger,
59
- private streamDebounceTime: number = 100
136
+ // Client-side SSP aggregation throttle: coalesces the in-browser
137
+ // StreamProcessor's per-record stream updates per query before notifying
138
+ // readers, so a burst of synced records repaints the UI once per window
139
+ // rather than row-by-row. 50ms keeps it responsive while still batching the
140
+ // initial-sync stream. Override via `SyncedDbConfig.streamDebounceTime`.
141
+ private streamDebounceTime: number = 50
60
142
  ) {
61
143
  this.logger = logger.child({ service: 'DataModule' });
62
144
  }
63
145
 
64
- async init(): Promise<void> {
65
- this.logger.info({ Category: 'spooky-client::DataModule::init' }, 'DataModule initialized');
146
+ async init(sessionId: string): Promise<void> {
147
+ this.sessionId = sessionId;
148
+ this.logger.info(
149
+ { sessionId, Category: 'sp00ky-client::DataModule::init' },
150
+ 'DataModule initialized'
151
+ );
152
+ }
153
+
154
+ /**
155
+ * Update the session salt used in query-id hashing. Call this when the
156
+ * SurrealDB session changes (sign-in, sign-out, reconnect). Subsequently
157
+ * registered queries will get fresh, session-scoped IDs.
158
+ */
159
+ setSessionId(sessionId: string): void {
160
+ this.sessionId = sessionId;
161
+ }
162
+
163
+ /** Shared-tabs: bake this tab's identity into mutation ids so a rollback of
164
+ * a follower's mutation routes back to the tab that made it. */
165
+ setTabId(tabId: string): void {
166
+ this.tabId = tabId;
167
+ }
168
+
169
+ /**
170
+ * Update the authenticated user record id. Pass `null` on sign-out.
171
+ * Read by `Sp00kySync.listRefTable()` so the LIVE subscription and
172
+ * the poll route to the same per-user `_00_list_ref_user_<id>` the
173
+ * SSP writes to.
174
+ */
175
+ setCurrentUserId(userId: string | null): void {
176
+ this.currentUserId = userId;
177
+ }
178
+
179
+ /** Read-only view of the authenticated user id used for per-user
180
+ * `_00_list_ref` routing. Other modules consult this so they pick the
181
+ * same table name DataModule does. */
182
+ getCurrentUserId(): string | null {
183
+ return this.currentUserId;
66
184
  }
67
185
 
68
186
  // ==================== QUERY MANAGEMENT ====================
@@ -74,19 +192,22 @@ export class DataModule<S extends SchemaStructure> {
74
192
  tableName: T,
75
193
  surqlString: string,
76
194
  params: Record<string, any>,
77
- ttl: QueryTimeToLive
195
+ ttl: QueryTimeToLive,
196
+ plan?: QueryPlan
78
197
  ): Promise<QueryHash> {
79
198
  const hash = await this.calculateHash({ surql: surqlString, params });
80
199
  this.logger.debug(
81
- { hash, Category: 'spooky-client::DataModule::query' },
200
+ { hash, Category: 'sp00ky-client::DataModule::query' },
82
201
  'Query Initialization: started'
83
202
  );
84
203
 
85
- const recordId = new RecordId('_spooky_query', hash);
204
+ // `_00_query` stays the single shared registration table in both
205
+ // ref-modes; the per-user split happens only on `_00_list_ref`.
206
+ const recordId = new RecordId('_00_query', hash);
86
207
 
87
208
  if (this.activeQueries.has(hash)) {
88
209
  this.logger.debug(
89
- { hash, Category: 'spooky-client::DataModule::query' },
210
+ { hash, Category: 'sp00ky-client::DataModule::query' },
90
211
  'Query Initialization: exists, returning'
91
212
  );
92
213
  return hash;
@@ -95,7 +216,7 @@ export class DataModule<S extends SchemaStructure> {
95
216
  // Another call is already creating this query — wait for it
96
217
  if (this.pendingQueries.has(hash)) {
97
218
  this.logger.debug(
98
- { hash, Category: 'spooky-client::DataModule::query' },
219
+ { hash, Category: 'sp00ky-client::DataModule::query' },
99
220
  'Query Initialization: pending, waiting for existing creation'
100
221
  );
101
222
  await this.pendingQueries.get(hash);
@@ -103,12 +224,20 @@ export class DataModule<S extends SchemaStructure> {
103
224
  }
104
225
 
105
226
  this.logger.debug(
106
- { hash, Category: 'spooky-client::DataModule::query' },
227
+ { hash, Category: 'sp00ky-client::DataModule::query' },
107
228
  'Query Initialization: not found, creating new query'
108
229
  );
109
230
 
110
231
  // Create the query and track the pending promise
111
- const promise = this.createAndRegisterQuery<T>(hash, recordId, surqlString, params, ttl, tableName);
232
+ const promise = this.createAndRegisterQuery<T>(
233
+ hash,
234
+ recordId,
235
+ surqlString,
236
+ params,
237
+ ttl,
238
+ tableName,
239
+ plan
240
+ );
112
241
  this.pendingQueries.set(hash, promise);
113
242
  try {
114
243
  await promise;
@@ -147,11 +276,95 @@ export class DataModule<S extends SchemaStructure> {
147
276
  subs.delete(callback);
148
277
  if (subs.size === 0) {
149
278
  this.subscriptions.delete(queryHash);
279
+ // NOTE: intentionally do NOT tear down the query / free its in-browser
280
+ // SSP view here. The subscriber-gated heartbeat (startTTLHeartbeat)
281
+ // already self-stops once subscribers hit 0, so an abandoned query
282
+ // stops being kept alive and the server's TTL sweep removes it. Freeing
283
+ // the local view on every last-unsubscribe caused re-registration churn
284
+ // on navigation (open A → leave → open A again re-registers), which is
285
+ // a flakiness risk for no real benefit — keep the local view resident.
150
286
  }
151
287
  }
152
288
  };
153
289
  }
154
290
 
291
+ /**
292
+ * Subscribe to a query's fetch-status changes (idle/fetching).
293
+ * With `{ immediate: true }` the callback fires synchronously with the
294
+ * current status (defaults to `idle` if the query isn't registered yet).
295
+ */
296
+ subscribeStatus(
297
+ queryHash: string,
298
+ callback: QueryStatusCallback,
299
+ options: { immediate?: boolean } = {}
300
+ ): () => void {
301
+ if (!this.statusSubscriptions.has(queryHash)) {
302
+ this.statusSubscriptions.set(queryHash, new Set());
303
+ }
304
+ this.statusSubscriptions.get(queryHash)?.add(callback);
305
+
306
+ if (options.immediate) {
307
+ callback(this.activeQueries.get(queryHash)?.status ?? 'idle');
308
+ }
309
+
310
+ return () => {
311
+ const subs = this.statusSubscriptions.get(queryHash);
312
+ if (subs) {
313
+ subs.delete(callback);
314
+ if (subs.size === 0) {
315
+ this.statusSubscriptions.delete(queryHash);
316
+ }
317
+ }
318
+ };
319
+ }
320
+
321
+ /**
322
+ * Set a query's fetch status and notify status observers (DevTools +
323
+ * `subscribeStatus` listeners). No-op when the status is unchanged or the
324
+ * query is unknown.
325
+ */
326
+ setQueryStatus(queryHash: string, status: QueryStatus): void {
327
+ const queryState = this.activeQueries.get(queryHash);
328
+ if (!queryState || queryState.status === status) {
329
+ return;
330
+ }
331
+ queryState.status = status;
332
+
333
+ this.onQueryStatusChange?.(queryHash, status);
334
+
335
+ const subs = this.statusSubscriptions.get(queryHash);
336
+ if (subs) {
337
+ for (const callback of subs) {
338
+ callback(status);
339
+ }
340
+ }
341
+ }
342
+
343
+ /**
344
+ * Enter a fetch cycle for a query. Refcounted: registration and concurrent
345
+ * poll/LIVE sync rounds can overlap on the same hash, and only the OUTERMOST
346
+ * cycle may flip the status — 0→1 emits `fetching`, and `endFetching`'s 1→0
347
+ * emits `idle`. Always pair with `endFetching` in a `finally`.
348
+ */
349
+ beginFetching(queryHash: string): void {
350
+ const depth = this.fetchDepth.get(queryHash) ?? 0;
351
+ this.fetchDepth.set(queryHash, depth + 1);
352
+ if (depth === 0) {
353
+ this.setQueryStatus(queryHash, 'fetching');
354
+ }
355
+ }
356
+
357
+ /** Leave a fetch cycle started with {@link beginFetching}; emits `idle` on the last exit. */
358
+ endFetching(queryHash: string): void {
359
+ const depth = this.fetchDepth.get(queryHash) ?? 0;
360
+ if (depth <= 1) {
361
+ this.fetchDepth.delete(queryHash);
362
+ this.setQueryStatus(queryHash, 'idle');
363
+ return;
364
+ }
365
+ this.fetchDepth.set(queryHash, depth - 1);
366
+ }
367
+
155
368
  /**
156
369
  * Subscribe to mutations (for sync)
157
370
  */
@@ -168,67 +381,208 @@ export class DataModule<S extends SchemaStructure> {
168
381
  async onStreamUpdate(update: StreamUpdate): Promise<void> {
169
382
  const { queryHash, op } = update;
170
383
 
171
- // Only debounce UPDATE operations
172
- // CREATE and DELETE should propagate immediately
173
- if (op === 'UPDATE') {
174
- // Clear existing timer if any
175
- if (this.debounceTimers.has(queryHash)) {
176
- clearTimeout(this.debounceTimers.get(queryHash)!);
384
+ // DELETE propagates immediately — a removed row should disappear without
385
+ // waiting on a debounce.
386
+ //
387
+ // CREATE and UPDATE are coalesced per query on a trailing timer. A list's
388
+ // rows stream in from sync as many small `_00_list_ref` diffs, each its own
389
+ // `cache.saveBatch` → one stream update per chunk. `ingestMany` only
390
+ // coalesces records ingested synchronously together, so chunks spread over
391
+ // time each re-materialize and re-notify — a 50-row window can fire 30+
392
+ // updates as it fills. Each StreamUpdate carries the full materialized
393
+ // `localArray`, so the latest one already reflects every prior chunk: keep
394
+ // only it and fire once on the trailing edge, settling the query in a couple
395
+ // of notifications instead of one per chunk.
396
+ if (op === 'DELETE') {
397
+ const existing = this.debounceTimers.get(queryHash);
398
+ if (existing) {
399
+ clearTimeout(existing);
400
+ this.debounceTimers.delete(queryHash);
177
401
  }
402
+ // The DELETE update carries the full latest localArray, so the coalesced
403
+ // CREATE/UPDATE it supersedes is already reflected — drop it.
404
+ this.pendingStreamUpdates.delete(queryHash);
405
+ await this.processStreamUpdate(update);
406
+ return;
407
+ }
178
408
 
179
- // Set new timer
180
- const timer = setTimeout(async () => {
181
- this.debounceTimers.delete(queryHash);
182
- await this.processStreamUpdate(update);
183
- }, this.streamDebounceTime);
409
+ // Clear existing timer if any
410
+ if (this.debounceTimers.has(queryHash)) {
411
+ // oxlint-disable-next-line no-non-null-assertion -- guarded by .has() check above
412
+ clearTimeout(this.debounceTimers.get(queryHash)!);
413
+ }
184
414
 
185
- this.debounceTimers.set(queryHash, timer);
186
- } else {
187
- // CREATE and DELETE - process immediately
415
+ // Set new timer
416
+ this.pendingStreamUpdates.set(queryHash, update);
417
+ const timer = setTimeout(async () => {
418
+ this.debounceTimers.delete(queryHash);
419
+ this.pendingStreamUpdates.delete(queryHash);
188
420
  await this.processStreamUpdate(update);
421
+ }, this.streamDebounceTime);
422
+
423
+ this.debounceTimers.set(queryHash, timer);
424
+ }
425
+
426
+ /**
427
+ * Process a query's pending (debounced) stream update NOW instead of on the
428
+ * trailing edge. Called by the sync engine before it flips a query back to
429
+ * `idle`, so the status change never races ahead of the rows it fetched.
430
+ * No-op when nothing is pending. The pending entry is removed before the
431
+ * await so a concurrently-firing timer can't process it twice.
432
+ */
433
+ async flushPendingStreamUpdate(queryHash: string): Promise<void> {
434
+ const timer = this.debounceTimers.get(queryHash);
435
+ if (timer) {
436
+ clearTimeout(timer);
437
+ this.debounceTimers.delete(queryHash);
189
438
  }
439
+ const pending = this.pendingStreamUpdates.get(queryHash);
440
+ if (!pending) return;
441
+ this.pendingStreamUpdates.delete(queryHash);
442
+ await this.processStreamUpdate(pending);
443
+ }
444
+
445
+ // Materialize a query's result rows from the local DB. For a windowed query
446
+ // (`LIMIT n START m`, m>0) the original surql is NOT re-run — re-applying
447
+ // `START m` against the shared local store would skip the window's own rows
448
+ // (sparse windowing) and return nothing. Instead we select the window's
449
+ // record id-set directly, preferring the server's `list_ref` (`remoteArray`,
450
+ // authoritative) over the in-browser SSP's view (`sspArray`), and re-apply
451
+ // the original ORDER BY for stable order.
452
+ private async materializeRecords(
453
+ queryState: QueryState,
454
+ sspArray?: Array<[string, number]>
455
+ ): Promise<Record<string, any>[]> {
456
+ const t0 = performance.now();
457
+ const plan = queryState.config.plan;
458
+ const windowMat = buildWindowMaterialization(queryState.config.surql);
459
+ let records: Record<string, any>[];
460
+ if (windowMat) {
461
+ const win =
462
+ (queryState.config.remoteArray?.length && queryState.config.remoteArray) ||
463
+ (sspArray?.length && sspArray) ||
464
+ queryState.config.localArray ||
465
+ [];
466
+ const winIds = win.map(([id]) => parseRecordIdString(id));
467
+ if (plan) {
468
+ // Engine-neutral window materialization: select exactly the id-set,
469
+ // keeping ORDER BY + relations. Works on any local engine.
470
+ const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
471
+ records = await this.local.select(winPlan, queryState.config.params);
472
+ } else {
473
+ const [rows] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
474
+ ...queryState.config.params,
475
+ __win: winIds,
476
+ });
477
+ records = rows || [];
478
+ }
479
+ } else if (plan) {
480
+ records = await this.local.select(plan, queryState.config.params);
481
+ } else {
482
+ const [rows] = await this.local.query<[Record<string, any>[]]>(
483
+ queryState.config.surql,
484
+ queryState.config.params
485
+ );
486
+ records = rows || [];
487
+ }
488
+ // Local SurrealDB record-fetch time → DevTools "localFetch" phase.
489
+ this.recordPhase(queryState, 'localFetch', performance.now() - t0);
490
+ return records;
190
491
  }
191
492
 
192
493
  private async processStreamUpdate(update: StreamUpdate): Promise<void> {
193
- const { queryHash, localArray } = update;
494
+ const { queryHash, localArray, materializationTimeMs } = update;
194
495
  const queryState = this.activeQueries.get(queryHash);
195
496
  if (!queryState) {
196
497
  this.logger.warn(
197
- { queryHash, Category: 'spooky-client::DataModule::onStreamUpdate' },
498
+ { queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
198
499
  'Received update for unknown query. Skipping...'
199
500
  );
200
501
  return;
201
502
  }
202
503
 
203
- try {
204
- // Fetch updated records
205
- const [records] = await this.local.query<[Record<string, any>[]]>(
206
- queryState.config.surql,
207
- queryState.config.params
208
- );
504
+ // Update the rolling materialization-sample window before the work that
505
+ // could throw, so the percentiles still move when the downstream local
506
+ // query fails (the materialization step itself ran).
507
+ if (typeof materializationTimeMs === 'number') {
508
+ queryState.materializationSamples.push(materializationTimeMs);
509
+ if (queryState.materializationSamples.length > MATERIALIZATION_SAMPLE_WINDOW) {
510
+ queryState.materializationSamples.shift();
511
+ }
512
+ queryState.lastIngestLatencyMs = materializationTimeMs;
513
+ }
514
+ // Record the SSP internal sub-phase timings (from the WASM binding) so
515
+ // DevTools can attribute ingest cost to store-apply vs circuit-step vs transform.
516
+ if (typeof update.storeApplyMs === 'number')
517
+ this.recordPhase(queryState, 'sspStoreApply', update.storeApplyMs);
518
+ if (typeof update.circuitStepMs === 'number')
519
+ this.recordPhase(queryState, 'sspCircuitStep', update.circuitStepMs);
520
+ if (typeof update.transformMs === 'number')
521
+ this.recordPhase(queryState, 'sspTransform', update.transformMs);
522
+ const percentiles = this.computeMaterializationPercentiles(queryState.materializationSamples);
523
+
524
+ // Fence against bucket switches: this update's `localArray` came from the
525
+ // pre-switch SSP circuit; applying it after a switch would show (and
526
+ // persist) the previous user's ids in the new bucket.
527
+ const epoch = this.local.epoch;
209
528
 
210
- // Update state
211
- const newRecords = records || [];
529
+ try {
530
+ // Materialize the query's rows. For a windowed (offset) query, re-running
531
+ // the original surql would re-apply `START n` against the shared local DB
532
+ // and skip the window's rows entirely; instead select the SSP's
533
+ // materialized window id-set (`localArray`) directly, re-applying the
534
+ // original ORDER BY for stable display order. Non-offset queries keep the
535
+ // normal re-query path.
536
+ const newRecords = await this.materializeRecords(queryState, localArray);
537
+ if (epoch !== this.local.epoch) return;
212
538
  queryState.config.localArray = localArray;
213
- await this.local.query(surql.seal(surql.updateSet('id', ['localArray'])), {
214
- id: queryState.config.id,
215
- localArray,
216
- });
217
539
 
218
- // Skip notification if records haven't changed
219
540
  const prevJson = JSON.stringify(queryState.records);
220
541
  const newJson = JSON.stringify(newRecords);
221
542
  queryState.records = newRecords;
222
- if (prevJson === newJson) {
543
+ const recordsChanged = prevJson !== newJson;
544
+
545
+ // updateCount counts user-visible updates (matches the prior semantic),
546
+ // while the materialization sample/percentiles already moved above for
547
+ // every observed engine step.
548
+ if (recordsChanged) {
549
+ queryState.updateCount++;
550
+ queryState.lastUpdatedAt = Date.now();
551
+ }
552
+
553
+ await this.local.query(
554
+ surql.seal(
555
+ surql.updateSet('id', [
556
+ 'localArray',
557
+ 'rowCount',
558
+ 'updateCount',
559
+ 'lastIngestLatency',
560
+ 'materializationP55',
561
+ 'materializationP90',
562
+ 'materializationP99',
563
+ ])
564
+ ),
565
+ {
566
+ id: queryState.config.id,
567
+ localArray,
568
+ rowCount: localArray.length,
569
+ updateCount: queryState.updateCount,
570
+ lastIngestLatency: queryState.lastIngestLatencyMs,
571
+ materializationP55: percentiles.p55,
572
+ materializationP90: percentiles.p90,
573
+ materializationP99: percentiles.p99,
574
+ },
575
+ { epoch }
576
+ );
577
+
578
+ if (!recordsChanged) {
223
579
  this.logger.debug(
224
- { queryHash, Category: 'spooky-client::DataModule::onStreamUpdate' },
580
+ { queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
225
581
  'Query records unchanged, skipping notification'
226
582
  );
227
583
  return;
228
584
  }
229
585
 
230
- queryState.updateCount++;
231
-
232
586
  // Notify subscribers
233
587
  const subscribers = this.subscriptions.get(queryHash);
234
588
  if (subscribers) {
@@ -240,17 +594,108 @@ export class DataModule<S extends SchemaStructure> {
240
594
  this.logger.debug(
241
595
  {
242
596
  queryHash,
243
- recordCount: records?.length,
244
- Category: 'spooky-client::DataModule::onStreamUpdate',
597
+ recordCount: newRecords?.length,
598
+ Category: 'sp00ky-client::DataModule::onStreamUpdate',
245
599
  },
246
600
  'Query updated from stream'
247
601
  );
248
602
  } catch (err) {
603
+ if (err instanceof StaleEpochError) {
604
+ this.logger.debug(
605
+ { queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
606
+ 'Dropped stream update from before a bucket switch'
607
+ );
608
+ return;
609
+ }
610
+ queryState.errorCount++;
249
611
  this.logger.error(
250
- { err, queryHash, Category: 'spooky-client::DataModule::onStreamUpdate' },
612
+ { err, queryHash, Category: 'sp00ky-client::DataModule::onStreamUpdate' },
251
613
  'Failed to fetch records for stream update'
252
614
  );
615
+ // Best-effort persist of the bumped errorCount; swallow secondary
616
+ // failures to avoid masking the original error in logs.
617
+ try {
618
+ await this.local.query(surql.seal(surql.updateSet('id', ['errorCount'])), {
619
+ id: queryState.config.id,
620
+ errorCount: queryState.errorCount,
621
+ });
622
+ } catch (persistErr) {
623
+ this.logger.warn(
624
+ {
625
+ err: persistErr,
626
+ queryHash,
627
+ Category: 'sp00ky-client::DataModule::onStreamUpdate',
628
+ },
629
+ 'Failed to persist incremented errorCount'
630
+ );
631
+ }
632
+ }
633
+ }
634
+
635
+ /**
636
+ * Compute p55/p90/p99 from a rolling window of materialization samples.
637
+ * Returns nulls for any percentile that has no samples yet so SurrealDB
638
+ * `option<float>` columns stay NONE rather than 0 before the first ingest.
639
+ */
640
+ private computeMaterializationPercentiles(samples: number[]): {
641
+ p55: number | null;
642
+ p90: number | null;
643
+ p99: number | null;
644
+ } {
645
+ if (samples.length === 0) {
646
+ return { p55: null, p90: null, p99: null };
253
647
  }
648
+ const sorted = [...samples].sort((a, b) => a - b);
649
+ const pick = (q: number) => {
650
+ const idx = Math.min(sorted.length - 1, Math.floor(q * sorted.length));
651
+ return sorted[idx]!;
652
+ };
653
+ return { p55: pick(0.55), p90: pick(0.9), p99: pick(0.99) };
654
+ }
655
+
656
+ /** Record a per-phase timing sample (ms) on a query's rolling window. */
657
+ private recordPhase(qs: QueryState, phase: string, ms: number): void {
658
+ if (!Number.isFinite(ms)) return;
659
+ const arr = qs.phaseSamples[phase] ?? (qs.phaseSamples[phase] = []);
660
+ pushSample(arr, ms);
661
+ qs.phaseLast[phase] = ms;
662
+ }
663
+
664
+ /** Record the remote record-fetch time (ms) for a query. Called by the sync engine. */
665
+ recordRemoteFetch(hash: string, ms: number): void {
666
+ const qs = this.activeQueries.get(hash);
667
+ if (qs) this.recordPhase(qs, 'remoteFetch', ms);
668
+ }
669
+
670
+ /**
671
+ * Record the frontend reconcile time (ms) for a query. Called from `useQuery`
672
+ * via `Sp00kyClient.reportFrontendTiming` after it applies an update to its store.
673
+ */
674
+ recordFrontendTiming(hash: string, ms: number): void {
675
+ const qs = this.activeQueries.get(hash);
676
+ if (qs) this.recordPhase(qs, 'frontend', ms);
677
+ }
678
+
679
+ /**
680
+ * Build the per-query processing-time breakdown surfaced to the DevTools panel
681
+ * and the MCP. `ssp` is the WASM-ingest wall time (from `materializationSamples`);
682
+ * the rest come from the per-phase rolling windows + one-shot registration timings.
683
+ */
684
+ phaseTimings(q: QueryState): QueryTimings {
685
+ const stat = (phase: string) =>
686
+ phaseStatOf(q.phaseSamples[phase] ?? [], q.phaseLast[phase] ?? null);
687
+ return {
688
+ ssp: phaseStatOf(q.materializationSamples, q.lastIngestLatencyMs),
689
+ sspStoreApply: stat('sspStoreApply'),
690
+ sspCircuitStep: stat('sspCircuitStep'),
691
+ sspTransform: stat('sspTransform'),
692
+ localFetch: stat('localFetch'),
693
+ remoteFetch: stat('remoteFetch'),
694
+ frontend: stat('frontend'),
695
+ registration: q.registrationTimings,
696
+ updateCount: q.updateCount,
697
+ errorCount: q.errorCount,
698
+ };
254
699
  }
255
700
 
256
701
  /**
@@ -260,6 +705,263 @@ export class DataModule<S extends SchemaStructure> {
260
705
  return this.activeQueries.get(hash);
261
706
  }
262
707
 
708
+ /**
709
+ * Cold-query guard for instant-hydrate: true when the query exists, hasn't been
710
+ * hydrated, and has NOT yet fetched its server result (`remoteArray` empty).
711
+ * We gate on `remoteArray`, not local `records`: a windowed query is often
712
+ * partially pre-seeded from the circuit (e.g. the dashboard's 5-row preview),
713
+ * but it still hasn't loaded its own full window from the server — so it should
714
+ * still hydrate. A warm re-subscribe (remoteArray already populated) is skipped.
715
+ */
716
+ isCold(hash: string): boolean {
717
+ const qs = this.activeQueries.get(hash);
718
+ return !!qs && !qs.hydrated && (qs.config.remoteArray?.length ?? 0) === 0;
719
+ }
720
+
721
+ /**
722
+ * Walk a hydrated record's fields and append any EMBEDDED child records to
723
+ * `batch` (recursing for nested related fields). An embedded child is a
724
+ * value that is itself a record — a non-null object whose `id` is a
725
+ * `RecordId` — or an array of such records (one-to-many vs one-to-one). A
726
+ * bare `RecordId` (a foreign-key reference) or any other value is skipped,
727
+ * so this never mistakes a FK column for an embedded body. Children are
728
+ * keyed by their own `record.id.table`, versioned by `_00_rv`, and cleaned
729
+ * to their table's real columns (which strips the alias/related fields).
730
+ * `seen` dedupes within the batch.
731
+ */
732
+ private collectEmbeddedChildren(
733
+ record: Record<string, any>,
734
+ batch: CacheRecord[],
735
+ seen: Set<string>
736
+ ): void {
737
+ const isEmbeddedRecord = (v: unknown): v is RecordWithId =>
738
+ !!v &&
739
+ typeof v === 'object' &&
740
+ !(v instanceof RecordId) &&
741
+ (v as { id?: unknown }).id instanceof RecordId;
742
+
743
+ for (const value of Object.values(record)) {
744
+ const children = Array.isArray(value)
745
+ ? value.filter(isEmbeddedRecord)
746
+ : isEmbeddedRecord(value)
747
+ ? [value]
748
+ : [];
749
+ for (const child of children) {
750
+ const key = encodeRecordId(child.id);
751
+ if (seen.has(key)) continue;
752
+ seen.add(key);
753
+ // Recurse FIRST so nested grandchildren are captured before `cleanRecord`
754
+ // strips this child's alias fields.
755
+ this.collectEmbeddedChildren(child, batch, seen);
756
+ const table = child.id.table.toString();
757
+ const tableSchema = this.schema.tables.find((t) => t.name === table);
758
+ batch.push({
759
+ table,
760
+ op: 'CREATE',
761
+ // Flatten the child too: a nested comment carries its own embedded
762
+ // `author` object which the schemafull `record<user>` field would
763
+ // reject. (Its author is captured as a separate row by the recursion
764
+ // above.)
765
+ record: this.flattenRelationsForStorage(child, tableSchema),
766
+ version: (child._00_rv as number) || 1,
767
+ });
768
+ }
769
+ }
770
+ }
771
+
772
+ /**
773
+ * Prepare a subquery-bearing row (preload / hydration) for the schemafull
774
+ * local store: replace an embedded FORWARD-relation object (`author = { id, … }`)
775
+ * with its RecordId so a `record<…>` field coerces, and DROP reverse-subquery
776
+ * ARRAYS (`comments = [ … ]`) since their rows are cached separately as their
777
+ * own bodies. A flat record — as the live `SELECT * FROM $ids` sync returns,
778
+ * with relations already RecordIds — passes through unchanged.
779
+ */
780
+ private flattenRelationsForStorage(
781
+ record: Record<string, any>,
782
+ tableSchema?: { columns: Record<string, any> }
783
+ ): RecordWithId {
784
+ const isEmbeddedRecord = (v: unknown): boolean =>
785
+ !!v &&
786
+ typeof v === 'object' &&
787
+ !(v instanceof RecordId) &&
788
+ (v as { id?: unknown }).id instanceof RecordId;
789
+ const flat: Record<string, any> = {};
790
+ for (const [k, v] of Object.entries(record)) {
791
+ if (isEmbeddedRecord(v)) {
792
+ flat[k] = (v as { id: unknown }).id;
793
+ } else if (Array.isArray(v) && v.some(isEmbeddedRecord)) {
794
+ continue;
795
+ } else {
796
+ flat[k] = v;
797
+ }
798
+ }
799
+ return (tableSchema ? cleanRecord(tableSchema.columns, flat) : flat) as RecordWithId;
800
+ }
801
+
802
+ /**
803
+ * Instant-hydrate: ingest rows fetched one-shot from the remote (the query's own
804
+ * surql run directly) so the query DISPLAYS immediately, while the full realtime
805
+ * registration proceeds in the background. Ingests with versions (`_00_rv`) so the
806
+ * later `syncRecords` dedup skips re-pulling unchanged bodies, and seeds
807
+ * `remoteArray` so windowed queries materialize the correct window (no sparse
808
+ * local-circuit issue). Runs at most once per query (the `hydrated` flag).
809
+ */
810
+ async applyHydration(hash: string, rows: RecordWithId[]): Promise<void> {
811
+ const queryState = this.activeQueries.get(hash);
812
+ if (!queryState) return;
813
+ queryState.hydrated = true; // run-once, even when the remote returns nothing
814
+ if (rows.length === 0) return;
815
+
816
+ const epoch = this.local.epoch;
817
+ const tableName = queryState.config.tableName;
818
+ await this.buildAndSaveCacheBatch(tableName, rows);
819
+ // Bucket switched while we persisted: these rows were fetched under the
820
+ // previous auth context — don't let them prime the new bucket's query
821
+ // state; the rebind's re-registration refills it. (saveBatch's own epoch
822
+ // fence usually catches this, but the switch can land between it and here.)
823
+ if (epoch !== this.local.epoch) return;
824
+
825
+ // Prime remoteArray from the hydrated id+version pairs: `materializeRecords`
826
+ // prefers it for windowed queries (correct window) and it feeds the version
827
+ // dedup. Registration later overwrites it with the authoritative `_00_list_ref`.
828
+ queryState.config.remoteArray = rows.map(
829
+ (r) => [encodeRecordId(r.id), (r._00_rv as number) || 1] as [string, number]
830
+ );
831
+
832
+ queryState.records = await this.materializeRecords(queryState);
833
+ const subscribers = this.subscriptions.get(hash);
834
+ if (subscribers) {
835
+ for (const cb of subscribers) cb(queryState.records);
836
+ }
837
+ }
838
+
839
+ /**
840
+ * Build the cache batch for a set of one-shot rows and persist it to the
841
+ * local DB + in-browser SSP. Maps each row to a `CREATE` op on its own table
842
+ * and extracts EMBEDDED related children (any nesting depth) as their own
843
+ * records — a `.related()` query returns its children embedded, and a later
844
+ * correlated re-materialization needs them present as standalone rows.
845
+ * Shared by `applyHydration` (live registration) and `persistSnapshot`
846
+ * (preload).
847
+ */
848
+ private async buildAndSaveCacheBatch(tableName: string, rows: RecordWithId[]): Promise<void> {
849
+ const tableSchema = this.schema.tables.find((t) => t.name === tableName);
850
+ const batch: CacheRecord[] = rows.map((record) => ({
851
+ table: tableName,
852
+ op: 'CREATE' as const,
853
+ // Flatten embedded relations first: a preload/hydration row can carry a
854
+ // forward relation as a full nested OBJECT (`author = { id, … }`) and a
855
+ // reverse subquery as an array of objects (`comments = [ … ]`). The
856
+ // schemafull local field is `record<user>`, which rejects an object
857
+ // (`Couldn't coerce … found { id: …, username: … }`) and would throw the
858
+ // WHOLE batch. Store the parent with `author` as its RecordId and drop the
859
+ // subquery arrays — the children are cached as their own rows below.
860
+ record: this.flattenRelationsForStorage(record, tableSchema),
861
+ version: (record._00_rv as number) || 1,
862
+ }));
863
+
864
+ const seen = new Set<string>(rows.map((r) => encodeRecordId(r.id)));
865
+ for (const record of rows) {
866
+ this.collectEmbeddedChildren(record, batch, seen);
867
+ }
868
+
869
+ await this.cache.saveBatch(batch);
870
+ }
871
+
872
+ /**
873
+ * Preload/prewarm: persist one-shot rows (and their embedded related children)
874
+ * into the local cache WITHOUT registering a query — no `activeQueries` entry,
875
+ * no `_00_query` view, no TTL heartbeat. The rows live in the local DB as
876
+ * ordinary bodies (never GC'd on their own) so a later `useQuery` seeds its
877
+ * first paint from them instantly, then registers a live view to freshen.
878
+ */
879
+ async persistSnapshot(tableName: string, rows: RecordWithId[]): Promise<void> {
880
+ if (rows.length === 0) return;
881
+ await this.buildAndSaveCacheBatch(tableName, rows);
882
+ }
883
+
884
+ /**
885
+ * Read the durable preload freshness marker for a query hash, or null if this
886
+ * query was never preloaded in the current bucket. Co-located with the cached
887
+ * rows (per-bucket `_00_preload` table) so a bucket switch that clears the
888
+ * data also clears the marker — a stale marker can't claim "warm" when the
889
+ * rows are gone. Any read error is treated as cold.
890
+ */
891
+ async getPreloadMarker(hash: string): Promise<{ fetchedAt: number; rowCount: number } | null> {
892
+ try {
893
+ // Pass a real RecordId: a bare string id hits the SurrealDB engine's
894
+ // `FROM ONLY $__id` as a plain string, which "selects" the string itself
895
+ // (a truthy non-row) instead of the record — misread as a warm marker.
896
+ const row = await this.local.getById('_00_preload', new RecordId('_00_preload', hash));
897
+ if (!row || typeof row !== 'object') return null;
898
+ return {
899
+ fetchedAt: Number((row as any).fetchedAt) || 0,
900
+ rowCount: Number((row as any).rowCount) || 0,
901
+ };
902
+ } catch {
903
+ return null;
904
+ }
905
+ }
906
+
907
+ /** Stamp the preload freshness marker after a successful snapshot fetch. */
908
+ async writePreloadMarker(hash: string, rowCount: number): Promise<void> {
909
+ // RecordId, not a bare string: the SurrealDB engine binds the id verbatim,
910
+ // and `UPSERT <string>` is an InternalError — the marker silently never
911
+ // landed (the write is awaited inside preload's best-effort catch).
912
+ await this.local.upsert(
913
+ '_00_preload',
914
+ new RecordId('_00_preload', hash),
915
+ { fetchedAt: Date.now(), rowCount },
916
+ 'replace'
917
+ );
918
+ }
919
+
920
+ /** True while ≥1 live subscriber is watching this query (refcount guard). */
921
+ hasSubscribers(hash: string): boolean {
922
+ return (this.subscriptions.get(hash)?.size ?? 0) > 0;
923
+ }
924
+
925
+ /**
926
+ * Opt-in eager teardown for a query whose LAST subscriber just left — used by
927
+ * viewport-windowed lists to cancel off-screen windows instead of leaving
928
+ * their remote views to expire on the TTL sweep. No-op while any subscriber
929
+ * remains (refcount). Only enqueues the remote cleanup here; the local WASM
930
+ * view + in-memory state are freed in {@link finalizeDeregister} after the
931
+ * remote delete completes, so a re-subscribe in between aborts/heals it.
932
+ *
933
+ * NOTE: most queries should NOT use this — the default keep-alive on
934
+ * unsubscribe avoids re-registration churn on navigation.
935
+ */
936
+ deregisterQuery(hash: string): void {
937
+ if (this.hasSubscribers(hash)) return;
938
+ if (!this.activeQueries.has(hash)) return;
939
+ this.onDeregister?.(hash);
940
+ }
941
+
942
+ /**
943
+ * Final local teardown after the remote `_00_query` row was deleted: free the
944
+ * WASM view, heartbeat timer, debounce timer, and in-memory state. Caller
945
+ * (`cleanupQuery`) guarantees no subscriber remains.
946
+ */
947
+ finalizeDeregister(hash: string): void {
948
+ const qs = this.activeQueries.get(hash);
949
+ if (qs?.ttlTimer) {
950
+ clearTimeout(qs.ttlTimer);
951
+ qs.ttlTimer = null;
952
+ }
953
+ const debounce = this.debounceTimers.get(hash);
954
+ if (debounce) {
955
+ clearTimeout(debounce);
956
+ this.debounceTimers.delete(hash);
957
+ }
958
+ this.pendingStreamUpdates.delete(hash);
959
+ this.fetchDepth.delete(hash);
960
+ this.cache.unregisterQuery(hash);
961
+ this.activeQueries.delete(hash);
962
+ this.subscriptions.delete(hash);
963
+ }
964
+
263
965
  /**
264
966
  * Get query state by id (for sync and devtools)
265
967
  */
@@ -274,36 +976,163 @@ export class DataModule<S extends SchemaStructure> {
274
976
  return Array.from(this.activeQueries.values());
275
977
  }
276
978
 
979
+ getActiveQueryHashes(): QueryHash[] {
980
+ return Array.from(this.activeQueries.keys());
981
+ }
982
+
277
983
  async updateQueryLocalArray(id: string, localArray: RecordVersionArray): Promise<void> {
278
984
  const queryState = this.activeQueries.get(id);
279
985
  if (!queryState) {
280
986
  this.logger.warn(
281
- { id, Category: 'spooky-client::DataModule::updateQueryLocalArray' },
987
+ { id, Category: 'sp00ky-client::DataModule::updateQueryLocalArray' },
282
988
  'Query to update local array not found'
283
989
  );
284
990
  return;
285
991
  }
992
+ const epoch = this.local.epoch;
286
993
  queryState.config.localArray = localArray;
287
- await this.local.query(surql.seal(surql.updateSet('id', ['localArray'])), {
288
- id: queryState.config.id,
289
- localArray,
290
- });
994
+ try {
995
+ await this.local.query(
996
+ surql.seal(surql.updateSet('id', ['localArray'])),
997
+ {
998
+ id: queryState.config.id,
999
+ localArray,
1000
+ },
1001
+ { epoch }
1002
+ );
1003
+ } catch (err) {
1004
+ if (err instanceof StaleEpochError) return;
1005
+ throw err;
1006
+ }
291
1007
  }
292
1008
 
293
1009
  async updateQueryRemoteArray(hash: string, remoteArray: RecordVersionArray): Promise<void> {
294
1010
  const queryState = this.getQueryByHash(hash);
295
1011
  if (!queryState) {
296
1012
  this.logger.warn(
297
- { hash, Category: 'spooky-client::DataModule::updateQueryRemoteArray' },
1013
+ { hash, Category: 'sp00ky-client::DataModule::updateQueryRemoteArray' },
298
1014
  'Query to update remote array not found'
299
1015
  );
300
1016
  return;
301
1017
  }
1018
+ const epoch = this.local.epoch;
302
1019
  queryState.config.remoteArray = remoteArray;
303
- await this.local.query(surql.seal(surql.updateSet('id', ['remoteArray'])), {
304
- id: queryState.config.id,
305
- remoteArray,
306
- });
1020
+ try {
1021
+ await this.local.query(
1022
+ surql.seal(surql.updateSet('id', ['remoteArray'])),
1023
+ {
1024
+ id: queryState.config.id,
1025
+ remoteArray,
1026
+ },
1027
+ { epoch }
1028
+ );
1029
+ } catch (err) {
1030
+ if (err instanceof StaleEpochError) return;
1031
+ throw err;
1032
+ }
1033
+ }
1034
+
1035
+ /**
1036
+ * Cancel every armed timer ahead of a local-bucket switch: stream-update
1037
+ * debounce timers (their pending updates carry the OLD bucket's id-sets) and
1038
+ * per-query TTL heartbeats (they'd refresh the previous user's remote
1039
+ * `_00_query` rows under the new session). The rebind re-arms heartbeats.
1040
+ */
1041
+ quiesce(): void {
1042
+ for (const timer of this.debounceTimers.values()) {
1043
+ clearTimeout(timer);
1044
+ }
1045
+ this.debounceTimers.clear();
1046
+ this.pendingStreamUpdates.clear();
1047
+ this.fetchDepth.clear();
1048
+ for (const queryState of this.activeQueries.values()) {
1049
+ if (queryState.ttlTimer) {
1050
+ clearTimeout(queryState.ttlTimer);
1051
+ queryState.ttlTimer = null;
1052
+ }
1053
+ }
1054
+ }
1055
+
1056
+ /**
1057
+ * Re-home every active query in a freshly-opened bucket, KEEPING its hash —
1058
+ * `useQuery` subscriptions are keyed by hash and don't re-register on auth
1059
+ * changes, so the hooks must stay attached. Per query:
1060
+ * 1. reset the sync arrays + hydration flag and drop the previous user's
1061
+ * records, notifying subscribers with the new-bucket materialization
1062
+ * (usually empty) so their rows leave the UI immediately;
1063
+ * 2. recreate the `_00_query` row in the new bucket;
1064
+ * 3. re-register the SSP view on the (fresh, post-reset) processor — this
1065
+ * also rebinds the view to the NEW `$auth` context;
1066
+ * 4. restart the TTL heartbeat.
1067
+ * Returns the hashes so the caller can enqueue remote re-registration, which
1068
+ * refills records from the server via the normal register→sync→notify path.
1069
+ */
1070
+ async rebindAfterBucketSwitch(): Promise<QueryHash[]> {
1071
+ const hashes: QueryHash[] = [];
1072
+ for (const [hash, queryState] of this.activeQueries.entries()) {
1073
+ const config = queryState.config;
1074
+ config.localArray = [];
1075
+ config.remoteArray = [];
1076
+ config.subqueryRemoteArray = undefined;
1077
+ queryState.hydrated = false;
1078
+ queryState.syncNotified = false;
1079
+ queryState.records = [];
1080
+ // Via setQueryStatus (not a bare assignment) so status observers see the
1081
+ // flip back to a loading state.
1082
+ this.setQueryStatus(hash, 'fetching');
1083
+
1084
+ try {
1085
+ await withRetry(this.logger, () =>
1086
+ this.local.query<[QueryConfigRecord]>(surql.seal(surql.create('id', 'data')), {
1087
+ id: config.id,
1088
+ data: {
1089
+ surql: config.surql,
1090
+ params: config.params,
1091
+ localArray: [],
1092
+ remoteArray: [],
1093
+ lastActiveAt: new Date(),
1094
+ createdAt: new Date(),
1095
+ ttl: config.ttl,
1096
+ tableName: config.tableName,
1097
+ updateCount: queryState.updateCount,
1098
+ rowCount: 0,
1099
+ errorCount: queryState.errorCount,
1100
+ },
1101
+ })
1102
+ );
1103
+
1104
+ const { localArray } = this.cache.registerQuery({
1105
+ queryHash: hash,
1106
+ surql: config.surql,
1107
+ params: config.params,
1108
+ ttl: new Duration(config.ttl),
1109
+ lastActiveAt: new Date(),
1110
+ });
1111
+ config.localArray = localArray;
1112
+ await this.local.query(surql.seal(surql.updateSet('id', ['localArray', 'rowCount'])), {
1113
+ id: config.id,
1114
+ localArray,
1115
+ rowCount: localArray.length,
1116
+ });
1117
+ } catch (err) {
1118
+ this.logger.error(
1119
+ { err, hash, Category: 'sp00ky-client::DataModule::rebindAfterBucketSwitch' },
1120
+ 'Failed to rebind query after bucket switch; remote re-registration will retry'
1121
+ );
1122
+ }
1123
+
1124
+ // Notify AFTER the SSP re-registration so a subscriber that re-reads
1125
+ // synchronously sees consistent (empty) state.
1126
+ const subscribers = this.subscriptions.get(hash);
1127
+ if (subscribers) {
1128
+ for (const callback of subscribers) {
1129
+ callback(queryState.records);
1130
+ }
1131
+ }
1132
+ this.startTTLHeartbeat(queryState, hash);
1133
+ hashes.push(hash);
1134
+ }
1135
+ return hashes;
307
1136
  }
308
1137
 
309
1138
  /**
@@ -313,20 +1142,29 @@ export class DataModule<S extends SchemaStructure> {
313
1142
  async notifyQuerySynced(queryHash: string): Promise<void> {
314
1143
  const queryState = this.activeQueries.get(queryHash);
315
1144
  if (!queryState) return;
316
-
317
- // Re-query local DB for latest data
318
- const [records] = await this.local.query<[Record<string, any>[]]>(
319
- queryState.config.surql,
320
- queryState.config.params
321
- );
322
- const newRecords = records || [];
1145
+ const epoch = this.local.epoch;
1146
+
1147
+ // Re-query local DB for latest data (windowed queries materialize from the
1148
+ // list_ref window so they resolve even if the in-browser SSP never emits —
1149
+ // it can't compute a high offset whose preceding rows aren't resident).
1150
+ const newRecords = await this.materializeRecords(queryState);
1151
+ // Bucket switched while we materialized: these rows mix old-bucket reads
1152
+ // with new-bucket state — drop them; the rebind/re-registration re-emits.
1153
+ if (epoch !== this.local.epoch) return;
323
1154
  const changed = JSON.stringify(queryState.records) !== JSON.stringify(newRecords);
324
1155
  queryState.records = newRecords;
325
1156
 
326
- // Notify if data changed OR if this is the first sync (updateCount === 0)
327
- // The latter handles "query truly has no results" so UI can stop loading
328
- if (changed || queryState.updateCount === 0) {
1157
+ // Notify if data changed OR if this registration lifetime hasn't emitted a
1158
+ // post-sync notification yet. The latter handles "query truly has no
1159
+ // results" so the UI can stop loading — gated on the in-memory
1160
+ // `syncNotified` flag rather than `updateCount === 0`, because updateCount
1161
+ // is PERSISTED across deregister/re-register: a re-registered empty window
1162
+ // (updateCount > 0, records unchanged) would otherwise never emit and its
1163
+ // subscribers would show a loading state forever.
1164
+ if (changed || !queryState.syncNotified) {
1165
+ queryState.syncNotified = true;
329
1166
  queryState.updateCount++;
1167
+ queryState.lastUpdatedAt = Date.now();
330
1168
  const subscribers = this.subscriptions.get(queryHash);
331
1169
  if (subscribers) {
332
1170
  for (const callback of subscribers) {
@@ -344,6 +1182,24 @@ export class DataModule<S extends SchemaStructure> {
344
1182
  data: RoutePayload<S, B, R>,
345
1183
  options?: RunOptions
346
1184
  ): Promise<void> {
1185
+ const { tableName, record } = this.buildJobRecord(backend, path, data, options);
1186
+ const recordId = `${tableName}:${generateId()}`;
1187
+ await this.create(recordId, record);
1188
+ }
1189
+
1190
+ /**
1191
+ * Build the outbox job record + resolve its table for a backend route.
1192
+ *
1193
+ * Every job is a single execution. Recurring work is declared server-side
1194
+ * (`schedules:` in sp00ky.yml) and the scheduler creates a fresh row per cycle,
1195
+ * so nothing here needs to know about schedules.
1196
+ */
1197
+ private buildJobRecord<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(
1198
+ backend: B,
1199
+ path: R,
1200
+ data: RoutePayload<S, B, R>,
1201
+ options?: RunOptions
1202
+ ): { tableName: string; record: Record<string, unknown> } {
347
1203
  const route = this.schema.backends?.[backend]?.routes?.[path];
348
1204
  if (!route) {
349
1205
  throw new Error(`Route ${backend}.${path} not found`);
@@ -366,16 +1222,28 @@ export class DataModule<S extends SchemaStructure> {
366
1222
  const record: Record<string, unknown> = {
367
1223
  path,
368
1224
  payload: JSON.stringify(payload),
1225
+ // Set explicitly: the schema's DEFAULT ALWAYS "pending" is server-side
1226
+ // only. An optimistic local create without it surfaces with status
1227
+ // undefined, so in-flight indicators keyed on pending/processing stay
1228
+ // off until the first server echo (seconds on a delayed job).
1229
+ status: 'pending',
369
1230
  max_retries: options?.max_retries ?? 3,
370
1231
  retry_strategy: options?.retry_strategy ?? 'linear',
371
1232
  };
372
1233
 
1234
+ if (options?.timeout != null) {
1235
+ record.timeout = options.timeout;
1236
+ }
1237
+
1238
+ if (options?.delay != null) {
1239
+ record.delay = options.delay;
1240
+ }
1241
+
373
1242
  if (options?.assignedTo) {
374
1243
  record.assigned_to = options.assignedTo;
375
1244
  }
376
1245
 
377
- const recordId = `${tableName}:${generateId()}`;
378
- await this.create(recordId, record);
1246
+ return { tableName, record };
379
1247
  }
380
1248
 
381
1249
  // ==================== MUTATION MANAGEMENT ====================
@@ -392,7 +1260,7 @@ export class DataModule<S extends SchemaStructure> {
392
1260
 
393
1261
  const rid = parseRecordIdString(id);
394
1262
  const params = parseParams(tableSchema.columns, data);
395
- const mutationId = parseRecordIdString(`_spooky_pending_mutations:${Date.now()}`);
1263
+ const mutationId = parseRecordIdString(mintMutationId(this.tabId));
396
1264
 
397
1265
  const dataKeys = Object.keys(params).map((key) => ({ key, variable: `data_${key}` }));
398
1266
  const prefixedParams = Object.fromEntries(
@@ -410,6 +1278,12 @@ export class DataModule<S extends SchemaStructure> {
410
1278
  this.local.execute(query, {
411
1279
  id: rid,
412
1280
  mid: mutationId,
1281
+ // The record itself is written with per-key `data_<field>` vars
1282
+ // (`createSet`), but the OUTBOX row needs the whole payload in one
1283
+ // `data` field so the mutation can be replayed later. Without this the
1284
+ // row is written with `data` unbound: the create becomes unsendable and
1285
+ // blocks the queue behind it.
1286
+ data: params,
413
1287
  ...prefixedParams,
414
1288
  })
415
1289
  );
@@ -441,7 +1315,7 @@ export class DataModule<S extends SchemaStructure> {
441
1315
  callback([mutationEvent]);
442
1316
  }
443
1317
 
444
- this.logger.debug({ id, Category: 'spooky-client::DataModule::create' }, 'Record created');
1318
+ this.logger.debug({ id, Category: 'sp00ky-client::DataModule::create' }, 'Record created');
445
1319
 
446
1320
  return target;
447
1321
  }
@@ -463,7 +1337,10 @@ export class DataModule<S extends SchemaStructure> {
463
1337
 
464
1338
  const rid = parseRecordIdString(id);
465
1339
  const params = parseParams(tableSchema.columns, data);
466
- const mutationId = parseRecordIdString(`_spooky_pending_mutations:${Date.now()}`);
1340
+ const mutationId = parseRecordIdString(mintMutationId(this.tabId));
1341
+
1342
+ // Note: CRDT state is pushed directly to the _00_crdt table by CrdtField.pushToRemote(),
1343
+ // NOT through the record update pipeline. This keeps the record data clean.
467
1344
 
468
1345
  // Capture current record state before mutation for rollback support
469
1346
  const [beforeRecord] = await withRetry(this.logger, () =>
@@ -472,7 +1349,7 @@ export class DataModule<S extends SchemaStructure> {
472
1349
 
473
1350
  const query = surql.seal<{ target: T }>(
474
1351
  surql.tx([
475
- surql.updateSet('id', [{ statement: 'spooky_rv += 1' }]),
1352
+ surql.updateSet('id', [{ statement: '_00_rv += 1' }]),
476
1353
  surql.let('updated', surql.updateMerge('id', 'data')),
477
1354
  surql.createMutation('update', 'mid', 'id', 'data'),
478
1355
  surql.returnObject([{ key: 'target', variable: 'updated' }]),
@@ -496,8 +1373,8 @@ export class DataModule<S extends SchemaStructure> {
496
1373
  updatedFields[key] = (target as Record<string, any>)[key];
497
1374
  }
498
1375
  }
499
- if ('spooky_rv' in (target as Record<string, any>)) {
500
- updatedFields.spooky_rv = (target as Record<string, any>).spooky_rv;
1376
+ if ('_00_rv' in (target as Record<string, any>)) {
1377
+ updatedFields._00_rv = (target as Record<string, any>)._00_rv;
501
1378
  }
502
1379
  this.replaceRecordInQueries(updatedFields);
503
1380
 
@@ -509,7 +1386,7 @@ export class DataModule<S extends SchemaStructure> {
509
1386
  table: table,
510
1387
  op: 'UPDATE',
511
1388
  record: parsedRecord,
512
- version: target.spooky_rv as number,
1389
+ version: target._00_rv as number,
513
1390
  },
514
1391
  true
515
1392
  );
@@ -531,7 +1408,7 @@ export class DataModule<S extends SchemaStructure> {
531
1408
  callback([mutationEvent]);
532
1409
  }
533
1410
 
534
- this.logger.debug({ id, Category: 'spooky-client::DataModule::update' }, 'Record updated');
1411
+ this.logger.debug({ id, Category: 'sp00ky-client::DataModule::update' }, 'Record updated');
535
1412
 
536
1413
  return target;
537
1414
  }
@@ -547,14 +1424,53 @@ export class DataModule<S extends SchemaStructure> {
547
1424
  }
548
1425
 
549
1426
  const rid = parseRecordIdString(id);
550
- const mutationId = parseRecordIdString(`_spooky_pending_mutations:${Date.now()}`);
1427
+ const mutationId = parseRecordIdString(mintMutationId(this.tabId));
1428
+
1429
+ // Fetch the record before deleting so DBSP can match it against query predicates
1430
+ const [beforeRecords] = await this.local.query<[Record<string, any>[]]>(
1431
+ 'SELECT * FROM ONLY $id',
1432
+ { id: rid }
1433
+ );
1434
+ const beforeRecord = beforeRecords ?? {};
551
1435
 
552
1436
  const query = surql.seal<void>(
553
1437
  surql.tx([surql.delete('id'), surql.createMutation('delete', 'mid', 'id')])
554
1438
  );
555
1439
 
556
1440
  await withRetry(this.logger, () => this.local.execute(query, { id: rid, mid: mutationId }));
557
- await this.cache.delete(table, id, true);
1441
+
1442
+ // The local DELETE has now committed. Everything below must reflect that in
1443
+ // active live queries — so the deleted row disappears optimistically without
1444
+ // a reload — even if the optimistic SSP-view ingest below fails. Previously a
1445
+ // throw from `cache.delete` (the WASM ingest) aborted `delete()` after the
1446
+ // commit, so the manual notify loop never ran and the row lingered on screen
1447
+ // until reload. Ingesting the delete into the in-browser SSP view is
1448
+ // best-effort: the manual re-materialize reads the local DB (which already
1449
+ // excludes the row), so the result is correct regardless.
1450
+ try {
1451
+ await this.cache.delete(table, id, true, beforeRecord);
1452
+ } catch (err) {
1453
+ this.logger.error(
1454
+ { err, id, Category: 'sp00ky-client::DataModule::delete' },
1455
+ 'SSP delete-ingest failed; relying on query re-materialize to reflect the delete'
1456
+ );
1457
+ }
1458
+
1459
+ // DBSP may not emit view updates for DELETE ops — manually notify all queries
1460
+ // that reference this table. Each is isolated so one failing re-materialize
1461
+ // can't stop the others (or the sync emit below) from running.
1462
+ for (const [queryHash, queryState] of this.activeQueries) {
1463
+ if (queryState.config.tableName === tableName) {
1464
+ try {
1465
+ await this.notifyQuerySynced(queryHash);
1466
+ } catch (err) {
1467
+ this.logger.error(
1468
+ { err, queryHash, Category: 'sp00ky-client::DataModule::delete' },
1469
+ 'notifyQuerySynced failed after delete'
1470
+ );
1471
+ }
1472
+ }
1473
+ }
558
1474
 
559
1475
  // Emit mutation event
560
1476
  const mutationEvent: DeleteEvent = {
@@ -567,7 +1483,7 @@ export class DataModule<S extends SchemaStructure> {
567
1483
  callback([mutationEvent]);
568
1484
  }
569
1485
 
570
- this.logger.debug({ id, Category: 'spooky-client::DataModule::delete' }, 'Record deleted');
1486
+ this.logger.debug({ id, Category: 'sp00ky-client::DataModule::delete' }, 'Record deleted');
571
1487
  }
572
1488
 
573
1489
  // ==================== ROLLBACK METHODS ====================
@@ -579,19 +1495,17 @@ export class DataModule<S extends SchemaStructure> {
579
1495
  const id = encodeRecordId(recordId);
580
1496
 
581
1497
  try {
582
- await withRetry(this.logger, () =>
583
- this.local.query('DELETE $id', { id: recordId })
584
- );
1498
+ await withRetry(this.logger, () => this.local.query('DELETE $id', { id: recordId }));
585
1499
  await this.cache.delete(tableName, id, true);
586
1500
  this.removeRecordFromQueries(recordId);
587
1501
 
588
1502
  this.logger.info(
589
- { id, tableName, Category: 'spooky-client::DataModule::rollbackCreate' },
1503
+ { id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
590
1504
  'Rolled back optimistic create'
591
1505
  );
592
1506
  } catch (err) {
593
1507
  this.logger.error(
594
- { err, id, tableName, Category: 'spooky-client::DataModule::rollbackCreate' },
1508
+ { err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackCreate' },
595
1509
  'Failed to rollback create'
596
1510
  );
597
1511
  }
@@ -626,7 +1540,7 @@ export class DataModule<S extends SchemaStructure> {
626
1540
  table: tableName,
627
1541
  op: 'UPDATE',
628
1542
  record: parsedRecord,
629
- version: (beforeRecord.spooky_rv as number) || 1,
1543
+ version: (beforeRecord._00_rv as number) || 1,
630
1544
  },
631
1545
  true
632
1546
  );
@@ -635,12 +1549,12 @@ export class DataModule<S extends SchemaStructure> {
635
1549
  await this.replaceRecordInQueries(beforeRecord);
636
1550
 
637
1551
  this.logger.info(
638
- { id, tableName, Category: 'spooky-client::DataModule::rollbackUpdate' },
1552
+ { id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
639
1553
  'Rolled back optimistic update'
640
1554
  );
641
1555
  } catch (err) {
642
1556
  this.logger.error(
643
- { err, id, tableName, Category: 'spooky-client::DataModule::rollbackUpdate' },
1557
+ { err, id, tableName, Category: 'sp00ky-client::DataModule::rollbackUpdate' },
644
1558
  'Failed to rollback update'
645
1559
  );
646
1560
  }
@@ -678,7 +1592,8 @@ export class DataModule<S extends SchemaStructure> {
678
1592
  surqlString: string,
679
1593
  params: Record<string, any>,
680
1594
  ttl: QueryTimeToLive,
681
- tableName: T
1595
+ tableName: T,
1596
+ plan?: QueryPlan
682
1597
  ): Promise<QueryHash> {
683
1598
  const queryState = await this.createNewQuery<T>({
684
1599
  recordId,
@@ -686,31 +1601,76 @@ export class DataModule<S extends SchemaStructure> {
686
1601
  params,
687
1602
  ttl,
688
1603
  tableName,
1604
+ plan,
689
1605
  });
690
1606
 
691
- const { localArray } = this.cache.registerQuery({
1607
+ const t0 = performance.now();
1608
+ const { localArray, registrationTimings } = this.cache.registerQuery({
692
1609
  queryHash: hash,
693
1610
  surql: surqlString,
694
1611
  params,
695
1612
  ttl: new Duration(ttl),
696
1613
  lastActiveAt: new Date(),
697
1614
  });
1615
+ const registrationTime = performance.now() - t0;
1616
+
1617
+ // Record the one-shot SSP registration timings (parse/plan/snapshot from the
1618
+ // WASM binding) + the register_view wall time for DevTools.
1619
+ queryState.registrationTimings = {
1620
+ parseMs: registrationTimings?.parseMs ?? null,
1621
+ planMs: registrationTimings?.planMs ?? null,
1622
+ snapshotMs: registrationTimings?.snapshotMs ?? null,
1623
+ wallMs: registrationTime,
1624
+ };
698
1625
 
699
1626
  await withRetry(this.logger, () =>
700
- this.local.query(surql.seal(surql.updateSet('id', ['localArray'])), {
701
- id: recordId,
702
- localArray,
703
- })
1627
+ this.local.query(
1628
+ surql.seal(surql.updateSet('id', ['localArray', 'registrationTime', 'rowCount'])),
1629
+ {
1630
+ id: recordId,
1631
+ localArray,
1632
+ registrationTime,
1633
+ rowCount: localArray.length,
1634
+ }
1635
+ )
704
1636
  );
705
1637
 
1638
+ // Windowed (`START n`) queries skipped the raw initial load in
1639
+ // createNewQuery (O(offset) + wrong rows for sparse windows). Seed the
1640
+ // initial rows now from the SSP's window id-set (`localArray`) via the same
1641
+ // window-materialization path the stream updates use — O(window), and the
1642
+ // ids are already the correct window — so the first paint isn't empty while
1643
+ // the remote `_00_list_ref` syncs in.
1644
+ const windowMat = buildWindowMaterialization(surqlString);
1645
+ if (windowMat && localArray.length > 0) {
1646
+ try {
1647
+ const winIds = localArray.map(([id]) => parseRecordIdString(id));
1648
+ if (plan) {
1649
+ const winPlan = buildWindowMaterializationPlan(plan, winIds) ?? { ...plan, ids: winIds };
1650
+ queryState.records = await this.local.select(winPlan, params);
1651
+ } else {
1652
+ const [seeded] = await this.local.query<[Record<string, any>[]]>(windowMat.query, {
1653
+ ...params,
1654
+ __win: winIds,
1655
+ });
1656
+ queryState.records = seeded || [];
1657
+ }
1658
+ } catch (err) {
1659
+ this.logger.warn(
1660
+ { err, hash, Category: 'sp00ky-client::DataModule::createAndRegisterQuery' },
1661
+ 'Failed to seed windowed initial records from localArray'
1662
+ );
1663
+ }
1664
+ }
1665
+
706
1666
  this.activeQueries.set(hash, queryState);
707
- this.startTTLHeartbeat(queryState);
1667
+ this.startTTLHeartbeat(queryState, hash);
708
1668
  this.logger.debug(
709
1669
  {
710
1670
  hash,
711
1671
  tableName,
712
1672
  recordCount: queryState.records.length,
713
- Category: 'spooky-client::DataModule::query',
1673
+ Category: 'sp00ky-client::DataModule::query',
714
1674
  },
715
1675
  'Query registered'
716
1676
  );
@@ -724,14 +1684,25 @@ export class DataModule<S extends SchemaStructure> {
724
1684
  params,
725
1685
  ttl,
726
1686
  tableName,
1687
+ plan,
727
1688
  }: {
728
1689
  recordId: RecordId;
729
1690
  surql: string;
730
1691
  params: Record<string, any>;
731
1692
  ttl: QueryTimeToLive;
732
1693
  tableName: T;
1694
+ plan?: QueryPlan;
733
1695
  }): Promise<QueryState> {
734
- const tableSchema = this.schema.tables.find((t) => t.name === tableName);
1696
+ // `_00_*` meta tables (feature flags, app releases) are framework-owned:
1697
+ // they exist in the client db schema by construction but are never part of
1698
+ // the app's generated `schema.tables`, so give them an empty column map
1699
+ // instead of the not-found error (which silently broke every meta-table
1700
+ // live query, e.g. feature flags never updating on this path).
1701
+ const tableSchema =
1702
+ this.schema.tables.find((t) => t.name === tableName) ??
1703
+ (String(tableName).startsWith('_00_')
1704
+ ? ({ name: tableName, columns: {} } as any)
1705
+ : undefined);
735
1706
  if (!tableSchema) {
736
1707
  throw new Error(`Table ${tableName} not found`);
737
1708
  }
@@ -752,8 +1723,12 @@ export class DataModule<S extends SchemaStructure> {
752
1723
  localArray: [],
753
1724
  remoteArray: [],
754
1725
  lastActiveAt: new Date(),
1726
+ createdAt: new Date(),
755
1727
  ttl,
756
1728
  tableName,
1729
+ updateCount: 0,
1730
+ rowCount: 0,
1731
+ errorCount: 0,
757
1732
  },
758
1733
  })
759
1734
  );
@@ -763,62 +1738,109 @@ export class DataModule<S extends SchemaStructure> {
763
1738
  const config: QueryConfig = {
764
1739
  ...configRecord,
765
1740
  id: recordId,
1741
+ // In-memory only — carries the engine-neutral plan so non-SurrealQL local
1742
+ // engines materialize via `select(plan)` instead of parsing `surql`.
1743
+ plan,
766
1744
  params: parseParams(tableSchema.columns, configRecord.params),
767
1745
  };
768
1746
 
769
1747
  let records: Record<string, any>[] = [];
770
- try {
771
- const [result] = await this.local.query<[Record<string, any>[]]>(surqlString, params);
772
- records = result || [];
773
- } catch (err) {
774
- this.logger.warn(
775
- { err, Category: 'spooky-client::DataModule::createNewQuery' },
776
- 'Failed to load initial cached records'
777
- );
1748
+ // Windowed (`START n`) queries: do NOT seed from the raw surql here. Running
1749
+ // `… LIMIT n START m` against the shared local store is O(m) — it sorts and
1750
+ // skips m rows on every window open — AND returns the wrong rows for sparse
1751
+ // windows (the reason `buildWindowMaterialization` exists). Those windows are
1752
+ // seeded from the SSP `localArray` in `createAndRegisterQuery` instead.
1753
+ if (buildWindowMaterialization(surqlString) === null) {
1754
+ try {
1755
+ // Prefer the engine-neutral plan (required for non-SurrealQL engines);
1756
+ // fall back to running the raw surql on the SurrealDB engine.
1757
+ const result = plan
1758
+ ? await this.local.select(plan, params)
1759
+ : (await this.local.query<[Record<string, any>[]]>(surqlString, params))[0];
1760
+ records = result || [];
1761
+ } catch (err) {
1762
+ this.logger.warn(
1763
+ { err, Category: 'sp00ky-client::DataModule::createNewQuery' },
1764
+ 'Failed to load initial cached records'
1765
+ );
1766
+ }
778
1767
  }
779
1768
 
1769
+ // Persisted counters survive a restart even though the rolling
1770
+ // sample window is rebuilt from scratch in memory.
1771
+ const persistedUpdateCount =
1772
+ typeof (configRecord as any)?.updateCount === 'number'
1773
+ ? (configRecord as any).updateCount
1774
+ : 0;
1775
+ const persistedErrorCount =
1776
+ typeof (configRecord as any)?.errorCount === 'number' ? (configRecord as any).errorCount : 0;
1777
+
780
1778
  return {
781
1779
  config,
782
1780
  records,
783
1781
  ttlTimer: null,
784
1782
  ttlDurationMs: parseDuration(ttl),
785
- updateCount: 0,
1783
+ updateCount: persistedUpdateCount,
1784
+ lastUpdatedAt: null,
1785
+ materializationSamples: [],
1786
+ lastIngestLatencyMs: null,
1787
+ errorCount: persistedErrorCount,
1788
+ // Born `fetching`, not `idle`: every cold registration is followed by a
1789
+ // `register` down-event whose lifecycle (Sp00kySync.registerQuery) resolves
1790
+ // the status to `idle` once the initial sync completed. Starting idle left
1791
+ // a gap where a fresh windowed query looked settled while still empty.
1792
+ status: 'fetching',
1793
+ phaseSamples: {},
1794
+ phaseLast: {},
1795
+ registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
786
1796
  };
787
1797
  }
788
1798
 
789
1799
  private async calculateHash(data: any): Promise<string> {
790
- const content = JSON.stringify(data);
1800
+ // sessionId is part of the hash so the same logical query from two
1801
+ // sessions (e.g. two browser tabs of the same user) lands on different
1802
+ // `_00_query` rows and doesn't fight over a shared one.
1803
+ const content = JSON.stringify({ ...data, sessionId: this.sessionId });
791
1804
  const msgBuffer = new TextEncoder().encode(content);
792
1805
  const hashBuffer = await crypto.subtle.digest('SHA-256', msgBuffer);
793
1806
  const hashArray = Array.from(new Uint8Array(hashBuffer));
794
1807
  return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('');
795
1808
  }
796
1809
 
797
- private startTTLHeartbeat(queryState: QueryState): void {
1810
+ private startTTLHeartbeat(queryState: QueryState, hash: QueryHash): void {
798
1811
  if (queryState.ttlTimer) return;
799
1812
 
800
1813
  const heartbeatTime = Math.floor(queryState.ttlDurationMs * 0.9);
801
1814
 
802
1815
  queryState.ttlTimer = setTimeout(() => {
803
- // TODO: Emit heartbeat event for sync
1816
+ queryState.ttlTimer = null;
1817
+ // Only keep the remote query alive while something is actually watching
1818
+ // it. With the server now sweeping ALL expired views (not just in-circuit
1819
+ // ones), an un-refreshed query WOULD be swept after its TTL — so a live
1820
+ // subscriber must heartbeat. An abandoned query (no subscribers) is left
1821
+ // to expire and get swept; its local view was already torn down at the
1822
+ // last unsubscribe, so we just stop the timer here.
1823
+ const subscriberCount = this.subscriptions.get(hash)?.size ?? 0;
1824
+ if (subscriberCount === 0) {
1825
+ this.logger.debug(
1826
+ { hash, Category: 'sp00ky-client::DataModule::startTTLHeartbeat' },
1827
+ 'TTL heartbeat: no subscribers, stopping'
1828
+ );
1829
+ return;
1830
+ }
1831
+ this.onHeartbeat?.(hash);
804
1832
  this.logger.debug(
805
1833
  {
1834
+ hash,
806
1835
  id: encodeRecordId(queryState.config.id),
807
- Category: 'spooky-client::DataModule::startTTLHeartbeat',
1836
+ Category: 'sp00ky-client::DataModule::startTTLHeartbeat',
808
1837
  },
809
- 'TTL heartbeat'
1838
+ 'TTL heartbeat sent'
810
1839
  );
811
- this.startTTLHeartbeat(queryState);
1840
+ this.startTTLHeartbeat(queryState, hash);
812
1841
  }, heartbeatTime);
813
1842
  }
814
1843
 
815
- private stopTTLHeartbeat(queryState: QueryState): void {
816
- if (queryState.ttlTimer) {
817
- clearTimeout(queryState.ttlTimer);
818
- queryState.ttlTimer = null;
819
- }
820
- }
821
-
822
1844
  private async replaceRecordInQueries(record: Record<string, any>): Promise<void> {
823
1845
  for (const [queryHash, queryState] of this.activeQueries.entries()) {
824
1846
  const index = queryState.records.findIndex((r) => r.id === record.id);
@@ -851,7 +1873,7 @@ export function parseUpdateOptions(
851
1873
  const delay = options.debounced !== true ? (options.debounced?.delay ?? 200) : 200;
852
1874
  const keyType = options.debounced !== true ? (options.debounced?.key ?? id) : id;
853
1875
  const key =
854
- keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).sort().join('#')}` : id;
1876
+ keyType === 'recordId_x_fields' ? `${id}::${Object.keys(data).toSorted().join('#')}` : id;
855
1877
 
856
1878
  pushEventOptions = {
857
1879
  debounced: {