@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2514 -58
  3. package/dist/index.js +12561 -2449
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +101 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +127 -24
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +164 -82
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +180 -0
  65. package/src/modules/sync/queue/queue-down.ts +80 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1726 -99
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -17
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +110 -14
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1693 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -1,13 +1,14 @@
1
- import { LocalDatabaseService } from '../../services/database/index';
2
- import {
1
+ import type { LocalStore } from '../../services/database/index';
2
+ import { StaleEpochError } from '../../services/database/index';
3
+ import type {
3
4
  StreamProcessorService,
4
5
  StreamUpdate,
5
6
  StreamUpdateReceiver,
6
7
  } from '../../services/stream-processor/index';
7
- import { Logger } from '../../services/logger/index';
8
+ import type { Logger } from '../../services/logger/index';
8
9
  import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
9
- import { CacheRecord, QueryConfig } from './types';
10
- import { RecordVersionArray } from '../../types';
10
+ import type { CacheRecord, QueryConfig } from './types';
11
+ import type { RecordVersionArray } from '../../types';
11
12
 
12
13
  export * from './types';
13
14
 
@@ -17,13 +18,29 @@ export * from './types';
17
18
  * Single responsibility: Handle all local storage operations and DBSP ingestion.
18
19
  * This module acts as the bridge between data operations and persistence.
19
20
  */
21
+ /** One ingested change, in exactly the shape `ingestMany` consumes. Shared
22
+ * with the tabs protocol so a leader can relay its ingests to followers. */
23
+ export interface CacheIngestTuple {
24
+ table: string;
25
+ op: 'CREATE' | 'UPDATE' | 'DELETE';
26
+ id: string;
27
+ record: Record<string, unknown>;
28
+ }
29
+
20
30
  export class CacheModule implements StreamUpdateReceiver {
21
31
  private logger: Logger;
22
32
  private streamUpdateCallback: (update: StreamUpdate) => void;
23
33
  private versionLookups: Record<string, number> = {};
34
+ /** Shared-tabs leader: fan every committed ingest out to follower circuits.
35
+ * Fired AFTER the local tx (the rows are already in the shared store, so a
36
+ * follower only needs the circuit feed). A follower relays its own
37
+ * mutations to the leader the same way, see {@link setIngestRelay}. */
38
+ private ingestRelay: ((tuples: CacheIngestTuple[]) => void) | null = null;
39
+ /** See {@link setIngestRelay}. */
40
+ private relayLocalWritesOnly = false;
24
41
 
25
42
  constructor(
26
- private local: LocalDatabaseService,
43
+ private local: LocalStore,
27
44
  private streamProcessor: StreamProcessorService,
28
45
  streamUpdateCallback: (update: StreamUpdate) => void,
29
46
  logger: Logger
@@ -43,17 +60,69 @@ export class CacheModule implements StreamUpdateReceiver {
43
60
  {
44
61
  queryHash: update.queryHash,
45
62
  arrayLength: update.localArray?.length,
46
- Category: 'spooky-client::CacheModule::onStreamUpdate',
63
+ Category: 'sp00ky-client::CacheModule::onStreamUpdate',
47
64
  },
48
65
  'Stream update received'
49
66
  );
50
67
  this.streamUpdateCallback(update);
51
68
  }
52
69
 
70
+ /**
71
+ * Fan every committed ingest out to the other tabs. The leader relays
72
+ * everything (its sync fetches are the only copy the followers get). A
73
+ * follower relays with `localWritesOnly`: just the mutation path, which is
74
+ * the only thing it knows that the leader does not. Its sync-fetched
75
+ * batches are the leader's data coming back and must not be re-broadcast,
76
+ * or every follower registration would fan its whole working set to
77
+ * every tab.
78
+ */
79
+ setIngestRelay(
80
+ cb: ((tuples: CacheIngestTuple[]) => void) | null,
81
+ opts: { localWritesOnly?: boolean } = {}
82
+ ): void {
83
+ this.ingestRelay = cb;
84
+ this.relayLocalWritesOnly = opts.localWritesOnly === true;
85
+ }
86
+
87
+ /**
88
+ * Shared-tabs follower: feed relayed tuples into THIS tab's circuit only.
89
+ * The rows are already in the shared store (the leader wrote them), so no
90
+ * local write happens here; the normal chain then runs: SSP -> stream update
91
+ * -> DataModule debounce -> materializeRecords (re-reads via the port
92
+ * transport) -> this tab's subscriptions fire with this tab's hashes.
93
+ */
94
+ applyRelayedIngest(tuples: CacheIngestTuple[]): void {
95
+ for (const t of tuples) {
96
+ const rv = (t.record as { _00_rv?: number } | undefined)?._00_rv;
97
+ if (t.op === 'DELETE') delete this.versionLookups[t.id];
98
+ else if (typeof rv === 'number') this.versionLookups[t.id] = rv;
99
+ }
100
+ this.streamProcessor.ingestMany(tuples as Parameters<StreamProcessorService['ingestMany']>[0]);
101
+ }
102
+
53
103
  public lookup(recordId: string): number {
54
104
  return this.versionLookups[recordId] ?? 0;
55
105
  }
56
106
 
107
+ /**
108
+ * Seed the version memo from rows the circuit was primed with out of the
109
+ * local store, so the first post-reload sync diff does not re-download
110
+ * bodies the browser already has. Only rows the prime actually put into the
111
+ * circuit belong here: a memo entry with no circuit row would make the diff
112
+ * flag the id forever while nothing ever fetches it.
113
+ */
114
+ public primeVersions(entries: [string, number][]): void {
115
+ for (const [id, rv] of entries) {
116
+ if (rv > (this.versionLookups[id] ?? 0)) this.versionLookups[id] = rv;
117
+ }
118
+ }
119
+
120
+ /** Drop the version cache on a bucket switch — a stale version would make
121
+ * the sync diff skip fetching a body the new bucket legitimately needs. */
122
+ public clearVersionLookups(): void {
123
+ this.versionLookups = {};
124
+ }
125
+
57
126
  /**
58
127
  * Save a single record to local DB and ingest into DBSP
59
128
  * Used by mutations (create/update)
@@ -70,10 +139,17 @@ export class CacheModule implements StreamUpdateReceiver {
70
139
  async saveBatch(records: CacheRecord[], skipDbInsert: boolean = false): Promise<void> {
71
140
  if (records.length === 0) return;
72
141
 
142
+ // Fence against bucket switches: this batch's records were derived from
143
+ // reads against the CURRENT store/user. If a switch lands while we await
144
+ // the (gated) local write, the write throws StaleEpochError and the whole
145
+ // batch — including the SSP ingest — is dropped: the new bucket re-syncs
146
+ // its own data from the server.
147
+ const epoch = this.local.epoch;
148
+
73
149
  this.logger.debug(
74
150
  {
75
151
  count: records.length,
76
- Category: 'spooky-client::CacheModule::saveBatch',
152
+ Category: 'sp00ky-client::CacheModule::saveBatch',
77
153
  },
78
154
  'Saving record batch'
79
155
  );
@@ -85,7 +161,7 @@ export class CacheModule implements StreamUpdateReceiver {
85
161
  ...record,
86
162
  record: {
87
163
  ...record.record,
88
- spooky_rv: record.version,
164
+ _00_rv: record.version,
89
165
  },
90
166
  };
91
167
  });
@@ -94,40 +170,69 @@ export class CacheModule implements StreamUpdateReceiver {
94
170
  const query = surql.seal<void>(
95
171
  surql.tx(
96
172
  populatedRecords.map((_, i) => {
97
- return surql.upsert(`id${i}`, `content${i}`);
173
+ // MERGE, not REPLACE: the remote payload omits local-only
174
+ // fields (`_00_crdt`, `_00_cursor`) injected by the CLI's
175
+ // local schema, so REPLACE would wipe the persisted CRDT
176
+ // snapshot on every sync-down round-trip and break offline
177
+ // reload of formatted text.
178
+ return surql.upsertMerge(`id${i}`, `content${i}`);
98
179
  })
99
180
  )
100
181
  );
101
182
 
102
- const params = populatedRecords.reduce(
103
- (acc, record, i) => {
104
- const { id, ...content } = record.record;
105
- return {
106
- ...acc,
107
- [`id${i}`]: id,
108
- [`content${i}`]: content,
109
- };
110
- },
111
- {} as Record<string, any>
112
- );
183
+ // Filled in place, NOT with a spread-per-iteration reduce: spreading the
184
+ // accumulator copies every key written so far on each record, which is
185
+ // O(n^2) in the batch size. On a cold start that batches thousands of
186
+ // rows (a game library, a player-name registry) it was the single
187
+ // biggest main-thread cost of the whole boot - ~36% of samples, seconds
188
+ // of blocking - for a loop that does no real work.
189
+ const params: Record<string, any> = {};
190
+ for (let i = 0; i < populatedRecords.length; i++) {
191
+ const { id, ...content } = populatedRecords[i].record;
192
+ params[`id${i}`] = id;
193
+ params[`content${i}`] = content;
194
+ }
113
195
 
114
- await this.local.execute(query, params);
196
+ await this.local.execute(query, params, { epoch });
115
197
  }
116
198
 
117
- // 2. Batch ingest into DBSP (use populatedRecords which has spooky_rv set)
118
- for (const record of populatedRecords) {
199
+ // Late fence for the skipDbInsert path (no gated write above to trip on).
200
+ if (this.local.epoch !== epoch) throw new StaleEpochError();
201
+
202
+ // 2. Bulk ingest into DBSP (use populatedRecords which has _00_rv set).
203
+ // ingestMany coalesces the per-record stream updates into a single
204
+ // notification per affected query — the UI then updates once, after the
205
+ // whole batch is ingested, instead of row-by-row.
206
+ const versionOf = new Map<string, number>();
207
+ const bulk = populatedRecords.map((record) => {
119
208
  const recordId = encodeRecordId(record.record.id);
120
- this.versionLookups[recordId] = record.version;
121
- this.streamProcessor.ingest(record.table, record.op, recordId, record.record);
209
+ versionOf.set(recordId, record.version);
210
+ return { table: record.table, op: record.op, id: recordId, record: record.record };
211
+ });
212
+ // Memo and relay only what the circuit actually took: a chunk that
213
+ // failed mid-batch is neither "known" here nor fanned out to followers.
214
+ const ingested = this.streamProcessor.ingestMany(bulk);
215
+ for (const t of ingested) this.versionLookups[t.id] = versionOf.get(t.id) ?? 0;
216
+ // `skipDbInsert` is exactly the mutation path (the tx already wrote the
217
+ // row); sync-fetched batches pass false.
218
+ if (ingested.length > 0 && (!this.relayLocalWritesOnly || skipDbInsert)) {
219
+ this.ingestRelay?.(ingested as CacheIngestTuple[]);
122
220
  }
123
221
 
124
222
  this.logger.debug(
125
- { count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
223
+ { count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
126
224
  'Batch saved successfully'
127
225
  );
128
226
  } catch (err) {
227
+ if (err instanceof StaleEpochError) {
228
+ this.logger.debug(
229
+ { count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
230
+ 'Dropped batch from before a bucket switch'
231
+ );
232
+ return;
233
+ }
129
234
  this.logger.error(
130
- { err, count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
235
+ { err, count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
131
236
  'Failed to save batch'
132
237
  );
133
238
  throw err;
@@ -137,29 +242,41 @@ export class CacheModule implements StreamUpdateReceiver {
137
242
  /**
138
243
  * Delete a record from local DB and ingest deletion into DBSP
139
244
  */
140
- async delete(table: string, id: string, skipDbDelete: boolean = false): Promise<void> {
245
+ async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
141
246
  this.logger.debug(
142
- { table, id, Category: 'spooky-client::CacheModule::delete' },
247
+ { table, id, Category: 'sp00ky-client::CacheModule::delete' },
143
248
  'Deleting record'
144
249
  );
145
250
 
251
+ const epoch = this.local.epoch;
146
252
  try {
147
253
  // 1. Delete from local database
148
254
  if (!skipDbDelete) {
149
- await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
255
+ await this.local.query('DELETE $id', { id: parseRecordIdString(id) }, { epoch });
150
256
  }
257
+ if (this.local.epoch !== epoch) throw new StaleEpochError();
151
258
 
152
- // 2. Ingest deletion into DBSP
259
+ // 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
153
260
  delete this.versionLookups[id];
154
- await this.streamProcessor.ingest(table, 'DELETE', id, {});
261
+ this.streamProcessor.ingestMany([{ table, op: 'DELETE', id, record: recordData }]);
262
+ if (!this.relayLocalWritesOnly || skipDbDelete) {
263
+ this.ingestRelay?.([{ table, op: 'DELETE', id, record: recordData }]);
264
+ }
155
265
 
156
266
  this.logger.debug(
157
- { table, id, Category: 'spooky-client::CacheModule::delete' },
267
+ { table, id, Category: 'sp00ky-client::CacheModule::delete' },
158
268
  'Record deleted successfully'
159
269
  );
160
270
  } catch (err) {
271
+ if (err instanceof StaleEpochError) {
272
+ this.logger.debug(
273
+ { table, id, Category: 'sp00ky-client::CacheModule::delete' },
274
+ 'Dropped delete from before a bucket switch'
275
+ );
276
+ return;
277
+ }
161
278
  this.logger.error(
162
- { err, table, id, Category: 'spooky-client::CacheModule::delete' },
279
+ { err, table, id, Category: 'sp00ky-client::CacheModule::delete' },
163
280
  'Failed to delete record'
164
281
  );
165
282
  throw err;
@@ -170,12 +287,15 @@ export class CacheModule implements StreamUpdateReceiver {
170
287
  * Register a query with DBSP to create a materialized view
171
288
  * Returns the initial result array
172
289
  */
173
- registerQuery(config: QueryConfig): { localArray: RecordVersionArray } {
290
+ registerQuery(config: QueryConfig): {
291
+ localArray: RecordVersionArray;
292
+ registrationTimings?: { parseMs: number; planMs: number; snapshotMs: number };
293
+ } {
174
294
  this.logger.debug(
175
295
  {
176
296
  queryHash: config.queryHash,
177
297
  surql: config.surql,
178
- Category: 'spooky-client::CacheModule::registerQuery',
298
+ Category: 'sp00ky-client::CacheModule::registerQuery',
179
299
  },
180
300
  'Registering query'
181
301
  );
@@ -202,15 +322,15 @@ export class CacheModule implements StreamUpdateReceiver {
202
322
  {
203
323
  queryHash: config.queryHash,
204
324
  arrayLength: update.localArray?.length,
205
- Category: 'spooky-client::CacheModule::registerQuery',
325
+ Category: 'sp00ky-client::CacheModule::registerQuery',
206
326
  },
207
327
  'Query registered successfully'
208
328
  );
209
329
 
210
- return { localArray: update.localArray };
330
+ return { localArray: update.localArray, registrationTimings: update.registration };
211
331
  } catch (err) {
212
332
  this.logger.error(
213
- { err, queryHash: config.queryHash, Category: 'spooky-client::CacheModule::registerQuery' },
333
+ { err, queryHash: config.queryHash, Category: 'sp00ky-client::CacheModule::registerQuery' },
214
334
  'Failed to register query'
215
335
  );
216
336
  throw err;
@@ -222,18 +342,18 @@ export class CacheModule implements StreamUpdateReceiver {
222
342
  */
223
343
  unregisterQuery(queryHash: string): void {
224
344
  this.logger.debug(
225
- { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
345
+ { queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
226
346
  'Unregistering query'
227
347
  );
228
348
  try {
229
349
  this.streamProcessor.unregisterQueryPlan(queryHash);
230
350
  this.logger.debug(
231
- { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
351
+ { queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
232
352
  'Query unregistered successfully'
233
353
  );
234
354
  } catch (err) {
235
355
  this.logger.error(
236
- { err, queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
356
+ { err, queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
237
357
  'Failed to unregister query'
238
358
  );
239
359
  }
@@ -1,5 +1,5 @@
1
- import { RecordId, Duration } from 'surrealdb';
2
- import { QueryTimeToLive, RecordVersionArray } from '../../types';
1
+ import type { RecordId, Duration } from 'surrealdb';
2
+ import type { QueryTimeToLive } from '../../types';
3
3
 
4
4
  export type RecordWithId = Record<string, any> & { id: RecordId<string> };
5
5
 
@@ -0,0 +1,294 @@
1
+ import type { LoroDoc } from 'loro-crdt';
2
+ import type { LocalStore, RemoteDatabaseService } from '../../services/database/index';
3
+ import type { Logger } from '../../services/logger/index';
4
+ import { parseRecordIdString } from '../../utils/index';
5
+
6
+ // ==================== CURSOR UTILITIES ====================
7
+
8
+ export const CURSOR_COLORS = [
9
+ '#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
10
+ '#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
11
+ ];
12
+
13
+ export function cursorColorFromName(name: string): string {
14
+ let hash = 0;
15
+ for (let i = 0; i < name.length; i++) {
16
+ hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
17
+ }
18
+ return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
19
+ }
20
+
21
+ // ==================== CRDT FIELD ====================
22
+
23
+ export class CrdtField {
24
+ private doc: LoroDoc;
25
+ private pushTimer: ReturnType<typeof setTimeout> | null = null;
26
+ private local: LocalStore | null = null;
27
+ private remote: RemoteDatabaseService | null = null;
28
+ private recordId: string | null = null;
29
+ private sessionId: string = '';
30
+ private unsubscribe: (() => void) | null = null;
31
+ private lastPushTime = 0;
32
+ private lastCursorPushTime = 0;
33
+ private loadedFromCrdt = false;
34
+ private pushRetryCount = 0;
35
+ private logger: Logger | null;
36
+ private cursorsEnabled: boolean;
37
+ /** Remote-push debounce. Local writes happen immediately on every Loro
38
+ * update; the remote UPSERT is coalesced over this window. Configured
39
+ * via `Sp00kyConfig.crdtDebounceMs`, default 500. */
40
+ private remoteDebounceMs: number = 500;
41
+
42
+ private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
43
+ private pendingCursorUpdate: Uint8Array | null = null;
44
+
45
+ /** Callback set by the editor to receive remote cursor updates.
46
+ * Any cursor data that arrived before this callback was set will be replayed. */
47
+ set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
48
+ this._onCursorUpdate = cb;
49
+ if (cb && this.pendingCursorUpdate) {
50
+ try { cb(this.pendingCursorUpdate); } catch (e) {
51
+ this.logger?.warn(
52
+ { error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
53
+ 'Failed to replay pending cursor update'
54
+ );
55
+ }
56
+ this.pendingCursorUpdate = null;
57
+ }
58
+ }
59
+
60
+ get onCursorUpdate() { return this._onCursorUpdate; }
61
+
62
+ /**
63
+ * @param LoroDocClass the `LoroDoc` constructor, injected by the caller after
64
+ * awaiting {@link loadLoro} — keeps `loro-crdt` out of this module's static
65
+ * import graph so it only ships to apps that use CRDT fields.
66
+ */
67
+ constructor(
68
+ private fieldName: string,
69
+ cursorsEnabled: boolean,
70
+ LoroDocClass: typeof LoroDoc,
71
+ initialState?: Uint8Array,
72
+ logger?: Logger | null,
73
+ ) {
74
+ if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(fieldName)) {
75
+ throw new Error(
76
+ `CrdtField: refusing unsafe field identifier '${fieldName}' — must match [a-zA-Z_][a-zA-Z0-9_]*`
77
+ );
78
+ }
79
+ this.logger = logger ?? null;
80
+ this.cursorsEnabled = cursorsEnabled;
81
+ this.doc = new LoroDocClass();
82
+ if (initialState && initialState.length > 0) {
83
+ // Tolerance: catch bad-snapshot data (corrupt blob, stale legacy
84
+ // value left over from a pre-`bytes` migration) so the editor still
85
+ // mounts. Without this guard the rejection bubbles through
86
+ // `useCrdtField` → permanent fallback `<p>`, with no cursor.
87
+ try {
88
+ this.doc.import(initialState);
89
+ this.loadedFromCrdt = true;
90
+ } catch (e) {
91
+ this.logger?.warn(
92
+ {
93
+ error: e,
94
+ fieldName,
95
+ Category: 'sp00ky-client::CrdtField::constructor',
96
+ },
97
+ 'Initial CRDT state is not a valid LoroDoc snapshot — starting empty and will seed from fallback text'
98
+ );
99
+ }
100
+ }
101
+ }
102
+
103
+ getDoc(): LoroDoc { return this.doc; }
104
+
105
+ /** Whether the LoroDoc was loaded from saved CRDT state */
106
+ hasContent(): boolean {
107
+ return this.loadedFromCrdt;
108
+ }
109
+
110
+ startSync(
111
+ local: LocalStore,
112
+ remote: RemoteDatabaseService,
113
+ recordId: string,
114
+ sessionId: string,
115
+ debounceMs: number,
116
+ ): void {
117
+ this.local = local;
118
+ this.remote = remote;
119
+ this.recordId = recordId;
120
+ this.sessionId = sessionId;
121
+ this.remoteDebounceMs = debounceMs;
122
+ // Every local Loro update writes the snapshot to the local cache
123
+ // *immediately* (so reload/offline see the latest text), then
124
+ // schedules a debounced push to remote. The local UPSERT is cheap —
125
+ // it's an in-memory SurrealKV write — but errors are swallowed so a
126
+ // bad write never blocks user input.
127
+ this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
128
+ void this.persistLocal();
129
+ this.scheduleRemotePush();
130
+ });
131
+ }
132
+
133
+ /**
134
+ * Stop syncing this field. Flushes one final remote push by default so the
135
+ * last keystrokes aren't lost. Pass `{ flush: false }` on a bucket switch —
136
+ * the remote session already belongs to the NEXT user, and pushing this
137
+ * (previous user's) snapshot under it would clobber the record remotely.
138
+ */
139
+ stopSync(options: { flush?: boolean } = {}): void {
140
+ const flush = options.flush ?? true;
141
+ if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
142
+ if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
143
+ if (flush && this.remote && this.recordId) { void this.pushToRemote(); }
144
+ }
145
+
146
+ importRemote(state: Uint8Array): void {
147
+ // Echo suppression: skip imports within `remoteDebounceMs + 200` of
148
+ // our own push. The +200 guards against round-trip jitter where our
149
+ // own write echoes back from the LIVE feed before the debounce
150
+ // window closes.
151
+ if (Date.now() - this.lastPushTime < this.remoteDebounceMs + 200) return;
152
+ try {
153
+ this.doc.import(state);
154
+ // Persist the merged snapshot locally so the next reload/offline
155
+ // open sees the freshest converged state without waiting for the
156
+ // LIVE feed.
157
+ void this.persistLocal();
158
+ } catch (e) {
159
+ this.logger?.warn(
160
+ { error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
161
+ 'Failed to import remote CRDT state'
162
+ );
163
+ }
164
+ }
165
+
166
+ exportSnapshot(): Uint8Array {
167
+ return this.doc.export({ mode: 'snapshot' });
168
+ }
169
+
170
+ /** Push this session's cursor blob into the parent row at
171
+ * `<field>.cursors[$sid]`. No-op when cursors aren't enabled on this
172
+ * field — the editor still calls this method optimistically, but
173
+ * without `@cursor` on the schema there's nowhere to store the blob.
174
+ * The UPDATE itself fires the parent table's LIVE feed, so other
175
+ * browsers receive the cursor change without a separate `_00_rv` bump. */
176
+ async pushCursorState(encoded: Uint8Array): Promise<void> {
177
+ if (!this.remote || !this.recordId) return;
178
+ if (!this.cursorsEnabled) return;
179
+ this.lastCursorPushTime = Date.now();
180
+ try {
181
+ const state = encodeBase64(encoded);
182
+ await this.remote.query(
183
+ `UPDATE $id SET ${this.fieldName}.cursors[$sid] = $state RETURN NONE;`,
184
+ {
185
+ id: parseRecordIdString(this.recordId),
186
+ sid: this.sessionId,
187
+ state,
188
+ }
189
+ );
190
+ } catch (e) {
191
+ this.logger?.warn(
192
+ { error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
193
+ 'Failed to push cursor state'
194
+ );
195
+ }
196
+ }
197
+
198
+ /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
199
+ importRemoteCursor(base64State: string): void {
200
+ if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
201
+ try {
202
+ const data = decodeBase64(base64State);
203
+ if (this._onCursorUpdate) {
204
+ this._onCursorUpdate(data);
205
+ } else {
206
+ // Only keep the latest cursor state — older positions are useless
207
+ this.pendingCursorUpdate = data;
208
+ }
209
+ } catch (e) {
210
+ this.logger?.warn(
211
+ { error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
212
+ 'Failed to apply remote cursor data'
213
+ );
214
+ }
215
+ }
216
+
217
+ private scheduleRemotePush(): void {
218
+ if (this.pushTimer) clearTimeout(this.pushTimer);
219
+ this.pushTimer = setTimeout(() => void this.pushToRemote(), this.remoteDebounceMs);
220
+ }
221
+
222
+ /** SET path inside a parent row for the current snapshot. `@crdt`-only
223
+ * fields hold the snapshot directly (`<field>`); `@crdt @cursor`
224
+ * fields hold a `{ state, cursors }` object so the snapshot lives at
225
+ * `<field>.state` next to per-session cursor blobs. */
226
+ private statePath(): string {
227
+ return this.cursorsEnabled ? `${this.fieldName}.state` : this.fieldName;
228
+ }
229
+
230
+ /** Mirror the LoroDoc snapshot into the parent row locally. Runs on
231
+ * every local update and every remote import so reloads (online or
232
+ * offline) see the freshest content immediately. Failures are
233
+ * swallowed — a stale local write must never block user input. */
234
+ private async persistLocal(): Promise<void> {
235
+ if (!this.local || !this.recordId) return;
236
+ try {
237
+ await this.local.query(
238
+ `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
239
+ { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
240
+ );
241
+ } catch (e) {
242
+ this.logger?.debug(
243
+ { error: e, Category: 'sp00ky-client::CrdtField::persistLocal' },
244
+ 'Local CRDT persist failed (best-effort)'
245
+ );
246
+ }
247
+ }
248
+
249
+ private async pushToRemote(): Promise<void> {
250
+ if (!this.remote || !this.recordId) return;
251
+ this.lastPushTime = Date.now();
252
+ try {
253
+ // The UPDATE on the parent fires the parent table's LIVE feed,
254
+ // so cross-browser receivers see this change directly in the LIVE
255
+ // payload — no separate `_00_rv` bump or sidecar UPSERT.
256
+ await this.remote.query(
257
+ `UPDATE $id SET ${this.statePath()} = $state RETURN NONE;`,
258
+ { id: parseRecordIdString(this.recordId), state: this.exportSnapshot() }
259
+ );
260
+ this.pushRetryCount = 0;
261
+ } catch (e) {
262
+ this.logger?.warn(
263
+ { error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
264
+ 'Failed to push CRDT state to remote'
265
+ );
266
+ // Bounded retry. Offline first-loads will exhaust this and stop
267
+ // hammering; the next user keystroke (or the next time a remote
268
+ // event lands once we're back online) will kick off another push.
269
+ if (this.pushRetryCount < 2) {
270
+ this.pushRetryCount++;
271
+ this.scheduleRemotePush();
272
+ }
273
+ }
274
+ }
275
+ }
276
+
277
+ export function decodeBase64(b64: string): Uint8Array {
278
+ if (typeof atob === 'function') {
279
+ const binary = atob(b64);
280
+ const bytes = new Uint8Array(binary.length);
281
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
282
+ return bytes;
283
+ }
284
+ return new Uint8Array(Buffer.from(b64, 'base64'));
285
+ }
286
+
287
+ export function encodeBase64(bytes: Uint8Array): string {
288
+ if (typeof btoa === 'function') {
289
+ let binary = '';
290
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
291
+ return btoa(binary);
292
+ }
293
+ return Buffer.from(bytes).toString('base64');
294
+ }