@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2514 -58
  3. package/dist/index.js +12561 -2449
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +101 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +127 -24
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +164 -82
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +180 -0
  65. package/src/modules/sync/queue/queue-down.ts +80 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1726 -99
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -17
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +110 -14
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1693 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -0,0 +1,249 @@
1
+ import { describe, it, expect, vi, beforeEach } from 'vitest';
2
+ import { RecordId } from 'surrealdb';
3
+ import { Sp00kySync } from './sync';
4
+ import type { IngestTuple } from '../../services/tabs/protocol';
5
+
6
+ // Shared-tabs sync roles. A follower's optimistic write used to enter only
7
+ // its own circuit: the leader and every other follower learned of it after the
8
+ // server round-trip, through a LIVE event SurrealDB v3 sometimes drops. Now the
9
+ // follower posts the ingested tuples to the leader, which feeds its own circuit
10
+ // and fans them out (excluding the origin), and the leader tells every follower
11
+ // when a push settled so the row does not blink out of the render set.
12
+
13
+ function makeLogger(): any {
14
+ const logger: any = {
15
+ child: () => logger,
16
+ debug: () => {},
17
+ info: () => {},
18
+ warn: () => {},
19
+ error: () => {},
20
+ trace: () => {},
21
+ };
22
+ return logger;
23
+ }
24
+
25
+ const tuple = (op: IngestTuple['op'], id = 'thread:c'): IngestTuple => ({
26
+ table: 'thread',
27
+ op,
28
+ id,
29
+ record: { id, _00_rv: 1 },
30
+ });
31
+
32
+ function makeSync() {
33
+ const queryId = new RecordId('_00_query', 'h1');
34
+ const queryState: any = {
35
+ config: {
36
+ id: queryId,
37
+ localArray: [['thread:a', 1]],
38
+ remoteArray: [['thread:a', 1]],
39
+ membershipKnown: true,
40
+ membershipKey: 'stable-key',
41
+ },
42
+ };
43
+ const updateQueryRemoteArray = vi.fn(async (_h: string, next: any) => {
44
+ queryState.config.remoteArray = next;
45
+ });
46
+ const dataModule: any = {
47
+ getQueryById: vi.fn((id: RecordId) => (String(id.id) === 'h1' ? queryState : undefined)),
48
+ getQueryByHash: vi.fn().mockReturnValue(queryState),
49
+ updateQueryRemoteArray,
50
+ notifyQuerySynced: vi.fn().mockResolvedValue(undefined),
51
+ notifyTableQueries: vi.fn().mockResolvedValue(undefined),
52
+ noteWriteSettled: vi.fn(),
53
+ getActiveQueryHashes: () => ['h1'],
54
+ getPendingRecordIds: async () => ({ writes: new Set(), deletes: new Set() }),
55
+ };
56
+ const cache: any = { applyRelayedIngest: vi.fn() };
57
+ const sync = new Sp00kySync(
58
+ {} as any,
59
+ { query: vi.fn() } as any,
60
+ cache,
61
+ dataModule,
62
+ {} as any,
63
+ makeLogger()
64
+ );
65
+ (sync as any).runSyncForQuery = vi.fn().mockResolvedValue(undefined);
66
+ (sync as any).upQueue.enqueueFromDatabase = vi.fn().mockResolvedValue(undefined);
67
+ (sync as any).scheduler.enqueueMutation = vi.fn();
68
+
69
+ const hub: any = {
70
+ onFollowerMessage: null,
71
+ relayIngest: vi.fn(),
72
+ broadcast: vi.fn(),
73
+ sendTo: vi.fn(),
74
+ };
75
+ const forwarder: any = {
76
+ onLeaderMessage: null,
77
+ mutationEnqueued: vi.fn(),
78
+ ingest: vi.fn(),
79
+ };
80
+ return { sync, cache, dataModule, hub, forwarder, queryState, updateQueryRemoteArray };
81
+ }
82
+
83
+ describe('shared-tabs leader', () => {
84
+ beforeEach(() => vi.clearAllMocks());
85
+
86
+ it('applies a follower ingest to its own circuit and relays it to the other followers', () => {
87
+ const { sync, cache, hub } = makeSync();
88
+ sync.promoteToLeader(hub);
89
+ const tuples = [tuple('CREATE')];
90
+
91
+ hub.onFollowerMessage('tab-2', { type: 'ingest', tuples });
92
+
93
+ expect(cache.applyRelayedIngest).toHaveBeenCalledWith(tuples);
94
+ expect(hub.relayIngest).toHaveBeenCalledWith(tuples, 'tab-2');
95
+ });
96
+
97
+ it('re-materializes the table queries for a relayed DELETE', async () => {
98
+ const { sync, dataModule, hub } = makeSync();
99
+ sync.promoteToLeader(hub);
100
+
101
+ hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('DELETE')] });
102
+ await Promise.resolve();
103
+
104
+ expect(dataModule.notifyTableQueries).toHaveBeenCalledWith('thread');
105
+ });
106
+
107
+ it('does not re-materialize for a relayed CREATE/UPDATE (the stream update covers it)', async () => {
108
+ const { sync, dataModule, hub } = makeSync();
109
+ sync.promoteToLeader(hub);
110
+
111
+ hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('UPDATE')] });
112
+ await Promise.resolve();
113
+
114
+ expect(dataModule.notifyTableQueries).not.toHaveBeenCalled();
115
+ });
116
+
117
+ it('treats a follower write as activity for the poll backoff', () => {
118
+ const { sync, hub } = makeSync();
119
+ sync.promoteToLeader(hub);
120
+
121
+ (sync as any).listRefIdleStreak = 7;
122
+ hub.onFollowerMessage('tab-2', { type: 'ingest', tuples: [tuple('CREATE')] });
123
+ expect((sync as any).listRefIdleStreak).toBe(0);
124
+
125
+ (sync as any).listRefIdleStreak = 7;
126
+ hub.onFollowerMessage('tab-2', {
127
+ type: 'mutation-enqueued',
128
+ mutationId: '_00_pending_mutations:1_0001_tab-2',
129
+ });
130
+ expect((sync as any).listRefIdleStreak).toBe(0);
131
+ expect((sync as any).upQueue.enqueueFromDatabase).toHaveBeenCalledWith(
132
+ '_00_pending_mutations:1_0001_tab-2'
133
+ );
134
+ });
135
+
136
+ it('notes a settled mutation locally and broadcasts it to every follower', () => {
137
+ const { sync, dataModule, hub } = makeSync();
138
+ sync.promoteToLeader(hub);
139
+
140
+ (sync as any).handleMutationSettled({
141
+ type: 'create',
142
+ mutation_id: new RecordId('_00_pending_mutations', '1_0001_tab-2'),
143
+ record_id: new RecordId('thread', 'c'),
144
+ });
145
+
146
+ expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'create');
147
+ expect(hub.broadcast).toHaveBeenCalledWith({
148
+ type: 'mutation-settled',
149
+ mutationId: '_00_pending_mutations:1_0001_tab-2',
150
+ recordId: 'thread:c',
151
+ eventType: 'create',
152
+ });
153
+ });
154
+
155
+ it('solo: a settled mutation is noted locally with no hub to broadcast on', () => {
156
+ const { sync, dataModule } = makeSync();
157
+ (sync as any).handleMutationSettled({
158
+ type: 'update',
159
+ mutation_id: new RecordId('_00_pending_mutations', '1_0001_solo'),
160
+ record_id: new RecordId('thread', 'c'),
161
+ });
162
+ expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'update');
163
+ });
164
+ });
165
+
166
+ describe('shared-tabs follower', () => {
167
+ beforeEach(() => vi.clearAllMocks());
168
+
169
+ it('starts the local settled-write grace on mutation-settled', () => {
170
+ const { sync, dataModule, forwarder } = makeSync();
171
+ sync.demoteToFollower(forwarder);
172
+
173
+ forwarder.onLeaderMessage({
174
+ type: 'mutation-settled',
175
+ mutationId: '_00_pending_mutations:1_0001_tab-1',
176
+ recordId: 'thread:c',
177
+ eventType: 'delete',
178
+ });
179
+
180
+ expect(dataModule.noteWriteSettled).toHaveBeenCalledWith('thread:c', 'delete');
181
+ });
182
+
183
+ it('feeds a relayed ingest to the local circuit', async () => {
184
+ const { sync, cache, dataModule, forwarder } = makeSync();
185
+ sync.demoteToFollower(forwarder);
186
+ const tuples = [tuple('DELETE')];
187
+
188
+ forwarder.onLeaderMessage({ type: 'ingest-relay', tuples, leadershipId: 1, seq: 1 });
189
+ await Promise.resolve();
190
+
191
+ expect(cache.applyRelayedIngest).toHaveBeenCalledWith(tuples);
192
+ expect(dataModule.notifyTableQueries).toHaveBeenCalledWith('thread');
193
+ });
194
+
195
+ it('applies a relayed list_ref change for one of its own queries', async () => {
196
+ const { sync, forwarder, updateQueryRemoteArray } = makeSync();
197
+ sync.demoteToFollower(forwarder);
198
+
199
+ forwarder.onLeaderMessage({
200
+ type: 'list-ref-change',
201
+ action: 'CREATE',
202
+ queryId: '_00_query:h1',
203
+ recordId: 'thread:c',
204
+ version: 1,
205
+ parent: false,
206
+ });
207
+ await new Promise((r) => setTimeout(r, 0));
208
+
209
+ expect(updateQueryRemoteArray).toHaveBeenCalledWith('h1', [
210
+ ['thread:a', 1],
211
+ ['thread:c', 1],
212
+ ]);
213
+ expect((sync as any).runSyncForQuery).toHaveBeenCalled();
214
+ });
215
+
216
+ it('ignores a relayed list_ref change for another tab query', async () => {
217
+ const { sync, forwarder, updateQueryRemoteArray } = makeSync();
218
+ sync.demoteToFollower(forwarder);
219
+
220
+ forwarder.onLeaderMessage({
221
+ type: 'list-ref-change',
222
+ action: 'CREATE',
223
+ queryId: '_00_query:foreign',
224
+ recordId: 'thread:c',
225
+ version: 1,
226
+ parent: false,
227
+ });
228
+ await new Promise((r) => setTimeout(r, 0));
229
+
230
+ expect(updateQueryRemoteArray).not.toHaveBeenCalled();
231
+ expect((sync as any).runSyncForQuery).not.toHaveBeenCalled();
232
+ });
233
+
234
+ it('forwards mutation ids to the leader instead of queueing locally', async () => {
235
+ const { sync, forwarder } = makeSync();
236
+ sync.demoteToFollower(forwarder);
237
+
238
+ await sync.enqueueMutation([
239
+ {
240
+ type: 'create',
241
+ mutation_id: new RecordId('_00_pending_mutations', '1_0001_tab-2'),
242
+ record_id: new RecordId('thread', 'c'),
243
+ } as any,
244
+ ]);
245
+
246
+ expect(forwarder.mutationEnqueued).toHaveBeenCalledWith('_00_pending_mutations:1_0001_tab-2');
247
+ expect((sync as any).scheduler.enqueueMutation).not.toHaveBeenCalled();
248
+ });
249
+ });