@spooky-sync/core 0.0.1-canary.22 → 0.0.1-canary.221

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2615 -58
  3. package/dist/index.js +12714 -2408
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +141 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +138 -25
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +165 -86
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +228 -0
  65. package/src/modules/sync/queue/queue-down.ts +108 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1851 -98
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -16
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +192 -15
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1720 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -1,20 +1,67 @@
1
+ import type {
2
+ Diagnostic,
3
+ SurrealEvents} from 'surrealdb';
1
4
  import {
2
5
  applyDiagnostics,
3
6
  createRemoteEngines,
4
- Diagnostic,
5
7
  Surreal,
6
- SurrealTransaction,
7
8
  } from 'surrealdb';
8
- import { SpookyConfig } from '../../types';
9
- import { Logger } from '../logger/index';
9
+ import type { ConnectionState, ReconnectConfig, Sp00kyConfig } from '../../types';
10
+ import type { Logger } from '../logger/index';
10
11
  import { AbstractDatabaseService } from './database';
11
12
  import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
12
13
 
14
+ /** Defaults for {@link ReconnectConfig}. See that type for the rationale. */
15
+ export const RECONNECT_DEFAULTS = {
16
+ attempts: -1,
17
+ retryDelayMax: 15_000,
18
+ heartbeatIntervalMs: 20_000,
19
+ heartbeatTimeoutMs: 10_000,
20
+ superviseRetryDelayMaxMs: 15_000,
21
+ } as const;
22
+
23
+ /** Fill in {@link RECONNECT_DEFAULTS} for every field the caller omitted. */
24
+ export function resolveReconnectConfig(
25
+ input?: ReconnectConfig
26
+ ): Required<ReconnectConfig> {
27
+ return { ...RECONNECT_DEFAULTS, ...input };
28
+ }
29
+
30
+ /** Transport events the SDK publishes, mapped 1:1 to {@link ConnectionState}. */
31
+ export type RemoteConnectionEvent = ConnectionState | 'error';
32
+
13
33
  export class RemoteDatabaseService extends AbstractDatabaseService {
14
- private config: SpookyConfig<any>['database'];
34
+ private config: Sp00kyConfig<any>['database'];
15
35
  protected eventType = DatabaseEventTypes.RemoteQuery;
36
+ private readonly reconnectConfig: Required<ReconnectConfig>;
37
+ /**
38
+ * In-flight `connect()`, so concurrent callers (boot + supervisor revive +
39
+ * an `online` event landing at the same moment) share one attempt instead of
40
+ * racing two sockets. Cleared on settle, so a later call always reconnects.
41
+ */
42
+ private connecting: Promise<void> | null = null;
43
+ /**
44
+ * The token to re-authenticate a freshly opened socket with, kept current by
45
+ * {@link setAuthToken}.
46
+ *
47
+ * `config.token` is fixed at construction and most apps never set it: they
48
+ * sign in later, which authenticates the socket that happens to be open at
49
+ * the time. That is enough for the SDK's OWN reconnects (it replays
50
+ * `version`/`use`/`authenticate` itself), but not for the supervisor's revive
51
+ * loop, which builds a socket from scratch. Without this the revived socket
52
+ * came back UNAUTHENTICATED and stayed that way for the life of the page,
53
+ * while the client's `currentUserId` — restored from local storage — kept
54
+ * reporting the user as signed in.
55
+ *
56
+ * The visible damage is silent and total: `fn::query::register` sends
57
+ * `<string>($auth.id OR '')`, so every view registered afterwards is stamped
58
+ * with an empty identity, and every `$auth.id` predicate in it resolves
59
+ * false. Public tables keep returning rows while everything owned by the user
60
+ * returns nothing, on a page that still looks signed in.
61
+ */
62
+ private authToken: string | null = null;
16
63
 
17
- constructor(config: SpookyConfig<any>['database'], logger: Logger) {
64
+ constructor(config: Sp00kyConfig<any>['database'], logger: Logger) {
18
65
  const events = createDatabaseEventSystem();
19
66
  super(
20
67
  new Surreal({
@@ -29,7 +76,7 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
29
76
  type,
30
77
  phase,
31
78
  service: 'surrealdb:remote',
32
- Category: 'spooky-client::RemoteDatabaseService::diagnostics',
79
+ Category: 'sp00ky-client::RemoteDatabaseService::diagnostics',
33
80
  },
34
81
  `Remote SurrealDB diagnostics captured ${type}:${phase}`
35
82
  );
@@ -41,26 +88,156 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
41
88
  events
42
89
  );
43
90
  this.config = config;
91
+ this.reconnectConfig = resolveReconnectConfig(config.reconnect);
92
+ this.queryTimeoutMs = Math.max(0, config.queryTimeoutMs ?? 60_000);
44
93
  }
45
94
 
46
- getConfig(): SpookyConfig<any>['database'] {
95
+ getConfig(): Sp00kyConfig<any>['database'] {
47
96
  return this.config;
48
97
  }
49
98
 
99
+ /**
100
+ * Send one SurrealQL statement so that it survives the page going away.
101
+ *
102
+ * A WebSocket `send()` during `pagehide` is not guaranteed to flush — the
103
+ * browser may tear the socket down first, and the frame is simply lost.
104
+ * Measured: an unload-time release over the live socket reached the server
105
+ * zero times out of one. `fetch` with `keepalive` is the primitive the
106
+ * platform actually guarantees here, so this goes over SurrealDB's HTTP
107
+ * `/sql` endpoint instead of the RPC socket.
108
+ *
109
+ * Best-effort by design: no await, no retry, errors swallowed. Every caller
110
+ * must have a server-side fallback that makes a lost beacon a non-event.
111
+ */
112
+ beaconSql(sql: string): void {
113
+ try {
114
+ const { endpoint, namespace, database } = this.getConfig();
115
+ if (!endpoint || typeof fetch !== 'function') return;
116
+
117
+ // `ws(s)://host/rpc` is the socket; `http(s)://host/sql` is the same
118
+ // server's statement endpoint.
119
+ const url = new URL(endpoint);
120
+ url.protocol = url.protocol === 'wss:' ? 'https:' : url.protocol === 'ws:' ? 'http:' : url.protocol;
121
+ url.pathname = url.pathname.replace(/\/rpc\/?$/, '') + '/sql';
122
+
123
+ const headers: Record<string, string> = {
124
+ Accept: 'application/json',
125
+ 'Content-Type': 'text/plain',
126
+ };
127
+ if (namespace) headers['surreal-ns'] = namespace;
128
+ if (database) headers['surreal-db'] = database;
129
+ // Without this the statement runs unauthenticated and `$auth.id` is NONE,
130
+ // which for a per-user release means it matches nothing.
131
+ if (this.authToken) headers.Authorization = `Bearer ${this.authToken}`;
132
+
133
+ void fetch(url.toString(), {
134
+ method: 'POST',
135
+ headers,
136
+ body: sql,
137
+ keepalive: true,
138
+ credentials: 'omit',
139
+ }).catch(() => {});
140
+ } catch {
141
+ // Malformed endpoint, no fetch, blocked by CSP: the caller's fallback owns it.
142
+ }
143
+ }
144
+
145
+ /**
146
+ * Record the token every future connect should authenticate with, or `null`
147
+ * on sign-out. See {@link authToken}.
148
+ *
149
+ * Does not touch the CURRENT socket: callers authenticate that themselves
150
+ * (sign-in and session restore both already do). This only makes the next
151
+ * from-scratch connect reproduce that state.
152
+ */
153
+ setAuthToken(token: string | null): void {
154
+ this.authToken = token;
155
+ }
156
+
157
+ /** Resolved reconnect tunables; the supervisor reads its own knobs here. */
158
+ getReconnectConfig(): Required<ReconnectConfig> {
159
+ return this.reconnectConfig;
160
+ }
161
+
162
+ /** Current transport state as reported by the SDK. */
163
+ getStatus(): ConnectionState {
164
+ return this.client.status as ConnectionState;
165
+ }
166
+
167
+ /**
168
+ * Observe transport events. Thin passthrough so callers (the supervisor,
169
+ * sync, CRDT) don't have to reach through `getClient()`.
170
+ */
171
+ subscribeConnection<K extends RemoteConnectionEvent>(
172
+ event: K,
173
+ cb: (...payload: SurrealEvents[K]) => void
174
+ ): () => void {
175
+ return this.client.subscribe(event, cb);
176
+ }
177
+
178
+ /**
179
+ * Tear the socket down on purpose. Used by the heartbeat watchdog when a
180
+ * socket stops answering but never closes: `close()` makes the SDK publish
181
+ * `disconnected`, which is what drives the supervisor's revive loop.
182
+ */
183
+ async forceClose(): Promise<void> {
184
+ try {
185
+ await this.client.close();
186
+ } catch (err) {
187
+ this.logger.debug(
188
+ { err, Category: 'sp00ky-client::RemoteDatabaseService::forceClose' },
189
+ 'forceClose failed; treating the socket as gone anyway'
190
+ );
191
+ }
192
+ }
193
+
194
+ /**
195
+ * Open (or re-open) the remote connection.
196
+ *
197
+ * Safe to call repeatedly: concurrent calls share the in-flight attempt, and
198
+ * a call after a `disconnected` builds a fresh socket. `use()` and
199
+ * `authenticate()` are re-applied here for the cold path; the SDK also
200
+ * replays them itself on its own internal reconnects.
201
+ */
50
202
  async connect(): Promise<void> {
51
- const { endpoint, token, namespace, database } = this.getConfig();
203
+ if (this.connecting) return this.connecting;
204
+ this.connecting = this.doConnect().finally(() => {
205
+ this.connecting = null;
206
+ });
207
+ return this.connecting;
208
+ }
209
+
210
+ private async doConnect(): Promise<void> {
211
+ const { endpoint, namespace, database } = this.getConfig();
212
+ // The live token wins over the constructor-time one: this connect may be
213
+ // the supervisor rebuilding a socket long after sign-in. See `authToken`.
214
+ const token = this.authToken ?? this.getConfig().token;
52
215
  if (endpoint) {
53
216
  this.logger.info(
54
217
  {
55
218
  endpoint,
56
219
  namespace,
57
220
  database,
58
- Category: 'spooky-client::RemoteDatabaseService::connect',
221
+ Category: 'sp00ky-client::RemoteDatabaseService::connect',
59
222
  },
60
223
  'Connecting to remote database'
61
224
  );
62
225
  try {
63
- await this.client.connect(endpoint);
226
+ // Without explicit options the SDK caps itself at 5 attempts (~62s of
227
+ // outage) and then gives up permanently — nothing would re-open the
228
+ // socket after that. `attempts: -1` keeps it trying; the supervisor
229
+ // still covers the case where the SDK terminates the engine because its
230
+ // post-reconnect handshake threw.
231
+ await this.client.connect(endpoint, {
232
+ reconnect: {
233
+ enabled: true,
234
+ attempts: this.reconnectConfig.attempts,
235
+ retryDelay: 1_000,
236
+ retryDelayMax: this.reconnectConfig.retryDelayMax,
237
+ retryDelayMultiplier: 2,
238
+ retryDelayJitter: 0.1,
239
+ },
240
+ });
64
241
  await this.client.use({
65
242
  namespace,
66
243
  database,
@@ -68,25 +245,25 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
68
245
 
69
246
  if (token) {
70
247
  this.logger.debug(
71
- { Category: 'spooky-client::RemoteDatabaseService::connect' },
248
+ { Category: 'sp00ky-client::RemoteDatabaseService::connect' },
72
249
  'Authenticating with token'
73
250
  );
74
251
  await this.client.authenticate(token);
75
252
  }
76
253
  this.logger.info(
77
- { Category: 'spooky-client::RemoteDatabaseService::connect' },
254
+ { Category: 'sp00ky-client::RemoteDatabaseService::connect' },
78
255
  'Connected to remote database'
79
256
  );
80
257
  } catch (err) {
81
258
  this.logger.error(
82
- { err, Category: 'spooky-client::RemoteDatabaseService::connect' },
259
+ { err, Category: 'sp00ky-client::RemoteDatabaseService::connect' },
83
260
  'Failed to connect to remote database'
84
261
  );
85
262
  throw err;
86
263
  }
87
264
  } else {
88
265
  this.logger.warn(
89
- { Category: 'spooky-client::RemoteDatabaseService::connect' },
266
+ { Category: 'sp00ky-client::RemoteDatabaseService::connect' },
90
267
  'No endpoint configured for remote database'
91
268
  );
92
269
  }