@spooky-sync/core 0.0.1-canary.7 → 0.0.1-canary.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/AGENTS.md +56 -0
  2. package/dist/index.d.ts +299 -369
  3. package/dist/index.js +2278 -399
  4. package/dist/otel/index.d.ts +21 -0
  5. package/dist/otel/index.js +86 -0
  6. package/dist/types.d.ts +460 -0
  7. package/package.json +37 -7
  8. package/skills/sp00ky-core/SKILL.md +258 -0
  9. package/skills/sp00ky-core/references/auth.md +98 -0
  10. package/skills/sp00ky-core/references/config.md +76 -0
  11. package/src/build-globals.d.ts +12 -0
  12. package/src/events/events.test.ts +2 -1
  13. package/src/events/index.ts +3 -0
  14. package/src/index.ts +3 -2
  15. package/src/modules/auth/events/index.ts +2 -1
  16. package/src/modules/auth/index.ts +17 -20
  17. package/src/modules/cache/index.ts +41 -29
  18. package/src/modules/cache/types.ts +2 -2
  19. package/src/modules/crdt/crdt-field.ts +281 -0
  20. package/src/modules/crdt/crdt-hydration.test.ts +206 -0
  21. package/src/modules/crdt/index.ts +352 -0
  22. package/src/modules/data/data.status.test.ts +108 -0
  23. package/src/modules/data/index.ts +662 -108
  24. package/src/modules/data/window-query.test.ts +52 -0
  25. package/src/modules/data/window-query.ts +130 -0
  26. package/src/modules/devtools/index.ts +89 -21
  27. package/src/modules/devtools/versions.test.ts +74 -0
  28. package/src/modules/devtools/versions.ts +81 -0
  29. package/src/modules/ref-tables.test.ts +56 -0
  30. package/src/modules/ref-tables.ts +57 -0
  31. package/src/modules/sync/engine.ts +97 -37
  32. package/src/modules/sync/events/index.ts +3 -2
  33. package/src/modules/sync/queue/queue-down.ts +5 -4
  34. package/src/modules/sync/queue/queue-up.ts +14 -13
  35. package/src/modules/sync/scheduler.ts +2 -2
  36. package/src/modules/sync/sync.ts +553 -58
  37. package/src/modules/sync/utils.test.ts +239 -2
  38. package/src/modules/sync/utils.ts +166 -17
  39. package/src/otel/index.ts +127 -0
  40. package/src/services/database/database.ts +11 -11
  41. package/src/services/database/events/index.ts +2 -1
  42. package/src/services/database/local-migrator.ts +21 -21
  43. package/src/services/database/local.test.ts +33 -0
  44. package/src/services/database/local.ts +192 -32
  45. package/src/services/database/remote.ts +13 -13
  46. package/src/services/logger/index.ts +6 -101
  47. package/src/services/persistence/localstorage.ts +2 -2
  48. package/src/services/persistence/resilient.ts +41 -0
  49. package/src/services/persistence/surrealdb.ts +9 -9
  50. package/src/services/stream-processor/index.ts +205 -36
  51. package/src/services/stream-processor/permissions.test.ts +47 -0
  52. package/src/services/stream-processor/permissions.ts +53 -0
  53. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  54. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  55. package/src/services/stream-processor/wasm-types.ts +18 -2
  56. package/src/sp00ky.auth-order.test.ts +65 -0
  57. package/src/sp00ky.ts +582 -0
  58. package/src/types.ts +132 -13
  59. package/src/utils/index.ts +35 -13
  60. package/src/utils/parser.ts +3 -2
  61. package/src/utils/surql.ts +24 -15
  62. package/src/utils/withRetry.test.ts +1 -1
  63. package/tsdown.config.ts +55 -1
  64. package/src/spooky.ts +0 -392
@@ -5,8 +5,15 @@ import {
5
5
  applyRecordVersionDiff,
6
6
  createDiffFromDbOp,
7
7
  ArraySyncer,
8
+ resolveListRefPollInterval,
9
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS,
10
+ buildListRefSelect,
11
+ nextPollDelayMs,
12
+ listRefPollDelayMs,
13
+ LIST_REF_POLL_MAX_INTERVAL_MS,
14
+ recordVersionArraysEqual,
8
15
  } from './utils';
9
- import { RecordVersionArray, RecordVersionDiff } from '../../types';
16
+ import type { RecordVersionArray, RecordVersionDiff } from '../../types';
10
17
  import { encodeRecordId } from '../../utils/index';
11
18
 
12
19
  function rid(table: string, id: string): RecordId<string> {
@@ -305,7 +312,237 @@ describe('ArraySyncer', () => {
305
312
  expect(diff!.removed).toHaveLength(3);
306
313
  // Check they come in sorted order
307
314
  const removedIds = diff!.removed.map((r) => encodeRecordId(r));
308
- const sorted = [...removedIds].sort();
315
+ const sorted = [...removedIds].toSorted();
309
316
  expect(removedIds).toEqual(sorted);
310
317
  });
311
318
  });
319
+
320
+ describe('resolveListRefPollInterval', () => {
321
+ it('returns the default when no option is provided', () => {
322
+ expect(resolveListRefPollInterval()).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
323
+ expect(resolveListRefPollInterval(undefined)).toBe(
324
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
325
+ );
326
+ });
327
+
328
+ it('respects a positive override', () => {
329
+ expect(resolveListRefPollInterval(1000)).toBe(1000);
330
+ expect(resolveListRefPollInterval(5000)).toBe(5000);
331
+ });
332
+
333
+ it('falls back to default for zero, negative, NaN, or Infinity', () => {
334
+ // Accepting these would either silently disable polling or busy-loop
335
+ // the event loop, so they fall back to the default.
336
+ expect(resolveListRefPollInterval(0)).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
337
+ expect(resolveListRefPollInterval(-500)).toBe(
338
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
339
+ );
340
+ expect(resolveListRefPollInterval(NaN)).toBe(
341
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
342
+ );
343
+ expect(resolveListRefPollInterval(Infinity)).toBe(
344
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
345
+ );
346
+ });
347
+
348
+ it('locks the default at 500ms so accidental tuning trips this test', () => {
349
+ expect(DEFAULT_LIST_REF_POLL_INTERVAL_MS).toBe(500);
350
+ });
351
+ });
352
+
353
+ describe('buildListRefSelect', () => {
354
+ it('substitutes the table name', () => {
355
+ expect(buildListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
356
+ expect(buildListRefSelect('_00_list_ref_user_abc')).toContain(
357
+ 'FROM _00_list_ref_user_abc '
358
+ );
359
+ });
360
+
361
+ it('filters by the bound query id', () => {
362
+ expect(buildListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
363
+ });
364
+
365
+ it('excludes subquery rows via parent IS NONE', () => {
366
+ // Without this predicate the poll would surface child records (the
367
+ // subquery-projection rows the SSP also writes into list_ref) as
368
+ // spurious "added" diffs against `localArray` every tick.
369
+ expect(buildListRefSelect('_00_list_ref')).toContain('parent IS NONE');
370
+ });
371
+
372
+ it('selects only `out` and `version`', () => {
373
+ // The diff only needs the record id and its version; pulling more
374
+ // fields would bloat the per-tick query traffic.
375
+ const sql = buildListRefSelect('_00_list_ref');
376
+ expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
377
+ });
378
+ });
379
+
380
+ describe('nextPollDelayMs', () => {
381
+ it('returns the base interval when LIVE has never delivered', () => {
382
+ expect(
383
+ nextPollDelayMs({
384
+ now: 10_000,
385
+ lastLiveEventAt: null,
386
+ baseIntervalMs: 500,
387
+ })
388
+ ).toBe(500);
389
+ });
390
+
391
+ it('returns the healthy interval when a LIVE event fired within the cooldown', () => {
392
+ expect(
393
+ nextPollDelayMs({
394
+ now: 10_000,
395
+ lastLiveEventAt: 8_000, // 2s ago, well within 5s cooldown
396
+ baseIntervalMs: 500,
397
+ })
398
+ ).toBe(5_000);
399
+ });
400
+
401
+ it('snaps back to the base interval after the cooldown elapses', () => {
402
+ expect(
403
+ nextPollDelayMs({
404
+ now: 20_000,
405
+ lastLiveEventAt: 10_000, // 10s ago, well past cooldown
406
+ baseIntervalMs: 500,
407
+ })
408
+ ).toBe(500);
409
+ });
410
+
411
+ it('treats the cooldown boundary as expired (>= cooldownMs returns base)', () => {
412
+ // Exactly at the boundary, the LIVE feed is considered cold so we
413
+ // resume the aggressive cadence. Picks the same direction as
414
+ // `if (sinceLive >= cooldownMs)`.
415
+ expect(
416
+ nextPollDelayMs({
417
+ now: 15_000,
418
+ lastLiveEventAt: 10_000, // exactly 5s ago
419
+ baseIntervalMs: 500,
420
+ cooldownMs: 5_000,
421
+ })
422
+ ).toBe(500);
423
+ });
424
+
425
+ it('clamps a clock-skew negative sinceLive to the base interval', () => {
426
+ // If a stale timestamp puts the LIVE event in the future, we
427
+ // should not interpret that as "LIVE healthy" — treat it as
428
+ // unknown and use the conservative aggressive cadence.
429
+ expect(
430
+ nextPollDelayMs({
431
+ now: 10_000,
432
+ lastLiveEventAt: 20_000, // future timestamp
433
+ baseIntervalMs: 500,
434
+ })
435
+ ).toBe(500);
436
+ });
437
+
438
+ it('never widens below an aggressively-configured base interval', () => {
439
+ // If the caller picked a faster base (e.g. 100ms) on purpose,
440
+ // the healthy path should not silently slow things back to 5s.
441
+ // The healthy interval is clamped to at least base.
442
+ expect(
443
+ nextPollDelayMs({
444
+ now: 10_000,
445
+ lastLiveEventAt: 9_500,
446
+ baseIntervalMs: 8_000,
447
+ })
448
+ ).toBe(8_000);
449
+ });
450
+
451
+ it('respects custom cooldown and healthy-interval values', () => {
452
+ expect(
453
+ nextPollDelayMs({
454
+ now: 10_000,
455
+ lastLiveEventAt: 9_500, // 0.5s ago
456
+ baseIntervalMs: 250,
457
+ cooldownMs: 1_000,
458
+ healthyIntervalMs: 2_000,
459
+ })
460
+ ).toBe(2_000);
461
+ expect(
462
+ nextPollDelayMs({
463
+ now: 10_000,
464
+ lastLiveEventAt: 8_500, // 1.5s ago > cooldown
465
+ baseIntervalMs: 250,
466
+ cooldownMs: 1_000,
467
+ healthyIntervalMs: 2_000,
468
+ })
469
+ ).toBe(250);
470
+ });
471
+ });
472
+
473
+ describe('listRefPollDelayMs', () => {
474
+ it('returns the base interval at idle streak 0 (something just happened)', () => {
475
+ expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 500 })).toBe(500);
476
+ // Negative streak is defensively treated as "active" too.
477
+ expect(listRefPollDelayMs({ idleStreak: -3, baseIntervalMs: 500 })).toBe(500);
478
+ });
479
+
480
+ it('doubles per idle streak (exponential backoff)', () => {
481
+ expect(listRefPollDelayMs({ idleStreak: 1, baseIntervalMs: 500 })).toBe(1_000);
482
+ expect(listRefPollDelayMs({ idleStreak: 2, baseIntervalMs: 500 })).toBe(2_000);
483
+ expect(listRefPollDelayMs({ idleStreak: 3, baseIntervalMs: 500 })).toBe(4_000);
484
+ });
485
+
486
+ it('caps at LIST_REF_POLL_MAX_INTERVAL_MS (5s) once the doubling exceeds it', () => {
487
+ // streak 4 would be 8000ms uncapped → clamped to 5000.
488
+ expect(listRefPollDelayMs({ idleStreak: 4, baseIntervalMs: 500 })).toBe(5_000);
489
+ expect(listRefPollDelayMs({ idleStreak: 50, baseIntervalMs: 500 })).toBe(5_000);
490
+ expect(LIST_REF_POLL_MAX_INTERVAL_MS).toBe(5_000);
491
+ });
492
+
493
+ it('never returns below the configured base, even when base exceeds the cap', () => {
494
+ // An aggressively-large base must not be implicitly shrunk by the cap.
495
+ expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 8_000 })).toBe(8_000);
496
+ expect(listRefPollDelayMs({ idleStreak: 5, baseIntervalMs: 8_000 })).toBe(8_000);
497
+ });
498
+
499
+ it('respects a custom max interval', () => {
500
+ expect(
501
+ listRefPollDelayMs({ idleStreak: 10, baseIntervalMs: 500, maxIntervalMs: 3_000 })
502
+ ).toBe(3_000);
503
+ });
504
+
505
+ it('does not overflow for a very long idle streak', () => {
506
+ // 2^1000 would be Infinity; the exponent clamp keeps it finite and capped.
507
+ expect(listRefPollDelayMs({ idleStreak: 1_000, baseIntervalMs: 500 })).toBe(5_000);
508
+ });
509
+ });
510
+
511
+ describe('recordVersionArraysEqual', () => {
512
+ it('treats the same reference and identical contents as equal', () => {
513
+ const a: RecordVersionArray = [['game:1', 1], ['game:2', 3]];
514
+ expect(recordVersionArraysEqual(a, a)).toBe(true);
515
+ expect(recordVersionArraysEqual(a, [['game:1', 1], ['game:2', 3]])).toBe(true);
516
+ });
517
+
518
+ it('is order-insensitive (the list_ref SELECT has no ORDER BY)', () => {
519
+ expect(
520
+ recordVersionArraysEqual(
521
+ [['game:1', 1], ['game:2', 3]],
522
+ [['game:2', 3], ['game:1', 1]]
523
+ )
524
+ ).toBe(true);
525
+ });
526
+
527
+ it('returns false when a version differs for the same id', () => {
528
+ expect(
529
+ recordVersionArraysEqual([['game:1', 1]], [['game:1', 2]])
530
+ ).toBe(false);
531
+ });
532
+
533
+ it('returns false on length mismatch', () => {
534
+ expect(
535
+ recordVersionArraysEqual([['game:1', 1]], [['game:1', 1], ['game:2', 1]])
536
+ ).toBe(false);
537
+ });
538
+
539
+ it('returns false when an id is present in one but not the other', () => {
540
+ expect(
541
+ recordVersionArraysEqual([['game:1', 1]], [['game:2', 1]])
542
+ ).toBe(false);
543
+ });
544
+
545
+ it('treats two empty arrays as equal', () => {
546
+ expect(recordVersionArraysEqual([], [])).toBe(true);
547
+ });
548
+ });
@@ -1,5 +1,5 @@
1
- import { RecordId } from 'surrealdb';
2
- import { RecordVersionArray, RecordVersionDiff } from '../../types';
1
+ import type { RecordId } from 'surrealdb';
2
+ import type { RecordVersionArray, RecordVersionDiff } from '../../types';
3
3
  import { parseRecordIdString, encodeRecordId } from '../../utils/index';
4
4
 
5
5
  export class ArraySyncer {
@@ -8,8 +8,8 @@ export class ArraySyncer {
8
8
  private needsSort = false;
9
9
 
10
10
  constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
11
- this.remoteArray = remoteArray.sort((a, b) => a[0].localeCompare(b[0]));
12
- this.localArray = localArray.sort((a, b) => a[0].localeCompare(b[0]));
11
+ this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
12
+ this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
13
13
  }
14
14
 
15
15
  /**
@@ -49,7 +49,6 @@ export class ArraySyncer {
49
49
  this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
50
50
  this.needsSort = false;
51
51
  }
52
- console.log('xxxx555', this.localArray, this.remoteArray);
53
52
  const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
54
53
  return diff;
55
54
  }
@@ -93,10 +92,12 @@ export function diffRecordVersionArray(
93
92
  return {
94
93
  added: added.map((id) => ({
95
94
  id: parseRecordIdString(id),
95
+ // oxlint-disable-next-line no-non-null-assertion
96
96
  version: remoteMap.get(id)!,
97
97
  })),
98
98
  updated: updated.map((id) => ({
99
99
  id: parseRecordIdString(id),
100
+ // oxlint-disable-next-line no-non-null-assertion
100
101
  version: remoteMap.get(id)!,
101
102
  })),
102
103
  removed: removed.map(parseRecordIdString),
@@ -127,7 +128,7 @@ export function applyRecordVersionDiff(
127
128
  currentMap.set(encodeRecordId(item.id), item.version);
128
129
  }
129
130
 
130
- return Array.from(currentMap).sort((a, b) => a[0].localeCompare(b[0]));
131
+ return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
131
132
  }
132
133
 
133
134
  export function createDiffFromDbOp(
@@ -136,17 +137,14 @@ export function createDiffFromDbOp(
136
137
  version: number,
137
138
  versions?: RecordVersionArray
138
139
  ): RecordVersionDiff {
139
- // Version guard: skip stale CREATE/UPDATE, but always process DELETE
140
- if (op !== 'DELETE') {
141
- const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
142
-
143
- if (old && old[1] >= version) {
144
- return {
145
- added: [],
146
- updated: [],
147
- removed: [],
148
- };
149
- }
140
+ const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
141
+
142
+ if (old && old[1] >= version) {
143
+ return {
144
+ added: [],
145
+ updated: [],
146
+ removed: [],
147
+ };
150
148
  }
151
149
 
152
150
  if (op === 'CREATE') {
@@ -169,3 +167,154 @@ export function createDiffFromDbOp(
169
167
  };
170
168
  }
171
169
  }
170
+
171
+ /**
172
+ * Default cadence for the `_00_list_ref_user_<id>` poll fallback. The
173
+ * poll is the safety net for SurrealDB v3's occasionally-dropped LIVE
174
+ * deliveries; 500ms is aggressive enough to feel real-time on the
175
+ * happy path while keeping the per-session query load bounded.
176
+ */
177
+ export const DEFAULT_LIST_REF_POLL_INTERVAL_MS = 500;
178
+
179
+ /**
180
+ * Build the SurrealQL select that powers both the initial-fetch and
181
+ * the periodic poll of `_00_list_ref[_user_<id>]`. The `parent IS NONE`
182
+ * predicate excludes subquery entries (rows with `parent_rel` set)
183
+ * because the client's `RecordVersionArray` only tracks primary rows;
184
+ * including subquery rows would surface them as spurious "added"
185
+ * diffs every tick.
186
+ */
187
+ export function buildListRefSelect(table: string): string {
188
+ return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
189
+ }
190
+
191
+ /**
192
+ * Resolve the effective list-ref poll interval. Negative or zero
193
+ * values fall back to the default — accepting them would either
194
+ * disable polling silently or busy-loop the event loop.
195
+ */
196
+ export function resolveListRefPollInterval(opt?: number): number {
197
+ if (typeof opt !== 'number' || !Number.isFinite(opt) || opt <= 0) {
198
+ return DEFAULT_LIST_REF_POLL_INTERVAL_MS;
199
+ }
200
+ return opt;
201
+ }
202
+
203
+ /**
204
+ * When the LIVE feed has delivered an event within this window, treat
205
+ * the LIVE subscription as healthy and back the poll off to
206
+ * {@link LIVE_HEALTHY_POLL_INTERVAL_MS}. As soon as LIVE quiets for
207
+ * longer than this, the poll snaps back to the aggressive default.
208
+ */
209
+ export const LIVE_HEALTHY_COOLDOWN_MS = 5_000;
210
+
211
+ /**
212
+ * Poll interval while LIVE is delivering events. The aggressive
213
+ * default 500ms costs ~120 queries / minute / session — wasted work
214
+ * when LIVE is already covering us. 5s keeps the safety net in place
215
+ * at 1/10th the load.
216
+ */
217
+ export const LIVE_HEALTHY_POLL_INTERVAL_MS = 5_000;
218
+
219
+ /**
220
+ * Pick the next poll delay based on whether LIVE has been healthy
221
+ * recently. If a LIVE event fired within `cooldownMs`, use the slow
222
+ * (`healthyIntervalMs`) cadence; otherwise the fast (`baseIntervalMs`)
223
+ * cadence. Pure so it's unit-testable; `Sp00kySync.startListRefPoll`
224
+ * calls it from a self-rescheduling timer.
225
+ *
226
+ * The healthy interval is clamped to at least `baseIntervalMs` so
227
+ * configuring an aggressive base (e.g. 100ms) never gets implicitly
228
+ * widened by this helper.
229
+ *
230
+ * @deprecated Superseded by {@link listRefPollDelayMs}, which backs the
231
+ * poll off based on observed change activity (LIVE *or* poll-detected)
232
+ * rather than LIVE liveness alone — the cross-session LIVE-permission gap
233
+ * means LIVE often never fires, so this helper would keep the poll pinned
234
+ * at the aggressive base forever even on a fully idle page. Kept (and
235
+ * tested) for reference.
236
+ */
237
+ export function nextPollDelayMs(args: {
238
+ now: number;
239
+ lastLiveEventAt: number | null;
240
+ baseIntervalMs: number;
241
+ cooldownMs?: number;
242
+ healthyIntervalMs?: number;
243
+ }): number {
244
+ const {
245
+ now,
246
+ lastLiveEventAt,
247
+ baseIntervalMs,
248
+ cooldownMs = LIVE_HEALTHY_COOLDOWN_MS,
249
+ healthyIntervalMs = LIVE_HEALTHY_POLL_INTERVAL_MS,
250
+ } = args;
251
+ if (lastLiveEventAt === null) return baseIntervalMs;
252
+ const sinceLive = now - lastLiveEventAt;
253
+ if (sinceLive < 0 || sinceLive >= cooldownMs) return baseIntervalMs;
254
+ return Math.max(healthyIntervalMs, baseIntervalMs);
255
+ }
256
+
257
+ /**
258
+ * Ceiling for the adaptive list_ref poll backoff. An idle page (no LIVE
259
+ * events, no poll-detected list_ref changes) coasts up to this cadence;
260
+ * the existing healthy-LIVE safety net runs at the same 5s, so this keeps
261
+ * the worst-case catch-up latency for a missed cross-session change at the
262
+ * cadence the codebase already treats as acceptable.
263
+ */
264
+ export const LIST_REF_POLL_MAX_INTERVAL_MS = 5_000;
265
+
266
+ /**
267
+ * Adaptive poll delay: stay at the responsive `baseIntervalMs` while
268
+ * changes are arriving, and exponentially back off toward `maxIntervalMs`
269
+ * while the `_00_list_ref` is quiet.
270
+ *
271
+ * `idleStreak` is the count of consecutive poll cycles that observed *no*
272
+ * change. `Sp00kySync` resets it to 0 whenever a poll detects a real
273
+ * remoteArray change OR a LIVE event lands, so any activity snaps the poll
274
+ * straight back to `baseIntervalMs`. A streak of 0 (something just
275
+ * happened) → base; otherwise `base * 2^streak` capped at `maxIntervalMs`.
276
+ *
277
+ * This replaces {@link nextPollDelayMs}: the old helper slowed the poll
278
+ * only while LIVE was *delivering*, but the cross-session LIVE-permission
279
+ * gap means LIVE frequently never fires here, so it left a fully idle page
280
+ * polling every `base` ms forever (the "continuous queries while idle"
281
+ * symptom). Backing off on observed idleness instead covers the
282
+ * LIVE-healthy case for free (LIVE applies the change → the next poll sees
283
+ * nothing new → the streak grows → it backs off).
284
+ */
285
+ export function listRefPollDelayMs(args: {
286
+ idleStreak: number;
287
+ baseIntervalMs: number;
288
+ maxIntervalMs?: number;
289
+ }): number {
290
+ const { idleStreak, baseIntervalMs, maxIntervalMs = LIST_REF_POLL_MAX_INTERVAL_MS } = args;
291
+ const cap = Math.max(baseIntervalMs, maxIntervalMs);
292
+ if (idleStreak <= 0) return baseIntervalMs;
293
+ // 2^streak grows fast; clamp the exponent so it can't overflow on a
294
+ // page left idle for a very long time.
295
+ const exponent = Math.min(idleStreak, 30);
296
+ return Math.min(baseIntervalMs * 2 ** exponent, cap);
297
+ }
298
+
299
+ /**
300
+ * Order-insensitive equality for two `RecordVersionArray`s (each a list of
301
+ * `[recordIdString, version]`). The `_00_list_ref` SELECT has no `ORDER
302
+ * BY`, so row order can differ between polls without anything having
303
+ * actually changed — comparing as an id→version map avoids false
304
+ * "changed" verdicts that would defeat the idle backoff. Record ids are
305
+ * unique within a query's list_ref, so a map is a faithful representation.
306
+ */
307
+ export function recordVersionArraysEqual(
308
+ a: RecordVersionArray,
309
+ b: RecordVersionArray
310
+ ): boolean {
311
+ if (a === b) return true;
312
+ if (a.length !== b.length) return false;
313
+ const byId = new Map<string, number>();
314
+ for (const [id, version] of a) byId.set(id, version);
315
+ for (const [id, version] of b) {
316
+ // `get` returns undefined for a missing id, which never === a number.
317
+ if (byId.get(id) !== version) return false;
318
+ }
319
+ return true;
320
+ }
@@ -0,0 +1,127 @@
1
+ import type { Level } from 'pino';
2
+ import type { PinoTransmit } from '../types';
3
+
4
+ // Map pino levels to OTEL severity numbers
5
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
6
+ function mapLevelToSeverityNumber(level: string): number {
7
+ switch (level) {
8
+ case 'trace':
9
+ return 1;
10
+ case 'debug':
11
+ return 5;
12
+ case 'info':
13
+ return 9;
14
+ case 'warn':
15
+ return 13;
16
+ case 'error':
17
+ return 17;
18
+ case 'fatal':
19
+ return 21;
20
+ default:
21
+ return 9;
22
+ }
23
+ }
24
+
25
+ async function loadOtelModules(otelEndpoint: string) {
26
+ const [{ LoggerProvider, BatchLogRecordProcessor }, { OTLPLogExporter }, { resourceFromAttributes }, { ATTR_SERVICE_NAME }] =
27
+ await Promise.all([
28
+ import('@opentelemetry/sdk-logs'),
29
+ import('@opentelemetry/exporter-logs-otlp-proto'),
30
+ import('@opentelemetry/resources'),
31
+ import('@opentelemetry/semantic-conventions'),
32
+ ]);
33
+
34
+ const resource = resourceFromAttributes({
35
+ [ATTR_SERVICE_NAME]: 'sp00ky-client',
36
+ });
37
+
38
+ const exporter = new OTLPLogExporter({
39
+ url: otelEndpoint,
40
+ });
41
+
42
+ const loggerProvider = new LoggerProvider({
43
+ resource,
44
+ processors: [new BatchLogRecordProcessor(exporter)],
45
+ });
46
+
47
+ const otelLoggerCache: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
48
+
49
+ return (category: string) => {
50
+ if (!otelLoggerCache[category]) {
51
+ otelLoggerCache[category] = loggerProvider.getLogger(category);
52
+ }
53
+ return otelLoggerCache[category];
54
+ };
55
+ }
56
+
57
+ /**
58
+ * Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
59
+ *
60
+ * @example
61
+ * ```ts
62
+ * import { createOtelTransmit } from '@spooky-sync/core/otel';
63
+ *
64
+ * new Sp00kyClient({
65
+ * // ...
66
+ * otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
67
+ * });
68
+ * ```
69
+ */
70
+ export function createOtelTransmit(endpoint: string, level: Level = 'info'): PinoTransmit {
71
+ // Start loading OTel modules eagerly (don't await — we're synchronous)
72
+ const otelReady = loadOtelModules(endpoint);
73
+
74
+ return {
75
+ level: level,
76
+ send: (levelLabel: string, logEvent: any) => {
77
+ // oxlint-disable-next-line promise/always-return
78
+ otelReady.then((getOtelLogger) => {
79
+ try {
80
+ const messages = [...logEvent.messages];
81
+ const severityNumber = mapLevelToSeverityNumber(levelLabel);
82
+
83
+ // Construct the message body
84
+ let body = '';
85
+ const lastMsg = messages.pop();
86
+
87
+ if (typeof lastMsg === 'string') {
88
+ body = lastMsg;
89
+ } else if (lastMsg) {
90
+ body = JSON.stringify(lastMsg);
91
+ }
92
+
93
+ let category = 'sp00ky-client::unknown';
94
+
95
+ const attributes = {};
96
+ for (const msg of messages) {
97
+ if (typeof msg === 'object') {
98
+ if (msg.Category) {
99
+ category = msg.Category;
100
+ delete msg.Category;
101
+ }
102
+ Object.assign(attributes, msg);
103
+ }
104
+ }
105
+
106
+ // Emit to OTEL SDK
107
+ getOtelLogger(category).emit({
108
+ severityNumber: severityNumber,
109
+ severityText: levelLabel.toUpperCase(),
110
+ body: body,
111
+ attributes: {
112
+ ...logEvent.bindings[0],
113
+ ...attributes,
114
+ },
115
+ timestamp: new Date(logEvent.ts),
116
+ });
117
+ } catch (e) {
118
+ // oxlint-disable-next-line no-console
119
+ console.warn('Failed to transmit log to OTEL endpoint', e);
120
+ }
121
+ }).catch((e) => {
122
+ // oxlint-disable-next-line no-console
123
+ console.warn('Failed to load OpenTelemetry modules', e);
124
+ });
125
+ },
126
+ };
127
+ }
@@ -1,11 +1,9 @@
1
- import { Surreal, SurrealTransaction } from 'surrealdb';
2
- import { createLogger, Logger } from '../logger/index';
3
- import {
1
+ import type { Surreal, SurrealTransaction } from 'surrealdb';
2
+ import type { Logger } from '../logger/index';
3
+ import type {
4
4
  DatabaseEventSystem,
5
- DatabaseEventTypes,
6
- DatabaseQueryEventPayload,
7
- } from './events/index';
8
- import { SealedQuery } from '../../utils/surql';
5
+ DatabaseEventTypes} from './events/index';
6
+ import type { SealedQuery } from '../../utils/surql';
9
7
 
10
8
  export abstract class AbstractDatabaseService {
11
9
  protected client: Surreal;
@@ -43,11 +41,12 @@ export abstract class AbstractDatabaseService {
43
41
  async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
44
42
  return new Promise((resolve, reject) => {
45
43
  this.queryQueue = this.queryQueue
44
+ // oxlint-disable-next-line promise/always-return
46
45
  .then(async () => {
47
46
  const startTime = performance.now();
48
47
  try {
49
48
  this.logger.debug(
50
- { query, vars, Category: 'spooky-client::Database::query' },
49
+ { query, vars, Category: 'sp00ky-client::Database::query' },
51
50
  'Executing query'
52
51
  );
53
52
  const pending = this.client.query(query, vars);
@@ -67,7 +66,7 @@ export abstract class AbstractDatabaseService {
67
66
 
68
67
  resolve(result);
69
68
  this.logger.trace(
70
- { query, result, Category: 'spooky-client::Database::query' },
69
+ { query, result, Category: 'sp00ky-client::Database::query' },
71
70
  'Query executed successfully'
72
71
  );
73
72
  } catch (err) {
@@ -84,9 +83,10 @@ export abstract class AbstractDatabaseService {
84
83
  });
85
84
 
86
85
  this.logger.error(
87
- { query, vars, err, Category: 'spooky-client::Database::query' },
86
+ { query, vars, err, Category: 'sp00ky-client::Database::query' },
88
87
  'Query execution failed'
89
88
  );
89
+ // oxlint-disable-next-line no-multiple-resolved -- resolve/reject are in try/catch, mutually exclusive
90
90
  reject(err);
91
91
  }
92
92
  })
@@ -102,7 +102,7 @@ export abstract class AbstractDatabaseService {
102
102
  }
103
103
 
104
104
  async close(): Promise<void> {
105
- this.logger.info({ Category: 'spooky-client::Database::close' }, 'Closing database connection');
105
+ this.logger.info({ Category: 'sp00ky-client::Database::close' }, 'Closing database connection');
106
106
  await this.client.close();
107
107
  }
108
108
  }
@@ -1,4 +1,5 @@
1
- import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
1
+ import type { EventDefinition, EventSystem } from '../../../events/index';
2
+ import { createEventSystem } from '../../../events/index';
2
3
 
3
4
  export const DatabaseEventTypes = {
4
5
  LocalQuery: 'DATABASE_LOCAL_QUERY',