@spooky-sync/core 0.0.1-canary.9 → 0.0.1-canary.91

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/AGENTS.md +56 -0
  2. package/dist/index.d.ts +907 -339
  3. package/dist/index.js +2837 -402
  4. package/dist/otel/index.d.ts +21 -0
  5. package/dist/otel/index.js +86 -0
  6. package/dist/types.d.ts +543 -0
  7. package/package.json +39 -9
  8. package/skills/sp00ky-core/SKILL.md +258 -0
  9. package/skills/sp00ky-core/references/auth.md +98 -0
  10. package/skills/sp00ky-core/references/config.md +76 -0
  11. package/src/build-globals.d.ts +12 -0
  12. package/src/events/events.test.ts +2 -1
  13. package/src/events/index.ts +3 -0
  14. package/src/index.ts +9 -2
  15. package/src/modules/auth/events/index.ts +2 -1
  16. package/src/modules/auth/index.ts +59 -20
  17. package/src/modules/cache/index.ts +41 -29
  18. package/src/modules/cache/types.ts +2 -2
  19. package/src/modules/crdt/crdt-field.ts +281 -0
  20. package/src/modules/crdt/crdt-hydration.test.ts +206 -0
  21. package/src/modules/crdt/index.ts +352 -0
  22. package/src/modules/data/data.status.test.ts +108 -0
  23. package/src/modules/data/index.ts +732 -109
  24. package/src/modules/data/window-query.test.ts +52 -0
  25. package/src/modules/data/window-query.ts +130 -0
  26. package/src/modules/devtools/index.ts +115 -21
  27. package/src/modules/devtools/versions.test.ts +74 -0
  28. package/src/modules/devtools/versions.ts +81 -0
  29. package/src/modules/feature-flag/index.ts +166 -0
  30. package/src/modules/ref-tables.test.ts +66 -0
  31. package/src/modules/ref-tables.ts +69 -0
  32. package/src/modules/sync/engine.ts +101 -37
  33. package/src/modules/sync/events/index.ts +9 -2
  34. package/src/modules/sync/queue/queue-down.ts +5 -4
  35. package/src/modules/sync/queue/queue-up.ts +14 -13
  36. package/src/modules/sync/scheduler.ts +40 -3
  37. package/src/modules/sync/sync.ts +893 -59
  38. package/src/modules/sync/utils.test.ts +269 -2
  39. package/src/modules/sync/utils.ts +182 -17
  40. package/src/otel/index.ts +127 -0
  41. package/src/services/database/database.ts +11 -11
  42. package/src/services/database/events/index.ts +2 -1
  43. package/src/services/database/local-migrator.ts +21 -21
  44. package/src/services/database/local.test.ts +33 -0
  45. package/src/services/database/local.ts +192 -32
  46. package/src/services/database/remote.ts +13 -13
  47. package/src/services/logger/index.ts +6 -101
  48. package/src/services/persistence/localstorage.ts +2 -2
  49. package/src/services/persistence/resilient.ts +41 -0
  50. package/src/services/persistence/surrealdb.ts +9 -9
  51. package/src/services/stream-processor/index.ts +248 -37
  52. package/src/services/stream-processor/permissions.test.ts +47 -0
  53. package/src/services/stream-processor/permissions.ts +53 -0
  54. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  55. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  56. package/src/services/stream-processor/wasm-types.ts +18 -2
  57. package/src/sp00ky.auth-order.test.ts +65 -0
  58. package/src/sp00ky.ts +648 -0
  59. package/src/types.ts +192 -15
  60. package/src/utils/index.ts +35 -13
  61. package/src/utils/parser.ts +3 -2
  62. package/src/utils/surql.ts +24 -15
  63. package/src/utils/withRetry.test.ts +1 -1
  64. package/tsdown.config.ts +55 -1
  65. package/src/spooky.ts +0 -392
@@ -5,8 +5,16 @@ import {
5
5
  applyRecordVersionDiff,
6
6
  createDiffFromDbOp,
7
7
  ArraySyncer,
8
+ resolveListRefPollInterval,
9
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS,
10
+ buildListRefSelect,
11
+ buildSubqueryListRefSelect,
12
+ nextPollDelayMs,
13
+ listRefPollDelayMs,
14
+ LIST_REF_POLL_MAX_INTERVAL_MS,
15
+ recordVersionArraysEqual,
8
16
  } from './utils';
9
- import { RecordVersionArray, RecordVersionDiff } from '../../types';
17
+ import type { RecordVersionArray, RecordVersionDiff } from '../../types';
10
18
  import { encodeRecordId } from '../../utils/index';
11
19
 
12
20
  function rid(table: string, id: string): RecordId<string> {
@@ -305,7 +313,266 @@ describe('ArraySyncer', () => {
305
313
  expect(diff!.removed).toHaveLength(3);
306
314
  // Check they come in sorted order
307
315
  const removedIds = diff!.removed.map((r) => encodeRecordId(r));
308
- const sorted = [...removedIds].sort();
316
+ const sorted = [...removedIds].toSorted();
309
317
  expect(removedIds).toEqual(sorted);
310
318
  });
311
319
  });
320
+
321
+ describe('resolveListRefPollInterval', () => {
322
+ it('returns the default when no option is provided', () => {
323
+ expect(resolveListRefPollInterval()).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
324
+ expect(resolveListRefPollInterval(undefined)).toBe(
325
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
326
+ );
327
+ });
328
+
329
+ it('respects a positive override', () => {
330
+ expect(resolveListRefPollInterval(1000)).toBe(1000);
331
+ expect(resolveListRefPollInterval(5000)).toBe(5000);
332
+ });
333
+
334
+ it('falls back to default for zero, negative, NaN, or Infinity', () => {
335
+ // Accepting these would either silently disable polling or busy-loop
336
+ // the event loop, so they fall back to the default.
337
+ expect(resolveListRefPollInterval(0)).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
338
+ expect(resolveListRefPollInterval(-500)).toBe(
339
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
340
+ );
341
+ expect(resolveListRefPollInterval(NaN)).toBe(
342
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
343
+ );
344
+ expect(resolveListRefPollInterval(Infinity)).toBe(
345
+ DEFAULT_LIST_REF_POLL_INTERVAL_MS
346
+ );
347
+ });
348
+
349
+ it('locks the default at 500ms so accidental tuning trips this test', () => {
350
+ expect(DEFAULT_LIST_REF_POLL_INTERVAL_MS).toBe(500);
351
+ });
352
+ });
353
+
354
+ describe('buildListRefSelect', () => {
355
+ it('substitutes the table name', () => {
356
+ expect(buildListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
357
+ expect(buildListRefSelect('_00_list_ref_user_abc')).toContain(
358
+ 'FROM _00_list_ref_user_abc '
359
+ );
360
+ });
361
+
362
+ it('filters by the bound query id', () => {
363
+ expect(buildListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
364
+ });
365
+
366
+ it('excludes subquery rows via parent IS NONE', () => {
367
+ // Without this predicate the poll would surface child records (the
368
+ // subquery-projection rows the SSP also writes into list_ref) as
369
+ // spurious "added" diffs against `localArray` every tick.
370
+ expect(buildListRefSelect('_00_list_ref')).toContain('parent IS NONE');
371
+ });
372
+
373
+ it('selects only `out` and `version`', () => {
374
+ // The diff only needs the record id and its version; pulling more
375
+ // fields would bloat the per-tick query traffic.
376
+ const sql = buildListRefSelect('_00_list_ref');
377
+ expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
378
+ });
379
+ });
380
+
381
+ describe('buildSubqueryListRefSelect', () => {
382
+ it('substitutes the table name', () => {
383
+ expect(buildSubqueryListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
384
+ expect(buildSubqueryListRefSelect('_00_list_ref_user_abc')).toContain(
385
+ 'FROM _00_list_ref_user_abc '
386
+ );
387
+ });
388
+
389
+ it('filters by the bound query id', () => {
390
+ expect(buildSubqueryListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
391
+ });
392
+
393
+ it('selects ONLY subquery child rows via parent IS NOT NONE', () => {
394
+ // The mirror of `buildListRefSelect`'s `parent IS NONE`: the SSP writes
395
+ // `.related()` child rows into list_ref tagged with `parent`/`parent_rel`;
396
+ // this select pulls exactly those so their bodies can be synced into the
397
+ // local cache (separately from the primary window) and the correlated
398
+ // surql re-materializes with related data on a cold reload.
399
+ const sql = buildSubqueryListRefSelect('_00_list_ref');
400
+ expect(sql).toContain('parent IS NOT NONE');
401
+ expect(sql).not.toContain('parent IS NONE ');
402
+ });
403
+
404
+ it('selects only `out` and `version`', () => {
405
+ const sql = buildSubqueryListRefSelect('_00_list_ref');
406
+ expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
407
+ });
408
+ });
409
+
410
+ describe('nextPollDelayMs', () => {
411
+ it('returns the base interval when LIVE has never delivered', () => {
412
+ expect(
413
+ nextPollDelayMs({
414
+ now: 10_000,
415
+ lastLiveEventAt: null,
416
+ baseIntervalMs: 500,
417
+ })
418
+ ).toBe(500);
419
+ });
420
+
421
+ it('returns the healthy interval when a LIVE event fired within the cooldown', () => {
422
+ expect(
423
+ nextPollDelayMs({
424
+ now: 10_000,
425
+ lastLiveEventAt: 8_000, // 2s ago, well within 5s cooldown
426
+ baseIntervalMs: 500,
427
+ })
428
+ ).toBe(5_000);
429
+ });
430
+
431
+ it('snaps back to the base interval after the cooldown elapses', () => {
432
+ expect(
433
+ nextPollDelayMs({
434
+ now: 20_000,
435
+ lastLiveEventAt: 10_000, // 10s ago, well past cooldown
436
+ baseIntervalMs: 500,
437
+ })
438
+ ).toBe(500);
439
+ });
440
+
441
+ it('treats the cooldown boundary as expired (>= cooldownMs returns base)', () => {
442
+ // Exactly at the boundary, the LIVE feed is considered cold so we
443
+ // resume the aggressive cadence. Picks the same direction as
444
+ // `if (sinceLive >= cooldownMs)`.
445
+ expect(
446
+ nextPollDelayMs({
447
+ now: 15_000,
448
+ lastLiveEventAt: 10_000, // exactly 5s ago
449
+ baseIntervalMs: 500,
450
+ cooldownMs: 5_000,
451
+ })
452
+ ).toBe(500);
453
+ });
454
+
455
+ it('clamps a clock-skew negative sinceLive to the base interval', () => {
456
+ // If a stale timestamp puts the LIVE event in the future, we
457
+ // should not interpret that as "LIVE healthy" — treat it as
458
+ // unknown and use the conservative aggressive cadence.
459
+ expect(
460
+ nextPollDelayMs({
461
+ now: 10_000,
462
+ lastLiveEventAt: 20_000, // future timestamp
463
+ baseIntervalMs: 500,
464
+ })
465
+ ).toBe(500);
466
+ });
467
+
468
+ it('never widens below an aggressively-configured base interval', () => {
469
+ // If the caller picked a faster base (e.g. 100ms) on purpose,
470
+ // the healthy path should not silently slow things back to 5s.
471
+ // The healthy interval is clamped to at least base.
472
+ expect(
473
+ nextPollDelayMs({
474
+ now: 10_000,
475
+ lastLiveEventAt: 9_500,
476
+ baseIntervalMs: 8_000,
477
+ })
478
+ ).toBe(8_000);
479
+ });
480
+
481
+ it('respects custom cooldown and healthy-interval values', () => {
482
+ expect(
483
+ nextPollDelayMs({
484
+ now: 10_000,
485
+ lastLiveEventAt: 9_500, // 0.5s ago
486
+ baseIntervalMs: 250,
487
+ cooldownMs: 1_000,
488
+ healthyIntervalMs: 2_000,
489
+ })
490
+ ).toBe(2_000);
491
+ expect(
492
+ nextPollDelayMs({
493
+ now: 10_000,
494
+ lastLiveEventAt: 8_500, // 1.5s ago > cooldown
495
+ baseIntervalMs: 250,
496
+ cooldownMs: 1_000,
497
+ healthyIntervalMs: 2_000,
498
+ })
499
+ ).toBe(250);
500
+ });
501
+ });
502
+
503
+ describe('listRefPollDelayMs', () => {
504
+ it('returns the base interval at idle streak 0 (something just happened)', () => {
505
+ expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 500 })).toBe(500);
506
+ // Negative streak is defensively treated as "active" too.
507
+ expect(listRefPollDelayMs({ idleStreak: -3, baseIntervalMs: 500 })).toBe(500);
508
+ });
509
+
510
+ it('doubles per idle streak (exponential backoff)', () => {
511
+ expect(listRefPollDelayMs({ idleStreak: 1, baseIntervalMs: 500 })).toBe(1_000);
512
+ expect(listRefPollDelayMs({ idleStreak: 2, baseIntervalMs: 500 })).toBe(2_000);
513
+ expect(listRefPollDelayMs({ idleStreak: 3, baseIntervalMs: 500 })).toBe(4_000);
514
+ });
515
+
516
+ it('caps at LIST_REF_POLL_MAX_INTERVAL_MS (5s) once the doubling exceeds it', () => {
517
+ // streak 4 would be 8000ms uncapped → clamped to 5000.
518
+ expect(listRefPollDelayMs({ idleStreak: 4, baseIntervalMs: 500 })).toBe(5_000);
519
+ expect(listRefPollDelayMs({ idleStreak: 50, baseIntervalMs: 500 })).toBe(5_000);
520
+ expect(LIST_REF_POLL_MAX_INTERVAL_MS).toBe(5_000);
521
+ });
522
+
523
+ it('never returns below the configured base, even when base exceeds the cap', () => {
524
+ // An aggressively-large base must not be implicitly shrunk by the cap.
525
+ expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 8_000 })).toBe(8_000);
526
+ expect(listRefPollDelayMs({ idleStreak: 5, baseIntervalMs: 8_000 })).toBe(8_000);
527
+ });
528
+
529
+ it('respects a custom max interval', () => {
530
+ expect(
531
+ listRefPollDelayMs({ idleStreak: 10, baseIntervalMs: 500, maxIntervalMs: 3_000 })
532
+ ).toBe(3_000);
533
+ });
534
+
535
+ it('does not overflow for a very long idle streak', () => {
536
+ // 2^1000 would be Infinity; the exponent clamp keeps it finite and capped.
537
+ expect(listRefPollDelayMs({ idleStreak: 1_000, baseIntervalMs: 500 })).toBe(5_000);
538
+ });
539
+ });
540
+
541
+ describe('recordVersionArraysEqual', () => {
542
+ it('treats the same reference and identical contents as equal', () => {
543
+ const a: RecordVersionArray = [['game:1', 1], ['game:2', 3]];
544
+ expect(recordVersionArraysEqual(a, a)).toBe(true);
545
+ expect(recordVersionArraysEqual(a, [['game:1', 1], ['game:2', 3]])).toBe(true);
546
+ });
547
+
548
+ it('is order-insensitive (the list_ref SELECT has no ORDER BY)', () => {
549
+ expect(
550
+ recordVersionArraysEqual(
551
+ [['game:1', 1], ['game:2', 3]],
552
+ [['game:2', 3], ['game:1', 1]]
553
+ )
554
+ ).toBe(true);
555
+ });
556
+
557
+ it('returns false when a version differs for the same id', () => {
558
+ expect(
559
+ recordVersionArraysEqual([['game:1', 1]], [['game:1', 2]])
560
+ ).toBe(false);
561
+ });
562
+
563
+ it('returns false on length mismatch', () => {
564
+ expect(
565
+ recordVersionArraysEqual([['game:1', 1]], [['game:1', 1], ['game:2', 1]])
566
+ ).toBe(false);
567
+ });
568
+
569
+ it('returns false when an id is present in one but not the other', () => {
570
+ expect(
571
+ recordVersionArraysEqual([['game:1', 1]], [['game:2', 1]])
572
+ ).toBe(false);
573
+ });
574
+
575
+ it('treats two empty arrays as equal', () => {
576
+ expect(recordVersionArraysEqual([], [])).toBe(true);
577
+ });
578
+ });
@@ -1,5 +1,5 @@
1
- import { RecordId } from 'surrealdb';
2
- import { RecordVersionArray, RecordVersionDiff } from '../../types';
1
+ import type { RecordId } from 'surrealdb';
2
+ import type { RecordVersionArray, RecordVersionDiff } from '../../types';
3
3
  import { parseRecordIdString, encodeRecordId } from '../../utils/index';
4
4
 
5
5
  export class ArraySyncer {
@@ -8,8 +8,8 @@ export class ArraySyncer {
8
8
  private needsSort = false;
9
9
 
10
10
  constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
11
- this.remoteArray = remoteArray.sort((a, b) => a[0].localeCompare(b[0]));
12
- this.localArray = localArray.sort((a, b) => a[0].localeCompare(b[0]));
11
+ this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
12
+ this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
13
13
  }
14
14
 
15
15
  /**
@@ -49,7 +49,6 @@ export class ArraySyncer {
49
49
  this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
50
50
  this.needsSort = false;
51
51
  }
52
- console.log('xxxx555', this.localArray, this.remoteArray);
53
52
  const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
54
53
  return diff;
55
54
  }
@@ -93,10 +92,12 @@ export function diffRecordVersionArray(
93
92
  return {
94
93
  added: added.map((id) => ({
95
94
  id: parseRecordIdString(id),
95
+ // oxlint-disable-next-line no-non-null-assertion
96
96
  version: remoteMap.get(id)!,
97
97
  })),
98
98
  updated: updated.map((id) => ({
99
99
  id: parseRecordIdString(id),
100
+ // oxlint-disable-next-line no-non-null-assertion
100
101
  version: remoteMap.get(id)!,
101
102
  })),
102
103
  removed: removed.map(parseRecordIdString),
@@ -127,7 +128,7 @@ export function applyRecordVersionDiff(
127
128
  currentMap.set(encodeRecordId(item.id), item.version);
128
129
  }
129
130
 
130
- return Array.from(currentMap).sort((a, b) => a[0].localeCompare(b[0]));
131
+ return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
131
132
  }
132
133
 
133
134
  export function createDiffFromDbOp(
@@ -136,17 +137,14 @@ export function createDiffFromDbOp(
136
137
  version: number,
137
138
  versions?: RecordVersionArray
138
139
  ): RecordVersionDiff {
139
- // Version guard: skip stale CREATE/UPDATE, but always process DELETE
140
- if (op !== 'DELETE') {
141
- const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
142
-
143
- if (old && old[1] >= version) {
144
- return {
145
- added: [],
146
- updated: [],
147
- removed: [],
148
- };
149
- }
140
+ const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
141
+
142
+ if (old && old[1] >= version) {
143
+ return {
144
+ added: [],
145
+ updated: [],
146
+ removed: [],
147
+ };
150
148
  }
151
149
 
152
150
  if (op === 'CREATE') {
@@ -169,3 +167,170 @@ export function createDiffFromDbOp(
169
167
  };
170
168
  }
171
169
  }
170
+
171
+ /**
172
+ * Default cadence for the `_00_list_ref_user_<id>` poll fallback. The
173
+ * poll is the safety net for SurrealDB v3's occasionally-dropped LIVE
174
+ * deliveries; 500ms is aggressive enough to feel real-time on the
175
+ * happy path while keeping the per-session query load bounded.
176
+ */
177
+ export const DEFAULT_LIST_REF_POLL_INTERVAL_MS = 500;
178
+
179
+ /**
180
+ * Build the SurrealQL select that powers both the initial-fetch and
181
+ * the periodic poll of `_00_list_ref[_user_<id>]`. The `parent IS NONE`
182
+ * predicate excludes subquery entries (rows with `parent_rel` set)
183
+ * because the client's `RecordVersionArray` only tracks primary rows;
184
+ * including subquery rows would surface them as spurious "added"
185
+ * diffs every tick.
186
+ */
187
+ export function buildListRefSelect(table: string): string {
188
+ return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
189
+ }
190
+
191
+ /**
192
+ * Build the SurrealQL select for a query's SUBQUERY child edges — the
193
+ * mirror of {@link buildListRefSelect}. `.related()` queries register a
194
+ * correlated subquery; the SSP materializes each matched child as a
195
+ * `_00_list_ref` edge tagged with `parent`/`parent_rel` (see
196
+ * `apps/ssp` edge writer). `parent IS NONE` (the primary select) drops
197
+ * these, so their bodies never reach the local cache and a cold-reload
198
+ * re-materialization of the correlated surql yields empty related
199
+ * fields. This `parent IS NOT NONE` variant pulls the child `out`+`version`
200
+ * pairs (any nesting depth) so we can sync their bodies into the local
201
+ * store SEPARATELY from the primary window array.
202
+ */
203
+ export function buildSubqueryListRefSelect(table: string): string {
204
+ return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
205
+ }
206
+
207
+ /**
208
+ * Resolve the effective list-ref poll interval. Negative or zero
209
+ * values fall back to the default — accepting them would either
210
+ * disable polling silently or busy-loop the event loop.
211
+ */
212
+ export function resolveListRefPollInterval(opt?: number): number {
213
+ if (typeof opt !== 'number' || !Number.isFinite(opt) || opt <= 0) {
214
+ return DEFAULT_LIST_REF_POLL_INTERVAL_MS;
215
+ }
216
+ return opt;
217
+ }
218
+
219
+ /**
220
+ * When the LIVE feed has delivered an event within this window, treat
221
+ * the LIVE subscription as healthy and back the poll off to
222
+ * {@link LIVE_HEALTHY_POLL_INTERVAL_MS}. As soon as LIVE quiets for
223
+ * longer than this, the poll snaps back to the aggressive default.
224
+ */
225
+ export const LIVE_HEALTHY_COOLDOWN_MS = 5_000;
226
+
227
+ /**
228
+ * Poll interval while LIVE is delivering events. The aggressive
229
+ * default 500ms costs ~120 queries / minute / session — wasted work
230
+ * when LIVE is already covering us. 5s keeps the safety net in place
231
+ * at 1/10th the load.
232
+ */
233
+ export const LIVE_HEALTHY_POLL_INTERVAL_MS = 5_000;
234
+
235
+ /**
236
+ * Pick the next poll delay based on whether LIVE has been healthy
237
+ * recently. If a LIVE event fired within `cooldownMs`, use the slow
238
+ * (`healthyIntervalMs`) cadence; otherwise the fast (`baseIntervalMs`)
239
+ * cadence. Pure so it's unit-testable; `Sp00kySync.startListRefPoll`
240
+ * calls it from a self-rescheduling timer.
241
+ *
242
+ * The healthy interval is clamped to at least `baseIntervalMs` so
243
+ * configuring an aggressive base (e.g. 100ms) never gets implicitly
244
+ * widened by this helper.
245
+ *
246
+ * @deprecated Superseded by {@link listRefPollDelayMs}, which backs the
247
+ * poll off based on observed change activity (LIVE *or* poll-detected)
248
+ * rather than LIVE liveness alone — the cross-session LIVE-permission gap
249
+ * means LIVE often never fires, so this helper would keep the poll pinned
250
+ * at the aggressive base forever even on a fully idle page. Kept (and
251
+ * tested) for reference.
252
+ */
253
+ export function nextPollDelayMs(args: {
254
+ now: number;
255
+ lastLiveEventAt: number | null;
256
+ baseIntervalMs: number;
257
+ cooldownMs?: number;
258
+ healthyIntervalMs?: number;
259
+ }): number {
260
+ const {
261
+ now,
262
+ lastLiveEventAt,
263
+ baseIntervalMs,
264
+ cooldownMs = LIVE_HEALTHY_COOLDOWN_MS,
265
+ healthyIntervalMs = LIVE_HEALTHY_POLL_INTERVAL_MS,
266
+ } = args;
267
+ if (lastLiveEventAt === null) return baseIntervalMs;
268
+ const sinceLive = now - lastLiveEventAt;
269
+ if (sinceLive < 0 || sinceLive >= cooldownMs) return baseIntervalMs;
270
+ return Math.max(healthyIntervalMs, baseIntervalMs);
271
+ }
272
+
273
+ /**
274
+ * Ceiling for the adaptive list_ref poll backoff. An idle page (no LIVE
275
+ * events, no poll-detected list_ref changes) coasts up to this cadence;
276
+ * the existing healthy-LIVE safety net runs at the same 5s, so this keeps
277
+ * the worst-case catch-up latency for a missed cross-session change at the
278
+ * cadence the codebase already treats as acceptable.
279
+ */
280
+ export const LIST_REF_POLL_MAX_INTERVAL_MS = 5_000;
281
+
282
+ /**
283
+ * Adaptive poll delay: stay at the responsive `baseIntervalMs` while
284
+ * changes are arriving, and exponentially back off toward `maxIntervalMs`
285
+ * while the `_00_list_ref` is quiet.
286
+ *
287
+ * `idleStreak` is the count of consecutive poll cycles that observed *no*
288
+ * change. `Sp00kySync` resets it to 0 whenever a poll detects a real
289
+ * remoteArray change OR a LIVE event lands, so any activity snaps the poll
290
+ * straight back to `baseIntervalMs`. A streak of 0 (something just
291
+ * happened) → base; otherwise `base * 2^streak` capped at `maxIntervalMs`.
292
+ *
293
+ * This replaces {@link nextPollDelayMs}: the old helper slowed the poll
294
+ * only while LIVE was *delivering*, but the cross-session LIVE-permission
295
+ * gap means LIVE frequently never fires here, so it left a fully idle page
296
+ * polling every `base` ms forever (the "continuous queries while idle"
297
+ * symptom). Backing off on observed idleness instead covers the
298
+ * LIVE-healthy case for free (LIVE applies the change → the next poll sees
299
+ * nothing new → the streak grows → it backs off).
300
+ */
301
+ export function listRefPollDelayMs(args: {
302
+ idleStreak: number;
303
+ baseIntervalMs: number;
304
+ maxIntervalMs?: number;
305
+ }): number {
306
+ const { idleStreak, baseIntervalMs, maxIntervalMs = LIST_REF_POLL_MAX_INTERVAL_MS } = args;
307
+ const cap = Math.max(baseIntervalMs, maxIntervalMs);
308
+ if (idleStreak <= 0) return baseIntervalMs;
309
+ // 2^streak grows fast; clamp the exponent so it can't overflow on a
310
+ // page left idle for a very long time.
311
+ const exponent = Math.min(idleStreak, 30);
312
+ return Math.min(baseIntervalMs * 2 ** exponent, cap);
313
+ }
314
+
315
+ /**
316
+ * Order-insensitive equality for two `RecordVersionArray`s (each a list of
317
+ * `[recordIdString, version]`). The `_00_list_ref` SELECT has no `ORDER
318
+ * BY`, so row order can differ between polls without anything having
319
+ * actually changed — comparing as an id→version map avoids false
320
+ * "changed" verdicts that would defeat the idle backoff. Record ids are
321
+ * unique within a query's list_ref, so a map is a faithful representation.
322
+ */
323
+ export function recordVersionArraysEqual(
324
+ a: RecordVersionArray,
325
+ b: RecordVersionArray
326
+ ): boolean {
327
+ if (a === b) return true;
328
+ if (a.length !== b.length) return false;
329
+ const byId = new Map<string, number>();
330
+ for (const [id, version] of a) byId.set(id, version);
331
+ for (const [id, version] of b) {
332
+ // `get` returns undefined for a missing id, which never === a number.
333
+ if (byId.get(id) !== version) return false;
334
+ }
335
+ return true;
336
+ }
@@ -0,0 +1,127 @@
1
+ import type { Level } from 'pino';
2
+ import type { PinoTransmit } from '../types';
3
+
4
+ // Map pino levels to OTEL severity numbers
5
+ // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
6
+ function mapLevelToSeverityNumber(level: string): number {
7
+ switch (level) {
8
+ case 'trace':
9
+ return 1;
10
+ case 'debug':
11
+ return 5;
12
+ case 'info':
13
+ return 9;
14
+ case 'warn':
15
+ return 13;
16
+ case 'error':
17
+ return 17;
18
+ case 'fatal':
19
+ return 21;
20
+ default:
21
+ return 9;
22
+ }
23
+ }
24
+
25
+ async function loadOtelModules(otelEndpoint: string) {
26
+ const [{ LoggerProvider, BatchLogRecordProcessor }, { OTLPLogExporter }, { resourceFromAttributes }, { ATTR_SERVICE_NAME }] =
27
+ await Promise.all([
28
+ import('@opentelemetry/sdk-logs'),
29
+ import('@opentelemetry/exporter-logs-otlp-proto'),
30
+ import('@opentelemetry/resources'),
31
+ import('@opentelemetry/semantic-conventions'),
32
+ ]);
33
+
34
+ const resource = resourceFromAttributes({
35
+ [ATTR_SERVICE_NAME]: 'sp00ky-client',
36
+ });
37
+
38
+ const exporter = new OTLPLogExporter({
39
+ url: otelEndpoint,
40
+ });
41
+
42
+ const loggerProvider = new LoggerProvider({
43
+ resource,
44
+ processors: [new BatchLogRecordProcessor(exporter)],
45
+ });
46
+
47
+ const otelLoggerCache: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
48
+
49
+ return (category: string) => {
50
+ if (!otelLoggerCache[category]) {
51
+ otelLoggerCache[category] = loggerProvider.getLogger(category);
52
+ }
53
+ return otelLoggerCache[category];
54
+ };
55
+ }
56
+
57
+ /**
58
+ * Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
59
+ *
60
+ * @example
61
+ * ```ts
62
+ * import { createOtelTransmit } from '@spooky-sync/core/otel';
63
+ *
64
+ * new Sp00kyClient({
65
+ * // ...
66
+ * otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
67
+ * });
68
+ * ```
69
+ */
70
+ export function createOtelTransmit(endpoint: string, level: Level = 'info'): PinoTransmit {
71
+ // Start loading OTel modules eagerly (don't await — we're synchronous)
72
+ const otelReady = loadOtelModules(endpoint);
73
+
74
+ return {
75
+ level: level,
76
+ send: (levelLabel: string, logEvent: any) => {
77
+ // oxlint-disable-next-line promise/always-return
78
+ otelReady.then((getOtelLogger) => {
79
+ try {
80
+ const messages = [...logEvent.messages];
81
+ const severityNumber = mapLevelToSeverityNumber(levelLabel);
82
+
83
+ // Construct the message body
84
+ let body = '';
85
+ const lastMsg = messages.pop();
86
+
87
+ if (typeof lastMsg === 'string') {
88
+ body = lastMsg;
89
+ } else if (lastMsg) {
90
+ body = JSON.stringify(lastMsg);
91
+ }
92
+
93
+ let category = 'sp00ky-client::unknown';
94
+
95
+ const attributes = {};
96
+ for (const msg of messages) {
97
+ if (typeof msg === 'object') {
98
+ if (msg.Category) {
99
+ category = msg.Category;
100
+ delete msg.Category;
101
+ }
102
+ Object.assign(attributes, msg);
103
+ }
104
+ }
105
+
106
+ // Emit to OTEL SDK
107
+ getOtelLogger(category).emit({
108
+ severityNumber: severityNumber,
109
+ severityText: levelLabel.toUpperCase(),
110
+ body: body,
111
+ attributes: {
112
+ ...logEvent.bindings[0],
113
+ ...attributes,
114
+ },
115
+ timestamp: new Date(logEvent.ts),
116
+ });
117
+ } catch (e) {
118
+ // oxlint-disable-next-line no-console
119
+ console.warn('Failed to transmit log to OTEL endpoint', e);
120
+ }
121
+ }).catch((e) => {
122
+ // oxlint-disable-next-line no-console
123
+ console.warn('Failed to load OpenTelemetry modules', e);
124
+ });
125
+ },
126
+ };
127
+ }