@spooky-sync/core 0.0.1-canary.8 → 0.0.1-canary.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +854 -339
- package/dist/index.js +2633 -396
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/types.d.ts +494 -0
- package/package.json +39 -9
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +41 -29
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +281 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +352 -0
- package/src/modules/data/data.status.test.ts +108 -0
- package/src/modules/data/index.ts +726 -108
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +130 -0
- package/src/modules/devtools/index.ts +89 -21
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.ts +166 -0
- package/src/modules/ref-tables.test.ts +56 -0
- package/src/modules/ref-tables.ts +57 -0
- package/src/modules/sync/engine.ts +97 -37
- package/src/modules/sync/events/index.ts +3 -2
- package/src/modules/sync/queue/queue-down.ts +5 -4
- package/src/modules/sync/queue/queue-up.ts +14 -13
- package/src/modules/sync/scheduler.ts +2 -2
- package/src/modules/sync/sync.ts +676 -58
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/local-migrator.ts +21 -21
- package/src/services/database/local.test.ts +33 -0
- package/src/services/database/local.ts +192 -32
- package/src/services/database/remote.ts +13 -13
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +9 -9
- package/src/services/stream-processor/index.ts +248 -37
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +65 -0
- package/src/sp00ky.ts +625 -0
- package/src/types.ts +140 -13
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +55 -1
- package/src/spooky.ts +0 -392
|
@@ -5,8 +5,16 @@ import {
|
|
|
5
5
|
applyRecordVersionDiff,
|
|
6
6
|
createDiffFromDbOp,
|
|
7
7
|
ArraySyncer,
|
|
8
|
+
resolveListRefPollInterval,
|
|
9
|
+
DEFAULT_LIST_REF_POLL_INTERVAL_MS,
|
|
10
|
+
buildListRefSelect,
|
|
11
|
+
buildSubqueryListRefSelect,
|
|
12
|
+
nextPollDelayMs,
|
|
13
|
+
listRefPollDelayMs,
|
|
14
|
+
LIST_REF_POLL_MAX_INTERVAL_MS,
|
|
15
|
+
recordVersionArraysEqual,
|
|
8
16
|
} from './utils';
|
|
9
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
17
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
10
18
|
import { encodeRecordId } from '../../utils/index';
|
|
11
19
|
|
|
12
20
|
function rid(table: string, id: string): RecordId<string> {
|
|
@@ -305,7 +313,266 @@ describe('ArraySyncer', () => {
|
|
|
305
313
|
expect(diff!.removed).toHaveLength(3);
|
|
306
314
|
// Check they come in sorted order
|
|
307
315
|
const removedIds = diff!.removed.map((r) => encodeRecordId(r));
|
|
308
|
-
const sorted = [...removedIds].
|
|
316
|
+
const sorted = [...removedIds].toSorted();
|
|
309
317
|
expect(removedIds).toEqual(sorted);
|
|
310
318
|
});
|
|
311
319
|
});
|
|
320
|
+
|
|
321
|
+
describe('resolveListRefPollInterval', () => {
|
|
322
|
+
it('returns the default when no option is provided', () => {
|
|
323
|
+
expect(resolveListRefPollInterval()).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
|
|
324
|
+
expect(resolveListRefPollInterval(undefined)).toBe(
|
|
325
|
+
DEFAULT_LIST_REF_POLL_INTERVAL_MS
|
|
326
|
+
);
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
it('respects a positive override', () => {
|
|
330
|
+
expect(resolveListRefPollInterval(1000)).toBe(1000);
|
|
331
|
+
expect(resolveListRefPollInterval(5000)).toBe(5000);
|
|
332
|
+
});
|
|
333
|
+
|
|
334
|
+
it('falls back to default for zero, negative, NaN, or Infinity', () => {
|
|
335
|
+
// Accepting these would either silently disable polling or busy-loop
|
|
336
|
+
// the event loop, so they fall back to the default.
|
|
337
|
+
expect(resolveListRefPollInterval(0)).toBe(DEFAULT_LIST_REF_POLL_INTERVAL_MS);
|
|
338
|
+
expect(resolveListRefPollInterval(-500)).toBe(
|
|
339
|
+
DEFAULT_LIST_REF_POLL_INTERVAL_MS
|
|
340
|
+
);
|
|
341
|
+
expect(resolveListRefPollInterval(NaN)).toBe(
|
|
342
|
+
DEFAULT_LIST_REF_POLL_INTERVAL_MS
|
|
343
|
+
);
|
|
344
|
+
expect(resolveListRefPollInterval(Infinity)).toBe(
|
|
345
|
+
DEFAULT_LIST_REF_POLL_INTERVAL_MS
|
|
346
|
+
);
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it('locks the default at 500ms so accidental tuning trips this test', () => {
|
|
350
|
+
expect(DEFAULT_LIST_REF_POLL_INTERVAL_MS).toBe(500);
|
|
351
|
+
});
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
describe('buildListRefSelect', () => {
|
|
355
|
+
it('substitutes the table name', () => {
|
|
356
|
+
expect(buildListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
|
|
357
|
+
expect(buildListRefSelect('_00_list_ref_user_abc')).toContain(
|
|
358
|
+
'FROM _00_list_ref_user_abc '
|
|
359
|
+
);
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
it('filters by the bound query id', () => {
|
|
363
|
+
expect(buildListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
it('excludes subquery rows via parent IS NONE', () => {
|
|
367
|
+
// Without this predicate the poll would surface child records (the
|
|
368
|
+
// subquery-projection rows the SSP also writes into list_ref) as
|
|
369
|
+
// spurious "added" diffs against `localArray` every tick.
|
|
370
|
+
expect(buildListRefSelect('_00_list_ref')).toContain('parent IS NONE');
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
it('selects only `out` and `version`', () => {
|
|
374
|
+
// The diff only needs the record id and its version; pulling more
|
|
375
|
+
// fields would bloat the per-tick query traffic.
|
|
376
|
+
const sql = buildListRefSelect('_00_list_ref');
|
|
377
|
+
expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
|
|
378
|
+
});
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
describe('buildSubqueryListRefSelect', () => {
|
|
382
|
+
it('substitutes the table name', () => {
|
|
383
|
+
expect(buildSubqueryListRefSelect('_00_list_ref')).toContain('FROM _00_list_ref ');
|
|
384
|
+
expect(buildSubqueryListRefSelect('_00_list_ref_user_abc')).toContain(
|
|
385
|
+
'FROM _00_list_ref_user_abc '
|
|
386
|
+
);
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
it('filters by the bound query id', () => {
|
|
390
|
+
expect(buildSubqueryListRefSelect('_00_list_ref')).toContain('WHERE in = $in');
|
|
391
|
+
});
|
|
392
|
+
|
|
393
|
+
it('selects ONLY subquery child rows via parent IS NOT NONE', () => {
|
|
394
|
+
// The mirror of `buildListRefSelect`'s `parent IS NONE`: the SSP writes
|
|
395
|
+
// `.related()` child rows into list_ref tagged with `parent`/`parent_rel`;
|
|
396
|
+
// this select pulls exactly those so their bodies can be synced into the
|
|
397
|
+
// local cache (separately from the primary window) and the correlated
|
|
398
|
+
// surql re-materializes with related data on a cold reload.
|
|
399
|
+
const sql = buildSubqueryListRefSelect('_00_list_ref');
|
|
400
|
+
expect(sql).toContain('parent IS NOT NONE');
|
|
401
|
+
expect(sql).not.toContain('parent IS NONE ');
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
it('selects only `out` and `version`', () => {
|
|
405
|
+
const sql = buildSubqueryListRefSelect('_00_list_ref');
|
|
406
|
+
expect(sql.startsWith('SELECT out, version FROM ')).toBe(true);
|
|
407
|
+
});
|
|
408
|
+
});
|
|
409
|
+
|
|
410
|
+
describe('nextPollDelayMs', () => {
|
|
411
|
+
it('returns the base interval when LIVE has never delivered', () => {
|
|
412
|
+
expect(
|
|
413
|
+
nextPollDelayMs({
|
|
414
|
+
now: 10_000,
|
|
415
|
+
lastLiveEventAt: null,
|
|
416
|
+
baseIntervalMs: 500,
|
|
417
|
+
})
|
|
418
|
+
).toBe(500);
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
it('returns the healthy interval when a LIVE event fired within the cooldown', () => {
|
|
422
|
+
expect(
|
|
423
|
+
nextPollDelayMs({
|
|
424
|
+
now: 10_000,
|
|
425
|
+
lastLiveEventAt: 8_000, // 2s ago, well within 5s cooldown
|
|
426
|
+
baseIntervalMs: 500,
|
|
427
|
+
})
|
|
428
|
+
).toBe(5_000);
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
it('snaps back to the base interval after the cooldown elapses', () => {
|
|
432
|
+
expect(
|
|
433
|
+
nextPollDelayMs({
|
|
434
|
+
now: 20_000,
|
|
435
|
+
lastLiveEventAt: 10_000, // 10s ago, well past cooldown
|
|
436
|
+
baseIntervalMs: 500,
|
|
437
|
+
})
|
|
438
|
+
).toBe(500);
|
|
439
|
+
});
|
|
440
|
+
|
|
441
|
+
it('treats the cooldown boundary as expired (>= cooldownMs returns base)', () => {
|
|
442
|
+
// Exactly at the boundary, the LIVE feed is considered cold so we
|
|
443
|
+
// resume the aggressive cadence. Picks the same direction as
|
|
444
|
+
// `if (sinceLive >= cooldownMs)`.
|
|
445
|
+
expect(
|
|
446
|
+
nextPollDelayMs({
|
|
447
|
+
now: 15_000,
|
|
448
|
+
lastLiveEventAt: 10_000, // exactly 5s ago
|
|
449
|
+
baseIntervalMs: 500,
|
|
450
|
+
cooldownMs: 5_000,
|
|
451
|
+
})
|
|
452
|
+
).toBe(500);
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it('clamps a clock-skew negative sinceLive to the base interval', () => {
|
|
456
|
+
// If a stale timestamp puts the LIVE event in the future, we
|
|
457
|
+
// should not interpret that as "LIVE healthy" — treat it as
|
|
458
|
+
// unknown and use the conservative aggressive cadence.
|
|
459
|
+
expect(
|
|
460
|
+
nextPollDelayMs({
|
|
461
|
+
now: 10_000,
|
|
462
|
+
lastLiveEventAt: 20_000, // future timestamp
|
|
463
|
+
baseIntervalMs: 500,
|
|
464
|
+
})
|
|
465
|
+
).toBe(500);
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
it('never widens below an aggressively-configured base interval', () => {
|
|
469
|
+
// If the caller picked a faster base (e.g. 100ms) on purpose,
|
|
470
|
+
// the healthy path should not silently slow things back to 5s.
|
|
471
|
+
// The healthy interval is clamped to at least base.
|
|
472
|
+
expect(
|
|
473
|
+
nextPollDelayMs({
|
|
474
|
+
now: 10_000,
|
|
475
|
+
lastLiveEventAt: 9_500,
|
|
476
|
+
baseIntervalMs: 8_000,
|
|
477
|
+
})
|
|
478
|
+
).toBe(8_000);
|
|
479
|
+
});
|
|
480
|
+
|
|
481
|
+
it('respects custom cooldown and healthy-interval values', () => {
|
|
482
|
+
expect(
|
|
483
|
+
nextPollDelayMs({
|
|
484
|
+
now: 10_000,
|
|
485
|
+
lastLiveEventAt: 9_500, // 0.5s ago
|
|
486
|
+
baseIntervalMs: 250,
|
|
487
|
+
cooldownMs: 1_000,
|
|
488
|
+
healthyIntervalMs: 2_000,
|
|
489
|
+
})
|
|
490
|
+
).toBe(2_000);
|
|
491
|
+
expect(
|
|
492
|
+
nextPollDelayMs({
|
|
493
|
+
now: 10_000,
|
|
494
|
+
lastLiveEventAt: 8_500, // 1.5s ago > cooldown
|
|
495
|
+
baseIntervalMs: 250,
|
|
496
|
+
cooldownMs: 1_000,
|
|
497
|
+
healthyIntervalMs: 2_000,
|
|
498
|
+
})
|
|
499
|
+
).toBe(250);
|
|
500
|
+
});
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
describe('listRefPollDelayMs', () => {
|
|
504
|
+
it('returns the base interval at idle streak 0 (something just happened)', () => {
|
|
505
|
+
expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 500 })).toBe(500);
|
|
506
|
+
// Negative streak is defensively treated as "active" too.
|
|
507
|
+
expect(listRefPollDelayMs({ idleStreak: -3, baseIntervalMs: 500 })).toBe(500);
|
|
508
|
+
});
|
|
509
|
+
|
|
510
|
+
it('doubles per idle streak (exponential backoff)', () => {
|
|
511
|
+
expect(listRefPollDelayMs({ idleStreak: 1, baseIntervalMs: 500 })).toBe(1_000);
|
|
512
|
+
expect(listRefPollDelayMs({ idleStreak: 2, baseIntervalMs: 500 })).toBe(2_000);
|
|
513
|
+
expect(listRefPollDelayMs({ idleStreak: 3, baseIntervalMs: 500 })).toBe(4_000);
|
|
514
|
+
});
|
|
515
|
+
|
|
516
|
+
it('caps at LIST_REF_POLL_MAX_INTERVAL_MS (5s) once the doubling exceeds it', () => {
|
|
517
|
+
// streak 4 would be 8000ms uncapped → clamped to 5000.
|
|
518
|
+
expect(listRefPollDelayMs({ idleStreak: 4, baseIntervalMs: 500 })).toBe(5_000);
|
|
519
|
+
expect(listRefPollDelayMs({ idleStreak: 50, baseIntervalMs: 500 })).toBe(5_000);
|
|
520
|
+
expect(LIST_REF_POLL_MAX_INTERVAL_MS).toBe(5_000);
|
|
521
|
+
});
|
|
522
|
+
|
|
523
|
+
it('never returns below the configured base, even when base exceeds the cap', () => {
|
|
524
|
+
// An aggressively-large base must not be implicitly shrunk by the cap.
|
|
525
|
+
expect(listRefPollDelayMs({ idleStreak: 0, baseIntervalMs: 8_000 })).toBe(8_000);
|
|
526
|
+
expect(listRefPollDelayMs({ idleStreak: 5, baseIntervalMs: 8_000 })).toBe(8_000);
|
|
527
|
+
});
|
|
528
|
+
|
|
529
|
+
it('respects a custom max interval', () => {
|
|
530
|
+
expect(
|
|
531
|
+
listRefPollDelayMs({ idleStreak: 10, baseIntervalMs: 500, maxIntervalMs: 3_000 })
|
|
532
|
+
).toBe(3_000);
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it('does not overflow for a very long idle streak', () => {
|
|
536
|
+
// 2^1000 would be Infinity; the exponent clamp keeps it finite and capped.
|
|
537
|
+
expect(listRefPollDelayMs({ idleStreak: 1_000, baseIntervalMs: 500 })).toBe(5_000);
|
|
538
|
+
});
|
|
539
|
+
});
|
|
540
|
+
|
|
541
|
+
describe('recordVersionArraysEqual', () => {
|
|
542
|
+
it('treats the same reference and identical contents as equal', () => {
|
|
543
|
+
const a: RecordVersionArray = [['game:1', 1], ['game:2', 3]];
|
|
544
|
+
expect(recordVersionArraysEqual(a, a)).toBe(true);
|
|
545
|
+
expect(recordVersionArraysEqual(a, [['game:1', 1], ['game:2', 3]])).toBe(true);
|
|
546
|
+
});
|
|
547
|
+
|
|
548
|
+
it('is order-insensitive (the list_ref SELECT has no ORDER BY)', () => {
|
|
549
|
+
expect(
|
|
550
|
+
recordVersionArraysEqual(
|
|
551
|
+
[['game:1', 1], ['game:2', 3]],
|
|
552
|
+
[['game:2', 3], ['game:1', 1]]
|
|
553
|
+
)
|
|
554
|
+
).toBe(true);
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
it('returns false when a version differs for the same id', () => {
|
|
558
|
+
expect(
|
|
559
|
+
recordVersionArraysEqual([['game:1', 1]], [['game:1', 2]])
|
|
560
|
+
).toBe(false);
|
|
561
|
+
});
|
|
562
|
+
|
|
563
|
+
it('returns false on length mismatch', () => {
|
|
564
|
+
expect(
|
|
565
|
+
recordVersionArraysEqual([['game:1', 1]], [['game:1', 1], ['game:2', 1]])
|
|
566
|
+
).toBe(false);
|
|
567
|
+
});
|
|
568
|
+
|
|
569
|
+
it('returns false when an id is present in one but not the other', () => {
|
|
570
|
+
expect(
|
|
571
|
+
recordVersionArraysEqual([['game:1', 1]], [['game:2', 1]])
|
|
572
|
+
).toBe(false);
|
|
573
|
+
});
|
|
574
|
+
|
|
575
|
+
it('treats two empty arrays as equal', () => {
|
|
576
|
+
expect(recordVersionArraysEqual([], [])).toBe(true);
|
|
577
|
+
});
|
|
578
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { RecordId } from 'surrealdb';
|
|
2
|
-
import { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
1
|
+
import type { RecordId } from 'surrealdb';
|
|
2
|
+
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
3
3
|
import { parseRecordIdString, encodeRecordId } from '../../utils/index';
|
|
4
4
|
|
|
5
5
|
export class ArraySyncer {
|
|
@@ -8,8 +8,8 @@ export class ArraySyncer {
|
|
|
8
8
|
private needsSort = false;
|
|
9
9
|
|
|
10
10
|
constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
|
|
11
|
-
this.remoteArray = remoteArray.
|
|
12
|
-
this.localArray = localArray.
|
|
11
|
+
this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
12
|
+
this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
/**
|
|
@@ -49,7 +49,6 @@ export class ArraySyncer {
|
|
|
49
49
|
this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
|
|
50
50
|
this.needsSort = false;
|
|
51
51
|
}
|
|
52
|
-
console.log('xxxx555', this.localArray, this.remoteArray);
|
|
53
52
|
const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
|
|
54
53
|
return diff;
|
|
55
54
|
}
|
|
@@ -93,10 +92,12 @@ export function diffRecordVersionArray(
|
|
|
93
92
|
return {
|
|
94
93
|
added: added.map((id) => ({
|
|
95
94
|
id: parseRecordIdString(id),
|
|
95
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
96
96
|
version: remoteMap.get(id)!,
|
|
97
97
|
})),
|
|
98
98
|
updated: updated.map((id) => ({
|
|
99
99
|
id: parseRecordIdString(id),
|
|
100
|
+
// oxlint-disable-next-line no-non-null-assertion
|
|
100
101
|
version: remoteMap.get(id)!,
|
|
101
102
|
})),
|
|
102
103
|
removed: removed.map(parseRecordIdString),
|
|
@@ -127,7 +128,7 @@ export function applyRecordVersionDiff(
|
|
|
127
128
|
currentMap.set(encodeRecordId(item.id), item.version);
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
return Array.from(currentMap).
|
|
131
|
+
return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
|
|
131
132
|
}
|
|
132
133
|
|
|
133
134
|
export function createDiffFromDbOp(
|
|
@@ -136,17 +137,14 @@ export function createDiffFromDbOp(
|
|
|
136
137
|
version: number,
|
|
137
138
|
versions?: RecordVersionArray
|
|
138
139
|
): RecordVersionDiff {
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
removed: [],
|
|
148
|
-
};
|
|
149
|
-
}
|
|
140
|
+
const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
|
|
141
|
+
|
|
142
|
+
if (old && old[1] >= version) {
|
|
143
|
+
return {
|
|
144
|
+
added: [],
|
|
145
|
+
updated: [],
|
|
146
|
+
removed: [],
|
|
147
|
+
};
|
|
150
148
|
}
|
|
151
149
|
|
|
152
150
|
if (op === 'CREATE') {
|
|
@@ -169,3 +167,170 @@ export function createDiffFromDbOp(
|
|
|
169
167
|
};
|
|
170
168
|
}
|
|
171
169
|
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Default cadence for the `_00_list_ref_user_<id>` poll fallback. The
|
|
173
|
+
* poll is the safety net for SurrealDB v3's occasionally-dropped LIVE
|
|
174
|
+
* deliveries; 500ms is aggressive enough to feel real-time on the
|
|
175
|
+
* happy path while keeping the per-session query load bounded.
|
|
176
|
+
*/
|
|
177
|
+
export const DEFAULT_LIST_REF_POLL_INTERVAL_MS = 500;
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Build the SurrealQL select that powers both the initial-fetch and
|
|
181
|
+
* the periodic poll of `_00_list_ref[_user_<id>]`. The `parent IS NONE`
|
|
182
|
+
* predicate excludes subquery entries (rows with `parent_rel` set)
|
|
183
|
+
* because the client's `RecordVersionArray` only tracks primary rows;
|
|
184
|
+
* including subquery rows would surface them as spurious "added"
|
|
185
|
+
* diffs every tick.
|
|
186
|
+
*/
|
|
187
|
+
export function buildListRefSelect(table: string): string {
|
|
188
|
+
return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NONE`;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Build the SurrealQL select for a query's SUBQUERY child edges — the
|
|
193
|
+
* mirror of {@link buildListRefSelect}. `.related()` queries register a
|
|
194
|
+
* correlated subquery; the SSP materializes each matched child as a
|
|
195
|
+
* `_00_list_ref` edge tagged with `parent`/`parent_rel` (see
|
|
196
|
+
* `apps/ssp` edge writer). `parent IS NONE` (the primary select) drops
|
|
197
|
+
* these, so their bodies never reach the local cache and a cold-reload
|
|
198
|
+
* re-materialization of the correlated surql yields empty related
|
|
199
|
+
* fields. This `parent IS NOT NONE` variant pulls the child `out`+`version`
|
|
200
|
+
* pairs (any nesting depth) so we can sync their bodies into the local
|
|
201
|
+
* store SEPARATELY from the primary window array.
|
|
202
|
+
*/
|
|
203
|
+
export function buildSubqueryListRefSelect(table: string): string {
|
|
204
|
+
return `SELECT out, version FROM ${table} WHERE in = $in AND parent IS NOT NONE`;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Resolve the effective list-ref poll interval. Negative or zero
|
|
209
|
+
* values fall back to the default — accepting them would either
|
|
210
|
+
* disable polling silently or busy-loop the event loop.
|
|
211
|
+
*/
|
|
212
|
+
export function resolveListRefPollInterval(opt?: number): number {
|
|
213
|
+
if (typeof opt !== 'number' || !Number.isFinite(opt) || opt <= 0) {
|
|
214
|
+
return DEFAULT_LIST_REF_POLL_INTERVAL_MS;
|
|
215
|
+
}
|
|
216
|
+
return opt;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* When the LIVE feed has delivered an event within this window, treat
|
|
221
|
+
* the LIVE subscription as healthy and back the poll off to
|
|
222
|
+
* {@link LIVE_HEALTHY_POLL_INTERVAL_MS}. As soon as LIVE quiets for
|
|
223
|
+
* longer than this, the poll snaps back to the aggressive default.
|
|
224
|
+
*/
|
|
225
|
+
export const LIVE_HEALTHY_COOLDOWN_MS = 5_000;
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Poll interval while LIVE is delivering events. The aggressive
|
|
229
|
+
* default 500ms costs ~120 queries / minute / session — wasted work
|
|
230
|
+
* when LIVE is already covering us. 5s keeps the safety net in place
|
|
231
|
+
* at 1/10th the load.
|
|
232
|
+
*/
|
|
233
|
+
export const LIVE_HEALTHY_POLL_INTERVAL_MS = 5_000;
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Pick the next poll delay based on whether LIVE has been healthy
|
|
237
|
+
* recently. If a LIVE event fired within `cooldownMs`, use the slow
|
|
238
|
+
* (`healthyIntervalMs`) cadence; otherwise the fast (`baseIntervalMs`)
|
|
239
|
+
* cadence. Pure so it's unit-testable; `Sp00kySync.startListRefPoll`
|
|
240
|
+
* calls it from a self-rescheduling timer.
|
|
241
|
+
*
|
|
242
|
+
* The healthy interval is clamped to at least `baseIntervalMs` so
|
|
243
|
+
* configuring an aggressive base (e.g. 100ms) never gets implicitly
|
|
244
|
+
* widened by this helper.
|
|
245
|
+
*
|
|
246
|
+
* @deprecated Superseded by {@link listRefPollDelayMs}, which backs the
|
|
247
|
+
* poll off based on observed change activity (LIVE *or* poll-detected)
|
|
248
|
+
* rather than LIVE liveness alone — the cross-session LIVE-permission gap
|
|
249
|
+
* means LIVE often never fires, so this helper would keep the poll pinned
|
|
250
|
+
* at the aggressive base forever even on a fully idle page. Kept (and
|
|
251
|
+
* tested) for reference.
|
|
252
|
+
*/
|
|
253
|
+
export function nextPollDelayMs(args: {
|
|
254
|
+
now: number;
|
|
255
|
+
lastLiveEventAt: number | null;
|
|
256
|
+
baseIntervalMs: number;
|
|
257
|
+
cooldownMs?: number;
|
|
258
|
+
healthyIntervalMs?: number;
|
|
259
|
+
}): number {
|
|
260
|
+
const {
|
|
261
|
+
now,
|
|
262
|
+
lastLiveEventAt,
|
|
263
|
+
baseIntervalMs,
|
|
264
|
+
cooldownMs = LIVE_HEALTHY_COOLDOWN_MS,
|
|
265
|
+
healthyIntervalMs = LIVE_HEALTHY_POLL_INTERVAL_MS,
|
|
266
|
+
} = args;
|
|
267
|
+
if (lastLiveEventAt === null) return baseIntervalMs;
|
|
268
|
+
const sinceLive = now - lastLiveEventAt;
|
|
269
|
+
if (sinceLive < 0 || sinceLive >= cooldownMs) return baseIntervalMs;
|
|
270
|
+
return Math.max(healthyIntervalMs, baseIntervalMs);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Ceiling for the adaptive list_ref poll backoff. An idle page (no LIVE
|
|
275
|
+
* events, no poll-detected list_ref changes) coasts up to this cadence;
|
|
276
|
+
* the existing healthy-LIVE safety net runs at the same 5s, so this keeps
|
|
277
|
+
* the worst-case catch-up latency for a missed cross-session change at the
|
|
278
|
+
* cadence the codebase already treats as acceptable.
|
|
279
|
+
*/
|
|
280
|
+
export const LIST_REF_POLL_MAX_INTERVAL_MS = 5_000;
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Adaptive poll delay: stay at the responsive `baseIntervalMs` while
|
|
284
|
+
* changes are arriving, and exponentially back off toward `maxIntervalMs`
|
|
285
|
+
* while the `_00_list_ref` is quiet.
|
|
286
|
+
*
|
|
287
|
+
* `idleStreak` is the count of consecutive poll cycles that observed *no*
|
|
288
|
+
* change. `Sp00kySync` resets it to 0 whenever a poll detects a real
|
|
289
|
+
* remoteArray change OR a LIVE event lands, so any activity snaps the poll
|
|
290
|
+
* straight back to `baseIntervalMs`. A streak of 0 (something just
|
|
291
|
+
* happened) → base; otherwise `base * 2^streak` capped at `maxIntervalMs`.
|
|
292
|
+
*
|
|
293
|
+
* This replaces {@link nextPollDelayMs}: the old helper slowed the poll
|
|
294
|
+
* only while LIVE was *delivering*, but the cross-session LIVE-permission
|
|
295
|
+
* gap means LIVE frequently never fires here, so it left a fully idle page
|
|
296
|
+
* polling every `base` ms forever (the "continuous queries while idle"
|
|
297
|
+
* symptom). Backing off on observed idleness instead covers the
|
|
298
|
+
* LIVE-healthy case for free (LIVE applies the change → the next poll sees
|
|
299
|
+
* nothing new → the streak grows → it backs off).
|
|
300
|
+
*/
|
|
301
|
+
export function listRefPollDelayMs(args: {
|
|
302
|
+
idleStreak: number;
|
|
303
|
+
baseIntervalMs: number;
|
|
304
|
+
maxIntervalMs?: number;
|
|
305
|
+
}): number {
|
|
306
|
+
const { idleStreak, baseIntervalMs, maxIntervalMs = LIST_REF_POLL_MAX_INTERVAL_MS } = args;
|
|
307
|
+
const cap = Math.max(baseIntervalMs, maxIntervalMs);
|
|
308
|
+
if (idleStreak <= 0) return baseIntervalMs;
|
|
309
|
+
// 2^streak grows fast; clamp the exponent so it can't overflow on a
|
|
310
|
+
// page left idle for a very long time.
|
|
311
|
+
const exponent = Math.min(idleStreak, 30);
|
|
312
|
+
return Math.min(baseIntervalMs * 2 ** exponent, cap);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Order-insensitive equality for two `RecordVersionArray`s (each a list of
|
|
317
|
+
* `[recordIdString, version]`). The `_00_list_ref` SELECT has no `ORDER
|
|
318
|
+
* BY`, so row order can differ between polls without anything having
|
|
319
|
+
* actually changed — comparing as an id→version map avoids false
|
|
320
|
+
* "changed" verdicts that would defeat the idle backoff. Record ids are
|
|
321
|
+
* unique within a query's list_ref, so a map is a faithful representation.
|
|
322
|
+
*/
|
|
323
|
+
export function recordVersionArraysEqual(
|
|
324
|
+
a: RecordVersionArray,
|
|
325
|
+
b: RecordVersionArray
|
|
326
|
+
): boolean {
|
|
327
|
+
if (a === b) return true;
|
|
328
|
+
if (a.length !== b.length) return false;
|
|
329
|
+
const byId = new Map<string, number>();
|
|
330
|
+
for (const [id, version] of a) byId.set(id, version);
|
|
331
|
+
for (const [id, version] of b) {
|
|
332
|
+
// `get` returns undefined for a missing id, which never === a number.
|
|
333
|
+
if (byId.get(id) !== version) return false;
|
|
334
|
+
}
|
|
335
|
+
return true;
|
|
336
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import type { Level } from 'pino';
|
|
2
|
+
import type { PinoTransmit } from '../types';
|
|
3
|
+
|
|
4
|
+
// Map pino levels to OTEL severity numbers
|
|
5
|
+
// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#severity-fields
|
|
6
|
+
function mapLevelToSeverityNumber(level: string): number {
|
|
7
|
+
switch (level) {
|
|
8
|
+
case 'trace':
|
|
9
|
+
return 1;
|
|
10
|
+
case 'debug':
|
|
11
|
+
return 5;
|
|
12
|
+
case 'info':
|
|
13
|
+
return 9;
|
|
14
|
+
case 'warn':
|
|
15
|
+
return 13;
|
|
16
|
+
case 'error':
|
|
17
|
+
return 17;
|
|
18
|
+
case 'fatal':
|
|
19
|
+
return 21;
|
|
20
|
+
default:
|
|
21
|
+
return 9;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
async function loadOtelModules(otelEndpoint: string) {
|
|
26
|
+
const [{ LoggerProvider, BatchLogRecordProcessor }, { OTLPLogExporter }, { resourceFromAttributes }, { ATTR_SERVICE_NAME }] =
|
|
27
|
+
await Promise.all([
|
|
28
|
+
import('@opentelemetry/sdk-logs'),
|
|
29
|
+
import('@opentelemetry/exporter-logs-otlp-proto'),
|
|
30
|
+
import('@opentelemetry/resources'),
|
|
31
|
+
import('@opentelemetry/semantic-conventions'),
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
const resource = resourceFromAttributes({
|
|
35
|
+
[ATTR_SERVICE_NAME]: 'sp00ky-client',
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const exporter = new OTLPLogExporter({
|
|
39
|
+
url: otelEndpoint,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
const loggerProvider = new LoggerProvider({
|
|
43
|
+
resource,
|
|
44
|
+
processors: [new BatchLogRecordProcessor(exporter)],
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const otelLoggerCache: Record<string, ReturnType<typeof loggerProvider.getLogger>> = {};
|
|
48
|
+
|
|
49
|
+
return (category: string) => {
|
|
50
|
+
if (!otelLoggerCache[category]) {
|
|
51
|
+
otelLoggerCache[category] = loggerProvider.getLogger(category);
|
|
52
|
+
}
|
|
53
|
+
return otelLoggerCache[category];
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Creates a pino browser transmit object that forwards logs to an OpenTelemetry collector.
|
|
59
|
+
*
|
|
60
|
+
* @example
|
|
61
|
+
* ```ts
|
|
62
|
+
* import { createOtelTransmit } from '@spooky-sync/core/otel';
|
|
63
|
+
*
|
|
64
|
+
* new Sp00kyClient({
|
|
65
|
+
* // ...
|
|
66
|
+
* otelTransmit: createOtelTransmit('http://localhost:4318/v1/logs'),
|
|
67
|
+
* });
|
|
68
|
+
* ```
|
|
69
|
+
*/
|
|
70
|
+
export function createOtelTransmit(endpoint: string, level: Level = 'info'): PinoTransmit {
|
|
71
|
+
// Start loading OTel modules eagerly (don't await — we're synchronous)
|
|
72
|
+
const otelReady = loadOtelModules(endpoint);
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
level: level,
|
|
76
|
+
send: (levelLabel: string, logEvent: any) => {
|
|
77
|
+
// oxlint-disable-next-line promise/always-return
|
|
78
|
+
otelReady.then((getOtelLogger) => {
|
|
79
|
+
try {
|
|
80
|
+
const messages = [...logEvent.messages];
|
|
81
|
+
const severityNumber = mapLevelToSeverityNumber(levelLabel);
|
|
82
|
+
|
|
83
|
+
// Construct the message body
|
|
84
|
+
let body = '';
|
|
85
|
+
const lastMsg = messages.pop();
|
|
86
|
+
|
|
87
|
+
if (typeof lastMsg === 'string') {
|
|
88
|
+
body = lastMsg;
|
|
89
|
+
} else if (lastMsg) {
|
|
90
|
+
body = JSON.stringify(lastMsg);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
let category = 'sp00ky-client::unknown';
|
|
94
|
+
|
|
95
|
+
const attributes = {};
|
|
96
|
+
for (const msg of messages) {
|
|
97
|
+
if (typeof msg === 'object') {
|
|
98
|
+
if (msg.Category) {
|
|
99
|
+
category = msg.Category;
|
|
100
|
+
delete msg.Category;
|
|
101
|
+
}
|
|
102
|
+
Object.assign(attributes, msg);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Emit to OTEL SDK
|
|
107
|
+
getOtelLogger(category).emit({
|
|
108
|
+
severityNumber: severityNumber,
|
|
109
|
+
severityText: levelLabel.toUpperCase(),
|
|
110
|
+
body: body,
|
|
111
|
+
attributes: {
|
|
112
|
+
...logEvent.bindings[0],
|
|
113
|
+
...attributes,
|
|
114
|
+
},
|
|
115
|
+
timestamp: new Date(logEvent.ts),
|
|
116
|
+
});
|
|
117
|
+
} catch (e) {
|
|
118
|
+
// oxlint-disable-next-line no-console
|
|
119
|
+
console.warn('Failed to transmit log to OTEL endpoint', e);
|
|
120
|
+
}
|
|
121
|
+
}).catch((e) => {
|
|
122
|
+
// oxlint-disable-next-line no-console
|
|
123
|
+
console.warn('Failed to load OpenTelemetry modules', e);
|
|
124
|
+
});
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|