@spooky-sync/core 0.0.1-canary.84 → 0.0.1-canary.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { A as UpEvent, C as RunOptions, D as StoreType, E as Sp00kyQueryResultPromise, M as SyncEventSystem, N as EventDefinition, O as TimingPhase, P as EventSystem, S as RegistrationTimings, T as Sp00kyQueryResult, _ as QueryTimeToLive, a as MutationCallback, b as RecordVersionArray, c as PersistenceClient, d as QueryConfig, f as QueryConfigRecord, g as QueryStatusCallback, h as QueryStatus, i as MATERIALIZATION_SAMPLE_WINDOW, j as Logger$1, k as UpdateOptions, l as PhaseStat, m as QueryState, n as EventSubscriptionOptions, o as MutationEvent, p as QueryHash, r as Level, s as MutationEventType, t as DebounceOptions, u as PinoTransmit, v as QueryTimings, w as Sp00kyConfig, x as RecordVersionDiff, y as QueryUpdateCallback } from "./types.js";
1
+ import { A as SyncHealthStatus, C as RunOptions, D as StoreType, E as Sp00kyQueryResultPromise, F as SyncEventSystem, I as EventDefinition, L as EventSystem, M as UpdateOptions, N as UpEvent, O as SyncHealth, P as Logger$1, S as RegistrationTimings, T as Sp00kyQueryResult, _ as QueryTimeToLive, a as MutationCallback, b as RecordVersionArray, c as PersistenceClient, d as QueryConfig, f as QueryConfigRecord, g as QueryStatusCallback, h as QueryStatus, i as MATERIALIZATION_SAMPLE_WINDOW, j as TimingPhase, k as SyncHealthConfig, l as PhaseStat, m as QueryState, n as EventSubscriptionOptions, o as MutationEvent, p as QueryHash, r as Level, s as MutationEventType, t as DebounceOptions, u as PinoTransmit, v as QueryTimings, w as Sp00kyConfig, x as RecordVersionDiff, y as QueryUpdateCallback } from "./types.js";
2
2
  import * as surrealdb0 from "surrealdb";
3
3
  import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
4
4
  import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
@@ -586,6 +586,12 @@ interface Sp00kySyncOptions {
586
586
  * Defaults to `false`.
587
587
  */
588
588
  anonymousLiveQueries?: boolean;
589
+ /**
590
+ * Consecutive failed sync rounds before sync health flips to `degraded`.
591
+ * `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
592
+ * Defaults to `3`.
593
+ */
594
+ degradeAfterConsecutiveFailures?: number;
589
595
  }
590
596
  /**
591
597
  * The main synchronization engine for Sp00ky.
@@ -627,6 +633,39 @@ declare class Sp00kySync<S extends SchemaStructure> {
627
633
  get isSyncing(): boolean;
628
634
  get pendingMutationCount(): number;
629
635
  subscribeToPendingMutations(cb: (count: number) => void): () => void;
636
+ private readonly degradeAfterFailures;
637
+ private consecutiveSyncFailures;
638
+ private syncHealthStatus;
639
+ private lastSyncErrorKind;
640
+ private lastSyncErrorMessage;
641
+ private selfHealTimer;
642
+ private selfHealAttempts;
643
+ private static readonly SELF_HEAL_BASE_MS;
644
+ private static readonly SELF_HEAL_MAX_MS;
645
+ /** Current sync-health snapshot. */
646
+ get syncHealth(): SyncHealth;
647
+ /**
648
+ * Observe sync health. The callback fires immediately with the current
649
+ * status and again on every healthy↔degraded transition. Returns an
650
+ * unsubscribe. Mirrors {@link subscribeToPendingMutations}.
651
+ */
652
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
653
+ private emitSyncHealth;
654
+ /**
655
+ * Fed by the scheduler once per drained sync round. Individual failures are
656
+ * absorbed by the queue's retry; only a run of `degradeAfterFailures`
657
+ * consecutive failures flips the status to `degraded`, and the next clean
658
+ * round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
659
+ * is 0).
660
+ */
661
+ private recordSyncOutcome;
662
+ /**
663
+ * Begin self-heal retries (no-op if already running). Started on the
664
+ * healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
665
+ */
666
+ private startSelfHeal;
667
+ private scheduleSelfHeal;
668
+ private stopSelfHeal;
630
669
  constructor(local: LocalDatabaseService, remote: RemoteDatabaseService, cache: CacheModule, dataModule: DataModule<S>, schema: S, logger: Logger$1, options?: Sp00kySyncOptions);
631
670
  /**
632
671
  * Initializes the synchronization system.
@@ -1043,6 +1082,13 @@ declare class Sp00kyClient<S extends SchemaStructure> {
1043
1082
  * suite can guard the pre-emptive path against regression. */
1044
1083
  get liveRetryCount(): number;
1045
1084
  subscribeToPendingMutations(cb: (count: number) => void): () => void;
1085
+ /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
1086
+ get syncHealth(): SyncHealth;
1087
+ /**
1088
+ * Observe sync health. Fires immediately with the current status and again
1089
+ * on every healthy↔degraded transition. Returns an unsubscribe.
1090
+ */
1091
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
1046
1092
  constructor(config: Sp00kyConfig<S>);
1047
1093
  /**
1048
1094
  * Setup direct callbacks instead of event subscriptions
@@ -1130,4 +1176,4 @@ declare function textToHtml(text: string): string;
1130
1176
  */
1131
1177
 
1132
1178
  //#endregion
1133
- export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
1179
+ export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
package/dist/index.js CHANGED
@@ -1948,13 +1948,15 @@ function createSyncQueueEventSystem() {
1948
1948
  const SyncEventTypes = {
1949
1949
  QueryUpdated: "SYNC_QUERY_UPDATED",
1950
1950
  RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED",
1951
- MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK"
1951
+ MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK",
1952
+ SyncHealthChanged: "SYNC_HEALTH_CHANGED"
1952
1953
  };
1953
1954
  function createSyncEventSystem() {
1954
1955
  return createEventSystem([
1955
1956
  SyncEventTypes.QueryUpdated,
1956
1957
  SyncEventTypes.RemoteDataIngested,
1957
- SyncEventTypes.MutationRolledBack
1958
+ SyncEventTypes.MutationRolledBack,
1959
+ SyncEventTypes.SyncHealthChanged
1958
1960
  ]);
1959
1961
  }
1960
1962
 
@@ -2489,13 +2491,14 @@ var SyncEngine = class {
2489
2491
  var SyncScheduler = class {
2490
2492
  isSyncingUp = false;
2491
2493
  isSyncingDown = false;
2492
- constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback) {
2494
+ constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
2493
2495
  this.upQueue = upQueue;
2494
2496
  this.downQueue = downQueue;
2495
2497
  this.onProcessUp = onProcessUp;
2496
2498
  this.onProcessDown = onProcessDown;
2497
2499
  this.logger = logger;
2498
2500
  this.onRollback = onRollback;
2501
+ this.onSyncOutcome = onSyncOutcome;
2499
2502
  }
2500
2503
  async init() {
2501
2504
  await this.upQueue.loadFromDatabase();
@@ -2520,8 +2523,19 @@ var SyncScheduler = class {
2520
2523
  async syncUp() {
2521
2524
  if (this.isSyncingUp) return;
2522
2525
  this.isSyncingUp = true;
2526
+ let processedAny = false;
2523
2527
  try {
2524
- while (this.upQueue.size > 0) await this.upQueue.next(this.onProcessUp, this.onRollback);
2528
+ while (this.upQueue.size > 0) {
2529
+ await this.upQueue.next(this.onProcessUp, this.onRollback);
2530
+ processedAny = true;
2531
+ }
2532
+ if (processedAny) this.onSyncOutcome?.(true);
2533
+ } catch (error) {
2534
+ this.onSyncOutcome?.(false, error);
2535
+ this.logger.debug({
2536
+ error,
2537
+ Category: "sp00ky-client::SyncScheduler::syncUp"
2538
+ }, "syncUp halted on a queue error; item re-queued, will retry on next trigger");
2525
2539
  } finally {
2526
2540
  this.isSyncingUp = false;
2527
2541
  this.syncDown();
@@ -2534,11 +2548,20 @@ var SyncScheduler = class {
2534
2548
  if (this.isSyncingDown) return;
2535
2549
  if (this.upQueue.size > 0) return;
2536
2550
  this.isSyncingDown = true;
2551
+ let processedAny = false;
2537
2552
  try {
2538
2553
  while (this.downQueue.size > 0) {
2539
2554
  if (this.upQueue.size > 0) break;
2540
2555
  await this.downQueue.next(this.onProcessDown);
2556
+ processedAny = true;
2541
2557
  }
2558
+ if (processedAny) this.onSyncOutcome?.(true);
2559
+ } catch (error) {
2560
+ this.onSyncOutcome?.(false, error);
2561
+ this.logger.debug({
2562
+ error,
2563
+ Category: "sp00ky-client::SyncScheduler::syncDown"
2564
+ }, "syncDown halted on a queue error; item re-queued, will retry on next trigger");
2542
2565
  } finally {
2543
2566
  this.isSyncingDown = false;
2544
2567
  }
@@ -2603,7 +2626,7 @@ function listRefTableFor(mode, userId) {
2603
2626
  * Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
2604
2627
  * @template S The schema structure type.
2605
2628
  */
2606
- var Sp00kySync = class {
2629
+ var Sp00kySync = class Sp00kySync {
2607
2630
  upQueue;
2608
2631
  downQueue;
2609
2632
  isInit = false;
@@ -2647,6 +2670,123 @@ var Sp00kySync = class {
2647
2670
  this.upQueue.events.unsubscribe(id2);
2648
2671
  };
2649
2672
  }
2673
+ degradeAfterFailures;
2674
+ consecutiveSyncFailures = 0;
2675
+ syncHealthStatus = "healthy";
2676
+ lastSyncErrorKind;
2677
+ lastSyncErrorMessage;
2678
+ selfHealTimer = null;
2679
+ selfHealAttempts = 0;
2680
+ static SELF_HEAL_BASE_MS = 2e3;
2681
+ static SELF_HEAL_MAX_MS = 3e4;
2682
+ /** Current sync-health snapshot. */
2683
+ get syncHealth() {
2684
+ return {
2685
+ status: this.syncHealthStatus,
2686
+ consecutiveFailures: this.consecutiveSyncFailures,
2687
+ kind: this.syncHealthStatus === "degraded" ? this.lastSyncErrorKind : void 0,
2688
+ error: this.syncHealthStatus === "degraded" ? this.lastSyncErrorMessage : void 0
2689
+ };
2690
+ }
2691
+ /**
2692
+ * Observe sync health. The callback fires immediately with the current
2693
+ * status and again on every healthy↔degraded transition. Returns an
2694
+ * unsubscribe. Mirrors {@link subscribeToPendingMutations}.
2695
+ */
2696
+ subscribeToSyncHealth(cb) {
2697
+ cb(this.syncHealth);
2698
+ const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) => cb(event.payload));
2699
+ return () => this.events.unsubscribe(id);
2700
+ }
2701
+ emitSyncHealth() {
2702
+ this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
2703
+ }
2704
+ /**
2705
+ * Fed by the scheduler once per drained sync round. Individual failures are
2706
+ * absorbed by the queue's retry; only a run of `degradeAfterFailures`
2707
+ * consecutive failures flips the status to `degraded`, and the next clean
2708
+ * round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
2709
+ * is 0).
2710
+ */
2711
+ recordSyncOutcome(ok, error) {
2712
+ if (this.degradeAfterFailures <= 0) return;
2713
+ if (ok) {
2714
+ if (this.consecutiveSyncFailures === 0) return;
2715
+ this.consecutiveSyncFailures = 0;
2716
+ if (this.syncHealthStatus !== "healthy") {
2717
+ this.syncHealthStatus = "healthy";
2718
+ this.lastSyncErrorKind = void 0;
2719
+ this.lastSyncErrorMessage = void 0;
2720
+ this.stopSelfHeal();
2721
+ this.logger.info({ Category: "sp00ky-client::Sp00kySync::syncHealth" }, "Sync recovered; health back to healthy");
2722
+ this.emitSyncHealth();
2723
+ }
2724
+ return;
2725
+ }
2726
+ this.consecutiveSyncFailures++;
2727
+ this.lastSyncErrorKind = classifySyncError(error);
2728
+ this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
2729
+ if (this.syncHealthStatus !== "degraded" && this.consecutiveSyncFailures >= this.degradeAfterFailures) {
2730
+ this.syncHealthStatus = "degraded";
2731
+ this.logger.warn({
2732
+ consecutiveFailures: this.consecutiveSyncFailures,
2733
+ kind: this.lastSyncErrorKind,
2734
+ error,
2735
+ Category: "sp00ky-client::Sp00kySync::syncHealth"
2736
+ }, "Sync degraded after sustained failures");
2737
+ this.emitSyncHealth();
2738
+ this.startSelfHeal();
2739
+ }
2740
+ }
2741
+ /**
2742
+ * Begin self-heal retries (no-op if already running). Started on the
2743
+ * healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
2744
+ */
2745
+ startSelfHeal() {
2746
+ if (this.selfHealTimer !== null) return;
2747
+ this.selfHealAttempts = 0;
2748
+ this.scheduleSelfHeal();
2749
+ }
2750
+ scheduleSelfHeal() {
2751
+ const delay = Math.min(Sp00kySync.SELF_HEAL_MAX_MS, Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts);
2752
+ this.selfHealTimer = setTimeout(async () => {
2753
+ this.selfHealTimer = null;
2754
+ if (this.syncHealthStatus !== "degraded") return;
2755
+ this.selfHealAttempts++;
2756
+ this.logger.debug({
2757
+ attempt: this.selfHealAttempts,
2758
+ delayMs: delay,
2759
+ Category: "sp00ky-client::Sp00kySync::selfHeal"
2760
+ }, "Self-heal: re-driving sync while degraded");
2761
+ try {
2762
+ if (this.upQueue.size > 0) await this.scheduler.syncUp();
2763
+ else if (this.downQueue.size > 0) await this.scheduler.syncDown();
2764
+ else {
2765
+ const hashes = this.dataModule.getActiveQueryHashes();
2766
+ if (hashes.length > 0) {
2767
+ for (const hash of hashes) this.scheduler.enqueueDownEvent({
2768
+ type: "register",
2769
+ payload: { hash }
2770
+ });
2771
+ await this.scheduler.syncDown();
2772
+ } else {
2773
+ await this.remote.query("RETURN true");
2774
+ this.recordSyncOutcome(true);
2775
+ }
2776
+ }
2777
+ } catch (err) {
2778
+ this.recordSyncOutcome(false, err);
2779
+ }
2780
+ if (this.syncHealthStatus === "degraded") this.scheduleSelfHeal();
2781
+ }, delay);
2782
+ }
2783
+ stopSelfHeal() {
2784
+ if (this.selfHealTimer !== null) {
2785
+ clearTimeout(this.selfHealTimer);
2786
+ this.selfHealTimer = null;
2787
+ }
2788
+ this.selfHealAttempts = 0;
2789
+ }
2650
2790
  constructor(local, remote, cache, dataModule, schema, logger, options) {
2651
2791
  this.local = local;
2652
2792
  this.remote = remote;
@@ -2657,9 +2797,10 @@ var Sp00kySync = class {
2657
2797
  this.upQueue = new UpQueue(this.local, this.logger);
2658
2798
  this.downQueue = new DownQueue(this.local, this.logger);
2659
2799
  this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
2660
- this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this));
2800
+ this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this), this.recordSyncOutcome.bind(this));
2661
2801
  this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
2662
2802
  this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
2803
+ this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
2663
2804
  }
2664
2805
  /**
2665
2806
  * Initializes the synchronization system.
@@ -3343,8 +3484,8 @@ function parseBackendInfo(raw) {
3343
3484
 
3344
3485
  //#endregion
3345
3486
  //#region src/modules/devtools/index.ts
3346
- const CORE_VERSION = "0.0.1-canary.84";
3347
- const WASM_VERSION = "0.0.1-canary.84";
3487
+ const CORE_VERSION = "0.0.1-canary.86";
3488
+ const WASM_VERSION = "0.0.1-canary.86";
3348
3489
  const SURREAL_VERSION = "3.0.3";
3349
3490
  var DevToolsService = class {
3350
3491
  eventsHistory = [];
@@ -5170,6 +5311,17 @@ var Sp00kyClient = class {
5170
5311
  subscribeToPendingMutations(cb) {
5171
5312
  return this.sync.subscribeToPendingMutations(cb);
5172
5313
  }
5314
+ /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
5315
+ get syncHealth() {
5316
+ return this.sync.syncHealth;
5317
+ }
5318
+ /**
5319
+ * Observe sync health. Fires immediately with the current status and again
5320
+ * on every healthy↔degraded transition. Returns an unsubscribe.
5321
+ */
5322
+ subscribeToSyncHealth(cb) {
5323
+ return this.sync.subscribeToSyncHealth(cb);
5324
+ }
5173
5325
  constructor(config) {
5174
5326
  this.config = config;
5175
5327
  const logger = createLogger(config.logLevel ?? "info", config.otelTransmit);
@@ -5197,7 +5349,8 @@ var Sp00kyClient = class {
5197
5349
  this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
5198
5350
  this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger, {
5199
5351
  refSyncIntervalMs: this.config.refSyncIntervalMs,
5200
- anonymousLiveQueries: this.config.enableAnonymousLiveQueries
5352
+ anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
5353
+ degradeAfterConsecutiveFailures: this.config.syncHealth === false ? 0 : this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3
5201
5354
  });
5202
5355
  this.featureFlags = new FeatureFlagModule({
5203
5356
  dataModule: this.dataModule,
package/dist/types.d.ts CHANGED
@@ -128,6 +128,7 @@ declare const SyncEventTypes: {
128
128
  readonly QueryUpdated: "SYNC_QUERY_UPDATED";
129
129
  readonly RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED";
130
130
  readonly MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK";
131
+ readonly SyncHealthChanged: "SYNC_HEALTH_CHANGED";
131
132
  };
132
133
  type SyncEventTypeMap = {
133
134
  [SyncEventTypes.QueryUpdated]: EventDefinition<typeof SyncEventTypes.QueryUpdated, {
@@ -146,6 +147,7 @@ type SyncEventTypeMap = {
146
147
  recordId: string;
147
148
  error: string;
148
149
  }>;
150
+ [SyncEventTypes.SyncHealthChanged]: EventDefinition<typeof SyncEventTypes.SyncHealthChanged, SyncHealth>;
149
151
  };
150
152
  type SyncEventSystem = EventSystem<SyncEventTypeMap>;
151
153
  //#endregion
@@ -301,6 +303,41 @@ interface Sp00kyConfig<S extends SchemaStructure> {
301
303
  * one-shot but never sync live.
302
304
  */
303
305
  enableAnonymousLiveQueries?: boolean;
306
+ /**
307
+ * Surface sustained sync failures as a "degraded" health status that the app
308
+ * can observe via `subscribeToSyncHealth` (or the client-solid
309
+ * `useSyncStatus` hook) to render a "can't reach the server" banner.
310
+ *
311
+ * Individual failures — a transient remote 500 on query registration, a
312
+ * dropped WebSocket, etc. — are always swallowed and retried; they never
313
+ * throw at the app. This only controls when a *run* of consecutive failures
314
+ * is reported. Status flips back to `healthy` on the next successful sync
315
+ * round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
316
+ * (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
317
+ */
318
+ syncHealth?: SyncHealthConfig | false;
319
+ }
320
+ /** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
321
+ interface SyncHealthConfig {
322
+ /**
323
+ * Number of consecutive failed sync rounds (up or down) before the status
324
+ * flips from `healthy` to `degraded`. A single transient failure is absorbed
325
+ * by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
326
+ * disables degraded reporting entirely.
327
+ */
328
+ degradeAfterConsecutiveFailures?: number;
329
+ }
330
+ type SyncHealthStatus = 'healthy' | 'degraded';
331
+ /** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
332
+ interface SyncHealth {
333
+ /** `'degraded'` once consecutive failures cross the configured threshold. */
334
+ status: SyncHealthStatus;
335
+ /** Consecutive failed sync rounds at the moment of this report. */
336
+ consecutiveFailures: number;
337
+ /** Classification of the most recent failure (only set while `degraded`). */
338
+ kind?: 'network' | 'application';
339
+ /** Message of the most recent failure (only set while `degraded`). */
340
+ error?: string;
304
341
  }
305
342
  type QueryHash = string;
306
343
  type RecordVersionArray = Array<[string, number]>;
@@ -501,4 +538,4 @@ interface DebounceOptions {
501
538
  delay?: number;
502
539
  }
503
540
  //#endregion
504
- export { UpEvent as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as M, EventDefinition as N, TimingPhase as O, EventSystem as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, Logger$1 as j, UpdateOptions as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
541
+ export { SyncHealthStatus as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as F, EventDefinition as I, EventSystem as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, Logger$1 as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@spooky-sync/core",
3
- "version": "0.0.1-canary.84",
3
+ "version": "0.0.1-canary.86",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -59,8 +59,8 @@
59
59
  }
60
60
  },
61
61
  "dependencies": {
62
- "@spooky-sync/query-builder": "0.0.1-canary.84",
63
- "@spooky-sync/ssp-wasm": "0.0.1-canary.84",
62
+ "@spooky-sync/query-builder": "0.0.1-canary.86",
63
+ "@spooky-sync/ssp-wasm": "0.0.1-canary.86",
64
64
  "@surrealdb/wasm": "^3.0.3",
65
65
  "fast-json-patch": "^3.1.1",
66
66
  "loro-crdt": "^1.5.6",
@@ -1,6 +1,6 @@
1
1
  import type { EventDefinition, EventSystem } from '../../../events/index';
2
2
  import { createEventSystem } from '../../../events/index';
3
- import type { RecordVersionArray } from '../../../types';
3
+ import type { RecordVersionArray, SyncHealth } from '../../../types';
4
4
 
5
5
  export const SyncQueueEventTypes = {
6
6
  MutationEnqueued: 'MUTATION_ENQUEUED',
@@ -37,6 +37,7 @@ export const SyncEventTypes = {
37
37
  QueryUpdated: 'SYNC_QUERY_UPDATED',
38
38
  RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
39
39
  MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
40
+ SyncHealthChanged: 'SYNC_HEALTH_CHANGED',
40
41
  } as const;
41
42
 
42
43
  export type SyncEventTypeMap = {
@@ -65,6 +66,10 @@ export type SyncEventTypeMap = {
65
66
  error: string;
66
67
  }
67
68
  >;
69
+ [SyncEventTypes.SyncHealthChanged]: EventDefinition<
70
+ typeof SyncEventTypes.SyncHealthChanged,
71
+ SyncHealth
72
+ >;
68
73
  };
69
74
 
70
75
  export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
@@ -74,5 +79,6 @@ export function createSyncEventSystem(): SyncEventSystem {
74
79
  SyncEventTypes.QueryUpdated,
75
80
  SyncEventTypes.RemoteDataIngested,
76
81
  SyncEventTypes.MutationRolledBack,
82
+ SyncEventTypes.SyncHealthChanged,
77
83
  ]);
78
84
  }
@@ -16,7 +16,12 @@ export class SyncScheduler {
16
16
  private onProcessUp: (event: UpEvent) => Promise<void>,
17
17
  private onProcessDown: (event: DownEvent) => Promise<void>,
18
18
  private logger: Logger,
19
- private onRollback?: RollbackCallback
19
+ private onRollback?: RollbackCallback,
20
+ // Reports the outcome of each drained sync round (one syncUp/syncDown pass
21
+ // that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
22
+ // with the error when the round halted on a failure. Drives the consumer's
23
+ // sync-health tracking; empty/no-op rounds report nothing.
24
+ private onSyncOutcome?: (ok: boolean, error?: unknown) => void
20
25
  ) {}
21
26
 
22
27
  async init() {
@@ -50,10 +55,26 @@ export class SyncScheduler {
50
55
  async syncUp() {
51
56
  if (this.isSyncingUp) return;
52
57
  this.isSyncingUp = true;
58
+ let processedAny = false;
53
59
  try {
54
60
  while (this.upQueue.size > 0) {
55
61
  await this.upQueue.next(this.onProcessUp, this.onRollback);
62
+ processedAny = true;
56
63
  }
64
+ if (processedAny) this.onSyncOutcome?.(true);
65
+ } catch (error) {
66
+ this.onSyncOutcome?.(false, error);
67
+ // syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
68
+ // (broadcast synchronously, return value dropped) and is also kicked off
69
+ // via `void this.syncDown()` below. A rejection escaping here therefore
70
+ // surfaces as an *unhandled promise rejection* in the console rather than
71
+ // anything a caller can catch. UpQueue.next already logs the failing item
72
+ // (and re-queues it for retry on the next trigger), so swallow here to
73
+ // keep the failure contained instead of leaking it globally.
74
+ this.logger.debug(
75
+ { error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
76
+ 'syncUp halted on a queue error; item re-queued, will retry on next trigger'
77
+ );
57
78
  } finally {
58
79
  this.isSyncingUp = false;
59
80
  void this.syncDown();
@@ -68,11 +89,27 @@ export class SyncScheduler {
68
89
  if (this.upQueue.size > 0) return;
69
90
 
70
91
  this.isSyncingDown = true;
92
+ let processedAny = false;
71
93
  try {
72
94
  while (this.downQueue.size > 0) {
73
95
  if (this.upQueue.size > 0) break;
74
96
  await this.downQueue.next(this.onProcessDown);
97
+ processedAny = true;
75
98
  }
99
+ if (processedAny) this.onSyncOutcome?.(true);
100
+ } catch (error) {
101
+ this.onSyncOutcome?.(false, error);
102
+ // Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
103
+ // subscriber (and is also called via `void this.syncDown()`), so a thrown
104
+ // error here becomes an unhandled rejection. The canonical case is a
105
+ // transient remote 500 on `fn::query::register` — DownQueue.next logs it
106
+ // and re-queues the event at the head; we just stop draining this pass and
107
+ // let the next enqueue retry, without spamming the console with an
108
+ // "Uncaught (in promise) ... 500 Internal Server Error".
109
+ this.logger.debug(
110
+ { error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
111
+ 'syncDown halted on a queue error; item re-queued, will retry on next trigger'
112
+ );
76
113
  } finally {
77
114
  this.isSyncingDown = false;
78
115
  }
@@ -1,5 +1,5 @@
1
1
  import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
2
- import type { RecordVersionArray, RecordVersionDiff } from '../../types';
2
+ import type { RecordVersionArray, RecordVersionDiff, SyncHealth, SyncHealthStatus } from '../../types';
3
3
  import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
4
4
  import type { Logger } from '../../services/logger/index';
5
5
  import type { DownEvent, UpEvent} from './queue/index';
@@ -20,7 +20,7 @@ import { SyncScheduler } from './scheduler';
20
20
  import type { SchemaStructure } from '@spooky-sync/query-builder';
21
21
  import type { CacheModule } from '../cache/index';
22
22
  import type { DataModule } from '../data/index';
23
- import { encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
23
+ import { classifySyncError, encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
24
24
  import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
25
25
 
26
26
  /**
@@ -40,6 +40,12 @@ export interface Sp00kySyncOptions {
40
40
  * Defaults to `false`.
41
41
  */
42
42
  anonymousLiveQueries?: boolean;
43
+ /**
44
+ * Consecutive failed sync rounds before sync health flips to `degraded`.
45
+ * `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
46
+ * Defaults to `3`.
47
+ */
48
+ degradeAfterConsecutiveFailures?: number;
43
49
  }
44
50
 
45
51
  /**
@@ -154,6 +160,165 @@ export class Sp00kySync<S extends SchemaStructure> {
154
160
  };
155
161
  }
156
162
 
163
+ // ---- Sync health -------------------------------------------------------
164
+ // `0` disables degraded reporting (config `syncHealth: false`). Resolved
165
+ // from config in Sp00kyClient and passed through the constructor options.
166
+ private readonly degradeAfterFailures: number;
167
+ private consecutiveSyncFailures = 0;
168
+ private syncHealthStatus: SyncHealthStatus = 'healthy';
169
+ private lastSyncErrorKind: 'network' | 'application' | undefined;
170
+ private lastSyncErrorMessage: string | undefined;
171
+
172
+ // Self-heal: while degraded, re-drive sync on an exponential backoff so the
173
+ // app recovers on its own — even when the socket never actually dropped (in
174
+ // that case no `connected` event fires, so this re-registration is the ONLY
175
+ // thing that re-probes the server). Started on the degrade transition,
176
+ // cleared on recovery. Capped cadence so a long outage doesn't busy-loop.
177
+ private selfHealTimer: ReturnType<typeof setTimeout> | null = null;
178
+ private selfHealAttempts = 0;
179
+ private static readonly SELF_HEAL_BASE_MS = 2_000;
180
+ private static readonly SELF_HEAL_MAX_MS = 30_000;
181
+
182
+ /** Current sync-health snapshot. */
183
+ get syncHealth(): SyncHealth {
184
+ return {
185
+ status: this.syncHealthStatus,
186
+ consecutiveFailures: this.consecutiveSyncFailures,
187
+ kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
188
+ error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
189
+ };
190
+ }
191
+
192
+ /**
193
+ * Observe sync health. The callback fires immediately with the current
194
+ * status and again on every healthy↔degraded transition. Returns an
195
+ * unsubscribe. Mirrors {@link subscribeToPendingMutations}.
196
+ */
197
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
198
+ cb(this.syncHealth);
199
+ const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
200
+ cb(event.payload)
201
+ );
202
+ return () => this.events.unsubscribe(id);
203
+ }
204
+
205
+ private emitSyncHealth(): void {
206
+ this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
207
+ }
208
+
209
+ /**
210
+ * Fed by the scheduler once per drained sync round. Individual failures are
211
+ * absorbed by the queue's retry; only a run of `degradeAfterFailures`
212
+ * consecutive failures flips the status to `degraded`, and the next clean
213
+ * round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
214
+ * is 0).
215
+ */
216
+ private recordSyncOutcome(ok: boolean, error?: unknown): void {
217
+ if (this.degradeAfterFailures <= 0) return;
218
+ if (ok) {
219
+ if (this.consecutiveSyncFailures === 0) return;
220
+ this.consecutiveSyncFailures = 0;
221
+ if (this.syncHealthStatus !== 'healthy') {
222
+ this.syncHealthStatus = 'healthy';
223
+ this.lastSyncErrorKind = undefined;
224
+ this.lastSyncErrorMessage = undefined;
225
+ this.stopSelfHeal();
226
+ this.logger.info(
227
+ { Category: 'sp00ky-client::Sp00kySync::syncHealth' },
228
+ 'Sync recovered; health back to healthy'
229
+ );
230
+ this.emitSyncHealth();
231
+ }
232
+ return;
233
+ }
234
+ this.consecutiveSyncFailures++;
235
+ this.lastSyncErrorKind = classifySyncError(error);
236
+ this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
237
+ if (
238
+ this.syncHealthStatus !== 'degraded' &&
239
+ this.consecutiveSyncFailures >= this.degradeAfterFailures
240
+ ) {
241
+ this.syncHealthStatus = 'degraded';
242
+ this.logger.warn(
243
+ {
244
+ consecutiveFailures: this.consecutiveSyncFailures,
245
+ kind: this.lastSyncErrorKind,
246
+ error,
247
+ Category: 'sp00ky-client::Sp00kySync::syncHealth',
248
+ },
249
+ 'Sync degraded after sustained failures'
250
+ );
251
+ this.emitSyncHealth();
252
+ this.startSelfHeal();
253
+ }
254
+ }
255
+
256
+ /**
257
+ * Begin self-heal retries (no-op if already running). Started on the
258
+ * healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
259
+ */
260
+ private startSelfHeal(): void {
261
+ if (this.selfHealTimer !== null) return;
262
+ this.selfHealAttempts = 0;
263
+ this.scheduleSelfHeal();
264
+ }
265
+
266
+ private scheduleSelfHeal(): void {
267
+ const delay = Math.min(
268
+ Sp00kySync.SELF_HEAL_MAX_MS,
269
+ Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts
270
+ );
271
+ this.selfHealTimer = setTimeout(async () => {
272
+ this.selfHealTimer = null;
273
+ if (this.syncHealthStatus !== 'degraded') return;
274
+ this.selfHealAttempts++;
275
+ this.logger.debug(
276
+ { attempt: this.selfHealAttempts, delayMs: delay, Category: 'sp00ky-client::Sp00kySync::selfHeal' },
277
+ 'Self-heal: re-driving sync while degraded'
278
+ );
279
+ try {
280
+ // Retry whatever is still queued first; the failing op (register or
281
+ // mutation) was re-queued by the queue, so this re-probes the server
282
+ // and reports the outcome through the scheduler → recordSyncOutcome.
283
+ if (this.upQueue.size > 0) {
284
+ await this.scheduler.syncUp();
285
+ } else if (this.downQueue.size > 0) {
286
+ await this.scheduler.syncDown();
287
+ } else {
288
+ // Nothing queued (e.g. the failing op was rolled back + dropped):
289
+ // re-register active queries — mirroring the reconnect handler — so
290
+ // there's a concrete op whose success flips health. If there are no
291
+ // active queries either, probe connectivity directly.
292
+ const hashes = this.dataModule.getActiveQueryHashes();
293
+ if (hashes.length > 0) {
294
+ for (const hash of hashes) {
295
+ this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
296
+ }
297
+ await this.scheduler.syncDown();
298
+ } else {
299
+ await this.remote.query('RETURN true');
300
+ this.recordSyncOutcome(true);
301
+ }
302
+ }
303
+ } catch (err) {
304
+ // Only the direct connectivity probe can throw here (syncUp/syncDown
305
+ // swallow + self-report); treat a probe failure as another failed round.
306
+ this.recordSyncOutcome(false, err);
307
+ }
308
+ // Keep retrying until recovery. recordSyncOutcome(true) calls stopSelfHeal
309
+ // (clearing any pending timer), so only continue while still degraded.
310
+ if (this.syncHealthStatus === 'degraded') this.scheduleSelfHeal();
311
+ }, delay);
312
+ }
313
+
314
+ private stopSelfHeal(): void {
315
+ if (this.selfHealTimer !== null) {
316
+ clearTimeout(this.selfHealTimer);
317
+ this.selfHealTimer = null;
318
+ }
319
+ this.selfHealAttempts = 0;
320
+ }
321
+
157
322
  constructor(
158
323
  private local: LocalDatabaseService,
159
324
  private remote: RemoteDatabaseService,
@@ -173,10 +338,12 @@ export class Sp00kySync<S extends SchemaStructure> {
173
338
  this.processUpEvent.bind(this),
174
339
  this.processDownEvent.bind(this),
175
340
  this.logger,
176
- this.handleRollback.bind(this)
341
+ this.handleRollback.bind(this),
342
+ this.recordSyncOutcome.bind(this)
177
343
  );
178
344
  this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
179
345
  this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
346
+ this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
180
347
  }
181
348
 
182
349
  /**
package/src/sp00ky.ts CHANGED
@@ -6,7 +6,8 @@ import type {
6
6
  Sp00kyQueryResultPromise,
7
7
  PersistenceClient,
8
8
  UpdateOptions,
9
- RunOptions} from './types';
9
+ RunOptions,
10
+ SyncHealth} from './types';
10
11
  import {
11
12
  LocalDatabaseService,
12
13
  LocalMigrator,
@@ -128,6 +129,19 @@ export class Sp00kyClient<S extends SchemaStructure> {
128
129
  return this.sync.subscribeToPendingMutations(cb);
129
130
  }
130
131
 
132
+ /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
133
+ get syncHealth(): SyncHealth {
134
+ return this.sync.syncHealth;
135
+ }
136
+
137
+ /**
138
+ * Observe sync health. Fires immediately with the current status and again
139
+ * on every healthy↔degraded transition. Returns an unsubscribe.
140
+ */
141
+ subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
142
+ return this.sync.subscribeToSyncHealth(cb);
143
+ }
144
+
131
145
  constructor(private config: Sp00kyConfig<S>) {
132
146
  const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
133
147
  this.logger = logger.child({ service: 'Sp00kyClient' });
@@ -206,6 +220,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
206
220
  {
207
221
  refSyncIntervalMs: this.config.refSyncIntervalMs,
208
222
  anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
223
+ // `syncHealth: false` (or `{ degradeAfterConsecutiveFailures: 0 }`)
224
+ // disables degraded reporting; otherwise default to 3.
225
+ degradeAfterConsecutiveFailures:
226
+ this.config.syncHealth === false
227
+ ? 0
228
+ : this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
209
229
  }
210
230
  );
211
231
 
package/src/types.ts CHANGED
@@ -153,6 +153,44 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
153
153
  * one-shot but never sync live.
154
154
  */
155
155
  enableAnonymousLiveQueries?: boolean;
156
+ /**
157
+ * Surface sustained sync failures as a "degraded" health status that the app
158
+ * can observe via `subscribeToSyncHealth` (or the client-solid
159
+ * `useSyncStatus` hook) to render a "can't reach the server" banner.
160
+ *
161
+ * Individual failures — a transient remote 500 on query registration, a
162
+ * dropped WebSocket, etc. — are always swallowed and retried; they never
163
+ * throw at the app. This only controls when a *run* of consecutive failures
164
+ * is reported. Status flips back to `healthy` on the next successful sync
165
+ * round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
166
+ * (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
167
+ */
168
+ syncHealth?: SyncHealthConfig | false;
169
+ }
170
+
171
+ /** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
172
+ export interface SyncHealthConfig {
173
+ /**
174
+ * Number of consecutive failed sync rounds (up or down) before the status
175
+ * flips from `healthy` to `degraded`. A single transient failure is absorbed
176
+ * by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
177
+ * disables degraded reporting entirely.
178
+ */
179
+ degradeAfterConsecutiveFailures?: number;
180
+ }
181
+
182
+ export type SyncHealthStatus = 'healthy' | 'degraded';
183
+
184
+ /** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
185
+ export interface SyncHealth {
186
+ /** `'degraded'` once consecutive failures cross the configured threshold. */
187
+ status: SyncHealthStatus;
188
+ /** Consecutive failed sync rounds at the moment of this report. */
189
+ consecutiveFailures: number;
190
+ /** Classification of the most recent failure (only set while `degraded`). */
191
+ kind?: 'network' | 'application';
192
+ /** Message of the most recent failure (only set while `degraded`). */
193
+ error?: string;
156
194
  }
157
195
 
158
196
  export type QueryHash = string;