@spooky-sync/core 0.0.1-canary.84 → 0.0.1-canary.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +48 -2
- package/dist/index.js +162 -9
- package/dist/types.d.ts +38 -1
- package/package.json +3 -3
- package/src/modules/sync/events/index.ts +7 -1
- package/src/modules/sync/scheduler.ts +38 -1
- package/src/modules/sync/sync.ts +170 -3
- package/src/sp00ky.ts +21 -1
- package/src/types.ts +38 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as SyncHealthStatus, C as RunOptions, D as StoreType, E as Sp00kyQueryResultPromise, F as SyncEventSystem, I as EventDefinition, L as EventSystem, M as UpdateOptions, N as UpEvent, O as SyncHealth, P as Logger$1, S as RegistrationTimings, T as Sp00kyQueryResult, _ as QueryTimeToLive, a as MutationCallback, b as RecordVersionArray, c as PersistenceClient, d as QueryConfig, f as QueryConfigRecord, g as QueryStatusCallback, h as QueryStatus, i as MATERIALIZATION_SAMPLE_WINDOW, j as TimingPhase, k as SyncHealthConfig, l as PhaseStat, m as QueryState, n as EventSubscriptionOptions, o as MutationEvent, p as QueryHash, r as Level, s as MutationEventType, t as DebounceOptions, u as PinoTransmit, v as QueryTimings, w as Sp00kyConfig, x as RecordVersionDiff, y as QueryUpdateCallback } from "./types.js";
|
|
2
2
|
import * as surrealdb0 from "surrealdb";
|
|
3
3
|
import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
|
|
4
4
|
import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
|
|
@@ -586,6 +586,12 @@ interface Sp00kySyncOptions {
|
|
|
586
586
|
* Defaults to `false`.
|
|
587
587
|
*/
|
|
588
588
|
anonymousLiveQueries?: boolean;
|
|
589
|
+
/**
|
|
590
|
+
* Consecutive failed sync rounds before sync health flips to `degraded`.
|
|
591
|
+
* `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
|
|
592
|
+
* Defaults to `3`.
|
|
593
|
+
*/
|
|
594
|
+
degradeAfterConsecutiveFailures?: number;
|
|
589
595
|
}
|
|
590
596
|
/**
|
|
591
597
|
* The main synchronization engine for Sp00ky.
|
|
@@ -627,6 +633,39 @@ declare class Sp00kySync<S extends SchemaStructure> {
|
|
|
627
633
|
get isSyncing(): boolean;
|
|
628
634
|
get pendingMutationCount(): number;
|
|
629
635
|
subscribeToPendingMutations(cb: (count: number) => void): () => void;
|
|
636
|
+
private readonly degradeAfterFailures;
|
|
637
|
+
private consecutiveSyncFailures;
|
|
638
|
+
private syncHealthStatus;
|
|
639
|
+
private lastSyncErrorKind;
|
|
640
|
+
private lastSyncErrorMessage;
|
|
641
|
+
private selfHealTimer;
|
|
642
|
+
private selfHealAttempts;
|
|
643
|
+
private static readonly SELF_HEAL_BASE_MS;
|
|
644
|
+
private static readonly SELF_HEAL_MAX_MS;
|
|
645
|
+
/** Current sync-health snapshot. */
|
|
646
|
+
get syncHealth(): SyncHealth;
|
|
647
|
+
/**
|
|
648
|
+
* Observe sync health. The callback fires immediately with the current
|
|
649
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
650
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
651
|
+
*/
|
|
652
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
|
|
653
|
+
private emitSyncHealth;
|
|
654
|
+
/**
|
|
655
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
656
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
657
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
658
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
659
|
+
* is 0).
|
|
660
|
+
*/
|
|
661
|
+
private recordSyncOutcome;
|
|
662
|
+
/**
|
|
663
|
+
* Begin self-heal retries (no-op if already running). Started on the
|
|
664
|
+
* healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
|
|
665
|
+
*/
|
|
666
|
+
private startSelfHeal;
|
|
667
|
+
private scheduleSelfHeal;
|
|
668
|
+
private stopSelfHeal;
|
|
630
669
|
constructor(local: LocalDatabaseService, remote: RemoteDatabaseService, cache: CacheModule, dataModule: DataModule<S>, schema: S, logger: Logger$1, options?: Sp00kySyncOptions);
|
|
631
670
|
/**
|
|
632
671
|
* Initializes the synchronization system.
|
|
@@ -1043,6 +1082,13 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
1043
1082
|
* suite can guard the pre-emptive path against regression. */
|
|
1044
1083
|
get liveRetryCount(): number;
|
|
1045
1084
|
subscribeToPendingMutations(cb: (count: number) => void): () => void;
|
|
1085
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
1086
|
+
get syncHealth(): SyncHealth;
|
|
1087
|
+
/**
|
|
1088
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
1089
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
1090
|
+
*/
|
|
1091
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
|
|
1046
1092
|
constructor(config: Sp00kyConfig<S>);
|
|
1047
1093
|
/**
|
|
1048
1094
|
* Setup direct callbacks instead of event subscriptions
|
|
@@ -1130,4 +1176,4 @@ declare function textToHtml(text: string): string;
|
|
|
1130
1176
|
*/
|
|
1131
1177
|
|
|
1132
1178
|
//#endregion
|
|
1133
|
-
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
|
1179
|
+
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
package/dist/index.js
CHANGED
|
@@ -1948,13 +1948,15 @@ function createSyncQueueEventSystem() {
|
|
|
1948
1948
|
const SyncEventTypes = {
|
|
1949
1949
|
QueryUpdated: "SYNC_QUERY_UPDATED",
|
|
1950
1950
|
RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED",
|
|
1951
|
-
MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK"
|
|
1951
|
+
MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK",
|
|
1952
|
+
SyncHealthChanged: "SYNC_HEALTH_CHANGED"
|
|
1952
1953
|
};
|
|
1953
1954
|
function createSyncEventSystem() {
|
|
1954
1955
|
return createEventSystem([
|
|
1955
1956
|
SyncEventTypes.QueryUpdated,
|
|
1956
1957
|
SyncEventTypes.RemoteDataIngested,
|
|
1957
|
-
SyncEventTypes.MutationRolledBack
|
|
1958
|
+
SyncEventTypes.MutationRolledBack,
|
|
1959
|
+
SyncEventTypes.SyncHealthChanged
|
|
1958
1960
|
]);
|
|
1959
1961
|
}
|
|
1960
1962
|
|
|
@@ -2489,13 +2491,14 @@ var SyncEngine = class {
|
|
|
2489
2491
|
var SyncScheduler = class {
|
|
2490
2492
|
isSyncingUp = false;
|
|
2491
2493
|
isSyncingDown = false;
|
|
2492
|
-
constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback) {
|
|
2494
|
+
constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
|
|
2493
2495
|
this.upQueue = upQueue;
|
|
2494
2496
|
this.downQueue = downQueue;
|
|
2495
2497
|
this.onProcessUp = onProcessUp;
|
|
2496
2498
|
this.onProcessDown = onProcessDown;
|
|
2497
2499
|
this.logger = logger;
|
|
2498
2500
|
this.onRollback = onRollback;
|
|
2501
|
+
this.onSyncOutcome = onSyncOutcome;
|
|
2499
2502
|
}
|
|
2500
2503
|
async init() {
|
|
2501
2504
|
await this.upQueue.loadFromDatabase();
|
|
@@ -2520,8 +2523,19 @@ var SyncScheduler = class {
|
|
|
2520
2523
|
async syncUp() {
|
|
2521
2524
|
if (this.isSyncingUp) return;
|
|
2522
2525
|
this.isSyncingUp = true;
|
|
2526
|
+
let processedAny = false;
|
|
2523
2527
|
try {
|
|
2524
|
-
while (this.upQueue.size > 0)
|
|
2528
|
+
while (this.upQueue.size > 0) {
|
|
2529
|
+
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
2530
|
+
processedAny = true;
|
|
2531
|
+
}
|
|
2532
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
2533
|
+
} catch (error) {
|
|
2534
|
+
this.onSyncOutcome?.(false, error);
|
|
2535
|
+
this.logger.debug({
|
|
2536
|
+
error,
|
|
2537
|
+
Category: "sp00ky-client::SyncScheduler::syncUp"
|
|
2538
|
+
}, "syncUp halted on a queue error; item re-queued, will retry on next trigger");
|
|
2525
2539
|
} finally {
|
|
2526
2540
|
this.isSyncingUp = false;
|
|
2527
2541
|
this.syncDown();
|
|
@@ -2534,11 +2548,20 @@ var SyncScheduler = class {
|
|
|
2534
2548
|
if (this.isSyncingDown) return;
|
|
2535
2549
|
if (this.upQueue.size > 0) return;
|
|
2536
2550
|
this.isSyncingDown = true;
|
|
2551
|
+
let processedAny = false;
|
|
2537
2552
|
try {
|
|
2538
2553
|
while (this.downQueue.size > 0) {
|
|
2539
2554
|
if (this.upQueue.size > 0) break;
|
|
2540
2555
|
await this.downQueue.next(this.onProcessDown);
|
|
2556
|
+
processedAny = true;
|
|
2541
2557
|
}
|
|
2558
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
2559
|
+
} catch (error) {
|
|
2560
|
+
this.onSyncOutcome?.(false, error);
|
|
2561
|
+
this.logger.debug({
|
|
2562
|
+
error,
|
|
2563
|
+
Category: "sp00ky-client::SyncScheduler::syncDown"
|
|
2564
|
+
}, "syncDown halted on a queue error; item re-queued, will retry on next trigger");
|
|
2542
2565
|
} finally {
|
|
2543
2566
|
this.isSyncingDown = false;
|
|
2544
2567
|
}
|
|
@@ -2603,7 +2626,7 @@ function listRefTableFor(mode, userId) {
|
|
|
2603
2626
|
* Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
|
|
2604
2627
|
* @template S The schema structure type.
|
|
2605
2628
|
*/
|
|
2606
|
-
var Sp00kySync = class {
|
|
2629
|
+
var Sp00kySync = class Sp00kySync {
|
|
2607
2630
|
upQueue;
|
|
2608
2631
|
downQueue;
|
|
2609
2632
|
isInit = false;
|
|
@@ -2647,6 +2670,123 @@ var Sp00kySync = class {
|
|
|
2647
2670
|
this.upQueue.events.unsubscribe(id2);
|
|
2648
2671
|
};
|
|
2649
2672
|
}
|
|
2673
|
+
degradeAfterFailures;
|
|
2674
|
+
consecutiveSyncFailures = 0;
|
|
2675
|
+
syncHealthStatus = "healthy";
|
|
2676
|
+
lastSyncErrorKind;
|
|
2677
|
+
lastSyncErrorMessage;
|
|
2678
|
+
selfHealTimer = null;
|
|
2679
|
+
selfHealAttempts = 0;
|
|
2680
|
+
static SELF_HEAL_BASE_MS = 2e3;
|
|
2681
|
+
static SELF_HEAL_MAX_MS = 3e4;
|
|
2682
|
+
/** Current sync-health snapshot. */
|
|
2683
|
+
get syncHealth() {
|
|
2684
|
+
return {
|
|
2685
|
+
status: this.syncHealthStatus,
|
|
2686
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
2687
|
+
kind: this.syncHealthStatus === "degraded" ? this.lastSyncErrorKind : void 0,
|
|
2688
|
+
error: this.syncHealthStatus === "degraded" ? this.lastSyncErrorMessage : void 0
|
|
2689
|
+
};
|
|
2690
|
+
}
|
|
2691
|
+
/**
|
|
2692
|
+
* Observe sync health. The callback fires immediately with the current
|
|
2693
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
2694
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
2695
|
+
*/
|
|
2696
|
+
subscribeToSyncHealth(cb) {
|
|
2697
|
+
cb(this.syncHealth);
|
|
2698
|
+
const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) => cb(event.payload));
|
|
2699
|
+
return () => this.events.unsubscribe(id);
|
|
2700
|
+
}
|
|
2701
|
+
emitSyncHealth() {
|
|
2702
|
+
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
2703
|
+
}
|
|
2704
|
+
/**
|
|
2705
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
2706
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
2707
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
2708
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
2709
|
+
* is 0).
|
|
2710
|
+
*/
|
|
2711
|
+
recordSyncOutcome(ok, error) {
|
|
2712
|
+
if (this.degradeAfterFailures <= 0) return;
|
|
2713
|
+
if (ok) {
|
|
2714
|
+
if (this.consecutiveSyncFailures === 0) return;
|
|
2715
|
+
this.consecutiveSyncFailures = 0;
|
|
2716
|
+
if (this.syncHealthStatus !== "healthy") {
|
|
2717
|
+
this.syncHealthStatus = "healthy";
|
|
2718
|
+
this.lastSyncErrorKind = void 0;
|
|
2719
|
+
this.lastSyncErrorMessage = void 0;
|
|
2720
|
+
this.stopSelfHeal();
|
|
2721
|
+
this.logger.info({ Category: "sp00ky-client::Sp00kySync::syncHealth" }, "Sync recovered; health back to healthy");
|
|
2722
|
+
this.emitSyncHealth();
|
|
2723
|
+
}
|
|
2724
|
+
return;
|
|
2725
|
+
}
|
|
2726
|
+
this.consecutiveSyncFailures++;
|
|
2727
|
+
this.lastSyncErrorKind = classifySyncError(error);
|
|
2728
|
+
this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
|
|
2729
|
+
if (this.syncHealthStatus !== "degraded" && this.consecutiveSyncFailures >= this.degradeAfterFailures) {
|
|
2730
|
+
this.syncHealthStatus = "degraded";
|
|
2731
|
+
this.logger.warn({
|
|
2732
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
2733
|
+
kind: this.lastSyncErrorKind,
|
|
2734
|
+
error,
|
|
2735
|
+
Category: "sp00ky-client::Sp00kySync::syncHealth"
|
|
2736
|
+
}, "Sync degraded after sustained failures");
|
|
2737
|
+
this.emitSyncHealth();
|
|
2738
|
+
this.startSelfHeal();
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
/**
|
|
2742
|
+
* Begin self-heal retries (no-op if already running). Started on the
|
|
2743
|
+
* healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
|
|
2744
|
+
*/
|
|
2745
|
+
startSelfHeal() {
|
|
2746
|
+
if (this.selfHealTimer !== null) return;
|
|
2747
|
+
this.selfHealAttempts = 0;
|
|
2748
|
+
this.scheduleSelfHeal();
|
|
2749
|
+
}
|
|
2750
|
+
scheduleSelfHeal() {
|
|
2751
|
+
const delay = Math.min(Sp00kySync.SELF_HEAL_MAX_MS, Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts);
|
|
2752
|
+
this.selfHealTimer = setTimeout(async () => {
|
|
2753
|
+
this.selfHealTimer = null;
|
|
2754
|
+
if (this.syncHealthStatus !== "degraded") return;
|
|
2755
|
+
this.selfHealAttempts++;
|
|
2756
|
+
this.logger.debug({
|
|
2757
|
+
attempt: this.selfHealAttempts,
|
|
2758
|
+
delayMs: delay,
|
|
2759
|
+
Category: "sp00ky-client::Sp00kySync::selfHeal"
|
|
2760
|
+
}, "Self-heal: re-driving sync while degraded");
|
|
2761
|
+
try {
|
|
2762
|
+
if (this.upQueue.size > 0) await this.scheduler.syncUp();
|
|
2763
|
+
else if (this.downQueue.size > 0) await this.scheduler.syncDown();
|
|
2764
|
+
else {
|
|
2765
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
2766
|
+
if (hashes.length > 0) {
|
|
2767
|
+
for (const hash of hashes) this.scheduler.enqueueDownEvent({
|
|
2768
|
+
type: "register",
|
|
2769
|
+
payload: { hash }
|
|
2770
|
+
});
|
|
2771
|
+
await this.scheduler.syncDown();
|
|
2772
|
+
} else {
|
|
2773
|
+
await this.remote.query("RETURN true");
|
|
2774
|
+
this.recordSyncOutcome(true);
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
} catch (err) {
|
|
2778
|
+
this.recordSyncOutcome(false, err);
|
|
2779
|
+
}
|
|
2780
|
+
if (this.syncHealthStatus === "degraded") this.scheduleSelfHeal();
|
|
2781
|
+
}, delay);
|
|
2782
|
+
}
|
|
2783
|
+
stopSelfHeal() {
|
|
2784
|
+
if (this.selfHealTimer !== null) {
|
|
2785
|
+
clearTimeout(this.selfHealTimer);
|
|
2786
|
+
this.selfHealTimer = null;
|
|
2787
|
+
}
|
|
2788
|
+
this.selfHealAttempts = 0;
|
|
2789
|
+
}
|
|
2650
2790
|
constructor(local, remote, cache, dataModule, schema, logger, options) {
|
|
2651
2791
|
this.local = local;
|
|
2652
2792
|
this.remote = remote;
|
|
@@ -2657,9 +2797,10 @@ var Sp00kySync = class {
|
|
|
2657
2797
|
this.upQueue = new UpQueue(this.local, this.logger);
|
|
2658
2798
|
this.downQueue = new DownQueue(this.local, this.logger);
|
|
2659
2799
|
this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
|
|
2660
|
-
this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this));
|
|
2800
|
+
this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this), this.recordSyncOutcome.bind(this));
|
|
2661
2801
|
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
2662
2802
|
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
2803
|
+
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
2663
2804
|
}
|
|
2664
2805
|
/**
|
|
2665
2806
|
* Initializes the synchronization system.
|
|
@@ -3343,8 +3484,8 @@ function parseBackendInfo(raw) {
|
|
|
3343
3484
|
|
|
3344
3485
|
//#endregion
|
|
3345
3486
|
//#region src/modules/devtools/index.ts
|
|
3346
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
3347
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
3487
|
+
const CORE_VERSION = "0.0.1-canary.86";
|
|
3488
|
+
const WASM_VERSION = "0.0.1-canary.86";
|
|
3348
3489
|
const SURREAL_VERSION = "3.0.3";
|
|
3349
3490
|
var DevToolsService = class {
|
|
3350
3491
|
eventsHistory = [];
|
|
@@ -5170,6 +5311,17 @@ var Sp00kyClient = class {
|
|
|
5170
5311
|
subscribeToPendingMutations(cb) {
|
|
5171
5312
|
return this.sync.subscribeToPendingMutations(cb);
|
|
5172
5313
|
}
|
|
5314
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
5315
|
+
get syncHealth() {
|
|
5316
|
+
return this.sync.syncHealth;
|
|
5317
|
+
}
|
|
5318
|
+
/**
|
|
5319
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
5320
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
5321
|
+
*/
|
|
5322
|
+
subscribeToSyncHealth(cb) {
|
|
5323
|
+
return this.sync.subscribeToSyncHealth(cb);
|
|
5324
|
+
}
|
|
5173
5325
|
constructor(config) {
|
|
5174
5326
|
this.config = config;
|
|
5175
5327
|
const logger = createLogger(config.logLevel ?? "info", config.otelTransmit);
|
|
@@ -5197,7 +5349,8 @@ var Sp00kyClient = class {
|
|
|
5197
5349
|
this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
|
|
5198
5350
|
this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger, {
|
|
5199
5351
|
refSyncIntervalMs: this.config.refSyncIntervalMs,
|
|
5200
|
-
anonymousLiveQueries: this.config.enableAnonymousLiveQueries
|
|
5352
|
+
anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
|
|
5353
|
+
degradeAfterConsecutiveFailures: this.config.syncHealth === false ? 0 : this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3
|
|
5201
5354
|
});
|
|
5202
5355
|
this.featureFlags = new FeatureFlagModule({
|
|
5203
5356
|
dataModule: this.dataModule,
|
package/dist/types.d.ts
CHANGED
|
@@ -128,6 +128,7 @@ declare const SyncEventTypes: {
|
|
|
128
128
|
readonly QueryUpdated: "SYNC_QUERY_UPDATED";
|
|
129
129
|
readonly RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED";
|
|
130
130
|
readonly MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK";
|
|
131
|
+
readonly SyncHealthChanged: "SYNC_HEALTH_CHANGED";
|
|
131
132
|
};
|
|
132
133
|
type SyncEventTypeMap = {
|
|
133
134
|
[SyncEventTypes.QueryUpdated]: EventDefinition<typeof SyncEventTypes.QueryUpdated, {
|
|
@@ -146,6 +147,7 @@ type SyncEventTypeMap = {
|
|
|
146
147
|
recordId: string;
|
|
147
148
|
error: string;
|
|
148
149
|
}>;
|
|
150
|
+
[SyncEventTypes.SyncHealthChanged]: EventDefinition<typeof SyncEventTypes.SyncHealthChanged, SyncHealth>;
|
|
149
151
|
};
|
|
150
152
|
type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
151
153
|
//#endregion
|
|
@@ -301,6 +303,41 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
301
303
|
* one-shot but never sync live.
|
|
302
304
|
*/
|
|
303
305
|
enableAnonymousLiveQueries?: boolean;
|
|
306
|
+
/**
|
|
307
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
308
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
309
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
310
|
+
*
|
|
311
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
312
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
313
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
314
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
315
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
316
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
317
|
+
*/
|
|
318
|
+
syncHealth?: SyncHealthConfig | false;
|
|
319
|
+
}
|
|
320
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
321
|
+
interface SyncHealthConfig {
|
|
322
|
+
/**
|
|
323
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
324
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
325
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
326
|
+
* disables degraded reporting entirely.
|
|
327
|
+
*/
|
|
328
|
+
degradeAfterConsecutiveFailures?: number;
|
|
329
|
+
}
|
|
330
|
+
type SyncHealthStatus = 'healthy' | 'degraded';
|
|
331
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
332
|
+
interface SyncHealth {
|
|
333
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
334
|
+
status: SyncHealthStatus;
|
|
335
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
336
|
+
consecutiveFailures: number;
|
|
337
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
338
|
+
kind?: 'network' | 'application';
|
|
339
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
340
|
+
error?: string;
|
|
304
341
|
}
|
|
305
342
|
type QueryHash = string;
|
|
306
343
|
type RecordVersionArray = Array<[string, number]>;
|
|
@@ -501,4 +538,4 @@ interface DebounceOptions {
|
|
|
501
538
|
delay?: number;
|
|
502
539
|
}
|
|
503
540
|
//#endregion
|
|
504
|
-
export {
|
|
541
|
+
export { SyncHealthStatus as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as F, EventDefinition as I, EventSystem as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, Logger$1 as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.86",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -59,8 +59,8 @@
|
|
|
59
59
|
}
|
|
60
60
|
},
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
63
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
62
|
+
"@spooky-sync/query-builder": "0.0.1-canary.86",
|
|
63
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.86",
|
|
64
64
|
"@surrealdb/wasm": "^3.0.3",
|
|
65
65
|
"fast-json-patch": "^3.1.1",
|
|
66
66
|
"loro-crdt": "^1.5.6",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
2
|
import { createEventSystem } from '../../../events/index';
|
|
3
|
-
import type { RecordVersionArray } from '../../../types';
|
|
3
|
+
import type { RecordVersionArray, SyncHealth } from '../../../types';
|
|
4
4
|
|
|
5
5
|
export const SyncQueueEventTypes = {
|
|
6
6
|
MutationEnqueued: 'MUTATION_ENQUEUED',
|
|
@@ -37,6 +37,7 @@ export const SyncEventTypes = {
|
|
|
37
37
|
QueryUpdated: 'SYNC_QUERY_UPDATED',
|
|
38
38
|
RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
|
|
39
39
|
MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
|
|
40
|
+
SyncHealthChanged: 'SYNC_HEALTH_CHANGED',
|
|
40
41
|
} as const;
|
|
41
42
|
|
|
42
43
|
export type SyncEventTypeMap = {
|
|
@@ -65,6 +66,10 @@ export type SyncEventTypeMap = {
|
|
|
65
66
|
error: string;
|
|
66
67
|
}
|
|
67
68
|
>;
|
|
69
|
+
[SyncEventTypes.SyncHealthChanged]: EventDefinition<
|
|
70
|
+
typeof SyncEventTypes.SyncHealthChanged,
|
|
71
|
+
SyncHealth
|
|
72
|
+
>;
|
|
68
73
|
};
|
|
69
74
|
|
|
70
75
|
export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
@@ -74,5 +79,6 @@ export function createSyncEventSystem(): SyncEventSystem {
|
|
|
74
79
|
SyncEventTypes.QueryUpdated,
|
|
75
80
|
SyncEventTypes.RemoteDataIngested,
|
|
76
81
|
SyncEventTypes.MutationRolledBack,
|
|
82
|
+
SyncEventTypes.SyncHealthChanged,
|
|
77
83
|
]);
|
|
78
84
|
}
|
|
@@ -16,7 +16,12 @@ export class SyncScheduler {
|
|
|
16
16
|
private onProcessUp: (event: UpEvent) => Promise<void>,
|
|
17
17
|
private onProcessDown: (event: DownEvent) => Promise<void>,
|
|
18
18
|
private logger: Logger,
|
|
19
|
-
private onRollback?: RollbackCallback
|
|
19
|
+
private onRollback?: RollbackCallback,
|
|
20
|
+
// Reports the outcome of each drained sync round (one syncUp/syncDown pass
|
|
21
|
+
// that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
|
|
22
|
+
// with the error when the round halted on a failure. Drives the consumer's
|
|
23
|
+
// sync-health tracking; empty/no-op rounds report nothing.
|
|
24
|
+
private onSyncOutcome?: (ok: boolean, error?: unknown) => void
|
|
20
25
|
) {}
|
|
21
26
|
|
|
22
27
|
async init() {
|
|
@@ -50,10 +55,26 @@ export class SyncScheduler {
|
|
|
50
55
|
async syncUp() {
|
|
51
56
|
if (this.isSyncingUp) return;
|
|
52
57
|
this.isSyncingUp = true;
|
|
58
|
+
let processedAny = false;
|
|
53
59
|
try {
|
|
54
60
|
while (this.upQueue.size > 0) {
|
|
55
61
|
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
62
|
+
processedAny = true;
|
|
56
63
|
}
|
|
64
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.onSyncOutcome?.(false, error);
|
|
67
|
+
// syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
|
|
68
|
+
// (broadcast synchronously, return value dropped) and is also kicked off
|
|
69
|
+
// via `void this.syncDown()` below. A rejection escaping here therefore
|
|
70
|
+
// surfaces as an *unhandled promise rejection* in the console rather than
|
|
71
|
+
// anything a caller can catch. UpQueue.next already logs the failing item
|
|
72
|
+
// (and re-queues it for retry on the next trigger), so swallow here to
|
|
73
|
+
// keep the failure contained instead of leaking it globally.
|
|
74
|
+
this.logger.debug(
|
|
75
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
|
|
76
|
+
'syncUp halted on a queue error; item re-queued, will retry on next trigger'
|
|
77
|
+
);
|
|
57
78
|
} finally {
|
|
58
79
|
this.isSyncingUp = false;
|
|
59
80
|
void this.syncDown();
|
|
@@ -68,11 +89,27 @@ export class SyncScheduler {
|
|
|
68
89
|
if (this.upQueue.size > 0) return;
|
|
69
90
|
|
|
70
91
|
this.isSyncingDown = true;
|
|
92
|
+
let processedAny = false;
|
|
71
93
|
try {
|
|
72
94
|
while (this.downQueue.size > 0) {
|
|
73
95
|
if (this.upQueue.size > 0) break;
|
|
74
96
|
await this.downQueue.next(this.onProcessDown);
|
|
97
|
+
processedAny = true;
|
|
75
98
|
}
|
|
99
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.onSyncOutcome?.(false, error);
|
|
102
|
+
// Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
|
|
103
|
+
// subscriber (and is also called via `void this.syncDown()`), so a thrown
|
|
104
|
+
// error here becomes an unhandled rejection. The canonical case is a
|
|
105
|
+
// transient remote 500 on `fn::query::register` — DownQueue.next logs it
|
|
106
|
+
// and re-queues the event at the head; we just stop draining this pass and
|
|
107
|
+
// let the next enqueue retry, without spamming the console with an
|
|
108
|
+
// "Uncaught (in promise) ... 500 Internal Server Error".
|
|
109
|
+
this.logger.debug(
|
|
110
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
|
|
111
|
+
'syncDown halted on a queue error; item re-queued, will retry on next trigger'
|
|
112
|
+
);
|
|
76
113
|
} finally {
|
|
77
114
|
this.isSyncingDown = false;
|
|
78
115
|
}
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
-
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
2
|
+
import type { RecordVersionArray, RecordVersionDiff, SyncHealth, SyncHealthStatus } from '../../types';
|
|
3
3
|
import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
|
|
4
4
|
import type { Logger } from '../../services/logger/index';
|
|
5
5
|
import type { DownEvent, UpEvent} from './queue/index';
|
|
@@ -20,7 +20,7 @@ import { SyncScheduler } from './scheduler';
|
|
|
20
20
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
21
21
|
import type { CacheModule } from '../cache/index';
|
|
22
22
|
import type { DataModule } from '../data/index';
|
|
23
|
-
import { encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
|
|
23
|
+
import { classifySyncError, encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
|
|
24
24
|
import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
25
25
|
|
|
26
26
|
/**
|
|
@@ -40,6 +40,12 @@ export interface Sp00kySyncOptions {
|
|
|
40
40
|
* Defaults to `false`.
|
|
41
41
|
*/
|
|
42
42
|
anonymousLiveQueries?: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Consecutive failed sync rounds before sync health flips to `degraded`.
|
|
45
|
+
* `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
|
|
46
|
+
* Defaults to `3`.
|
|
47
|
+
*/
|
|
48
|
+
degradeAfterConsecutiveFailures?: number;
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
/**
|
|
@@ -154,6 +160,165 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
154
160
|
};
|
|
155
161
|
}
|
|
156
162
|
|
|
163
|
+
// ---- Sync health -------------------------------------------------------
|
|
164
|
+
// `0` disables degraded reporting (config `syncHealth: false`). Resolved
|
|
165
|
+
// from config in Sp00kyClient and passed through the constructor options.
|
|
166
|
+
private readonly degradeAfterFailures: number;
|
|
167
|
+
private consecutiveSyncFailures = 0;
|
|
168
|
+
private syncHealthStatus: SyncHealthStatus = 'healthy';
|
|
169
|
+
private lastSyncErrorKind: 'network' | 'application' | undefined;
|
|
170
|
+
private lastSyncErrorMessage: string | undefined;
|
|
171
|
+
|
|
172
|
+
// Self-heal: while degraded, re-drive sync on an exponential backoff so the
|
|
173
|
+
// app recovers on its own — even when the socket never actually dropped (in
|
|
174
|
+
// that case no `connected` event fires, so this re-registration is the ONLY
|
|
175
|
+
// thing that re-probes the server). Started on the degrade transition,
|
|
176
|
+
// cleared on recovery. Capped cadence so a long outage doesn't busy-loop.
|
|
177
|
+
private selfHealTimer: ReturnType<typeof setTimeout> | null = null;
|
|
178
|
+
private selfHealAttempts = 0;
|
|
179
|
+
private static readonly SELF_HEAL_BASE_MS = 2_000;
|
|
180
|
+
private static readonly SELF_HEAL_MAX_MS = 30_000;
|
|
181
|
+
|
|
182
|
+
/** Current sync-health snapshot. */
|
|
183
|
+
get syncHealth(): SyncHealth {
|
|
184
|
+
return {
|
|
185
|
+
status: this.syncHealthStatus,
|
|
186
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
187
|
+
kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
|
|
188
|
+
error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Observe sync health. The callback fires immediately with the current
|
|
194
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
195
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
196
|
+
*/
|
|
197
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
|
|
198
|
+
cb(this.syncHealth);
|
|
199
|
+
const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
|
|
200
|
+
cb(event.payload)
|
|
201
|
+
);
|
|
202
|
+
return () => this.events.unsubscribe(id);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
private emitSyncHealth(): void {
|
|
206
|
+
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
211
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
212
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
213
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
214
|
+
* is 0).
|
|
215
|
+
*/
|
|
216
|
+
private recordSyncOutcome(ok: boolean, error?: unknown): void {
|
|
217
|
+
if (this.degradeAfterFailures <= 0) return;
|
|
218
|
+
if (ok) {
|
|
219
|
+
if (this.consecutiveSyncFailures === 0) return;
|
|
220
|
+
this.consecutiveSyncFailures = 0;
|
|
221
|
+
if (this.syncHealthStatus !== 'healthy') {
|
|
222
|
+
this.syncHealthStatus = 'healthy';
|
|
223
|
+
this.lastSyncErrorKind = undefined;
|
|
224
|
+
this.lastSyncErrorMessage = undefined;
|
|
225
|
+
this.stopSelfHeal();
|
|
226
|
+
this.logger.info(
|
|
227
|
+
{ Category: 'sp00ky-client::Sp00kySync::syncHealth' },
|
|
228
|
+
'Sync recovered; health back to healthy'
|
|
229
|
+
);
|
|
230
|
+
this.emitSyncHealth();
|
|
231
|
+
}
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
this.consecutiveSyncFailures++;
|
|
235
|
+
this.lastSyncErrorKind = classifySyncError(error);
|
|
236
|
+
this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
|
|
237
|
+
if (
|
|
238
|
+
this.syncHealthStatus !== 'degraded' &&
|
|
239
|
+
this.consecutiveSyncFailures >= this.degradeAfterFailures
|
|
240
|
+
) {
|
|
241
|
+
this.syncHealthStatus = 'degraded';
|
|
242
|
+
this.logger.warn(
|
|
243
|
+
{
|
|
244
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
245
|
+
kind: this.lastSyncErrorKind,
|
|
246
|
+
error,
|
|
247
|
+
Category: 'sp00ky-client::Sp00kySync::syncHealth',
|
|
248
|
+
},
|
|
249
|
+
'Sync degraded after sustained failures'
|
|
250
|
+
);
|
|
251
|
+
this.emitSyncHealth();
|
|
252
|
+
this.startSelfHeal();
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Begin self-heal retries (no-op if already running). Started on the
|
|
258
|
+
* healthy→degraded transition; {@link recordSyncOutcome} stops it on recovery.
|
|
259
|
+
*/
|
|
260
|
+
private startSelfHeal(): void {
|
|
261
|
+
if (this.selfHealTimer !== null) return;
|
|
262
|
+
this.selfHealAttempts = 0;
|
|
263
|
+
this.scheduleSelfHeal();
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
private scheduleSelfHeal(): void {
|
|
267
|
+
const delay = Math.min(
|
|
268
|
+
Sp00kySync.SELF_HEAL_MAX_MS,
|
|
269
|
+
Sp00kySync.SELF_HEAL_BASE_MS * 2 ** this.selfHealAttempts
|
|
270
|
+
);
|
|
271
|
+
this.selfHealTimer = setTimeout(async () => {
|
|
272
|
+
this.selfHealTimer = null;
|
|
273
|
+
if (this.syncHealthStatus !== 'degraded') return;
|
|
274
|
+
this.selfHealAttempts++;
|
|
275
|
+
this.logger.debug(
|
|
276
|
+
{ attempt: this.selfHealAttempts, delayMs: delay, Category: 'sp00ky-client::Sp00kySync::selfHeal' },
|
|
277
|
+
'Self-heal: re-driving sync while degraded'
|
|
278
|
+
);
|
|
279
|
+
try {
|
|
280
|
+
// Retry whatever is still queued first; the failing op (register or
|
|
281
|
+
// mutation) was re-queued by the queue, so this re-probes the server
|
|
282
|
+
// and reports the outcome through the scheduler → recordSyncOutcome.
|
|
283
|
+
if (this.upQueue.size > 0) {
|
|
284
|
+
await this.scheduler.syncUp();
|
|
285
|
+
} else if (this.downQueue.size > 0) {
|
|
286
|
+
await this.scheduler.syncDown();
|
|
287
|
+
} else {
|
|
288
|
+
// Nothing queued (e.g. the failing op was rolled back + dropped):
|
|
289
|
+
// re-register active queries — mirroring the reconnect handler — so
|
|
290
|
+
// there's a concrete op whose success flips health. If there are no
|
|
291
|
+
// active queries either, probe connectivity directly.
|
|
292
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
293
|
+
if (hashes.length > 0) {
|
|
294
|
+
for (const hash of hashes) {
|
|
295
|
+
this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
296
|
+
}
|
|
297
|
+
await this.scheduler.syncDown();
|
|
298
|
+
} else {
|
|
299
|
+
await this.remote.query('RETURN true');
|
|
300
|
+
this.recordSyncOutcome(true);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
} catch (err) {
|
|
304
|
+
// Only the direct connectivity probe can throw here (syncUp/syncDown
|
|
305
|
+
// swallow + self-report); treat a probe failure as another failed round.
|
|
306
|
+
this.recordSyncOutcome(false, err);
|
|
307
|
+
}
|
|
308
|
+
// Keep retrying until recovery. recordSyncOutcome(true) calls stopSelfHeal
|
|
309
|
+
// (clearing any pending timer), so only continue while still degraded.
|
|
310
|
+
if (this.syncHealthStatus === 'degraded') this.scheduleSelfHeal();
|
|
311
|
+
}, delay);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
private stopSelfHeal(): void {
|
|
315
|
+
if (this.selfHealTimer !== null) {
|
|
316
|
+
clearTimeout(this.selfHealTimer);
|
|
317
|
+
this.selfHealTimer = null;
|
|
318
|
+
}
|
|
319
|
+
this.selfHealAttempts = 0;
|
|
320
|
+
}
|
|
321
|
+
|
|
157
322
|
constructor(
|
|
158
323
|
private local: LocalDatabaseService,
|
|
159
324
|
private remote: RemoteDatabaseService,
|
|
@@ -173,10 +338,12 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
173
338
|
this.processUpEvent.bind(this),
|
|
174
339
|
this.processDownEvent.bind(this),
|
|
175
340
|
this.logger,
|
|
176
|
-
this.handleRollback.bind(this)
|
|
341
|
+
this.handleRollback.bind(this),
|
|
342
|
+
this.recordSyncOutcome.bind(this)
|
|
177
343
|
);
|
|
178
344
|
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
179
345
|
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
346
|
+
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
180
347
|
}
|
|
181
348
|
|
|
182
349
|
/**
|
package/src/sp00ky.ts
CHANGED
|
@@ -6,7 +6,8 @@ import type {
|
|
|
6
6
|
Sp00kyQueryResultPromise,
|
|
7
7
|
PersistenceClient,
|
|
8
8
|
UpdateOptions,
|
|
9
|
-
RunOptions
|
|
9
|
+
RunOptions,
|
|
10
|
+
SyncHealth} from './types';
|
|
10
11
|
import {
|
|
11
12
|
LocalDatabaseService,
|
|
12
13
|
LocalMigrator,
|
|
@@ -128,6 +129,19 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
128
129
|
return this.sync.subscribeToPendingMutations(cb);
|
|
129
130
|
}
|
|
130
131
|
|
|
132
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
133
|
+
get syncHealth(): SyncHealth {
|
|
134
|
+
return this.sync.syncHealth;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
139
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
140
|
+
*/
|
|
141
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
|
|
142
|
+
return this.sync.subscribeToSyncHealth(cb);
|
|
143
|
+
}
|
|
144
|
+
|
|
131
145
|
constructor(private config: Sp00kyConfig<S>) {
|
|
132
146
|
const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
|
|
133
147
|
this.logger = logger.child({ service: 'Sp00kyClient' });
|
|
@@ -206,6 +220,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
206
220
|
{
|
|
207
221
|
refSyncIntervalMs: this.config.refSyncIntervalMs,
|
|
208
222
|
anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
|
|
223
|
+
// `syncHealth: false` (or `{ degradeAfterConsecutiveFailures: 0 }`)
|
|
224
|
+
// disables degraded reporting; otherwise default to 3.
|
|
225
|
+
degradeAfterConsecutiveFailures:
|
|
226
|
+
this.config.syncHealth === false
|
|
227
|
+
? 0
|
|
228
|
+
: this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
|
|
209
229
|
}
|
|
210
230
|
);
|
|
211
231
|
|
package/src/types.ts
CHANGED
|
@@ -153,6 +153,44 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
153
153
|
* one-shot but never sync live.
|
|
154
154
|
*/
|
|
155
155
|
enableAnonymousLiveQueries?: boolean;
|
|
156
|
+
/**
|
|
157
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
158
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
159
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
160
|
+
*
|
|
161
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
162
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
163
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
164
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
165
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
166
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
167
|
+
*/
|
|
168
|
+
syncHealth?: SyncHealthConfig | false;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
172
|
+
export interface SyncHealthConfig {
|
|
173
|
+
/**
|
|
174
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
175
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
176
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
177
|
+
* disables degraded reporting entirely.
|
|
178
|
+
*/
|
|
179
|
+
degradeAfterConsecutiveFailures?: number;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export type SyncHealthStatus = 'healthy' | 'degraded';
|
|
183
|
+
|
|
184
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
185
|
+
export interface SyncHealth {
|
|
186
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
187
|
+
status: SyncHealthStatus;
|
|
188
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
189
|
+
consecutiveFailures: number;
|
|
190
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
191
|
+
kind?: 'network' | 'application';
|
|
192
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
193
|
+
error?: string;
|
|
156
194
|
}
|
|
157
195
|
|
|
158
196
|
export type QueryHash = string;
|