@spooky-sync/core 0.0.1-canary.84 → 0.0.1-canary.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +37 -2
- package/dist/index.js +106 -8
- package/dist/types.d.ts +38 -1
- package/package.json +3 -3
- package/src/modules/sync/events/index.ts +7 -1
- package/src/modules/sync/scheduler.ts +38 -1
- package/src/modules/sync/sync.ts +92 -3
- package/src/sp00ky.ts +21 -1
- package/src/types.ts +38 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as SyncHealthStatus, C as RunOptions, D as StoreType, E as Sp00kyQueryResultPromise, F as SyncEventSystem, I as EventDefinition, L as EventSystem, M as UpdateOptions, N as UpEvent, O as SyncHealth, P as Logger$1, S as RegistrationTimings, T as Sp00kyQueryResult, _ as QueryTimeToLive, a as MutationCallback, b as RecordVersionArray, c as PersistenceClient, d as QueryConfig, f as QueryConfigRecord, g as QueryStatusCallback, h as QueryStatus, i as MATERIALIZATION_SAMPLE_WINDOW, j as TimingPhase, k as SyncHealthConfig, l as PhaseStat, m as QueryState, n as EventSubscriptionOptions, o as MutationEvent, p as QueryHash, r as Level, s as MutationEventType, t as DebounceOptions, u as PinoTransmit, v as QueryTimings, w as Sp00kyConfig, x as RecordVersionDiff, y as QueryUpdateCallback } from "./types.js";
|
|
2
2
|
import * as surrealdb0 from "surrealdb";
|
|
3
3
|
import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
|
|
4
4
|
import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, GetTable, QueryBuilder, QueryOptions, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
|
|
@@ -586,6 +586,12 @@ interface Sp00kySyncOptions {
|
|
|
586
586
|
* Defaults to `false`.
|
|
587
587
|
*/
|
|
588
588
|
anonymousLiveQueries?: boolean;
|
|
589
|
+
/**
|
|
590
|
+
* Consecutive failed sync rounds before sync health flips to `degraded`.
|
|
591
|
+
* `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
|
|
592
|
+
* Defaults to `3`.
|
|
593
|
+
*/
|
|
594
|
+
degradeAfterConsecutiveFailures?: number;
|
|
589
595
|
}
|
|
590
596
|
/**
|
|
591
597
|
* The main synchronization engine for Sp00ky.
|
|
@@ -627,6 +633,28 @@ declare class Sp00kySync<S extends SchemaStructure> {
|
|
|
627
633
|
get isSyncing(): boolean;
|
|
628
634
|
get pendingMutationCount(): number;
|
|
629
635
|
subscribeToPendingMutations(cb: (count: number) => void): () => void;
|
|
636
|
+
private readonly degradeAfterFailures;
|
|
637
|
+
private consecutiveSyncFailures;
|
|
638
|
+
private syncHealthStatus;
|
|
639
|
+
private lastSyncErrorKind;
|
|
640
|
+
private lastSyncErrorMessage;
|
|
641
|
+
/** Current sync-health snapshot. */
|
|
642
|
+
get syncHealth(): SyncHealth;
|
|
643
|
+
/**
|
|
644
|
+
* Observe sync health. The callback fires immediately with the current
|
|
645
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
646
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
647
|
+
*/
|
|
648
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
|
|
649
|
+
private emitSyncHealth;
|
|
650
|
+
/**
|
|
651
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
652
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
653
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
654
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
655
|
+
* is 0).
|
|
656
|
+
*/
|
|
657
|
+
private recordSyncOutcome;
|
|
630
658
|
constructor(local: LocalDatabaseService, remote: RemoteDatabaseService, cache: CacheModule, dataModule: DataModule<S>, schema: S, logger: Logger$1, options?: Sp00kySyncOptions);
|
|
631
659
|
/**
|
|
632
660
|
* Initializes the synchronization system.
|
|
@@ -1043,6 +1071,13 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
1043
1071
|
* suite can guard the pre-emptive path against regression. */
|
|
1044
1072
|
get liveRetryCount(): number;
|
|
1045
1073
|
subscribeToPendingMutations(cb: (count: number) => void): () => void;
|
|
1074
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
1075
|
+
get syncHealth(): SyncHealth;
|
|
1076
|
+
/**
|
|
1077
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
1078
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
1079
|
+
*/
|
|
1080
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
|
|
1046
1081
|
constructor(config: Sp00kyConfig<S>);
|
|
1047
1082
|
/**
|
|
1048
1083
|
* Setup direct callbacks instead of event subscriptions
|
|
@@ -1130,4 +1165,4 @@ declare function textToHtml(text: string): string;
|
|
|
1130
1165
|
*/
|
|
1131
1166
|
|
|
1132
1167
|
//#endregion
|
|
1133
|
-
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
|
1168
|
+
export { AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, textToHtml };
|
package/dist/index.js
CHANGED
|
@@ -1948,13 +1948,15 @@ function createSyncQueueEventSystem() {
|
|
|
1948
1948
|
const SyncEventTypes = {
|
|
1949
1949
|
QueryUpdated: "SYNC_QUERY_UPDATED",
|
|
1950
1950
|
RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED",
|
|
1951
|
-
MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK"
|
|
1951
|
+
MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK",
|
|
1952
|
+
SyncHealthChanged: "SYNC_HEALTH_CHANGED"
|
|
1952
1953
|
};
|
|
1953
1954
|
function createSyncEventSystem() {
|
|
1954
1955
|
return createEventSystem([
|
|
1955
1956
|
SyncEventTypes.QueryUpdated,
|
|
1956
1957
|
SyncEventTypes.RemoteDataIngested,
|
|
1957
|
-
SyncEventTypes.MutationRolledBack
|
|
1958
|
+
SyncEventTypes.MutationRolledBack,
|
|
1959
|
+
SyncEventTypes.SyncHealthChanged
|
|
1958
1960
|
]);
|
|
1959
1961
|
}
|
|
1960
1962
|
|
|
@@ -2489,13 +2491,14 @@ var SyncEngine = class {
|
|
|
2489
2491
|
var SyncScheduler = class {
|
|
2490
2492
|
isSyncingUp = false;
|
|
2491
2493
|
isSyncingDown = false;
|
|
2492
|
-
constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback) {
|
|
2494
|
+
constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
|
|
2493
2495
|
this.upQueue = upQueue;
|
|
2494
2496
|
this.downQueue = downQueue;
|
|
2495
2497
|
this.onProcessUp = onProcessUp;
|
|
2496
2498
|
this.onProcessDown = onProcessDown;
|
|
2497
2499
|
this.logger = logger;
|
|
2498
2500
|
this.onRollback = onRollback;
|
|
2501
|
+
this.onSyncOutcome = onSyncOutcome;
|
|
2499
2502
|
}
|
|
2500
2503
|
async init() {
|
|
2501
2504
|
await this.upQueue.loadFromDatabase();
|
|
@@ -2520,8 +2523,19 @@ var SyncScheduler = class {
|
|
|
2520
2523
|
async syncUp() {
|
|
2521
2524
|
if (this.isSyncingUp) return;
|
|
2522
2525
|
this.isSyncingUp = true;
|
|
2526
|
+
let processedAny = false;
|
|
2523
2527
|
try {
|
|
2524
|
-
while (this.upQueue.size > 0)
|
|
2528
|
+
while (this.upQueue.size > 0) {
|
|
2529
|
+
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
2530
|
+
processedAny = true;
|
|
2531
|
+
}
|
|
2532
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
2533
|
+
} catch (error) {
|
|
2534
|
+
this.onSyncOutcome?.(false, error);
|
|
2535
|
+
this.logger.debug({
|
|
2536
|
+
error,
|
|
2537
|
+
Category: "sp00ky-client::SyncScheduler::syncUp"
|
|
2538
|
+
}, "syncUp halted on a queue error; item re-queued, will retry on next trigger");
|
|
2525
2539
|
} finally {
|
|
2526
2540
|
this.isSyncingUp = false;
|
|
2527
2541
|
this.syncDown();
|
|
@@ -2534,11 +2548,20 @@ var SyncScheduler = class {
|
|
|
2534
2548
|
if (this.isSyncingDown) return;
|
|
2535
2549
|
if (this.upQueue.size > 0) return;
|
|
2536
2550
|
this.isSyncingDown = true;
|
|
2551
|
+
let processedAny = false;
|
|
2537
2552
|
try {
|
|
2538
2553
|
while (this.downQueue.size > 0) {
|
|
2539
2554
|
if (this.upQueue.size > 0) break;
|
|
2540
2555
|
await this.downQueue.next(this.onProcessDown);
|
|
2556
|
+
processedAny = true;
|
|
2541
2557
|
}
|
|
2558
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
2559
|
+
} catch (error) {
|
|
2560
|
+
this.onSyncOutcome?.(false, error);
|
|
2561
|
+
this.logger.debug({
|
|
2562
|
+
error,
|
|
2563
|
+
Category: "sp00ky-client::SyncScheduler::syncDown"
|
|
2564
|
+
}, "syncDown halted on a queue error; item re-queued, will retry on next trigger");
|
|
2542
2565
|
} finally {
|
|
2543
2566
|
this.isSyncingDown = false;
|
|
2544
2567
|
}
|
|
@@ -2647,6 +2670,68 @@ var Sp00kySync = class {
|
|
|
2647
2670
|
this.upQueue.events.unsubscribe(id2);
|
|
2648
2671
|
};
|
|
2649
2672
|
}
|
|
2673
|
+
degradeAfterFailures;
|
|
2674
|
+
consecutiveSyncFailures = 0;
|
|
2675
|
+
syncHealthStatus = "healthy";
|
|
2676
|
+
lastSyncErrorKind;
|
|
2677
|
+
lastSyncErrorMessage;
|
|
2678
|
+
/** Current sync-health snapshot. */
|
|
2679
|
+
get syncHealth() {
|
|
2680
|
+
return {
|
|
2681
|
+
status: this.syncHealthStatus,
|
|
2682
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
2683
|
+
kind: this.syncHealthStatus === "degraded" ? this.lastSyncErrorKind : void 0,
|
|
2684
|
+
error: this.syncHealthStatus === "degraded" ? this.lastSyncErrorMessage : void 0
|
|
2685
|
+
};
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* Observe sync health. The callback fires immediately with the current
|
|
2689
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
2690
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
2691
|
+
*/
|
|
2692
|
+
subscribeToSyncHealth(cb) {
|
|
2693
|
+
cb(this.syncHealth);
|
|
2694
|
+
const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) => cb(event.payload));
|
|
2695
|
+
return () => this.events.unsubscribe(id);
|
|
2696
|
+
}
|
|
2697
|
+
emitSyncHealth() {
|
|
2698
|
+
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
2699
|
+
}
|
|
2700
|
+
/**
|
|
2701
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
2702
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
2703
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
2704
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
2705
|
+
* is 0).
|
|
2706
|
+
*/
|
|
2707
|
+
recordSyncOutcome(ok, error) {
|
|
2708
|
+
if (this.degradeAfterFailures <= 0) return;
|
|
2709
|
+
if (ok) {
|
|
2710
|
+
if (this.consecutiveSyncFailures === 0) return;
|
|
2711
|
+
this.consecutiveSyncFailures = 0;
|
|
2712
|
+
if (this.syncHealthStatus !== "healthy") {
|
|
2713
|
+
this.syncHealthStatus = "healthy";
|
|
2714
|
+
this.lastSyncErrorKind = void 0;
|
|
2715
|
+
this.lastSyncErrorMessage = void 0;
|
|
2716
|
+
this.logger.info({ Category: "sp00ky-client::Sp00kySync::syncHealth" }, "Sync recovered; health back to healthy");
|
|
2717
|
+
this.emitSyncHealth();
|
|
2718
|
+
}
|
|
2719
|
+
return;
|
|
2720
|
+
}
|
|
2721
|
+
this.consecutiveSyncFailures++;
|
|
2722
|
+
this.lastSyncErrorKind = classifySyncError(error);
|
|
2723
|
+
this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
|
|
2724
|
+
if (this.syncHealthStatus !== "degraded" && this.consecutiveSyncFailures >= this.degradeAfterFailures) {
|
|
2725
|
+
this.syncHealthStatus = "degraded";
|
|
2726
|
+
this.logger.warn({
|
|
2727
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
2728
|
+
kind: this.lastSyncErrorKind,
|
|
2729
|
+
error,
|
|
2730
|
+
Category: "sp00ky-client::Sp00kySync::syncHealth"
|
|
2731
|
+
}, "Sync degraded after sustained failures");
|
|
2732
|
+
this.emitSyncHealth();
|
|
2733
|
+
}
|
|
2734
|
+
}
|
|
2650
2735
|
constructor(local, remote, cache, dataModule, schema, logger, options) {
|
|
2651
2736
|
this.local = local;
|
|
2652
2737
|
this.remote = remote;
|
|
@@ -2657,9 +2742,10 @@ var Sp00kySync = class {
|
|
|
2657
2742
|
this.upQueue = new UpQueue(this.local, this.logger);
|
|
2658
2743
|
this.downQueue = new DownQueue(this.local, this.logger);
|
|
2659
2744
|
this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
|
|
2660
|
-
this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this));
|
|
2745
|
+
this.scheduler = new SyncScheduler(this.upQueue, this.downQueue, this.processUpEvent.bind(this), this.processDownEvent.bind(this), this.logger, this.handleRollback.bind(this), this.recordSyncOutcome.bind(this));
|
|
2661
2746
|
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
2662
2747
|
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
2748
|
+
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
2663
2749
|
}
|
|
2664
2750
|
/**
|
|
2665
2751
|
* Initializes the synchronization system.
|
|
@@ -3343,8 +3429,8 @@ function parseBackendInfo(raw) {
|
|
|
3343
3429
|
|
|
3344
3430
|
//#endregion
|
|
3345
3431
|
//#region src/modules/devtools/index.ts
|
|
3346
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
3347
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
3432
|
+
const CORE_VERSION = "0.0.1-canary.85";
|
|
3433
|
+
const WASM_VERSION = "0.0.1-canary.85";
|
|
3348
3434
|
const SURREAL_VERSION = "3.0.3";
|
|
3349
3435
|
var DevToolsService = class {
|
|
3350
3436
|
eventsHistory = [];
|
|
@@ -5170,6 +5256,17 @@ var Sp00kyClient = class {
|
|
|
5170
5256
|
subscribeToPendingMutations(cb) {
|
|
5171
5257
|
return this.sync.subscribeToPendingMutations(cb);
|
|
5172
5258
|
}
|
|
5259
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
5260
|
+
get syncHealth() {
|
|
5261
|
+
return this.sync.syncHealth;
|
|
5262
|
+
}
|
|
5263
|
+
/**
|
|
5264
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
5265
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
5266
|
+
*/
|
|
5267
|
+
subscribeToSyncHealth(cb) {
|
|
5268
|
+
return this.sync.subscribeToSyncHealth(cb);
|
|
5269
|
+
}
|
|
5173
5270
|
constructor(config) {
|
|
5174
5271
|
this.config = config;
|
|
5175
5272
|
const logger = createLogger(config.logLevel ?? "info", config.otelTransmit);
|
|
@@ -5197,7 +5294,8 @@ var Sp00kyClient = class {
|
|
|
5197
5294
|
this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
|
|
5198
5295
|
this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger, {
|
|
5199
5296
|
refSyncIntervalMs: this.config.refSyncIntervalMs,
|
|
5200
|
-
anonymousLiveQueries: this.config.enableAnonymousLiveQueries
|
|
5297
|
+
anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
|
|
5298
|
+
degradeAfterConsecutiveFailures: this.config.syncHealth === false ? 0 : this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3
|
|
5201
5299
|
});
|
|
5202
5300
|
this.featureFlags = new FeatureFlagModule({
|
|
5203
5301
|
dataModule: this.dataModule,
|
package/dist/types.d.ts
CHANGED
|
@@ -128,6 +128,7 @@ declare const SyncEventTypes: {
|
|
|
128
128
|
readonly QueryUpdated: "SYNC_QUERY_UPDATED";
|
|
129
129
|
readonly RemoteDataIngested: "SYNC_REMOTE_DATA_INGESTED";
|
|
130
130
|
readonly MutationRolledBack: "SYNC_MUTATION_ROLLED_BACK";
|
|
131
|
+
readonly SyncHealthChanged: "SYNC_HEALTH_CHANGED";
|
|
131
132
|
};
|
|
132
133
|
type SyncEventTypeMap = {
|
|
133
134
|
[SyncEventTypes.QueryUpdated]: EventDefinition<typeof SyncEventTypes.QueryUpdated, {
|
|
@@ -146,6 +147,7 @@ type SyncEventTypeMap = {
|
|
|
146
147
|
recordId: string;
|
|
147
148
|
error: string;
|
|
148
149
|
}>;
|
|
150
|
+
[SyncEventTypes.SyncHealthChanged]: EventDefinition<typeof SyncEventTypes.SyncHealthChanged, SyncHealth>;
|
|
149
151
|
};
|
|
150
152
|
type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
151
153
|
//#endregion
|
|
@@ -301,6 +303,41 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
301
303
|
* one-shot but never sync live.
|
|
302
304
|
*/
|
|
303
305
|
enableAnonymousLiveQueries?: boolean;
|
|
306
|
+
/**
|
|
307
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
308
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
309
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
310
|
+
*
|
|
311
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
312
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
313
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
314
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
315
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
316
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
317
|
+
*/
|
|
318
|
+
syncHealth?: SyncHealthConfig | false;
|
|
319
|
+
}
|
|
320
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
321
|
+
interface SyncHealthConfig {
|
|
322
|
+
/**
|
|
323
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
324
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
325
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
326
|
+
* disables degraded reporting entirely.
|
|
327
|
+
*/
|
|
328
|
+
degradeAfterConsecutiveFailures?: number;
|
|
329
|
+
}
|
|
330
|
+
type SyncHealthStatus = 'healthy' | 'degraded';
|
|
331
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
332
|
+
interface SyncHealth {
|
|
333
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
334
|
+
status: SyncHealthStatus;
|
|
335
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
336
|
+
consecutiveFailures: number;
|
|
337
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
338
|
+
kind?: 'network' | 'application';
|
|
339
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
340
|
+
error?: string;
|
|
304
341
|
}
|
|
305
342
|
type QueryHash = string;
|
|
306
343
|
type RecordVersionArray = Array<[string, number]>;
|
|
@@ -501,4 +538,4 @@ interface DebounceOptions {
|
|
|
501
538
|
delay?: number;
|
|
502
539
|
}
|
|
503
540
|
//#endregion
|
|
504
|
-
export {
|
|
541
|
+
export { SyncHealthStatus as A, RunOptions as C, StoreType as D, Sp00kyQueryResultPromise as E, SyncEventSystem as F, EventDefinition as I, EventSystem as L, UpdateOptions as M, UpEvent as N, SyncHealth as O, Logger$1 as P, RegistrationTimings as S, Sp00kyQueryResult as T, QueryTimeToLive as _, MutationCallback as a, RecordVersionArray as b, PersistenceClient as c, QueryConfig as d, QueryConfigRecord as f, QueryStatusCallback as g, QueryStatus as h, MATERIALIZATION_SAMPLE_WINDOW as i, TimingPhase as j, SyncHealthConfig as k, PhaseStat as l, QueryState as m, EventSubscriptionOptions as n, MutationEvent as o, QueryHash as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryTimings as v, Sp00kyConfig as w, RecordVersionDiff as x, QueryUpdateCallback as y };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.85",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -59,8 +59,8 @@
|
|
|
59
59
|
}
|
|
60
60
|
},
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
63
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
62
|
+
"@spooky-sync/query-builder": "0.0.1-canary.85",
|
|
63
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.85",
|
|
64
64
|
"@surrealdb/wasm": "^3.0.3",
|
|
65
65
|
"fast-json-patch": "^3.1.1",
|
|
66
66
|
"loro-crdt": "^1.5.6",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { EventDefinition, EventSystem } from '../../../events/index';
|
|
2
2
|
import { createEventSystem } from '../../../events/index';
|
|
3
|
-
import type { RecordVersionArray } from '../../../types';
|
|
3
|
+
import type { RecordVersionArray, SyncHealth } from '../../../types';
|
|
4
4
|
|
|
5
5
|
export const SyncQueueEventTypes = {
|
|
6
6
|
MutationEnqueued: 'MUTATION_ENQUEUED',
|
|
@@ -37,6 +37,7 @@ export const SyncEventTypes = {
|
|
|
37
37
|
QueryUpdated: 'SYNC_QUERY_UPDATED',
|
|
38
38
|
RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
|
|
39
39
|
MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
|
|
40
|
+
SyncHealthChanged: 'SYNC_HEALTH_CHANGED',
|
|
40
41
|
} as const;
|
|
41
42
|
|
|
42
43
|
export type SyncEventTypeMap = {
|
|
@@ -65,6 +66,10 @@ export type SyncEventTypeMap = {
|
|
|
65
66
|
error: string;
|
|
66
67
|
}
|
|
67
68
|
>;
|
|
69
|
+
[SyncEventTypes.SyncHealthChanged]: EventDefinition<
|
|
70
|
+
typeof SyncEventTypes.SyncHealthChanged,
|
|
71
|
+
SyncHealth
|
|
72
|
+
>;
|
|
68
73
|
};
|
|
69
74
|
|
|
70
75
|
export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
|
|
@@ -74,5 +79,6 @@ export function createSyncEventSystem(): SyncEventSystem {
|
|
|
74
79
|
SyncEventTypes.QueryUpdated,
|
|
75
80
|
SyncEventTypes.RemoteDataIngested,
|
|
76
81
|
SyncEventTypes.MutationRolledBack,
|
|
82
|
+
SyncEventTypes.SyncHealthChanged,
|
|
77
83
|
]);
|
|
78
84
|
}
|
|
@@ -16,7 +16,12 @@ export class SyncScheduler {
|
|
|
16
16
|
private onProcessUp: (event: UpEvent) => Promise<void>,
|
|
17
17
|
private onProcessDown: (event: DownEvent) => Promise<void>,
|
|
18
18
|
private logger: Logger,
|
|
19
|
-
private onRollback?: RollbackCallback
|
|
19
|
+
private onRollback?: RollbackCallback,
|
|
20
|
+
// Reports the outcome of each drained sync round (one syncUp/syncDown pass
|
|
21
|
+
// that actually processed ≥1 item): `ok=true` on a clean drain, `ok=false`
|
|
22
|
+
// with the error when the round halted on a failure. Drives the consumer's
|
|
23
|
+
// sync-health tracking; empty/no-op rounds report nothing.
|
|
24
|
+
private onSyncOutcome?: (ok: boolean, error?: unknown) => void
|
|
20
25
|
) {}
|
|
21
26
|
|
|
22
27
|
async init() {
|
|
@@ -50,10 +55,26 @@ export class SyncScheduler {
|
|
|
50
55
|
async syncUp() {
|
|
51
56
|
if (this.isSyncingUp) return;
|
|
52
57
|
this.isSyncingUp = true;
|
|
58
|
+
let processedAny = false;
|
|
53
59
|
try {
|
|
54
60
|
while (this.upQueue.size > 0) {
|
|
55
61
|
await this.upQueue.next(this.onProcessUp, this.onRollback);
|
|
62
|
+
processedAny = true;
|
|
56
63
|
}
|
|
64
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
this.onSyncOutcome?.(false, error);
|
|
67
|
+
// syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
|
|
68
|
+
// (broadcast synchronously, return value dropped) and is also kicked off
|
|
69
|
+
// via `void this.syncDown()` below. A rejection escaping here therefore
|
|
70
|
+
// surfaces as an *unhandled promise rejection* in the console rather than
|
|
71
|
+
// anything a caller can catch. UpQueue.next already logs the failing item
|
|
72
|
+
// (and re-queues it for retry on the next trigger), so swallow here to
|
|
73
|
+
// keep the failure contained instead of leaking it globally.
|
|
74
|
+
this.logger.debug(
|
|
75
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncUp' },
|
|
76
|
+
'syncUp halted on a queue error; item re-queued, will retry on next trigger'
|
|
77
|
+
);
|
|
57
78
|
} finally {
|
|
58
79
|
this.isSyncingUp = false;
|
|
59
80
|
void this.syncDown();
|
|
@@ -68,11 +89,27 @@ export class SyncScheduler {
|
|
|
68
89
|
if (this.upQueue.size > 0) return;
|
|
69
90
|
|
|
70
91
|
this.isSyncingDown = true;
|
|
92
|
+
let processedAny = false;
|
|
71
93
|
try {
|
|
72
94
|
while (this.downQueue.size > 0) {
|
|
73
95
|
if (this.upQueue.size > 0) break;
|
|
74
96
|
await this.downQueue.next(this.onProcessDown);
|
|
97
|
+
processedAny = true;
|
|
75
98
|
}
|
|
99
|
+
if (processedAny) this.onSyncOutcome?.(true);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
this.onSyncOutcome?.(false, error);
|
|
102
|
+
// Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
|
|
103
|
+
// subscriber (and is also called via `void this.syncDown()`), so a thrown
|
|
104
|
+
// error here becomes an unhandled rejection. The canonical case is a
|
|
105
|
+
// transient remote 500 on `fn::query::register` — DownQueue.next logs it
|
|
106
|
+
// and re-queues the event at the head; we just stop draining this pass and
|
|
107
|
+
// let the next enqueue retry, without spamming the console with an
|
|
108
|
+
// "Uncaught (in promise) ... 500 Internal Server Error".
|
|
109
|
+
this.logger.debug(
|
|
110
|
+
{ error, Category: 'sp00ky-client::SyncScheduler::syncDown' },
|
|
111
|
+
'syncDown halted on a queue error; item re-queued, will retry on next trigger'
|
|
112
|
+
);
|
|
76
113
|
} finally {
|
|
77
114
|
this.isSyncingDown = false;
|
|
78
115
|
}
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
|
|
2
|
-
import type { RecordVersionArray, RecordVersionDiff } from '../../types';
|
|
2
|
+
import type { RecordVersionArray, RecordVersionDiff, SyncHealth, SyncHealthStatus } from '../../types';
|
|
3
3
|
import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
|
|
4
4
|
import type { Logger } from '../../services/logger/index';
|
|
5
5
|
import type { DownEvent, UpEvent} from './queue/index';
|
|
@@ -20,7 +20,7 @@ import { SyncScheduler } from './scheduler';
|
|
|
20
20
|
import type { SchemaStructure } from '@spooky-sync/query-builder';
|
|
21
21
|
import type { CacheModule } from '../cache/index';
|
|
22
22
|
import type { DataModule } from '../data/index';
|
|
23
|
-
import { encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
|
|
23
|
+
import { classifySyncError, encodeRecordId, extractIdPart, extractTablePart, surql } from '../../utils/index';
|
|
24
24
|
import { ANON_USER_ID, DEFAULT_REF_MODE, listRefTableFor, RefMode } from '../ref-tables';
|
|
25
25
|
|
|
26
26
|
/**
|
|
@@ -40,6 +40,12 @@ export interface Sp00kySyncOptions {
|
|
|
40
40
|
* Defaults to `false`.
|
|
41
41
|
*/
|
|
42
42
|
anonymousLiveQueries?: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Consecutive failed sync rounds before sync health flips to `degraded`.
|
|
45
|
+
* `0` disables degraded reporting. See {@link Sp00kyConfig.syncHealth}.
|
|
46
|
+
* Defaults to `3`.
|
|
47
|
+
*/
|
|
48
|
+
degradeAfterConsecutiveFailures?: number;
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
/**
|
|
@@ -154,6 +160,87 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
154
160
|
};
|
|
155
161
|
}
|
|
156
162
|
|
|
163
|
+
// ---- Sync health -------------------------------------------------------
|
|
164
|
+
// `0` disables degraded reporting (config `syncHealth: false`). Resolved
|
|
165
|
+
// from config in Sp00kyClient and passed through the constructor options.
|
|
166
|
+
private readonly degradeAfterFailures: number;
|
|
167
|
+
private consecutiveSyncFailures = 0;
|
|
168
|
+
private syncHealthStatus: SyncHealthStatus = 'healthy';
|
|
169
|
+
private lastSyncErrorKind: 'network' | 'application' | undefined;
|
|
170
|
+
private lastSyncErrorMessage: string | undefined;
|
|
171
|
+
|
|
172
|
+
/** Current sync-health snapshot. */
|
|
173
|
+
get syncHealth(): SyncHealth {
|
|
174
|
+
return {
|
|
175
|
+
status: this.syncHealthStatus,
|
|
176
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
177
|
+
kind: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorKind : undefined,
|
|
178
|
+
error: this.syncHealthStatus === 'degraded' ? this.lastSyncErrorMessage : undefined,
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Observe sync health. The callback fires immediately with the current
|
|
184
|
+
* status and again on every healthy↔degraded transition. Returns an
|
|
185
|
+
* unsubscribe. Mirrors {@link subscribeToPendingMutations}.
|
|
186
|
+
*/
|
|
187
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
|
|
188
|
+
cb(this.syncHealth);
|
|
189
|
+
const id = this.events.subscribe(SyncEventTypes.SyncHealthChanged, (event) =>
|
|
190
|
+
cb(event.payload)
|
|
191
|
+
);
|
|
192
|
+
return () => this.events.unsubscribe(id);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private emitSyncHealth(): void {
|
|
196
|
+
this.events.emit(SyncEventTypes.SyncHealthChanged, this.syncHealth);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Fed by the scheduler once per drained sync round. Individual failures are
|
|
201
|
+
* absorbed by the queue's retry; only a run of `degradeAfterFailures`
|
|
202
|
+
* consecutive failures flips the status to `degraded`, and the next clean
|
|
203
|
+
* round flips it back. No-op when reporting is disabled (`degradeAfterFailures`
|
|
204
|
+
* is 0).
|
|
205
|
+
*/
|
|
206
|
+
private recordSyncOutcome(ok: boolean, error?: unknown): void {
|
|
207
|
+
if (this.degradeAfterFailures <= 0) return;
|
|
208
|
+
if (ok) {
|
|
209
|
+
if (this.consecutiveSyncFailures === 0) return;
|
|
210
|
+
this.consecutiveSyncFailures = 0;
|
|
211
|
+
if (this.syncHealthStatus !== 'healthy') {
|
|
212
|
+
this.syncHealthStatus = 'healthy';
|
|
213
|
+
this.lastSyncErrorKind = undefined;
|
|
214
|
+
this.lastSyncErrorMessage = undefined;
|
|
215
|
+
this.logger.info(
|
|
216
|
+
{ Category: 'sp00ky-client::Sp00kySync::syncHealth' },
|
|
217
|
+
'Sync recovered; health back to healthy'
|
|
218
|
+
);
|
|
219
|
+
this.emitSyncHealth();
|
|
220
|
+
}
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
this.consecutiveSyncFailures++;
|
|
224
|
+
this.lastSyncErrorKind = classifySyncError(error);
|
|
225
|
+
this.lastSyncErrorMessage = error instanceof Error ? error.message : String(error);
|
|
226
|
+
if (
|
|
227
|
+
this.syncHealthStatus !== 'degraded' &&
|
|
228
|
+
this.consecutiveSyncFailures >= this.degradeAfterFailures
|
|
229
|
+
) {
|
|
230
|
+
this.syncHealthStatus = 'degraded';
|
|
231
|
+
this.logger.warn(
|
|
232
|
+
{
|
|
233
|
+
consecutiveFailures: this.consecutiveSyncFailures,
|
|
234
|
+
kind: this.lastSyncErrorKind,
|
|
235
|
+
error,
|
|
236
|
+
Category: 'sp00ky-client::Sp00kySync::syncHealth',
|
|
237
|
+
},
|
|
238
|
+
'Sync degraded after sustained failures'
|
|
239
|
+
);
|
|
240
|
+
this.emitSyncHealth();
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
157
244
|
constructor(
|
|
158
245
|
private local: LocalDatabaseService,
|
|
159
246
|
private remote: RemoteDatabaseService,
|
|
@@ -173,10 +260,12 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
173
260
|
this.processUpEvent.bind(this),
|
|
174
261
|
this.processDownEvent.bind(this),
|
|
175
262
|
this.logger,
|
|
176
|
-
this.handleRollback.bind(this)
|
|
263
|
+
this.handleRollback.bind(this),
|
|
264
|
+
this.recordSyncOutcome.bind(this)
|
|
177
265
|
);
|
|
178
266
|
this.refSyncIntervalMs = resolveListRefPollInterval(options?.refSyncIntervalMs);
|
|
179
267
|
this.anonLiveEnabled = options?.anonymousLiveQueries ?? false;
|
|
268
|
+
this.degradeAfterFailures = Math.max(0, options?.degradeAfterConsecutiveFailures ?? 3);
|
|
180
269
|
}
|
|
181
270
|
|
|
182
271
|
/**
|
package/src/sp00ky.ts
CHANGED
|
@@ -6,7 +6,8 @@ import type {
|
|
|
6
6
|
Sp00kyQueryResultPromise,
|
|
7
7
|
PersistenceClient,
|
|
8
8
|
UpdateOptions,
|
|
9
|
-
RunOptions
|
|
9
|
+
RunOptions,
|
|
10
|
+
SyncHealth} from './types';
|
|
10
11
|
import {
|
|
11
12
|
LocalDatabaseService,
|
|
12
13
|
LocalMigrator,
|
|
@@ -128,6 +129,19 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
128
129
|
return this.sync.subscribeToPendingMutations(cb);
|
|
129
130
|
}
|
|
130
131
|
|
|
132
|
+
/** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
|
|
133
|
+
get syncHealth(): SyncHealth {
|
|
134
|
+
return this.sync.syncHealth;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Observe sync health. Fires immediately with the current status and again
|
|
139
|
+
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
140
|
+
*/
|
|
141
|
+
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
|
|
142
|
+
return this.sync.subscribeToSyncHealth(cb);
|
|
143
|
+
}
|
|
144
|
+
|
|
131
145
|
constructor(private config: Sp00kyConfig<S>) {
|
|
132
146
|
const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
|
|
133
147
|
this.logger = logger.child({ service: 'Sp00kyClient' });
|
|
@@ -206,6 +220,12 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
206
220
|
{
|
|
207
221
|
refSyncIntervalMs: this.config.refSyncIntervalMs,
|
|
208
222
|
anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
|
|
223
|
+
// `syncHealth: false` (or `{ degradeAfterConsecutiveFailures: 0 }`)
|
|
224
|
+
// disables degraded reporting; otherwise default to 3.
|
|
225
|
+
degradeAfterConsecutiveFailures:
|
|
226
|
+
this.config.syncHealth === false
|
|
227
|
+
? 0
|
|
228
|
+
: this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
|
|
209
229
|
}
|
|
210
230
|
);
|
|
211
231
|
|
package/src/types.ts
CHANGED
|
@@ -153,6 +153,44 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
153
153
|
* one-shot but never sync live.
|
|
154
154
|
*/
|
|
155
155
|
enableAnonymousLiveQueries?: boolean;
|
|
156
|
+
/**
|
|
157
|
+
* Surface sustained sync failures as a "degraded" health status that the app
|
|
158
|
+
* can observe via `subscribeToSyncHealth` (or the client-solid
|
|
159
|
+
* `useSyncStatus` hook) to render a "can't reach the server" banner.
|
|
160
|
+
*
|
|
161
|
+
* Individual failures — a transient remote 500 on query registration, a
|
|
162
|
+
* dropped WebSocket, etc. — are always swallowed and retried; they never
|
|
163
|
+
* throw at the app. This only controls when a *run* of consecutive failures
|
|
164
|
+
* is reported. Status flips back to `healthy` on the next successful sync
|
|
165
|
+
* round. Defaults to `{ degradeAfterConsecutiveFailures: 3 }`; pass `false`
|
|
166
|
+
* (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
|
|
167
|
+
*/
|
|
168
|
+
syncHealth?: SyncHealthConfig | false;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
|
|
172
|
+
export interface SyncHealthConfig {
|
|
173
|
+
/**
|
|
174
|
+
* Number of consecutive failed sync rounds (up or down) before the status
|
|
175
|
+
* flips from `healthy` to `degraded`. A single transient failure is absorbed
|
|
176
|
+
* by the retry; only a sustained run trips the banner. Defaults to `3`. `0`
|
|
177
|
+
* disables degraded reporting entirely.
|
|
178
|
+
*/
|
|
179
|
+
degradeAfterConsecutiveFailures?: number;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
export type SyncHealthStatus = 'healthy' | 'degraded';
|
|
183
|
+
|
|
184
|
+
/** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
|
|
185
|
+
export interface SyncHealth {
|
|
186
|
+
/** `'degraded'` once consecutive failures cross the configured threshold. */
|
|
187
|
+
status: SyncHealthStatus;
|
|
188
|
+
/** Consecutive failed sync rounds at the moment of this report. */
|
|
189
|
+
consecutiveFailures: number;
|
|
190
|
+
/** Classification of the most recent failure (only set while `degraded`). */
|
|
191
|
+
kind?: 'network' | 'application';
|
|
192
|
+
/** Message of the most recent failure (only set while `degraded`). */
|
|
193
|
+
error?: string;
|
|
156
194
|
}
|
|
157
195
|
|
|
158
196
|
export type QueryHash = string;
|