@spooky-sync/core 0.0.1-canary.163 → 0.0.1-canary.165
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +36 -0
- package/dist/index.js +88 -17
- package/package.json +3 -3
- package/src/modules/sync/sync.ts +25 -2
- package/src/services/blobs/blob-cache.test.ts +18 -0
- package/src/services/blobs/blob-cache.ts +20 -2
- package/src/services/database/connection-supervisor.test.ts +34 -0
- package/src/services/database/connection-supervisor.ts +66 -2
- package/src/sp00ky.ts +19 -14
package/dist/index.d.ts
CHANGED
|
@@ -119,8 +119,12 @@ declare class ConnectionSupervisor {
|
|
|
119
119
|
private disposed;
|
|
120
120
|
private heartbeatTimer;
|
|
121
121
|
private heartbeatInFlight;
|
|
122
|
+
/** Consecutive failed probes. See {@link FAILURES_BEFORE_TEARDOWN}. */
|
|
123
|
+
private heartbeatFailures;
|
|
122
124
|
private reviveTimer;
|
|
123
125
|
private reviveAttempts;
|
|
126
|
+
/** Timestamp of the last wake-triggered probe, for rate limiting. */
|
|
127
|
+
private lastWakeProbeAt;
|
|
124
128
|
private reviving;
|
|
125
129
|
/**
|
|
126
130
|
* Set while the browser reports itself offline. Retrying a socket against a
|
|
@@ -129,6 +133,23 @@ declare class ConnectionSupervisor {
|
|
|
129
133
|
private suspended;
|
|
130
134
|
private teardown;
|
|
131
135
|
private static readonly REVIVE_BASE_MS;
|
|
136
|
+
/**
|
|
137
|
+
* How many consecutive heartbeat failures it takes to tear the socket down.
|
|
138
|
+
*
|
|
139
|
+
* The probe rides the same serialized queue as every other RPC (deliberately
|
|
140
|
+
* — see {@link beat}), which means it cannot distinguish a WEDGED queue from
|
|
141
|
+
* a merely BUSY one. A single slow window (a large sync burst, one heavy
|
|
142
|
+
* app query) used to be enough to force-close a perfectly healthy socket,
|
|
143
|
+
* and the resulting reconnect re-registered every active query about a
|
|
144
|
+
* second later. That self-inflicted teardown manufactured the very reconnect
|
|
145
|
+
* storms this class exists to survive. A genuinely dead socket still fails
|
|
146
|
+
* every probe, so it is torn down one interval later than before.
|
|
147
|
+
*/
|
|
148
|
+
private static readonly FAILURES_BEFORE_TEARDOWN;
|
|
149
|
+
/** Retry delay after an inconclusive (first) heartbeat failure. */
|
|
150
|
+
private static readonly HEARTBEAT_RETRY_MS;
|
|
151
|
+
/** Floor between probes triggered by wake events (tab focus, pageshow). */
|
|
152
|
+
private static readonly WAKE_PROBE_MIN_INTERVAL_MS;
|
|
132
153
|
constructor(remote: RemoteDatabaseService, logger: Logger$1, config?: Required<ReconnectConfig>);
|
|
133
154
|
/** Latest observed transport state. */
|
|
134
155
|
get connection(): ConnectionState;
|
|
@@ -1068,6 +1089,14 @@ declare class Sp00kySync<S extends SchemaStructure> {
|
|
|
1068
1089
|
* initial connect. See {@link subscribeToReconnect}.
|
|
1069
1090
|
*/
|
|
1070
1091
|
private needsResubscribe;
|
|
1092
|
+
/** When the last reconnect-driven full refetch ran, for burst coalescing. */
|
|
1093
|
+
private lastReconnectRefetchAt;
|
|
1094
|
+
/**
|
|
1095
|
+
* Minimum gap between reconnect-driven full refetches. Long enough to absorb
|
|
1096
|
+
* a flapping socket (the SDK reconnect ladder starts at 1s), short enough
|
|
1097
|
+
* that a genuine drop minutes later still refetches.
|
|
1098
|
+
*/
|
|
1099
|
+
private static readonly RECONNECT_REFETCH_COOLDOWN_MS;
|
|
1071
1100
|
events: SyncEventSystem;
|
|
1072
1101
|
private currentUserId;
|
|
1073
1102
|
private tabRole;
|
|
@@ -1883,6 +1912,13 @@ declare class BlobCache {
|
|
|
1883
1912
|
private readonly inflight;
|
|
1884
1913
|
private flushTimer;
|
|
1885
1914
|
private readonly onPageHide;
|
|
1915
|
+
/**
|
|
1916
|
+
* Resolves once the manifest has been reconciled against disk. Reads await
|
|
1917
|
+
* it, so `start()` does NOT have to be awaited on the boot path — blocking
|
|
1918
|
+
* boot on an OPFS directory walk delayed the WebSocket connect (and with it
|
|
1919
|
+
* the connection supervisor) for no benefit.
|
|
1920
|
+
*/
|
|
1921
|
+
private ready;
|
|
1886
1922
|
private hits;
|
|
1887
1923
|
private misses;
|
|
1888
1924
|
private evictedEntries;
|
package/dist/index.js
CHANGED
|
@@ -1081,8 +1081,12 @@ var ConnectionSupervisor = class ConnectionSupervisor {
|
|
|
1081
1081
|
disposed = false;
|
|
1082
1082
|
heartbeatTimer = null;
|
|
1083
1083
|
heartbeatInFlight = false;
|
|
1084
|
+
/** Consecutive failed probes. See {@link FAILURES_BEFORE_TEARDOWN}. */
|
|
1085
|
+
heartbeatFailures = 0;
|
|
1084
1086
|
reviveTimer = null;
|
|
1085
1087
|
reviveAttempts = 0;
|
|
1088
|
+
/** Timestamp of the last wake-triggered probe, for rate limiting. */
|
|
1089
|
+
lastWakeProbeAt = 0;
|
|
1086
1090
|
reviving = false;
|
|
1087
1091
|
/**
|
|
1088
1092
|
* Set while the browser reports itself offline. Retrying a socket against a
|
|
@@ -1091,6 +1095,23 @@ var ConnectionSupervisor = class ConnectionSupervisor {
|
|
|
1091
1095
|
suspended = false;
|
|
1092
1096
|
teardown = [];
|
|
1093
1097
|
static REVIVE_BASE_MS = 1e3;
|
|
1098
|
+
/**
|
|
1099
|
+
* How many consecutive heartbeat failures it takes to tear the socket down.
|
|
1100
|
+
*
|
|
1101
|
+
* The probe rides the same serialized queue as every other RPC (deliberately
|
|
1102
|
+
* — see {@link beat}), which means it cannot distinguish a WEDGED queue from
|
|
1103
|
+
* a merely BUSY one. A single slow window (a large sync burst, one heavy
|
|
1104
|
+
* app query) used to be enough to force-close a perfectly healthy socket,
|
|
1105
|
+
* and the resulting reconnect re-registered every active query about a
|
|
1106
|
+
* second later. That self-inflicted teardown manufactured the very reconnect
|
|
1107
|
+
* storms this class exists to survive. A genuinely dead socket still fails
|
|
1108
|
+
* every probe, so it is torn down one interval later than before.
|
|
1109
|
+
*/
|
|
1110
|
+
static FAILURES_BEFORE_TEARDOWN = 2;
|
|
1111
|
+
/** Retry delay after an inconclusive (first) heartbeat failure. */
|
|
1112
|
+
static HEARTBEAT_RETRY_MS = 5e3;
|
|
1113
|
+
/** Floor between probes triggered by wake events (tab focus, pageshow). */
|
|
1114
|
+
static WAKE_PROBE_MIN_INTERVAL_MS = 1e4;
|
|
1094
1115
|
constructor(remote, logger, config) {
|
|
1095
1116
|
this.remote = remote;
|
|
1096
1117
|
this.logger = logger.child({ service: "ConnectionSupervisor" });
|
|
@@ -1244,12 +1265,26 @@ var ConnectionSupervisor = class ConnectionSupervisor {
|
|
|
1244
1265
|
this.heartbeatInFlight = true;
|
|
1245
1266
|
try {
|
|
1246
1267
|
await withTimeout(this.remote.query("RETURN true"), this.config.heartbeatTimeoutMs, `Heartbeat timed out after ${this.config.heartbeatTimeoutMs}ms`);
|
|
1268
|
+
this.heartbeatFailures = 0;
|
|
1247
1269
|
this.startHeartbeat();
|
|
1248
1270
|
} catch (err) {
|
|
1271
|
+
this.heartbeatFailures++;
|
|
1272
|
+
if (this.heartbeatFailures < ConnectionSupervisor.FAILURES_BEFORE_TEARDOWN) {
|
|
1273
|
+
this.logger.debug({
|
|
1274
|
+
err,
|
|
1275
|
+
failures: this.heartbeatFailures,
|
|
1276
|
+
Category: "sp00ky-client::ConnectionSupervisor::heartbeat"
|
|
1277
|
+
}, "Heartbeat failed; re-probing before tearing the socket down");
|
|
1278
|
+
this.stopHeartbeat();
|
|
1279
|
+
if (!this.disposed && !this.suspended) this.heartbeatTimer = setTimeout(() => void this.beat(), Math.min(ConnectionSupervisor.HEARTBEAT_RETRY_MS, this.config.heartbeatIntervalMs));
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1249
1282
|
this.logger.warn({
|
|
1250
1283
|
err,
|
|
1284
|
+
failures: this.heartbeatFailures,
|
|
1251
1285
|
Category: "sp00ky-client::ConnectionSupervisor::heartbeat"
|
|
1252
|
-
}, "Heartbeat failed; tearing the socket down to force a reconnect");
|
|
1286
|
+
}, "Heartbeat failed repeatedly; tearing the socket down to force a reconnect");
|
|
1287
|
+
this.heartbeatFailures = 0;
|
|
1253
1288
|
await this.remote.forceClose();
|
|
1254
1289
|
if (this.remote.getStatus() !== "connected") this.scheduleRevive();
|
|
1255
1290
|
} finally {
|
|
@@ -1297,6 +1332,9 @@ var ConnectionSupervisor = class ConnectionSupervisor {
|
|
|
1297
1332
|
*/
|
|
1298
1333
|
wake(reason) {
|
|
1299
1334
|
if (this.disposed || this.suspended) return;
|
|
1335
|
+
const now = Date.now();
|
|
1336
|
+
if (this.remote.getStatus() === "connected" && now - this.lastWakeProbeAt < ConnectionSupervisor.WAKE_PROBE_MIN_INTERVAL_MS) return;
|
|
1337
|
+
this.lastWakeProbeAt = now;
|
|
1300
1338
|
this.logger.debug({
|
|
1301
1339
|
reason,
|
|
1302
1340
|
state: this.state,
|
|
@@ -5277,6 +5315,14 @@ var Sp00kySync = class Sp00kySync {
|
|
|
5277
5315
|
* initial connect. See {@link subscribeToReconnect}.
|
|
5278
5316
|
*/
|
|
5279
5317
|
needsResubscribe = false;
|
|
5318
|
+
/** When the last reconnect-driven full refetch ran, for burst coalescing. */
|
|
5319
|
+
lastReconnectRefetchAt = 0;
|
|
5320
|
+
/**
|
|
5321
|
+
* Minimum gap between reconnect-driven full refetches. Long enough to absorb
|
|
5322
|
+
* a flapping socket (the SDK reconnect ladder starts at 1s), short enough
|
|
5323
|
+
* that a genuine drop minutes later still refetches.
|
|
5324
|
+
*/
|
|
5325
|
+
static RECONNECT_REFETCH_COOLDOWN_MS = 1e4;
|
|
5280
5326
|
events = createSyncEventSystem();
|
|
5281
5327
|
currentUserId = null;
|
|
5282
5328
|
tabRole = "solo";
|
|
@@ -5902,8 +5948,21 @@ var Sp00kySync = class Sp00kySync {
|
|
|
5902
5948
|
client.subscribe("connected", () => {
|
|
5903
5949
|
if (!this.needsResubscribe) return;
|
|
5904
5950
|
this.needsResubscribe = false;
|
|
5905
|
-
|
|
5906
|
-
|
|
5951
|
+
const sinceLast = Date.now() - this.lastReconnectRefetchAt;
|
|
5952
|
+
if (sinceLast < Sp00kySync.RECONNECT_REFETCH_COOLDOWN_MS) {
|
|
5953
|
+
this.logger.debug({
|
|
5954
|
+
sinceLast,
|
|
5955
|
+
Category: "sp00ky-client::Sp00kySync::onReconnect"
|
|
5956
|
+
}, "Reconnected again within the cooldown; skipping duplicate refetch");
|
|
5957
|
+
return;
|
|
5958
|
+
}
|
|
5959
|
+
this.lastReconnectRefetchAt = Date.now();
|
|
5960
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
5961
|
+
this.logger.info({
|
|
5962
|
+
queries: hashes.length,
|
|
5963
|
+
Category: "sp00ky-client::Sp00kySync::onReconnect"
|
|
5964
|
+
}, "Remote reconnected, refetching active queries");
|
|
5965
|
+
for (const hash of hashes) this.scheduler.enqueueDownEvent({
|
|
5907
5966
|
type: "register",
|
|
5908
5967
|
payload: { hash }
|
|
5909
5968
|
});
|
|
@@ -6484,8 +6543,8 @@ async function walkOpfs(maxEntries = 2e3, maxDepth = 8) {
|
|
|
6484
6543
|
|
|
6485
6544
|
//#endregion
|
|
6486
6545
|
//#region src/modules/devtools/index.ts
|
|
6487
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
6488
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
6546
|
+
const CORE_VERSION = "0.0.1-canary.165";
|
|
6547
|
+
const WASM_VERSION = "0.0.1-canary.165";
|
|
6489
6548
|
const SURREAL_VERSION = "3.0.3";
|
|
6490
6549
|
var DevToolsService = class DevToolsService {
|
|
6491
6550
|
eventsHistory = [];
|
|
@@ -10071,6 +10130,13 @@ var BlobCache = class {
|
|
|
10071
10130
|
onPageHide = () => {
|
|
10072
10131
|
this.manifest.flush();
|
|
10073
10132
|
};
|
|
10133
|
+
/**
|
|
10134
|
+
* Resolves once the manifest has been reconciled against disk. Reads await
|
|
10135
|
+
* it, so `start()` does NOT have to be awaited on the boot path — blocking
|
|
10136
|
+
* boot on an OPFS directory walk delayed the WebSocket connect (and with it
|
|
10137
|
+
* the connection supervisor) for no benefit.
|
|
10138
|
+
*/
|
|
10139
|
+
ready = Promise.resolve();
|
|
10074
10140
|
hits = 0;
|
|
10075
10141
|
misses = 0;
|
|
10076
10142
|
evictedEntries = 0;
|
|
@@ -10102,6 +10168,7 @@ var BlobCache = class {
|
|
|
10102
10168
|
* Returns null when the file does not exist remotely and is not cached.
|
|
10103
10169
|
*/
|
|
10104
10170
|
async read(key, options = {}) {
|
|
10171
|
+
await this.ready;
|
|
10105
10172
|
const id = blobKeyId(key);
|
|
10106
10173
|
const persist = options.persist !== false && !this.persistDisabled;
|
|
10107
10174
|
if (!options.reload) {
|
|
@@ -10413,7 +10480,8 @@ var BlobCache = class {
|
|
|
10413
10480
|
* it lands on is the one the store was constructed with. */
|
|
10414
10481
|
async start(namespace) {
|
|
10415
10482
|
this.store.setNamespace(namespace);
|
|
10416
|
-
|
|
10483
|
+
this.ready = this.reconcile().catch(() => {});
|
|
10484
|
+
await this.ready;
|
|
10417
10485
|
}
|
|
10418
10486
|
/** Repoint at another local bucket. The bytes of the old one stay on disk so
|
|
10419
10487
|
* switching back (or signing back in) is still warm. */
|
|
@@ -10425,7 +10493,8 @@ var BlobCache = class {
|
|
|
10425
10493
|
this.manifest.reset();
|
|
10426
10494
|
this.store.setNamespace(namespace);
|
|
10427
10495
|
this.persistPaused = false;
|
|
10428
|
-
|
|
10496
|
+
this.ready = this.reconcile().catch(() => {});
|
|
10497
|
+
await this.ready;
|
|
10429
10498
|
}
|
|
10430
10499
|
setMaxBytes(maxBytes) {
|
|
10431
10500
|
this.maxBytes = maxBytes;
|
|
@@ -10815,7 +10884,7 @@ var Sp00kyClient = class {
|
|
|
10815
10884
|
return new TabsCoordinator({
|
|
10816
10885
|
tabId,
|
|
10817
10886
|
fingerprint: computeTabsFingerprint({
|
|
10818
|
-
coreVersion: "0.0.1-canary.
|
|
10887
|
+
coreVersion: "0.0.1-canary.165",
|
|
10819
10888
|
schemaHash: hash53(this.config.schemaSurql),
|
|
10820
10889
|
endpoint: this.config.database.endpoint ?? "",
|
|
10821
10890
|
namespace: this.config.database.namespace,
|
|
@@ -10954,18 +11023,20 @@ var Sp00kyClient = class {
|
|
|
10954
11023
|
await this.migrator.provision(this.config.schemaSurql);
|
|
10955
11024
|
this.logger.debug({ Category: "sp00ky-client::Sp00kyClient::init" }, "Schema provisioned");
|
|
10956
11025
|
}
|
|
10957
|
-
try {
|
|
10958
|
-
this.blobs.setMaxBytes(await resolveBlobBudget(this.config.blobCache?.maxBytes));
|
|
10959
|
-
await this.blobs.start(bootBucket);
|
|
10960
|
-
} catch (e) {
|
|
10961
|
-
this.logger.warn({
|
|
10962
|
-
err: e,
|
|
10963
|
-
Category: "sp00ky-client::Sp00kyClient::init"
|
|
10964
|
-
}, "Blob cache failed to start; bucket files will not be cached locally");
|
|
10965
|
-
}
|
|
10966
11026
|
await this.remote.connect();
|
|
10967
11027
|
this.connectionSupervisor.start();
|
|
10968
11028
|
this.logger.debug({ Category: "sp00ky-client::Sp00kyClient::init" }, "Remote database connected");
|
|
11029
|
+
(async () => {
|
|
11030
|
+
try {
|
|
11031
|
+
this.blobs.setMaxBytes(await resolveBlobBudget(this.config.blobCache?.maxBytes));
|
|
11032
|
+
await this.blobs.start(bootBucket);
|
|
11033
|
+
} catch (e) {
|
|
11034
|
+
this.logger.warn({
|
|
11035
|
+
err: e,
|
|
11036
|
+
Category: "sp00ky-client::Sp00kyClient::init"
|
|
11037
|
+
}, "Blob cache failed to start; bucket files will not be cached locally");
|
|
11038
|
+
}
|
|
11039
|
+
})();
|
|
10969
11040
|
this.streamProcessor.setStateKeySuffix(bootBucket);
|
|
10970
11041
|
await this.streamProcessor.init();
|
|
10971
11042
|
this.streamProcessor.setPermissions(extractSelectPermissions(this.config.schemaSurql));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.165",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.165",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.165",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"fast-json-patch": "^3.1.1",
|
package/src/modules/sync/sync.ts
CHANGED
|
@@ -107,6 +107,14 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
107
107
|
* initial connect. See {@link subscribeToReconnect}.
|
|
108
108
|
*/
|
|
109
109
|
private needsResubscribe: boolean = false;
|
|
110
|
+
/** When the last reconnect-driven full refetch ran, for burst coalescing. */
|
|
111
|
+
private lastReconnectRefetchAt = 0;
|
|
112
|
+
/**
|
|
113
|
+
* Minimum gap between reconnect-driven full refetches. Long enough to absorb
|
|
114
|
+
* a flapping socket (the SDK reconnect ladder starts at 1s), short enough
|
|
115
|
+
* that a genuine drop minutes later still refetches.
|
|
116
|
+
*/
|
|
117
|
+
private static readonly RECONNECT_REFETCH_COOLDOWN_MS = 10_000;
|
|
110
118
|
public events = createSyncEventSystem();
|
|
111
119
|
|
|
112
120
|
// Auth identity that drives per-user `_00_list_ref_user_<id>` routing
|
|
@@ -1034,11 +1042,26 @@ export class Sp00kySync<S extends SchemaStructure> {
|
|
|
1034
1042
|
client.subscribe('connected', () => {
|
|
1035
1043
|
if (!this.needsResubscribe) return;
|
|
1036
1044
|
this.needsResubscribe = false;
|
|
1045
|
+
// A flapping socket produces reconnecting -> connected repeatedly, and
|
|
1046
|
+
// each cycle used to re-register EVERY active query (a busy app has
|
|
1047
|
+
// dozens). That is the "everything reloads about a second after a blip"
|
|
1048
|
+
// symptom: the SDK's retryDelay is 1s, so the refetch lands right after
|
|
1049
|
+
// the drop the user never saw. Collapse bursts into one refetch.
|
|
1050
|
+
const sinceLast = Date.now() - this.lastReconnectRefetchAt;
|
|
1051
|
+
if (sinceLast < Sp00kySync.RECONNECT_REFETCH_COOLDOWN_MS) {
|
|
1052
|
+
this.logger.debug(
|
|
1053
|
+
{ sinceLast, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1054
|
+
'Reconnected again within the cooldown; skipping duplicate refetch'
|
|
1055
|
+
);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
this.lastReconnectRefetchAt = Date.now();
|
|
1059
|
+
const hashes = this.dataModule.getActiveQueryHashes();
|
|
1037
1060
|
this.logger.info(
|
|
1038
|
-
{ Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1061
|
+
{ queries: hashes.length, Category: 'sp00ky-client::Sp00kySync::onReconnect' },
|
|
1039
1062
|
'Remote reconnected, refetching active queries'
|
|
1040
1063
|
);
|
|
1041
|
-
for (const hash of
|
|
1064
|
+
for (const hash of hashes) {
|
|
1042
1065
|
this.scheduler.enqueueDownEvent({ type: 'register', payload: { hash } });
|
|
1043
1066
|
}
|
|
1044
1067
|
// The WS reconnect leaves the server-side LIVE subscription dead — the
|
|
@@ -304,6 +304,24 @@ describe('BlobCache namespaces', () => {
|
|
|
304
304
|
expect(fetchRemote).toHaveBeenCalledTimes(2);
|
|
305
305
|
});
|
|
306
306
|
|
|
307
|
+
it('a read issued before start() finishes waits for the reconcile', async () => {
|
|
308
|
+
// Boot fires start() without awaiting it (the OPFS walk must not delay the
|
|
309
|
+
// WebSocket connect). A read landing mid-walk must still see the rebuilt
|
|
310
|
+
// manifest rather than refetching a file that is already on disk.
|
|
311
|
+
const { cache, manifest, local, fetchRemote } = setup();
|
|
312
|
+
await cache.read(KEY);
|
|
313
|
+
await cache.flush();
|
|
314
|
+
local.rows.clear();
|
|
315
|
+
manifest.reset();
|
|
316
|
+
|
|
317
|
+
const starting = cache.start('user-1');
|
|
318
|
+
const readDuringStart = cache.read(KEY);
|
|
319
|
+
await Promise.all([starting, readDuringStart]);
|
|
320
|
+
|
|
321
|
+
expect(await readDuringStart).not.toBeNull();
|
|
322
|
+
expect(fetchRemote).toHaveBeenCalledTimes(1);
|
|
323
|
+
});
|
|
324
|
+
|
|
307
325
|
it('clear() drops the current namespace', async () => {
|
|
308
326
|
const { cache, store } = setup();
|
|
309
327
|
await cache.read(KEY);
|
|
@@ -127,6 +127,14 @@ export class BlobCache {
|
|
|
127
127
|
void this.manifest.flush();
|
|
128
128
|
};
|
|
129
129
|
|
|
130
|
+
/**
|
|
131
|
+
* Resolves once the manifest has been reconciled against disk. Reads await
|
|
132
|
+
* it, so `start()` does NOT have to be awaited on the boot path — blocking
|
|
133
|
+
* boot on an OPFS directory walk delayed the WebSocket connect (and with it
|
|
134
|
+
* the connection supervisor) for no benefit.
|
|
135
|
+
*/
|
|
136
|
+
private ready: Promise<void> = Promise.resolve();
|
|
137
|
+
|
|
130
138
|
private hits = 0;
|
|
131
139
|
private misses = 0;
|
|
132
140
|
private evictedEntries = 0;
|
|
@@ -167,6 +175,10 @@ export class BlobCache {
|
|
|
167
175
|
* Returns null when the file does not exist remotely and is not cached.
|
|
168
176
|
*/
|
|
169
177
|
async read(key: BlobKey, options: BlobReadOptions = {}): Promise<Blob | null> {
|
|
178
|
+
// Boot fires `start()` without awaiting it; a read that lands first must
|
|
179
|
+
// not race the reconcile, or it would refetch a file already on disk and
|
|
180
|
+
// then overwrite the row reconcile is about to rebuild.
|
|
181
|
+
await this.ready;
|
|
170
182
|
const id = blobKeyId(key);
|
|
171
183
|
const persist = options.persist !== false && !this.persistDisabled;
|
|
172
184
|
|
|
@@ -514,7 +526,12 @@ export class BlobCache {
|
|
|
514
526
|
* it lands on is the one the store was constructed with. */
|
|
515
527
|
async start(namespace: string): Promise<void> {
|
|
516
528
|
this.store.setNamespace(namespace);
|
|
517
|
-
|
|
529
|
+
// Publish the in-flight reconcile so reads issued before it finishes queue
|
|
530
|
+
// behind it instead of racing it. Swallow here: `reconcile()` already logs,
|
|
531
|
+
// and an unhandled rejection on a fire-and-forget boot call must not
|
|
532
|
+
// surface as a global error.
|
|
533
|
+
this.ready = this.reconcile().catch(() => {});
|
|
534
|
+
await this.ready;
|
|
518
535
|
}
|
|
519
536
|
|
|
520
537
|
/** Repoint at another local bucket. The bytes of the old one stay on disk so
|
|
@@ -527,7 +544,8 @@ export class BlobCache {
|
|
|
527
544
|
this.manifest.reset();
|
|
528
545
|
this.store.setNamespace(namespace);
|
|
529
546
|
this.persistPaused = false;
|
|
530
|
-
|
|
547
|
+
this.ready = this.reconcile().catch(() => {});
|
|
548
|
+
await this.ready;
|
|
531
549
|
}
|
|
532
550
|
|
|
533
551
|
setMaxBytes(maxBytes: number): void {
|
|
@@ -142,7 +142,13 @@ describe('ConnectionSupervisor', () => {
|
|
|
142
142
|
expect(remote.query).toHaveBeenCalledWith('RETURN true');
|
|
143
143
|
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
144
144
|
|
|
145
|
+
// One failure is inconclusive — the probe shares a queue with ordinary
|
|
146
|
+
// traffic, so it re-probes rather than tearing down a possibly-fine socket.
|
|
145
147
|
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatTimeoutMs);
|
|
148
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
149
|
+
|
|
150
|
+
// The second consecutive failure is the one that tears it down.
|
|
151
|
+
await vi.advanceTimersByTimeAsync(5_000 + CONFIG.heartbeatTimeoutMs);
|
|
146
152
|
expect(remote.forceClose).toHaveBeenCalledTimes(1);
|
|
147
153
|
|
|
148
154
|
// The forced close published `disconnected`, so the revive loop takes over.
|
|
@@ -152,6 +158,34 @@ describe('ConnectionSupervisor', () => {
|
|
|
152
158
|
sup.dispose();
|
|
153
159
|
});
|
|
154
160
|
|
|
161
|
+
it('does not tear down a healthy socket after a single slow probe', async () => {
|
|
162
|
+
// The regression this guards: the heartbeat rides the same serialized queue
|
|
163
|
+
// as every other RPC, so one busy window (a big sync burst) used to
|
|
164
|
+
// force-close a working connection — and the reconnect then re-registered
|
|
165
|
+
// every active query about a second later.
|
|
166
|
+
const { remote, emit } = makeRemote();
|
|
167
|
+
let calls = 0;
|
|
168
|
+
remote.query.mockImplementation(() => {
|
|
169
|
+
calls++;
|
|
170
|
+
// First probe hangs past its deadline, the next answers normally.
|
|
171
|
+
return calls === 1 ? new Promise(() => {}) : Promise.resolve(true);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
const sup = makeSupervisor(remote);
|
|
175
|
+
emit('connected');
|
|
176
|
+
sup.start();
|
|
177
|
+
|
|
178
|
+
await vi.advanceTimersByTimeAsync(CONFIG.heartbeatIntervalMs + CONFIG.heartbeatTimeoutMs);
|
|
179
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
180
|
+
|
|
181
|
+
// The retry succeeds, so the socket survives and heartbeating continues.
|
|
182
|
+
await vi.advanceTimersByTimeAsync(5_000 + CONFIG.heartbeatIntervalMs * 2);
|
|
183
|
+
expect(remote.forceClose).not.toHaveBeenCalled();
|
|
184
|
+
expect(sup.connection).toBe('connected');
|
|
185
|
+
|
|
186
|
+
sup.dispose();
|
|
187
|
+
});
|
|
188
|
+
|
|
155
189
|
it('keeps heartbeating while the connection is healthy', async () => {
|
|
156
190
|
const { remote, emit } = makeRemote();
|
|
157
191
|
const sup = makeSupervisor(remote);
|
|
@@ -40,9 +40,13 @@ export class ConnectionSupervisor {
|
|
|
40
40
|
|
|
41
41
|
private heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
|
|
42
42
|
private heartbeatInFlight = false;
|
|
43
|
+
/** Consecutive failed probes. See {@link FAILURES_BEFORE_TEARDOWN}. */
|
|
44
|
+
private heartbeatFailures = 0;
|
|
43
45
|
|
|
44
46
|
private reviveTimer: ReturnType<typeof setTimeout> | null = null;
|
|
45
47
|
private reviveAttempts = 0;
|
|
48
|
+
/** Timestamp of the last wake-triggered probe, for rate limiting. */
|
|
49
|
+
private lastWakeProbeAt = 0;
|
|
46
50
|
private reviving = false;
|
|
47
51
|
/**
|
|
48
52
|
* Set while the browser reports itself offline. Retrying a socket against a
|
|
@@ -53,6 +57,23 @@ export class ConnectionSupervisor {
|
|
|
53
57
|
private teardown: Array<() => void> = [];
|
|
54
58
|
|
|
55
59
|
private static readonly REVIVE_BASE_MS = 1_000;
|
|
60
|
+
/**
|
|
61
|
+
* How many consecutive heartbeat failures it takes to tear the socket down.
|
|
62
|
+
*
|
|
63
|
+
* The probe rides the same serialized queue as every other RPC (deliberately
|
|
64
|
+
* — see {@link beat}), which means it cannot distinguish a WEDGED queue from
|
|
65
|
+
* a merely BUSY one. A single slow window (a large sync burst, one heavy
|
|
66
|
+
* app query) used to be enough to force-close a perfectly healthy socket,
|
|
67
|
+
* and the resulting reconnect re-registered every active query about a
|
|
68
|
+
* second later. That self-inflicted teardown manufactured the very reconnect
|
|
69
|
+
* storms this class exists to survive. A genuinely dead socket still fails
|
|
70
|
+
* every probe, so it is torn down one interval later than before.
|
|
71
|
+
*/
|
|
72
|
+
private static readonly FAILURES_BEFORE_TEARDOWN = 2;
|
|
73
|
+
/** Retry delay after an inconclusive (first) heartbeat failure. */
|
|
74
|
+
private static readonly HEARTBEAT_RETRY_MS = 5_000;
|
|
75
|
+
/** Floor between probes triggered by wake events (tab focus, pageshow). */
|
|
76
|
+
private static readonly WAKE_PROBE_MIN_INTERVAL_MS = 10_000;
|
|
56
77
|
|
|
57
78
|
constructor(
|
|
58
79
|
private readonly remote: RemoteDatabaseService,
|
|
@@ -264,12 +285,40 @@ export class ConnectionSupervisor {
|
|
|
264
285
|
this.config.heartbeatTimeoutMs,
|
|
265
286
|
`Heartbeat timed out after ${this.config.heartbeatTimeoutMs}ms`
|
|
266
287
|
);
|
|
288
|
+
this.heartbeatFailures = 0;
|
|
267
289
|
this.startHeartbeat();
|
|
268
290
|
} catch (err) {
|
|
291
|
+
this.heartbeatFailures++;
|
|
292
|
+
if (this.heartbeatFailures < ConnectionSupervisor.FAILURES_BEFORE_TEARDOWN) {
|
|
293
|
+
// Inconclusive: the probe shares a queue with ordinary traffic, so this
|
|
294
|
+
// may just be a busy window rather than a dead socket. Re-probe soon
|
|
295
|
+
// instead of tearing down a connection that is probably fine.
|
|
296
|
+
this.logger.debug(
|
|
297
|
+
{
|
|
298
|
+
err,
|
|
299
|
+
failures: this.heartbeatFailures,
|
|
300
|
+
Category: 'sp00ky-client::ConnectionSupervisor::heartbeat',
|
|
301
|
+
},
|
|
302
|
+
'Heartbeat failed; re-probing before tearing the socket down'
|
|
303
|
+
);
|
|
304
|
+
this.stopHeartbeat();
|
|
305
|
+
if (!this.disposed && !this.suspended) {
|
|
306
|
+
this.heartbeatTimer = setTimeout(
|
|
307
|
+
() => void this.beat(),
|
|
308
|
+
Math.min(ConnectionSupervisor.HEARTBEAT_RETRY_MS, this.config.heartbeatIntervalMs)
|
|
309
|
+
);
|
|
310
|
+
}
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
269
313
|
this.logger.warn(
|
|
270
|
-
{
|
|
271
|
-
|
|
314
|
+
{
|
|
315
|
+
err,
|
|
316
|
+
failures: this.heartbeatFailures,
|
|
317
|
+
Category: 'sp00ky-client::ConnectionSupervisor::heartbeat',
|
|
318
|
+
},
|
|
319
|
+
'Heartbeat failed repeatedly; tearing the socket down to force a reconnect'
|
|
272
320
|
);
|
|
321
|
+
this.heartbeatFailures = 0;
|
|
273
322
|
// Force the `close` the transport never delivered. The resulting
|
|
274
323
|
// `disconnected` event drives the revive loop.
|
|
275
324
|
await this.remote.forceClose();
|
|
@@ -332,6 +381,21 @@ export class ConnectionSupervisor {
|
|
|
332
381
|
*/
|
|
333
382
|
private wake(reason: string): void {
|
|
334
383
|
if (this.disposed || this.suspended) return;
|
|
384
|
+
// `visibilitychange` fires on every alt-tab, every window minimise and every
|
|
385
|
+
// focus change. Probing a connected socket each time meant an ordinary
|
|
386
|
+
// afternoon of tab-switching issued a steady stream of extra RPCs, each of
|
|
387
|
+
// which could trip the heartbeat teardown above. A healthy socket does not
|
|
388
|
+
// become unhealthy because the user looked away for four seconds, so probes
|
|
389
|
+
// from wake triggers are rate-limited; a genuinely dead socket is still
|
|
390
|
+
// caught by the regular heartbeat.
|
|
391
|
+
const now = Date.now();
|
|
392
|
+
if (
|
|
393
|
+
this.remote.getStatus() === 'connected' &&
|
|
394
|
+
now - this.lastWakeProbeAt < ConnectionSupervisor.WAKE_PROBE_MIN_INTERVAL_MS
|
|
395
|
+
) {
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
this.lastWakeProbeAt = now;
|
|
335
399
|
this.logger.debug(
|
|
336
400
|
{ reason, state: this.state, Category: 'sp00ky-client::ConnectionSupervisor::wake' },
|
|
337
401
|
'Wake trigger; probing the connection'
|
package/src/sp00ky.ts
CHANGED
|
@@ -706,20 +706,6 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
706
706
|
this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Schema provisioned');
|
|
707
707
|
}
|
|
708
708
|
|
|
709
|
-
// After the store is open (the manifest lives in it) and after
|
|
710
|
-
// provisioning (`_00_blob` has to exist), before any query can ask for a
|
|
711
|
-
// file. Best-effort: a cold blob cache is a slow first paint, not a
|
|
712
|
-
// broken client.
|
|
713
|
-
try {
|
|
714
|
-
this.blobs.setMaxBytes(await resolveBlobBudget(this.config.blobCache?.maxBytes));
|
|
715
|
-
await this.blobs.start(bootBucket);
|
|
716
|
-
} catch (e) {
|
|
717
|
-
this.logger.warn(
|
|
718
|
-
{ err: e, Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
719
|
-
'Blob cache failed to start; bucket files will not be cached locally'
|
|
720
|
-
);
|
|
721
|
-
}
|
|
722
|
-
|
|
723
709
|
await this.remote.connect();
|
|
724
710
|
// Start supervising only after the first connect succeeds, so a boot-time
|
|
725
711
|
// failure surfaces as a thrown init() rather than being silently absorbed
|
|
@@ -730,6 +716,25 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
730
716
|
'Remote database connected'
|
|
731
717
|
);
|
|
732
718
|
|
|
719
|
+
// Warm the blob cache in the background. Deliberately NOT awaited, and
|
|
720
|
+
// deliberately after `remote.connect()`: this walks the OPFS directory to
|
|
721
|
+
// rebuild the manifest, and awaiting it ahead of the socket delayed the
|
|
722
|
+
// connect (and the connection supervisor with it) for no benefit. Reads
|
|
723
|
+
// await `BlobCache.ready` internally, so a bucket read that lands mid-walk
|
|
724
|
+
// still sees a reconciled manifest. Best-effort: a cold blob cache is a
|
|
725
|
+
// slow first image, not a broken client.
|
|
726
|
+
void (async () => {
|
|
727
|
+
try {
|
|
728
|
+
this.blobs.setMaxBytes(await resolveBlobBudget(this.config.blobCache?.maxBytes));
|
|
729
|
+
await this.blobs.start(bootBucket);
|
|
730
|
+
} catch (e) {
|
|
731
|
+
this.logger.warn(
|
|
732
|
+
{ err: e, Category: 'sp00ky-client::Sp00kyClient::init' },
|
|
733
|
+
'Blob cache failed to start; bucket files will not be cached locally'
|
|
734
|
+
);
|
|
735
|
+
}
|
|
736
|
+
})();
|
|
737
|
+
|
|
733
738
|
this.streamProcessor.setStateKeySuffix(bootBucket);
|
|
734
739
|
await this.streamProcessor.init();
|
|
735
740
|
// Seed table `select` permissions from the schema before any query is
|