@syncular/client 0.49.1 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/dist/availability.d.ts +1 -1
- package/dist/availability.js +1 -1
- package/dist/errors.d.ts +5 -0
- package/dist/errors.js +5 -0
- package/dist/leader-lock.d.ts +18 -7
- package/dist/leader-lock.js +40 -22
- package/dist/multi-tab.d.ts +63 -1
- package/dist/multi-tab.js +148 -29
- package/dist/reactive-store.js +2 -1
- package/dist/wasm-database.d.ts +6 -0
- package/dist/wasm-database.js +36 -6
- package/dist/worker-entry.js +1 -0
- package/dist/worker-host.d.ts +23 -1
- package/dist/worker-host.js +53 -2
- package/dist/worker-protocol.d.ts +6 -0
- package/package.json +3 -3
- package/src/availability.ts +3 -2
- package/src/errors.ts +6 -0
- package/src/leader-lock.ts +64 -34
- package/src/multi-tab.ts +233 -23
- package/src/reactive-store.ts +2 -1
- package/src/wasm-database.ts +64 -6
- package/src/worker-entry.ts +1 -0
- package/src/worker-host.ts +82 -4
- package/src/worker-protocol.ts +6 -0
package/src/multi-tab.ts
CHANGED
|
@@ -82,6 +82,12 @@ export const LEADER_INCOMPATIBLE_CODE = 'client.leader_incompatible';
|
|
|
82
82
|
/** Client-local: the leader that received a call stopped leading before it answered. */
|
|
83
83
|
export const LEADER_HANDOVER_CODE = 'client.leader_handover';
|
|
84
84
|
|
|
85
|
+
/** Client-local: another tab took this tab's leadership with `takeOver()`. */
|
|
86
|
+
export const LEADERSHIP_TAKEN_CODE = 'client.leadership_taken';
|
|
87
|
+
|
|
88
|
+
/** Client-local: `takeOver()` was called while its conditions do not hold. */
|
|
89
|
+
export const TAKEOVER_REFUSED_CODE = 'client.takeover_refused';
|
|
90
|
+
|
|
85
91
|
/**
|
|
86
92
|
* Version of the leader/follower wire and the forwarded handle API. Bump it
|
|
87
93
|
* with every change to a message shape or to a forwarded method's arguments
|
|
@@ -141,6 +147,12 @@ export type LeadershipState =
|
|
|
141
147
|
/** Whether the tab holding the database runs an older or a newer build. */
|
|
142
148
|
readonly leader: 'older' | 'newer';
|
|
143
149
|
readonly retryable: true;
|
|
150
|
+
}
|
|
151
|
+
| {
|
|
152
|
+
readonly state: 'blocked';
|
|
153
|
+
readonly reason: 'leadership-taken';
|
|
154
|
+
readonly code: typeof LEADERSHIP_TAKEN_CODE;
|
|
155
|
+
readonly retryable: true;
|
|
144
156
|
};
|
|
145
157
|
|
|
146
158
|
// ---------------------------------------------------------------------------
|
|
@@ -944,6 +956,13 @@ export interface TabLeaderContext<E> {
|
|
|
944
956
|
readonly epoch: number;
|
|
945
957
|
/** This tab's id. The leader's own calls arrive under it. */
|
|
946
958
|
readonly tabId: string;
|
|
959
|
+
/**
|
|
960
|
+
* True when this tab took the lock from an unreachable leader with
|
|
961
|
+
* `takeOver()`. That leader's worker can still hold the database files;
|
|
962
|
+
* the core must take over its storage lock and retry opening until the
|
|
963
|
+
* old worker releases them.
|
|
964
|
+
*/
|
|
965
|
+
readonly takeover: boolean;
|
|
947
966
|
/**
|
|
948
967
|
* The leader lock. The coordinator releases it after {@link TabLeader.close}
|
|
949
968
|
* resolves. Release it earlier only when the core is already gone, such as a
|
|
@@ -972,10 +991,48 @@ export interface TabLeader {
|
|
|
972
991
|
* lock.
|
|
973
992
|
*/
|
|
974
993
|
tabClosed?(tabId: string): void;
|
|
975
|
-
/**
|
|
994
|
+
/**
|
|
995
|
+
* Stop the core. The coordinator releases the leader lock afterwards. It
|
|
996
|
+
* also runs when another tab took the lock: the core must not write again.
|
|
997
|
+
*/
|
|
976
998
|
close(): Promise<void>;
|
|
977
999
|
}
|
|
978
1000
|
|
|
1001
|
+
/**
|
|
1002
|
+
* A follower's link to its leader went `leader-unreachable`, or left that
|
|
1003
|
+
* state. Times are `clock.now()` milliseconds (wall-clock by default).
|
|
1004
|
+
*/
|
|
1005
|
+
export type LeaderUnreachableEvent =
|
|
1006
|
+
| {
|
|
1007
|
+
readonly phase: 'started';
|
|
1008
|
+
readonly at: number;
|
|
1009
|
+
/** The earliest time `takeOver()` is allowed. */
|
|
1010
|
+
readonly takeOverAt: number;
|
|
1011
|
+
}
|
|
1012
|
+
| {
|
|
1013
|
+
readonly phase: 'ended';
|
|
1014
|
+
readonly at: number;
|
|
1015
|
+
readonly durationMs: number;
|
|
1016
|
+
/**
|
|
1017
|
+
* `rebound`: the leader answered again. `promoted`: its lock passed to
|
|
1018
|
+
* this tab. `taken-over`: this tab's `takeOver()` succeeded.
|
|
1019
|
+
* `incompatible`: a leader of another build answered. `closed`: this
|
|
1020
|
+
* tab closed first.
|
|
1021
|
+
*/
|
|
1022
|
+
readonly outcome:
|
|
1023
|
+
| 'rebound'
|
|
1024
|
+
| 'promoted'
|
|
1025
|
+
| 'taken-over'
|
|
1026
|
+
| 'incompatible'
|
|
1027
|
+
| 'closed';
|
|
1028
|
+
};
|
|
1029
|
+
|
|
1030
|
+
/** Time source and timer seam; tests inject a manual clock. */
|
|
1031
|
+
export interface TabClock {
|
|
1032
|
+
now(): number;
|
|
1033
|
+
schedule(callback: () => void, delayMs: number): () => void;
|
|
1034
|
+
}
|
|
1035
|
+
|
|
979
1036
|
export interface TabCoordinatorConfig<E> {
|
|
980
1037
|
/** Web Lock name of the leadership; the channel and tab locks derive from it. */
|
|
981
1038
|
readonly name: string;
|
|
@@ -990,11 +1047,19 @@ export interface TabCoordinatorConfig<E> {
|
|
|
990
1047
|
/** Events the leader broadcast, or sent to this tab. */
|
|
991
1048
|
onEvent?(event: E): void;
|
|
992
1049
|
onStateChange?(state: LeadershipState): void;
|
|
1050
|
+
/** Start and end of every `leader-unreachable` period, for host telemetry. */
|
|
1051
|
+
onLeaderUnreachable?(event: LeaderUnreachableEvent): void;
|
|
993
1052
|
/** Defaults to Web Locks; tests inject a name-keyed exclusive lock. */
|
|
994
1053
|
readonly lock?: LeaderLock;
|
|
995
1054
|
readonly channelFactory?: (name: string) => CrossTabChannel<E>;
|
|
996
1055
|
/** Liveness and handover-queue deadline; defaults to 10 s. */
|
|
997
1056
|
readonly callTimeoutMs?: number;
|
|
1057
|
+
/**
|
|
1058
|
+
* How long a follower must have been `leader-unreachable` before
|
|
1059
|
+
* `takeOver()` is allowed; defaults to twice `callTimeoutMs`.
|
|
1060
|
+
*/
|
|
1061
|
+
readonly takeOverAfterMs?: number;
|
|
1062
|
+
readonly clock?: TabClock;
|
|
998
1063
|
}
|
|
999
1064
|
|
|
1000
1065
|
/** One tab's end of the coordinator, the same in leader and follower tabs. */
|
|
@@ -1003,6 +1068,15 @@ export interface TabCoordinator {
|
|
|
1003
1068
|
readonly state: LeadershipState;
|
|
1004
1069
|
/** Run a call on the leader's core, in this tab or over the channel. */
|
|
1005
1070
|
call(method: string, args: readonly unknown[]): Promise<unknown>;
|
|
1071
|
+
/**
|
|
1072
|
+
* Take the lock from a leader that has been `leader-unreachable` for
|
|
1073
|
+
* `takeOverAfterMs`, and start the core in this tab. The old leader reports
|
|
1074
|
+
* `leadership-taken` and closes its core once it runs again. Rejects with
|
|
1075
|
+
* `client.takeover_refused` (`details.reason`: `not-unreachable`,
|
|
1076
|
+
* `too-early`, `in-progress` or `unsupported`) when the conditions do not
|
|
1077
|
+
* hold, and with the core's start error when the core cannot start.
|
|
1078
|
+
*/
|
|
1079
|
+
takeOver(): Promise<void>;
|
|
1006
1080
|
/** Stop the core (leader) or the link (follower), and release this tab's locks. */
|
|
1007
1081
|
close(): Promise<void>;
|
|
1008
1082
|
}
|
|
@@ -1011,6 +1085,18 @@ function tabLockName(name: string, tabId: string): string {
|
|
|
1011
1085
|
return `${name}:tab:${tabId}`;
|
|
1012
1086
|
}
|
|
1013
1087
|
|
|
1088
|
+
function takeOverRefused(
|
|
1089
|
+
reason: 'not-unreachable' | 'too-early' | 'in-progress' | 'unsupported',
|
|
1090
|
+
takeOverAt?: number,
|
|
1091
|
+
): ClientSyncError {
|
|
1092
|
+
return new ClientSyncError(
|
|
1093
|
+
TAKEOVER_REFUSED_CODE,
|
|
1094
|
+
'takeOver() requires a leader that has been unreachable for takeOverAfterMs',
|
|
1095
|
+
reason === 'too-early' || reason === 'in-progress',
|
|
1096
|
+
{ reason, ...(takeOverAt !== undefined ? { takeOverAt } : {}) },
|
|
1097
|
+
);
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1014
1100
|
/**
|
|
1015
1101
|
* Elect one leader tab per `name` and run `lead` there; every other tab
|
|
1016
1102
|
* forwards its calls to that core and receives its events. When the leader
|
|
@@ -1021,6 +1107,16 @@ export async function createTabCoordinator<E>(
|
|
|
1021
1107
|
config: TabCoordinatorConfig<E>,
|
|
1022
1108
|
): Promise<TabCoordinator> {
|
|
1023
1109
|
const lock = config.lock ?? webLocksLeaderLock();
|
|
1110
|
+
const clock: TabClock = config.clock ?? {
|
|
1111
|
+
now: () => Date.now(),
|
|
1112
|
+
schedule: (callback, delayMs) => {
|
|
1113
|
+
const timer = setTimeout(callback, delayMs);
|
|
1114
|
+
return () => clearTimeout(timer);
|
|
1115
|
+
},
|
|
1116
|
+
};
|
|
1117
|
+
const callTimeoutMs =
|
|
1118
|
+
config.callTimeoutMs ?? DEFAULT_FOLLOWER_CALL_TIMEOUT_MS;
|
|
1119
|
+
const takeOverAfterMs = config.takeOverAfterMs ?? 2 * callTimeoutMs;
|
|
1024
1120
|
const openChannel = config.channelFactory ?? broadcastChannelFactory<E>();
|
|
1025
1121
|
const channelName = multiTabChannelName(config.name);
|
|
1026
1122
|
const identity: TabIdentity = {
|
|
@@ -1037,9 +1133,43 @@ export async function createTabCoordinator<E>(
|
|
|
1037
1133
|
| undefined;
|
|
1038
1134
|
let follower: FollowerLink<E> | undefined;
|
|
1039
1135
|
let closed = false;
|
|
1136
|
+
let takingOver = false;
|
|
1137
|
+
/** Start of the current `leader-unreachable` period. */
|
|
1138
|
+
let unreachableSince: number | undefined;
|
|
1040
1139
|
|
|
1140
|
+
const report = (event: LeaderUnreachableEvent): void => {
|
|
1141
|
+
try {
|
|
1142
|
+
config.onLeaderUnreachable?.(event);
|
|
1143
|
+
} catch {
|
|
1144
|
+
// A telemetry listener must never break cross-tab coordination.
|
|
1145
|
+
}
|
|
1146
|
+
};
|
|
1147
|
+
const endUnreachable = (
|
|
1148
|
+
outcome: Extract<LeaderUnreachableEvent, { phase: 'ended' }>['outcome'],
|
|
1149
|
+
): void => {
|
|
1150
|
+
if (unreachableSince === undefined) return;
|
|
1151
|
+
const at = clock.now();
|
|
1152
|
+
report({ phase: 'ended', at, durationMs: at - unreachableSince, outcome });
|
|
1153
|
+
unreachableSince = undefined;
|
|
1154
|
+
};
|
|
1041
1155
|
const setState = (next: LeadershipState): void => {
|
|
1042
1156
|
state = next;
|
|
1157
|
+
if (next.state === 'blocked' && next.reason === 'leader-unreachable') {
|
|
1158
|
+
if (unreachableSince === undefined) {
|
|
1159
|
+
unreachableSince = clock.now();
|
|
1160
|
+
report({
|
|
1161
|
+
phase: 'started',
|
|
1162
|
+
at: unreachableSince,
|
|
1163
|
+
takeOverAt: unreachableSince + takeOverAfterMs,
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
} else if (next.state === 'follower') {
|
|
1167
|
+
endUnreachable('rebound');
|
|
1168
|
+
} else if (next.state === 'leader') {
|
|
1169
|
+
endUnreachable(takingOver ? 'taken-over' : 'promoted');
|
|
1170
|
+
} else if (next.state === 'blocked') {
|
|
1171
|
+
endUnreachable('incompatible');
|
|
1172
|
+
}
|
|
1043
1173
|
try {
|
|
1044
1174
|
config.onStateChange?.(next);
|
|
1045
1175
|
} catch {
|
|
@@ -1050,12 +1180,29 @@ export async function createTabCoordinator<E>(
|
|
|
1050
1180
|
// Held until close: a leader waits on it to learn this tab is gone.
|
|
1051
1181
|
const tabLease = await lock.acquire(tabLockName(config.name, tabId));
|
|
1052
1182
|
|
|
1053
|
-
const lead = async (
|
|
1183
|
+
const lead = async (
|
|
1184
|
+
lease: LeaderLease,
|
|
1185
|
+
epoch: number,
|
|
1186
|
+
takeover: boolean,
|
|
1187
|
+
): Promise<void> => {
|
|
1054
1188
|
let bridge: LeaderBridge<E> | undefined;
|
|
1055
1189
|
let current = true;
|
|
1190
|
+
let stepDown: ((next: LeadershipState) => void) | undefined;
|
|
1191
|
+
let lostWhileStarting = false;
|
|
1192
|
+
const taken: LeadershipState = {
|
|
1193
|
+
state: 'blocked',
|
|
1194
|
+
reason: 'leadership-taken',
|
|
1195
|
+
code: LEADERSHIP_TAKEN_CODE,
|
|
1196
|
+
retryable: true,
|
|
1197
|
+
};
|
|
1198
|
+
void lease.lost?.then(() => {
|
|
1199
|
+
if (stepDown !== undefined) stepDown(taken);
|
|
1200
|
+
else lostWhileStarting = true;
|
|
1201
|
+
});
|
|
1056
1202
|
const leader = await config.lead({
|
|
1057
1203
|
epoch,
|
|
1058
1204
|
tabId,
|
|
1205
|
+
takeover,
|
|
1059
1206
|
lease,
|
|
1060
1207
|
broadcast: (event) => {
|
|
1061
1208
|
if (!current) return;
|
|
@@ -1068,8 +1215,9 @@ export async function createTabCoordinator<E>(
|
|
|
1068
1215
|
else bridge?.sendEvent(to, event);
|
|
1069
1216
|
},
|
|
1070
1217
|
});
|
|
1071
|
-
if (closed) {
|
|
1072
|
-
// Closed while the core was starting
|
|
1218
|
+
if (closed || lostWhileStarting) {
|
|
1219
|
+
// Closed while the core was starting, or another tab took the lock: a
|
|
1220
|
+
// follower link, if any, binds to that tab's announcement.
|
|
1073
1221
|
current = false;
|
|
1074
1222
|
await leader.close();
|
|
1075
1223
|
await lease.release();
|
|
@@ -1084,6 +1232,15 @@ export async function createTabCoordinator<E>(
|
|
|
1084
1232
|
},
|
|
1085
1233
|
};
|
|
1086
1234
|
leading = ending;
|
|
1235
|
+
stepDown = (next) => {
|
|
1236
|
+
// Stop the core and release the lock; this tab stays blocked until it
|
|
1237
|
+
// reloads.
|
|
1238
|
+
if (leading !== ending) return;
|
|
1239
|
+
leading = undefined;
|
|
1240
|
+
ending.end();
|
|
1241
|
+
setState(next);
|
|
1242
|
+
void leader.close().finally(() => lease.release());
|
|
1243
|
+
};
|
|
1087
1244
|
bridge = new LeaderBridge<E>({
|
|
1088
1245
|
channel: openChannel(channelName),
|
|
1089
1246
|
epoch,
|
|
@@ -1102,21 +1259,15 @@ export async function createTabCoordinator<E>(
|
|
|
1102
1259
|
},
|
|
1103
1260
|
}
|
|
1104
1261
|
: {}),
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
if (leading !== ending) return;
|
|
1109
|
-
leading = undefined;
|
|
1110
|
-
ending.end();
|
|
1111
|
-
setState({
|
|
1262
|
+
// A newer build asked for the core.
|
|
1263
|
+
onNewerTab: () =>
|
|
1264
|
+
stepDown?.({
|
|
1112
1265
|
state: 'blocked',
|
|
1113
1266
|
reason: 'leader-incompatible',
|
|
1114
1267
|
code: LEADER_INCOMPATIBLE_CODE,
|
|
1115
1268
|
leader: 'newer',
|
|
1116
1269
|
retryable: true,
|
|
1117
|
-
})
|
|
1118
|
-
void leader.close().finally(() => lease.release());
|
|
1119
|
-
},
|
|
1270
|
+
}),
|
|
1120
1271
|
});
|
|
1121
1272
|
// Calls queued for the next leader run on this tab's own core.
|
|
1122
1273
|
follower?.handOver((method, args) => leader.call(tabId, method, args));
|
|
@@ -1140,6 +1291,16 @@ export async function createTabCoordinator<E>(
|
|
|
1140
1291
|
}
|
|
1141
1292
|
if (leading !== undefined)
|
|
1142
1293
|
return leading.leader.call(tabId, method, args);
|
|
1294
|
+
if (follower !== undefined) return follower.call(method, args);
|
|
1295
|
+
if (state.state === 'blocked' && state.reason === 'leadership-taken') {
|
|
1296
|
+
return Promise.reject(
|
|
1297
|
+
new ClientSyncError(
|
|
1298
|
+
LEADERSHIP_TAKEN_CODE,
|
|
1299
|
+
'another tab took over the core this tab ran',
|
|
1300
|
+
true,
|
|
1301
|
+
),
|
|
1302
|
+
);
|
|
1303
|
+
}
|
|
1143
1304
|
if (state.state === 'blocked' && state.reason === 'leader-incompatible') {
|
|
1144
1305
|
return Promise.reject(
|
|
1145
1306
|
new ClientSyncError(
|
|
@@ -1149,14 +1310,62 @@ export async function createTabCoordinator<E>(
|
|
|
1149
1310
|
),
|
|
1150
1311
|
);
|
|
1151
1312
|
}
|
|
1152
|
-
if (follower !== undefined) return follower.call(method, args);
|
|
1153
1313
|
return Promise.reject(
|
|
1154
1314
|
new ClientSyncError(WORKER_FAILED_CODE, 'no core serves this tab'),
|
|
1155
1315
|
);
|
|
1156
1316
|
},
|
|
1317
|
+
takeOver: async () => {
|
|
1318
|
+
if (closed) {
|
|
1319
|
+
throw new ClientSyncError(
|
|
1320
|
+
WORKER_FAILED_CODE,
|
|
1321
|
+
'the tab coordinator is closed',
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
const link = follower;
|
|
1325
|
+
if (
|
|
1326
|
+
link === undefined ||
|
|
1327
|
+
unreachableSince === undefined ||
|
|
1328
|
+
state.state !== 'blocked' ||
|
|
1329
|
+
state.reason !== 'leader-unreachable'
|
|
1330
|
+
) {
|
|
1331
|
+
throw takeOverRefused('not-unreachable');
|
|
1332
|
+
}
|
|
1333
|
+
if (takingOver) throw takeOverRefused('in-progress');
|
|
1334
|
+
const takeOverAt = unreachableSince + takeOverAfterMs;
|
|
1335
|
+
if (clock.now() < takeOverAt)
|
|
1336
|
+
throw takeOverRefused('too-early', takeOverAt);
|
|
1337
|
+
if (lock.steal === undefined) throw takeOverRefused('unsupported');
|
|
1338
|
+
takingOver = true;
|
|
1339
|
+
try {
|
|
1340
|
+
const lease = await lock.steal(config.name);
|
|
1341
|
+
if (closed) {
|
|
1342
|
+
await lease.release();
|
|
1343
|
+
return;
|
|
1344
|
+
}
|
|
1345
|
+
try {
|
|
1346
|
+
// Exceed every epoch this tab saw, so the old leader's late replies
|
|
1347
|
+
// and events are discarded everywhere.
|
|
1348
|
+
await lead(lease, link.maxEpochSeen + 1, true);
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
await lease.release();
|
|
1351
|
+
throw error;
|
|
1352
|
+
}
|
|
1353
|
+
if (!closed && leading === undefined) {
|
|
1354
|
+
// A third tab took the lock while this tab's core was starting.
|
|
1355
|
+
throw new ClientSyncError(
|
|
1356
|
+
LEADERSHIP_TAKEN_CODE,
|
|
1357
|
+
'another tab took over the core this tab ran',
|
|
1358
|
+
true,
|
|
1359
|
+
);
|
|
1360
|
+
}
|
|
1361
|
+
} finally {
|
|
1362
|
+
takingOver = false;
|
|
1363
|
+
}
|
|
1364
|
+
},
|
|
1157
1365
|
close: async () => {
|
|
1158
1366
|
if (closed) return;
|
|
1159
1367
|
closed = true;
|
|
1368
|
+
endUnreachable('closed');
|
|
1160
1369
|
follower?.close();
|
|
1161
1370
|
follower = undefined;
|
|
1162
1371
|
const ending = leading;
|
|
@@ -1177,7 +1386,7 @@ export async function createTabCoordinator<E>(
|
|
|
1177
1386
|
: await lock.acquire(config.name);
|
|
1178
1387
|
if (lease !== undefined) {
|
|
1179
1388
|
try {
|
|
1180
|
-
await lead(lease, 0);
|
|
1389
|
+
await lead(lease, 0, false);
|
|
1181
1390
|
} catch (error) {
|
|
1182
1391
|
await lease.release();
|
|
1183
1392
|
await tabLease.release();
|
|
@@ -1193,15 +1402,16 @@ export async function createTabCoordinator<E>(
|
|
|
1193
1402
|
onEvent: (event) => config.onEvent?.(event),
|
|
1194
1403
|
onLeaderChange: () => {},
|
|
1195
1404
|
onStateChange: setState,
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
: {}),
|
|
1405
|
+
callTimeoutMs,
|
|
1406
|
+
schedule: clock.schedule,
|
|
1199
1407
|
});
|
|
1200
1408
|
follower = link;
|
|
1201
|
-
// Granted only when the
|
|
1409
|
+
// Granted only when the lock is released: the leader's tab closed, or this
|
|
1410
|
+
// tab's own failed takeover let go of it.
|
|
1202
1411
|
void lock.acquire(config.name).then(async (promoted) => {
|
|
1203
|
-
if (closed) {
|
|
1204
|
-
//
|
|
1412
|
+
if (closed || follower !== link) {
|
|
1413
|
+
// Closed, or this tab already leads through takeOver(): promoting would
|
|
1414
|
+
// hold the lock forever.
|
|
1205
1415
|
await promoted.release();
|
|
1206
1416
|
return;
|
|
1207
1417
|
}
|
|
@@ -1210,7 +1420,7 @@ export async function createTabCoordinator<E>(
|
|
|
1210
1420
|
const epoch = link.maxEpochSeen + 1;
|
|
1211
1421
|
link.unbind();
|
|
1212
1422
|
try {
|
|
1213
|
-
await lead(promoted, epoch);
|
|
1423
|
+
await lead(promoted, epoch, false);
|
|
1214
1424
|
} catch {
|
|
1215
1425
|
// The core failed to start: release so the next tab tries.
|
|
1216
1426
|
await promoted.release();
|
package/src/reactive-store.ts
CHANGED
|
@@ -377,7 +377,8 @@ function localReadsBlocked(availability: SyncAvailability): boolean {
|
|
|
377
377
|
return (
|
|
378
378
|
availability.state === 'blocked' &&
|
|
379
379
|
(availability.reason === 'leader-unreachable' ||
|
|
380
|
-
availability.reason === 'leader-incompatible'
|
|
380
|
+
availability.reason === 'leader-incompatible' ||
|
|
381
|
+
availability.reason === 'leadership-taken')
|
|
381
382
|
);
|
|
382
383
|
}
|
|
383
384
|
|
package/src/wasm-database.ts
CHANGED
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
ClientSyncError,
|
|
37
37
|
invalidRequest,
|
|
38
38
|
STORAGE_BUSY_CODE,
|
|
39
|
+
STORAGE_FENCED_CODE,
|
|
39
40
|
STORAGE_UNAVAILABLE_CODE,
|
|
40
41
|
} from './errors';
|
|
41
42
|
|
|
@@ -121,6 +122,15 @@ function coerceParams(params: readonly SqlValue[]): unknown[] {
|
|
|
121
122
|
});
|
|
122
123
|
}
|
|
123
124
|
|
|
125
|
+
/**
|
|
126
|
+
* Set when another worker took the pool's storage lock. Shared by a database
|
|
127
|
+
* and its siblings: every one of them refuses statements from then on.
|
|
128
|
+
*/
|
|
129
|
+
interface StorageFence {
|
|
130
|
+
fenced: boolean;
|
|
131
|
+
readonly siblings: Set<Oo1Database>;
|
|
132
|
+
}
|
|
133
|
+
|
|
124
134
|
class WasmClientDatabase implements ClientDatabase {
|
|
125
135
|
readonly #db: Oo1Database;
|
|
126
136
|
readonly #sqlite3: Sqlite3Static;
|
|
@@ -128,6 +138,7 @@ class WasmClientDatabase implements ClientDatabase {
|
|
|
128
138
|
readonly #sah:
|
|
129
139
|
| { readonly util: SahPoolUtil; readonly lease: LeaderLease }
|
|
130
140
|
| undefined;
|
|
141
|
+
readonly #fence: StorageFence | undefined;
|
|
131
142
|
#closed = false;
|
|
132
143
|
#tx = { depth: 0 };
|
|
133
144
|
|
|
@@ -135,13 +146,35 @@ class WasmClientDatabase implements ClientDatabase {
|
|
|
135
146
|
db: Oo1Database,
|
|
136
147
|
sqlite3: Sqlite3Static,
|
|
137
148
|
sah?: { readonly util: SahPoolUtil; readonly lease: LeaderLease },
|
|
149
|
+
fence?: StorageFence,
|
|
138
150
|
) {
|
|
139
151
|
this.#db = db;
|
|
140
152
|
this.#sqlite3 = sqlite3;
|
|
141
153
|
this.#sah = sah;
|
|
154
|
+
this.#fence = fence;
|
|
155
|
+
// A takeover stole the storage lock. The new owner cannot open the pool
|
|
156
|
+
// while these SAHs are held, so close them before any further statement:
|
|
157
|
+
// the lost lease runs as its own task, never inside a transaction.
|
|
158
|
+
void sah?.lease.lost?.then(() => {
|
|
159
|
+
if (fence === undefined || this.#closed) return;
|
|
160
|
+
fence.fenced = true;
|
|
161
|
+
for (const sibling of fence.siblings) sibling.close();
|
|
162
|
+
fence.siblings.clear();
|
|
163
|
+
this.close();
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
#assertOwned(): void {
|
|
168
|
+
if (this.#fence?.fenced === true) {
|
|
169
|
+
throw new ClientSyncError(
|
|
170
|
+
STORAGE_FENCED_CODE,
|
|
171
|
+
'another tab took over this persistent database',
|
|
172
|
+
);
|
|
173
|
+
}
|
|
142
174
|
}
|
|
143
175
|
|
|
144
176
|
exec(sql: string, params: readonly SqlValue[] = []): void {
|
|
177
|
+
this.#assertOwned();
|
|
145
178
|
this.#db.exec({
|
|
146
179
|
sql,
|
|
147
180
|
...(params.length > 0 ? { bind: coerceParams(params) } : {}),
|
|
@@ -151,6 +184,7 @@ class WasmClientDatabase implements ClientDatabase {
|
|
|
151
184
|
}
|
|
152
185
|
|
|
153
186
|
query(sql: string, params: readonly SqlValue[] = []): SqlRow[] {
|
|
187
|
+
this.#assertOwned();
|
|
154
188
|
const resultRows: unknown[] = [];
|
|
155
189
|
this.#db.exec({
|
|
156
190
|
sql,
|
|
@@ -162,6 +196,7 @@ class WasmClientDatabase implements ClientDatabase {
|
|
|
162
196
|
}
|
|
163
197
|
|
|
164
198
|
transaction<T>(fn: () => T): T {
|
|
199
|
+
this.#assertOwned();
|
|
165
200
|
return runTransaction(this.#tx, (sql) => this.exec(sql), fn);
|
|
166
201
|
}
|
|
167
202
|
|
|
@@ -183,9 +218,18 @@ class WasmClientDatabase implements ClientDatabase {
|
|
|
183
218
|
db.close();
|
|
184
219
|
throw error;
|
|
185
220
|
}
|
|
221
|
+
this.#fence?.siblings.add(db);
|
|
186
222
|
return {
|
|
187
|
-
database: new WasmClientDatabase(
|
|
188
|
-
|
|
223
|
+
database: new WasmClientDatabase(
|
|
224
|
+
db,
|
|
225
|
+
this.#sqlite3,
|
|
226
|
+
undefined,
|
|
227
|
+
this.#fence,
|
|
228
|
+
),
|
|
229
|
+
close: () => {
|
|
230
|
+
this.#fence?.siblings.delete(db);
|
|
231
|
+
db.close();
|
|
232
|
+
},
|
|
189
233
|
removeFile: () => {
|
|
190
234
|
sah.util.unlink(fileName);
|
|
191
235
|
},
|
|
@@ -289,6 +333,12 @@ export interface PersistentWasmDatabaseOptions {
|
|
|
289
333
|
readonly directory?: string;
|
|
290
334
|
/** Initial SAH pool capacity (sqlite-wasm default: 6). */
|
|
291
335
|
readonly initialCapacity?: number;
|
|
336
|
+
/**
|
|
337
|
+
* Take the storage lock from the worker that holds it, which fences that
|
|
338
|
+
* worker's database. Its SAHs stay held until it runs again, so the open
|
|
339
|
+
* fails with a retryable `client.storage_busy` until then.
|
|
340
|
+
*/
|
|
341
|
+
readonly takeOver?: boolean;
|
|
292
342
|
}
|
|
293
343
|
|
|
294
344
|
function inWorkerContext(): boolean {
|
|
@@ -444,9 +494,12 @@ export async function openPersistentWasmDatabase(
|
|
|
444
494
|
);
|
|
445
495
|
// The document's leader lease can end before its worker's SAHs close. This
|
|
446
496
|
// worker-owned lock has exactly the same lifetime as the physical pool.
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
|
|
497
|
+
const storageLock = webLocksLeaderLock();
|
|
498
|
+
const lockName = `syncular-opfs/${directory}`;
|
|
499
|
+
const lease =
|
|
500
|
+
options?.takeOver === true
|
|
501
|
+
? await storageLock.steal(lockName)
|
|
502
|
+
: await storageLock.acquire(lockName);
|
|
450
503
|
let util: SahPoolUtil | undefined;
|
|
451
504
|
let db: Oo1Database | undefined;
|
|
452
505
|
try {
|
|
@@ -473,7 +526,12 @@ export async function openPersistentWasmDatabase(
|
|
|
473
526
|
await util.unpauseVfs();
|
|
474
527
|
db = new util.OpfsSAHPoolDb(`/${name}.db`);
|
|
475
528
|
configureSahCrashRecovery(db, sqlite3);
|
|
476
|
-
return new WasmClientDatabase(
|
|
529
|
+
return new WasmClientDatabase(
|
|
530
|
+
db,
|
|
531
|
+
sqlite3,
|
|
532
|
+
{ util, lease },
|
|
533
|
+
{ fenced: false, siblings: new Set() },
|
|
534
|
+
);
|
|
477
535
|
} catch (error) {
|
|
478
536
|
db?.close();
|
|
479
537
|
util?.pauseVfs();
|
package/src/worker-entry.ts
CHANGED