@procedures/store 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +109 -2
- package/dist/.tsbuildinfo +1 -1
- package/dist/indexeddb/idb.d.ts +100 -0
- package/dist/indexeddb/idb.d.ts.map +1 -0
- package/dist/indexeddb/idb.js +137 -0
- package/dist/indexeddb/idb.js.map +1 -0
- package/dist/indexeddb/program-store.d.ts +42 -0
- package/dist/indexeddb/program-store.d.ts.map +1 -0
- package/dist/indexeddb/program-store.js +71 -0
- package/dist/indexeddb/program-store.js.map +1 -0
- package/dist/indexeddb/queue-store.d.ts +38 -0
- package/dist/indexeddb/queue-store.d.ts.map +1 -0
- package/dist/indexeddb/queue-store.js +73 -0
- package/dist/indexeddb/queue-store.js.map +1 -0
- package/dist/indexeddb.d.ts +19 -0
- package/dist/indexeddb.d.ts.map +1 -0
- package/dist/indexeddb.js +17 -0
- package/dist/indexeddb.js.map +1 -0
- package/dist/program-store/file.d.ts +5 -0
- package/dist/program-store/file.d.ts.map +1 -1
- package/dist/program-store/file.js +5 -2
- package/dist/program-store/file.js.map +1 -1
- package/dist/program-store/procedures.d.ts.map +1 -1
- package/dist/program-store/procedures.js +5 -2
- package/dist/program-store/procedures.js.map +1 -1
- package/dist/storage/api.d.ts +18 -7
- package/dist/storage/api.d.ts.map +1 -1
- package/dist/storage/api.js +34 -3
- package/dist/storage/api.js.map +1 -1
- package/dist/storage/factory.d.ts +13 -2
- package/dist/storage/factory.d.ts.map +1 -1
- package/dist/storage/factory.js +12 -4
- package/dist/storage/factory.js.map +1 -1
- package/dist/storage/file-queue.d.ts +32 -0
- package/dist/storage/file-queue.d.ts.map +1 -0
- package/dist/storage/file-queue.js +101 -0
- package/dist/storage/file-queue.js.map +1 -0
- package/dist/storage/hybrid.d.ts +355 -42
- package/dist/storage/hybrid.d.ts.map +1 -1
- package/dist/storage/hybrid.js +1093 -250
- package/dist/storage/hybrid.js.map +1 -1
- package/dist/storage/index.d.ts +4 -2
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +5 -1
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/queue.d.ts +67 -0
- package/dist/storage/queue.d.ts.map +1 -0
- package/dist/storage/queue.js +85 -0
- package/dist/storage/queue.js.map +1 -0
- package/dist/storage/synced-registry.d.ts +18 -2
- package/dist/storage/synced-registry.d.ts.map +1 -1
- package/dist/storage/synced-registry.js +37 -3
- package/dist/storage/synced-registry.js.map +1 -1
- package/dist/storage/types.d.ts +5 -2
- package/dist/storage/types.d.ts.map +1 -1
- package/package.json +16 -6
- package/src/indexeddb/idb.ts +190 -0
- package/src/indexeddb/program-store.ts +80 -0
- package/src/indexeddb/queue-store.ts +80 -0
- package/src/indexeddb.ts +19 -0
- package/src/program-store/file.ts +5 -2
- package/src/program-store/procedures.ts +5 -2
- package/src/storage/api.ts +50 -10
- package/src/storage/factory.ts +21 -4
- package/src/storage/file-queue.ts +103 -0
- package/src/storage/hybrid.ts +1142 -274
- package/src/storage/index.ts +15 -2
- package/src/storage/queue.ts +128 -0
- package/src/storage/synced-registry.ts +49 -4
- package/src/storage/types.ts +6 -2
package/dist/storage/hybrid.js
CHANGED
|
@@ -1,36 +1,111 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Hybrid Storage Implementation
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* A local copy (an `InMemoryStorage`, or the `local` storage that the application gives) in
|
|
5
|
+
* front of a remote `CollectionStorage` (often an `ApiStorage`). The storage has four modes, and
|
|
6
|
+
* `setMode()` changes the mode while the storage operates (roadmap R28):
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* -
|
|
11
|
-
* -
|
|
12
|
-
* -
|
|
8
|
+
* | Mode | Reads | Writes | Remote calls |
|
|
9
|
+
* |---|---|---|---|
|
|
10
|
+
* | `local-only` | the local copy | the local copy and the queue | none |
|
|
11
|
+
* | `write-through` | the local copy, the remote on a miss | the local copy, then the remote | yes |
|
|
12
|
+
* | `write-back` | the local copy, the remote on a miss | the local copy and the queue, a timer sends it | yes |
|
|
13
|
+
* | `remote-only` | the remote | the remote | yes, and no local copy |
|
|
14
|
+
*
|
|
15
|
+
* - The queue is durable through the `queueStore` that the application gives.
|
|
16
|
+
* - A sync asks for the changes since a version when the remote has `getChangesSince`.
|
|
17
|
+
* - With `versionOf`, a conflict is a change on both sides, and `conflictResolution` decides.
|
|
18
|
+
* - `pullInterval` keeps the local copy fresh.
|
|
13
19
|
*/
|
|
14
20
|
import { InMemoryStorage } from "@procedures/collections";
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
21
|
+
import { optionalMs } from "@procedures/client";
|
|
22
|
+
import { MemoryQueueStore, idsOf, withoutId } from "./queue.js";
|
|
23
|
+
const MODES = new Set(["local-only", "write-through", "write-back", "remote-only"]);
|
|
24
|
+
/**
|
|
25
|
+
* The remote did not take a write of the write-through mode. The write is in the local cache and
|
|
26
|
+
* in the queue, and the storage sends it later (roadmap R2). `cause` is the error of the remote.
|
|
27
|
+
*/
|
|
28
|
+
export class PendingWriteError extends Error {
|
|
29
|
+
name = "PendingWriteError";
|
|
30
|
+
code = "WRITE_PENDING";
|
|
31
|
+
pending = true;
|
|
32
|
+
constructor(operation, cause) {
|
|
33
|
+
super(`The remote did not take the ${operation}: it waits in the queue`, { cause });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** The queue of operations is full, so the storage refuses the write (roadmap R2). */
|
|
37
|
+
export class QueueFullError extends Error {
|
|
38
|
+
name = "QueueFullError";
|
|
39
|
+
code = "QUEUE_FULL";
|
|
40
|
+
constructor(max) {
|
|
41
|
+
super(`The queue of operations is full (${max} operations): the storage refuses the write`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** The remote gave items without a key, so a sync cannot know which local items to keep. */
|
|
45
|
+
export class MissingKeysError extends Error {
|
|
46
|
+
name = "MissingKeysError";
|
|
47
|
+
code = "MISSING_KEYS";
|
|
48
|
+
constructor(count) {
|
|
49
|
+
super(`${count} remote items have no key. Give the option keyOf, or a remote storage with getEntries(). The sync changed nothing.`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The queue store failed (roadmap R28). A write that the store did not keep is undone and not
|
|
54
|
+
* acknowledged. A storage whose store cannot load refuses each operation.
|
|
55
|
+
*/
|
|
56
|
+
export class QueueStoreError extends Error {
|
|
57
|
+
name = "QueueStoreError";
|
|
58
|
+
code = "QUEUE_STORE";
|
|
59
|
+
constructor(action, cause) {
|
|
60
|
+
super(`The queue store could not ${action === "load" ? "load the queue" : "keep the queue"}`, { cause });
|
|
28
61
|
}
|
|
29
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* The sync found conflicts and `conflictResolution` is "error" (roadmap R28). The local values and
|
|
65
|
+
* their queued writes stay. `ids` are the ids of the conflicts.
|
|
66
|
+
*/
|
|
67
|
+
export class ConflictError extends Error {
|
|
68
|
+
ids;
|
|
69
|
+
name = "ConflictError";
|
|
70
|
+
code = "SYNC_CONFLICT";
|
|
71
|
+
constructor(ids) {
|
|
72
|
+
super(`Conflict detected between local and remote versions of ${ids.length} items: ${ids.slice(0, 10).join(", ")}`);
|
|
73
|
+
this.ids = ids;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* `setMode()` could not change the mode (roadmap R28). The mode did not change. `cause` is the
|
|
78
|
+
* error of the flush or of the sync.
|
|
79
|
+
*/
|
|
80
|
+
export class ModeSwitchError extends Error {
|
|
81
|
+
from;
|
|
82
|
+
to;
|
|
83
|
+
name = "ModeSwitchError";
|
|
84
|
+
code = "MODE_SWITCH_FAILED";
|
|
85
|
+
constructor(from, to, cause) {
|
|
86
|
+
super(`The storage could not change from ${from} to ${to}: the mode is still ${from}`, { cause });
|
|
87
|
+
this.from = from;
|
|
88
|
+
this.to = to;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const defaultIsUnavailable = (error) => {
|
|
92
|
+
if (typeof error !== "object" || error === null)
|
|
93
|
+
return true;
|
|
94
|
+
const { retryable, code } = error;
|
|
95
|
+
return retryable === true || code === undefined;
|
|
96
|
+
};
|
|
97
|
+
/** True when a remote does not know a procedure: an older server without `changes`. */
|
|
98
|
+
const isNotFound = (error) => typeof error === "object" && error !== null && error.code === "NOT_FOUND";
|
|
30
99
|
const defaultKeyOf = (item) => {
|
|
31
100
|
const id = item?.id;
|
|
32
101
|
return typeof id === "string" && id.length > 0 ? id : undefined;
|
|
33
102
|
};
|
|
103
|
+
function checkMode(mode) {
|
|
104
|
+
if (typeof mode !== "string" || !MODES.has(mode)) {
|
|
105
|
+
throw new TypeError(`Unknown mode ${JSON.stringify(mode)}: use local-only, write-through, write-back or remote-only`);
|
|
106
|
+
}
|
|
107
|
+
return mode;
|
|
108
|
+
}
|
|
34
109
|
/**
|
|
35
110
|
* Hybrid storage combining local cache with remote persistence.
|
|
36
111
|
*
|
|
@@ -42,9 +117,9 @@ const defaultKeyOf = (item) => {
|
|
|
42
117
|
*
|
|
43
118
|
* const remote = new ApiStorage(client, { collection: "users" });
|
|
44
119
|
* const hybrid = new HybridStorage(remote, {
|
|
45
|
-
*
|
|
120
|
+
* mode: "write-through",
|
|
46
121
|
* conflictResolution: "remote",
|
|
47
|
-
* syncInterval:
|
|
122
|
+
* syncInterval: seconds(5)
|
|
48
123
|
* });
|
|
49
124
|
*
|
|
50
125
|
* // Fast local reads
|
|
@@ -52,20 +127,58 @@ const defaultKeyOf = (item) => {
|
|
|
52
127
|
*
|
|
53
128
|
* // Writes go to both local and remote
|
|
54
129
|
* await hybrid.set("123", { name: "John" });
|
|
130
|
+
*
|
|
131
|
+
* // Offline: no remote call. The writes wait in the queue.
|
|
132
|
+
* await hybrid.setMode("local-only");
|
|
133
|
+
* // Online again: the switch sends the queue, then syncs the local copy
|
|
134
|
+
* await hybrid.setMode("write-through");
|
|
55
135
|
* ```
|
|
56
136
|
*/
|
|
57
137
|
export class HybridStorage {
|
|
58
138
|
local;
|
|
59
139
|
remote;
|
|
140
|
+
queueStore;
|
|
60
141
|
options;
|
|
142
|
+
mode;
|
|
61
143
|
offlineOps = [];
|
|
144
|
+
/** The operations in `offlineOps` that the queue store does not have yet */
|
|
145
|
+
unpersisted = new Set();
|
|
146
|
+
/** True when the queue store can differ from `offlineOps`: a failed write, or sent operations */
|
|
147
|
+
storeDirty = false;
|
|
148
|
+
/** The calls of the queue store, one at a time */
|
|
149
|
+
storeChain = Promise.resolve();
|
|
150
|
+
/** The push timer of the write-back mode */
|
|
62
151
|
syncTimer = null;
|
|
152
|
+
/** The periodic pull */
|
|
153
|
+
pullTimer = null;
|
|
154
|
+
/** True while pushQueue sends the queue: the first operation can be on its way */
|
|
155
|
+
pushing = false;
|
|
156
|
+
/** The timer that sends the queue of the write-through mode after a failure */
|
|
157
|
+
drainTimer = null;
|
|
158
|
+
/** The load of the queue. The writes wait for it. */
|
|
159
|
+
loaded;
|
|
160
|
+
/** The load and the first sync. The reads wait for it, so a read never sees an empty cache by mistake. */
|
|
161
|
+
ready;
|
|
162
|
+
/** The error of the load (of the queue store, or of the local copy): the storage refuses each operation */
|
|
163
|
+
loadError;
|
|
164
|
+
/** A mode switch that runs. The operations wait for it. */
|
|
165
|
+
gate = Promise.resolve();
|
|
166
|
+
/** The operations that run, and the switches that wait for them to end */
|
|
167
|
+
active = 0;
|
|
168
|
+
idleWaiters = [];
|
|
63
169
|
isOnline = true;
|
|
170
|
+
closed = false;
|
|
64
171
|
/** A counter of local writes for each id, so a sync does not restore an older remote value */
|
|
65
172
|
writeVersions = new Map();
|
|
66
173
|
clearVersion = 0;
|
|
67
174
|
/** The running sync: syncs run one at a time */
|
|
68
175
|
syncChain = Promise.resolve();
|
|
176
|
+
/** The remote version of the last delta pull */
|
|
177
|
+
cursor;
|
|
178
|
+
/** True when the remote has `getChangesSince` but its server does not know `changes` */
|
|
179
|
+
changesUnsupported = false;
|
|
180
|
+
/** With `versionOf`: the last known remote version of each id. `null`: the remote has no item. */
|
|
181
|
+
bases = new Map();
|
|
69
182
|
stats = {
|
|
70
183
|
cacheHits: 0,
|
|
71
184
|
cacheMisses: 0,
|
|
@@ -73,28 +186,135 @@ export class HybridStorage {
|
|
|
73
186
|
conflicts: 0,
|
|
74
187
|
};
|
|
75
188
|
constructor(remote, options = {}) {
|
|
76
|
-
this.local = new InMemoryStorage();
|
|
189
|
+
this.local = options.local ?? new InMemoryStorage();
|
|
77
190
|
this.remote = remote;
|
|
191
|
+
this.queueStore = options.queueStore ?? new MemoryQueueStore();
|
|
192
|
+
this.mode = checkMode(options.mode ?? options.writeStrategy ?? "write-through");
|
|
78
193
|
this.options = {
|
|
79
|
-
writeStrategy: options.writeStrategy ?? "write-through",
|
|
80
194
|
conflictResolution: options.conflictResolution ?? "remote",
|
|
81
195
|
...(options.mergeFn !== undefined && { mergeFn: options.mergeFn }),
|
|
82
196
|
keyOf: options.keyOf ?? defaultKeyOf,
|
|
83
|
-
|
|
197
|
+
...(options.versionOf !== undefined && { versionOf: options.versionOf }),
|
|
198
|
+
syncInterval: optionalMs(options.syncInterval, "syncInterval") ?? 5000,
|
|
199
|
+
pullInterval: optionalMs(options.pullInterval, "pullInterval") ?? 0,
|
|
84
200
|
syncOnInit: options.syncOnInit ?? true,
|
|
85
201
|
maxOfflineOps: options.maxOfflineOps ?? 1000,
|
|
86
202
|
offlineQueue: options.offlineQueue ?? true,
|
|
203
|
+
isUnavailable: options.isUnavailable ?? defaultIsUnavailable,
|
|
87
204
|
};
|
|
88
|
-
//
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
205
|
+
// The queue first, then the first sync. The reads wait for both (roadmap R2, R28).
|
|
206
|
+
this.loaded = this.loadQueue();
|
|
207
|
+
this.ready = this.loaded.then(() => this.initialSync());
|
|
208
|
+
this.updateTimers();
|
|
209
|
+
}
|
|
210
|
+
//
|
|
211
|
+
// ═══ Modes ═══
|
|
212
|
+
//
|
|
213
|
+
/** The mode of the storage now. */
|
|
214
|
+
getMode() {
|
|
215
|
+
return this.mode;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* This method changes the mode while the storage operates (roadmap R28).
|
|
219
|
+
*
|
|
220
|
+
* - The switch waits for the operations that run. The operations that start later wait for
|
|
221
|
+
* the switch. Switches and syncs run one at a time.
|
|
222
|
+
* - A switch to a remote mode sends the queue first. A switch to `local-only` makes no remote
|
|
223
|
+
* call: the queue stays until the next remote mode.
|
|
224
|
+
* - A switch to `remote-only` drops the local copy, after the queue reached the remote.
|
|
225
|
+
* - A switch from `local-only` or `remote-only` to a mode with a local copy syncs the local
|
|
226
|
+
* copy. The reads wait for the sync.
|
|
227
|
+
* - When the flush or the sync fails, the mode does not change, and the switch throws
|
|
228
|
+
* `ModeSwitchError`.
|
|
229
|
+
*/
|
|
230
|
+
async setMode(mode) {
|
|
231
|
+
const target = checkMode(mode);
|
|
232
|
+
const previous = this.gate;
|
|
233
|
+
let release;
|
|
234
|
+
this.gate = new Promise((resolve) => (release = resolve));
|
|
235
|
+
try {
|
|
236
|
+
await previous;
|
|
237
|
+
await this.loaded;
|
|
238
|
+
if (this.loadError)
|
|
239
|
+
throw this.loadError;
|
|
240
|
+
await this.idle();
|
|
241
|
+
await this.serialize(() => this.switchTo(target));
|
|
242
|
+
}
|
|
243
|
+
finally {
|
|
244
|
+
release();
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
async switchTo(target) {
|
|
248
|
+
const from = this.mode;
|
|
249
|
+
if (target === from)
|
|
250
|
+
return;
|
|
251
|
+
try {
|
|
252
|
+
if (target !== "local-only") {
|
|
253
|
+
// The queue reaches the remote before the mode changes
|
|
254
|
+
await this.pushQueue();
|
|
255
|
+
await this.ensureStoreClean();
|
|
256
|
+
}
|
|
257
|
+
if (target === "remote-only") {
|
|
258
|
+
await this.dropLocal();
|
|
259
|
+
}
|
|
260
|
+
else if (from === "remote-only" || from === "local-only") {
|
|
261
|
+
if (from === "remote-only")
|
|
262
|
+
await this.rebuildLocal();
|
|
263
|
+
await this.pullRemote(from === "remote-only");
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
throw new ModeSwitchError(from, target, error);
|
|
268
|
+
}
|
|
269
|
+
this.mode = target;
|
|
270
|
+
this.updateTimers();
|
|
271
|
+
}
|
|
272
|
+
/** This method drops the local copy. A read in flight does not fill it again. */
|
|
273
|
+
async dropLocal() {
|
|
274
|
+
this.clearVersion++;
|
|
275
|
+
this.cursor = undefined;
|
|
276
|
+
await this.local.clear();
|
|
277
|
+
}
|
|
278
|
+
/** This method makes the local copy from the queue only, before a full sync. */
|
|
279
|
+
async rebuildLocal() {
|
|
280
|
+
await this.dropLocal();
|
|
281
|
+
for (const op of this.offlineOps)
|
|
282
|
+
await this.applyToLocal(op);
|
|
283
|
+
}
|
|
284
|
+
/** This method starts and stops the timers for the mode. */
|
|
285
|
+
updateTimers() {
|
|
286
|
+
if (this.closed)
|
|
287
|
+
return;
|
|
288
|
+
const push = this.mode === "write-back" && this.options.syncInterval > 0;
|
|
289
|
+
if (push && !this.syncTimer) {
|
|
290
|
+
this.syncTimer = setInterval(() => {
|
|
291
|
+
this.syncToRemote().catch((err) => {
|
|
292
|
+
console.error("Periodic sync failed:", err);
|
|
293
|
+
});
|
|
294
|
+
}, this.options.syncInterval);
|
|
295
|
+
// The timer must not keep the process alive (deep dive DATA-9)
|
|
296
|
+
this.syncTimer.unref?.();
|
|
93
297
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
this.
|
|
97
|
-
|
|
298
|
+
else if (!push && this.syncTimer) {
|
|
299
|
+
clearInterval(this.syncTimer);
|
|
300
|
+
this.syncTimer = null;
|
|
301
|
+
}
|
|
302
|
+
const pull = this.options.pullInterval > 0 && (this.mode === "write-through" || this.mode === "write-back");
|
|
303
|
+
if (pull && !this.pullTimer) {
|
|
304
|
+
this.pullTimer = setInterval(() => {
|
|
305
|
+
this.syncFromRemote().catch((err) => {
|
|
306
|
+
console.error("Periodic pull failed:", err);
|
|
307
|
+
});
|
|
308
|
+
}, this.options.pullInterval);
|
|
309
|
+
this.pullTimer.unref?.();
|
|
310
|
+
}
|
|
311
|
+
else if (!pull && this.pullTimer) {
|
|
312
|
+
clearInterval(this.pullTimer);
|
|
313
|
+
this.pullTimer = null;
|
|
314
|
+
}
|
|
315
|
+
if (this.mode !== "write-through" && this.drainTimer) {
|
|
316
|
+
clearTimeout(this.drainTimer);
|
|
317
|
+
this.drainTimer = null;
|
|
98
318
|
}
|
|
99
319
|
}
|
|
100
320
|
//
|
|
@@ -102,188 +322,272 @@ export class HybridStorage {
|
|
|
102
322
|
//
|
|
103
323
|
/**
|
|
104
324
|
* The local item, or else the remote item (which the cache then keeps). When the remote
|
|
105
|
-
* cannot answer, the storage is offline and the result is undefined.
|
|
325
|
+
* cannot answer, the storage is offline and the result is undefined. In `local-only` mode, a
|
|
326
|
+
* miss is undefined. In `remote-only` mode, the remote answers, and its error passes through.
|
|
106
327
|
*/
|
|
107
328
|
async get(id) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
if (this.hasPendingOp(id)) {
|
|
117
|
-
return undefined;
|
|
118
|
-
}
|
|
119
|
-
const version = this.versionOf(id);
|
|
120
|
-
try {
|
|
121
|
-
const remote = await this.remote.get(id);
|
|
122
|
-
this.isOnline = true;
|
|
123
|
-
if (remote !== undefined && this.versionOf(id) === version) {
|
|
124
|
-
await this.local.set(id, remote);
|
|
329
|
+
return this.run(true, async () => {
|
|
330
|
+
if (this.mode === "remote-only")
|
|
331
|
+
return this.remoteRead(() => this.remote.get(id));
|
|
332
|
+
// Try local cache first
|
|
333
|
+
const cached = await this.local.get(id);
|
|
334
|
+
if (cached !== undefined) {
|
|
335
|
+
this.stats.cacheHits++;
|
|
336
|
+
return cached;
|
|
125
337
|
}
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
338
|
+
this.stats.cacheMisses++;
|
|
339
|
+
// A pending local delete means "no item": do not read it back from the remote
|
|
340
|
+
if (this.mode === "local-only" || this.hasPendingOp(id)) {
|
|
341
|
+
return undefined;
|
|
342
|
+
}
|
|
343
|
+
const version = this.writeKey(id);
|
|
344
|
+
try {
|
|
345
|
+
const remote = await this.remote.get(id);
|
|
346
|
+
this.isOnline = true;
|
|
347
|
+
if (this.writeKey(id) === version && !this.hasPendingOp(id)) {
|
|
348
|
+
if (remote !== undefined)
|
|
349
|
+
await this.local.set(id, remote);
|
|
350
|
+
this.noteBase(id, remote);
|
|
351
|
+
}
|
|
352
|
+
return remote;
|
|
353
|
+
}
|
|
354
|
+
catch {
|
|
355
|
+
this.isOnline = false;
|
|
356
|
+
return undefined;
|
|
357
|
+
}
|
|
358
|
+
});
|
|
132
359
|
}
|
|
133
360
|
async getAll() {
|
|
134
|
-
|
|
135
|
-
|
|
361
|
+
return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.getAll()) : this.local.getAll()));
|
|
362
|
+
}
|
|
363
|
+
/** Every item with its id. */
|
|
364
|
+
async getEntries() {
|
|
365
|
+
return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remoteEntries()) : this.local.getEntries()));
|
|
136
366
|
}
|
|
137
367
|
async find(predicate) {
|
|
138
|
-
|
|
139
|
-
return this.local.find(predicate);
|
|
368
|
+
return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.find(predicate)) : this.local.find(predicate));
|
|
140
369
|
}
|
|
141
370
|
async has(id) {
|
|
142
|
-
return this.local.has(id);
|
|
371
|
+
return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.has(id)) : this.local.has(id)));
|
|
143
372
|
}
|
|
144
373
|
async size() {
|
|
145
|
-
return this.local.size();
|
|
374
|
+
return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.size()) : this.local.size()));
|
|
375
|
+
}
|
|
376
|
+
async getBatch(ids) {
|
|
377
|
+
return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.getBatch(ids)) : this.local.getBatch(ids));
|
|
378
|
+
}
|
|
379
|
+
/** A read of the remote-only mode. A queue that a restart loaded reaches the remote first. */
|
|
380
|
+
async remoteRead(read) {
|
|
381
|
+
if (this.offlineOps.length > 0)
|
|
382
|
+
await this.serialize(() => this.pushQueue());
|
|
383
|
+
try {
|
|
384
|
+
const result = await read();
|
|
385
|
+
this.isOnline = true;
|
|
386
|
+
return result;
|
|
387
|
+
}
|
|
388
|
+
catch (error) {
|
|
389
|
+
if (!(error instanceof MissingKeysError))
|
|
390
|
+
this.isOnline = false;
|
|
391
|
+
throw error;
|
|
392
|
+
}
|
|
146
393
|
}
|
|
147
394
|
//
|
|
148
395
|
// ═══ Write Operations ═══
|
|
149
396
|
//
|
|
150
397
|
async set(id, value) {
|
|
151
|
-
this.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
// Write-through: sync to remote immediately
|
|
156
|
-
try {
|
|
157
|
-
await this.remote.set(id, value);
|
|
158
|
-
}
|
|
159
|
-
catch (error) {
|
|
160
|
-
this.handleRemoteFailure("set", { id, value });
|
|
161
|
-
throw error;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
else {
|
|
165
|
-
// Write-back: queue for later sync
|
|
166
|
-
this.queueOperation({
|
|
167
|
-
type: "set",
|
|
168
|
-
data: { id, value },
|
|
169
|
-
timestamp: Date.now(),
|
|
170
|
-
});
|
|
171
|
-
}
|
|
398
|
+
await this.write("set", { id, value }, [id], async () => {
|
|
399
|
+
this.bump(id);
|
|
400
|
+
await this.local.set(id, value);
|
|
401
|
+
}, () => this.remote.set(id, value));
|
|
172
402
|
}
|
|
173
403
|
async delete(id) {
|
|
174
|
-
this.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
await this.remote.delete(id);
|
|
179
|
-
}
|
|
180
|
-
catch (error) {
|
|
181
|
-
this.handleRemoteFailure("delete", { id });
|
|
182
|
-
throw error;
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
else {
|
|
186
|
-
this.queueOperation({
|
|
187
|
-
type: "delete",
|
|
188
|
-
data: { id },
|
|
189
|
-
timestamp: Date.now(),
|
|
190
|
-
});
|
|
191
|
-
}
|
|
192
|
-
return existed;
|
|
404
|
+
return this.write("delete", { id }, [id], async () => {
|
|
405
|
+
this.bump(id);
|
|
406
|
+
return this.local.delete(id);
|
|
407
|
+
}, () => this.remote.delete(id));
|
|
193
408
|
}
|
|
194
409
|
async clear() {
|
|
195
|
-
this.
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
await this.remote.clear();
|
|
200
|
-
}
|
|
201
|
-
catch (error) {
|
|
202
|
-
this.handleRemoteFailure("clear", {});
|
|
203
|
-
throw error;
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
else {
|
|
207
|
-
this.queueOperation({
|
|
208
|
-
type: "clear",
|
|
209
|
-
data: {},
|
|
210
|
-
timestamp: Date.now(),
|
|
211
|
-
});
|
|
212
|
-
}
|
|
410
|
+
await this.write("clear", {}, "*", async () => {
|
|
411
|
+
this.clearVersion++;
|
|
412
|
+
await this.local.clear();
|
|
413
|
+
}, () => this.remote.clear());
|
|
213
414
|
}
|
|
214
415
|
//
|
|
215
416
|
// ═══ Bulk Operations ═══
|
|
216
417
|
//
|
|
217
418
|
async setBatch(items) {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
await this.remote.setBatch(items);
|
|
224
|
-
}
|
|
225
|
-
catch (error) {
|
|
226
|
-
this.handleRemoteFailure("setBatch", { items });
|
|
227
|
-
throw error;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
else {
|
|
231
|
-
this.queueOperation({
|
|
232
|
-
type: "setBatch",
|
|
233
|
-
data: { items },
|
|
234
|
-
timestamp: Date.now(),
|
|
235
|
-
});
|
|
236
|
-
}
|
|
419
|
+
await this.write("setBatch", { items }, items.map(([id]) => id), async () => {
|
|
420
|
+
for (const [id] of items)
|
|
421
|
+
this.bump(id);
|
|
422
|
+
await this.local.setBatch(items);
|
|
423
|
+
}, () => this.remote.setBatch(items));
|
|
237
424
|
}
|
|
238
425
|
async deleteBatch(ids) {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
426
|
+
return this.write("deleteBatch", { ids }, ids, async () => {
|
|
427
|
+
for (const id of ids)
|
|
428
|
+
this.bump(id);
|
|
429
|
+
return this.local.deleteBatch(ids);
|
|
430
|
+
}, () => this.remote.deleteBatch(ids));
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* One write, in the mode of the storage. The result is the result of the local copy, or of the
|
|
434
|
+
* remote in `remote-only` mode.
|
|
435
|
+
*
|
|
436
|
+
* - `remote-only`: the remote only. Its error passes through.
|
|
437
|
+
* - `write-back` and `local-only`: the local copy and the queue. A full queue refuses the write
|
|
438
|
+
* before the local change. The queue store keeps the operation before the write ends.
|
|
439
|
+
* - `write-through`, the remote refuses the write: the local change is undone, and the error
|
|
440
|
+
* of the remote passes through.
|
|
441
|
+
* - `write-through`, the remote is not available: with the queue, the write waits in it, a timer
|
|
442
|
+
* sends it later, and the write throws `PendingWriteError`. Without the queue, or with a full
|
|
443
|
+
* queue, the local change is undone.
|
|
444
|
+
*
|
|
445
|
+
* Before, a failed write-through stayed in the local cache and in the queue, no timer sent the
|
|
446
|
+
* queue, a second try queued the operation again, and a full queue dropped the oldest operation
|
|
447
|
+
* without a word (roadmap R2).
|
|
448
|
+
*/
|
|
449
|
+
write(type, data, ids, applyLocal, applyRemote) {
|
|
450
|
+
return this.run(false, async () => {
|
|
451
|
+
const op = { type, data, timestamp: Date.now() };
|
|
452
|
+
if (this.mode === "remote-only") {
|
|
453
|
+
const send = async () => {
|
|
454
|
+
await this.ensureStoreClean();
|
|
455
|
+
const result = await applyRemote();
|
|
456
|
+
this.noteRemoteWrite(op);
|
|
457
|
+
return result;
|
|
458
|
+
};
|
|
459
|
+
try {
|
|
460
|
+
const result = this.offlineOps.length > 0
|
|
461
|
+
? await this.serialize(async () => {
|
|
462
|
+
await this.pushQueue();
|
|
463
|
+
return send();
|
|
464
|
+
})
|
|
465
|
+
: await send();
|
|
466
|
+
this.isOnline = true;
|
|
467
|
+
return result;
|
|
468
|
+
}
|
|
469
|
+
catch (error) {
|
|
470
|
+
if (!(error instanceof QueueStoreError) && this.options.isUnavailable(error))
|
|
471
|
+
this.isOnline = false;
|
|
472
|
+
throw error;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if (this.mode === "write-back" || this.mode === "local-only") {
|
|
476
|
+
this.ensureQueueRoom(op);
|
|
477
|
+
this.attachBase(op);
|
|
478
|
+
const undo = await this.snapshot(ids);
|
|
479
|
+
const result = await applyLocal();
|
|
480
|
+
await this.enqueue(op, undo);
|
|
481
|
+
return result;
|
|
482
|
+
}
|
|
483
|
+
const undo = await this.snapshot(ids);
|
|
484
|
+
const result = await applyLocal();
|
|
243
485
|
try {
|
|
244
|
-
|
|
486
|
+
if (this.offlineOps.length > 0) {
|
|
487
|
+
// The queued operations are older: they reach the remote first, or the older value of a
|
|
488
|
+
// queued write replaces this one later. A failed send keeps this write behind them.
|
|
489
|
+
await this.serialize(async () => {
|
|
490
|
+
await this.pushQueue();
|
|
491
|
+
await this.ensureStoreClean();
|
|
492
|
+
await applyRemote();
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
else {
|
|
496
|
+
await this.ensureStoreClean();
|
|
497
|
+
await applyRemote();
|
|
498
|
+
}
|
|
499
|
+
this.isOnline = true;
|
|
500
|
+
this.noteRemoteWrite(op);
|
|
245
501
|
}
|
|
246
502
|
catch (error) {
|
|
247
|
-
|
|
503
|
+
if (!(error instanceof QueueStoreError) &&
|
|
504
|
+
this.options.offlineQueue &&
|
|
505
|
+
this.options.isUnavailable(error) &&
|
|
506
|
+
this.hasQueueRoom(op)) {
|
|
507
|
+
this.isOnline = false;
|
|
508
|
+
console.error(`Remote ${type} failed: the operation waits in the queue`);
|
|
509
|
+
this.attachBase(op);
|
|
510
|
+
await this.enqueue(op, undo);
|
|
511
|
+
this.scheduleDrain();
|
|
512
|
+
throw new PendingWriteError(type, error);
|
|
513
|
+
}
|
|
514
|
+
await undo();
|
|
248
515
|
throw error;
|
|
249
516
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
this.queueOperation({
|
|
253
|
-
type: "deleteBatch",
|
|
254
|
-
data: { ids },
|
|
255
|
-
timestamp: Date.now(),
|
|
256
|
-
});
|
|
257
|
-
}
|
|
258
|
-
return deleted;
|
|
517
|
+
return result;
|
|
518
|
+
});
|
|
259
519
|
}
|
|
260
|
-
|
|
261
|
-
|
|
520
|
+
/**
|
|
521
|
+
* The local values of the ids before a write, and a function that puts them back. It puts back
|
|
522
|
+
* only the ids that no later write changed.
|
|
523
|
+
*/
|
|
524
|
+
async snapshot(ids) {
|
|
525
|
+
const before = ids === "*" ? new Map(await this.local.getEntries()) : await this.local.getBatch(ids);
|
|
526
|
+
const touched = ids === "*" ? [...before.keys()] : ids;
|
|
527
|
+
// The versions after the write: this write adds one to each id (clear adds to clearVersion)
|
|
528
|
+
const versions = new Map(touched.map((id) => [id, (this.writeVersions.get(id) ?? 0) + (ids === "*" ? 0 : 1)]));
|
|
529
|
+
const clearVersion = this.clearVersion + (ids === "*" ? 1 : 0);
|
|
530
|
+
return async () => {
|
|
531
|
+
if (this.clearVersion !== clearVersion)
|
|
532
|
+
return;
|
|
533
|
+
for (const id of touched) {
|
|
534
|
+
if ((this.writeVersions.get(id) ?? 0) !== versions.get(id))
|
|
535
|
+
continue;
|
|
536
|
+
const value = before.get(id);
|
|
537
|
+
if (value === undefined)
|
|
538
|
+
await this.local.delete(id);
|
|
539
|
+
else
|
|
540
|
+
await this.local.set(id, value);
|
|
541
|
+
}
|
|
542
|
+
};
|
|
262
543
|
}
|
|
263
544
|
//
|
|
264
545
|
// ═══ Sync Operations ═══
|
|
265
546
|
//
|
|
266
547
|
/**
|
|
267
|
-
* Sync
|
|
268
|
-
*
|
|
548
|
+
* Sync from the remote to the local copy, in the modes `write-through` and `write-back`. The
|
|
549
|
+
* other modes have no local copy to sync (`remote-only`) or make no remote call (`local-only`):
|
|
550
|
+
* the call does nothing.
|
|
269
551
|
*
|
|
270
|
-
* -
|
|
552
|
+
* - The sync asks for the changes since the version of the last sync when the remote has
|
|
553
|
+
* `getChangesSince`. Else it reads all entries.
|
|
554
|
+
* - Each remote item goes into the cache under its id.
|
|
271
555
|
* - A local item that the remote does not have is removed, unless a queued local operation
|
|
272
|
-
* changes it
|
|
556
|
+
* changes it.
|
|
273
557
|
* - An id that a local write changed during the sync keeps the local value: the remote value
|
|
274
558
|
* is older.
|
|
275
559
|
* - Syncs run one at a time.
|
|
276
560
|
*/
|
|
277
561
|
syncFromRemote() {
|
|
278
|
-
return this.serialize(() =>
|
|
562
|
+
return this.serialize(async () => {
|
|
563
|
+
await this.loaded;
|
|
564
|
+
if (this.loadError)
|
|
565
|
+
throw this.loadError;
|
|
566
|
+
if (this.mode === "write-through" || this.mode === "write-back")
|
|
567
|
+
await this.pullRemote(false);
|
|
568
|
+
});
|
|
279
569
|
}
|
|
280
570
|
/**
|
|
281
571
|
* Sync all pending offline operations to remote.
|
|
282
572
|
* Processes queued operations in order. When one fails, it and the later operations stay in
|
|
283
|
-
* the queue: the operations that reached the remote are not sent again.
|
|
573
|
+
* the queue: the operations that reached the remote are not sent again. In `local-only` mode,
|
|
574
|
+
* the call does nothing.
|
|
284
575
|
*/
|
|
285
576
|
syncToRemote() {
|
|
286
|
-
return this.serialize(() =>
|
|
577
|
+
return this.serialize(async () => {
|
|
578
|
+
await this.loaded;
|
|
579
|
+
if (this.loadError)
|
|
580
|
+
throw this.loadError;
|
|
581
|
+
if (this.mode !== "local-only")
|
|
582
|
+
await this.pushQueue();
|
|
583
|
+
});
|
|
584
|
+
}
|
|
585
|
+
/**
|
|
586
|
+
* This method sends the queue to the remote (roadmap R28). `ProcedureServer.useStorage` uses it. In
|
|
587
|
+
* `local-only` mode, it sends nothing: the queue waits for a remote mode.
|
|
588
|
+
*/
|
|
589
|
+
flush() {
|
|
590
|
+
return this.syncToRemote();
|
|
287
591
|
}
|
|
288
592
|
/**
|
|
289
593
|
* Force a full sync (both directions).
|
|
@@ -295,14 +599,26 @@ export class HybridStorage {
|
|
|
295
599
|
//
|
|
296
600
|
// ═══ Lifecycle & Metadata ═══
|
|
297
601
|
//
|
|
602
|
+
/**
|
|
603
|
+
* This method stops the timers, sends the queue (not in `local-only` mode), and closes the remote and the
|
|
604
|
+
* local copy. The queue store keeps the operations that the remote did not take. A durable local
|
|
605
|
+
* copy keeps its items.
|
|
606
|
+
*/
|
|
298
607
|
async close() {
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
608
|
+
this.closed = true;
|
|
609
|
+
for (const timer of [this.syncTimer, this.pullTimer]) {
|
|
610
|
+
if (timer)
|
|
611
|
+
clearInterval(timer);
|
|
612
|
+
}
|
|
613
|
+
this.syncTimer = null;
|
|
614
|
+
this.pullTimer = null;
|
|
615
|
+
if (this.drainTimer) {
|
|
616
|
+
clearTimeout(this.drainTimer);
|
|
617
|
+
this.drainTimer = null;
|
|
303
618
|
}
|
|
619
|
+
await this.loaded;
|
|
304
620
|
// Sync pending operations before closing
|
|
305
|
-
if (this.offlineOps.length > 0) {
|
|
621
|
+
if (this.offlineOps.length > 0 && this.mode !== "local-only") {
|
|
306
622
|
try {
|
|
307
623
|
await this.syncToRemote();
|
|
308
624
|
}
|
|
@@ -310,8 +626,14 @@ export class HybridStorage {
|
|
|
310
626
|
console.error("Failed to sync before close:", error);
|
|
311
627
|
}
|
|
312
628
|
}
|
|
629
|
+
await this.storeChain;
|
|
313
630
|
// Close remote storage (an ApiStorage leaves the caller's client open)
|
|
314
|
-
|
|
631
|
+
try {
|
|
632
|
+
await this.remote.close();
|
|
633
|
+
}
|
|
634
|
+
finally {
|
|
635
|
+
await this.local.close();
|
|
636
|
+
}
|
|
315
637
|
}
|
|
316
638
|
async getMetadata() {
|
|
317
639
|
const totalOps = this.stats.cacheHits + this.stats.cacheMisses;
|
|
@@ -328,7 +650,8 @@ export class HybridStorage {
|
|
|
328
650
|
conflicts: this.stats.conflicts,
|
|
329
651
|
pendingOps: this.offlineOps.length,
|
|
330
652
|
isOnline: this.isOnline,
|
|
331
|
-
|
|
653
|
+
mode: this.mode,
|
|
654
|
+
writeStrategy: this.mode,
|
|
332
655
|
conflictResolution: this.options.conflictResolution,
|
|
333
656
|
},
|
|
334
657
|
};
|
|
@@ -336,68 +659,378 @@ export class HybridStorage {
|
|
|
336
659
|
//
|
|
337
660
|
// ═══ Internal Helpers ═══
|
|
338
661
|
//
|
|
339
|
-
|
|
662
|
+
/**
|
|
663
|
+
* This method starts a public operation: after the load of the queue (a write) or the first sync (a read),
|
|
664
|
+
* and never during a mode switch.
|
|
665
|
+
*/
|
|
666
|
+
async run(read, work) {
|
|
667
|
+
await (read ? this.ready : this.loaded);
|
|
668
|
+
for (;;) {
|
|
669
|
+
const gate = this.gate;
|
|
670
|
+
await gate;
|
|
671
|
+
// No new switch started while this operation waited
|
|
672
|
+
if (this.gate === gate)
|
|
673
|
+
break;
|
|
674
|
+
}
|
|
675
|
+
if (this.loadError)
|
|
676
|
+
throw this.loadError;
|
|
677
|
+
this.active++;
|
|
678
|
+
try {
|
|
679
|
+
return await work();
|
|
680
|
+
}
|
|
681
|
+
finally {
|
|
682
|
+
if (--this.active === 0) {
|
|
683
|
+
for (const resolve of this.idleWaiters.splice(0))
|
|
684
|
+
resolve();
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
/** A promise that resolves when no operation runs. */
|
|
689
|
+
idle() {
|
|
690
|
+
if (this.active === 0)
|
|
691
|
+
return Promise.resolve();
|
|
692
|
+
return new Promise((resolve) => this.idleWaiters.push(resolve));
|
|
693
|
+
}
|
|
694
|
+
/** This method loads the queue, and puts its writes in the local copy. */
|
|
695
|
+
async loadQueue() {
|
|
696
|
+
let ops;
|
|
697
|
+
try {
|
|
698
|
+
ops = await this.queueStore.load();
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
this.loadError = new QueueStoreError("load", error);
|
|
702
|
+
console.error("The queue store could not load the queue: the storage refuses each operation", error);
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
this.offlineOps = ops.slice();
|
|
706
|
+
try {
|
|
707
|
+
for (const op of this.offlineOps) {
|
|
708
|
+
for (const [id, version] of op.base ?? []) {
|
|
709
|
+
if (!this.bases.has(id))
|
|
710
|
+
this.bases.set(id, version);
|
|
711
|
+
}
|
|
712
|
+
// A durable local copy can hold the operation already: each operation gives the same
|
|
713
|
+
// result two times
|
|
714
|
+
await this.applyToLocal(op);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
catch (error) {
|
|
718
|
+
this.loadError = new Error("The local copy could not take the loaded queue: the storage refuses each operation", { cause: error });
|
|
719
|
+
console.error(this.loadError.message, error);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
/** The first sync: send the loaded queue, then read the remote. */
|
|
723
|
+
async initialSync() {
|
|
724
|
+
if (this.loadError)
|
|
725
|
+
return;
|
|
726
|
+
if (this.options.syncOnInit && this.mode !== "local-only") {
|
|
727
|
+
if (this.offlineOps.length > 0) {
|
|
728
|
+
await this.syncToRemote().catch((err) => {
|
|
729
|
+
console.error("Failed to send the queue on init:", err);
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
if (this.mode === "write-through" || this.mode === "write-back") {
|
|
733
|
+
await this.syncFromRemote().catch((err) => {
|
|
734
|
+
console.error("Failed to sync on init:", err);
|
|
735
|
+
});
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
if (this.offlineOps.length > 0)
|
|
739
|
+
this.scheduleDrain();
|
|
740
|
+
}
|
|
741
|
+
/** This method applies a queued operation to the local copy. */
|
|
742
|
+
async applyToLocal(op) {
|
|
743
|
+
const data = op.data;
|
|
744
|
+
switch (op.type) {
|
|
745
|
+
case "set":
|
|
746
|
+
await this.local.set(data.id, data.value);
|
|
747
|
+
break;
|
|
748
|
+
case "delete":
|
|
749
|
+
await this.local.delete(data.id);
|
|
750
|
+
break;
|
|
751
|
+
case "setBatch":
|
|
752
|
+
await this.local.setBatch(data.items);
|
|
753
|
+
break;
|
|
754
|
+
case "deleteBatch":
|
|
755
|
+
await this.local.deleteBatch(data.ids);
|
|
756
|
+
break;
|
|
757
|
+
case "clear":
|
|
758
|
+
await this.local.clear();
|
|
759
|
+
break;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* One pull. `full`: ask for all items (after the local copy was dropped).
|
|
764
|
+
*
|
|
765
|
+
* - The versions of the local writes at the start: a write during the pull keeps its value.
|
|
766
|
+
* - The cursor moves only after a pull without conflicts, so a conflict of the "error" policy
|
|
767
|
+
* comes again at the next pull.
|
|
768
|
+
*/
|
|
769
|
+
async pullRemote(full) {
|
|
340
770
|
const versions = new Map(this.writeVersions);
|
|
341
771
|
const clearVersion = this.clearVersion;
|
|
342
772
|
// True when a local write changed the id after the sync started
|
|
343
773
|
const changedLocally = (id) => this.clearVersion !== clearVersion || (this.writeVersions.get(id) ?? 0) !== (versions.get(id) ?? 0);
|
|
344
|
-
let
|
|
774
|
+
let changes;
|
|
345
775
|
try {
|
|
346
|
-
|
|
776
|
+
changes = await this.fetchChanges(full);
|
|
347
777
|
}
|
|
348
778
|
catch (error) {
|
|
349
|
-
|
|
779
|
+
if (!(error instanceof MissingKeysError))
|
|
780
|
+
this.isOnline = false;
|
|
350
781
|
throw error;
|
|
351
782
|
}
|
|
783
|
+
const conflicts = [];
|
|
784
|
+
let queueChanged = false;
|
|
352
785
|
const remoteIds = new Set();
|
|
353
|
-
for (const item of
|
|
354
|
-
const id = this.options.keyOf(item);
|
|
355
|
-
if (!id) {
|
|
356
|
-
console.warn("Item has no key, skipping:", item);
|
|
357
|
-
continue;
|
|
358
|
-
}
|
|
786
|
+
for (const [id, item] of changes.changed) {
|
|
359
787
|
remoteIds.add(id);
|
|
360
|
-
if (
|
|
361
|
-
|
|
788
|
+
if (await this.pullItem(id, item, changedLocally, conflicts))
|
|
789
|
+
queueChanged = true;
|
|
790
|
+
}
|
|
791
|
+
// The remote deleted these items
|
|
792
|
+
const gone = changes.full
|
|
793
|
+
? (await this.local.getEntries()).map(([id]) => id).filter((id) => !remoteIds.has(id))
|
|
794
|
+
: changes.deleted;
|
|
795
|
+
for (const id of gone) {
|
|
796
|
+
if (await this.pullDelete(id, changedLocally, conflicts))
|
|
797
|
+
queueChanged = true;
|
|
798
|
+
}
|
|
799
|
+
if (queueChanged) {
|
|
800
|
+
this.storeDirty = true;
|
|
801
|
+
await this.persistAll().catch((error) => {
|
|
802
|
+
console.error("The queue store could not keep the queue after the sync:", error);
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
this.stats.syncCount++;
|
|
806
|
+
this.isOnline = true;
|
|
807
|
+
if (conflicts.length > 0)
|
|
808
|
+
throw new ConflictError(conflicts);
|
|
809
|
+
if (changes.version !== undefined)
|
|
810
|
+
this.cursor = changes.version;
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* The remote changes: since the cursor from `getChangesSince` when the remote has it, or else
|
|
814
|
+
* all entries.
|
|
815
|
+
*/
|
|
816
|
+
async fetchChanges(full) {
|
|
817
|
+
if (this.remote.getChangesSince && !this.changesUnsupported) {
|
|
818
|
+
try {
|
|
819
|
+
return await this.remote.getChangesSince(full ? undefined : this.cursor);
|
|
362
820
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
821
|
+
catch (error) {
|
|
822
|
+
if (!isNotFound(error))
|
|
823
|
+
throw error;
|
|
824
|
+
// A server without the procedure `changes`: read all entries from now on
|
|
825
|
+
this.changesUnsupported = true;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
return { full: true, changed: await this.remoteEntries(), deleted: [] };
|
|
829
|
+
}
|
|
830
|
+
/** This method applies one remote item. It gives true when the queue changed. */
|
|
831
|
+
async pullItem(id, item, changedLocally, conflicts) {
|
|
832
|
+
if (changedLocally(id))
|
|
833
|
+
return false;
|
|
834
|
+
const localItem = await this.local.get(id);
|
|
835
|
+
if (changedLocally(id))
|
|
836
|
+
return false;
|
|
837
|
+
const remoteVersion = this.versionOfItem(item);
|
|
838
|
+
if (this.hasPendingOp(id)) {
|
|
839
|
+
// Without versions, the queued write wins, as before
|
|
840
|
+
if (remoteVersion === undefined)
|
|
841
|
+
return false;
|
|
842
|
+
// The remote did not change since the last sync, or the storage never saw a remote version
|
|
843
|
+
// of the id: the queued write wins
|
|
844
|
+
const base = this.bases.get(id);
|
|
845
|
+
if (base === undefined || base === remoteVersion)
|
|
846
|
+
return false;
|
|
847
|
+
if (localItem !== undefined && this.versionOfItem(localItem) === remoteVersion) {
|
|
848
|
+
this.bases.set(id, remoteVersion);
|
|
849
|
+
return false;
|
|
367
850
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
851
|
+
return this.resolvePending(id, localItem, item, remoteVersion, conflicts);
|
|
852
|
+
}
|
|
853
|
+
const localVersion = localItem === undefined ? undefined : this.versionOfItem(localItem);
|
|
854
|
+
if (localItem === undefined) {
|
|
855
|
+
await this.local.set(id, item);
|
|
856
|
+
}
|
|
857
|
+
else if (remoteVersion !== undefined && localVersion !== undefined) {
|
|
858
|
+
// Only the remote changed: it is not a conflict
|
|
859
|
+
if (localVersion !== remoteVersion)
|
|
860
|
+
await this.local.set(id, item);
|
|
861
|
+
}
|
|
862
|
+
else if (JSON.stringify(localItem) !== JSON.stringify(item)) {
|
|
863
|
+
// Conflict - resolve according to strategy
|
|
864
|
+
if (this.options.conflictResolution === "error") {
|
|
372
865
|
this.stats.conflicts++;
|
|
866
|
+
conflicts.push(id);
|
|
867
|
+
return false;
|
|
373
868
|
}
|
|
869
|
+
const resolved = this.resolveConflict(localItem, item);
|
|
870
|
+
await this.local.set(id, resolved);
|
|
871
|
+
this.stats.conflicts++;
|
|
374
872
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
873
|
+
if (remoteVersion !== undefined)
|
|
874
|
+
this.bases.set(id, remoteVersion);
|
|
875
|
+
return false;
|
|
876
|
+
}
|
|
877
|
+
/** This method applies one remote delete. It gives true when the queue changed. */
|
|
878
|
+
async pullDelete(id, changedLocally, conflicts) {
|
|
879
|
+
if (changedLocally(id))
|
|
880
|
+
return false;
|
|
881
|
+
if (!this.hasPendingOp(id)) {
|
|
882
|
+
await this.local.delete(id);
|
|
883
|
+
if (this.options.versionOf)
|
|
884
|
+
this.bases.set(id, null);
|
|
885
|
+
return false;
|
|
886
|
+
}
|
|
887
|
+
// Without versions, the queued write wins, as before. With versions, the remote deleted the
|
|
888
|
+
// item only if the last sync saw a version of it.
|
|
889
|
+
const base = this.bases.get(id);
|
|
890
|
+
if (!this.options.versionOf || base === undefined || base === null)
|
|
891
|
+
return false;
|
|
892
|
+
const localItem = await this.local.get(id);
|
|
893
|
+
if (changedLocally(id))
|
|
894
|
+
return false;
|
|
895
|
+
if (localItem === undefined) {
|
|
896
|
+
// Both sides deleted the item
|
|
897
|
+
this.bases.set(id, null);
|
|
898
|
+
return false;
|
|
899
|
+
}
|
|
900
|
+
this.stats.conflicts++;
|
|
901
|
+
switch (this.options.conflictResolution) {
|
|
902
|
+
case "remote":
|
|
903
|
+
if (!this.dropQueued(id))
|
|
904
|
+
break;
|
|
378
905
|
await this.local.delete(id);
|
|
906
|
+
this.bases.set(id, null);
|
|
907
|
+
return true;
|
|
908
|
+
case "error":
|
|
909
|
+
conflicts.push(id);
|
|
910
|
+
return false;
|
|
911
|
+
default:
|
|
912
|
+
// "local", and "merge" with one deleted side: the queued write wins
|
|
913
|
+
break;
|
|
914
|
+
}
|
|
915
|
+
this.bases.set(id, null);
|
|
916
|
+
return false;
|
|
917
|
+
}
|
|
918
|
+
/**
|
|
919
|
+
* A queued local write of an id whose remote version changed: a conflict, and the policy
|
|
920
|
+
* decides. True when the queue changed.
|
|
921
|
+
*/
|
|
922
|
+
async resolvePending(id, localItem, remoteItem, remoteVersion, conflicts) {
|
|
923
|
+
this.stats.conflicts++;
|
|
924
|
+
switch (this.options.conflictResolution) {
|
|
925
|
+
case "remote":
|
|
926
|
+
// The remote value replaces the queued write. A queued clear cannot lose one id.
|
|
927
|
+
if (!this.dropQueued(id))
|
|
928
|
+
break;
|
|
929
|
+
await this.local.set(id, remoteItem);
|
|
930
|
+
this.bases.set(id, remoteVersion);
|
|
931
|
+
return true;
|
|
932
|
+
case "merge": {
|
|
933
|
+
if (localItem === undefined)
|
|
934
|
+
break;
|
|
935
|
+
const merged = this.resolveConflict(localItem, remoteItem);
|
|
936
|
+
if (!this.dropQueued(id))
|
|
937
|
+
break;
|
|
938
|
+
await this.local.set(id, merged);
|
|
939
|
+
this.bases.set(id, remoteVersion);
|
|
940
|
+
this.offlineOps.push({ type: "set", data: { id, value: merged }, timestamp: Date.now(), base: [[id, remoteVersion]] });
|
|
941
|
+
return true;
|
|
379
942
|
}
|
|
943
|
+
case "error":
|
|
944
|
+
conflicts.push(id);
|
|
945
|
+
return false;
|
|
946
|
+
default:
|
|
947
|
+
break;
|
|
380
948
|
}
|
|
381
|
-
this
|
|
382
|
-
this.
|
|
949
|
+
// "local": the queued write wins, and it replaces this remote version
|
|
950
|
+
this.bases.set(id, remoteVersion);
|
|
951
|
+
return false;
|
|
952
|
+
}
|
|
953
|
+
/**
|
|
954
|
+
* This method removes an id from the queued operations. It gives false when a queued clear touches it: nothing
|
|
955
|
+
* changes then.
|
|
956
|
+
*/
|
|
957
|
+
dropQueued(id) {
|
|
958
|
+
if (this.offlineOps.some((op) => op.type === "clear"))
|
|
959
|
+
return false;
|
|
960
|
+
const next = [];
|
|
961
|
+
for (const op of this.offlineOps) {
|
|
962
|
+
const kept = withoutId(op, id);
|
|
963
|
+
if (kept)
|
|
964
|
+
next.push(kept);
|
|
965
|
+
}
|
|
966
|
+
this.offlineOps = next;
|
|
967
|
+
return true;
|
|
968
|
+
}
|
|
969
|
+
/**
|
|
970
|
+
* The remote items with their ids: from `getEntries()` when the remote has it, or else from
|
|
971
|
+
* `getAll()` and `keyOf`. An item without a key stops the sync before any change. Before, the
|
|
972
|
+
* sync skipped such items and then deleted each local item that the remote list did not have,
|
|
973
|
+
* so the remote list without ids emptied the cache (roadmap R2).
|
|
974
|
+
*/
|
|
975
|
+
async remoteEntries() {
|
|
976
|
+
if (this.remote.getEntries)
|
|
977
|
+
return this.remote.getEntries();
|
|
978
|
+
const items = await this.remote.getAll();
|
|
979
|
+
const entries = [];
|
|
980
|
+
let missing = 0;
|
|
981
|
+
for (const item of items) {
|
|
982
|
+
const id = this.options.keyOf(item);
|
|
983
|
+
if (id)
|
|
984
|
+
entries.push([id, item]);
|
|
985
|
+
else
|
|
986
|
+
missing++;
|
|
987
|
+
}
|
|
988
|
+
if (missing > 0)
|
|
989
|
+
throw new MissingKeysError(missing);
|
|
990
|
+
return entries;
|
|
383
991
|
}
|
|
992
|
+
/**
|
|
993
|
+
* This method sends the queue. The operations that reached the remote leave the queue. The queue store
|
|
994
|
+
* gets the change at the end.
|
|
995
|
+
*/
|
|
384
996
|
async pushQueue() {
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
997
|
+
this.pushing = true;
|
|
998
|
+
let sent = 0;
|
|
999
|
+
try {
|
|
1000
|
+
while (this.offlineOps.length > 0) {
|
|
1001
|
+
const op = this.offlineOps[0];
|
|
1002
|
+
if (this.unpersisted.has(op)) {
|
|
1003
|
+
// The write of this operation waits for the queue store
|
|
1004
|
+
await this.storeChain;
|
|
1005
|
+
continue;
|
|
1006
|
+
}
|
|
1007
|
+
try {
|
|
1008
|
+
await this.executeOperation(op);
|
|
1009
|
+
}
|
|
1010
|
+
catch (error) {
|
|
1011
|
+
// This operation and the later ones stay queued
|
|
1012
|
+
this.isOnline = false;
|
|
1013
|
+
throw error;
|
|
1014
|
+
}
|
|
1015
|
+
this.noteRemoteWrite(op);
|
|
1016
|
+
// Remove the operation only after it reached the remote
|
|
1017
|
+
if (this.offlineOps[0] === op) {
|
|
1018
|
+
this.offlineOps.shift();
|
|
1019
|
+
sent++;
|
|
1020
|
+
this.storeDirty = true;
|
|
1021
|
+
}
|
|
394
1022
|
}
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
1023
|
+
this.isOnline = true;
|
|
1024
|
+
}
|
|
1025
|
+
finally {
|
|
1026
|
+
this.pushing = false;
|
|
1027
|
+
if (sent > 0) {
|
|
1028
|
+
await this.persistAll().catch((error) => {
|
|
1029
|
+
// The store keeps sent operations. A later direct write repairs it first.
|
|
1030
|
+
console.error("The queue store could not remove the sent operations:", error);
|
|
1031
|
+
});
|
|
398
1032
|
}
|
|
399
1033
|
}
|
|
400
|
-
this.isOnline = true;
|
|
401
1034
|
}
|
|
402
1035
|
/** Run syncs one at a time. */
|
|
403
1036
|
serialize(run) {
|
|
@@ -405,15 +1038,105 @@ export class HybridStorage {
|
|
|
405
1038
|
this.syncChain = next.catch(() => undefined);
|
|
406
1039
|
return next;
|
|
407
1040
|
}
|
|
408
|
-
async localIds() {
|
|
409
|
-
return [...this.local.keys()];
|
|
410
|
-
}
|
|
411
1041
|
bump(id) {
|
|
412
1042
|
this.writeVersions.set(id, (this.writeVersions.get(id) ?? 0) + 1);
|
|
413
1043
|
}
|
|
414
|
-
|
|
1044
|
+
/** The count of local writes of an id: it changes with each write and each clear. */
|
|
1045
|
+
writeKey(id) {
|
|
415
1046
|
return `${this.clearVersion}:${this.writeVersions.get(id) ?? 0}`;
|
|
416
1047
|
}
|
|
1048
|
+
versionOfItem(item) {
|
|
1049
|
+
return this.options.versionOf?.(item);
|
|
1050
|
+
}
|
|
1051
|
+
/** This method records the remote version of an id that the storage read. */
|
|
1052
|
+
noteBase(id, item) {
|
|
1053
|
+
if (!this.options.versionOf)
|
|
1054
|
+
return;
|
|
1055
|
+
if (item === undefined) {
|
|
1056
|
+
this.bases.set(id, null);
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
const version = this.versionOfItem(item);
|
|
1060
|
+
if (version === undefined)
|
|
1061
|
+
this.bases.delete(id);
|
|
1062
|
+
else
|
|
1063
|
+
this.bases.set(id, version);
|
|
1064
|
+
}
|
|
1065
|
+
/** This method records the remote versions after a write reached the remote. */
|
|
1066
|
+
noteRemoteWrite(op) {
|
|
1067
|
+
if (!this.options.versionOf)
|
|
1068
|
+
return;
|
|
1069
|
+
const data = op.data;
|
|
1070
|
+
switch (op.type) {
|
|
1071
|
+
case "set":
|
|
1072
|
+
this.noteBase(data.id, data.value);
|
|
1073
|
+
break;
|
|
1074
|
+
case "delete":
|
|
1075
|
+
this.bases.set(data.id, null);
|
|
1076
|
+
break;
|
|
1077
|
+
case "setBatch":
|
|
1078
|
+
for (const [id, value] of data.items)
|
|
1079
|
+
this.noteBase(id, value);
|
|
1080
|
+
break;
|
|
1081
|
+
case "deleteBatch":
|
|
1082
|
+
for (const id of data.ids)
|
|
1083
|
+
this.bases.set(id, null);
|
|
1084
|
+
break;
|
|
1085
|
+
case "clear":
|
|
1086
|
+
for (const id of this.bases.keys())
|
|
1087
|
+
this.bases.set(id, null);
|
|
1088
|
+
break;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
/**
|
|
1092
|
+
* This method puts in the operation the remote version that it changes, for each id: the version that an
|
|
1093
|
+
* older queued operation of the id writes, or else the known remote version. A storage that
|
|
1094
|
+
* loads the queue after a restart takes the version of the first operation of each id.
|
|
1095
|
+
*/
|
|
1096
|
+
attachBase(op) {
|
|
1097
|
+
if (!this.options.versionOf)
|
|
1098
|
+
return;
|
|
1099
|
+
const base = [];
|
|
1100
|
+
for (const id of idsOf(op)) {
|
|
1101
|
+
if (id === "*")
|
|
1102
|
+
continue;
|
|
1103
|
+
const version = this.expectedBase(id);
|
|
1104
|
+
if (version !== undefined)
|
|
1105
|
+
base.push([id, version]);
|
|
1106
|
+
}
|
|
1107
|
+
if (base.length > 0)
|
|
1108
|
+
op.base = base;
|
|
1109
|
+
}
|
|
1110
|
+
/** The remote version of an id after the queued operations, or undefined when it is not known. */
|
|
1111
|
+
expectedBase(id) {
|
|
1112
|
+
for (let index = this.offlineOps.length - 1; index >= 0; index--) {
|
|
1113
|
+
const op = this.offlineOps[index];
|
|
1114
|
+
const data = op.data;
|
|
1115
|
+
switch (op.type) {
|
|
1116
|
+
case "set":
|
|
1117
|
+
if (data.id === id)
|
|
1118
|
+
return this.versionOfItem(data.value);
|
|
1119
|
+
break;
|
|
1120
|
+
case "delete":
|
|
1121
|
+
if (data.id === id)
|
|
1122
|
+
return null;
|
|
1123
|
+
break;
|
|
1124
|
+
case "setBatch": {
|
|
1125
|
+
const item = (data.items ?? []).findLast(([itemId]) => itemId === id);
|
|
1126
|
+
if (item)
|
|
1127
|
+
return this.versionOfItem(item[1]);
|
|
1128
|
+
break;
|
|
1129
|
+
}
|
|
1130
|
+
case "deleteBatch":
|
|
1131
|
+
if ((data.ids ?? []).includes(id))
|
|
1132
|
+
return null;
|
|
1133
|
+
break;
|
|
1134
|
+
case "clear":
|
|
1135
|
+
return null;
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
return this.bases.get(id);
|
|
1139
|
+
}
|
|
417
1140
|
hasPendingOp(id) {
|
|
418
1141
|
return this.offlineOps.some((op) => {
|
|
419
1142
|
const ids = idsOf(op);
|
|
@@ -440,18 +1163,165 @@ export class HybridStorage {
|
|
|
440
1163
|
return remote;
|
|
441
1164
|
}
|
|
442
1165
|
}
|
|
1166
|
+
//
|
|
1167
|
+
// ═══ The queue and its store ═══
|
|
1168
|
+
//
|
|
443
1169
|
/**
|
|
444
|
-
*
|
|
1170
|
+
* This method puts an operation in the queue, then in the queue store. The write ends only after the store
|
|
1171
|
+
* kept it. When the store fails, the operation leaves the queue, the local change is undone,
|
|
1172
|
+
* and the write throws `QueueStoreError`.
|
|
1173
|
+
*
|
|
1174
|
+
* A set or a delete of one id replaces the queued set or delete of that id, when no later
|
|
1175
|
+
* operation touches the id: a second try of a write does not queue it two times.
|
|
445
1176
|
*/
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
1177
|
+
async enqueue(op, undo) {
|
|
1178
|
+
const index = this.replaceableIndex(op);
|
|
1179
|
+
let rollback;
|
|
1180
|
+
if (index >= 0) {
|
|
1181
|
+
const previous = this.offlineOps[index];
|
|
1182
|
+
// The new operation changes the same remote version as the operation that it replaces
|
|
1183
|
+
if (previous.base)
|
|
1184
|
+
op.base = previous.base;
|
|
1185
|
+
else
|
|
1186
|
+
delete op.base;
|
|
1187
|
+
this.offlineOps[index] = op;
|
|
1188
|
+
rollback = () => {
|
|
1189
|
+
const at = this.offlineOps.indexOf(op);
|
|
1190
|
+
if (at >= 0)
|
|
1191
|
+
this.offlineOps[at] = previous;
|
|
1192
|
+
};
|
|
1193
|
+
}
|
|
1194
|
+
else {
|
|
1195
|
+
try {
|
|
1196
|
+
this.ensureQueueRoom(op);
|
|
1197
|
+
}
|
|
1198
|
+
catch (error) {
|
|
1199
|
+
await undo();
|
|
1200
|
+
throw error;
|
|
1201
|
+
}
|
|
1202
|
+
this.offlineOps.push(op);
|
|
1203
|
+
rollback = () => {
|
|
1204
|
+
const at = this.offlineOps.indexOf(op);
|
|
1205
|
+
if (at >= 0)
|
|
1206
|
+
this.offlineOps.splice(at, 1);
|
|
1207
|
+
};
|
|
449
1208
|
}
|
|
450
|
-
this.
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
1209
|
+
this.unpersisted.add(op);
|
|
1210
|
+
try {
|
|
1211
|
+
await this.persist(op, index < 0, rollback);
|
|
1212
|
+
}
|
|
1213
|
+
catch (error) {
|
|
1214
|
+
await undo();
|
|
1215
|
+
throw error;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* This method writes one operation to the queue store: an append when the store holds each operation
|
|
1220
|
+
* before it, or else the whole queue. On a failure, `rollback` removes the operation from the
|
|
1221
|
+
* queue before the next call of the store.
|
|
1222
|
+
*/
|
|
1223
|
+
persist(op, appended, rollback) {
|
|
1224
|
+
const run = this.storeChain.then(async () => {
|
|
1225
|
+
// An earlier write of the whole queue kept it already
|
|
1226
|
+
if (!this.unpersisted.has(op))
|
|
1227
|
+
return;
|
|
1228
|
+
const at = this.offlineOps.indexOf(op);
|
|
1229
|
+
const canAppend = appended && !this.storeDirty && at >= 0 && this.offlineOps.slice(0, at).every((other) => !this.unpersisted.has(other));
|
|
1230
|
+
const snapshot = this.offlineOps.slice();
|
|
1231
|
+
try {
|
|
1232
|
+
if (canAppend) {
|
|
1233
|
+
await this.queueStore.append(op);
|
|
1234
|
+
this.unpersisted.delete(op);
|
|
1235
|
+
}
|
|
1236
|
+
else {
|
|
1237
|
+
await this.queueStore.replace(snapshot);
|
|
1238
|
+
for (const kept of snapshot)
|
|
1239
|
+
this.unpersisted.delete(kept);
|
|
1240
|
+
this.unpersisted.delete(op);
|
|
1241
|
+
this.storeDirty = false;
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
catch (error) {
|
|
1245
|
+
this.storeDirty = true;
|
|
1246
|
+
this.unpersisted.delete(op);
|
|
1247
|
+
rollback();
|
|
1248
|
+
throw new QueueStoreError("write", error);
|
|
1249
|
+
}
|
|
1250
|
+
});
|
|
1251
|
+
this.storeChain = run.catch(() => undefined);
|
|
1252
|
+
return run;
|
|
1253
|
+
}
|
|
1254
|
+
/** This method writes the whole queue to the queue store when the store can differ from the queue. */
|
|
1255
|
+
persistAll() {
|
|
1256
|
+
const run = this.storeChain.then(async () => {
|
|
1257
|
+
if (!this.storeDirty && this.unpersisted.size === 0)
|
|
1258
|
+
return;
|
|
1259
|
+
const snapshot = this.offlineOps.slice();
|
|
1260
|
+
try {
|
|
1261
|
+
if (snapshot.length === 0)
|
|
1262
|
+
await this.queueStore.clear();
|
|
1263
|
+
else
|
|
1264
|
+
await this.queueStore.replace(snapshot);
|
|
1265
|
+
for (const kept of snapshot)
|
|
1266
|
+
this.unpersisted.delete(kept);
|
|
1267
|
+
this.storeDirty = false;
|
|
1268
|
+
}
|
|
1269
|
+
catch (error) {
|
|
1270
|
+
this.storeDirty = true;
|
|
1271
|
+
throw new QueueStoreError("write", error);
|
|
1272
|
+
}
|
|
1273
|
+
});
|
|
1274
|
+
this.storeChain = run.catch(() => undefined);
|
|
1275
|
+
return run;
|
|
1276
|
+
}
|
|
1277
|
+
/**
|
|
1278
|
+
* Before a write goes to the remote directly, the queue store must not hold a sent operation:
|
|
1279
|
+
* after a restart, it would replace this newer write.
|
|
1280
|
+
*/
|
|
1281
|
+
async ensureStoreClean() {
|
|
1282
|
+
if (this.storeDirty)
|
|
1283
|
+
await this.persistAll();
|
|
1284
|
+
}
|
|
1285
|
+
/** The index of the queued operation that `op` replaces, or -1. */
|
|
1286
|
+
replaceableIndex(op) {
|
|
1287
|
+
if (op.type !== "set" && op.type !== "delete")
|
|
1288
|
+
return -1;
|
|
1289
|
+
const [id] = idsOf(op);
|
|
1290
|
+
if (id === undefined)
|
|
1291
|
+
return -1;
|
|
1292
|
+
for (let index = this.offlineOps.length - 1; index >= 0; index--) {
|
|
1293
|
+
const queued = this.offlineOps[index];
|
|
1294
|
+
const ids = idsOf(queued);
|
|
1295
|
+
if (!ids.includes(id) && !ids.includes("*"))
|
|
1296
|
+
continue;
|
|
1297
|
+
// pushQueue can be sending the first operation now: do not change it
|
|
1298
|
+
const sending = index === 0 && this.pushing;
|
|
1299
|
+
return !sending && (queued.type === "set" || queued.type === "delete") ? index : -1;
|
|
454
1300
|
}
|
|
1301
|
+
return -1;
|
|
1302
|
+
}
|
|
1303
|
+
/** True when the queue can take the operation. */
|
|
1304
|
+
hasQueueRoom(op) {
|
|
1305
|
+
return this.replaceableIndex(op) >= 0 || this.offlineOps.length < this.options.maxOfflineOps;
|
|
1306
|
+
}
|
|
1307
|
+
/** A full queue refuses the write. Before, it dropped the oldest operation (roadmap R2). */
|
|
1308
|
+
ensureQueueRoom(op) {
|
|
1309
|
+
if (!this.hasQueueRoom(op))
|
|
1310
|
+
throw new QueueFullError(this.options.maxOfflineOps);
|
|
1311
|
+
}
|
|
1312
|
+
/**
|
|
1313
|
+
* In write-through mode, a timer sends the queue after a failure, and again after each failed
|
|
1314
|
+
* send, until the queue is empty.
|
|
1315
|
+
*/
|
|
1316
|
+
scheduleDrain() {
|
|
1317
|
+
if (this.drainTimer || this.closed || this.mode !== "write-through")
|
|
1318
|
+
return;
|
|
1319
|
+
const delay = this.options.syncInterval > 0 ? this.options.syncInterval : 5000;
|
|
1320
|
+
this.drainTimer = setTimeout(() => {
|
|
1321
|
+
this.drainTimer = null;
|
|
1322
|
+
this.syncToRemote().catch(() => this.scheduleDrain());
|
|
1323
|
+
}, delay);
|
|
1324
|
+
this.drainTimer.unref?.();
|
|
455
1325
|
}
|
|
456
1326
|
/**
|
|
457
1327
|
* Execute a queued operation against remote storage.
|
|
@@ -476,33 +1346,6 @@ export class HybridStorage {
|
|
|
476
1346
|
break;
|
|
477
1347
|
}
|
|
478
1348
|
}
|
|
479
|
-
/**
|
|
480
|
-
* Handle remote operation failure.
|
|
481
|
-
*/
|
|
482
|
-
handleRemoteFailure(operation, data) {
|
|
483
|
-
console.error(`Remote ${operation} failed, marking offline`);
|
|
484
|
-
this.isOnline = false;
|
|
485
|
-
// Queue operation if offline queue enabled
|
|
486
|
-
if (this.options.offlineQueue) {
|
|
487
|
-
this.queueOperation({
|
|
488
|
-
type: operation,
|
|
489
|
-
data,
|
|
490
|
-
timestamp: Date.now(),
|
|
491
|
-
});
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
/**
|
|
495
|
-
* Start periodic background sync.
|
|
496
|
-
*/
|
|
497
|
-
startPeriodicSync() {
|
|
498
|
-
this.syncTimer = setInterval(() => {
|
|
499
|
-
this.syncToRemote().catch((err) => {
|
|
500
|
-
console.error("Periodic sync failed:", err);
|
|
501
|
-
});
|
|
502
|
-
}, this.options.syncInterval);
|
|
503
|
-
// The timer must not keep the process alive (deep dive DATA-9)
|
|
504
|
-
this.syncTimer.unref?.();
|
|
505
|
-
}
|
|
506
1349
|
/**
|
|
507
1350
|
* Get current online status.
|
|
508
1351
|
*/
|