@procedures/store 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +63 -0
  2. package/README.md +109 -2
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/indexeddb/idb.d.ts +100 -0
  5. package/dist/indexeddb/idb.d.ts.map +1 -0
  6. package/dist/indexeddb/idb.js +137 -0
  7. package/dist/indexeddb/idb.js.map +1 -0
  8. package/dist/indexeddb/program-store.d.ts +42 -0
  9. package/dist/indexeddb/program-store.d.ts.map +1 -0
  10. package/dist/indexeddb/program-store.js +71 -0
  11. package/dist/indexeddb/program-store.js.map +1 -0
  12. package/dist/indexeddb/queue-store.d.ts +38 -0
  13. package/dist/indexeddb/queue-store.d.ts.map +1 -0
  14. package/dist/indexeddb/queue-store.js +73 -0
  15. package/dist/indexeddb/queue-store.js.map +1 -0
  16. package/dist/indexeddb.d.ts +19 -0
  17. package/dist/indexeddb.d.ts.map +1 -0
  18. package/dist/indexeddb.js +17 -0
  19. package/dist/indexeddb.js.map +1 -0
  20. package/dist/program-store/file.d.ts +5 -0
  21. package/dist/program-store/file.d.ts.map +1 -1
  22. package/dist/program-store/file.js +5 -2
  23. package/dist/program-store/file.js.map +1 -1
  24. package/dist/program-store/procedures.d.ts.map +1 -1
  25. package/dist/program-store/procedures.js +5 -2
  26. package/dist/program-store/procedures.js.map +1 -1
  27. package/dist/storage/api.d.ts +18 -7
  28. package/dist/storage/api.d.ts.map +1 -1
  29. package/dist/storage/api.js +34 -3
  30. package/dist/storage/api.js.map +1 -1
  31. package/dist/storage/factory.d.ts +13 -2
  32. package/dist/storage/factory.d.ts.map +1 -1
  33. package/dist/storage/factory.js +12 -4
  34. package/dist/storage/factory.js.map +1 -1
  35. package/dist/storage/file-queue.d.ts +32 -0
  36. package/dist/storage/file-queue.d.ts.map +1 -0
  37. package/dist/storage/file-queue.js +101 -0
  38. package/dist/storage/file-queue.js.map +1 -0
  39. package/dist/storage/hybrid.d.ts +355 -42
  40. package/dist/storage/hybrid.d.ts.map +1 -1
  41. package/dist/storage/hybrid.js +1093 -250
  42. package/dist/storage/hybrid.js.map +1 -1
  43. package/dist/storage/index.d.ts +4 -2
  44. package/dist/storage/index.d.ts.map +1 -1
  45. package/dist/storage/index.js +5 -1
  46. package/dist/storage/index.js.map +1 -1
  47. package/dist/storage/queue.d.ts +67 -0
  48. package/dist/storage/queue.d.ts.map +1 -0
  49. package/dist/storage/queue.js +85 -0
  50. package/dist/storage/queue.js.map +1 -0
  51. package/dist/storage/synced-registry.d.ts +18 -2
  52. package/dist/storage/synced-registry.d.ts.map +1 -1
  53. package/dist/storage/synced-registry.js +37 -3
  54. package/dist/storage/synced-registry.js.map +1 -1
  55. package/dist/storage/types.d.ts +5 -2
  56. package/dist/storage/types.d.ts.map +1 -1
  57. package/package.json +16 -6
  58. package/src/indexeddb/idb.ts +190 -0
  59. package/src/indexeddb/program-store.ts +80 -0
  60. package/src/indexeddb/queue-store.ts +80 -0
  61. package/src/indexeddb.ts +19 -0
  62. package/src/program-store/file.ts +5 -2
  63. package/src/program-store/procedures.ts +5 -2
  64. package/src/storage/api.ts +50 -10
  65. package/src/storage/factory.ts +21 -4
  66. package/src/storage/file-queue.ts +103 -0
  67. package/src/storage/hybrid.ts +1142 -274
  68. package/src/storage/index.ts +15 -2
  69. package/src/storage/queue.ts +128 -0
  70. package/src/storage/synced-registry.ts +49 -4
  71. package/src/storage/types.ts +6 -2
@@ -1,36 +1,111 @@
1
1
  /**
2
2
  * Hybrid Storage Implementation
3
3
  *
4
- * Combines local cache (InMemoryStorage) with remote persistence (ApiStorage).
5
- * Provides fast local reads with remote backup and sync.
4
+ * A local copy (an `InMemoryStorage`, or the `local` storage that the application gives) in
5
+ * front of a remote `CollectionStorage` (often an `ApiStorage`). The storage has four modes, and
6
+ * `setMode()` changes the mode while the storage operates (roadmap R28):
6
7
  *
7
- * Features:
8
- * - Local cache for fast reads, with a remote read on a miss
9
- * - Write-through or write-back strategies
10
- * - Automatic sync on initialization
11
- * - Conflict resolution
12
- * - Offline operation queue
8
+ * | Mode | Reads | Writes | Remote calls |
9
+ * |---|---|---|---|
10
+ * | `local-only` | the local copy | the local copy and the queue | none |
11
+ * | `write-through` | the local copy, the remote on a miss | the local copy, then the remote | yes |
12
+ * | `write-back` | the local copy, the remote on a miss | the local copy and the queue, a timer sends it | yes |
13
+ * | `remote-only` | the remote | the remote | yes, and no local copy |
14
+ *
15
+ * - The queue is durable through the `queueStore` that the application gives.
16
+ * - A sync asks for the changes since a version when the remote has `getChangesSince`.
17
+ * - With `versionOf`, a conflict is a change on both sides, and `conflictResolution` decides.
18
+ * - `pullInterval` keeps the local copy fresh.
13
19
  */
14
20
  import { InMemoryStorage } from "@procedures/collections";
15
- /** The ids that an operation changes ("*" for clear). */
16
- function idsOf(op) {
17
- const data = op.data;
18
- switch (op.type) {
19
- case "set":
20
- case "delete":
21
- return data.id === undefined ? [] : [data.id];
22
- case "setBatch":
23
- return (data.items ?? []).map(([id]) => id);
24
- case "deleteBatch":
25
- return data.ids ?? [];
26
- case "clear":
27
- return ["*"];
21
+ import { optionalMs } from "@procedures/client";
22
+ import { MemoryQueueStore, idsOf, withoutId } from "./queue.js";
23
+ const MODES = new Set(["local-only", "write-through", "write-back", "remote-only"]);
24
+ /**
25
+ * The remote did not take a write of the write-through mode. The write is in the local cache and
26
+ * in the queue, and the storage sends it later (roadmap R2). `cause` is the error of the remote.
27
+ */
28
+ export class PendingWriteError extends Error {
29
+ name = "PendingWriteError";
30
+ code = "WRITE_PENDING";
31
+ pending = true;
32
+ constructor(operation, cause) {
33
+ super(`The remote did not take the ${operation}: it waits in the queue`, { cause });
34
+ }
35
+ }
36
+ /** The queue of operations is full, so the storage refuses the write (roadmap R2). */
37
+ export class QueueFullError extends Error {
38
+ name = "QueueFullError";
39
+ code = "QUEUE_FULL";
40
+ constructor(max) {
41
+ super(`The queue of operations is full (${max} operations): the storage refuses the write`);
42
+ }
43
+ }
44
+ /** The remote gave items without a key, so a sync cannot know which local items to keep. */
45
+ export class MissingKeysError extends Error {
46
+ name = "MissingKeysError";
47
+ code = "MISSING_KEYS";
48
+ constructor(count) {
49
+ super(`${count} remote items have no key. Give the option keyOf, or a remote storage with getEntries(). The sync changed nothing.`);
50
+ }
51
+ }
52
+ /**
53
+ * The queue store failed (roadmap R28). A write that the store did not keep is undone and not
54
+ * acknowledged. A storage whose store cannot load refuses each operation.
55
+ */
56
+ export class QueueStoreError extends Error {
57
+ name = "QueueStoreError";
58
+ code = "QUEUE_STORE";
59
+ constructor(action, cause) {
60
+ super(`The queue store could not ${action === "load" ? "load the queue" : "keep the queue"}`, { cause });
28
61
  }
29
62
  }
63
+ /**
64
+ * The sync found conflicts and `conflictResolution` is "error" (roadmap R28). The local values and
65
+ * their queued writes stay. `ids` are the ids of the conflicts.
66
+ */
67
+ export class ConflictError extends Error {
68
+ ids;
69
+ name = "ConflictError";
70
+ code = "SYNC_CONFLICT";
71
+ constructor(ids) {
72
+ super(`Conflict detected between local and remote versions of ${ids.length} items: ${ids.slice(0, 10).join(", ")}`);
73
+ this.ids = ids;
74
+ }
75
+ }
76
+ /**
77
+ * `setMode()` could not change the mode (roadmap R28). The mode did not change. `cause` is the
78
+ * error of the flush or of the sync.
79
+ */
80
+ export class ModeSwitchError extends Error {
81
+ from;
82
+ to;
83
+ name = "ModeSwitchError";
84
+ code = "MODE_SWITCH_FAILED";
85
+ constructor(from, to, cause) {
86
+ super(`The storage could not change from ${from} to ${to}: the mode is still ${from}`, { cause });
87
+ this.from = from;
88
+ this.to = to;
89
+ }
90
+ }
91
+ const defaultIsUnavailable = (error) => {
92
+ if (typeof error !== "object" || error === null)
93
+ return true;
94
+ const { retryable, code } = error;
95
+ return retryable === true || code === undefined;
96
+ };
97
+ /** True when a remote does not know a procedure: an older server without `changes`. */
98
+ const isNotFound = (error) => typeof error === "object" && error !== null && error.code === "NOT_FOUND";
30
99
  const defaultKeyOf = (item) => {
31
100
  const id = item?.id;
32
101
  return typeof id === "string" && id.length > 0 ? id : undefined;
33
102
  };
103
+ function checkMode(mode) {
104
+ if (typeof mode !== "string" || !MODES.has(mode)) {
105
+ throw new TypeError(`Unknown mode ${JSON.stringify(mode)}: use local-only, write-through, write-back or remote-only`);
106
+ }
107
+ return mode;
108
+ }
34
109
  /**
35
110
  * Hybrid storage combining local cache with remote persistence.
36
111
  *
@@ -42,9 +117,9 @@ const defaultKeyOf = (item) => {
42
117
  *
43
118
  * const remote = new ApiStorage(client, { collection: "users" });
44
119
  * const hybrid = new HybridStorage(remote, {
45
- * writeStrategy: "write-through",
120
+ * mode: "write-through",
46
121
  * conflictResolution: "remote",
47
- * syncInterval: 5000
122
+ * syncInterval: seconds(5)
48
123
  * });
49
124
  *
50
125
  * // Fast local reads
@@ -52,20 +127,58 @@ const defaultKeyOf = (item) => {
52
127
  *
53
128
  * // Writes go to both local and remote
54
129
  * await hybrid.set("123", { name: "John" });
130
+ *
131
+ * // Offline: no remote call. The writes wait in the queue.
132
+ * await hybrid.setMode("local-only");
133
+ * // Online again: the switch sends the queue, then syncs the local copy
134
+ * await hybrid.setMode("write-through");
55
135
  * ```
56
136
  */
57
137
  export class HybridStorage {
58
138
  local;
59
139
  remote;
140
+ queueStore;
60
141
  options;
142
+ mode;
61
143
  offlineOps = [];
144
+ /** The operations in `offlineOps` that the queue store does not have yet */
145
+ unpersisted = new Set();
146
+ /** True when the queue store can differ from `offlineOps`: a failed write, or sent operations */
147
+ storeDirty = false;
148
+ /** The calls of the queue store, one at a time */
149
+ storeChain = Promise.resolve();
150
+ /** The push timer of the write-back mode */
62
151
  syncTimer = null;
152
+ /** The periodic pull */
153
+ pullTimer = null;
154
+ /** True while pushQueue sends the queue: the first operation can be on its way */
155
+ pushing = false;
156
+ /** The timer that sends the queue of the write-through mode after a failure */
157
+ drainTimer = null;
158
+ /** The load of the queue. The writes wait for it. */
159
+ loaded;
160
+ /** The load and the first sync. The reads wait for it, so a read never sees an empty cache by mistake. */
161
+ ready;
162
+ /** The error of the load (of the queue store, or of the local copy): the storage refuses each operation */
163
+ loadError;
164
+ /** A mode switch that runs. The operations wait for it. */
165
+ gate = Promise.resolve();
166
+ /** The operations that run, and the switches that wait for them to end */
167
+ active = 0;
168
+ idleWaiters = [];
63
169
  isOnline = true;
170
+ closed = false;
64
171
  /** A counter of local writes for each id, so a sync does not restore an older remote value */
65
172
  writeVersions = new Map();
66
173
  clearVersion = 0;
67
174
  /** The running sync: syncs run one at a time */
68
175
  syncChain = Promise.resolve();
176
+ /** The remote version of the last delta pull */
177
+ cursor;
178
+ /** True when the remote has `getChangesSince` but its server does not know `changes` */
179
+ changesUnsupported = false;
180
+ /** With `versionOf`: the last known remote version of each id. `null`: the remote has no item. */
181
+ bases = new Map();
69
182
  stats = {
70
183
  cacheHits: 0,
71
184
  cacheMisses: 0,
@@ -73,28 +186,135 @@ export class HybridStorage {
73
186
  conflicts: 0,
74
187
  };
75
188
  constructor(remote, options = {}) {
76
- this.local = new InMemoryStorage();
189
+ this.local = options.local ?? new InMemoryStorage();
77
190
  this.remote = remote;
191
+ this.queueStore = options.queueStore ?? new MemoryQueueStore();
192
+ this.mode = checkMode(options.mode ?? options.writeStrategy ?? "write-through");
78
193
  this.options = {
79
- writeStrategy: options.writeStrategy ?? "write-through",
80
194
  conflictResolution: options.conflictResolution ?? "remote",
81
195
  ...(options.mergeFn !== undefined && { mergeFn: options.mergeFn }),
82
196
  keyOf: options.keyOf ?? defaultKeyOf,
83
- syncInterval: options.syncInterval ?? 5000,
197
+ ...(options.versionOf !== undefined && { versionOf: options.versionOf }),
198
+ syncInterval: optionalMs(options.syncInterval, "syncInterval") ?? 5000,
199
+ pullInterval: optionalMs(options.pullInterval, "pullInterval") ?? 0,
84
200
  syncOnInit: options.syncOnInit ?? true,
85
201
  maxOfflineOps: options.maxOfflineOps ?? 1000,
86
202
  offlineQueue: options.offlineQueue ?? true,
203
+ isUnavailable: options.isUnavailable ?? defaultIsUnavailable,
87
204
  };
88
- // Initialize with remote data
89
- if (this.options.syncOnInit) {
90
- this.syncFromRemote().catch((err) => {
91
- console.error("Failed to sync on init:", err);
92
- });
205
+ // The queue first, then the first sync. The reads wait for both (roadmap R2, R28).
206
+ this.loaded = this.loadQueue();
207
+ this.ready = this.loaded.then(() => this.initialSync());
208
+ this.updateTimers();
209
+ }
210
+ //
211
+ // ═══ Modes ═══
212
+ //
213
+ /** The mode of the storage now. */
214
+ getMode() {
215
+ return this.mode;
216
+ }
217
+ /**
218
+ * This method changes the mode while the storage operates (roadmap R28).
219
+ *
220
+ * - The switch waits for the operations that run. The operations that start later wait for
221
+ * the switch. Switches and syncs run one at a time.
222
+ * - A switch to a remote mode sends the queue first. A switch to `local-only` makes no remote
223
+ * call: the queue stays until the next remote mode.
224
+ * - A switch to `remote-only` drops the local copy, after the queue reached the remote.
225
+ * - A switch from `local-only` or `remote-only` to a mode with a local copy syncs the local
226
+ * copy. The reads wait for the sync.
227
+ * - When the flush or the sync fails, the mode does not change, and the switch throws
228
+ * `ModeSwitchError`.
229
+ */
230
+ async setMode(mode) {
231
+ const target = checkMode(mode);
232
+ const previous = this.gate;
233
+ let release;
234
+ this.gate = new Promise((resolve) => (release = resolve));
235
+ try {
236
+ await previous;
237
+ await this.loaded;
238
+ if (this.loadError)
239
+ throw this.loadError;
240
+ await this.idle();
241
+ await this.serialize(() => this.switchTo(target));
242
+ }
243
+ finally {
244
+ release();
245
+ }
246
+ }
247
+ async switchTo(target) {
248
+ const from = this.mode;
249
+ if (target === from)
250
+ return;
251
+ try {
252
+ if (target !== "local-only") {
253
+ // The queue reaches the remote before the mode changes
254
+ await this.pushQueue();
255
+ await this.ensureStoreClean();
256
+ }
257
+ if (target === "remote-only") {
258
+ await this.dropLocal();
259
+ }
260
+ else if (from === "remote-only" || from === "local-only") {
261
+ if (from === "remote-only")
262
+ await this.rebuildLocal();
263
+ await this.pullRemote(from === "remote-only");
264
+ }
265
+ }
266
+ catch (error) {
267
+ throw new ModeSwitchError(from, target, error);
268
+ }
269
+ this.mode = target;
270
+ this.updateTimers();
271
+ }
272
+ /** This method drops the local copy. A read in flight does not fill it again. */
273
+ async dropLocal() {
274
+ this.clearVersion++;
275
+ this.cursor = undefined;
276
+ await this.local.clear();
277
+ }
278
+ /** This method makes the local copy from the queue only, before a full sync. */
279
+ async rebuildLocal() {
280
+ await this.dropLocal();
281
+ for (const op of this.offlineOps)
282
+ await this.applyToLocal(op);
283
+ }
284
+ /** This method starts and stops the timers for the mode. */
285
+ updateTimers() {
286
+ if (this.closed)
287
+ return;
288
+ const push = this.mode === "write-back" && this.options.syncInterval > 0;
289
+ if (push && !this.syncTimer) {
290
+ this.syncTimer = setInterval(() => {
291
+ this.syncToRemote().catch((err) => {
292
+ console.error("Periodic sync failed:", err);
293
+ });
294
+ }, this.options.syncInterval);
295
+ // The timer must not keep the process alive (deep dive DATA-9)
296
+ this.syncTimer.unref?.();
93
297
  }
94
- // Start periodic sync for write-back mode
95
- if (this.options.writeStrategy === "write-back" &&
96
- this.options.syncInterval > 0) {
97
- this.startPeriodicSync();
298
+ else if (!push && this.syncTimer) {
299
+ clearInterval(this.syncTimer);
300
+ this.syncTimer = null;
301
+ }
302
+ const pull = this.options.pullInterval > 0 && (this.mode === "write-through" || this.mode === "write-back");
303
+ if (pull && !this.pullTimer) {
304
+ this.pullTimer = setInterval(() => {
305
+ this.syncFromRemote().catch((err) => {
306
+ console.error("Periodic pull failed:", err);
307
+ });
308
+ }, this.options.pullInterval);
309
+ this.pullTimer.unref?.();
310
+ }
311
+ else if (!pull && this.pullTimer) {
312
+ clearInterval(this.pullTimer);
313
+ this.pullTimer = null;
314
+ }
315
+ if (this.mode !== "write-through" && this.drainTimer) {
316
+ clearTimeout(this.drainTimer);
317
+ this.drainTimer = null;
98
318
  }
99
319
  }
100
320
  //
@@ -102,188 +322,272 @@ export class HybridStorage {
102
322
  //
103
323
  /**
104
324
  * The local item, or else the remote item (which the cache then keeps). When the remote
105
- * cannot answer, the storage is offline and the result is undefined.
325
+ * cannot answer, the storage is offline and the result is undefined. In `local-only` mode, a
326
+ * miss is undefined. In `remote-only` mode, the remote answers, and its error passes through.
106
327
  */
107
328
  async get(id) {
108
- // Try local cache first
109
- const cached = await this.local.get(id);
110
- if (cached !== undefined) {
111
- this.stats.cacheHits++;
112
- return cached;
113
- }
114
- this.stats.cacheMisses++;
115
- // A pending local delete means "no item": do not read it back from the remote
116
- if (this.hasPendingOp(id)) {
117
- return undefined;
118
- }
119
- const version = this.versionOf(id);
120
- try {
121
- const remote = await this.remote.get(id);
122
- this.isOnline = true;
123
- if (remote !== undefined && this.versionOf(id) === version) {
124
- await this.local.set(id, remote);
329
+ return this.run(true, async () => {
330
+ if (this.mode === "remote-only")
331
+ return this.remoteRead(() => this.remote.get(id));
332
+ // Try local cache first
333
+ const cached = await this.local.get(id);
334
+ if (cached !== undefined) {
335
+ this.stats.cacheHits++;
336
+ return cached;
125
337
  }
126
- return remote;
127
- }
128
- catch {
129
- this.isOnline = false;
130
- return undefined;
131
- }
338
+ this.stats.cacheMisses++;
339
+ // A pending local delete means "no item": do not read it back from the remote
340
+ if (this.mode === "local-only" || this.hasPendingOp(id)) {
341
+ return undefined;
342
+ }
343
+ const version = this.writeKey(id);
344
+ try {
345
+ const remote = await this.remote.get(id);
346
+ this.isOnline = true;
347
+ if (this.writeKey(id) === version && !this.hasPendingOp(id)) {
348
+ if (remote !== undefined)
349
+ await this.local.set(id, remote);
350
+ this.noteBase(id, remote);
351
+ }
352
+ return remote;
353
+ }
354
+ catch {
355
+ this.isOnline = false;
356
+ return undefined;
357
+ }
358
+ });
132
359
  }
133
360
  async getAll() {
134
- // Return local cache
135
- return this.local.getAll();
361
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.getAll()) : this.local.getAll()));
362
+ }
363
+ /** Every item with its id. */
364
+ async getEntries() {
365
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remoteEntries()) : this.local.getEntries()));
136
366
  }
137
367
  async find(predicate) {
138
- // Search local cache
139
- return this.local.find(predicate);
368
+ return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.find(predicate)) : this.local.find(predicate));
140
369
  }
141
370
  async has(id) {
142
- return this.local.has(id);
371
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.has(id)) : this.local.has(id)));
143
372
  }
144
373
  async size() {
145
- return this.local.size();
374
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.size()) : this.local.size()));
375
+ }
376
+ async getBatch(ids) {
377
+ return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.getBatch(ids)) : this.local.getBatch(ids));
378
+ }
379
+ /** A read of the remote-only mode. A queue that a restart loaded reaches the remote first. */
380
+ async remoteRead(read) {
381
+ if (this.offlineOps.length > 0)
382
+ await this.serialize(() => this.pushQueue());
383
+ try {
384
+ const result = await read();
385
+ this.isOnline = true;
386
+ return result;
387
+ }
388
+ catch (error) {
389
+ if (!(error instanceof MissingKeysError))
390
+ this.isOnline = false;
391
+ throw error;
392
+ }
146
393
  }
147
394
  //
148
395
  // ═══ Write Operations ═══
149
396
  //
150
397
  async set(id, value) {
151
- this.bump(id);
152
- // Always write to local immediately
153
- await this.local.set(id, value);
154
- if (this.options.writeStrategy === "write-through") {
155
- // Write-through: sync to remote immediately
156
- try {
157
- await this.remote.set(id, value);
158
- }
159
- catch (error) {
160
- this.handleRemoteFailure("set", { id, value });
161
- throw error;
162
- }
163
- }
164
- else {
165
- // Write-back: queue for later sync
166
- this.queueOperation({
167
- type: "set",
168
- data: { id, value },
169
- timestamp: Date.now(),
170
- });
171
- }
398
+ await this.write("set", { id, value }, [id], async () => {
399
+ this.bump(id);
400
+ await this.local.set(id, value);
401
+ }, () => this.remote.set(id, value));
172
402
  }
173
403
  async delete(id) {
174
- this.bump(id);
175
- const existed = await this.local.delete(id);
176
- if (this.options.writeStrategy === "write-through") {
177
- try {
178
- await this.remote.delete(id);
179
- }
180
- catch (error) {
181
- this.handleRemoteFailure("delete", { id });
182
- throw error;
183
- }
184
- }
185
- else {
186
- this.queueOperation({
187
- type: "delete",
188
- data: { id },
189
- timestamp: Date.now(),
190
- });
191
- }
192
- return existed;
404
+ return this.write("delete", { id }, [id], async () => {
405
+ this.bump(id);
406
+ return this.local.delete(id);
407
+ }, () => this.remote.delete(id));
193
408
  }
194
409
  async clear() {
195
- this.clearVersion++;
196
- await this.local.clear();
197
- if (this.options.writeStrategy === "write-through") {
198
- try {
199
- await this.remote.clear();
200
- }
201
- catch (error) {
202
- this.handleRemoteFailure("clear", {});
203
- throw error;
204
- }
205
- }
206
- else {
207
- this.queueOperation({
208
- type: "clear",
209
- data: {},
210
- timestamp: Date.now(),
211
- });
212
- }
410
+ await this.write("clear", {}, "*", async () => {
411
+ this.clearVersion++;
412
+ await this.local.clear();
413
+ }, () => this.remote.clear());
213
414
  }
214
415
  //
215
416
  // ═══ Bulk Operations ═══
216
417
  //
217
418
  async setBatch(items) {
218
- for (const [id] of items)
219
- this.bump(id);
220
- await this.local.setBatch(items);
221
- if (this.options.writeStrategy === "write-through") {
222
- try {
223
- await this.remote.setBatch(items);
224
- }
225
- catch (error) {
226
- this.handleRemoteFailure("setBatch", { items });
227
- throw error;
228
- }
229
- }
230
- else {
231
- this.queueOperation({
232
- type: "setBatch",
233
- data: { items },
234
- timestamp: Date.now(),
235
- });
236
- }
419
+ await this.write("setBatch", { items }, items.map(([id]) => id), async () => {
420
+ for (const [id] of items)
421
+ this.bump(id);
422
+ await this.local.setBatch(items);
423
+ }, () => this.remote.setBatch(items));
237
424
  }
238
425
  async deleteBatch(ids) {
239
- for (const id of ids)
240
- this.bump(id);
241
- const deleted = await this.local.deleteBatch(ids);
242
- if (this.options.writeStrategy === "write-through") {
426
+ return this.write("deleteBatch", { ids }, ids, async () => {
427
+ for (const id of ids)
428
+ this.bump(id);
429
+ return this.local.deleteBatch(ids);
430
+ }, () => this.remote.deleteBatch(ids));
431
+ }
432
+ /**
433
+ * One write, in the mode of the storage. The result is the result of the local copy, or of the
434
+ * remote in `remote-only` mode.
435
+ *
436
+ * - `remote-only`: the remote only. Its error passes through.
437
+ * - `write-back` and `local-only`: the local copy and the queue. A full queue refuses the write
438
+ * before the local change. The queue store keeps the operation before the write ends.
439
+ * - `write-through`, the remote refuses the write: the local change is undone, and the error
440
+ * of the remote passes through.
441
+ * - `write-through`, the remote is not available: with the queue, the write waits in it, a timer
442
+ * sends it later, and the write throws `PendingWriteError`. Without the queue, or with a full
443
+ * queue, the local change is undone.
444
+ *
445
+ * Before, a failed write-through stayed in the local cache and in the queue, no timer sent the
446
+ * queue, a second try queued the operation again, and a full queue dropped the oldest operation
447
+ * without a word (roadmap R2).
448
+ */
449
+ write(type, data, ids, applyLocal, applyRemote) {
450
+ return this.run(false, async () => {
451
+ const op = { type, data, timestamp: Date.now() };
452
+ if (this.mode === "remote-only") {
453
+ const send = async () => {
454
+ await this.ensureStoreClean();
455
+ const result = await applyRemote();
456
+ this.noteRemoteWrite(op);
457
+ return result;
458
+ };
459
+ try {
460
+ const result = this.offlineOps.length > 0
461
+ ? await this.serialize(async () => {
462
+ await this.pushQueue();
463
+ return send();
464
+ })
465
+ : await send();
466
+ this.isOnline = true;
467
+ return result;
468
+ }
469
+ catch (error) {
470
+ if (!(error instanceof QueueStoreError) && this.options.isUnavailable(error))
471
+ this.isOnline = false;
472
+ throw error;
473
+ }
474
+ }
475
+ if (this.mode === "write-back" || this.mode === "local-only") {
476
+ this.ensureQueueRoom(op);
477
+ this.attachBase(op);
478
+ const undo = await this.snapshot(ids);
479
+ const result = await applyLocal();
480
+ await this.enqueue(op, undo);
481
+ return result;
482
+ }
483
+ const undo = await this.snapshot(ids);
484
+ const result = await applyLocal();
243
485
  try {
244
- await this.remote.deleteBatch(ids);
486
+ if (this.offlineOps.length > 0) {
487
+ // The queued operations are older: they reach the remote first, or the older value of a
488
+ // queued write replaces this one later. A failed send keeps this write behind them.
489
+ await this.serialize(async () => {
490
+ await this.pushQueue();
491
+ await this.ensureStoreClean();
492
+ await applyRemote();
493
+ });
494
+ }
495
+ else {
496
+ await this.ensureStoreClean();
497
+ await applyRemote();
498
+ }
499
+ this.isOnline = true;
500
+ this.noteRemoteWrite(op);
245
501
  }
246
502
  catch (error) {
247
- this.handleRemoteFailure("deleteBatch", { ids });
503
+ if (!(error instanceof QueueStoreError) &&
504
+ this.options.offlineQueue &&
505
+ this.options.isUnavailable(error) &&
506
+ this.hasQueueRoom(op)) {
507
+ this.isOnline = false;
508
+ console.error(`Remote ${type} failed: the operation waits in the queue`);
509
+ this.attachBase(op);
510
+ await this.enqueue(op, undo);
511
+ this.scheduleDrain();
512
+ throw new PendingWriteError(type, error);
513
+ }
514
+ await undo();
248
515
  throw error;
249
516
  }
250
- }
251
- else {
252
- this.queueOperation({
253
- type: "deleteBatch",
254
- data: { ids },
255
- timestamp: Date.now(),
256
- });
257
- }
258
- return deleted;
517
+ return result;
518
+ });
259
519
  }
260
- getBatch(ids) {
261
- return this.local.getBatch(ids);
520
+ /**
521
+ * The local values of the ids before a write, and a function that puts them back. It puts back
522
+ * only the ids that no later write changed.
523
+ */
524
+ async snapshot(ids) {
525
+ const before = ids === "*" ? new Map(await this.local.getEntries()) : await this.local.getBatch(ids);
526
+ const touched = ids === "*" ? [...before.keys()] : ids;
527
+ // The versions after the write: this write adds one to each id (clear adds to clearVersion)
528
+ const versions = new Map(touched.map((id) => [id, (this.writeVersions.get(id) ?? 0) + (ids === "*" ? 0 : 1)]));
529
+ const clearVersion = this.clearVersion + (ids === "*" ? 1 : 0);
530
+ return async () => {
531
+ if (this.clearVersion !== clearVersion)
532
+ return;
533
+ for (const id of touched) {
534
+ if ((this.writeVersions.get(id) ?? 0) !== versions.get(id))
535
+ continue;
536
+ const value = before.get(id);
537
+ if (value === undefined)
538
+ await this.local.delete(id);
539
+ else
540
+ await this.local.set(id, value);
541
+ }
542
+ };
262
543
  }
263
544
  //
264
545
  // ═══ Sync Operations ═══
265
546
  //
266
547
  /**
267
- * Sync all data from remote to local.
268
- * Handles conflicts according to conflict resolution strategy.
548
+ * Sync from the remote to the local copy, in the modes `write-through` and `write-back`. The
549
+ * other modes have no local copy to sync (`remote-only`) or make no remote call (`local-only`):
550
+ * the call does nothing.
269
551
  *
270
- * - Each remote item goes into the cache under `keyOf(item)`.
552
+ * - The sync asks for the changes since the version of the last sync when the remote has
553
+ * `getChangesSince`. Else it reads all entries.
554
+ * - Each remote item goes into the cache under its id.
271
555
  * - A local item that the remote does not have is removed, unless a queued local operation
272
- * changes it (the remote deleted it).
556
+ * changes it.
273
557
  * - An id that a local write changed during the sync keeps the local value: the remote value
274
558
  * is older.
275
559
  * - Syncs run one at a time.
276
560
  */
277
561
  syncFromRemote() {
278
- return this.serialize(() => this.pullRemote());
562
+ return this.serialize(async () => {
563
+ await this.loaded;
564
+ if (this.loadError)
565
+ throw this.loadError;
566
+ if (this.mode === "write-through" || this.mode === "write-back")
567
+ await this.pullRemote(false);
568
+ });
279
569
  }
280
570
  /**
281
571
  * Sync all pending offline operations to remote.
282
572
  * Processes queued operations in order. When one fails, it and the later operations stay in
283
- * the queue: the operations that reached the remote are not sent again.
573
+ * the queue: the operations that reached the remote are not sent again. In `local-only` mode,
574
+ * the call does nothing.
284
575
  */
285
576
  syncToRemote() {
286
- return this.serialize(() => this.pushQueue());
577
+ return this.serialize(async () => {
578
+ await this.loaded;
579
+ if (this.loadError)
580
+ throw this.loadError;
581
+ if (this.mode !== "local-only")
582
+ await this.pushQueue();
583
+ });
584
+ }
585
+ /**
586
+ * This method sends the queue to the remote (roadmap R28). `ProcedureServer.useStorage` uses it. In
587
+ * `local-only` mode, it sends nothing: the queue waits for a remote mode.
588
+ */
589
+ flush() {
590
+ return this.syncToRemote();
287
591
  }
288
592
  /**
289
593
  * Force a full sync (both directions).
@@ -295,14 +599,26 @@ export class HybridStorage {
295
599
  //
296
600
  // ═══ Lifecycle & Metadata ═══
297
601
  //
602
+ /**
603
+ * This method stops the timers, sends the queue (not in `local-only` mode), and closes the remote and the
604
+ * local copy. The queue store keeps the operations that the remote did not take. A durable local
605
+ * copy keeps its items.
606
+ */
298
607
  async close() {
299
- // Stop periodic sync
300
- if (this.syncTimer) {
301
- clearInterval(this.syncTimer);
302
- this.syncTimer = null;
608
+ this.closed = true;
609
+ for (const timer of [this.syncTimer, this.pullTimer]) {
610
+ if (timer)
611
+ clearInterval(timer);
612
+ }
613
+ this.syncTimer = null;
614
+ this.pullTimer = null;
615
+ if (this.drainTimer) {
616
+ clearTimeout(this.drainTimer);
617
+ this.drainTimer = null;
303
618
  }
619
+ await this.loaded;
304
620
  // Sync pending operations before closing
305
- if (this.offlineOps.length > 0) {
621
+ if (this.offlineOps.length > 0 && this.mode !== "local-only") {
306
622
  try {
307
623
  await this.syncToRemote();
308
624
  }
@@ -310,8 +626,14 @@ export class HybridStorage {
310
626
  console.error("Failed to sync before close:", error);
311
627
  }
312
628
  }
629
+ await this.storeChain;
313
630
  // Close remote storage (an ApiStorage leaves the caller's client open)
314
- await this.remote.close();
631
+ try {
632
+ await this.remote.close();
633
+ }
634
+ finally {
635
+ await this.local.close();
636
+ }
315
637
  }
316
638
  async getMetadata() {
317
639
  const totalOps = this.stats.cacheHits + this.stats.cacheMisses;
@@ -328,7 +650,8 @@ export class HybridStorage {
328
650
  conflicts: this.stats.conflicts,
329
651
  pendingOps: this.offlineOps.length,
330
652
  isOnline: this.isOnline,
331
- writeStrategy: this.options.writeStrategy,
653
+ mode: this.mode,
654
+ writeStrategy: this.mode,
332
655
  conflictResolution: this.options.conflictResolution,
333
656
  },
334
657
  };
@@ -336,68 +659,378 @@ export class HybridStorage {
336
659
  //
337
660
  // ═══ Internal Helpers ═══
338
661
  //
339
- async pullRemote() {
662
+ /**
663
+ * This method starts a public operation: after the load of the queue (a write) or the first sync (a read),
664
+ * and never during a mode switch.
665
+ */
666
+ async run(read, work) {
667
+ await (read ? this.ready : this.loaded);
668
+ for (;;) {
669
+ const gate = this.gate;
670
+ await gate;
671
+ // No new switch started while this operation waited
672
+ if (this.gate === gate)
673
+ break;
674
+ }
675
+ if (this.loadError)
676
+ throw this.loadError;
677
+ this.active++;
678
+ try {
679
+ return await work();
680
+ }
681
+ finally {
682
+ if (--this.active === 0) {
683
+ for (const resolve of this.idleWaiters.splice(0))
684
+ resolve();
685
+ }
686
+ }
687
+ }
688
+ /** A promise that resolves when no operation runs. */
689
+ idle() {
690
+ if (this.active === 0)
691
+ return Promise.resolve();
692
+ return new Promise((resolve) => this.idleWaiters.push(resolve));
693
+ }
694
+ /** This method loads the queue, and puts its writes in the local copy. */
695
+ async loadQueue() {
696
+ let ops;
697
+ try {
698
+ ops = await this.queueStore.load();
699
+ }
700
+ catch (error) {
701
+ this.loadError = new QueueStoreError("load", error);
702
+ console.error("The queue store could not load the queue: the storage refuses each operation", error);
703
+ return;
704
+ }
705
+ this.offlineOps = ops.slice();
706
+ try {
707
+ for (const op of this.offlineOps) {
708
+ for (const [id, version] of op.base ?? []) {
709
+ if (!this.bases.has(id))
710
+ this.bases.set(id, version);
711
+ }
712
+ // A durable local copy can hold the operation already: each operation gives the same
713
+ // result two times
714
+ await this.applyToLocal(op);
715
+ }
716
+ }
717
+ catch (error) {
718
+ this.loadError = new Error("The local copy could not take the loaded queue: the storage refuses each operation", { cause: error });
719
+ console.error(this.loadError.message, error);
720
+ }
721
+ }
722
+ /** The first sync: send the loaded queue, then read the remote. */
723
+ async initialSync() {
724
+ if (this.loadError)
725
+ return;
726
+ if (this.options.syncOnInit && this.mode !== "local-only") {
727
+ if (this.offlineOps.length > 0) {
728
+ await this.syncToRemote().catch((err) => {
729
+ console.error("Failed to send the queue on init:", err);
730
+ });
731
+ }
732
+ if (this.mode === "write-through" || this.mode === "write-back") {
733
+ await this.syncFromRemote().catch((err) => {
734
+ console.error("Failed to sync on init:", err);
735
+ });
736
+ }
737
+ }
738
+ if (this.offlineOps.length > 0)
739
+ this.scheduleDrain();
740
+ }
741
+ /** This method applies a queued operation to the local copy. */
742
+ async applyToLocal(op) {
743
+ const data = op.data;
744
+ switch (op.type) {
745
+ case "set":
746
+ await this.local.set(data.id, data.value);
747
+ break;
748
+ case "delete":
749
+ await this.local.delete(data.id);
750
+ break;
751
+ case "setBatch":
752
+ await this.local.setBatch(data.items);
753
+ break;
754
+ case "deleteBatch":
755
+ await this.local.deleteBatch(data.ids);
756
+ break;
757
+ case "clear":
758
+ await this.local.clear();
759
+ break;
760
+ }
761
+ }
762
+ /**
763
+ * One pull. `full`: ask for all items (after the local copy was dropped).
764
+ *
765
+ * - The versions of the local writes at the start: a write during the pull keeps its value.
766
+ * - The cursor moves only after a pull without conflicts, so a conflict of the "error" policy
767
+ * comes again at the next pull.
768
+ */
769
+ async pullRemote(full) {
340
770
  const versions = new Map(this.writeVersions);
341
771
  const clearVersion = this.clearVersion;
342
772
  // True when a local write changed the id after the sync started
343
773
  const changedLocally = (id) => this.clearVersion !== clearVersion || (this.writeVersions.get(id) ?? 0) !== (versions.get(id) ?? 0);
344
- let remoteData;
774
+ let changes;
345
775
  try {
346
- remoteData = await this.remote.getAll();
776
+ changes = await this.fetchChanges(full);
347
777
  }
348
778
  catch (error) {
349
- this.isOnline = false;
779
+ if (!(error instanceof MissingKeysError))
780
+ this.isOnline = false;
350
781
  throw error;
351
782
  }
783
+ const conflicts = [];
784
+ let queueChanged = false;
352
785
  const remoteIds = new Set();
353
- for (const item of remoteData) {
354
- const id = this.options.keyOf(item);
355
- if (!id) {
356
- console.warn("Item has no key, skipping:", item);
357
- continue;
358
- }
786
+ for (const [id, item] of changes.changed) {
359
787
  remoteIds.add(id);
360
- if (changedLocally(id) || this.hasPendingOp(id)) {
361
- continue;
788
+ if (await this.pullItem(id, item, changedLocally, conflicts))
789
+ queueChanged = true;
790
+ }
791
+ // The remote deleted these items
792
+ const gone = changes.full
793
+ ? (await this.local.getEntries()).map(([id]) => id).filter((id) => !remoteIds.has(id))
794
+ : changes.deleted;
795
+ for (const id of gone) {
796
+ if (await this.pullDelete(id, changedLocally, conflicts))
797
+ queueChanged = true;
798
+ }
799
+ if (queueChanged) {
800
+ this.storeDirty = true;
801
+ await this.persistAll().catch((error) => {
802
+ console.error("The queue store could not keep the queue after the sync:", error);
803
+ });
804
+ }
805
+ this.stats.syncCount++;
806
+ this.isOnline = true;
807
+ if (conflicts.length > 0)
808
+ throw new ConflictError(conflicts);
809
+ if (changes.version !== undefined)
810
+ this.cursor = changes.version;
811
+ }
812
+ /**
813
+ * The remote changes: since the cursor from `getChangesSince` when the remote has it, or else
814
+ * all entries.
815
+ */
816
+ async fetchChanges(full) {
817
+ if (this.remote.getChangesSince && !this.changesUnsupported) {
818
+ try {
819
+ return await this.remote.getChangesSince(full ? undefined : this.cursor);
362
820
  }
363
- const localItem = await this.local.get(id);
364
- if (localItem === undefined) {
365
- // No local version - just store remote
366
- await this.local.set(id, item);
821
+ catch (error) {
822
+ if (!isNotFound(error))
823
+ throw error;
824
+ // A server without the procedure `changes`: read all entries from now on
825
+ this.changesUnsupported = true;
826
+ }
827
+ }
828
+ return { full: true, changed: await this.remoteEntries(), deleted: [] };
829
+ }
830
+ /** This method applies one remote item. It gives true when the queue changed. */
831
+ async pullItem(id, item, changedLocally, conflicts) {
832
+ if (changedLocally(id))
833
+ return false;
834
+ const localItem = await this.local.get(id);
835
+ if (changedLocally(id))
836
+ return false;
837
+ const remoteVersion = this.versionOfItem(item);
838
+ if (this.hasPendingOp(id)) {
839
+ // Without versions, the queued write wins, as before
840
+ if (remoteVersion === undefined)
841
+ return false;
842
+ // The remote did not change since the last sync, or the storage never saw a remote version
843
+ // of the id: the queued write wins
844
+ const base = this.bases.get(id);
845
+ if (base === undefined || base === remoteVersion)
846
+ return false;
847
+ if (localItem !== undefined && this.versionOfItem(localItem) === remoteVersion) {
848
+ this.bases.set(id, remoteVersion);
849
+ return false;
367
850
  }
368
- else if (JSON.stringify(localItem) !== JSON.stringify(item)) {
369
- // Conflict - resolve according to strategy
370
- const resolved = this.resolveConflict(localItem, item);
371
- await this.local.set(id, resolved);
851
+ return this.resolvePending(id, localItem, item, remoteVersion, conflicts);
852
+ }
853
+ const localVersion = localItem === undefined ? undefined : this.versionOfItem(localItem);
854
+ if (localItem === undefined) {
855
+ await this.local.set(id, item);
856
+ }
857
+ else if (remoteVersion !== undefined && localVersion !== undefined) {
858
+ // Only the remote changed: it is not a conflict
859
+ if (localVersion !== remoteVersion)
860
+ await this.local.set(id, item);
861
+ }
862
+ else if (JSON.stringify(localItem) !== JSON.stringify(item)) {
863
+ // Conflict - resolve according to strategy
864
+ if (this.options.conflictResolution === "error") {
372
865
  this.stats.conflicts++;
866
+ conflicts.push(id);
867
+ return false;
373
868
  }
869
+ const resolved = this.resolveConflict(localItem, item);
870
+ await this.local.set(id, resolved);
871
+ this.stats.conflicts++;
374
872
  }
375
- // The remote deleted these items
376
- for (const id of await this.localIds()) {
377
- if (!remoteIds.has(id) && !changedLocally(id) && !this.hasPendingOp(id)) {
873
+ if (remoteVersion !== undefined)
874
+ this.bases.set(id, remoteVersion);
875
+ return false;
876
+ }
877
+ /** This method applies one remote delete. It gives true when the queue changed. */
878
+ async pullDelete(id, changedLocally, conflicts) {
879
+ if (changedLocally(id))
880
+ return false;
881
+ if (!this.hasPendingOp(id)) {
882
+ await this.local.delete(id);
883
+ if (this.options.versionOf)
884
+ this.bases.set(id, null);
885
+ return false;
886
+ }
887
+ // Without versions, the queued write wins, as before. With versions, the remote deleted the
888
+ // item only if the last sync saw a version of it.
889
+ const base = this.bases.get(id);
890
+ if (!this.options.versionOf || base === undefined || base === null)
891
+ return false;
892
+ const localItem = await this.local.get(id);
893
+ if (changedLocally(id))
894
+ return false;
895
+ if (localItem === undefined) {
896
+ // Both sides deleted the item
897
+ this.bases.set(id, null);
898
+ return false;
899
+ }
900
+ this.stats.conflicts++;
901
+ switch (this.options.conflictResolution) {
902
+ case "remote":
903
+ if (!this.dropQueued(id))
904
+ break;
378
905
  await this.local.delete(id);
906
+ this.bases.set(id, null);
907
+ return true;
908
+ case "error":
909
+ conflicts.push(id);
910
+ return false;
911
+ default:
912
+ // "local", and "merge" with one deleted side: the queued write wins
913
+ break;
914
+ }
915
+ this.bases.set(id, null);
916
+ return false;
917
+ }
918
+ /**
919
+ * A queued local write of an id whose remote version changed: a conflict, and the policy
920
+ * decides. True when the queue changed.
921
+ */
922
+ async resolvePending(id, localItem, remoteItem, remoteVersion, conflicts) {
923
+ this.stats.conflicts++;
924
+ switch (this.options.conflictResolution) {
925
+ case "remote":
926
+ // The remote value replaces the queued write. A queued clear cannot lose one id.
927
+ if (!this.dropQueued(id))
928
+ break;
929
+ await this.local.set(id, remoteItem);
930
+ this.bases.set(id, remoteVersion);
931
+ return true;
932
+ case "merge": {
933
+ if (localItem === undefined)
934
+ break;
935
+ const merged = this.resolveConflict(localItem, remoteItem);
936
+ if (!this.dropQueued(id))
937
+ break;
938
+ await this.local.set(id, merged);
939
+ this.bases.set(id, remoteVersion);
940
+ this.offlineOps.push({ type: "set", data: { id, value: merged }, timestamp: Date.now(), base: [[id, remoteVersion]] });
941
+ return true;
379
942
  }
943
+ case "error":
944
+ conflicts.push(id);
945
+ return false;
946
+ default:
947
+ break;
380
948
  }
381
- this.stats.syncCount++;
382
- this.isOnline = true;
949
+ // "local": the queued write wins, and it replaces this remote version
950
+ this.bases.set(id, remoteVersion);
951
+ return false;
952
+ }
953
+ /**
954
+ * This method removes an id from the queued operations. It gives false when a queued clear touches it: nothing
955
+ * changes then.
956
+ */
957
+ dropQueued(id) {
958
+ if (this.offlineOps.some((op) => op.type === "clear"))
959
+ return false;
960
+ const next = [];
961
+ for (const op of this.offlineOps) {
962
+ const kept = withoutId(op, id);
963
+ if (kept)
964
+ next.push(kept);
965
+ }
966
+ this.offlineOps = next;
967
+ return true;
968
+ }
969
+ /**
970
+ * The remote items with their ids: from `getEntries()` when the remote has it, or else from
971
+ * `getAll()` and `keyOf`. An item without a key stops the sync before any change. Before, the
972
+ * sync skipped such items and then deleted each local item that the remote list did not have,
973
+ * so the remote list without ids emptied the cache (roadmap R2).
974
+ */
975
+ async remoteEntries() {
976
+ if (this.remote.getEntries)
977
+ return this.remote.getEntries();
978
+ const items = await this.remote.getAll();
979
+ const entries = [];
980
+ let missing = 0;
981
+ for (const item of items) {
982
+ const id = this.options.keyOf(item);
983
+ if (id)
984
+ entries.push([id, item]);
985
+ else
986
+ missing++;
987
+ }
988
+ if (missing > 0)
989
+ throw new MissingKeysError(missing);
990
+ return entries;
383
991
  }
992
+ /**
993
+ * This method sends the queue. The operations that reached the remote leave the queue. The queue store
994
+ * gets the change at the end.
995
+ */
384
996
  async pushQueue() {
385
- while (this.offlineOps.length > 0) {
386
- const op = this.offlineOps[0];
387
- try {
388
- await this.executeOperation(op);
389
- }
390
- catch (error) {
391
- // This operation and the later ones stay queued
392
- this.isOnline = false;
393
- throw error;
997
+ this.pushing = true;
998
+ let sent = 0;
999
+ try {
1000
+ while (this.offlineOps.length > 0) {
1001
+ const op = this.offlineOps[0];
1002
+ if (this.unpersisted.has(op)) {
1003
+ // The write of this operation waits for the queue store
1004
+ await this.storeChain;
1005
+ continue;
1006
+ }
1007
+ try {
1008
+ await this.executeOperation(op);
1009
+ }
1010
+ catch (error) {
1011
+ // This operation and the later ones stay queued
1012
+ this.isOnline = false;
1013
+ throw error;
1014
+ }
1015
+ this.noteRemoteWrite(op);
1016
+ // Remove the operation only after it reached the remote
1017
+ if (this.offlineOps[0] === op) {
1018
+ this.offlineOps.shift();
1019
+ sent++;
1020
+ this.storeDirty = true;
1021
+ }
394
1022
  }
395
- // Remove the operation only after it reached the remote
396
- if (this.offlineOps[0] === op) {
397
- this.offlineOps.shift();
1023
+ this.isOnline = true;
1024
+ }
1025
+ finally {
1026
+ this.pushing = false;
1027
+ if (sent > 0) {
1028
+ await this.persistAll().catch((error) => {
1029
+ // The store keeps sent operations. A later direct write repairs it first.
1030
+ console.error("The queue store could not remove the sent operations:", error);
1031
+ });
398
1032
  }
399
1033
  }
400
- this.isOnline = true;
401
1034
  }
402
1035
  /** Run syncs one at a time. */
403
1036
  serialize(run) {
@@ -405,15 +1038,105 @@ export class HybridStorage {
405
1038
  this.syncChain = next.catch(() => undefined);
406
1039
  return next;
407
1040
  }
408
- async localIds() {
409
- return [...this.local.keys()];
410
- }
411
1041
  bump(id) {
412
1042
  this.writeVersions.set(id, (this.writeVersions.get(id) ?? 0) + 1);
413
1043
  }
414
- versionOf(id) {
1044
+ /** The count of local writes of an id: it changes with each write and each clear. */
1045
+ writeKey(id) {
415
1046
  return `${this.clearVersion}:${this.writeVersions.get(id) ?? 0}`;
416
1047
  }
1048
+ versionOfItem(item) {
1049
+ return this.options.versionOf?.(item);
1050
+ }
1051
+ /** This method records the remote version of an id that the storage read. */
1052
+ noteBase(id, item) {
1053
+ if (!this.options.versionOf)
1054
+ return;
1055
+ if (item === undefined) {
1056
+ this.bases.set(id, null);
1057
+ return;
1058
+ }
1059
+ const version = this.versionOfItem(item);
1060
+ if (version === undefined)
1061
+ this.bases.delete(id);
1062
+ else
1063
+ this.bases.set(id, version);
1064
+ }
1065
+ /** This method records the remote versions after a write reached the remote. */
1066
+ noteRemoteWrite(op) {
1067
+ if (!this.options.versionOf)
1068
+ return;
1069
+ const data = op.data;
1070
+ switch (op.type) {
1071
+ case "set":
1072
+ this.noteBase(data.id, data.value);
1073
+ break;
1074
+ case "delete":
1075
+ this.bases.set(data.id, null);
1076
+ break;
1077
+ case "setBatch":
1078
+ for (const [id, value] of data.items)
1079
+ this.noteBase(id, value);
1080
+ break;
1081
+ case "deleteBatch":
1082
+ for (const id of data.ids)
1083
+ this.bases.set(id, null);
1084
+ break;
1085
+ case "clear":
1086
+ for (const id of this.bases.keys())
1087
+ this.bases.set(id, null);
1088
+ break;
1089
+ }
1090
+ }
1091
+ /**
1092
+ * This method puts in the operation the remote version that it changes, for each id: the version that an
1093
+ * older queued operation of the id writes, or else the known remote version. A storage that
1094
+ * loads the queue after a restart takes the version of the first operation of each id.
1095
+ */
1096
+ attachBase(op) {
1097
+ if (!this.options.versionOf)
1098
+ return;
1099
+ const base = [];
1100
+ for (const id of idsOf(op)) {
1101
+ if (id === "*")
1102
+ continue;
1103
+ const version = this.expectedBase(id);
1104
+ if (version !== undefined)
1105
+ base.push([id, version]);
1106
+ }
1107
+ if (base.length > 0)
1108
+ op.base = base;
1109
+ }
1110
+ /** The remote version of an id after the queued operations, or undefined when it is not known. */
1111
+ expectedBase(id) {
1112
+ for (let index = this.offlineOps.length - 1; index >= 0; index--) {
1113
+ const op = this.offlineOps[index];
1114
+ const data = op.data;
1115
+ switch (op.type) {
1116
+ case "set":
1117
+ if (data.id === id)
1118
+ return this.versionOfItem(data.value);
1119
+ break;
1120
+ case "delete":
1121
+ if (data.id === id)
1122
+ return null;
1123
+ break;
1124
+ case "setBatch": {
1125
+ const item = (data.items ?? []).findLast(([itemId]) => itemId === id);
1126
+ if (item)
1127
+ return this.versionOfItem(item[1]);
1128
+ break;
1129
+ }
1130
+ case "deleteBatch":
1131
+ if ((data.ids ?? []).includes(id))
1132
+ return null;
1133
+ break;
1134
+ case "clear":
1135
+ return null;
1136
+ }
1137
+ }
1138
+ return this.bases.get(id);
1139
+ }
417
1140
  hasPendingOp(id) {
418
1141
  return this.offlineOps.some((op) => {
419
1142
  const ids = idsOf(op);
@@ -440,18 +1163,165 @@ export class HybridStorage {
440
1163
  return remote;
441
1164
  }
442
1165
  }
1166
+ //
1167
+ // ═══ The queue and its store ═══
1168
+ //
443
1169
  /**
444
- * Queue operation for later sync.
1170
+ * This method puts an operation in the queue, then in the queue store. The write ends only after the store
1171
+ * kept it. When the store fails, the operation leaves the queue, the local change is undone,
1172
+ * and the write throws `QueueStoreError`.
1173
+ *
1174
+ * A set or a delete of one id replaces the queued set or delete of that id, when no later
1175
+ * operation touches the id: a second try of a write does not queue it two times.
445
1176
  */
446
- queueOperation(op) {
447
- if (!this.options.offlineQueue) {
448
- return;
1177
+ async enqueue(op, undo) {
1178
+ const index = this.replaceableIndex(op);
1179
+ let rollback;
1180
+ if (index >= 0) {
1181
+ const previous = this.offlineOps[index];
1182
+ // The new operation changes the same remote version as the operation that it replaces
1183
+ if (previous.base)
1184
+ op.base = previous.base;
1185
+ else
1186
+ delete op.base;
1187
+ this.offlineOps[index] = op;
1188
+ rollback = () => {
1189
+ const at = this.offlineOps.indexOf(op);
1190
+ if (at >= 0)
1191
+ this.offlineOps[at] = previous;
1192
+ };
1193
+ }
1194
+ else {
1195
+ try {
1196
+ this.ensureQueueRoom(op);
1197
+ }
1198
+ catch (error) {
1199
+ await undo();
1200
+ throw error;
1201
+ }
1202
+ this.offlineOps.push(op);
1203
+ rollback = () => {
1204
+ const at = this.offlineOps.indexOf(op);
1205
+ if (at >= 0)
1206
+ this.offlineOps.splice(at, 1);
1207
+ };
449
1208
  }
450
- this.offlineOps.push(op);
451
- // Enforce max queue size
452
- if (this.offlineOps.length > this.options.maxOfflineOps) {
453
- this.offlineOps.shift(); // Remove oldest
1209
+ this.unpersisted.add(op);
1210
+ try {
1211
+ await this.persist(op, index < 0, rollback);
1212
+ }
1213
+ catch (error) {
1214
+ await undo();
1215
+ throw error;
1216
+ }
1217
+ }
1218
+ /**
1219
+ * This method writes one operation to the queue store: an append when the store holds each operation
1220
+ * before it, or else the whole queue. On a failure, `rollback` removes the operation from the
1221
+ * queue before the next call of the store.
1222
+ */
1223
+ persist(op, appended, rollback) {
1224
+ const run = this.storeChain.then(async () => {
1225
+ // An earlier write of the whole queue kept it already
1226
+ if (!this.unpersisted.has(op))
1227
+ return;
1228
+ const at = this.offlineOps.indexOf(op);
1229
+ const canAppend = appended && !this.storeDirty && at >= 0 && this.offlineOps.slice(0, at).every((other) => !this.unpersisted.has(other));
1230
+ const snapshot = this.offlineOps.slice();
1231
+ try {
1232
+ if (canAppend) {
1233
+ await this.queueStore.append(op);
1234
+ this.unpersisted.delete(op);
1235
+ }
1236
+ else {
1237
+ await this.queueStore.replace(snapshot);
1238
+ for (const kept of snapshot)
1239
+ this.unpersisted.delete(kept);
1240
+ this.unpersisted.delete(op);
1241
+ this.storeDirty = false;
1242
+ }
1243
+ }
1244
+ catch (error) {
1245
+ this.storeDirty = true;
1246
+ this.unpersisted.delete(op);
1247
+ rollback();
1248
+ throw new QueueStoreError("write", error);
1249
+ }
1250
+ });
1251
+ this.storeChain = run.catch(() => undefined);
1252
+ return run;
1253
+ }
1254
+ /** This method writes the whole queue to the queue store when the store can differ from the queue. */
1255
+ persistAll() {
1256
+ const run = this.storeChain.then(async () => {
1257
+ if (!this.storeDirty && this.unpersisted.size === 0)
1258
+ return;
1259
+ const snapshot = this.offlineOps.slice();
1260
+ try {
1261
+ if (snapshot.length === 0)
1262
+ await this.queueStore.clear();
1263
+ else
1264
+ await this.queueStore.replace(snapshot);
1265
+ for (const kept of snapshot)
1266
+ this.unpersisted.delete(kept);
1267
+ this.storeDirty = false;
1268
+ }
1269
+ catch (error) {
1270
+ this.storeDirty = true;
1271
+ throw new QueueStoreError("write", error);
1272
+ }
1273
+ });
1274
+ this.storeChain = run.catch(() => undefined);
1275
+ return run;
1276
+ }
1277
+ /**
1278
+ * Before a write goes to the remote directly, the queue store must not hold a sent operation:
1279
+ * after a restart, it would replace this newer write.
1280
+ */
1281
+ async ensureStoreClean() {
1282
+ if (this.storeDirty)
1283
+ await this.persistAll();
1284
+ }
1285
+ /** The index of the queued operation that `op` replaces, or -1. */
1286
+ replaceableIndex(op) {
1287
+ if (op.type !== "set" && op.type !== "delete")
1288
+ return -1;
1289
+ const [id] = idsOf(op);
1290
+ if (id === undefined)
1291
+ return -1;
1292
+ for (let index = this.offlineOps.length - 1; index >= 0; index--) {
1293
+ const queued = this.offlineOps[index];
1294
+ const ids = idsOf(queued);
1295
+ if (!ids.includes(id) && !ids.includes("*"))
1296
+ continue;
1297
+ // pushQueue can be sending the first operation now: do not change it
1298
+ const sending = index === 0 && this.pushing;
1299
+ return !sending && (queued.type === "set" || queued.type === "delete") ? index : -1;
454
1300
  }
1301
+ return -1;
1302
+ }
1303
+ /** True when the queue can take the operation. */
1304
+ hasQueueRoom(op) {
1305
+ return this.replaceableIndex(op) >= 0 || this.offlineOps.length < this.options.maxOfflineOps;
1306
+ }
1307
+ /** A full queue refuses the write. Before, it dropped the oldest operation (roadmap R2). */
1308
+ ensureQueueRoom(op) {
1309
+ if (!this.hasQueueRoom(op))
1310
+ throw new QueueFullError(this.options.maxOfflineOps);
1311
+ }
1312
+ /**
1313
+ * In write-through mode, a timer sends the queue after a failure, and again after each failed
1314
+ * send, until the queue is empty.
1315
+ */
1316
+ scheduleDrain() {
1317
+ if (this.drainTimer || this.closed || this.mode !== "write-through")
1318
+ return;
1319
+ const delay = this.options.syncInterval > 0 ? this.options.syncInterval : 5000;
1320
+ this.drainTimer = setTimeout(() => {
1321
+ this.drainTimer = null;
1322
+ this.syncToRemote().catch(() => this.scheduleDrain());
1323
+ }, delay);
1324
+ this.drainTimer.unref?.();
455
1325
  }
456
1326
  /**
457
1327
  * Execute a queued operation against remote storage.
@@ -476,33 +1346,6 @@ export class HybridStorage {
476
1346
  break;
477
1347
  }
478
1348
  }
479
- /**
480
- * Handle remote operation failure.
481
- */
482
- handleRemoteFailure(operation, data) {
483
- console.error(`Remote ${operation} failed, marking offline`);
484
- this.isOnline = false;
485
- // Queue operation if offline queue enabled
486
- if (this.options.offlineQueue) {
487
- this.queueOperation({
488
- type: operation,
489
- data,
490
- timestamp: Date.now(),
491
- });
492
- }
493
- }
494
- /**
495
- * Start periodic background sync.
496
- */
497
- startPeriodicSync() {
498
- this.syncTimer = setInterval(() => {
499
- this.syncToRemote().catch((err) => {
500
- console.error("Periodic sync failed:", err);
501
- });
502
- }, this.options.syncInterval);
503
- // The timer must not keep the process alive (deep dive DATA-9)
504
- this.syncTimer.unref?.();
505
- }
506
1349
  /**
507
1350
  * Get current online status.
508
1351
  */