@procedures/store 0.2.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +79 -0
  2. package/README.md +109 -2
  3. package/dist/.tsbuildinfo +1 -1
  4. package/dist/indexeddb/idb.d.ts +100 -0
  5. package/dist/indexeddb/idb.d.ts.map +1 -0
  6. package/dist/indexeddb/idb.js +137 -0
  7. package/dist/indexeddb/idb.js.map +1 -0
  8. package/dist/indexeddb/program-store.d.ts +42 -0
  9. package/dist/indexeddb/program-store.d.ts.map +1 -0
  10. package/dist/indexeddb/program-store.js +71 -0
  11. package/dist/indexeddb/program-store.js.map +1 -0
  12. package/dist/indexeddb/queue-store.d.ts +38 -0
  13. package/dist/indexeddb/queue-store.d.ts.map +1 -0
  14. package/dist/indexeddb/queue-store.js +73 -0
  15. package/dist/indexeddb/queue-store.js.map +1 -0
  16. package/dist/indexeddb.d.ts +19 -0
  17. package/dist/indexeddb.d.ts.map +1 -0
  18. package/dist/indexeddb.js +17 -0
  19. package/dist/indexeddb.js.map +1 -0
  20. package/dist/program-store/file.d.ts +5 -0
  21. package/dist/program-store/file.d.ts.map +1 -1
  22. package/dist/program-store/file.js +5 -2
  23. package/dist/program-store/file.js.map +1 -1
  24. package/dist/storage/api.d.ts +18 -7
  25. package/dist/storage/api.d.ts.map +1 -1
  26. package/dist/storage/api.js +34 -3
  27. package/dist/storage/api.js.map +1 -1
  28. package/dist/storage/factory.d.ts +13 -2
  29. package/dist/storage/factory.d.ts.map +1 -1
  30. package/dist/storage/factory.js +12 -4
  31. package/dist/storage/factory.js.map +1 -1
  32. package/dist/storage/file-queue.d.ts +32 -0
  33. package/dist/storage/file-queue.d.ts.map +1 -0
  34. package/dist/storage/file-queue.js +101 -0
  35. package/dist/storage/file-queue.js.map +1 -0
  36. package/dist/storage/hybrid.d.ts +361 -42
  37. package/dist/storage/hybrid.d.ts.map +1 -1
  38. package/dist/storage/hybrid.js +1102 -251
  39. package/dist/storage/hybrid.js.map +1 -1
  40. package/dist/storage/index.d.ts +4 -2
  41. package/dist/storage/index.d.ts.map +1 -1
  42. package/dist/storage/index.js +5 -1
  43. package/dist/storage/index.js.map +1 -1
  44. package/dist/storage/queue.d.ts +67 -0
  45. package/dist/storage/queue.d.ts.map +1 -0
  46. package/dist/storage/queue.js +85 -0
  47. package/dist/storage/queue.js.map +1 -0
  48. package/dist/storage/synced-registry.d.ts +18 -2
  49. package/dist/storage/synced-registry.d.ts.map +1 -1
  50. package/dist/storage/synced-registry.js +37 -3
  51. package/dist/storage/synced-registry.js.map +1 -1
  52. package/dist/storage/types.d.ts +5 -2
  53. package/dist/storage/types.d.ts.map +1 -1
  54. package/package.json +16 -6
  55. package/src/indexeddb/idb.ts +190 -0
  56. package/src/indexeddb/program-store.ts +80 -0
  57. package/src/indexeddb/queue-store.ts +80 -0
  58. package/src/indexeddb.ts +19 -0
  59. package/src/program-store/file.ts +5 -2
  60. package/src/storage/api.ts +50 -10
  61. package/src/storage/factory.ts +21 -4
  62. package/src/storage/file-queue.ts +103 -0
  63. package/src/storage/hybrid.ts +1151 -275
  64. package/src/storage/index.ts +15 -2
  65. package/src/storage/queue.ts +128 -0
  66. package/src/storage/synced-registry.ts +49 -4
  67. package/src/storage/types.ts +6 -2
@@ -1,36 +1,111 @@
1
1
  /**
2
2
  * Hybrid Storage Implementation
3
3
  *
4
- * Combines local cache (InMemoryStorage) with remote persistence (ApiStorage).
5
- * Provides fast local reads with remote backup and sync.
4
+ * A local copy (an `InMemoryStorage`, or the `local` storage that the application gives) in
5
+ * front of a remote `CollectionStorage` (often an `ApiStorage`). The storage has four modes, and
6
+ * `setMode()` changes the mode while the storage operates (roadmap R28):
6
7
  *
7
- * Features:
8
- * - Local cache for fast reads, with a remote read on a miss
9
- * - Write-through or write-back strategies
10
- * - Automatic sync on initialization
11
- * - Conflict resolution
12
- * - Offline operation queue
8
+ * | Mode | Reads | Writes | Remote calls |
9
+ * |---|---|---|---|
10
+ * | `local-only` | the local copy | the local copy and the queue | none |
11
+ * | `write-through` | the local copy, the remote on a miss | the local copy, then the remote | yes |
12
+ * | `write-back` | the local copy, the remote on a miss | the local copy and the queue, a timer sends it | yes |
13
+ * | `remote-only` | the remote | the remote | yes, and no local copy |
14
+ *
15
+ * - The queue is durable through the `queueStore` that the application gives.
16
+ * - A sync asks for the changes since a version when the remote has `getChangesSince`.
17
+ * - With `versionOf`, a conflict is a change on both sides, and `conflictResolution` decides.
18
+ * - `pullInterval` keeps the local copy fresh.
13
19
  */
14
20
  import { InMemoryStorage } from "@procedures/collections";
15
- /** The ids that an operation changes ("*" for clear). */
16
- function idsOf(op) {
17
- const data = op.data;
18
- switch (op.type) {
19
- case "set":
20
- case "delete":
21
- return data.id === undefined ? [] : [data.id];
22
- case "setBatch":
23
- return (data.items ?? []).map(([id]) => id);
24
- case "deleteBatch":
25
- return data.ids ?? [];
26
- case "clear":
27
- return ["*"];
21
+ import { optionalMs } from "@procedures/client";
22
+ import { MemoryQueueStore, idsOf, withoutId } from "./queue.js";
23
+ const MODES = new Set(["local-only", "write-through", "write-back", "remote-only"]);
24
+ /**
25
+ * The remote did not take a write of the write-through mode. The write is in the local cache and
26
+ * in the queue, and the storage sends it later (roadmap R2). `cause` is the error of the remote.
27
+ */
28
+ export class PendingWriteError extends Error {
29
+ name = "PendingWriteError";
30
+ code = "WRITE_PENDING";
31
+ pending = true;
32
+ constructor(operation, cause) {
33
+ super(`The remote did not take the ${operation}: it waits in the queue`, { cause });
34
+ }
35
+ }
36
+ /** The queue of operations is full, so the storage refuses the write (roadmap R2). */
37
+ export class QueueFullError extends Error {
38
+ name = "QueueFullError";
39
+ code = "QUEUE_FULL";
40
+ constructor(max) {
41
+ super(`The queue of operations is full (${max} operations): the storage refuses the write`);
42
+ }
43
+ }
44
+ /** The remote gave items without a key, so a sync cannot know which local items to keep. */
45
+ export class MissingKeysError extends Error {
46
+ name = "MissingKeysError";
47
+ code = "MISSING_KEYS";
48
+ constructor(count) {
49
+ super(`${count} remote items have no key. Give the option keyOf, or a remote storage with getEntries(). The sync changed nothing.`);
50
+ }
51
+ }
52
+ /**
53
+ * The queue store failed (roadmap R28). A write that the store did not keep is undone and not
54
+ * acknowledged. A storage whose store cannot load refuses each operation.
55
+ */
56
+ export class QueueStoreError extends Error {
57
+ name = "QueueStoreError";
58
+ code = "QUEUE_STORE";
59
+ constructor(action, cause) {
60
+ super(`The queue store could not ${action === "load" ? "load the queue" : "keep the queue"}`, { cause });
28
61
  }
29
62
  }
63
+ /**
64
+ * The sync found conflicts and `conflictResolution` is "error" (roadmap R28). The local values and
65
+ * their queued writes stay. `ids` are the ids of the conflicts.
66
+ */
67
+ export class ConflictError extends Error {
68
+ ids;
69
+ name = "ConflictError";
70
+ code = "SYNC_CONFLICT";
71
+ constructor(ids) {
72
+ super(`Conflict detected between local and remote versions of ${ids.length} items: ${ids.slice(0, 10).join(", ")}`);
73
+ this.ids = ids;
74
+ }
75
+ }
76
+ /**
77
+ * `setMode()` could not change the mode (roadmap R28). The mode did not change. `cause` is the
78
+ * error of the flush or of the sync.
79
+ */
80
+ export class ModeSwitchError extends Error {
81
+ from;
82
+ to;
83
+ name = "ModeSwitchError";
84
+ code = "MODE_SWITCH_FAILED";
85
+ constructor(from, to, cause) {
86
+ super(`The storage could not change from ${from} to ${to}: the mode is still ${from}`, { cause });
87
+ this.from = from;
88
+ this.to = to;
89
+ }
90
+ }
91
+ const defaultIsUnavailable = (error) => {
92
+ if (typeof error !== "object" || error === null)
93
+ return true;
94
+ const { retryable, code } = error;
95
+ return retryable === true || code === undefined;
96
+ };
97
+ /** True when a remote does not know a procedure: an older server without `changes`. */
98
+ const isNotFound = (error) => typeof error === "object" && error !== null && error.code === "NOT_FOUND";
30
99
  const defaultKeyOf = (item) => {
31
100
  const id = item?.id;
32
101
  return typeof id === "string" && id.length > 0 ? id : undefined;
33
102
  };
103
+ function checkMode(mode) {
104
+ if (typeof mode !== "string" || !MODES.has(mode)) {
105
+ throw new TypeError(`Unknown mode ${JSON.stringify(mode)}: use local-only, write-through, write-back or remote-only`);
106
+ }
107
+ return mode;
108
+ }
34
109
  /**
35
110
  * Hybrid storage combining local cache with remote persistence.
36
111
  *
@@ -42,9 +117,9 @@ const defaultKeyOf = (item) => {
42
117
  *
43
118
  * const remote = new ApiStorage(client, { collection: "users" });
44
119
  * const hybrid = new HybridStorage(remote, {
45
- * writeStrategy: "write-through",
120
+ * mode: "write-through",
46
121
  * conflictResolution: "remote",
47
- * syncInterval: 5000
122
+ * syncInterval: seconds(5)
48
123
  * });
49
124
  *
50
125
  * // Fast local reads
@@ -52,20 +127,58 @@ const defaultKeyOf = (item) => {
52
127
  *
53
128
  * // Writes go to both local and remote
54
129
  * await hybrid.set("123", { name: "John" });
130
+ *
131
+ * // Offline: no remote call. The writes wait in the queue.
132
+ * await hybrid.setMode("local-only");
133
+ * // Online again: the switch sends the queue, then syncs the local copy
134
+ * await hybrid.setMode("write-through");
55
135
  * ```
56
136
  */
57
137
  export class HybridStorage {
58
138
  local;
59
139
  remote;
140
+ queueStore;
60
141
  options;
142
+ mode;
61
143
  offlineOps = [];
144
+ /** The operations in `offlineOps` that the queue store does not have yet */
145
+ unpersisted = new Set();
146
+ /** True when the queue store can differ from `offlineOps`: a failed write, or sent operations */
147
+ storeDirty = false;
148
+ /** The calls of the queue store, one at a time */
149
+ storeChain = Promise.resolve();
150
+ /** The push timer of the write-back mode */
62
151
  syncTimer = null;
152
+ /** The periodic pull */
153
+ pullTimer = null;
154
+ /** True while pushQueue sends the queue: the first operation can be on its way */
155
+ pushing = false;
156
+ /** The timer that sends the queue of the write-through mode after a failure */
157
+ drainTimer = null;
158
+ /** The load of the queue. The writes wait for it. */
159
+ loaded;
160
+ /** The load and the first sync. The reads wait for it, so a read never sees an empty cache by mistake. */
161
+ ready;
162
+ /** The error of the load (of the queue store, or of the local copy): the storage refuses each operation */
163
+ loadError;
164
+ /** A mode switch that runs. The operations wait for it. */
165
+ gate = Promise.resolve();
166
+ /** The operations that run, and the switches that wait for them to end */
167
+ active = 0;
168
+ idleWaiters = [];
63
169
  isOnline = true;
170
+ closed = false;
64
171
  /** A counter of local writes for each id, so a sync does not restore an older remote value */
65
172
  writeVersions = new Map();
66
173
  clearVersion = 0;
67
174
  /** The running sync: syncs run one at a time */
68
175
  syncChain = Promise.resolve();
176
+ /** The remote version of the last delta pull */
177
+ cursor;
178
+ /** True when the remote has `getChangesSince` but its server does not know `changes` */
179
+ changesUnsupported = false;
180
+ /** With `versionOf`: the last known remote version of each id. `null`: the remote has no item. */
181
+ bases = new Map();
69
182
  stats = {
70
183
  cacheHits: 0,
71
184
  cacheMisses: 0,
@@ -73,28 +186,135 @@ export class HybridStorage {
73
186
  conflicts: 0,
74
187
  };
75
188
  constructor(remote, options = {}) {
76
- this.local = new InMemoryStorage();
189
+ this.local = options.local ?? new InMemoryStorage();
77
190
  this.remote = remote;
191
+ this.queueStore = options.queueStore ?? new MemoryQueueStore();
192
+ this.mode = checkMode(options.mode ?? options.writeStrategy ?? "write-through");
78
193
  this.options = {
79
- writeStrategy: options.writeStrategy ?? "write-through",
80
194
  conflictResolution: options.conflictResolution ?? "remote",
81
195
  ...(options.mergeFn !== undefined && { mergeFn: options.mergeFn }),
82
196
  keyOf: options.keyOf ?? defaultKeyOf,
83
- syncInterval: options.syncInterval ?? 5000,
197
+ ...(options.versionOf !== undefined && { versionOf: options.versionOf }),
198
+ syncInterval: optionalMs(options.syncInterval, "syncInterval") ?? 5000,
199
+ pullInterval: optionalMs(options.pullInterval, "pullInterval") ?? 0,
84
200
  syncOnInit: options.syncOnInit ?? true,
85
201
  maxOfflineOps: options.maxOfflineOps ?? 1000,
86
202
  offlineQueue: options.offlineQueue ?? true,
203
+ isUnavailable: options.isUnavailable ?? defaultIsUnavailable,
87
204
  };
88
- // Initialize with remote data
89
- if (this.options.syncOnInit) {
90
- this.syncFromRemote().catch((err) => {
91
- console.error("Failed to sync on init:", err);
92
- });
205
+ // The queue first, then the first sync. The reads wait for both (roadmap R2, R28).
206
+ this.loaded = this.loadQueue();
207
+ this.ready = this.loaded.then(() => this.initialSync());
208
+ this.updateTimers();
209
+ }
210
+ //
211
+ // ═══ Modes ═══
212
+ //
213
+ /** The mode of the storage now. */
214
+ getMode() {
215
+ return this.mode;
216
+ }
217
+ /**
218
+ * This method changes the mode while the storage operates (roadmap R28).
219
+ *
220
+ * - The switch waits for the operations that run. The operations that start later wait for
221
+ * the switch. Switches and syncs run one at a time.
222
+ * - A switch to a remote mode sends the queue first. A switch to `local-only` makes no remote
223
+ * call: the queue stays until the next remote mode.
224
+ * - A switch to `remote-only` drops the local copy, after the queue reached the remote.
225
+ * - A switch from `local-only` or `remote-only` to a mode with a local copy syncs the local
226
+ * copy. The reads wait for the sync.
227
+ * - When the flush or the sync fails, the mode does not change, and the switch throws
228
+ * `ModeSwitchError`.
229
+ */
230
+ async setMode(mode) {
231
+ const target = checkMode(mode);
232
+ const previous = this.gate;
233
+ let release;
234
+ this.gate = new Promise((resolve) => (release = resolve));
235
+ try {
236
+ await previous;
237
+ await this.loaded;
238
+ if (this.loadError)
239
+ throw this.loadError;
240
+ await this.idle();
241
+ await this.serialize(() => this.switchTo(target));
242
+ }
243
+ finally {
244
+ release();
245
+ }
246
+ }
247
+ async switchTo(target) {
248
+ const from = this.mode;
249
+ if (target === from)
250
+ return;
251
+ try {
252
+ if (target !== "local-only") {
253
+ // The queue reaches the remote before the mode changes
254
+ await this.pushQueue();
255
+ await this.ensureStoreClean();
256
+ }
257
+ if (target === "remote-only") {
258
+ await this.dropLocal();
259
+ }
260
+ else if (from === "remote-only" || from === "local-only") {
261
+ if (from === "remote-only")
262
+ await this.rebuildLocal();
263
+ await this.pullRemote(from === "remote-only");
264
+ }
93
265
  }
94
- // Start periodic sync for write-back mode
95
- if (this.options.writeStrategy === "write-back" &&
96
- this.options.syncInterval > 0) {
97
- this.startPeriodicSync();
266
+ catch (error) {
267
+ throw new ModeSwitchError(from, target, error);
268
+ }
269
+ this.mode = target;
270
+ this.updateTimers();
271
+ }
272
+ /** This method drops the local copy. A read in flight does not fill it again. */
273
+ async dropLocal() {
274
+ this.clearVersion++;
275
+ this.cursor = undefined;
276
+ await this.local.clear();
277
+ }
278
+ /** This method makes the local copy from the queue only, before a full sync. */
279
+ async rebuildLocal() {
280
+ await this.dropLocal();
281
+ for (const op of this.offlineOps)
282
+ await this.applyToLocal(op);
283
+ }
284
+ /** This method starts and stops the timers for the mode. */
285
+ updateTimers() {
286
+ if (this.closed)
287
+ return;
288
+ const push = this.mode === "write-back" && this.options.syncInterval > 0;
289
+ if (push && !this.syncTimer) {
290
+ this.syncTimer = setInterval(() => {
291
+ this.syncToRemote().catch((err) => {
292
+ console.error("Periodic sync failed:", err);
293
+ });
294
+ }, this.options.syncInterval);
295
+ // The timer must not keep the process alive (deep dive DATA-9)
296
+ this.syncTimer.unref?.();
297
+ }
298
+ else if (!push && this.syncTimer) {
299
+ clearInterval(this.syncTimer);
300
+ this.syncTimer = null;
301
+ }
302
+ const pull = this.options.pullInterval > 0 && (this.mode === "write-through" || this.mode === "write-back");
303
+ if (pull && !this.pullTimer) {
304
+ this.pullTimer = setInterval(() => {
305
+ this.syncFromRemote().catch((err) => {
306
+ console.error("Periodic pull failed:", err);
307
+ });
308
+ }, this.options.pullInterval);
309
+ this.pullTimer.unref?.();
310
+ }
311
+ else if (!pull && this.pullTimer) {
312
+ clearInterval(this.pullTimer);
313
+ this.pullTimer = null;
314
+ }
315
+ if (this.mode !== "write-through" && this.drainTimer) {
316
+ clearTimeout(this.drainTimer);
317
+ this.drainTimer = null;
98
318
  }
99
319
  }
100
320
  //
@@ -102,188 +322,272 @@ export class HybridStorage {
102
322
  //
103
323
  /**
104
324
  * The local item, or else the remote item (which the cache then keeps). When the remote
105
- * cannot answer, the storage is offline and the result is undefined.
325
+ * cannot answer, the storage is offline and the result is undefined. In `local-only` mode, a
326
+ * miss is undefined. In `remote-only` mode, the remote answers, and its error passes through.
106
327
  */
107
328
  async get(id) {
108
- // Try local cache first
109
- const cached = await this.local.get(id);
110
- if (cached !== undefined) {
111
- this.stats.cacheHits++;
112
- return cached;
113
- }
114
- this.stats.cacheMisses++;
115
- // A pending local delete means "no item": do not read it back from the remote
116
- if (this.hasPendingOp(id)) {
117
- return undefined;
118
- }
119
- const version = this.versionOf(id);
120
- try {
121
- const remote = await this.remote.get(id);
122
- this.isOnline = true;
123
- if (remote !== undefined && this.versionOf(id) === version) {
124
- await this.local.set(id, remote);
329
+ return this.run(true, async () => {
330
+ if (this.mode === "remote-only")
331
+ return this.remoteRead(() => this.remote.get(id));
332
+ // Try local cache first
333
+ const cached = await this.local.get(id);
334
+ if (cached !== undefined) {
335
+ this.stats.cacheHits++;
336
+ return cached;
125
337
  }
126
- return remote;
127
- }
128
- catch {
129
- this.isOnline = false;
130
- return undefined;
131
- }
338
+ this.stats.cacheMisses++;
339
+ // A pending local delete means "no item": do not read it back from the remote
340
+ if (this.mode === "local-only" || this.hasPendingOp(id)) {
341
+ return undefined;
342
+ }
343
+ const version = this.writeKey(id);
344
+ try {
345
+ const remote = await this.remote.get(id);
346
+ this.isOnline = true;
347
+ if (this.writeKey(id) === version && !this.hasPendingOp(id)) {
348
+ if (remote !== undefined)
349
+ await this.local.set(id, remote);
350
+ this.noteBase(id, remote);
351
+ }
352
+ return remote;
353
+ }
354
+ catch {
355
+ this.isOnline = false;
356
+ return undefined;
357
+ }
358
+ });
132
359
  }
133
360
  async getAll() {
134
- // Return local cache
135
- return this.local.getAll();
361
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.getAll()) : this.local.getAll()));
362
+ }
363
+ /** Every item with its id. */
364
+ async getEntries() {
365
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remoteEntries()) : this.local.getEntries()));
136
366
  }
137
367
  async find(predicate) {
138
- // Search local cache
139
- return this.local.find(predicate);
368
+ return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.find(predicate)) : this.local.find(predicate));
140
369
  }
141
370
  async has(id) {
142
- return this.local.has(id);
371
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.has(id)) : this.local.has(id)));
143
372
  }
144
373
  async size() {
145
- return this.local.size();
374
+ return this.run(true, () => (this.mode === "remote-only" ? this.remoteRead(() => this.remote.size()) : this.local.size()));
375
+ }
376
+ async getBatch(ids) {
377
+ return this.run(true, () => this.mode === "remote-only" ? this.remoteRead(() => this.remote.getBatch(ids)) : this.local.getBatch(ids));
378
+ }
379
+ /** A read of the remote-only mode. A queue that a restart loaded reaches the remote first. */
380
+ async remoteRead(read) {
381
+ if (this.offlineOps.length > 0)
382
+ await this.serialize(() => this.pushQueue());
383
+ try {
384
+ const result = await read();
385
+ this.isOnline = true;
386
+ return result;
387
+ }
388
+ catch (error) {
389
+ if (!(error instanceof MissingKeysError))
390
+ this.isOnline = false;
391
+ throw error;
392
+ }
146
393
  }
147
394
  //
148
395
  // ═══ Write Operations ═══
149
396
  //
150
397
  async set(id, value) {
151
- this.bump(id);
152
- // Always write to local immediately
153
- await this.local.set(id, value);
154
- if (this.options.writeStrategy === "write-through") {
155
- // Write-through: sync to remote immediately
156
- try {
157
- await this.remote.set(id, value);
158
- }
159
- catch (error) {
160
- this.handleRemoteFailure("set", { id, value });
161
- throw error;
162
- }
163
- }
164
- else {
165
- // Write-back: queue for later sync
166
- this.queueOperation({
167
- type: "set",
168
- data: { id, value },
169
- timestamp: Date.now(),
170
- });
171
- }
398
+ await this.write("set", { id, value }, [id], async () => {
399
+ this.bump(id);
400
+ await this.local.set(id, value);
401
+ }, () => this.remote.set(id, value));
172
402
  }
173
403
  async delete(id) {
174
- this.bump(id);
175
- const existed = await this.local.delete(id);
176
- if (this.options.writeStrategy === "write-through") {
177
- try {
178
- await this.remote.delete(id);
179
- }
180
- catch (error) {
181
- this.handleRemoteFailure("delete", { id });
182
- throw error;
183
- }
184
- }
185
- else {
186
- this.queueOperation({
187
- type: "delete",
188
- data: { id },
189
- timestamp: Date.now(),
190
- });
191
- }
192
- return existed;
404
+ return this.write("delete", { id }, [id], async () => {
405
+ this.bump(id);
406
+ return this.local.delete(id);
407
+ }, () => this.remote.delete(id));
193
408
  }
194
409
  async clear() {
195
- this.clearVersion++;
196
- await this.local.clear();
197
- if (this.options.writeStrategy === "write-through") {
198
- try {
199
- await this.remote.clear();
200
- }
201
- catch (error) {
202
- this.handleRemoteFailure("clear", {});
203
- throw error;
204
- }
205
- }
206
- else {
207
- this.queueOperation({
208
- type: "clear",
209
- data: {},
210
- timestamp: Date.now(),
211
- });
212
- }
410
+ await this.write("clear", {}, "*", async () => {
411
+ this.clearVersion++;
412
+ await this.local.clear();
413
+ }, () => this.remote.clear());
213
414
  }
214
415
  //
215
416
  // ═══ Bulk Operations ═══
216
417
  //
217
418
  async setBatch(items) {
218
- for (const [id] of items)
219
- this.bump(id);
220
- await this.local.setBatch(items);
221
- if (this.options.writeStrategy === "write-through") {
222
- try {
223
- await this.remote.setBatch(items);
224
- }
225
- catch (error) {
226
- this.handleRemoteFailure("setBatch", { items });
227
- throw error;
228
- }
229
- }
230
- else {
231
- this.queueOperation({
232
- type: "setBatch",
233
- data: { items },
234
- timestamp: Date.now(),
235
- });
236
- }
419
+ await this.write("setBatch", { items }, items.map(([id]) => id), async () => {
420
+ for (const [id] of items)
421
+ this.bump(id);
422
+ await this.local.setBatch(items);
423
+ }, () => this.remote.setBatch(items));
237
424
  }
238
425
  async deleteBatch(ids) {
239
- for (const id of ids)
240
- this.bump(id);
241
- const deleted = await this.local.deleteBatch(ids);
242
- if (this.options.writeStrategy === "write-through") {
426
+ return this.write("deleteBatch", { ids }, ids, async () => {
427
+ for (const id of ids)
428
+ this.bump(id);
429
+ return this.local.deleteBatch(ids);
430
+ }, () => this.remote.deleteBatch(ids));
431
+ }
432
+ /**
433
+ * One write, in the mode of the storage. The result is the result of the local copy, or of the
434
+ * remote in `remote-only` mode.
435
+ *
436
+ * - `remote-only`: the remote only. Its error passes through.
437
+ * - `write-back` and `local-only`: the local copy and the queue. A full queue refuses the write
438
+ * before the local change. The queue store keeps the operation before the write ends.
439
+ * - `write-through`, the remote refuses the write: the local change is undone, and the error
440
+ * of the remote passes through.
441
+ * - `write-through`, the remote is not available: with the queue, the write waits in it, a timer
442
+ * sends it later, and the write throws `PendingWriteError`. Without the queue, or with a full
443
+ * queue, the local change is undone.
444
+ *
445
+ * Before, a failed write-through stayed in the local cache and in the queue, no timer sent the
446
+ * queue, a second try queued the operation again, and a full queue dropped the oldest operation
447
+ * without a word (roadmap R2).
448
+ */
449
+ write(type, data, ids, applyLocal, applyRemote) {
450
+ return this.run(false, async () => {
451
+ const op = { type, data, timestamp: Date.now() };
452
+ if (this.mode === "remote-only") {
453
+ const send = async () => {
454
+ await this.ensureStoreClean();
455
+ const result = await applyRemote();
456
+ this.noteRemoteWrite(op);
457
+ return result;
458
+ };
459
+ try {
460
+ const result = this.offlineOps.length > 0
461
+ ? await this.serialize(async () => {
462
+ await this.pushQueue();
463
+ return send();
464
+ })
465
+ : await send();
466
+ this.isOnline = true;
467
+ return result;
468
+ }
469
+ catch (error) {
470
+ if (!(error instanceof QueueStoreError) && this.options.isUnavailable(error))
471
+ this.isOnline = false;
472
+ throw error;
473
+ }
474
+ }
475
+ if (this.mode === "write-back" || this.mode === "local-only") {
476
+ this.ensureQueueRoom(op);
477
+ this.attachBase(op);
478
+ const undo = await this.snapshot(ids);
479
+ const result = await applyLocal();
480
+ await this.enqueue(op, undo);
481
+ return result;
482
+ }
483
+ const undo = await this.snapshot(ids);
484
+ const result = await applyLocal();
243
485
  try {
244
- await this.remote.deleteBatch(ids);
486
+ if (this.offlineOps.length > 0) {
487
+ // The queued operations are older: they reach the remote first, or the older value of a
488
+ // queued write replaces this one later. A failed send keeps this write behind them.
489
+ await this.serialize(async () => {
490
+ await this.pushQueue();
491
+ await this.ensureStoreClean();
492
+ await applyRemote();
493
+ });
494
+ }
495
+ else {
496
+ await this.ensureStoreClean();
497
+ await applyRemote();
498
+ }
499
+ this.isOnline = true;
500
+ this.noteRemoteWrite(op);
245
501
  }
246
502
  catch (error) {
247
- this.handleRemoteFailure("deleteBatch", { ids });
503
+ if (!(error instanceof QueueStoreError) &&
504
+ this.options.offlineQueue &&
505
+ this.options.isUnavailable(error) &&
506
+ this.hasQueueRoom(op)) {
507
+ this.isOnline = false;
508
+ console.error(`Remote ${type} failed: the operation waits in the queue`);
509
+ this.attachBase(op);
510
+ await this.enqueue(op, undo);
511
+ this.scheduleDrain();
512
+ throw new PendingWriteError(type, error);
513
+ }
514
+ await undo();
248
515
  throw error;
249
516
  }
250
- }
251
- else {
252
- this.queueOperation({
253
- type: "deleteBatch",
254
- data: { ids },
255
- timestamp: Date.now(),
256
- });
257
- }
258
- return deleted;
517
+ return result;
518
+ });
259
519
  }
260
- getBatch(ids) {
261
- return this.local.getBatch(ids);
520
+ /**
521
+ * The local values of the ids before a write, and a function that puts them back. It puts back
522
+ * only the ids that no later write changed.
523
+ */
524
+ async snapshot(ids) {
525
+ const before = ids === "*" ? new Map(await this.local.getEntries()) : await this.local.getBatch(ids);
526
+ const touched = ids === "*" ? [...before.keys()] : ids;
527
+ // The versions after the write: this write adds one to each id (clear adds to clearVersion)
528
+ const versions = new Map(touched.map((id) => [id, (this.writeVersions.get(id) ?? 0) + (ids === "*" ? 0 : 1)]));
529
+ const clearVersion = this.clearVersion + (ids === "*" ? 1 : 0);
530
+ return async () => {
531
+ if (this.clearVersion !== clearVersion)
532
+ return;
533
+ for (const id of touched) {
534
+ if ((this.writeVersions.get(id) ?? 0) !== versions.get(id))
535
+ continue;
536
+ const value = before.get(id);
537
+ if (value === undefined)
538
+ await this.local.delete(id);
539
+ else
540
+ await this.local.set(id, value);
541
+ }
542
+ };
262
543
  }
263
544
  //
264
545
  // ═══ Sync Operations ═══
265
546
  //
266
547
  /**
267
- * Sync all data from remote to local.
268
- * Handles conflicts according to conflict resolution strategy.
548
+ * Sync from the remote to the local copy, in the modes `write-through` and `write-back`. The
549
+ * other modes have no local copy to sync (`remote-only`) or make no remote call (`local-only`):
550
+ * the call does nothing.
269
551
  *
270
- * - Each remote item goes into the cache under `keyOf(item)`.
552
+ * - The sync asks for the changes since the version of the last sync when the remote has
553
+ * `getChangesSince`. Else it reads all entries.
554
+ * - Each remote item goes into the cache under its id.
271
555
  * - A local item that the remote does not have is removed, unless a queued local operation
272
- * changes it (the remote deleted it).
556
+ * changes it.
273
557
  * - An id that a local write changed during the sync keeps the local value: the remote value
274
558
  * is older.
275
559
  * - Syncs run one at a time.
276
560
  */
277
561
  syncFromRemote() {
278
- return this.serialize(() => this.pullRemote());
562
+ return this.serialize(async () => {
563
+ await this.loaded;
564
+ if (this.loadError)
565
+ throw this.loadError;
566
+ if (this.mode === "write-through" || this.mode === "write-back")
567
+ await this.pullRemote(false);
568
+ });
279
569
  }
280
570
  /**
281
571
  * Sync all pending offline operations to remote.
282
572
  * Processes queued operations in order. When one fails, it and the later operations stay in
283
- * the queue: the operations that reached the remote are not sent again.
573
+ * the queue: the operations that reached the remote are not sent again. In `local-only` mode,
574
+ * the call does nothing.
284
575
  */
285
576
  syncToRemote() {
286
- return this.serialize(() => this.pushQueue());
577
+ return this.serialize(async () => {
578
+ await this.loaded;
579
+ if (this.loadError)
580
+ throw this.loadError;
581
+ if (this.mode !== "local-only")
582
+ await this.pushQueue();
583
+ });
584
+ }
585
+ /**
586
+ * This method sends the queue to the remote (roadmap R28). `ProcedureServer.useStorage` uses it. In
587
+ * `local-only` mode, it sends nothing: the queue waits for a remote mode.
588
+ */
589
+ flush() {
590
+ return this.syncToRemote();
287
591
  }
288
592
  /**
289
593
  * Force a full sync (both directions).
@@ -295,14 +599,26 @@ export class HybridStorage {
295
599
  //
296
600
  // ═══ Lifecycle & Metadata ═══
297
601
  //
602
+ /**
603
+ * This method stops the timers, sends the queue (not in `local-only` mode), and closes the remote and the
604
+ * local copy. The queue store keeps the operations that the remote did not take. A durable local
605
+ * copy keeps its items.
606
+ */
298
607
  async close() {
299
- // Stop periodic sync
300
- if (this.syncTimer) {
301
- clearInterval(this.syncTimer);
302
- this.syncTimer = null;
608
+ this.closed = true;
609
+ for (const timer of [this.syncTimer, this.pullTimer]) {
610
+ if (timer)
611
+ clearInterval(timer);
612
+ }
613
+ this.syncTimer = null;
614
+ this.pullTimer = null;
615
+ if (this.drainTimer) {
616
+ clearTimeout(this.drainTimer);
617
+ this.drainTimer = null;
303
618
  }
619
+ await this.loaded;
304
620
  // Sync pending operations before closing
305
- if (this.offlineOps.length > 0) {
621
+ if (this.offlineOps.length > 0 && this.mode !== "local-only") {
306
622
  try {
307
623
  await this.syncToRemote();
308
624
  }
@@ -310,17 +626,31 @@ export class HybridStorage {
310
626
  console.error("Failed to sync before close:", error);
311
627
  }
312
628
  }
629
+ await this.storeChain;
313
630
  // Close remote storage (an ApiStorage leaves the caller's client open)
314
- await this.remote.close();
631
+ try {
632
+ await this.remote.close();
633
+ }
634
+ finally {
635
+ await this.local.close();
636
+ }
315
637
  }
638
+ /**
639
+ * The metadata of the storage. `size` is the result of `size()`: the number of items of the
640
+ * storage, also in `remote-only` mode. `stats.localSize` is the number of items of the local
641
+ * copy. Before, `size` was the size of the local copy, which `remote-only` keeps empty (found by
642
+ * the contract suite of `@procedures/collections/testing`).
643
+ */
316
644
  async getMetadata() {
645
+ const currentSize = await this.size();
646
+ const localSize = await this.local.size();
317
647
  const totalOps = this.stats.cacheHits + this.stats.cacheMisses;
318
648
  const hitRate = totalOps > 0 ? (this.stats.cacheHits / totalOps) * 100 : 0;
319
- const currentSize = await this.local.size();
320
649
  return {
321
650
  type: "hybrid",
322
651
  size: currentSize,
323
652
  stats: {
653
+ localSize,
324
654
  hitRate: Math.round(hitRate * 100) / 100,
325
655
  cacheHits: this.stats.cacheHits,
326
656
  cacheMisses: this.stats.cacheMisses,
@@ -328,7 +658,8 @@ export class HybridStorage {
328
658
  conflicts: this.stats.conflicts,
329
659
  pendingOps: this.offlineOps.length,
330
660
  isOnline: this.isOnline,
331
- writeStrategy: this.options.writeStrategy,
661
+ mode: this.mode,
662
+ writeStrategy: this.mode,
332
663
  conflictResolution: this.options.conflictResolution,
333
664
  },
334
665
  };
@@ -336,68 +667,378 @@ export class HybridStorage {
336
667
  //
337
668
  // ═══ Internal Helpers ═══
338
669
  //
339
- async pullRemote() {
670
+ /**
671
+ * This method starts a public operation: after the load of the queue (a write) or the first sync (a read),
672
+ * and never during a mode switch.
673
+ */
674
+ async run(read, work) {
675
+ await (read ? this.ready : this.loaded);
676
+ for (;;) {
677
+ const gate = this.gate;
678
+ await gate;
679
+ // No new switch started while this operation waited
680
+ if (this.gate === gate)
681
+ break;
682
+ }
683
+ if (this.loadError)
684
+ throw this.loadError;
685
+ this.active++;
686
+ try {
687
+ return await work();
688
+ }
689
+ finally {
690
+ if (--this.active === 0) {
691
+ for (const resolve of this.idleWaiters.splice(0))
692
+ resolve();
693
+ }
694
+ }
695
+ }
696
+ /** A promise that resolves when no operation runs. */
697
+ idle() {
698
+ if (this.active === 0)
699
+ return Promise.resolve();
700
+ return new Promise((resolve) => this.idleWaiters.push(resolve));
701
+ }
702
+ /** This method loads the queue, and puts its writes in the local copy. */
703
+ async loadQueue() {
704
+ let ops;
705
+ try {
706
+ ops = await this.queueStore.load();
707
+ }
708
+ catch (error) {
709
+ this.loadError = new QueueStoreError("load", error);
710
+ console.error("The queue store could not load the queue: the storage refuses each operation", error);
711
+ return;
712
+ }
713
+ this.offlineOps = ops.slice();
714
+ try {
715
+ for (const op of this.offlineOps) {
716
+ for (const [id, version] of op.base ?? []) {
717
+ if (!this.bases.has(id))
718
+ this.bases.set(id, version);
719
+ }
720
+ // A durable local copy can hold the operation already: each operation gives the same
721
+ // result two times
722
+ await this.applyToLocal(op);
723
+ }
724
+ }
725
+ catch (error) {
726
+ this.loadError = new Error("The local copy could not take the loaded queue: the storage refuses each operation", { cause: error });
727
+ console.error(this.loadError.message, error);
728
+ }
729
+ }
730
+ /** The first sync: send the loaded queue, then read the remote. */
731
+ async initialSync() {
732
+ if (this.loadError)
733
+ return;
734
+ if (this.options.syncOnInit && this.mode !== "local-only") {
735
+ if (this.offlineOps.length > 0) {
736
+ await this.syncToRemote().catch((err) => {
737
+ console.error("Failed to send the queue on init:", err);
738
+ });
739
+ }
740
+ if (this.mode === "write-through" || this.mode === "write-back") {
741
+ await this.syncFromRemote().catch((err) => {
742
+ console.error("Failed to sync on init:", err);
743
+ });
744
+ }
745
+ }
746
+ if (this.offlineOps.length > 0)
747
+ this.scheduleDrain();
748
+ }
749
+ /** This method applies a queued operation to the local copy. */
750
+ async applyToLocal(op) {
751
+ const data = op.data;
752
+ switch (op.type) {
753
+ case "set":
754
+ await this.local.set(data.id, data.value);
755
+ break;
756
+ case "delete":
757
+ await this.local.delete(data.id);
758
+ break;
759
+ case "setBatch":
760
+ await this.local.setBatch(data.items);
761
+ break;
762
+ case "deleteBatch":
763
+ await this.local.deleteBatch(data.ids);
764
+ break;
765
+ case "clear":
766
+ await this.local.clear();
767
+ break;
768
+ }
769
+ }
770
+ /**
771
+ * One pull. `full`: ask for all items (after the local copy was dropped).
772
+ *
773
+ * - The versions of the local writes at the start: a write during the pull keeps its value.
774
+ * - The cursor moves only after a pull without conflicts, so a conflict of the "error" policy
775
+ * comes again at the next pull.
776
+ */
777
+ async pullRemote(full) {
340
778
  const versions = new Map(this.writeVersions);
341
779
  const clearVersion = this.clearVersion;
342
780
  // True when a local write changed the id after the sync started
343
781
  const changedLocally = (id) => this.clearVersion !== clearVersion || (this.writeVersions.get(id) ?? 0) !== (versions.get(id) ?? 0);
344
- let remoteData;
782
+ let changes;
345
783
  try {
346
- remoteData = await this.remote.getAll();
784
+ changes = await this.fetchChanges(full);
347
785
  }
348
786
  catch (error) {
349
- this.isOnline = false;
787
+ if (!(error instanceof MissingKeysError))
788
+ this.isOnline = false;
350
789
  throw error;
351
790
  }
791
+ const conflicts = [];
792
+ let queueChanged = false;
352
793
  const remoteIds = new Set();
353
- for (const item of remoteData) {
354
- const id = this.options.keyOf(item);
355
- if (!id) {
356
- console.warn("Item has no key, skipping:", item);
357
- continue;
358
- }
794
+ for (const [id, item] of changes.changed) {
359
795
  remoteIds.add(id);
360
- if (changedLocally(id) || this.hasPendingOp(id)) {
361
- continue;
796
+ if (await this.pullItem(id, item, changedLocally, conflicts))
797
+ queueChanged = true;
798
+ }
799
+ // The remote deleted these items
800
+ const gone = changes.full
801
+ ? (await this.local.getEntries()).map(([id]) => id).filter((id) => !remoteIds.has(id))
802
+ : changes.deleted;
803
+ for (const id of gone) {
804
+ if (await this.pullDelete(id, changedLocally, conflicts))
805
+ queueChanged = true;
806
+ }
807
+ if (queueChanged) {
808
+ this.storeDirty = true;
809
+ await this.persistAll().catch((error) => {
810
+ console.error("The queue store could not keep the queue after the sync:", error);
811
+ });
812
+ }
813
+ this.stats.syncCount++;
814
+ this.isOnline = true;
815
+ if (conflicts.length > 0)
816
+ throw new ConflictError(conflicts);
817
+ if (changes.version !== undefined)
818
+ this.cursor = changes.version;
819
+ }
820
+ /**
821
+ * The remote changes: since the cursor from `getChangesSince` when the remote has it, or else
822
+ * all entries.
823
+ */
824
+ async fetchChanges(full) {
825
+ if (this.remote.getChangesSince && !this.changesUnsupported) {
826
+ try {
827
+ return await this.remote.getChangesSince(full ? undefined : this.cursor);
362
828
  }
363
- const localItem = await this.local.get(id);
364
- if (localItem === undefined) {
365
- // No local version - just store remote
366
- await this.local.set(id, item);
829
+ catch (error) {
830
+ if (!isNotFound(error))
831
+ throw error;
832
+ // A server without the procedure `changes`: read all entries from now on
833
+ this.changesUnsupported = true;
367
834
  }
368
- else if (JSON.stringify(localItem) !== JSON.stringify(item)) {
369
- // Conflict - resolve according to strategy
370
- const resolved = this.resolveConflict(localItem, item);
371
- await this.local.set(id, resolved);
835
+ }
836
+ return { full: true, changed: await this.remoteEntries(), deleted: [] };
837
+ }
838
+ /** This method applies one remote item. It gives true when the queue changed. */
839
+ async pullItem(id, item, changedLocally, conflicts) {
840
+ if (changedLocally(id))
841
+ return false;
842
+ const localItem = await this.local.get(id);
843
+ if (changedLocally(id))
844
+ return false;
845
+ const remoteVersion = this.versionOfItem(item);
846
+ if (this.hasPendingOp(id)) {
847
+ // Without versions, the queued write wins, as before
848
+ if (remoteVersion === undefined)
849
+ return false;
850
+ // The remote did not change since the last sync, or the storage never saw a remote version
851
+ // of the id: the queued write wins
852
+ const base = this.bases.get(id);
853
+ if (base === undefined || base === remoteVersion)
854
+ return false;
855
+ if (localItem !== undefined && this.versionOfItem(localItem) === remoteVersion) {
856
+ this.bases.set(id, remoteVersion);
857
+ return false;
858
+ }
859
+ return this.resolvePending(id, localItem, item, remoteVersion, conflicts);
860
+ }
861
+ const localVersion = localItem === undefined ? undefined : this.versionOfItem(localItem);
862
+ if (localItem === undefined) {
863
+ await this.local.set(id, item);
864
+ }
865
+ else if (remoteVersion !== undefined && localVersion !== undefined) {
866
+ // Only the remote changed: it is not a conflict
867
+ if (localVersion !== remoteVersion)
868
+ await this.local.set(id, item);
869
+ }
870
+ else if (JSON.stringify(localItem) !== JSON.stringify(item)) {
871
+ // Conflict - resolve according to strategy
872
+ if (this.options.conflictResolution === "error") {
372
873
  this.stats.conflicts++;
874
+ conflicts.push(id);
875
+ return false;
373
876
  }
877
+ const resolved = this.resolveConflict(localItem, item);
878
+ await this.local.set(id, resolved);
879
+ this.stats.conflicts++;
374
880
  }
375
- // The remote deleted these items
376
- for (const id of await this.localIds()) {
377
- if (!remoteIds.has(id) && !changedLocally(id) && !this.hasPendingOp(id)) {
881
+ if (remoteVersion !== undefined)
882
+ this.bases.set(id, remoteVersion);
883
+ return false;
884
+ }
885
+ /** This method applies one remote delete. It gives true when the queue changed. */
886
+ async pullDelete(id, changedLocally, conflicts) {
887
+ if (changedLocally(id))
888
+ return false;
889
+ if (!this.hasPendingOp(id)) {
890
+ await this.local.delete(id);
891
+ if (this.options.versionOf)
892
+ this.bases.set(id, null);
893
+ return false;
894
+ }
895
+ // Without versions, the queued write wins, as before. With versions, the remote deleted the
896
+ // item only if the last sync saw a version of it.
897
+ const base = this.bases.get(id);
898
+ if (!this.options.versionOf || base === undefined || base === null)
899
+ return false;
900
+ const localItem = await this.local.get(id);
901
+ if (changedLocally(id))
902
+ return false;
903
+ if (localItem === undefined) {
904
+ // Both sides deleted the item
905
+ this.bases.set(id, null);
906
+ return false;
907
+ }
908
+ this.stats.conflicts++;
909
+ switch (this.options.conflictResolution) {
910
+ case "remote":
911
+ if (!this.dropQueued(id))
912
+ break;
378
913
  await this.local.delete(id);
914
+ this.bases.set(id, null);
915
+ return true;
916
+ case "error":
917
+ conflicts.push(id);
918
+ return false;
919
+ default:
920
+ // "local", and "merge" with one deleted side: the queued write wins
921
+ break;
922
+ }
923
+ this.bases.set(id, null);
924
+ return false;
925
+ }
926
+ /**
927
+ * A queued local write of an id whose remote version changed: a conflict, and the policy
928
+ * decides. True when the queue changed.
929
+ */
930
+ async resolvePending(id, localItem, remoteItem, remoteVersion, conflicts) {
931
+ this.stats.conflicts++;
932
+ switch (this.options.conflictResolution) {
933
+ case "remote":
934
+ // The remote value replaces the queued write. A queued clear cannot lose one id.
935
+ if (!this.dropQueued(id))
936
+ break;
937
+ await this.local.set(id, remoteItem);
938
+ this.bases.set(id, remoteVersion);
939
+ return true;
940
+ case "merge": {
941
+ if (localItem === undefined)
942
+ break;
943
+ const merged = this.resolveConflict(localItem, remoteItem);
944
+ if (!this.dropQueued(id))
945
+ break;
946
+ await this.local.set(id, merged);
947
+ this.bases.set(id, remoteVersion);
948
+ this.offlineOps.push({ type: "set", data: { id, value: merged }, timestamp: Date.now(), base: [[id, remoteVersion]] });
949
+ return true;
379
950
  }
951
+ case "error":
952
+ conflicts.push(id);
953
+ return false;
954
+ default:
955
+ break;
380
956
  }
381
- this.stats.syncCount++;
382
- this.isOnline = true;
957
+ // "local": the queued write wins, and it replaces this remote version
958
+ this.bases.set(id, remoteVersion);
959
+ return false;
960
+ }
961
+ /**
962
+ * This method removes an id from the queued operations. It gives false when a queued clear touches it: nothing
963
+ * changes then.
964
+ */
965
+ dropQueued(id) {
966
+ if (this.offlineOps.some((op) => op.type === "clear"))
967
+ return false;
968
+ const next = [];
969
+ for (const op of this.offlineOps) {
970
+ const kept = withoutId(op, id);
971
+ if (kept)
972
+ next.push(kept);
973
+ }
974
+ this.offlineOps = next;
975
+ return true;
976
+ }
977
+ /**
978
+ * The remote items with their ids: from `getEntries()` when the remote has it, or else from
979
+ * `getAll()` and `keyOf`. An item without a key stops the sync before any change. Before, the
980
+ * sync skipped such items and then deleted each local item that the remote list did not have,
981
+ * so the remote list without ids emptied the cache (roadmap R2).
982
+ */
983
+ async remoteEntries() {
984
+ if (this.remote.getEntries)
985
+ return this.remote.getEntries();
986
+ const items = await this.remote.getAll();
987
+ const entries = [];
988
+ let missing = 0;
989
+ for (const item of items) {
990
+ const id = this.options.keyOf(item);
991
+ if (id)
992
+ entries.push([id, item]);
993
+ else
994
+ missing++;
995
+ }
996
+ if (missing > 0)
997
+ throw new MissingKeysError(missing);
998
+ return entries;
383
999
  }
1000
+ /**
1001
+ * This method sends the queue. The operations that reached the remote leave the queue. The queue store
1002
+ * gets the change at the end.
1003
+ */
384
1004
  async pushQueue() {
385
- while (this.offlineOps.length > 0) {
386
- const op = this.offlineOps[0];
387
- try {
388
- await this.executeOperation(op);
389
- }
390
- catch (error) {
391
- // This operation and the later ones stay queued
392
- this.isOnline = false;
393
- throw error;
1005
+ this.pushing = true;
1006
+ let sent = 0;
1007
+ try {
1008
+ while (this.offlineOps.length > 0) {
1009
+ const op = this.offlineOps[0];
1010
+ if (this.unpersisted.has(op)) {
1011
+ // The write of this operation waits for the queue store
1012
+ await this.storeChain;
1013
+ continue;
1014
+ }
1015
+ try {
1016
+ await this.executeOperation(op);
1017
+ }
1018
+ catch (error) {
1019
+ // This operation and the later ones stay queued
1020
+ this.isOnline = false;
1021
+ throw error;
1022
+ }
1023
+ this.noteRemoteWrite(op);
1024
+ // Remove the operation only after it reached the remote
1025
+ if (this.offlineOps[0] === op) {
1026
+ this.offlineOps.shift();
1027
+ sent++;
1028
+ this.storeDirty = true;
1029
+ }
394
1030
  }
395
- // Remove the operation only after it reached the remote
396
- if (this.offlineOps[0] === op) {
397
- this.offlineOps.shift();
1031
+ this.isOnline = true;
1032
+ }
1033
+ finally {
1034
+ this.pushing = false;
1035
+ if (sent > 0) {
1036
+ await this.persistAll().catch((error) => {
1037
+ // The store keeps sent operations. A later direct write repairs it first.
1038
+ console.error("The queue store could not remove the sent operations:", error);
1039
+ });
398
1040
  }
399
1041
  }
400
- this.isOnline = true;
401
1042
  }
402
1043
  /** Run syncs one at a time. */
403
1044
  serialize(run) {
@@ -405,15 +1046,105 @@ export class HybridStorage {
405
1046
  this.syncChain = next.catch(() => undefined);
406
1047
  return next;
407
1048
  }
408
- async localIds() {
409
- return [...this.local.keys()];
410
- }
411
1049
  bump(id) {
412
1050
  this.writeVersions.set(id, (this.writeVersions.get(id) ?? 0) + 1);
413
1051
  }
414
- versionOf(id) {
1052
+ /** The count of local writes of an id: it changes with each write and each clear. */
1053
+ writeKey(id) {
415
1054
  return `${this.clearVersion}:${this.writeVersions.get(id) ?? 0}`;
416
1055
  }
1056
+ versionOfItem(item) {
1057
+ return this.options.versionOf?.(item);
1058
+ }
1059
+ /** This method records the remote version of an id that the storage read. */
1060
+ noteBase(id, item) {
1061
+ if (!this.options.versionOf)
1062
+ return;
1063
+ if (item === undefined) {
1064
+ this.bases.set(id, null);
1065
+ return;
1066
+ }
1067
+ const version = this.versionOfItem(item);
1068
+ if (version === undefined)
1069
+ this.bases.delete(id);
1070
+ else
1071
+ this.bases.set(id, version);
1072
+ }
1073
+ /** This method records the remote versions after a write reached the remote. */
1074
+ noteRemoteWrite(op) {
1075
+ if (!this.options.versionOf)
1076
+ return;
1077
+ const data = op.data;
1078
+ switch (op.type) {
1079
+ case "set":
1080
+ this.noteBase(data.id, data.value);
1081
+ break;
1082
+ case "delete":
1083
+ this.bases.set(data.id, null);
1084
+ break;
1085
+ case "setBatch":
1086
+ for (const [id, value] of data.items)
1087
+ this.noteBase(id, value);
1088
+ break;
1089
+ case "deleteBatch":
1090
+ for (const id of data.ids)
1091
+ this.bases.set(id, null);
1092
+ break;
1093
+ case "clear":
1094
+ for (const id of this.bases.keys())
1095
+ this.bases.set(id, null);
1096
+ break;
1097
+ }
1098
+ }
1099
+ /**
1100
+ * This method puts in the operation the remote version that it changes, for each id: the version that an
1101
+ * older queued operation of the id writes, or else the known remote version. A storage that
1102
+ * loads the queue after a restart takes the version of the first operation of each id.
1103
+ */
1104
+ attachBase(op) {
1105
+ if (!this.options.versionOf)
1106
+ return;
1107
+ const base = [];
1108
+ for (const id of idsOf(op)) {
1109
+ if (id === "*")
1110
+ continue;
1111
+ const version = this.expectedBase(id);
1112
+ if (version !== undefined)
1113
+ base.push([id, version]);
1114
+ }
1115
+ if (base.length > 0)
1116
+ op.base = base;
1117
+ }
1118
+ /** The remote version of an id after the queued operations, or undefined when it is not known. */
1119
+ expectedBase(id) {
1120
+ for (let index = this.offlineOps.length - 1; index >= 0; index--) {
1121
+ const op = this.offlineOps[index];
1122
+ const data = op.data;
1123
+ switch (op.type) {
1124
+ case "set":
1125
+ if (data.id === id)
1126
+ return this.versionOfItem(data.value);
1127
+ break;
1128
+ case "delete":
1129
+ if (data.id === id)
1130
+ return null;
1131
+ break;
1132
+ case "setBatch": {
1133
+ const item = (data.items ?? []).findLast(([itemId]) => itemId === id);
1134
+ if (item)
1135
+ return this.versionOfItem(item[1]);
1136
+ break;
1137
+ }
1138
+ case "deleteBatch":
1139
+ if ((data.ids ?? []).includes(id))
1140
+ return null;
1141
+ break;
1142
+ case "clear":
1143
+ return null;
1144
+ }
1145
+ }
1146
+ return this.bases.get(id);
1147
+ }
417
1148
  hasPendingOp(id) {
418
1149
  return this.offlineOps.some((op) => {
419
1150
  const ids = idsOf(op);
@@ -440,18 +1171,165 @@ export class HybridStorage {
440
1171
  return remote;
441
1172
  }
442
1173
  }
1174
+ //
1175
+ // ═══ The queue and its store ═══
1176
+ //
443
1177
  /**
444
- * Queue operation for later sync.
1178
+ * This method puts an operation in the queue, then in the queue store. The write ends only after the store
1179
+ * kept it. When the store fails, the operation leaves the queue, the local change is undone,
1180
+ * and the write throws `QueueStoreError`.
1181
+ *
1182
+ * A set or a delete of one id replaces the queued set or delete of that id, when no later
1183
+ * operation touches the id: a second try of a write does not queue it two times.
445
1184
  */
446
- queueOperation(op) {
447
- if (!this.options.offlineQueue) {
448
- return;
1185
+ async enqueue(op, undo) {
1186
+ const index = this.replaceableIndex(op);
1187
+ let rollback;
1188
+ if (index >= 0) {
1189
+ const previous = this.offlineOps[index];
1190
+ // The new operation changes the same remote version as the operation that it replaces
1191
+ if (previous.base)
1192
+ op.base = previous.base;
1193
+ else
1194
+ delete op.base;
1195
+ this.offlineOps[index] = op;
1196
+ rollback = () => {
1197
+ const at = this.offlineOps.indexOf(op);
1198
+ if (at >= 0)
1199
+ this.offlineOps[at] = previous;
1200
+ };
1201
+ }
1202
+ else {
1203
+ try {
1204
+ this.ensureQueueRoom(op);
1205
+ }
1206
+ catch (error) {
1207
+ await undo();
1208
+ throw error;
1209
+ }
1210
+ this.offlineOps.push(op);
1211
+ rollback = () => {
1212
+ const at = this.offlineOps.indexOf(op);
1213
+ if (at >= 0)
1214
+ this.offlineOps.splice(at, 1);
1215
+ };
449
1216
  }
450
- this.offlineOps.push(op);
451
- // Enforce max queue size
452
- if (this.offlineOps.length > this.options.maxOfflineOps) {
453
- this.offlineOps.shift(); // Remove oldest
1217
+ this.unpersisted.add(op);
1218
+ try {
1219
+ await this.persist(op, index < 0, rollback);
1220
+ }
1221
+ catch (error) {
1222
+ await undo();
1223
+ throw error;
1224
+ }
1225
+ }
1226
+ /**
1227
+ * This method writes one operation to the queue store: an append when the store holds each operation
1228
+ * before it, or else the whole queue. On a failure, `rollback` removes the operation from the
1229
+ * queue before the next call of the store.
1230
+ */
1231
+ persist(op, appended, rollback) {
1232
+ const run = this.storeChain.then(async () => {
1233
+ // An earlier write of the whole queue kept it already
1234
+ if (!this.unpersisted.has(op))
1235
+ return;
1236
+ const at = this.offlineOps.indexOf(op);
1237
+ const canAppend = appended && !this.storeDirty && at >= 0 && this.offlineOps.slice(0, at).every((other) => !this.unpersisted.has(other));
1238
+ const snapshot = this.offlineOps.slice();
1239
+ try {
1240
+ if (canAppend) {
1241
+ await this.queueStore.append(op);
1242
+ this.unpersisted.delete(op);
1243
+ }
1244
+ else {
1245
+ await this.queueStore.replace(snapshot);
1246
+ for (const kept of snapshot)
1247
+ this.unpersisted.delete(kept);
1248
+ this.unpersisted.delete(op);
1249
+ this.storeDirty = false;
1250
+ }
1251
+ }
1252
+ catch (error) {
1253
+ this.storeDirty = true;
1254
+ this.unpersisted.delete(op);
1255
+ rollback();
1256
+ throw new QueueStoreError("write", error);
1257
+ }
1258
+ });
1259
+ this.storeChain = run.catch(() => undefined);
1260
+ return run;
1261
+ }
1262
+ /** This method writes the whole queue to the queue store when the store can differ from the queue. */
1263
+ persistAll() {
1264
+ const run = this.storeChain.then(async () => {
1265
+ if (!this.storeDirty && this.unpersisted.size === 0)
1266
+ return;
1267
+ const snapshot = this.offlineOps.slice();
1268
+ try {
1269
+ if (snapshot.length === 0)
1270
+ await this.queueStore.clear();
1271
+ else
1272
+ await this.queueStore.replace(snapshot);
1273
+ for (const kept of snapshot)
1274
+ this.unpersisted.delete(kept);
1275
+ this.storeDirty = false;
1276
+ }
1277
+ catch (error) {
1278
+ this.storeDirty = true;
1279
+ throw new QueueStoreError("write", error);
1280
+ }
1281
+ });
1282
+ this.storeChain = run.catch(() => undefined);
1283
+ return run;
1284
+ }
1285
+ /**
1286
+ * Before a write goes to the remote directly, the queue store must not hold a sent operation:
1287
+ * after a restart, it would replace this newer write.
1288
+ */
1289
+ async ensureStoreClean() {
1290
+ if (this.storeDirty)
1291
+ await this.persistAll();
1292
+ }
1293
+ /** The index of the queued operation that `op` replaces, or -1. */
1294
+ replaceableIndex(op) {
1295
+ if (op.type !== "set" && op.type !== "delete")
1296
+ return -1;
1297
+ const [id] = idsOf(op);
1298
+ if (id === undefined)
1299
+ return -1;
1300
+ for (let index = this.offlineOps.length - 1; index >= 0; index--) {
1301
+ const queued = this.offlineOps[index];
1302
+ const ids = idsOf(queued);
1303
+ if (!ids.includes(id) && !ids.includes("*"))
1304
+ continue;
1305
+ // pushQueue can be sending the first operation now: do not change it
1306
+ const sending = index === 0 && this.pushing;
1307
+ return !sending && (queued.type === "set" || queued.type === "delete") ? index : -1;
454
1308
  }
1309
+ return -1;
1310
+ }
1311
+ /** True when the queue can take the operation. */
1312
+ hasQueueRoom(op) {
1313
+ return this.replaceableIndex(op) >= 0 || this.offlineOps.length < this.options.maxOfflineOps;
1314
+ }
1315
+ /** A full queue refuses the write. Before, it dropped the oldest operation (roadmap R2). */
1316
+ ensureQueueRoom(op) {
1317
+ if (!this.hasQueueRoom(op))
1318
+ throw new QueueFullError(this.options.maxOfflineOps);
1319
+ }
1320
+ /**
1321
+ * In write-through mode, a timer sends the queue after a failure, and again after each failed
1322
+ * send, until the queue is empty.
1323
+ */
1324
+ scheduleDrain() {
1325
+ if (this.drainTimer || this.closed || this.mode !== "write-through")
1326
+ return;
1327
+ const delay = this.options.syncInterval > 0 ? this.options.syncInterval : 5000;
1328
+ this.drainTimer = setTimeout(() => {
1329
+ this.drainTimer = null;
1330
+ this.syncToRemote().catch(() => this.scheduleDrain());
1331
+ }, delay);
1332
+ this.drainTimer.unref?.();
455
1333
  }
456
1334
  /**
457
1335
  * Execute a queued operation against remote storage.
@@ -476,33 +1354,6 @@ export class HybridStorage {
476
1354
  break;
477
1355
  }
478
1356
  }
479
- /**
480
- * Handle remote operation failure.
481
- */
482
- handleRemoteFailure(operation, data) {
483
- console.error(`Remote ${operation} failed, marking offline`);
484
- this.isOnline = false;
485
- // Queue operation if offline queue enabled
486
- if (this.options.offlineQueue) {
487
- this.queueOperation({
488
- type: operation,
489
- data,
490
- timestamp: Date.now(),
491
- });
492
- }
493
- }
494
- /**
495
- * Start periodic background sync.
496
- */
497
- startPeriodicSync() {
498
- this.syncTimer = setInterval(() => {
499
- this.syncToRemote().catch((err) => {
500
- console.error("Periodic sync failed:", err);
501
- });
502
- }, this.options.syncInterval);
503
- // The timer must not keep the process alive (deep dive DATA-9)
504
- this.syncTimer.unref?.();
505
- }
506
1357
  /**
507
1358
  * Get current online status.
508
1359
  */