@vrtmrz/livesync-commonlib 0.1.44 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +80 -4
  2. package/dist/API/DirectFileManipulatorV2.d.ts +2 -0
  3. package/dist/API/DirectFileManipulatorV2.js +3 -0
  4. package/dist/API/processSetting.d.ts +1 -0
  5. package/dist/common/configForDoc.d.ts +1 -0
  6. package/dist/common/configForDoc.js +11 -1
  7. package/dist/common/models/db.definition.d.ts +1 -0
  8. package/dist/common/models/setting.const.defaults.js +1 -0
  9. package/dist/common/models/setting.const.preferred.js +1 -3
  10. package/dist/common/models/setting.const.qr.js +1 -0
  11. package/dist/common/models/setting.policy.d.ts +4 -0
  12. package/dist/common/models/setting.policy.js +1 -0
  13. package/dist/common/models/setting.type.d.ts +2 -0
  14. package/dist/common/models/shared.definition.d.ts +20 -0
  15. package/dist/common/models/tweak.definition.d.ts +1 -0
  16. package/dist/common/settingConstants.d.ts +1 -0
  17. package/dist/common/settingConstants.js +6 -2
  18. package/dist/interfaces/FileHandler.d.ts +5 -1
  19. package/dist/interfaces/StorageEventManager.d.ts +5 -0
  20. package/dist/interfaces/StorageEventManager.js +7 -0
  21. package/dist/managers/EntryManager/EntryManagerImpls.js +3 -2
  22. package/dist/managers/StorageEventManager.d.ts +19 -10
  23. package/dist/managers/StorageEventManager.js +259 -189
  24. package/dist/pouchdb/LiveSyncDBFunctions.js +4 -0
  25. package/dist/pouchdb/StreamingFetch.d.ts +5 -1
  26. package/dist/pouchdb/StreamingFetch.js +529 -65
  27. package/dist/replication/LiveSyncAbstractReplicator.d.ts +2 -0
  28. package/dist/replication/LiveSyncAbstractReplicator.js +5 -2
  29. package/dist/replication/RemoteResource.d.ts +23 -2
  30. package/dist/replication/RemoteResource.js +6 -2
  31. package/dist/replication/couchdb/LiveSyncReplicator.d.ts +14 -0
  32. package/dist/replication/couchdb/LiveSyncReplicator.js +171 -3
  33. package/dist/replication/couchdb/fastForward.d.ts +19 -0
  34. package/dist/replication/couchdb/fastForward.js +119 -0
  35. package/dist/replication/couchdb/replicationProgress.d.ts +6 -0
  36. package/dist/replication/couchdb/replicationProgress.js +105 -0
  37. package/dist/replication/index.d.ts +4 -1
  38. package/dist/replication/joinedDevices.d.ts +25 -0
  39. package/dist/replication/joinedDevices.js +123 -0
  40. package/dist/replication/journal/JournalSyncCore.d.ts +2 -1
  41. package/dist/replication/journal/JournalSyncCore.js +11 -1
  42. package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +4 -0
  43. package/dist/replication/journal/LiveSyncJournalReplicator.js +56 -1
  44. package/dist/replication/journal/objectstore/JournalStorageAdapter.d.ts +3 -0
  45. package/dist/replication/journal/objectstore/MinioStorageAdapter.d.ts +2 -1
  46. package/dist/replication/journal/objectstore/MinioStorageAdapter.js +27 -1
  47. package/dist/replication/trystero/P2PRoomSessionOwner.d.ts +1 -1
  48. package/dist/replication/trystero/P2PRoomSessionOwner.js +27 -14
  49. package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
  50. package/dist/serviceFeatures/offlineScanner.js +19 -7
  51. package/dist/serviceModules/Rebuilder.js +41 -15
  52. package/dist/serviceModules/ServiceFileAccessBase.js +5 -0
  53. package/dist/serviceModules/ServiceFileHandlerBase.d.ts +8 -5
  54. package/dist/serviceModules/ServiceFileHandlerBase.js +75 -38
  55. package/dist/services/BrowserServices.d.ts +4 -2
  56. package/dist/services/BrowserServices.js +2 -5
  57. package/dist/services/HeadlessServices.d.ts +3 -0
  58. package/dist/services/HeadlessServices.js +2 -5
  59. package/dist/services/ServiceHub.d.ts +4 -4
  60. package/dist/services/base/AppLifecycleService.js +1 -1
  61. package/dist/services/base/ControlService.js +6 -2
  62. package/dist/services/base/FileProcessingService.d.ts +7 -1
  63. package/dist/services/base/FileProcessingService.js +9 -1
  64. package/dist/services/base/IService.d.ts +19 -19
  65. package/dist/services/base/ReplicatorService.d.ts +1 -0
  66. package/dist/services/base/ReplicatorService.remoteResourceResolver.js +1 -1
  67. package/dist/services/base/SettingService.d.ts +8 -18
  68. package/dist/services/base/SettingService.js +28 -46
  69. package/dist/services/implements/base/SvelteDialog.d.ts +2 -2
  70. package/dist/services/implements/base/UIService.d.ts +2 -0
  71. package/dist/services/implements/base/UIService.js +5 -0
  72. package/dist/services/implements/injectable/InjectableServiceHub.d.ts +4 -4
  73. package/dist/services/implements/injectable/InjectableServices.d.ts +2 -2
  74. package/dist/services/implements/injectable/InjectableSettingService.d.ts +0 -3
  75. package/dist/services/implements/injectable/InjectableSettingService.js +0 -11
  76. package/dist/services/lib/HandlerUtils.d.ts +60 -0
  77. package/dist/services/lib/HandlerUtils.js +46 -0
  78. package/dist/string_and_binary/chunks.js +7 -5
  79. package/dist/uiCommands.d.ts +26 -0
  80. package/dist/uiCommands.js +57 -0
  81. package/dist/worker/bgWorker.js +1 -1
  82. package/docs/development.md +15 -3
  83. package/docs/offline-scanner.md +15 -13
  84. package/docs/proven-in-use.md +38 -4
  85. package/docs/releasing.md +20 -13
  86. package/docs/settings-lifecycle.md +39 -5
  87. package/docs/storage-events-and-reflection.md +109 -10
  88. package/package.json +13 -18
  89. package/dist/services/base/ConfigService.d.ts +0 -7
  90. package/dist/services/base/ConfigService.js +0 -6
  91. package/dist/services/implements/browser/ConfigServiceBrowserCompat.d.ts +0 -16
  92. package/dist/services/implements/browser/ConfigServiceBrowserCompat.js +0 -23
@@ -8,8 +8,7 @@ import {
8
8
  } from "../common/types.js";
9
9
  import { delay, fireAndForget, isRemediationModeActive, throttle } from "../common/utils.js";
10
10
  import {} from "../common/types.js";
11
- import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
12
- import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
11
+ import { serialized } from "octagonal-wheels/concurrency/lock";
13
12
  import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
14
13
  import { promiseWithResolvers } from "octagonal-wheels/promises";
15
14
  import { StorageEventManager } from "../interfaces/StorageEventManager.js";
@@ -24,7 +23,8 @@ class StorageEventManagerBase extends StorageEventManager {
24
23
  storageAccess;
25
24
  adapter;
26
25
  get shouldBatchSave() {
27
- return this.settings?.batchSave && this.settings?.liveSync != true;
26
+ const settings = this.settings;
27
+ return settings?.liveSync === true ? settings.batchSaveInLiveSync === true : settings?.batchSave;
28
28
  }
29
29
  get batchSaveMinimumDelay() {
30
30
  return this.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay;
@@ -59,15 +59,10 @@ class StorageEventManagerBase extends StorageEventManager {
59
59
  return this.adapter.typeGuard.isFile(file);
60
60
  }
61
61
  updateStatus() {
62
- const allFileEventItems = this.bufferedQueuedItems.filter((e) => "args" in e);
63
- const allItems = allFileEventItems.filter((e) => !e.cancelled);
64
- const totalItems = allItems.length + this.concurrentProcessing.waiting;
65
- const processing = this.processingCount;
66
- const batchedCount = this._waitingMap.size;
67
62
  this.adapter.status.updateStatus({
68
- batched: batchedCount,
69
- processing,
70
- totalQueued: totalItems + batchedCount + processing
63
+ batched: this._waitingMap.size,
64
+ processing: this.processingCount,
65
+ totalQueued: [...this.pendingSnapshot.values()].filter((entry) => entry.operation && !entry.completed).length
71
66
  });
72
67
  }
73
68
  /**
@@ -82,7 +77,23 @@ class StorageEventManagerBase extends StorageEventManager {
82
77
  this.snapShotRestored = this._restoreFromSnapshot();
83
78
  return this.snapShotRestored;
84
79
  }
85
- async appendQueue(params, ctx) {
80
+ acceptingEvents = true;
81
+ admissions = /* @__PURE__ */ new Set();
82
+ appendQueue(params, ctx) {
83
+ if (!this.acceptingEvents) return Promise.resolve();
84
+ const admission = { acceptedAt: Date.now(), completion: Promise.resolve(), operations: [], flushRequested: false };
85
+ this.admissions.add(admission);
86
+ admission.completion = this.appendQueueInternal(params, ctx, admission);
87
+ void admission.completion.then(
88
+ () => this.admissions.delete(admission),
89
+ (error) => {
90
+ this.admissions.delete(admission);
91
+ this.reportOperationFailure(error);
92
+ }
93
+ );
94
+ return admission.completion;
95
+ }
96
+ async appendQueueInternal(params, ctx, admission) {
86
97
  const settings = this.settings;
87
98
  if (!settings.isConfigured) return;
88
99
  if (settings.suspendFileWatching) return;
@@ -161,7 +172,7 @@ class StorageEventManagerBase extends StorageEventManager {
161
172
  if (param.cachedData) {
162
173
  cache = param.cachedData;
163
174
  }
164
- void this.enqueue({
175
+ const item = {
165
176
  type,
166
177
  args: {
167
178
  file,
@@ -172,66 +183,126 @@ class StorageEventManagerBase extends StorageEventManager {
172
183
  },
173
184
  skipBatchWait: param.skipBatchWait,
174
185
  key: atomicKey
175
- });
186
+ };
187
+ this.enqueue(item);
188
+ const operation = this.operations.get(item);
189
+ if (operation) {
190
+ operation.acceptedAt = admission.acceptedAt;
191
+ if (admission.flushRequested) this.releaseOperations([operation]);
192
+ admission.operations.push(operation);
193
+ }
176
194
  }
177
195
  }
178
196
  // Cache file and waiting to can be proceed.
179
197
  bufferedQueuedItems = [];
180
- enqueue(newItem) {
181
- if (newItem.type == "DELETE" || newItem.type == "RENAME") {
182
- this.bufferedQueuedItems.push({
183
- type: TYPE_SENTINEL_FLUSH
184
- });
198
+ pendingSnapshot = /* @__PURE__ */ new Map();
199
+ operations = /* @__PURE__ */ new WeakMap();
200
+ latestByPath = /* @__PURE__ */ new Map();
201
+ nextSequence = 0;
202
+ snapshotTail = Promise.resolve();
203
+ dispatching = null;
204
+ retainEvent(event) {
205
+ const existing = this.pendingSnapshot.get(event);
206
+ if (existing) return existing;
207
+ const entry = { event, sequence: ++this.nextSequence, completed: false };
208
+ if (event.type !== TYPE_SENTINEL_FLUSH) {
209
+ const operation = this.operations.get(event) ?? {
210
+ event,
211
+ sequence: entry.sequence,
212
+ acceptedAt: Date.now(),
213
+ completion: promiseWithResolvers(),
214
+ prerequisites: /* @__PURE__ */ new Set(),
215
+ started: false,
216
+ flushRequested: false
217
+ };
218
+ entry.operation = operation;
219
+ this.operations.set(event, operation);
185
220
  }
221
+ this.pendingSnapshot.set(event, entry);
222
+ return entry;
223
+ }
224
+ reportOperationFailure(error) {
225
+ this._log("Storage operation remains pending after a failure", LOG_LEVEL_NOTICE);
226
+ this._log(error, LOG_LEVEL_VERBOSE);
227
+ }
228
+ failOperation(operation, error) {
229
+ if (operation.result) return;
230
+ operation.result = { ok: false, error };
231
+ operation.completion.resolve(operation.result);
232
+ this.reportOperationFailure(error);
186
233
  this.updateStatus();
234
+ }
235
+ enqueue(newItem) {
236
+ if (newItem.type === "DELETE" || newItem.type === "RENAME") {
237
+ const sentinel = { type: TYPE_SENTINEL_FLUSH };
238
+ this.retainEvent(sentinel);
239
+ this.bufferedQueuedItems.push(sentinel);
240
+ }
241
+ const operation = this.retainEvent(newItem).operation;
187
242
  this.bufferedQueuedItems.push(newItem);
188
- fireAndForget(() => this._takeSnapshot().then(() => this.runQueuedEvents()));
243
+ this.updateStatus();
244
+ fireAndForget(async () => {
245
+ try {
246
+ await this._takeSnapshot();
247
+ await this.runQueuedEvents();
248
+ } catch (error) {
249
+ this.failOperation(operation, error);
250
+ }
251
+ });
189
252
  }
190
- /**
191
- * Immediately take snapshot.
192
- */
193
253
  _triggerTakeSnapshot() {
194
- void this._takeSnapshot();
254
+ fireAndForget(() => this._takeSnapshot().catch((error) => this.reportOperationFailure(error)));
195
255
  }
196
- /**
197
- * Trigger taking snapshot after throttled period.
198
- */
199
256
  triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100);
200
- // Limit concurrent processing to reduce the IO load. file-processing + scheduler (1), so file events can be processed in 4 slots.
257
+ // Waiting timers do not occupy a processing slot.
201
258
  concurrentProcessing = Semaphore(5);
202
259
  _waitingMap = /* @__PURE__ */ new Map();
203
- _waitForIdle = null;
260
+ pendingOperations(before = Number.POSITIVE_INFINITY) {
261
+ return [...this.pendingSnapshot.values()].flatMap(
262
+ (entry) => entry.operation && entry.sequence < before ? [entry.operation] : []
263
+ );
264
+ }
265
+ releaseOperations(operations) {
266
+ for (const operation of operations) {
267
+ operation.flushRequested = true;
268
+ operation.flushSignal?.resolve();
269
+ const path = operation.event.args.file.path;
270
+ if (this._waitingMap.get(path)?.event === operation.event) this._proceedWaiting(path);
271
+ }
272
+ }
273
+ async waitForOperations(operations) {
274
+ this.releaseOperations(operations);
275
+ return await Promise.all(operations.map((operation) => operation.completion.promise));
276
+ }
204
277
  /**
205
- * Wait until all queued events are processed.
206
- * Subsequent new events will not be waited, but new events will not be added.
207
- * @returns
278
+ * Complete the finite set of operations accepted before this call.
279
+ * Later edits cannot replace or extend that set while it is being flushed.
208
280
  */
209
- waitForIdle() {
210
- if (this._waitingMap.size === 0) {
211
- return Promise.resolve();
212
- }
213
- if (this._waitForIdle) {
214
- return this._waitForIdle;
281
+ async waitForIdle() {
282
+ const admissions = [...this.admissions];
283
+ for (const admission of admissions) admission.flushRequested = true;
284
+ for (const item of this.bufferedQueuedItems) this.retainEvent(item);
285
+ const operations = this.pendingOperations();
286
+ this.releaseOperations(operations);
287
+ const admitted = await Promise.allSettled(admissions.map((admission) => admission.completion));
288
+ const boundary = [.../* @__PURE__ */ new Set([...operations, ...admissions.flatMap((admission) => admission.operations)])];
289
+ this.releaseOperations(boundary);
290
+ fireAndForget(this.runQueuedEvents().catch((error) => this.reportOperationFailure(error)));
291
+ const results = await this.waitForOperations(boundary);
292
+ const admissionFailure = admitted.find((result) => result.status === "rejected");
293
+ if (admissionFailure?.status === "rejected") throw admissionFailure.reason;
294
+ const failure = results.find((result) => !result.ok);
295
+ if (failure?.ok === false) throw failure.error;
296
+ }
297
+ /** Stop watcher admission before draining or preserving accepted work. */
298
+ async stopWatching() {
299
+ this.acceptingEvents = false;
300
+ try {
301
+ await this.waitForIdle();
302
+ } catch (error) {
303
+ this.reportOperationFailure(error);
215
304
  }
216
- const promises = [...this._waitingMap.entries()].map(([key, waitInfo]) => {
217
- return new Promise((resolve) => {
218
- waitInfo.canProceed.promise.then(() => {
219
- this._log(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG);
220
- }).catch((e) => {
221
- this._log(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO);
222
- this._log(e, LOG_LEVEL_VERBOSE);
223
- }).finally(() => {
224
- resolve();
225
- });
226
- this._proceedWaiting(key);
227
- });
228
- });
229
- const waitPromise = Promise.all(promises).then(() => {
230
- this._waitForIdle = null;
231
- this._log(`All wait for idle completed`, LOG_LEVEL_VERBOSE);
232
- });
233
- this._waitForIdle = waitPromise;
234
- return waitPromise;
305
+ await this._takeSnapshot();
235
306
  }
236
307
  /**
237
308
  * Proceed waiting for the given key immediately.
@@ -269,11 +340,13 @@ class StorageEventManagerBase extends StorageEventManager {
269
340
  }
270
341
  const resolver = promiseWithResolvers();
271
342
  const now = Date.now();
272
- const since = waitedSince ?? now;
343
+ const acceptedAt = this.operations.get(event)?.acceptedAt ?? now;
344
+ const since = waitedSince ?? acceptedAt;
273
345
  const elapsed = now - since;
274
346
  const maxDelay = this.batchSaveMaximumDelay * 1e3;
275
347
  const remainingDelay = Math.max(0, maxDelay - elapsed);
276
- const nextDelay = Math.min(remainingDelay, this.batchSaveMinimumDelay * 1e3);
348
+ const quietDelay = Math.max(0, this.batchSaveMinimumDelay * 1e3 - (now - acceptedAt));
349
+ const nextDelay = Math.min(remainingDelay, quietDelay);
277
350
  if (elapsed >= maxDelay) {
278
351
  this._log(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG);
279
352
  } else {
@@ -300,71 +373,91 @@ class StorageEventManagerBase extends StorageEventManager {
300
373
  * Process the given file event.
301
374
  */
302
375
  async processFileEvent(fei) {
303
- const releaser = await this.concurrentProcessing.acquire();
376
+ const entry = this.retainEvent(fei);
377
+ const operation = entry.operation;
378
+ if (operation.started || operation.result) {
379
+ const result = await operation.completion.promise;
380
+ if (result.ok === false) throw result.error;
381
+ return;
382
+ }
383
+ operation.started = true;
384
+ const path = fei.args.file.path;
385
+ const relatedPath = fei.type === "RENAME" ? fei.args.oldPath : fei.args.renameTarget;
386
+ const paths = relatedPath ? [path, relatedPath] : [path];
387
+ const waiting = this._waitingMap.get(path);
388
+ let since = operation.acceptedAt;
389
+ for (const eventPath of paths) {
390
+ const previous = this.latestByPath.get(eventPath);
391
+ if (!previous || previous.result?.ok) continue;
392
+ const canReplace = waiting?.event === previous.event && !previous.flushRequested && (previous.event.type === fei.type || fei.type === "DELETE");
393
+ if (canReplace) {
394
+ since = waiting.since;
395
+ previous.successor = operation;
396
+ for (const dependency of previous.prerequisites) operation.prerequisites.add(dependency);
397
+ this._cancelWaiting(eventPath);
398
+ } else {
399
+ operation.prerequisites.add(previous);
400
+ if (this._waitingMap.get(eventPath)?.event === previous.event) this._proceedWaiting(eventPath);
401
+ }
402
+ }
403
+ if (this._waitingMap.get(path) === waiting && waiting) this._proceedWaiting(path);
404
+ for (const eventPath of paths) this.latestByPath.set(eventPath, operation);
405
+ if (operation.flushRequested) this.releaseOperations([...operation.prerequisites]);
406
+ const shouldWait = this.shouldBatchSave && !fei.skipBatchWait && !operation.flushRequested && (fei.type === "CREATE" || fei.type === "CHANGED");
407
+ const gate = shouldWait ? this._addWaiting(path, fei, since).canProceed.promise : Promise.resolve(true);
304
408
  try {
305
- this.updateStatus();
306
- const filename = fei.args.file.path;
307
- const waitingKey = `${filename}`;
308
- const previous = this._waitingMap.get(waitingKey);
309
- let isShouldBeCancelled = fei.skipBatchWait || false;
310
- let previousPromise = Promise.resolve(true);
311
- let waitPromise = Promise.resolve(true);
312
- if (previous) {
313
- previousPromise = previous.canProceed.promise;
314
- if (isShouldBeCancelled) {
315
- this._log(
316
- `Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`,
317
- LOG_LEVEL_DEBUG
318
- );
319
- }
320
- if (!isShouldBeCancelled && fei.type === "DELETE") {
321
- this._log(
322
- `Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`,
323
- LOG_LEVEL_DEBUG
324
- );
325
- isShouldBeCancelled = true;
326
- }
327
- if (!isShouldBeCancelled && previous.type === fei.type) {
328
- this._log(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG);
329
- isShouldBeCancelled = true;
330
- }
331
- if (isShouldBeCancelled) {
332
- this._cancelWaiting(waitingKey);
333
- this._log(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG);
334
- isShouldBeCancelled = true;
409
+ await this._takeSnapshot();
410
+ const canProceed = await gate;
411
+ if (!canProceed) {
412
+ if (!operation.successor) throw new Error(`Storage operation cancelled without a successor: ${path}`);
413
+ const result = await operation.successor.completion.promise;
414
+ if (result.ok === false) throw result.error;
415
+ } else {
416
+ for (const prerequisite of operation.prerequisites) {
417
+ const result = await prerequisite.completion.promise;
418
+ if (result.ok === false) throw result.error;
335
419
  }
336
- if (!isShouldBeCancelled) {
337
- this._log(`Processing ${filename}: Waiting for previous to complete: ${fei.type}`, LOG_LEVEL_DEBUG);
338
- this._proceedWaiting(waitingKey);
339
- this._log(`Processing ${filename}: Previous completed: ${fei.type}`, LOG_LEVEL_DEBUG);
420
+ const release = await this.concurrentProcessing.acquire();
421
+ try {
422
+ await this.requestProcessQueue(fei);
423
+ } finally {
424
+ release();
340
425
  }
341
426
  }
342
- await previousPromise;
343
- if (this.shouldBatchSave && !fei.skipBatchWait) {
344
- if (fei.type == "CREATE" || fei.type == "CHANGED") {
345
- const info = this._addWaiting(waitingKey, fei, previous?.since);
346
- waitPromise = info.canProceed.promise;
347
- } else if (fei.type == "DELETE") {
348
- }
349
- this._log(`Processing ${filename}: Waiting for batch save: ${fei.type}`, LOG_LEVEL_DEBUG);
350
- const canProceed = await waitPromise;
351
- if (!canProceed) {
352
- this._log(`Processing ${filename}: Cancelled by new queue: ${fei.type}`, LOG_LEVEL_DEBUG);
353
- return;
354
- }
427
+ entry.completed = true;
428
+ try {
429
+ await this._takeSnapshot();
430
+ } catch (error) {
431
+ entry.completed = false;
432
+ throw error;
355
433
  }
356
- await this.requestProcessQueue(fei);
434
+ operation.result = { ok: true };
435
+ operation.completion.resolve(operation.result);
436
+ for (const eventPath of paths) {
437
+ if (this.latestByPath.get(eventPath) === operation) this.latestByPath.delete(eventPath);
438
+ }
439
+ } catch (error) {
440
+ if (this._waitingMap.get(path)?.event === fei) this._cancelWaiting(path);
441
+ this.failOperation(operation, error);
442
+ throw error;
357
443
  } finally {
358
- await this._takeSnapshot();
359
- releaser();
444
+ this.updateStatus();
360
445
  }
361
446
  }
362
447
  async _takeSnapshot() {
363
- const processingEvents = [...this._waitingMap.values()].map((e) => e.event);
364
- const waitingEvents = this.bufferedQueuedItems;
365
- const snapShot = [...processingEvents, ...waitingEvents];
366
- await this._saveSnapshot(snapShot);
367
- this.updateStatus();
448
+ for (const event of this.bufferedQueuedItems) this.retainEvent(event);
449
+ for (const waiting of this._waitingMap.values()) this.retainEvent(waiting.event);
450
+ const save = this.snapshotTail.then(async () => {
451
+ const entries = [...this.pendingSnapshot.values()];
452
+ const completed = entries.filter((entry) => entry.completed);
453
+ const snapshot = entries.filter((entry) => !entry.completed).map((entry) => entry.event);
454
+ await this._saveSnapshot(snapshot);
455
+ for (const entry of completed) this.pendingSnapshot.delete(entry.event);
456
+ this.updateStatus();
457
+ });
458
+ this.snapshotTail = save.catch(() => {
459
+ });
460
+ await save;
368
461
  }
369
462
  async _restoreFromSnapshot() {
370
463
  const snapShot = await this._loadSnapshot();
@@ -389,55 +482,42 @@ class StorageEventManagerBase extends StorageEventManager {
389
482
  * An individual restored operation failure is logged but does not prevent the
390
483
  * following Offline Scanner from reconciling the resulting current state.
391
484
  */
392
- runQueuedEvents(options = {}) {
393
- const waitForProcessing = options.waitForProcessing ?? false;
394
- return skipIfDuplicated("storage-event-manager-run-queued-events", async () => {
395
- let dispatched = [];
396
- const waitForDispatched = async () => {
397
- if (dispatched.length === 0) return;
398
- const pending = dispatched;
399
- dispatched = [];
400
- const results = await Promise.allSettled(pending.map(({ processing }) => processing));
401
- for (const [index, result] of results.entries()) {
402
- if (result.status === "rejected") {
403
- this._log(
404
- `Restored storage operation failed for ${pending[index].path}; the Offline Scanner will reconcile the current state`,
405
- LOG_LEVEL_NOTICE
406
- );
407
- this._log(result.reason, LOG_LEVEL_VERBOSE);
485
+ async runQueuedEvents(options = {}) {
486
+ for (const event of this.bufferedQueuedItems) this.retainEvent(event);
487
+ const accepted = this.pendingOperations();
488
+ if (!this.dispatching) {
489
+ const dispatch = async () => {
490
+ while (this.bufferedQueuedItems.length > 0) {
491
+ const event = this.bufferedQueuedItems.shift();
492
+ const entry = this.retainEvent(event);
493
+ if (event.type === TYPE_SENTINEL_FLUSH) {
494
+ const results = await this.waitForOperations(this.pendingOperations(entry.sequence));
495
+ if (results.every((result) => result.ok)) {
496
+ entry.completed = true;
497
+ try {
498
+ await this._takeSnapshot();
499
+ } catch (error) {
500
+ entry.completed = false;
501
+ this.reportOperationFailure(error);
502
+ }
503
+ }
504
+ } else if (!entry.operation.result) {
505
+ fireAndForget(this.processFileEvent(event).catch(() => {
506
+ }));
408
507
  }
409
508
  }
410
509
  };
411
- do {
412
- if (this.bufferedQueuedItems.length === 0) {
413
- break;
414
- }
415
- const fei = this.bufferedQueuedItems.shift();
416
- await this._takeSnapshot();
417
- this.updateStatus();
418
- const releaser = await this.concurrentProcessing.acquire();
419
- releaser();
420
- this.updateStatus();
421
- if (fei.type === TYPE_SENTINEL_FLUSH) {
422
- this._log(`Waiting for idle`, LOG_LEVEL_VERBOSE);
423
- if (waitForProcessing) {
424
- await waitForDispatched();
425
- }
426
- await this.waitForIdle();
427
- this.updateStatus();
428
- continue;
429
- }
430
- const processing = this.processFileEvent(fei);
431
- if (waitForProcessing) {
432
- dispatched.push({ path: fei.args.file.path, processing });
433
- } else {
434
- fireAndForget(processing);
435
- }
436
- } while (this.bufferedQueuedItems.length > 0);
437
- if (waitForProcessing) {
438
- await waitForDispatched();
510
+ this.dispatching = dispatch().finally(() => {
511
+ this.dispatching = null;
512
+ });
513
+ }
514
+ await this.dispatching;
515
+ if (options.waitForProcessing) {
516
+ const results = await this.waitForOperations(accepted);
517
+ for (const result of results) {
518
+ if (result.ok === false) this.reportOperationFailure(result.error);
439
519
  }
440
- });
520
+ }
441
521
  }
442
522
  processingCount = 0;
443
523
  async requestProcessQueue(fei) {
@@ -445,50 +525,40 @@ class StorageEventManagerBase extends StorageEventManager {
445
525
  this.processingCount++;
446
526
  this.updateStatus();
447
527
  await this.handleFileEvent(fei);
448
- await this._takeSnapshot();
449
528
  } finally {
450
529
  this.processingCount--;
451
530
  this.updateStatus();
452
531
  }
453
532
  }
454
533
  isWaiting(filename) {
455
- return isWaitingForTimeout(`storage-event-manager-batchsave-${filename}`);
534
+ return this._waitingMap.has(filename);
456
535
  }
457
536
  async handleFileEvent(queue) {
458
537
  const file = queue.args.file;
459
- const lockKey = `handleFile:${file.path}`;
460
- await serialized(lockKey, async () => {
461
- if (queue.cancelled) {
462
- this._log(`File event cancelled before processing: ${file.path}`, LOG_LEVEL_INFO);
463
- return;
464
- }
465
- if (queue.type == "INTERNAL" || file.isInternal) {
466
- await this.fileProcessing.processOptionalFileEvent(file.path);
467
- } else {
468
- const last = 0;
469
- if (queue.type == "DELETE") {
470
- await this.fileProcessing.processFileEvent(queue);
471
- } else {
472
- if (file.stat.mtime == last) {
473
- this._log(
474
- `File has been already scanned on ${queue.type}, skip: ${file.path}`,
475
- LOG_LEVEL_VERBOSE
476
- );
477
- return;
538
+ await serialized(`handleFile:${file.path}`, async () => {
539
+ if (queue.cancelled) throw new Error(`Storage operation has no confirmed result: ${file.path}`);
540
+ if (queue.type === "INTERNAL" || file.isInternal) {
541
+ const handler = this.fileProcessing.processOptionalFileEventWithResult;
542
+ if (handler && handler.hasHandlers !== false) {
543
+ const operation = this.operations.get(queue);
544
+ let whenFlushRequested;
545
+ if (operation) {
546
+ operation.flushSignal ??= promiseWithResolvers();
547
+ if (operation.flushRequested) operation.flushSignal.resolve();
548
+ whenFlushRequested = operation.flushSignal.promise;
478
549
  }
479
- if (!await this.fileProcessing.processFileEvent(queue)) {
480
- this._log(
481
- `STORAGE -> DB: Handler failed, cancel the relative operations: ${file.path}`,
482
- LOG_LEVEL_INFO
483
- );
484
- this.cancelRelativeEvent(queue);
485
- return;
550
+ const outcome = await handler(file.path, { whenFlushRequested });
551
+ if (outcome.kind === "veto") {
552
+ throw new Error(`Optional storage operation declined: ${file.path} (${outcome.value})`);
486
553
  }
554
+ } else {
555
+ await this.fileProcessing.processOptionalFileEvent(file.path);
487
556
  }
557
+ } else if (!await this.fileProcessing.processFileEvent(queue)) {
558
+ throw new Error(`Storage operation failed: ${file.path}`);
488
559
  }
489
560
  });
490
561
  this.updateStatus();
491
- return;
492
562
  }
493
563
  cancelRelativeEvent(item) {
494
564
  this._cancelWaiting(item.args.file.path);
@@ -86,6 +86,10 @@ async function ensureRemoteIsCompatible(infoSrc, setting, deviceNodeID, currentV
86
86
  writeMilestone = true;
87
87
  }
88
88
  }
89
+ if (nodeDeviceInfo.platform !== void 0 && info.platform !== nodeDeviceInfo.platform) {
90
+ remoteMilestone.node_info[deviceNodeID].platform = nodeDeviceInfo.platform;
91
+ writeMilestone = true;
92
+ }
89
93
  const diffLastConnected = now - (remoteMilestone.node_info[deviceNodeID].last_connected || 0);
90
94
  if (diffLastConnected > 6e4) {
91
95
  remoteMilestone.node_info[deviceNodeID].last_connected = now;
@@ -54,6 +54,10 @@ export type FetchChangesForInitialSyncProgress = {
54
54
  * @param decryptFunction Function to decrypt each document.
55
55
  * @param since Sequence ID to start fetching changes from (default is '0').
56
56
  * @param customHeaders Additional request headers required by the CouchDB endpoint or its reverse proxy.
57
+ * @param onStoredRevisions Observes revision identities after their complete local batch is persisted.
57
58
  */
58
- export declare function fetchChangesForInitialSync(downloadToDB: PouchDB.Database, remoteDbUrl: string, authHeader: string, decryptFunction: (doc: EntryDoc) => Promise<AnyEntry | EntryLeaf>, since?: number | string, onProgress?: (progress: FetchChangesForInitialSyncProgress) => void, onCheckpoint?: (sequence: DBSequence) => void | Promise<void>, customHeaders?: Record<string, string>): Promise<void>;
59
+ export declare function fetchChangesForInitialSync(downloadToDB: PouchDB.Database, remoteDbUrl: string, authHeader: string, decryptFunction: (doc: EntryDoc) => Promise<AnyEntry | EntryLeaf>, since?: number | string, onProgress?: (progress: FetchChangesForInitialSyncProgress) => void, onCheckpoint?: (sequence: DBSequence) => void | Promise<void>, customHeaders?: Record<string, string>, onStoredRevisions?: (revisions: readonly {
60
+ id: string;
61
+ rev: string;
62
+ }[]) => void | Promise<void>): Promise<void>;
59
63
  export {};