@vrtmrz/livesync-commonlib 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/API/processSetting.d.ts +2 -0
- package/dist/common/models/setting.const.defaults.js +2 -0
- package/dist/common/models/setting.const.qr.js +3 -0
- package/dist/common/models/setting.policy.d.ts +8 -0
- package/dist/common/models/setting.policy.js +2 -0
- package/dist/common/models/setting.type.d.ts +6 -0
- package/dist/common/models/shared.definition.d.ts +20 -0
- package/dist/common/models/tweak.definition.d.ts +2 -0
- package/dist/common/settingConstants.d.ts +2 -0
- package/dist/common/settingConstants.js +10 -2
- package/dist/interfaces/FileHandler.d.ts +7 -1
- package/dist/interfaces/StorageAccess.d.ts +2 -0
- package/dist/interfaces/StorageEventManager.d.ts +7 -0
- package/dist/interfaces/StorageEventManager.js +11 -0
- package/dist/managers/StorageEventManager.d.ts +20 -10
- package/dist/managers/StorageEventManager.js +268 -189
- package/dist/replication/LiveSyncAbstractReplicator.d.ts +2 -0
- package/dist/replication/LiveSyncAbstractReplicator.js +5 -2
- package/dist/replication/couchdb/LiveSyncReplicator.d.ts +6 -0
- package/dist/replication/couchdb/LiveSyncReplicator.js +39 -2
- package/dist/replication/couchdb/replicationProgress.d.ts +6 -0
- package/dist/replication/couchdb/replicationProgress.js +105 -0
- package/dist/replication/index.d.ts +1 -0
- package/dist/replication/trystero/TrysteroReplicator.d.ts +4 -0
- package/dist/serviceFeatures/offlineScanner.d.ts +4 -4
- package/dist/serviceFeatures/offlineScanner.js +30 -13
- package/dist/serviceFeatures/prepareDatabaseForUse.d.ts +3 -2
- package/dist/serviceFeatures/prepareDatabaseForUse.js +5 -4
- package/dist/serviceModules/ServiceFileAccessBase.d.ts +1 -0
- package/dist/serviceModules/ServiceFileAccessBase.js +8 -0
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +9 -5
- package/dist/serviceModules/ServiceFileHandlerBase.js +96 -38
- package/dist/services/base/AppLifecycleService.js +1 -1
- package/dist/services/base/ControlService.js +6 -2
- package/dist/services/base/DatabaseEventService.d.ts +1 -1
- package/dist/services/base/FileProcessingService.d.ts +7 -1
- package/dist/services/base/FileProcessingService.js +9 -1
- package/dist/services/base/IService.d.ts +12 -4
- package/dist/services/base/ReplicatorService.d.ts +1 -0
- package/dist/services/base/VaultScanResult.d.ts +5 -0
- package/dist/services/base/VaultService.d.ts +1 -1
- package/dist/services/lib/HandlerUtils.d.ts +60 -0
- package/dist/services/lib/HandlerUtils.js +46 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/offline-scanner.md +23 -13
- package/docs/settings-lifecycle.md +2 -0
- package/docs/storage-events-and-reflection.md +113 -10
- package/package.json +2 -2
|
@@ -8,8 +8,7 @@ import {
|
|
|
8
8
|
} from "../common/types.js";
|
|
9
9
|
import { delay, fireAndForget, isRemediationModeActive, throttle } from "../common/utils.js";
|
|
10
10
|
import {} from "../common/types.js";
|
|
11
|
-
import { serialized
|
|
12
|
-
import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
|
|
11
|
+
import { serialized } from "octagonal-wheels/concurrency/lock";
|
|
13
12
|
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
|
|
14
13
|
import { promiseWithResolvers } from "octagonal-wheels/promises";
|
|
15
14
|
import { StorageEventManager } from "../interfaces/StorageEventManager.js";
|
|
@@ -24,7 +23,8 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
24
23
|
storageAccess;
|
|
25
24
|
adapter;
|
|
26
25
|
get shouldBatchSave() {
|
|
27
|
-
|
|
26
|
+
const settings = this.settings;
|
|
27
|
+
return settings?.liveSync === true ? settings.batchSaveInLiveSync === true : settings?.batchSave;
|
|
28
28
|
}
|
|
29
29
|
get batchSaveMinimumDelay() {
|
|
30
30
|
return this.settings?.batchSaveMinimumDelay ?? DEFAULT_SETTINGS.batchSaveMinimumDelay;
|
|
@@ -59,15 +59,10 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
59
59
|
return this.adapter.typeGuard.isFile(file);
|
|
60
60
|
}
|
|
61
61
|
updateStatus() {
|
|
62
|
-
const allFileEventItems = this.bufferedQueuedItems.filter((e) => "args" in e);
|
|
63
|
-
const allItems = allFileEventItems.filter((e) => !e.cancelled);
|
|
64
|
-
const totalItems = allItems.length + this.concurrentProcessing.waiting;
|
|
65
|
-
const processing = this.processingCount;
|
|
66
|
-
const batchedCount = this._waitingMap.size;
|
|
67
62
|
this.adapter.status.updateStatus({
|
|
68
|
-
batched:
|
|
69
|
-
processing,
|
|
70
|
-
totalQueued:
|
|
63
|
+
batched: this._waitingMap.size,
|
|
64
|
+
processing: this.processingCount,
|
|
65
|
+
totalQueued: [...this.pendingSnapshot.values()].filter((entry) => entry.operation && !entry.completed).length
|
|
71
66
|
});
|
|
72
67
|
}
|
|
73
68
|
/**
|
|
@@ -82,7 +77,23 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
82
77
|
this.snapShotRestored = this._restoreFromSnapshot();
|
|
83
78
|
return this.snapShotRestored;
|
|
84
79
|
}
|
|
85
|
-
|
|
80
|
+
acceptingEvents = true;
|
|
81
|
+
admissions = /* @__PURE__ */ new Set();
|
|
82
|
+
appendQueue(params, ctx) {
|
|
83
|
+
if (!this.acceptingEvents) return Promise.resolve();
|
|
84
|
+
const admission = { acceptedAt: Date.now(), completion: Promise.resolve(), operations: [], flushRequested: false };
|
|
85
|
+
this.admissions.add(admission);
|
|
86
|
+
admission.completion = this.appendQueueInternal(params, ctx, admission);
|
|
87
|
+
void admission.completion.then(
|
|
88
|
+
() => this.admissions.delete(admission),
|
|
89
|
+
(error) => {
|
|
90
|
+
this.admissions.delete(admission);
|
|
91
|
+
this.reportOperationFailure(error);
|
|
92
|
+
}
|
|
93
|
+
);
|
|
94
|
+
return admission.completion;
|
|
95
|
+
}
|
|
96
|
+
async appendQueueInternal(params, ctx, admission) {
|
|
86
97
|
const settings = this.settings;
|
|
87
98
|
if (!settings.isConfigured) return;
|
|
88
99
|
if (settings.suspendFileWatching) return;
|
|
@@ -161,7 +172,7 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
161
172
|
if (param.cachedData) {
|
|
162
173
|
cache = param.cachedData;
|
|
163
174
|
}
|
|
164
|
-
|
|
175
|
+
const item = {
|
|
165
176
|
type,
|
|
166
177
|
args: {
|
|
167
178
|
file,
|
|
@@ -172,66 +183,126 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
172
183
|
},
|
|
173
184
|
skipBatchWait: param.skipBatchWait,
|
|
174
185
|
key: atomicKey
|
|
175
|
-
}
|
|
186
|
+
};
|
|
187
|
+
this.enqueue(item);
|
|
188
|
+
const operation = this.operations.get(item);
|
|
189
|
+
if (operation) {
|
|
190
|
+
operation.acceptedAt = admission.acceptedAt;
|
|
191
|
+
if (admission.flushRequested) this.releaseOperations([operation]);
|
|
192
|
+
admission.operations.push(operation);
|
|
193
|
+
}
|
|
176
194
|
}
|
|
177
195
|
}
|
|
178
196
|
// Cache file and waiting to can be proceed.
|
|
179
197
|
bufferedQueuedItems = [];
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
198
|
+
pendingSnapshot = /* @__PURE__ */ new Map();
|
|
199
|
+
operations = /* @__PURE__ */ new WeakMap();
|
|
200
|
+
latestByPath = /* @__PURE__ */ new Map();
|
|
201
|
+
nextSequence = 0;
|
|
202
|
+
snapshotTail = Promise.resolve();
|
|
203
|
+
dispatching = null;
|
|
204
|
+
retainEvent(event) {
|
|
205
|
+
const existing = this.pendingSnapshot.get(event);
|
|
206
|
+
if (existing) return existing;
|
|
207
|
+
const entry = { event, sequence: ++this.nextSequence, completed: false };
|
|
208
|
+
if (event.type !== TYPE_SENTINEL_FLUSH) {
|
|
209
|
+
const operation = this.operations.get(event) ?? {
|
|
210
|
+
event,
|
|
211
|
+
sequence: entry.sequence,
|
|
212
|
+
acceptedAt: Date.now(),
|
|
213
|
+
completion: promiseWithResolvers(),
|
|
214
|
+
prerequisites: /* @__PURE__ */ new Set(),
|
|
215
|
+
started: false,
|
|
216
|
+
flushRequested: false
|
|
217
|
+
};
|
|
218
|
+
entry.operation = operation;
|
|
219
|
+
this.operations.set(event, operation);
|
|
185
220
|
}
|
|
221
|
+
this.pendingSnapshot.set(event, entry);
|
|
222
|
+
return entry;
|
|
223
|
+
}
|
|
224
|
+
reportOperationFailure(error) {
|
|
225
|
+
this._log("Storage operation remains pending after a failure", LOG_LEVEL_NOTICE);
|
|
226
|
+
this._log(error, LOG_LEVEL_VERBOSE);
|
|
227
|
+
}
|
|
228
|
+
failOperation(operation, error) {
|
|
229
|
+
if (operation.result) return;
|
|
230
|
+
operation.result = { ok: false, error };
|
|
231
|
+
operation.completion.resolve(operation.result);
|
|
232
|
+
this.reportOperationFailure(error);
|
|
186
233
|
this.updateStatus();
|
|
234
|
+
}
|
|
235
|
+
enqueue(newItem) {
|
|
236
|
+
if (newItem.type === "DELETE" || newItem.type === "RENAME") {
|
|
237
|
+
const sentinel = { type: TYPE_SENTINEL_FLUSH };
|
|
238
|
+
this.retainEvent(sentinel);
|
|
239
|
+
this.bufferedQueuedItems.push(sentinel);
|
|
240
|
+
}
|
|
241
|
+
const operation = this.retainEvent(newItem).operation;
|
|
187
242
|
this.bufferedQueuedItems.push(newItem);
|
|
188
|
-
|
|
243
|
+
this.updateStatus();
|
|
244
|
+
fireAndForget(async () => {
|
|
245
|
+
try {
|
|
246
|
+
await this._takeSnapshot();
|
|
247
|
+
await this.runQueuedEvents();
|
|
248
|
+
} catch (error) {
|
|
249
|
+
this.failOperation(operation, error);
|
|
250
|
+
}
|
|
251
|
+
});
|
|
189
252
|
}
|
|
190
|
-
/**
|
|
191
|
-
* Immediately take snapshot.
|
|
192
|
-
*/
|
|
193
253
|
_triggerTakeSnapshot() {
|
|
194
|
-
|
|
254
|
+
fireAndForget(() => this._takeSnapshot().catch((error) => this.reportOperationFailure(error)));
|
|
195
255
|
}
|
|
196
|
-
/**
|
|
197
|
-
* Trigger taking snapshot after throttled period.
|
|
198
|
-
*/
|
|
199
256
|
triggerTakeSnapshot = throttle(() => this._triggerTakeSnapshot(), 100);
|
|
200
|
-
//
|
|
257
|
+
// Waiting timers do not occupy a processing slot.
|
|
201
258
|
concurrentProcessing = Semaphore(5);
|
|
202
259
|
_waitingMap = /* @__PURE__ */ new Map();
|
|
203
|
-
|
|
260
|
+
pendingOperations(before = Number.POSITIVE_INFINITY) {
|
|
261
|
+
return [...this.pendingSnapshot.values()].flatMap(
|
|
262
|
+
(entry) => entry.operation && entry.sequence < before ? [entry.operation] : []
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
releaseOperations(operations) {
|
|
266
|
+
for (const operation of operations) {
|
|
267
|
+
operation.flushRequested = true;
|
|
268
|
+
operation.flushSignal?.resolve();
|
|
269
|
+
const path = operation.event.args.file.path;
|
|
270
|
+
if (this._waitingMap.get(path)?.event === operation.event) this._proceedWaiting(path);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
async waitForOperations(operations) {
|
|
274
|
+
this.releaseOperations(operations);
|
|
275
|
+
return await Promise.all(operations.map((operation) => operation.completion.promise));
|
|
276
|
+
}
|
|
204
277
|
/**
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
* @returns
|
|
278
|
+
* Complete the finite set of operations accepted before this call.
|
|
279
|
+
* Later edits cannot replace or extend that set while it is being flushed.
|
|
208
280
|
*/
|
|
209
|
-
waitForIdle() {
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
281
|
+
async waitForIdle() {
|
|
282
|
+
const admissions = [...this.admissions];
|
|
283
|
+
for (const admission of admissions) admission.flushRequested = true;
|
|
284
|
+
for (const item of this.bufferedQueuedItems) this.retainEvent(item);
|
|
285
|
+
const operations = this.pendingOperations();
|
|
286
|
+
this.releaseOperations(operations);
|
|
287
|
+
const admitted = await Promise.allSettled(admissions.map((admission) => admission.completion));
|
|
288
|
+
const boundary = [.../* @__PURE__ */ new Set([...operations, ...admissions.flatMap((admission) => admission.operations)])];
|
|
289
|
+
this.releaseOperations(boundary);
|
|
290
|
+
fireAndForget(this.runQueuedEvents().catch((error) => this.reportOperationFailure(error)));
|
|
291
|
+
const results = await this.waitForOperations(boundary);
|
|
292
|
+
const admissionFailure = admitted.find((result) => result.status === "rejected");
|
|
293
|
+
if (admissionFailure?.status === "rejected") throw admissionFailure.reason;
|
|
294
|
+
const failure = results.find((result) => !result.ok);
|
|
295
|
+
if (failure?.ok === false) throw failure.error;
|
|
296
|
+
}
|
|
297
|
+
/** Stop watcher admission before draining or preserving accepted work. */
|
|
298
|
+
async stopWatching() {
|
|
299
|
+
this.acceptingEvents = false;
|
|
300
|
+
try {
|
|
301
|
+
await this.waitForIdle();
|
|
302
|
+
} catch (error) {
|
|
303
|
+
this.reportOperationFailure(error);
|
|
215
304
|
}
|
|
216
|
-
|
|
217
|
-
return new Promise((resolve) => {
|
|
218
|
-
waitInfo.canProceed.promise.then(() => {
|
|
219
|
-
this._log(`Processing ${key}: Wait for idle completed`, LOG_LEVEL_DEBUG);
|
|
220
|
-
}).catch((e) => {
|
|
221
|
-
this._log(`Processing ${key}: Wait for idle error`, LOG_LEVEL_INFO);
|
|
222
|
-
this._log(e, LOG_LEVEL_VERBOSE);
|
|
223
|
-
}).finally(() => {
|
|
224
|
-
resolve();
|
|
225
|
-
});
|
|
226
|
-
this._proceedWaiting(key);
|
|
227
|
-
});
|
|
228
|
-
});
|
|
229
|
-
const waitPromise = Promise.all(promises).then(() => {
|
|
230
|
-
this._waitForIdle = null;
|
|
231
|
-
this._log(`All wait for idle completed`, LOG_LEVEL_VERBOSE);
|
|
232
|
-
});
|
|
233
|
-
this._waitForIdle = waitPromise;
|
|
234
|
-
return waitPromise;
|
|
305
|
+
await this._takeSnapshot();
|
|
235
306
|
}
|
|
236
307
|
/**
|
|
237
308
|
* Proceed waiting for the given key immediately.
|
|
@@ -269,11 +340,13 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
269
340
|
}
|
|
270
341
|
const resolver = promiseWithResolvers();
|
|
271
342
|
const now = Date.now();
|
|
272
|
-
const
|
|
343
|
+
const acceptedAt = this.operations.get(event)?.acceptedAt ?? now;
|
|
344
|
+
const since = waitedSince ?? acceptedAt;
|
|
273
345
|
const elapsed = now - since;
|
|
274
346
|
const maxDelay = this.batchSaveMaximumDelay * 1e3;
|
|
275
347
|
const remainingDelay = Math.max(0, maxDelay - elapsed);
|
|
276
|
-
const
|
|
348
|
+
const quietDelay = Math.max(0, this.batchSaveMinimumDelay * 1e3 - (now - acceptedAt));
|
|
349
|
+
const nextDelay = Math.min(remainingDelay, quietDelay);
|
|
277
350
|
if (elapsed >= maxDelay) {
|
|
278
351
|
this._log(`Processing ${key}: Batch save maximum delay already exceeded: ${event.type}`, LOG_LEVEL_DEBUG);
|
|
279
352
|
} else {
|
|
@@ -300,71 +373,91 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
300
373
|
* Process the given file event.
|
|
301
374
|
*/
|
|
302
375
|
async processFileEvent(fei) {
|
|
303
|
-
const
|
|
376
|
+
const entry = this.retainEvent(fei);
|
|
377
|
+
const operation = entry.operation;
|
|
378
|
+
if (operation.started || operation.result) {
|
|
379
|
+
const result = await operation.completion.promise;
|
|
380
|
+
if (result.ok === false) throw result.error;
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
operation.started = true;
|
|
384
|
+
const path = fei.args.file.path;
|
|
385
|
+
const relatedPath = fei.type === "RENAME" ? fei.args.oldPath : fei.args.renameTarget;
|
|
386
|
+
const paths = relatedPath ? [path, relatedPath] : [path];
|
|
387
|
+
const waiting = this._waitingMap.get(path);
|
|
388
|
+
let since = operation.acceptedAt;
|
|
389
|
+
for (const eventPath of paths) {
|
|
390
|
+
const previous = this.latestByPath.get(eventPath);
|
|
391
|
+
if (!previous || previous.result?.ok) continue;
|
|
392
|
+
const canReplace = waiting?.event === previous.event && !previous.flushRequested && (previous.event.type === fei.type || fei.type === "DELETE");
|
|
393
|
+
if (canReplace) {
|
|
394
|
+
since = waiting.since;
|
|
395
|
+
previous.successor = operation;
|
|
396
|
+
for (const dependency of previous.prerequisites) operation.prerequisites.add(dependency);
|
|
397
|
+
this._cancelWaiting(eventPath);
|
|
398
|
+
} else {
|
|
399
|
+
operation.prerequisites.add(previous);
|
|
400
|
+
if (this._waitingMap.get(eventPath)?.event === previous.event) this._proceedWaiting(eventPath);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
if (this._waitingMap.get(path) === waiting && waiting) this._proceedWaiting(path);
|
|
404
|
+
for (const eventPath of paths) this.latestByPath.set(eventPath, operation);
|
|
405
|
+
if (operation.flushRequested) this.releaseOperations([...operation.prerequisites]);
|
|
406
|
+
const shouldWait = this.shouldBatchSave && !fei.skipBatchWait && !operation.flushRequested && (fei.type === "CREATE" || fei.type === "CHANGED");
|
|
407
|
+
const gate = shouldWait ? this._addWaiting(path, fei, since).canProceed.promise : Promise.resolve(true);
|
|
304
408
|
try {
|
|
305
|
-
this.
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
this._log(
|
|
316
|
-
`Processing ${filename}: Requested to perform immediately, cancelling previous waiting: ${fei.type}`,
|
|
317
|
-
LOG_LEVEL_DEBUG
|
|
318
|
-
);
|
|
319
|
-
}
|
|
320
|
-
if (!isShouldBeCancelled && fei.type === "DELETE") {
|
|
321
|
-
this._log(
|
|
322
|
-
`Processing ${filename}: DELETE requested, cancelling previous waiting: ${fei.type}`,
|
|
323
|
-
LOG_LEVEL_DEBUG
|
|
324
|
-
);
|
|
325
|
-
isShouldBeCancelled = true;
|
|
326
|
-
}
|
|
327
|
-
if (!isShouldBeCancelled && previous.type === fei.type) {
|
|
328
|
-
this._log(`Processing ${filename}: Cancelling previous waiting: ${fei.type}`, LOG_LEVEL_DEBUG);
|
|
329
|
-
isShouldBeCancelled = true;
|
|
330
|
-
}
|
|
331
|
-
if (isShouldBeCancelled) {
|
|
332
|
-
this._cancelWaiting(waitingKey);
|
|
333
|
-
this._log(`Processing ${filename}: Previous cancelled: ${fei.type}`, LOG_LEVEL_DEBUG);
|
|
334
|
-
isShouldBeCancelled = true;
|
|
409
|
+
await this._takeSnapshot();
|
|
410
|
+
const canProceed = await gate;
|
|
411
|
+
if (!canProceed) {
|
|
412
|
+
if (!operation.successor) throw new Error(`Storage operation cancelled without a successor: ${path}`);
|
|
413
|
+
const result = await operation.successor.completion.promise;
|
|
414
|
+
if (result.ok === false) throw result.error;
|
|
415
|
+
} else {
|
|
416
|
+
for (const prerequisite of operation.prerequisites) {
|
|
417
|
+
const result = await prerequisite.completion.promise;
|
|
418
|
+
if (result.ok === false) throw result.error;
|
|
335
419
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
this.
|
|
339
|
-
|
|
420
|
+
const release = await this.concurrentProcessing.acquire();
|
|
421
|
+
try {
|
|
422
|
+
await this.requestProcessQueue(fei);
|
|
423
|
+
} finally {
|
|
424
|
+
release();
|
|
340
425
|
}
|
|
341
426
|
}
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
return;
|
|
354
|
-
}
|
|
427
|
+
entry.completed = true;
|
|
428
|
+
try {
|
|
429
|
+
await this._takeSnapshot();
|
|
430
|
+
} catch (error) {
|
|
431
|
+
entry.completed = false;
|
|
432
|
+
throw error;
|
|
433
|
+
}
|
|
434
|
+
operation.result = { ok: true };
|
|
435
|
+
operation.completion.resolve(operation.result);
|
|
436
|
+
for (const eventPath of paths) {
|
|
437
|
+
if (this.latestByPath.get(eventPath) === operation) this.latestByPath.delete(eventPath);
|
|
355
438
|
}
|
|
356
|
-
|
|
439
|
+
} catch (error) {
|
|
440
|
+
if (this._waitingMap.get(path)?.event === fei) this._cancelWaiting(path);
|
|
441
|
+
this.failOperation(operation, error);
|
|
442
|
+
throw error;
|
|
357
443
|
} finally {
|
|
358
|
-
|
|
359
|
-
releaser();
|
|
444
|
+
this.updateStatus();
|
|
360
445
|
}
|
|
361
446
|
}
|
|
362
447
|
async _takeSnapshot() {
|
|
363
|
-
const
|
|
364
|
-
const
|
|
365
|
-
const
|
|
366
|
-
|
|
367
|
-
|
|
448
|
+
for (const event of this.bufferedQueuedItems) this.retainEvent(event);
|
|
449
|
+
for (const waiting of this._waitingMap.values()) this.retainEvent(waiting.event);
|
|
450
|
+
const save = this.snapshotTail.then(async () => {
|
|
451
|
+
const entries = [...this.pendingSnapshot.values()];
|
|
452
|
+
const completed = entries.filter((entry) => entry.completed);
|
|
453
|
+
const snapshot = entries.filter((entry) => !entry.completed).map((entry) => entry.event);
|
|
454
|
+
await this._saveSnapshot(snapshot);
|
|
455
|
+
for (const entry of completed) this.pendingSnapshot.delete(entry.event);
|
|
456
|
+
this.updateStatus();
|
|
457
|
+
});
|
|
458
|
+
this.snapshotTail = save.catch(() => {
|
|
459
|
+
});
|
|
460
|
+
await save;
|
|
368
461
|
}
|
|
369
462
|
async _restoreFromSnapshot() {
|
|
370
463
|
const snapShot = await this._loadSnapshot();
|
|
@@ -389,55 +482,42 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
389
482
|
* An individual restored operation failure is logged but does not prevent the
|
|
390
483
|
* following Offline Scanner from reconciling the resulting current state.
|
|
391
484
|
*/
|
|
392
|
-
runQueuedEvents(options = {}) {
|
|
393
|
-
const
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
485
|
+
async runQueuedEvents(options = {}) {
|
|
486
|
+
for (const event of this.bufferedQueuedItems) this.retainEvent(event);
|
|
487
|
+
const accepted = this.pendingOperations();
|
|
488
|
+
if (!this.dispatching) {
|
|
489
|
+
const dispatch = async () => {
|
|
490
|
+
while (this.bufferedQueuedItems.length > 0) {
|
|
491
|
+
const event = this.bufferedQueuedItems.shift();
|
|
492
|
+
const entry = this.retainEvent(event);
|
|
493
|
+
if (event.type === TYPE_SENTINEL_FLUSH) {
|
|
494
|
+
const results = await this.waitForOperations(this.pendingOperations(entry.sequence));
|
|
495
|
+
if (results.every((result) => result.ok)) {
|
|
496
|
+
entry.completed = true;
|
|
497
|
+
try {
|
|
498
|
+
await this._takeSnapshot();
|
|
499
|
+
} catch (error) {
|
|
500
|
+
entry.completed = false;
|
|
501
|
+
this.reportOperationFailure(error);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
} else if (!entry.operation.result) {
|
|
505
|
+
fireAndForget(this.processFileEvent(event).catch(() => {
|
|
506
|
+
}));
|
|
408
507
|
}
|
|
409
508
|
}
|
|
410
509
|
};
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
this.updateStatus();
|
|
421
|
-
if (fei.type === TYPE_SENTINEL_FLUSH) {
|
|
422
|
-
this._log(`Waiting for idle`, LOG_LEVEL_VERBOSE);
|
|
423
|
-
if (waitForProcessing) {
|
|
424
|
-
await waitForDispatched();
|
|
425
|
-
}
|
|
426
|
-
await this.waitForIdle();
|
|
427
|
-
this.updateStatus();
|
|
428
|
-
continue;
|
|
429
|
-
}
|
|
430
|
-
const processing = this.processFileEvent(fei);
|
|
431
|
-
if (waitForProcessing) {
|
|
432
|
-
dispatched.push({ path: fei.args.file.path, processing });
|
|
433
|
-
} else {
|
|
434
|
-
fireAndForget(processing);
|
|
435
|
-
}
|
|
436
|
-
} while (this.bufferedQueuedItems.length > 0);
|
|
437
|
-
if (waitForProcessing) {
|
|
438
|
-
await waitForDispatched();
|
|
510
|
+
this.dispatching = dispatch().finally(() => {
|
|
511
|
+
this.dispatching = null;
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
await this.dispatching;
|
|
515
|
+
if (options.waitForProcessing) {
|
|
516
|
+
const results = await this.waitForOperations(accepted);
|
|
517
|
+
for (const result of results) {
|
|
518
|
+
if (result.ok === false) this.reportOperationFailure(result.error);
|
|
439
519
|
}
|
|
440
|
-
}
|
|
520
|
+
}
|
|
441
521
|
}
|
|
442
522
|
processingCount = 0;
|
|
443
523
|
async requestProcessQueue(fei) {
|
|
@@ -445,50 +525,49 @@ class StorageEventManagerBase extends StorageEventManager {
|
|
|
445
525
|
this.processingCount++;
|
|
446
526
|
this.updateStatus();
|
|
447
527
|
await this.handleFileEvent(fei);
|
|
448
|
-
await this._takeSnapshot();
|
|
449
528
|
} finally {
|
|
450
529
|
this.processingCount--;
|
|
451
530
|
this.updateStatus();
|
|
452
531
|
}
|
|
453
532
|
}
|
|
533
|
+
hasPendingFileEvent(filename) {
|
|
534
|
+
for (const entry of this.pendingSnapshot.values()) {
|
|
535
|
+
if (entry.completed || entry.event.type === TYPE_SENTINEL_FLUSH) continue;
|
|
536
|
+
const event = entry.event;
|
|
537
|
+
const relatedPath = event.type === "RENAME" ? event.args.oldPath : event.args.renameTarget;
|
|
538
|
+
if ([event.args.file.path, relatedPath].some((path) => path && (filename === path || filename.startsWith(`${path}/`)))) return true;
|
|
539
|
+
}
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
454
542
|
isWaiting(filename) {
|
|
455
|
-
return
|
|
543
|
+
return this._waitingMap.has(filename);
|
|
456
544
|
}
|
|
457
545
|
async handleFileEvent(queue) {
|
|
458
546
|
const file = queue.args.file;
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
if (queue.
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
await this.fileProcessing.processFileEvent(queue);
|
|
471
|
-
} else {
|
|
472
|
-
if (file.stat.mtime == last) {
|
|
473
|
-
this._log(
|
|
474
|
-
`File has been already scanned on ${queue.type}, skip: ${file.path}`,
|
|
475
|
-
LOG_LEVEL_VERBOSE
|
|
476
|
-
);
|
|
477
|
-
return;
|
|
547
|
+
await serialized(`handleFile:${file.path}`, async () => {
|
|
548
|
+
if (queue.cancelled) throw new Error(`Storage operation has no confirmed result: ${file.path}`);
|
|
549
|
+
if (queue.type === "INTERNAL" || file.isInternal) {
|
|
550
|
+
const handler = this.fileProcessing.processOptionalFileEventWithResult;
|
|
551
|
+
if (handler && handler.hasHandlers !== false) {
|
|
552
|
+
const operation = this.operations.get(queue);
|
|
553
|
+
let whenFlushRequested;
|
|
554
|
+
if (operation) {
|
|
555
|
+
operation.flushSignal ??= promiseWithResolvers();
|
|
556
|
+
if (operation.flushRequested) operation.flushSignal.resolve();
|
|
557
|
+
whenFlushRequested = operation.flushSignal.promise;
|
|
478
558
|
}
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
LOG_LEVEL_INFO
|
|
483
|
-
);
|
|
484
|
-
this.cancelRelativeEvent(queue);
|
|
485
|
-
return;
|
|
559
|
+
const outcome = await handler(file.path, { whenFlushRequested });
|
|
560
|
+
if (outcome.kind === "veto") {
|
|
561
|
+
throw new Error(`Optional storage operation declined: ${file.path} (${outcome.value})`);
|
|
486
562
|
}
|
|
563
|
+
} else {
|
|
564
|
+
await this.fileProcessing.processOptionalFileEvent(file.path);
|
|
487
565
|
}
|
|
566
|
+
} else if (!await this.fileProcessing.processFileEvent(queue)) {
|
|
567
|
+
throw new Error(`Storage operation failed: ${file.path}`);
|
|
488
568
|
}
|
|
489
569
|
});
|
|
490
570
|
this.updateStatus();
|
|
491
|
-
return;
|
|
492
571
|
}
|
|
493
572
|
cancelRelativeEvent(item) {
|
|
494
573
|
this._cancelWaiting(item.args.file.path);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ReplicationStatics } from "../common/models/shared.definition.js";
|
|
1
2
|
import { type EntryDoc, type DatabaseConnectingStatus, type RemoteDBSettings, type TweakValues, type RemotePreferredTweakResult, type NodeData } from "../common/types.js";
|
|
2
3
|
import type { RequiredServices } from "../interfaces/ServiceModule.js";
|
|
3
4
|
import type { TranslationParameters } from "../services/base/MessageTranslator.js";
|
|
@@ -62,6 +63,7 @@ export declare abstract class LiveSyncAbstractReplicator implements ReplicatorIn
|
|
|
62
63
|
constructor(env: LiveSyncReplicatorEnv);
|
|
63
64
|
abstract terminateSync(): void | Promise<void>;
|
|
64
65
|
abstract openReplication(setting: RemoteDBSettings, keepAlive: boolean, showResult: boolean, ignoreCleanLock: boolean): Promise<void | boolean>;
|
|
66
|
+
protected getReplicationStatics(): ReplicationStatics;
|
|
65
67
|
updateInfo: () => void;
|
|
66
68
|
abstract tryConnectRemote(setting: RemoteDBSettings, showResult?: boolean): Promise<boolean>;
|
|
67
69
|
abstract replicateAllToServer(setting: RemoteDBSettings, showingNotice?: boolean): Promise<boolean>;
|
|
@@ -75,8 +75,8 @@ class LiveSyncAbstractReplicator {
|
|
|
75
75
|
constructor(env) {
|
|
76
76
|
this.env = env;
|
|
77
77
|
}
|
|
78
|
-
|
|
79
|
-
|
|
78
|
+
getReplicationStatics() {
|
|
79
|
+
return {
|
|
80
80
|
sent: this.docSent,
|
|
81
81
|
arrived: this.docArrived,
|
|
82
82
|
maxPullSeq: this.maxPullSeq,
|
|
@@ -85,6 +85,9 @@ class LiveSyncAbstractReplicator {
|
|
|
85
85
|
lastSyncPushSeq: this.lastSyncPushSeq,
|
|
86
86
|
syncStatus: this.syncStatus
|
|
87
87
|
};
|
|
88
|
+
}
|
|
89
|
+
updateInfo = () => {
|
|
90
|
+
this.env.services.replicator.replicationStatics.value = this.getReplicationStatics();
|
|
88
91
|
};
|
|
89
92
|
}
|
|
90
93
|
export {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ReplicationStatics } from "../../common/models/shared.definition.js";
|
|
1
2
|
import { type EntryDoc, type RemoteDBSettings, type EntryLeaf, type SyncParameters, type DatabaseEntry, type NodeData, type RemotePreferredTweakResult } from "../../common/types.js";
|
|
2
3
|
import type { InitialFetchFastForwardResource } from "../RemoteResource.js";
|
|
3
4
|
import { LiveSyncAbstractReplicator, type LiveSyncReplicatorEnv, type RemoteDBStatus } from "../LiveSyncAbstractReplicator.js";
|
|
@@ -45,6 +46,11 @@ export declare class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicato
|
|
|
45
46
|
private continuousTask;
|
|
46
47
|
private continuousStopWaiter;
|
|
47
48
|
private continuousStopRequested;
|
|
49
|
+
private continuousPhase?;
|
|
50
|
+
private couchDBProgress?;
|
|
51
|
+
protected getReplicationStatics(): ReplicationStatics;
|
|
52
|
+
private publishPreparationPhase;
|
|
53
|
+
private finishReplicationProgress;
|
|
48
54
|
isMobile(): boolean;
|
|
49
55
|
constructor(env: LiveSyncCouchDBReplicatorEnv);
|
|
50
56
|
/** Own the checkpoint handoff for one fresh, suspended Fast Fetch only. */
|