@vrtmrz/livesync-commonlib 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/API/processSetting.d.ts +1 -0
- package/dist/common/models/setting.const.defaults.js +1 -0
- package/dist/common/models/setting.const.qr.js +1 -0
- package/dist/common/models/setting.policy.d.ts +4 -0
- package/dist/common/models/setting.policy.js +1 -0
- package/dist/common/models/setting.type.d.ts +2 -0
- package/dist/common/models/shared.definition.d.ts +20 -0
- package/dist/common/models/tweak.definition.d.ts +1 -0
- package/dist/common/settingConstants.d.ts +1 -0
- package/dist/common/settingConstants.js +6 -2
- package/dist/interfaces/FileHandler.d.ts +5 -1
- package/dist/interfaces/StorageEventManager.d.ts +5 -0
- package/dist/interfaces/StorageEventManager.js +7 -0
- package/dist/managers/StorageEventManager.d.ts +19 -10
- package/dist/managers/StorageEventManager.js +259 -189
- package/dist/replication/LiveSyncAbstractReplicator.d.ts +2 -0
- package/dist/replication/LiveSyncAbstractReplicator.js +5 -2
- package/dist/replication/couchdb/LiveSyncReplicator.d.ts +6 -0
- package/dist/replication/couchdb/LiveSyncReplicator.js +39 -2
- package/dist/replication/couchdb/replicationProgress.d.ts +6 -0
- package/dist/replication/couchdb/replicationProgress.js +105 -0
- package/dist/replication/index.d.ts +1 -0
- package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
- package/dist/serviceFeatures/offlineScanner.js +19 -7
- package/dist/serviceModules/ServiceFileAccessBase.js +5 -0
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +8 -5
- package/dist/serviceModules/ServiceFileHandlerBase.js +75 -38
- package/dist/services/base/AppLifecycleService.js +1 -1
- package/dist/services/base/ControlService.js +6 -2
- package/dist/services/base/FileProcessingService.d.ts +7 -1
- package/dist/services/base/FileProcessingService.js +9 -1
- package/dist/services/base/IService.d.ts +9 -1
- package/dist/services/base/ReplicatorService.d.ts +1 -0
- package/dist/services/lib/HandlerUtils.d.ts +60 -0
- package/dist/services/lib/HandlerUtils.js +46 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/offline-scanner.md +15 -13
- package/docs/settings-lifecycle.md +2 -0
- package/docs/storage-events-and-reflection.md +109 -10
- package/package.json +2 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { observeReplicationProgress } from "./replicationProgress.js";
|
|
1
2
|
import {
|
|
2
3
|
VER,
|
|
3
4
|
MILESTONE_DOCID,
|
|
@@ -122,6 +123,24 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
122
123
|
continuousTask;
|
|
123
124
|
continuousStopWaiter;
|
|
124
125
|
continuousStopRequested = false;
|
|
126
|
+
continuousPhase;
|
|
127
|
+
couchDBProgress;
|
|
128
|
+
getReplicationStatics() {
|
|
129
|
+
return {
|
|
130
|
+
...super.getReplicationStatics(),
|
|
131
|
+
...this.couchDBProgress ? { couchDB: this.couchDBProgress } : {}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
publishPreparationPhase(phase) {
|
|
135
|
+
this.continuousPhase = phase;
|
|
136
|
+
this.couchDBProgress = { phase };
|
|
137
|
+
this.updateInfo();
|
|
138
|
+
}
|
|
139
|
+
finishReplicationProgress() {
|
|
140
|
+
if (!this.couchDBProgress || this.couchDBProgress.phase === "idle") return;
|
|
141
|
+
this.couchDBProgress = { ...this.couchDBProgress, phase: "idle" };
|
|
142
|
+
this.updateInfo();
|
|
143
|
+
}
|
|
125
144
|
isMobile() {
|
|
126
145
|
return this.env.services.API.isMobile();
|
|
127
146
|
}
|
|
@@ -241,6 +260,7 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
241
260
|
this.continuousStopRequested = true;
|
|
242
261
|
}
|
|
243
262
|
this.abortController();
|
|
263
|
+
this.finishReplicationProgress();
|
|
244
264
|
if (!task) {
|
|
245
265
|
return Promise.resolve();
|
|
246
266
|
}
|
|
@@ -352,6 +372,17 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
352
372
|
this.controller.abort();
|
|
353
373
|
}
|
|
354
374
|
this.controller = controller;
|
|
375
|
+
const progress = observeReplicationProgress(
|
|
376
|
+
syncHandler,
|
|
377
|
+
syncMode,
|
|
378
|
+
this.continuousPhase ?? "one-shot",
|
|
379
|
+
controller.signal,
|
|
380
|
+
(snapshot) => {
|
|
381
|
+
if (this.controller !== controller) return;
|
|
382
|
+
this.couchDBProgress = snapshot;
|
|
383
|
+
this.updateInfo();
|
|
384
|
+
}
|
|
385
|
+
);
|
|
355
386
|
const gen = genReplication(syncHandler, controller.signal);
|
|
356
387
|
try {
|
|
357
388
|
for await (const [type, e] of gen) {
|
|
@@ -361,6 +392,7 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
361
392
|
return "FAILED";
|
|
362
393
|
}
|
|
363
394
|
releaser();
|
|
395
|
+
if (controller.signal.aborted || this.controller !== controller) break;
|
|
364
396
|
switch (type) {
|
|
365
397
|
case "change":
|
|
366
398
|
if ("direction" in e) {
|
|
@@ -440,8 +472,9 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
440
472
|
Logger(ex, LOG_LEVEL_VERBOSE);
|
|
441
473
|
return "FAILED";
|
|
442
474
|
} finally {
|
|
443
|
-
|
|
444
|
-
|
|
475
|
+
progress.dispose();
|
|
476
|
+
controller.abort();
|
|
477
|
+
if (this.controller === controller) this.controller = void 0;
|
|
445
478
|
}
|
|
446
479
|
}
|
|
447
480
|
getEmptyMaxEntry(remoteID) {
|
|
@@ -1118,6 +1151,8 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1118
1151
|
if (this.continuousTask !== task) {
|
|
1119
1152
|
return;
|
|
1120
1153
|
}
|
|
1154
|
+
this.finishReplicationProgress();
|
|
1155
|
+
this.continuousPhase = void 0;
|
|
1121
1156
|
this.continuousTask = void 0;
|
|
1122
1157
|
this.continuousStopWaiter = void 0;
|
|
1123
1158
|
this.continuousStopRequested = false;
|
|
@@ -1131,6 +1166,7 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1131
1166
|
);
|
|
1132
1167
|
return false;
|
|
1133
1168
|
}
|
|
1169
|
+
this.publishPreparationPhase("catch-up");
|
|
1134
1170
|
const localDB = this.rawDatabase;
|
|
1135
1171
|
Logger(this.translate("liveSyncReplicator.beforeLiveSync"));
|
|
1136
1172
|
const caughtUp = await this.env.services.replicator.runFiniteReplicationActivity(
|
|
@@ -1146,6 +1182,7 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1146
1182
|
{ label: "replication" }
|
|
1147
1183
|
);
|
|
1148
1184
|
if (caughtUp && !this.continuousStopRequested) {
|
|
1185
|
+
this.publishPreparationPhase("continuous");
|
|
1149
1186
|
Logger(this.translate("liveSyncReplicator.liveSyncBegin"));
|
|
1150
1187
|
const ret = await this.checkReplicationConnectivity(setting, true, false, showResult);
|
|
1151
1188
|
if (ret === false || this.continuousStopRequested) {
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { EntryDoc } from "../../common/types.js";
|
|
2
|
+
import type { CouchDBReplicationPhase, CouchDBReplicationProgress } from "../../common/models/shared.definition.js";
|
|
3
|
+
/** Observe one handler without adding its high-volume checkpoint events to the replication inbox. */
|
|
4
|
+
export declare function observeReplicationProgress(handler: PouchDB.Replication.Sync<EntryDoc> | PouchDB.Replication.Replication<EntryDoc>, mode: "sync" | "pullOnly" | "pushOnly", phase: Exclude<CouchDBReplicationPhase, "idle">, signal: AbortSignal, publish: (progress: CouchDBReplicationProgress) => void): {
|
|
5
|
+
dispose(): void;
|
|
6
|
+
};
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { compatGlobal } from "../../common/coreEnvFunctions.js";
|
|
2
|
+
const PROGRESS_PUBLICATION_INTERVAL_MS = 200;
|
|
3
|
+
function asEmitter(value) {
|
|
4
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
5
|
+
if (!("on" in value) || typeof value.on !== "function") return void 0;
|
|
6
|
+
if (!("removeListener" in value) || typeof value.removeListener !== "function") return void 0;
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function isSequence(value) {
|
|
10
|
+
return typeof value === "string" || typeof value === "number" && Number.isFinite(value);
|
|
11
|
+
}
|
|
12
|
+
function observeReplicationProgress(handler, mode, phase, signal, publish) {
|
|
13
|
+
const progress = {};
|
|
14
|
+
const detach = [];
|
|
15
|
+
let timer;
|
|
16
|
+
let dirty = false;
|
|
17
|
+
let disposed = false;
|
|
18
|
+
let currentPhase = phase;
|
|
19
|
+
const cancelTimer = () => {
|
|
20
|
+
if (timer === void 0) return;
|
|
21
|
+
compatGlobal.clearTimeout(timer);
|
|
22
|
+
timer = void 0;
|
|
23
|
+
};
|
|
24
|
+
const flush = () => {
|
|
25
|
+
cancelTimer();
|
|
26
|
+
if (!dirty) return;
|
|
27
|
+
dirty = false;
|
|
28
|
+
publish({
|
|
29
|
+
phase: currentPhase,
|
|
30
|
+
...progress.pull ? { pull: { ...progress.pull } } : {},
|
|
31
|
+
...progress.push ? { push: { ...progress.push } } : {}
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
const schedule = () => {
|
|
35
|
+
dirty = true;
|
|
36
|
+
timer ??= compatGlobal.setTimeout(flush, PROGRESS_PUBLICATION_INTERVAL_MS);
|
|
37
|
+
};
|
|
38
|
+
const listen = (emitter, event, listener) => {
|
|
39
|
+
emitter.on(event, listener);
|
|
40
|
+
detach.push(() => emitter.removeListener(event, listener));
|
|
41
|
+
};
|
|
42
|
+
const observeDirection = (direction, emitter) => {
|
|
43
|
+
if (!emitter) return;
|
|
44
|
+
const value = { checkedChanges: 0, completedBatches: 0, checking: false };
|
|
45
|
+
progress[direction] = value;
|
|
46
|
+
listen(emitter, "checkpoint", (event) => {
|
|
47
|
+
if (disposed || signal.aborted || typeof event !== "object" || event === null) return;
|
|
48
|
+
if ("revs_diff" in event && typeof event.revs_diff === "object" && event.revs_diff !== null && "id" in event.revs_diff && typeof event.revs_diff.id === "string") {
|
|
49
|
+
value.checkedChanges++;
|
|
50
|
+
value.checking = true;
|
|
51
|
+
} else if ("start_next_batch" in event && isSequence(event.start_next_batch)) {
|
|
52
|
+
value.checking = true;
|
|
53
|
+
} else if ("checkpoint" in event && isSequence(event.checkpoint)) {
|
|
54
|
+
value.completedBatches++;
|
|
55
|
+
value.lastSeq = event.checkpoint;
|
|
56
|
+
} else {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
schedule();
|
|
60
|
+
});
|
|
61
|
+
const stopChecking = () => {
|
|
62
|
+
if (disposed || signal.aborted) return;
|
|
63
|
+
if (value.checking) {
|
|
64
|
+
value.checking = false;
|
|
65
|
+
dirty = true;
|
|
66
|
+
}
|
|
67
|
+
flush();
|
|
68
|
+
};
|
|
69
|
+
listen(emitter, "change", stopChecking);
|
|
70
|
+
listen(emitter, "paused", stopChecking);
|
|
71
|
+
};
|
|
72
|
+
const dispose = () => {
|
|
73
|
+
if (disposed) return;
|
|
74
|
+
disposed = true;
|
|
75
|
+
cancelTimer();
|
|
76
|
+
for (const remove of detach) remove();
|
|
77
|
+
signal.removeEventListener("abort", dispose);
|
|
78
|
+
currentPhase = "idle";
|
|
79
|
+
if (progress.pull) progress.pull.checking = false;
|
|
80
|
+
if (progress.push) progress.push.checking = false;
|
|
81
|
+
dirty = true;
|
|
82
|
+
flush();
|
|
83
|
+
};
|
|
84
|
+
if (mode === "sync") {
|
|
85
|
+
const duplex = handler;
|
|
86
|
+
observeDirection("pull", asEmitter(duplex.pull));
|
|
87
|
+
observeDirection("push", asEmitter(duplex.push));
|
|
88
|
+
} else {
|
|
89
|
+
observeDirection(mode === "pullOnly" ? "pull" : "push", asEmitter(handler));
|
|
90
|
+
}
|
|
91
|
+
const outer = asEmitter(handler);
|
|
92
|
+
if (outer) {
|
|
93
|
+
listen(outer, "complete", dispose);
|
|
94
|
+
listen(outer, "error", dispose);
|
|
95
|
+
listen(outer, "denied", dispose);
|
|
96
|
+
}
|
|
97
|
+
signal.addEventListener("abort", dispose, { once: true });
|
|
98
|
+
dirty = true;
|
|
99
|
+
if (signal.aborted) dispose();
|
|
100
|
+
else flush();
|
|
101
|
+
return { dispose };
|
|
102
|
+
}
|
|
103
|
+
export {
|
|
104
|
+
observeReplicationProgress
|
|
105
|
+
};
|
|
@@ -12,3 +12,4 @@ export type { CentralCompatibilityRecoveryHint, CentralCompatibilityRejectionRea
|
|
|
12
12
|
export { assessRemoteFeatureDocument, describeRemoteFeatureRejection, ENCRYPTED_INTERNAL_METADATA_FEATURE, NFC_PATH_IDENTITY_FEATURE, REMOTE_FEATURE_GENERATION, usesEncryptedInternalMetadata, } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
13
13
|
export type { RemoteFeatureAssessment } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
14
14
|
export type { JoinedDevice, JoinedDevicesSnapshot, ForgetJoinedDeviceResult } from "./joinedDevices.js";
|
|
15
|
+
export type { ReplicationStatics, CouchDBReplicationPhase, CouchDBReplicationProgress, ReplicationCheckpointProgress, } from "../common/models/shared.definition.js";
|
|
@@ -144,6 +144,7 @@ export declare class TrysteroReplicator {
|
|
|
144
144
|
idDerivationVersion: 0 | 1;
|
|
145
145
|
idDerivationProof: string;
|
|
146
146
|
liveSync?: boolean;
|
|
147
|
+
batchSaveInLiveSync?: boolean;
|
|
147
148
|
syncOnSave?: boolean;
|
|
148
149
|
syncOnStart?: boolean;
|
|
149
150
|
syncOnFileOpen?: boolean;
|
|
@@ -332,6 +333,7 @@ export declare class TrysteroReplicator {
|
|
|
332
333
|
idDerivationVersion: 0 | 1;
|
|
333
334
|
idDerivationProof: string;
|
|
334
335
|
liveSync?: boolean;
|
|
336
|
+
batchSaveInLiveSync?: boolean;
|
|
335
337
|
syncOnSave?: boolean;
|
|
336
338
|
syncOnStart?: boolean;
|
|
337
339
|
syncOnFileOpen?: boolean;
|
|
@@ -360,8 +360,7 @@ async function syncFileBetweenDBandStorage(host, log, file, doc) {
|
|
|
360
360
|
case BASE_IS_NEW:
|
|
361
361
|
if (!host.services.vault.isFileSizeTooLarge(file.stat.size)) {
|
|
362
362
|
log("STORAGE -> DB :" + file.path);
|
|
363
|
-
await host.serviceModules.fileHandler.storeFileToDB(file);
|
|
364
|
-
return FilePairProcessResults.COMPLETED;
|
|
363
|
+
return await host.serviceModules.fileHandler.storeFileToDB(file) ? FilePairProcessResults.COMPLETED : FilePairProcessResults.FAILED;
|
|
365
364
|
} else {
|
|
366
365
|
log(
|
|
367
366
|
`STORAGE -> DB : ${file.path} has been skipped due to file size exceeding the limit`,
|
|
@@ -390,11 +389,22 @@ async function syncFileBetweenDBandStorage(host, log, file, doc) {
|
|
|
390
389
|
return FilePairProcessResults.SKIPPED;
|
|
391
390
|
}
|
|
392
391
|
case EVEN:
|
|
393
|
-
if (doc.
|
|
394
|
-
|
|
392
|
+
if (host.services.vault.isFileSizeTooLarge(file.stat.size) || host.services.vault.isFileSizeTooLarge(doc.size)) {
|
|
393
|
+
log(`STORAGE ? DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
|
|
394
|
+
return FilePairProcessResults.SKIPPED;
|
|
395
|
+
}
|
|
396
|
+
if (doc._rev && await host.serviceModules.fileHandler.tryRecordUntrackedFileRevision(file, doc._rev)) {
|
|
397
|
+
log("STORAGE == DB :" + file.path, LOG_LEVEL_DEBUG);
|
|
398
|
+
return FilePairProcessResults.COMPLETED;
|
|
399
|
+
}
|
|
400
|
+
log("STORAGE ? DB :" + file.path + " Checking content", LOG_LEVEL_DEBUG);
|
|
401
|
+
try {
|
|
402
|
+
return await host.serviceModules.fileHandler.storeFileToDB(file, false, false, { revalidateStorage: true }) ? FilePairProcessResults.COMPLETED : FilePairProcessResults.FAILED;
|
|
403
|
+
} catch (error) {
|
|
404
|
+
log(`STORAGE ? DB : Could not reconcile ${file.path}`, LOG_LEVEL_NOTICE);
|
|
405
|
+
log(error, LOG_LEVEL_VERBOSE);
|
|
406
|
+
return FilePairProcessResults.FAILED;
|
|
395
407
|
}
|
|
396
|
-
log("STORAGE == DB :" + file.path, LOG_LEVEL_DEBUG);
|
|
397
|
-
return FilePairProcessResults.COMPLETED;
|
|
398
408
|
default:
|
|
399
409
|
log("STORAGE ?? DB :" + file.path + " Something got weird");
|
|
400
410
|
return FilePairProcessResults.FAILED;
|
|
@@ -501,7 +511,9 @@ async function collectDatabaseFiles(host, settings, log, showingNotice) {
|
|
|
501
511
|
async function updateToDatabase(host, log, logLevel, file) {
|
|
502
512
|
if (!host.services.vault.isFileSizeTooLarge(file.stat.size)) {
|
|
503
513
|
const path = file.path;
|
|
504
|
-
await host.serviceModules.fileHandler.storeFileToDB(file)
|
|
514
|
+
if (!await host.serviceModules.fileHandler.storeFileToDB(file)) {
|
|
515
|
+
return FilePairProcessResults.FAILED;
|
|
516
|
+
}
|
|
505
517
|
host.services.context.events.emitEvent("event-file-changed", { file: path, automated: true });
|
|
506
518
|
return FilePairProcessResults.COMPLETED;
|
|
507
519
|
} else {
|
|
@@ -15,6 +15,11 @@ class ServiceFileAccessBase extends ServiceModuleBase {
|
|
|
15
15
|
this.vaultAccess = services.vaultAccess;
|
|
16
16
|
services.appLifecycle.onFirstInitialise.addHandler(this._everyOnFirstInitialize.bind(this));
|
|
17
17
|
services.fileProcessing.commitPendingFileEvents.addHandler(this._everyCommitPendingFileEvent.bind(this));
|
|
18
|
+
services.appLifecycle.onBeforeUnload.addHandler(async () => {
|
|
19
|
+
if (this.vaultManager.stopWatching) await this.vaultManager.stopWatching();
|
|
20
|
+
else await this.vaultManager.waitForIdle();
|
|
21
|
+
return true;
|
|
22
|
+
});
|
|
18
23
|
}
|
|
19
24
|
restoreState() {
|
|
20
25
|
return this.vaultManager.restoreState();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AnyEntry, FilePath, FilePathWithPrefix, MetaEntry, UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../common/types.js";
|
|
2
2
|
import type { LiveSyncEventHub } from "../hub/hub.js";
|
|
3
|
-
import type { IFileHandler } from "../interfaces/FileHandler.js";
|
|
3
|
+
import type { IFileHandler, StoreFileOptions } from "../interfaces/FileHandler.js";
|
|
4
4
|
import { ServiceModuleBase } from "./ServiceModuleBase.js";
|
|
5
5
|
import type { APIService } from "../services/base/APIService.js";
|
|
6
6
|
import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess.js";
|
|
@@ -74,7 +74,8 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
74
74
|
/**
|
|
75
75
|
* Read the raw provenance value used by an optimistic storage snapshot.
|
|
76
76
|
* Unlike `getProvenance`, this does not validate or remove the referenced
|
|
77
|
-
* database revision
|
|
77
|
+
* database revision. Recovery callers require a readable value so a store
|
|
78
|
+
* failure cannot be mistaken for absent provenance.
|
|
78
79
|
*/
|
|
79
80
|
private readProvenanceSnapshot;
|
|
80
81
|
/**
|
|
@@ -99,7 +100,7 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
99
100
|
readFileFromStub(file: UXFileInfoStub | UXFileInfo): Promise<UXFileInfo>;
|
|
100
101
|
tryRecordUntrackedFileRevision(info: UXFileInfoStub | UXFileInfo, expectedRevision: string): Promise<boolean>;
|
|
101
102
|
private infoToStub;
|
|
102
|
-
storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean): Promise<boolean>;
|
|
103
|
+
storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean, options?: StoreFileOptions): Promise<boolean>;
|
|
103
104
|
storeFileToDBWithBaseRevision(info: UXFileInfoStub | UXFileInfo | FilePathWithPrefix, baseRevision: string, createIfDifferent?: boolean): Promise<boolean>;
|
|
104
105
|
/**
|
|
105
106
|
* Store current storage content below an exact live revision. A differing
|
|
@@ -110,11 +111,13 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
110
111
|
/**
|
|
111
112
|
* Save one storage snapshot while the caller holds its document lock.
|
|
112
113
|
* Ordinary saves read once inside the lock and leave retries disabled.
|
|
113
|
-
* Restored
|
|
114
|
-
*
|
|
114
|
+
* Restored start-up events and recovery scans retain bounded rereads
|
|
115
|
+
* because a watcher event may not cover a concurrent edit.
|
|
115
116
|
*/
|
|
116
117
|
private storeFileToDBFromRevision;
|
|
117
118
|
deleteFileFromDB(info: UXFileInfoStub | UXInternalFileInfoStub | FilePath): Promise<boolean>;
|
|
119
|
+
/** Complete a validated storage deletion when its database effect is already represented. */
|
|
120
|
+
private deleteFileFromDBIfPresent;
|
|
118
121
|
/**
|
|
119
122
|
* Record a storage deletion under the caller's document lock, preserving
|
|
120
123
|
* the displayed conflict branch when its provenance is known.
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { LOG_LEVEL_INFO, LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
|
|
2
2
|
import { serialized } from "octagonal-wheels/concurrency/lock";
|
|
3
3
|
import {
|
|
4
|
+
createBlob,
|
|
4
5
|
getDocDataAsArray,
|
|
5
6
|
isDocContentSame,
|
|
6
|
-
readAsBlob,
|
|
7
7
|
readContent
|
|
8
8
|
} from "../common/utils.js";
|
|
9
9
|
import { computeFileContentHash } from "../common/fileContentHash.js";
|
|
@@ -158,7 +158,7 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
158
158
|
* A valid saved content hash can prove unchanged bytes after revision
|
|
159
159
|
* pruning. It cannot reconstruct ancestry for an edited file.
|
|
160
160
|
*/
|
|
161
|
-
async classifyStorageContent(file, currentEntry, preferredPath, recorded, currentLoaded) {
|
|
161
|
+
async classifyStorageContent(file, currentEntry, preferredPath, recorded, currentLoaded, requireReadableContent = false) {
|
|
162
162
|
const path = preferredPath ?? file.path;
|
|
163
163
|
const recordedRevision = recorded?.revision;
|
|
164
164
|
if (recordedRevision) {
|
|
@@ -173,6 +173,9 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
173
173
|
}
|
|
174
174
|
if (!base) {
|
|
175
175
|
const meta = await this.db.fetchEntryMeta(file, recordedRevision, true);
|
|
176
|
+
if (requireReadableContent && meta && !meta.deleted && !meta._deleted) {
|
|
177
|
+
throw new Error(`Could not read the recorded file revision for ${file.path}`);
|
|
178
|
+
}
|
|
176
179
|
if ((!meta || meta.deleted || meta._deleted) && !hasContentHash(recorded)) await this.deleteProvenance(path);
|
|
177
180
|
}
|
|
178
181
|
}
|
|
@@ -189,13 +192,15 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
189
192
|
/**
|
|
190
193
|
* Read the raw provenance value used by an optimistic storage snapshot.
|
|
191
194
|
* Unlike `getProvenance`, this does not validate or remove the referenced
|
|
192
|
-
* database revision
|
|
195
|
+
* database revision. Recovery callers require a readable value so a store
|
|
196
|
+
* failure cannot be mistaken for absent provenance.
|
|
193
197
|
*/
|
|
194
|
-
async readProvenanceSnapshot(path) {
|
|
198
|
+
async readProvenanceSnapshot(path, requireReadable = false) {
|
|
195
199
|
try {
|
|
196
200
|
const record = await this.fileReflectionProvenance?.get(this.getProvenancePath(path));
|
|
197
201
|
return record ? { ...record } : void 0;
|
|
198
|
-
} catch {
|
|
202
|
+
} catch (error) {
|
|
203
|
+
if (requireReadable) throw error;
|
|
199
204
|
return void 0;
|
|
200
205
|
}
|
|
201
206
|
}
|
|
@@ -203,20 +208,20 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
203
208
|
* Check that storage still has the same path, bytes, timestamp, and
|
|
204
209
|
* provenance as a previously captured snapshot.
|
|
205
210
|
*/
|
|
206
|
-
async currentStorageSnapshotMatches(file, revision) {
|
|
211
|
+
async currentStorageSnapshotMatches(file, revision, requireReadableProvenance = false) {
|
|
207
212
|
const current = await this.storage.getStub(file.path);
|
|
208
213
|
if (!current || isFolderInfo(current) || current.path !== file.path) return false;
|
|
209
214
|
const readCurrent = await this.readFileFromStub(current);
|
|
210
|
-
return readCurrent.stat.mtime === file.stat.mtime && await isDocContentSame(readCurrent.body, file.body) && sameProvenance(await this.readProvenanceSnapshot(file.path), revision);
|
|
215
|
+
return readCurrent.stat.mtime === file.stat.mtime && await isDocContentSame(readCurrent.body, file.body) && sameProvenance(await this.readProvenanceSnapshot(file.path, requireReadableProvenance), revision);
|
|
211
216
|
}
|
|
212
217
|
/**
|
|
213
218
|
* Publish provenance after a durable database write if storage is unchanged.
|
|
214
219
|
* A failed revalidation leaves the database revision stored and asks the
|
|
215
220
|
* caller to reread storage before proceeding.
|
|
216
221
|
*/
|
|
217
|
-
async finaliseStoredRevision(file, observedProvenance, storedRevision) {
|
|
222
|
+
async finaliseStoredRevision(file, observedProvenance, storedRevision, requireReadableProvenance = false) {
|
|
218
223
|
const evidence = await this.contentEvidence(file.body);
|
|
219
|
-
if (await this.currentStorageSnapshotMatches(file, observedProvenance)) {
|
|
224
|
+
if (await this.currentStorageSnapshotMatches(file, observedProvenance, requireReadableProvenance)) {
|
|
220
225
|
await this.setProvenance(file.path, storedRevision, file.stat.mtime, file.body, evidence);
|
|
221
226
|
return true;
|
|
222
227
|
} else {
|
|
@@ -293,10 +298,10 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
293
298
|
const file = typeof info === "string" ? await this.storage.getFileStub(info) : info;
|
|
294
299
|
return file;
|
|
295
300
|
}
|
|
296
|
-
async storeFileToDB(info, force = false, onlyChunks = false) {
|
|
301
|
+
async storeFileToDB(info, force = false, onlyChunks = false, options = {}) {
|
|
297
302
|
return await this.serializedByFileEventPaths(
|
|
298
303
|
[getStoragePathFromUXFileInfo(info)],
|
|
299
|
-
() => this.storeFileToDBFromRevision(info, force, onlyChunks)
|
|
304
|
+
() => this.storeFileToDBFromRevision(info, force, onlyChunks, void 0, options.revalidateStorage)
|
|
300
305
|
);
|
|
301
306
|
}
|
|
302
307
|
async storeFileToDBWithBaseRevision(info, baseRevision, createIfDifferent = true) {
|
|
@@ -366,8 +371,8 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
366
371
|
/**
|
|
367
372
|
* Save one storage snapshot while the caller holds its document lock.
|
|
368
373
|
* Ordinary saves read once inside the lock and leave retries disabled.
|
|
369
|
-
* Restored
|
|
370
|
-
*
|
|
374
|
+
* Restored start-up events and recovery scans retain bounded rereads
|
|
375
|
+
* because a watcher event may not cover a concurrent edit.
|
|
371
376
|
*/
|
|
372
377
|
async storeFileToDBFromRevision(info, force = false, onlyChunks = false, preferredBasePath, retryIfStorageChanges = false, snapshotRetry = 0) {
|
|
373
378
|
const file = await this.infoToStub(info);
|
|
@@ -387,32 +392,36 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
387
392
|
return await this.db.createChunks(readFile2, force, true);
|
|
388
393
|
}
|
|
389
394
|
const observedProvenance = await this.readProvenanceSnapshot(
|
|
390
|
-
preferredBasePath ?? file.path
|
|
395
|
+
preferredBasePath ?? file.path,
|
|
396
|
+
retryIfStorageChanges
|
|
391
397
|
);
|
|
392
398
|
const readFile = await this.readFileFromStub(file);
|
|
393
399
|
const entry = await this.db.fetchEntry(file, void 0, true, true, true);
|
|
394
400
|
const currentEntry = entry || await this.db.fetchEntryMeta(file, void 0, true);
|
|
395
401
|
const conflictedRevs = await this.db.getConflictedRevs(file);
|
|
396
402
|
const isConflicted = conflictedRevs.length > 0;
|
|
397
|
-
|
|
403
|
+
const currentIsLive = currentEntry && !currentEntry.deleted && !currentEntry._deleted;
|
|
404
|
+
if (!force && (currentIsLive || retryIfStorageChanges)) {
|
|
405
|
+
if (retryIfStorageChanges && currentIsLive && !entry) return false;
|
|
398
406
|
const classification = await this.classifyStorageContent(
|
|
399
407
|
readFile,
|
|
400
408
|
currentEntry,
|
|
401
409
|
preferredBasePath,
|
|
402
410
|
observedProvenance,
|
|
403
|
-
entry ? { _rev: entry._rev, data: readContent(entry) } : void 0
|
|
411
|
+
entry ? { _rev: entry._rev, data: readContent(entry) } : void 0,
|
|
412
|
+
retryIfStorageChanges
|
|
404
413
|
);
|
|
405
|
-
const upgradeCurrentRecord = classification.kind === "unchanged" && !isConflicted && classification.revision === currentEntry._rev && !hasContentHash(observedProvenance);
|
|
414
|
+
const upgradeCurrentRecord = classification.kind === "unchanged" && !isConflicted && currentEntry && !currentEntry.deleted && !currentEntry._deleted && classification.revision === currentEntry._rev && !hasContentHash(observedProvenance);
|
|
406
415
|
const evidence = upgradeCurrentRecord || classification.kind === "matching-leaf" && classification.revision ? await this.contentEvidence(readFile.body) : void 0;
|
|
407
|
-
const snapshotMatches = retryIfStorageChanges || upgradeCurrentRecord ? await this.currentStorageSnapshotMatches(readFile, observedProvenance) : true;
|
|
416
|
+
const snapshotMatches = retryIfStorageChanges || upgradeCurrentRecord ? await this.currentStorageSnapshotMatches(readFile, observedProvenance, retryIfStorageChanges) : true;
|
|
408
417
|
if (retryIfStorageChanges && !snapshotMatches) {
|
|
409
418
|
return await this.retryCurrentStorage(file.path, preferredBasePath, snapshotRetry);
|
|
410
419
|
}
|
|
411
420
|
if (classification.kind === "unchanged") {
|
|
412
421
|
if (isConflicted) {
|
|
413
422
|
await this.queueConflictCheckFor(file.path);
|
|
414
|
-
} else if (classification.revision !== currentEntry._rev) {
|
|
415
|
-
if (!
|
|
423
|
+
} else if (!currentEntry || currentEntry.deleted || currentEntry._deleted || classification.revision !== currentEntry._rev) {
|
|
424
|
+
if (!currentEntry) return false;
|
|
416
425
|
return await this.dbToStorageCore(file.path, file);
|
|
417
426
|
} else {
|
|
418
427
|
if (upgradeCurrentRecord && snapshotMatches) {
|
|
@@ -430,7 +439,7 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
430
439
|
const storedRevision2 = classification.kind === "edited" ? await this.db.storeWithBaseRevision(readFile, classification.revision, true) : await this.db.storeIndependentRevision(readFile, true);
|
|
431
440
|
if (storedRevision2 === false) return false;
|
|
432
441
|
if (preferredBasePath && preferredBasePath !== file.path) await this.deleteProvenance(preferredBasePath);
|
|
433
|
-
if (retryIfStorageChanges && !await this.finaliseStoredRevision(readFile, observedProvenance, storedRevision2)) {
|
|
442
|
+
if (retryIfStorageChanges && !await this.finaliseStoredRevision(readFile, observedProvenance, storedRevision2, true)) {
|
|
434
443
|
return await this.retryCurrentStorage(file.path, preferredBasePath, snapshotRetry);
|
|
435
444
|
}
|
|
436
445
|
if (!retryIfStorageChanges) await this.setProvenance(file.path, storedRevision2, readFile.stat.mtime, readFile.body);
|
|
@@ -519,6 +528,12 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
519
528
|
() => this.deleteFileFromDBCore(info)
|
|
520
529
|
);
|
|
521
530
|
}
|
|
531
|
+
/** Complete a validated storage deletion when its database effect is already represented. */
|
|
532
|
+
async deleteFileFromDBIfPresent(info) {
|
|
533
|
+
const entry = await this.db.fetchEntryMeta(info, void 0, true);
|
|
534
|
+
if (!entry || entry.deleted || entry._deleted) return true;
|
|
535
|
+
return await this.deleteFileFromDBCore(info);
|
|
536
|
+
}
|
|
522
537
|
/**
|
|
523
538
|
* Record a storage deletion under the caller's document lock, preserving
|
|
524
539
|
* the displayed conflict branch when its provenance is known.
|
|
@@ -892,17 +907,17 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
892
907
|
this._log(`File ${path} is not exist on the database`, LOG_LEVEL_VERBOSE);
|
|
893
908
|
return false;
|
|
894
909
|
}
|
|
910
|
+
const docData = readContent(docRead);
|
|
911
|
+
const reflectedContent = createBlob(docData);
|
|
895
912
|
if (!settings.processSizeMismatchedFiles) {
|
|
896
|
-
if (docRead.size != 0 && docRead.size !==
|
|
913
|
+
if (docRead.size != 0 && docRead.size !== reflectedContent.size) {
|
|
897
914
|
this._log(
|
|
898
|
-
`File ${path} seems to be corrupted! Writing prevented. (${docRead.size} != ${
|
|
915
|
+
`File ${path} seems to be corrupted! Writing prevented. (${docRead.size} != ${reflectedContent.size})`,
|
|
899
916
|
LOG_LEVEL_NOTICE
|
|
900
917
|
);
|
|
901
918
|
return false;
|
|
902
919
|
}
|
|
903
920
|
}
|
|
904
|
-
const docData = readContent(docRead);
|
|
905
|
-
const reflectedContent = readAsBlob(docRead);
|
|
906
921
|
const reflectedEvidence = await this.contentEvidence(reflectedContent);
|
|
907
922
|
if (allowExistingConflicts && existDoc && !force) {
|
|
908
923
|
const readFile = await this.readFileFromStub(existDoc);
|
|
@@ -1080,14 +1095,23 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1080
1095
|
const relatedPath = type === "RENAME" ? eventItem.oldPath : type === "DELETE" ? eventItem.renameTarget : void 0;
|
|
1081
1096
|
const eventPaths = relatedPath === void 0 ? [path] : [path, relatedPath];
|
|
1082
1097
|
return await this.serializedByFileEventPaths(eventPaths, async () => {
|
|
1083
|
-
if (!await this.isCurrentPathSelected(path))
|
|
1098
|
+
if (!await this.isCurrentPathSelected(path)) {
|
|
1099
|
+
return shouldBeIgnored(path) || !await this.vault.isTargetFile(path, { skipCaseCollisionCheck: true });
|
|
1100
|
+
}
|
|
1084
1101
|
switch (type) {
|
|
1085
1102
|
case "CREATE":
|
|
1086
|
-
case "CHANGED":
|
|
1087
|
-
|
|
1103
|
+
case "CHANGED": {
|
|
1104
|
+
const current = this.getExactCurrentFile(await this.storage.getStub(path), path);
|
|
1105
|
+
if (!current || this.vault.isFileSizeTooLarge(current.stat.size)) return true;
|
|
1106
|
+
return await this.storeFileToDBFromRevision(current);
|
|
1107
|
+
}
|
|
1088
1108
|
case "DELETE":
|
|
1089
|
-
|
|
1090
|
-
|
|
1109
|
+
try {
|
|
1110
|
+
if (!await this.canRecordStorageDeletion(item)) return true;
|
|
1111
|
+
} catch {
|
|
1112
|
+
return false;
|
|
1113
|
+
}
|
|
1114
|
+
return await this.deleteFileFromDBIfPresent(item.args.file);
|
|
1091
1115
|
case "RENAME":
|
|
1092
1116
|
return await this.renameFileInDBCore(
|
|
1093
1117
|
item.args.file,
|
|
@@ -1137,7 +1161,7 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1137
1161
|
} catch (ex) {
|
|
1138
1162
|
this._log(`Could not validate deletion of ${path}; preserved the database entry`, LOG_LEVEL_NOTICE);
|
|
1139
1163
|
this._log(ex, LOG_LEVEL_VERBOSE);
|
|
1140
|
-
|
|
1164
|
+
throw ex;
|
|
1141
1165
|
}
|
|
1142
1166
|
}
|
|
1143
1167
|
/**
|
|
@@ -1222,15 +1246,19 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1222
1246
|
action = await this._planRestoredFileEvent(item, isSameDocument);
|
|
1223
1247
|
} catch (ex) {
|
|
1224
1248
|
this.logRestoredEventValidationFailure(path, ex);
|
|
1225
|
-
return
|
|
1249
|
+
return false;
|
|
1226
1250
|
}
|
|
1227
1251
|
switch (action.kind) {
|
|
1228
1252
|
case "none":
|
|
1229
1253
|
return true;
|
|
1230
|
-
case "
|
|
1231
|
-
return
|
|
1254
|
+
case "unresolved":
|
|
1255
|
+
return false;
|
|
1256
|
+
case "store": {
|
|
1257
|
+
const stored = await this.storeFileToDBFromRevision(action.file, false, false, void 0, true);
|
|
1258
|
+
return stored && action.complete !== false;
|
|
1259
|
+
}
|
|
1232
1260
|
case "delete":
|
|
1233
|
-
return await this.
|
|
1261
|
+
return await this.deleteFileFromDBIfPresent(action.path);
|
|
1234
1262
|
case "rename":
|
|
1235
1263
|
return await this.renameFileInDBCore(action.file, action.oldPath, true);
|
|
1236
1264
|
}
|
|
@@ -1266,6 +1294,10 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1266
1294
|
if (!currentTarget2 || !await this.isCurrentFileSelected(currentTarget2)) {
|
|
1267
1295
|
return { kind: "none" };
|
|
1268
1296
|
}
|
|
1297
|
+
const entry = await this.db.fetchEntryMeta(newPath, void 0, true);
|
|
1298
|
+
if (entry && !entry.deleted && !entry._deleted && this.storage.normalisePath(this.getPath(entry)) === this.storage.normalisePath(newPath)) {
|
|
1299
|
+
return { kind: "store", file: currentTarget2 };
|
|
1300
|
+
}
|
|
1269
1301
|
return { kind: "rename", file: currentTarget2, oldPath };
|
|
1270
1302
|
}
|
|
1271
1303
|
const currentTargetItem = await this.storage.getStub(newPath);
|
|
@@ -1283,11 +1315,16 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1283
1315
|
return { kind: "none" };
|
|
1284
1316
|
}
|
|
1285
1317
|
if (!sourceWasInspected || currentSourceItem !== null || !await this.canApplyRestoredDeletion(oldPath)) {
|
|
1318
|
+
return { kind: "store", file: currentTarget, complete: sourceWasInspected };
|
|
1319
|
+
}
|
|
1320
|
+
const sourceEntry = await this.db.fetchEntryMeta(oldPath, void 0, true);
|
|
1321
|
+
if (!sourceEntry || sourceEntry.deleted || sourceEntry._deleted) {
|
|
1286
1322
|
return { kind: "store", file: currentTarget };
|
|
1287
1323
|
}
|
|
1288
1324
|
return { kind: "rename", file: currentTarget, oldPath };
|
|
1289
1325
|
}
|
|
1290
|
-
if (!sourceWasInspected
|
|
1326
|
+
if (!sourceWasInspected) return { kind: "unresolved" };
|
|
1327
|
+
if (currentSourceItem !== null || !await this.canApplyRestoredDeletion(oldPath)) {
|
|
1291
1328
|
return { kind: "none" };
|
|
1292
1329
|
}
|
|
1293
1330
|
return { kind: "delete", path: oldPath };
|
|
@@ -1324,12 +1361,12 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
1324
1361
|
return await this.isCurrentPathSelected(path);
|
|
1325
1362
|
} catch (ex) {
|
|
1326
1363
|
this.logRestoredEventValidationFailure(path, ex);
|
|
1327
|
-
|
|
1364
|
+
throw ex;
|
|
1328
1365
|
}
|
|
1329
1366
|
}
|
|
1330
1367
|
logRestoredEventValidationFailure(path, ex) {
|
|
1331
1368
|
this._log(
|
|
1332
|
-
`Could not validate the saved storage operation for ${path} against current storage; the
|
|
1369
|
+
`Could not validate the saved storage operation for ${path} against current storage; the operation remains pending for recovery`,
|
|
1333
1370
|
LOG_LEVEL_NOTICE
|
|
1334
1371
|
);
|
|
1335
1372
|
this._log(ex, LOG_LEVEL_VERBOSE);
|
|
@@ -60,7 +60,7 @@ class AppLifecycleService extends ServiceBase {
|
|
|
60
60
|
* Event triggered before the plug-in is unloaded.
|
|
61
61
|
* This is typically used to perform any necessary cleanup or save state before the plug-in is unloaded.
|
|
62
62
|
*/
|
|
63
|
-
onBeforeUnload = handlers().
|
|
63
|
+
onBeforeUnload = handlers().allSettled("onBeforeUnload");
|
|
64
64
|
/**
|
|
65
65
|
* Event triggered when the plug-in is being unloaded.
|
|
66
66
|
*
|