@vrtmrz/livesync-commonlib 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -0
- package/dist/API/processSetting.d.ts +2 -0
- package/dist/common/models/setting.const.defaults.js +2 -0
- package/dist/common/models/setting.const.qr.js +3 -0
- package/dist/common/models/setting.policy.d.ts +8 -0
- package/dist/common/models/setting.policy.js +2 -0
- package/dist/common/models/setting.type.d.ts +6 -0
- package/dist/common/models/shared.definition.d.ts +20 -0
- package/dist/common/models/tweak.definition.d.ts +2 -0
- package/dist/common/settingConstants.d.ts +2 -0
- package/dist/common/settingConstants.js +10 -2
- package/dist/interfaces/FileHandler.d.ts +7 -1
- package/dist/interfaces/StorageAccess.d.ts +2 -0
- package/dist/interfaces/StorageEventManager.d.ts +7 -0
- package/dist/interfaces/StorageEventManager.js +11 -0
- package/dist/managers/StorageEventManager.d.ts +20 -10
- package/dist/managers/StorageEventManager.js +268 -189
- package/dist/replication/LiveSyncAbstractReplicator.d.ts +2 -0
- package/dist/replication/LiveSyncAbstractReplicator.js +5 -2
- package/dist/replication/couchdb/LiveSyncReplicator.d.ts +6 -0
- package/dist/replication/couchdb/LiveSyncReplicator.js +39 -2
- package/dist/replication/couchdb/replicationProgress.d.ts +6 -0
- package/dist/replication/couchdb/replicationProgress.js +105 -0
- package/dist/replication/index.d.ts +1 -0
- package/dist/replication/trystero/TrysteroReplicator.d.ts +4 -0
- package/dist/serviceFeatures/offlineScanner.d.ts +4 -4
- package/dist/serviceFeatures/offlineScanner.js +30 -13
- package/dist/serviceFeatures/prepareDatabaseForUse.d.ts +3 -2
- package/dist/serviceFeatures/prepareDatabaseForUse.js +5 -4
- package/dist/serviceModules/ServiceFileAccessBase.d.ts +1 -0
- package/dist/serviceModules/ServiceFileAccessBase.js +8 -0
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +9 -5
- package/dist/serviceModules/ServiceFileHandlerBase.js +96 -38
- package/dist/services/base/AppLifecycleService.js +1 -1
- package/dist/services/base/ControlService.js +6 -2
- package/dist/services/base/DatabaseEventService.d.ts +1 -1
- package/dist/services/base/FileProcessingService.d.ts +7 -1
- package/dist/services/base/FileProcessingService.js +9 -1
- package/dist/services/base/IService.d.ts +12 -4
- package/dist/services/base/ReplicatorService.d.ts +1 -0
- package/dist/services/base/VaultScanResult.d.ts +5 -0
- package/dist/services/base/VaultService.d.ts +1 -1
- package/dist/services/lib/HandlerUtils.d.ts +60 -0
- package/dist/services/lib/HandlerUtils.js +46 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/offline-scanner.md +23 -13
- package/docs/settings-lifecycle.md +2 -0
- package/docs/storage-events-and-reflection.md +113 -10
- package/package.json +2 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { observeReplicationProgress } from "./replicationProgress.js";
|
|
1
2
|
import {
|
|
2
3
|
VER,
|
|
3
4
|
MILESTONE_DOCID,
|
|
@@ -122,6 +123,24 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
122
123
|
continuousTask;
|
|
123
124
|
continuousStopWaiter;
|
|
124
125
|
continuousStopRequested = false;
|
|
126
|
+
continuousPhase;
|
|
127
|
+
couchDBProgress;
|
|
128
|
+
getReplicationStatics() {
|
|
129
|
+
return {
|
|
130
|
+
...super.getReplicationStatics(),
|
|
131
|
+
...this.couchDBProgress ? { couchDB: this.couchDBProgress } : {}
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
publishPreparationPhase(phase) {
|
|
135
|
+
this.continuousPhase = phase;
|
|
136
|
+
this.couchDBProgress = { phase };
|
|
137
|
+
this.updateInfo();
|
|
138
|
+
}
|
|
139
|
+
finishReplicationProgress() {
|
|
140
|
+
if (!this.couchDBProgress || this.couchDBProgress.phase === "idle") return;
|
|
141
|
+
this.couchDBProgress = { ...this.couchDBProgress, phase: "idle" };
|
|
142
|
+
this.updateInfo();
|
|
143
|
+
}
|
|
125
144
|
isMobile() {
|
|
126
145
|
return this.env.services.API.isMobile();
|
|
127
146
|
}
|
|
@@ -241,6 +260,7 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
241
260
|
this.continuousStopRequested = true;
|
|
242
261
|
}
|
|
243
262
|
this.abortController();
|
|
263
|
+
this.finishReplicationProgress();
|
|
244
264
|
if (!task) {
|
|
245
265
|
return Promise.resolve();
|
|
246
266
|
}
|
|
@@ -352,6 +372,17 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
352
372
|
this.controller.abort();
|
|
353
373
|
}
|
|
354
374
|
this.controller = controller;
|
|
375
|
+
const progress = observeReplicationProgress(
|
|
376
|
+
syncHandler,
|
|
377
|
+
syncMode,
|
|
378
|
+
this.continuousPhase ?? "one-shot",
|
|
379
|
+
controller.signal,
|
|
380
|
+
(snapshot) => {
|
|
381
|
+
if (this.controller !== controller) return;
|
|
382
|
+
this.couchDBProgress = snapshot;
|
|
383
|
+
this.updateInfo();
|
|
384
|
+
}
|
|
385
|
+
);
|
|
355
386
|
const gen = genReplication(syncHandler, controller.signal);
|
|
356
387
|
try {
|
|
357
388
|
for await (const [type, e] of gen) {
|
|
@@ -361,6 +392,7 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
361
392
|
return "FAILED";
|
|
362
393
|
}
|
|
363
394
|
releaser();
|
|
395
|
+
if (controller.signal.aborted || this.controller !== controller) break;
|
|
364
396
|
switch (type) {
|
|
365
397
|
case "change":
|
|
366
398
|
if ("direction" in e) {
|
|
@@ -440,8 +472,9 @@ class LiveSyncCouchDBReplicator extends LiveSyncAbstractReplicator {
|
|
|
440
472
|
Logger(ex, LOG_LEVEL_VERBOSE);
|
|
441
473
|
return "FAILED";
|
|
442
474
|
} finally {
|
|
443
|
-
|
|
444
|
-
|
|
475
|
+
progress.dispose();
|
|
476
|
+
controller.abort();
|
|
477
|
+
if (this.controller === controller) this.controller = void 0;
|
|
445
478
|
}
|
|
446
479
|
}
|
|
447
480
|
getEmptyMaxEntry(remoteID) {
|
|
@@ -1118,6 +1151,8 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1118
1151
|
if (this.continuousTask !== task) {
|
|
1119
1152
|
return;
|
|
1120
1153
|
}
|
|
1154
|
+
this.finishReplicationProgress();
|
|
1155
|
+
this.continuousPhase = void 0;
|
|
1121
1156
|
this.continuousTask = void 0;
|
|
1122
1157
|
this.continuousStopWaiter = void 0;
|
|
1123
1158
|
this.continuousStopRequested = false;
|
|
@@ -1131,6 +1166,7 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1131
1166
|
);
|
|
1132
1167
|
return false;
|
|
1133
1168
|
}
|
|
1169
|
+
this.publishPreparationPhase("catch-up");
|
|
1134
1170
|
const localDB = this.rawDatabase;
|
|
1135
1171
|
Logger(this.translate("liveSyncReplicator.beforeLiveSync"));
|
|
1136
1172
|
const caughtUp = await this.env.services.replicator.runFiniteReplicationActivity(
|
|
@@ -1146,6 +1182,7 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1146
1182
|
{ label: "replication" }
|
|
1147
1183
|
);
|
|
1148
1184
|
if (caughtUp && !this.continuousStopRequested) {
|
|
1185
|
+
this.publishPreparationPhase("continuous");
|
|
1149
1186
|
Logger(this.translate("liveSyncReplicator.liveSyncBegin"));
|
|
1150
1187
|
const ret = await this.checkReplicationConnectivity(setting, true, false, showResult);
|
|
1151
1188
|
if (ret === false || this.continuousStopRequested) {
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { EntryDoc } from "../../common/types.js";
|
|
2
|
+
import type { CouchDBReplicationPhase, CouchDBReplicationProgress } from "../../common/models/shared.definition.js";
|
|
3
|
+
/** Observe one handler without adding its high-volume checkpoint events to the replication inbox. */
|
|
4
|
+
export declare function observeReplicationProgress(handler: PouchDB.Replication.Sync<EntryDoc> | PouchDB.Replication.Replication<EntryDoc>, mode: "sync" | "pullOnly" | "pushOnly", phase: Exclude<CouchDBReplicationPhase, "idle">, signal: AbortSignal, publish: (progress: CouchDBReplicationProgress) => void): {
|
|
5
|
+
dispose(): void;
|
|
6
|
+
};
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { compatGlobal } from "../../common/coreEnvFunctions.js";
|
|
2
|
+
const PROGRESS_PUBLICATION_INTERVAL_MS = 200;
|
|
3
|
+
function asEmitter(value) {
|
|
4
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
5
|
+
if (!("on" in value) || typeof value.on !== "function") return void 0;
|
|
6
|
+
if (!("removeListener" in value) || typeof value.removeListener !== "function") return void 0;
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function isSequence(value) {
|
|
10
|
+
return typeof value === "string" || typeof value === "number" && Number.isFinite(value);
|
|
11
|
+
}
|
|
12
|
+
function observeReplicationProgress(handler, mode, phase, signal, publish) {
|
|
13
|
+
const progress = {};
|
|
14
|
+
const detach = [];
|
|
15
|
+
let timer;
|
|
16
|
+
let dirty = false;
|
|
17
|
+
let disposed = false;
|
|
18
|
+
let currentPhase = phase;
|
|
19
|
+
const cancelTimer = () => {
|
|
20
|
+
if (timer === void 0) return;
|
|
21
|
+
compatGlobal.clearTimeout(timer);
|
|
22
|
+
timer = void 0;
|
|
23
|
+
};
|
|
24
|
+
const flush = () => {
|
|
25
|
+
cancelTimer();
|
|
26
|
+
if (!dirty) return;
|
|
27
|
+
dirty = false;
|
|
28
|
+
publish({
|
|
29
|
+
phase: currentPhase,
|
|
30
|
+
...progress.pull ? { pull: { ...progress.pull } } : {},
|
|
31
|
+
...progress.push ? { push: { ...progress.push } } : {}
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
const schedule = () => {
|
|
35
|
+
dirty = true;
|
|
36
|
+
timer ??= compatGlobal.setTimeout(flush, PROGRESS_PUBLICATION_INTERVAL_MS);
|
|
37
|
+
};
|
|
38
|
+
const listen = (emitter, event, listener) => {
|
|
39
|
+
emitter.on(event, listener);
|
|
40
|
+
detach.push(() => emitter.removeListener(event, listener));
|
|
41
|
+
};
|
|
42
|
+
const observeDirection = (direction, emitter) => {
|
|
43
|
+
if (!emitter) return;
|
|
44
|
+
const value = { checkedChanges: 0, completedBatches: 0, checking: false };
|
|
45
|
+
progress[direction] = value;
|
|
46
|
+
listen(emitter, "checkpoint", (event) => {
|
|
47
|
+
if (disposed || signal.aborted || typeof event !== "object" || event === null) return;
|
|
48
|
+
if ("revs_diff" in event && typeof event.revs_diff === "object" && event.revs_diff !== null && "id" in event.revs_diff && typeof event.revs_diff.id === "string") {
|
|
49
|
+
value.checkedChanges++;
|
|
50
|
+
value.checking = true;
|
|
51
|
+
} else if ("start_next_batch" in event && isSequence(event.start_next_batch)) {
|
|
52
|
+
value.checking = true;
|
|
53
|
+
} else if ("checkpoint" in event && isSequence(event.checkpoint)) {
|
|
54
|
+
value.completedBatches++;
|
|
55
|
+
value.lastSeq = event.checkpoint;
|
|
56
|
+
} else {
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
schedule();
|
|
60
|
+
});
|
|
61
|
+
const stopChecking = () => {
|
|
62
|
+
if (disposed || signal.aborted) return;
|
|
63
|
+
if (value.checking) {
|
|
64
|
+
value.checking = false;
|
|
65
|
+
dirty = true;
|
|
66
|
+
}
|
|
67
|
+
flush();
|
|
68
|
+
};
|
|
69
|
+
listen(emitter, "change", stopChecking);
|
|
70
|
+
listen(emitter, "paused", stopChecking);
|
|
71
|
+
};
|
|
72
|
+
const dispose = () => {
|
|
73
|
+
if (disposed) return;
|
|
74
|
+
disposed = true;
|
|
75
|
+
cancelTimer();
|
|
76
|
+
for (const remove of detach) remove();
|
|
77
|
+
signal.removeEventListener("abort", dispose);
|
|
78
|
+
currentPhase = "idle";
|
|
79
|
+
if (progress.pull) progress.pull.checking = false;
|
|
80
|
+
if (progress.push) progress.push.checking = false;
|
|
81
|
+
dirty = true;
|
|
82
|
+
flush();
|
|
83
|
+
};
|
|
84
|
+
if (mode === "sync") {
|
|
85
|
+
const duplex = handler;
|
|
86
|
+
observeDirection("pull", asEmitter(duplex.pull));
|
|
87
|
+
observeDirection("push", asEmitter(duplex.push));
|
|
88
|
+
} else {
|
|
89
|
+
observeDirection(mode === "pullOnly" ? "pull" : "push", asEmitter(handler));
|
|
90
|
+
}
|
|
91
|
+
const outer = asEmitter(handler);
|
|
92
|
+
if (outer) {
|
|
93
|
+
listen(outer, "complete", dispose);
|
|
94
|
+
listen(outer, "error", dispose);
|
|
95
|
+
listen(outer, "denied", dispose);
|
|
96
|
+
}
|
|
97
|
+
signal.addEventListener("abort", dispose, { once: true });
|
|
98
|
+
dirty = true;
|
|
99
|
+
if (signal.aborted) dispose();
|
|
100
|
+
else flush();
|
|
101
|
+
return { dispose };
|
|
102
|
+
}
|
|
103
|
+
export {
|
|
104
|
+
observeReplicationProgress
|
|
105
|
+
};
|
|
@@ -12,3 +12,4 @@ export type { CentralCompatibilityRecoveryHint, CentralCompatibilityRejectionRea
|
|
|
12
12
|
export { assessRemoteFeatureDocument, describeRemoteFeatureRejection, ENCRYPTED_INTERNAL_METADATA_FEATURE, NFC_PATH_IDENTITY_FEATURE, REMOTE_FEATURE_GENERATION, usesEncryptedInternalMetadata, } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
13
13
|
export type { RemoteFeatureAssessment } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
14
14
|
export type { JoinedDevice, JoinedDevicesSnapshot, ForgetJoinedDeviceResult } from "./joinedDevices.js";
|
|
15
|
+
export type { ReplicationStatics, CouchDBReplicationPhase, CouchDBReplicationProgress, ReplicationCheckpointProgress, } from "../common/models/shared.definition.js";
|
|
@@ -144,8 +144,10 @@ export declare class TrysteroReplicator {
|
|
|
144
144
|
idDerivationVersion: 0 | 1;
|
|
145
145
|
idDerivationProof: string;
|
|
146
146
|
liveSync?: boolean;
|
|
147
|
+
batchSaveInLiveSync?: boolean;
|
|
147
148
|
syncOnSave?: boolean;
|
|
148
149
|
syncOnStart?: boolean;
|
|
150
|
+
strictContentCheckOnStartup?: boolean;
|
|
149
151
|
syncOnFileOpen?: boolean;
|
|
150
152
|
syncOnEditorSave?: boolean;
|
|
151
153
|
keepReplicationActiveInBackground?: boolean;
|
|
@@ -332,8 +334,10 @@ export declare class TrysteroReplicator {
|
|
|
332
334
|
idDerivationVersion: 0 | 1;
|
|
333
335
|
idDerivationProof: string;
|
|
334
336
|
liveSync?: boolean;
|
|
337
|
+
batchSaveInLiveSync?: boolean;
|
|
335
338
|
syncOnSave?: boolean;
|
|
336
339
|
syncOnStart?: boolean;
|
|
340
|
+
strictContentCheckOnStartup?: boolean;
|
|
337
341
|
syncOnFileOpen?: boolean;
|
|
338
342
|
syncOnEditorSave?: boolean;
|
|
339
343
|
keepReplicationActiveInBackground?: boolean;
|
|
@@ -3,7 +3,7 @@ import { type LogFunction } from "../services/lib/logUtils.js";
|
|
|
3
3
|
import type { NecessaryServices } from "../interfaces/ServiceModule.js";
|
|
4
4
|
import type { IPathService } from "../services/base/IService.js";
|
|
5
5
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
6
|
-
import { VaultScanResults, type VaultScanResult } from "../services/base/VaultScanResult.js";
|
|
6
|
+
import { VaultScanResults, type VaultScanResult, type VaultScanOptions } from "../services/base/VaultScanResult.js";
|
|
7
7
|
export { VaultScanResults };
|
|
8
8
|
export type { VaultScanResult };
|
|
9
9
|
/**
|
|
@@ -156,7 +156,7 @@ export declare function getPathFromEntry(host: {
|
|
|
156
156
|
* @param file Storage file information
|
|
157
157
|
* @param doc Database entry
|
|
158
158
|
*/
|
|
159
|
-
export declare function syncFileBetweenDBandStorage(host: NecessaryServices<"setting" | "vault" | "path", "storageAccess" | "fileHandler">, log: LogFunction, file: UXFileInfoStub, doc: MetaEntry): Promise<FilePairProcessResult>;
|
|
159
|
+
export declare function syncFileBetweenDBandStorage(host: NecessaryServices<"setting" | "vault" | "path", "storageAccess" | "fileHandler">, log: LogFunction, file: UXFileInfoStub, doc: MetaEntry, scanOptions?: VaultScanOptions): Promise<FilePairProcessResult>;
|
|
160
160
|
export declare function canProceedScan(host: NecessaryServices<"keyValueDB" | "setting", never>, errorManager: UnresolvedErrorManager, log: LogFunction, showingNotice?: boolean, ignoreSuspending?: boolean): boolean;
|
|
161
161
|
/**
|
|
162
162
|
* Normalise the logical comparison key and lower case it when filename case should be handled insensitively.
|
|
@@ -180,7 +180,7 @@ export declare function collectDatabaseFiles(host: NecessaryServices<"database"
|
|
|
180
180
|
}>;
|
|
181
181
|
export declare function updateToDatabase(host: NecessaryServices<"vault", "fileHandler">, log: LogFunction, logLevel: LOG_LEVEL, file: UXFileInfoStub): Promise<FilePairProcessResult>;
|
|
182
182
|
export declare function updateToStorage(host: NecessaryServices<"vault" | "path", "fileHandler">, log: LogFunction, logLevel: LOG_LEVEL, w: MetaEntry): Promise<FilePairProcessResult>;
|
|
183
|
-
export declare function syncStorageAndDatabase(host: NecessaryServices<"setting" | "vault" | "path", "storageAccess" | "fileHandler">, log: LogFunction, file: UXFileInfoStub, logLevel: LOG_LEVEL, doc: MetaEntry): Promise<FilePairProcessResult>;
|
|
183
|
+
export declare function syncStorageAndDatabase(host: NecessaryServices<"setting" | "vault" | "path", "storageAccess" | "fileHandler">, log: LogFunction, file: UXFileInfoStub, logLevel: LOG_LEVEL, doc: MetaEntry, scanOptions?: VaultScanOptions): Promise<FilePairProcessResult>;
|
|
184
184
|
export declare const FullScanModes: {
|
|
185
185
|
readonly DB_APPLY: "db-apply";
|
|
186
186
|
readonly NEWER_WINS: "newer-wins";
|
|
@@ -205,7 +205,7 @@ export declare const ExtraOnLocal: {
|
|
|
205
205
|
*/
|
|
206
206
|
readonly APPEND_STORAGE_ONLY: "append-storage-only";
|
|
207
207
|
};
|
|
208
|
-
export interface FullScanOptions {
|
|
208
|
+
export interface FullScanOptions extends VaultScanOptions {
|
|
209
209
|
mode: FullScanMode;
|
|
210
210
|
extraOnLocal?: (typeof ExtraOnLocal)[keyof typeof ExtraOnLocal];
|
|
211
211
|
extraOnRemote?: (typeof ExtraOnRemote)[keyof typeof ExtraOnRemote];
|
|
@@ -350,7 +350,7 @@ function getPathFromEntry(host, doc) {
|
|
|
350
350
|
const path = host.services.path.getPath(doc);
|
|
351
351
|
return path;
|
|
352
352
|
}
|
|
353
|
-
async function syncFileBetweenDBandStorage(host, log, file, doc) {
|
|
353
|
+
async function syncFileBetweenDBandStorage(host, log, file, doc, scanOptions) {
|
|
354
354
|
const docPath = getPathFromEntry(host, doc);
|
|
355
355
|
if (!doc) {
|
|
356
356
|
throw new Error(`Missing doc:${docPath}`);
|
|
@@ -360,8 +360,7 @@ async function syncFileBetweenDBandStorage(host, log, file, doc) {
|
|
|
360
360
|
case BASE_IS_NEW:
|
|
361
361
|
if (!host.services.vault.isFileSizeTooLarge(file.stat.size)) {
|
|
362
362
|
log("STORAGE -> DB :" + file.path);
|
|
363
|
-
await host.serviceModules.fileHandler.storeFileToDB(file);
|
|
364
|
-
return FilePairProcessResults.COMPLETED;
|
|
363
|
+
return await host.serviceModules.fileHandler.storeFileToDB(file) ? FilePairProcessResults.COMPLETED : FilePairProcessResults.FAILED;
|
|
365
364
|
} else {
|
|
366
365
|
log(
|
|
367
366
|
`STORAGE -> DB : ${file.path} has been skipped due to file size exceeding the limit`,
|
|
@@ -390,11 +389,26 @@ async function syncFileBetweenDBandStorage(host, log, file, doc) {
|
|
|
390
389
|
return FilePairProcessResults.SKIPPED;
|
|
391
390
|
}
|
|
392
391
|
case EVEN:
|
|
393
|
-
if (doc.
|
|
394
|
-
|
|
392
|
+
if (host.services.vault.isFileSizeTooLarge(file.stat.size) || host.services.vault.isFileSizeTooLarge(doc.size)) {
|
|
393
|
+
log(`STORAGE ? DB : ${file.path} has been skipped due to file size exceeding the limit`, LOG_LEVEL_NOTICE);
|
|
394
|
+
return FilePairProcessResults.SKIPPED;
|
|
395
|
+
}
|
|
396
|
+
if (scanOptions?.strictContentCheck === false && await host.serviceModules.fileHandler.canSkipUnchangedFile?.(file, doc)) {
|
|
397
|
+
log(`STORAGE ? DB : ${file.path} skipped using unchanged storage and revision metadata`, LOG_LEVEL_DEBUG);
|
|
398
|
+
return FilePairProcessResults.SKIPPED;
|
|
399
|
+
}
|
|
400
|
+
if (doc._rev && await host.serviceModules.fileHandler.tryRecordUntrackedFileRevision(file, doc._rev)) {
|
|
401
|
+
log("STORAGE == DB :" + file.path, LOG_LEVEL_DEBUG);
|
|
402
|
+
return FilePairProcessResults.COMPLETED;
|
|
403
|
+
}
|
|
404
|
+
log("STORAGE ? DB :" + file.path + " Checking content", LOG_LEVEL_DEBUG);
|
|
405
|
+
try {
|
|
406
|
+
return await host.serviceModules.fileHandler.storeFileToDB(file, false, false, { revalidateStorage: true }) ? FilePairProcessResults.COMPLETED : FilePairProcessResults.FAILED;
|
|
407
|
+
} catch (error) {
|
|
408
|
+
log(`STORAGE ? DB : Could not reconcile ${file.path}`, LOG_LEVEL_NOTICE);
|
|
409
|
+
log(error, LOG_LEVEL_VERBOSE);
|
|
410
|
+
return FilePairProcessResults.FAILED;
|
|
395
411
|
}
|
|
396
|
-
log("STORAGE == DB :" + file.path, LOG_LEVEL_DEBUG);
|
|
397
|
-
return FilePairProcessResults.COMPLETED;
|
|
398
412
|
default:
|
|
399
413
|
log("STORAGE ?? DB :" + file.path + " Something got weird");
|
|
400
414
|
return FilePairProcessResults.FAILED;
|
|
@@ -501,7 +515,9 @@ async function collectDatabaseFiles(host, settings, log, showingNotice) {
|
|
|
501
515
|
async function updateToDatabase(host, log, logLevel, file) {
|
|
502
516
|
if (!host.services.vault.isFileSizeTooLarge(file.stat.size)) {
|
|
503
517
|
const path = file.path;
|
|
504
|
-
await host.serviceModules.fileHandler.storeFileToDB(file)
|
|
518
|
+
if (!await host.serviceModules.fileHandler.storeFileToDB(file)) {
|
|
519
|
+
return FilePairProcessResults.FAILED;
|
|
520
|
+
}
|
|
505
521
|
host.services.context.events.emitEvent("event-file-changed", { file: path, automated: true });
|
|
506
522
|
return FilePairProcessResults.COMPLETED;
|
|
507
523
|
} else {
|
|
@@ -537,13 +553,13 @@ async function updateToStorage(host, log, logLevel, w) {
|
|
|
537
553
|
return FilePairProcessResults.FAILED;
|
|
538
554
|
}
|
|
539
555
|
}
|
|
540
|
-
async function syncStorageAndDatabase(host, log, file, logLevel, doc) {
|
|
556
|
+
async function syncStorageAndDatabase(host, log, file, logLevel, doc, scanOptions) {
|
|
541
557
|
if ((doc._conflicts?.length ?? 0) > 0) {
|
|
542
558
|
log(`SYNC DATABASE AND STORAGE: ${file.path} has conflicts. skipped`, LOG_LEVEL_INFO);
|
|
543
559
|
return FilePairProcessResults.SKIPPED;
|
|
544
560
|
}
|
|
545
561
|
if (!host.services.vault.isFileSizeTooLarge(file.stat.size) && !host.services.vault.isFileSizeTooLarge(doc.size)) {
|
|
546
|
-
return await syncFileBetweenDBandStorage(host, log, file, doc);
|
|
562
|
+
return await syncFileBetweenDBandStorage(host, log, file, doc, scanOptions);
|
|
547
563
|
} else {
|
|
548
564
|
log(
|
|
549
565
|
`SYNC DATABASE AND STORAGE: ${getPathFromEntry(host, doc)} has been skipped due to file size exceeding the limit`,
|
|
@@ -686,7 +702,7 @@ async function processFilePair(host, log, pair, options) {
|
|
|
686
702
|
if (!file || !doc) {
|
|
687
703
|
throw new Error(`Cannot compare freshness for ${path}`);
|
|
688
704
|
}
|
|
689
|
-
const syncResult = await syncStorageAndDatabase(host, log, file, LOG_LEVEL_INFO, doc);
|
|
705
|
+
const syncResult = await syncStorageAndDatabase(host, log, file, LOG_LEVEL_INFO, doc, options);
|
|
690
706
|
if (syncResult === FilePairProcessResults.COMPLETED) {
|
|
691
707
|
updateFileMTimeInMap(host, fileMapKey, Math.max(file.stat.mtime, doc.mtime));
|
|
692
708
|
}
|
|
@@ -910,11 +926,12 @@ async function performFullScan(host, log, errorManager, showingNoticeOrOptions =
|
|
|
910
926
|
function useOfflineScanner(host) {
|
|
911
927
|
const log = createInstanceLogFunction("SF:OfflineScanner", host.services.API);
|
|
912
928
|
const errorManager = new UnresolvedErrorManager(host.services.appLifecycle, host.services.context.events);
|
|
913
|
-
const handleScanVault = async (showingNotice, ignoreSuspending = false, continueOnFileFailure = false) => {
|
|
929
|
+
const handleScanVault = async (showingNotice, ignoreSuspending = false, continueOnFileFailure = false, scanOptions) => {
|
|
914
930
|
return await performFullScan(host, log, errorManager, {
|
|
915
931
|
showingNotice,
|
|
916
932
|
ignoreSuspending,
|
|
917
|
-
continueOnFileFailure
|
|
933
|
+
continueOnFileFailure,
|
|
934
|
+
strictContentCheck: scanOptions?.strictContentCheck
|
|
918
935
|
});
|
|
919
936
|
};
|
|
920
937
|
host.services.vault.scanVault.addHandler(handleScanVault);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { NecessaryServices } from "../interfaces/ServiceModule.js";
|
|
2
2
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
3
3
|
import { type LogFunction } from "../services/lib/logUtils.js";
|
|
4
|
-
import { type VaultScanResult } from "../services/base/VaultScanResult.js";
|
|
4
|
+
import { type VaultScanResult, type VaultScanOptions } from "../services/base/VaultScanResult.js";
|
|
5
5
|
/**
|
|
6
6
|
* Initialise the database and trigger a full vault scan.
|
|
7
7
|
* @param host Services container
|
|
@@ -11,9 +11,10 @@ import { type VaultScanResult } from "../services/base/VaultScanResult.js";
|
|
|
11
11
|
* @param reopenDatabase Whether to reopen the database connection
|
|
12
12
|
* @param ignoreSuspending Whether to ignore suspension settings
|
|
13
13
|
* @param continueOnFileFailure Whether individual file failures may satisfy application readiness
|
|
14
|
+
* @param scanOptions Optional content-check policy; omitted options keep strict checks
|
|
14
15
|
* @returns The initialisation outcome, including accepted individual file failures
|
|
15
16
|
*/
|
|
16
|
-
export declare function prepareDatabaseForUse(host: NecessaryServices<"appLifecycle" | "setting" | "vault" | "path" | "database" | "databaseEvents" | "fileProcessing" | "replicator", never>, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean): Promise<VaultScanResult>;
|
|
17
|
+
export declare function prepareDatabaseForUse(host: NecessaryServices<"appLifecycle" | "setting" | "vault" | "path" | "database" | "databaseEvents" | "fileProcessing" | "replicator", never>, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean, scanOptions?: VaultScanOptions): Promise<VaultScanResult>;
|
|
17
18
|
/**
|
|
18
19
|
* Associate the initialiser file feature with the app lifecycle events.
|
|
19
20
|
* This function binds initialization handlers to the appropriate lifecycle events.
|
|
@@ -6,7 +6,7 @@ function reportPreparationFailure(log, message) {
|
|
|
6
6
|
log(message, LOG_LEVEL_VERBOSE);
|
|
7
7
|
return false;
|
|
8
8
|
}
|
|
9
|
-
async function prepareDatabaseForUse(host, log, errorManager, showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false) {
|
|
9
|
+
async function prepareDatabaseForUse(host, log, errorManager, showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false, scanOptions) {
|
|
10
10
|
const appLifecycle = host.services.appLifecycle;
|
|
11
11
|
appLifecycle.resetIsReady();
|
|
12
12
|
try {
|
|
@@ -25,7 +25,7 @@ async function prepareDatabaseForUse(host, log, errorManager, showingNotice = fa
|
|
|
25
25
|
"Database preparation stopped because the local database is not ready."
|
|
26
26
|
);
|
|
27
27
|
}
|
|
28
|
-
const scanResult = await host.services.vault.scanVault(showingNotice, ignoreSuspending, continueOnFileFailure);
|
|
28
|
+
const scanResult = await host.services.vault.scanVault(showingNotice, ignoreSuspending, continueOnFileFailure, scanOptions);
|
|
29
29
|
if (scanResult === VaultScanResults.FAILED) {
|
|
30
30
|
return reportPreparationFailure(log, "Database preparation stopped because the Vault scan failed.");
|
|
31
31
|
}
|
|
@@ -57,7 +57,7 @@ async function prepareDatabaseForUse(host, log, errorManager, showingNotice = fa
|
|
|
57
57
|
function usePrepareDatabaseForUse(host) {
|
|
58
58
|
const log = createInstanceLogFunction("SF:prepareDatabaseForUse", host.services.API);
|
|
59
59
|
const errorManager = new UnresolvedErrorManager(host.services.appLifecycle, host.services.context.events);
|
|
60
|
-
const initialiseDatabaseHandler = async (showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false) => {
|
|
60
|
+
const initialiseDatabaseHandler = async (showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false, scanOptions) => {
|
|
61
61
|
return await prepareDatabaseForUse(
|
|
62
62
|
host,
|
|
63
63
|
log,
|
|
@@ -65,7 +65,8 @@ function usePrepareDatabaseForUse(host) {
|
|
|
65
65
|
showingNotice,
|
|
66
66
|
reopenDatabase,
|
|
67
67
|
ignoreSuspending,
|
|
68
|
-
continueOnFileFailure
|
|
68
|
+
continueOnFileFailure,
|
|
69
|
+
scanOptions
|
|
69
70
|
);
|
|
70
71
|
};
|
|
71
72
|
host.services.databaseEvents.initialiseDatabase.addHandler(initialiseDatabaseHandler);
|
|
@@ -26,6 +26,7 @@ export declare class ServiceFileAccessBase<TAdapter extends IFileSystemAdapter<a
|
|
|
26
26
|
private vault;
|
|
27
27
|
private setting;
|
|
28
28
|
constructor(services: StorageAccessBaseDependencies<TAdapter>);
|
|
29
|
+
hasPendingFileEvent(path: FilePath): boolean;
|
|
29
30
|
restoreState(): Promise<void>;
|
|
30
31
|
_everyOnFirstInitialize(): Promise<boolean>;
|
|
31
32
|
_everyCommitPendingFileEvent(): Promise<boolean>;
|
|
@@ -15,6 +15,14 @@ class ServiceFileAccessBase extends ServiceModuleBase {
|
|
|
15
15
|
this.vaultAccess = services.vaultAccess;
|
|
16
16
|
services.appLifecycle.onFirstInitialise.addHandler(this._everyOnFirstInitialize.bind(this));
|
|
17
17
|
services.fileProcessing.commitPendingFileEvents.addHandler(this._everyCommitPendingFileEvent.bind(this));
|
|
18
|
+
services.appLifecycle.onBeforeUnload.addHandler(async () => {
|
|
19
|
+
if (this.vaultManager.stopWatching) await this.vaultManager.stopWatching();
|
|
20
|
+
else await this.vaultManager.waitForIdle();
|
|
21
|
+
return true;
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
hasPendingFileEvent(path) {
|
|
25
|
+
return this.vaultManager.hasPendingFileEvent?.(path) ?? true;
|
|
18
26
|
}
|
|
19
27
|
restoreState() {
|
|
20
28
|
return this.vaultManager.restoreState();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { AnyEntry, FilePath, FilePathWithPrefix, MetaEntry, UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../common/types.js";
|
|
2
2
|
import type { LiveSyncEventHub } from "../hub/hub.js";
|
|
3
|
-
import type { IFileHandler } from "../interfaces/FileHandler.js";
|
|
3
|
+
import type { IFileHandler, StoreFileOptions } from "../interfaces/FileHandler.js";
|
|
4
4
|
import { ServiceModuleBase } from "./ServiceModuleBase.js";
|
|
5
5
|
import type { APIService } from "../services/base/APIService.js";
|
|
6
6
|
import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess.js";
|
|
@@ -74,7 +74,8 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
74
74
|
/**
|
|
75
75
|
* Read the raw provenance value used by an optimistic storage snapshot.
|
|
76
76
|
* Unlike `getProvenance`, this does not validate or remove the referenced
|
|
77
|
-
* database revision
|
|
77
|
+
* database revision. Recovery callers require a readable value so a store
|
|
78
|
+
* failure cannot be mistaken for absent provenance.
|
|
78
79
|
*/
|
|
79
80
|
private readProvenanceSnapshot;
|
|
80
81
|
/**
|
|
@@ -97,9 +98,10 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
97
98
|
getPath(entry: AnyEntry): FilePathWithPrefix;
|
|
98
99
|
getPathWithoutPrefix(entry: AnyEntry): FilePathWithPrefix;
|
|
99
100
|
readFileFromStub(file: UXFileInfoStub | UXFileInfo): Promise<UXFileInfo>;
|
|
101
|
+
canSkipUnchangedFile(info: UXFileInfoStub, expected: MetaEntry): Promise<boolean>;
|
|
100
102
|
tryRecordUntrackedFileRevision(info: UXFileInfoStub | UXFileInfo, expectedRevision: string): Promise<boolean>;
|
|
101
103
|
private infoToStub;
|
|
102
|
-
storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean): Promise<boolean>;
|
|
104
|
+
storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean, options?: StoreFileOptions): Promise<boolean>;
|
|
103
105
|
storeFileToDBWithBaseRevision(info: UXFileInfoStub | UXFileInfo | FilePathWithPrefix, baseRevision: string, createIfDifferent?: boolean): Promise<boolean>;
|
|
104
106
|
/**
|
|
105
107
|
* Store current storage content below an exact live revision. A differing
|
|
@@ -110,11 +112,13 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
110
112
|
/**
|
|
111
113
|
* Save one storage snapshot while the caller holds its document lock.
|
|
112
114
|
* Ordinary saves read once inside the lock and leave retries disabled.
|
|
113
|
-
* Restored
|
|
114
|
-
*
|
|
115
|
+
* Restored start-up events and recovery scans retain bounded rereads
|
|
116
|
+
* because a watcher event may not cover a concurrent edit.
|
|
115
117
|
*/
|
|
116
118
|
private storeFileToDBFromRevision;
|
|
117
119
|
deleteFileFromDB(info: UXFileInfoStub | UXInternalFileInfoStub | FilePath): Promise<boolean>;
|
|
120
|
+
/** Complete a validated storage deletion when its database effect is already represented. */
|
|
121
|
+
private deleteFileFromDBIfPresent;
|
|
118
122
|
/**
|
|
119
123
|
* Record a storage deletion under the caller's document lock, preserving
|
|
120
124
|
* the displayed conflict branch when its provenance is known.
|