@vrtmrz/livesync-commonlib 0.1.21 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/common/models/fileaccess.type.d.ts +2 -0
- package/dist/managers/StorageEventManager.js +1 -0
- package/dist/serviceFeatures/offlineScanner.d.ts +16 -8
- package/dist/serviceFeatures/offlineScanner.js +18 -6
- package/dist/serviceFeatures/prepareDatabaseForUse.d.ts +4 -2
- package/dist/serviceFeatures/prepareDatabaseForUse.js +60 -26
- package/dist/serviceFeatures/targetFilter.d.ts +2 -1
- package/dist/serviceFeatures/targetFilter.js +2 -1
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +1 -0
- package/dist/serviceModules/ServiceFileHandlerBase.js +47 -2
- package/dist/services/base/DatabaseEventService.d.ts +1 -1
- package/dist/services/base/DatabaseEventService.js +1 -1
- package/dist/services/base/IService.d.ts +8 -3
- package/dist/services/base/ReplicationService.readiness.js +1 -1
- package/dist/services/base/VaultScanResult.d.ts +13 -0
- package/dist/services/base/VaultScanResult.js +8 -0
- package/dist/services/base/VaultService.d.ts +5 -3
- package/dist/services/base/VaultService.js +2 -2
- package/dist/services/lib/HandlerUtils.d.ts +23 -1
- package/dist/services/lib/HandlerUtils.js +48 -0
- package/docs/development.md +2 -0
- package/docs/offline-scanner.md +159 -0
- package/docs/proven-in-use.md +8 -2
- package/docs/service-feature-composition.md +21 -8
- package/docs/storage-events-and-reflection.md +107 -0
- package/package.json +1 -1
|
@@ -53,6 +53,8 @@ export type FileEventArgs = {
|
|
|
53
53
|
file: UXFileInfoStub | UXInternalFileInfoStub;
|
|
54
54
|
cache?: CacheData;
|
|
55
55
|
oldPath?: string;
|
|
56
|
+
/** The destination of a rename converted to DELETE after target filtering. */
|
|
57
|
+
renameTarget?: string;
|
|
56
58
|
ctx?: unknown;
|
|
57
59
|
};
|
|
58
60
|
export type FileEventItem = {
|
|
@@ -2,6 +2,9 @@ import { type DocumentID, type FilePathWithPrefix, type FilePathWithPrefixLC, ty
|
|
|
2
2
|
import { type LogFunction } from "../services/lib/logUtils.js";
|
|
3
3
|
import type { NecessaryServices } from "../interfaces/ServiceModule.js";
|
|
4
4
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
5
|
+
import { VaultScanResults, type VaultScanResult } from "../services/base/VaultScanResult.js";
|
|
6
|
+
export { VaultScanResults };
|
|
7
|
+
export type { VaultScanResult };
|
|
5
8
|
/**
|
|
6
9
|
* Outcome of processing one storage/database pair during an offline scan.
|
|
7
10
|
*
|
|
@@ -200,6 +203,12 @@ export interface FullScanOptions {
|
|
|
200
203
|
omitEvents?: boolean;
|
|
201
204
|
showingNotice?: boolean;
|
|
202
205
|
ignoreSuspending?: boolean;
|
|
206
|
+
/**
|
|
207
|
+
* Allow a completed scan containing individual file-pair failures to
|
|
208
|
+
* satisfy ordinary application readiness. Scan preconditions remain
|
|
209
|
+
* strict, and failed pairs remain retryable.
|
|
210
|
+
*/
|
|
211
|
+
continueOnFileFailure?: boolean;
|
|
203
212
|
}
|
|
204
213
|
export type FullScanMode = (typeof FullScanModes)[keyof typeof FullScanModes];
|
|
205
214
|
type FilePair = {
|
|
@@ -225,9 +234,9 @@ export declare function resolveFilePairAction(state: FilePairState, options: Ful
|
|
|
225
234
|
* @param log Logging function
|
|
226
235
|
* @param errorManager Error manager
|
|
227
236
|
* @param options Full scan options
|
|
228
|
-
* @returns
|
|
237
|
+
* @returns The scan result for all file pairs
|
|
229
238
|
*/
|
|
230
|
-
export declare function synchroniseAllFilesBetweenDBandStorage(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, options: FullScanOptions): Promise<
|
|
239
|
+
export declare function synchroniseAllFilesBetweenDBandStorage(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, options: FullScanOptions): Promise<VaultScanResult>;
|
|
231
240
|
export declare function normaliseFullScanOptions(showingNoticeOrOptions: Partial<FullScanOptions> | boolean | undefined, ignoreSuspending?: boolean): FullScanOptions;
|
|
232
241
|
/**
|
|
233
242
|
* Remove one path from the Offline Scanner's durable last-seen state.
|
|
@@ -242,16 +251,15 @@ export declare function clearOfflineScannerLastSeen(host: NecessaryServices<"set
|
|
|
242
251
|
* @param host Services container
|
|
243
252
|
* @param log Logging function
|
|
244
253
|
* @param errorManager Error manager
|
|
245
|
-
* @param
|
|
246
|
-
* @param ignoreSuspending
|
|
247
|
-
* @returns
|
|
254
|
+
* @param showingNoticeOrOptions Full-scan options, or the legacy notice flag
|
|
255
|
+
* @param ignoreSuspending Legacy suspension flag used with the notice flag
|
|
256
|
+
* @returns The scan outcome, including accepted individual file failures
|
|
248
257
|
*/
|
|
249
|
-
export declare function performFullScan(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, options?: Partial<FullScanOptions>): Promise<
|
|
250
|
-
export declare function performFullScan(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, ignoreSuspending?: boolean): Promise<
|
|
258
|
+
export declare function performFullScan(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, options?: Partial<FullScanOptions>): Promise<VaultScanResult>;
|
|
259
|
+
export declare function performFullScan(host: NecessaryServices<"setting" | "vault" | "path" | "fileProcessing" | "database" | "keyValueDB", "storageAccess" | "fileHandler">, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, ignoreSuspending?: boolean): Promise<VaultScanResult>;
|
|
251
260
|
/**
|
|
252
261
|
* Associate the initialiser file feature with the app lifecycle events.
|
|
253
262
|
* This function binds initialization handlers to the appropriate lifecycle events.
|
|
254
263
|
* @param host Services container with required dependencies
|
|
255
264
|
*/
|
|
256
265
|
export declare function useOfflineScanner(host: NecessaryServices<"API" | "appLifecycle" | "setting" | "vault" | "path" | "database" | "databaseEvents" | "fileProcessing" | "keyValueDB" | "replicator", "storageAccess" | "fileHandler">): void;
|
|
257
|
-
export {};
|
|
@@ -15,6 +15,7 @@ import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.
|
|
|
15
15
|
import { compatGlobal } from "../common/coreEnvFunctions.js";
|
|
16
16
|
import { ICHeader, ICXHeader, PSCHeader } from "../common/models/fileaccess.const.js";
|
|
17
17
|
import { serialized } from "octagonal-wheels/concurrency/lock";
|
|
18
|
+
import { VaultScanResults } from "../services/base/VaultScanResult.js";
|
|
18
19
|
const FilePairProcessResults = {
|
|
19
20
|
COMPLETED: "completed",
|
|
20
21
|
SKIPPED: "skipped",
|
|
@@ -749,15 +750,16 @@ async function synchroniseAllFilesBetweenDBandStorage(host, log, errorManager, o
|
|
|
749
750
|
let skippedCount = 0;
|
|
750
751
|
let failedCount = 0;
|
|
751
752
|
let processedCount = 0;
|
|
752
|
-
for await (const result of withConcurrency(
|
|
753
|
+
for await (const { path, result } of withConcurrency(
|
|
753
754
|
pairs,
|
|
754
755
|
async (e) => {
|
|
756
|
+
const path2 = e.file?.path ?? getPathFromEntry(host, e.doc);
|
|
755
757
|
try {
|
|
756
|
-
return await processFilePair(host, log, e, options);
|
|
758
|
+
return { path: path2, result: await processFilePair(host, log, e, options) };
|
|
757
759
|
} catch (ex) {
|
|
758
760
|
log(`Error while synchronising files`, LOG_LEVEL_NOTICE);
|
|
759
761
|
log(ex, LOG_LEVEL_VERBOSE);
|
|
760
|
-
return FilePairProcessResults.FAILED;
|
|
762
|
+
return { path: path2, result: FilePairProcessResults.FAILED };
|
|
761
763
|
}
|
|
762
764
|
},
|
|
763
765
|
10
|
|
@@ -772,6 +774,10 @@ async function synchroniseAllFilesBetweenDBandStorage(host, log, errorManager, o
|
|
|
772
774
|
break;
|
|
773
775
|
case FilePairProcessResults.FAILED:
|
|
774
776
|
failedCount++;
|
|
777
|
+
log(
|
|
778
|
+
`Offline scan failed to synchronise ${path} between storage and the local database; this path remains eligible for a later scan.`,
|
|
779
|
+
LOG_LEVEL_VERBOSE
|
|
780
|
+
);
|
|
775
781
|
break;
|
|
776
782
|
}
|
|
777
783
|
if (processedCount % 25 === 0) {
|
|
@@ -788,7 +794,8 @@ async function synchroniseAllFilesBetweenDBandStorage(host, log, errorManager, o
|
|
|
788
794
|
"syncAll"
|
|
789
795
|
);
|
|
790
796
|
saveFileStatus(host, true);
|
|
791
|
-
|
|
797
|
+
if (failedCount === 0) return VaultScanResults.COMPLETED;
|
|
798
|
+
return options.continueOnFileFailure === true ? VaultScanResults.COMPLETED_WITH_FILE_FAILURES : VaultScanResults.FAILED;
|
|
792
799
|
}
|
|
793
800
|
function normaliseFullScanOptions(showingNoticeOrOptions, ignoreSuspending = false) {
|
|
794
801
|
if (typeof showingNoticeOrOptions === "object") {
|
|
@@ -898,8 +905,12 @@ async function performFullScan(host, log, errorManager, showingNoticeOrOptions =
|
|
|
898
905
|
function useOfflineScanner(host) {
|
|
899
906
|
const log = createInstanceLogFunction("SF:OfflineScanner", host.services.API);
|
|
900
907
|
const errorManager = new UnresolvedErrorManager(host.services.appLifecycle, host.services.context.events);
|
|
901
|
-
const handleScanVault = async (showingNotice, ignoreSuspending = false) => {
|
|
902
|
-
return await performFullScan(host, log, errorManager,
|
|
908
|
+
const handleScanVault = async (showingNotice, ignoreSuspending = false, continueOnFileFailure = false) => {
|
|
909
|
+
return await performFullScan(host, log, errorManager, {
|
|
910
|
+
showingNotice,
|
|
911
|
+
ignoreSuspending,
|
|
912
|
+
continueOnFileFailure
|
|
913
|
+
});
|
|
903
914
|
};
|
|
904
915
|
host.services.vault.scanVault.addHandler(handleScanVault);
|
|
905
916
|
}
|
|
@@ -912,6 +923,7 @@ export {
|
|
|
912
923
|
MetadataDocumentNamespaces,
|
|
913
924
|
MetadataDocumentRepairResults,
|
|
914
925
|
OfflineScanUnresolvedReasons,
|
|
926
|
+
VaultScanResults,
|
|
915
927
|
canProceedScan,
|
|
916
928
|
clearOfflineScannerLastSeen,
|
|
917
929
|
collectDatabaseFiles,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { NecessaryServices } from "../interfaces/ServiceModule.js";
|
|
2
2
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
3
3
|
import { type LogFunction } from "../services/lib/logUtils.js";
|
|
4
|
+
import { type VaultScanResult } from "../services/base/VaultScanResult.js";
|
|
4
5
|
/**
|
|
5
6
|
* Initialise the database and trigger a full vault scan.
|
|
6
7
|
* @param host Services container
|
|
@@ -9,9 +10,10 @@ import { type LogFunction } from "../services/lib/logUtils.js";
|
|
|
9
10
|
* @param showingNotice Whether to show notices during initialisation
|
|
10
11
|
* @param reopenDatabase Whether to reopen the database connection
|
|
11
12
|
* @param ignoreSuspending Whether to ignore suspension settings
|
|
12
|
-
* @
|
|
13
|
+
* @param continueOnFileFailure Whether individual file failures may satisfy application readiness
|
|
14
|
+
* @returns The initialisation outcome, including accepted individual file failures
|
|
13
15
|
*/
|
|
14
|
-
export declare function prepareDatabaseForUse(host: NecessaryServices<"appLifecycle" | "setting" | "vault" | "path" | "database" | "databaseEvents" | "fileProcessing" | "replicator", never>, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean): Promise<
|
|
16
|
+
export declare function prepareDatabaseForUse(host: NecessaryServices<"appLifecycle" | "setting" | "vault" | "path" | "database" | "databaseEvents" | "fileProcessing" | "replicator", never>, log: LogFunction, errorManager: UnresolvedErrorManager, showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean): Promise<VaultScanResult>;
|
|
15
17
|
/**
|
|
16
18
|
* Associate the initialiser file feature with the app lifecycle events.
|
|
17
19
|
* This function binds initialization handlers to the appropriate lifecycle events.
|
|
@@ -1,38 +1,72 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
|
|
2
2
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
3
3
|
import { createInstanceLogFunction } from "../services/lib/logUtils.js";
|
|
4
|
-
|
|
4
|
+
import { VaultScanResults } from "../services/base/VaultScanResult.js";
|
|
5
|
+
function reportPreparationFailure(log, message) {
|
|
6
|
+
log(message, LOG_LEVEL_VERBOSE);
|
|
7
|
+
return false;
|
|
8
|
+
}
|
|
9
|
+
async function prepareDatabaseForUse(host, log, errorManager, showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false) {
|
|
5
10
|
const appLifecycle = host.services.appLifecycle;
|
|
6
11
|
appLifecycle.resetIsReady();
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
12
|
+
try {
|
|
13
|
+
if (reopenDatabase && !await host.services.database.openDatabase({
|
|
14
|
+
databaseEvents: host.services.databaseEvents,
|
|
15
|
+
replicator: host.services.replicator
|
|
16
|
+
})) {
|
|
17
|
+
return reportPreparationFailure(
|
|
18
|
+
log,
|
|
19
|
+
"Database preparation stopped because the local database could not be opened."
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
if (!host.services.database.isDatabaseReady()) {
|
|
23
|
+
return reportPreparationFailure(
|
|
24
|
+
log,
|
|
25
|
+
"Database preparation stopped because the local database is not ready."
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
const scanResult = await host.services.vault.scanVault(showingNotice, ignoreSuspending, continueOnFileFailure);
|
|
29
|
+
if (scanResult === VaultScanResults.FAILED) {
|
|
30
|
+
return reportPreparationFailure(log, "Database preparation stopped because the Vault scan failed.");
|
|
31
|
+
}
|
|
32
|
+
if (scanResult === VaultScanResults.COMPLETED_WITH_FILE_FAILURES && continueOnFileFailure !== true) {
|
|
33
|
+
return reportPreparationFailure(
|
|
34
|
+
log,
|
|
35
|
+
"Database preparation stopped because the Vault scan could not process every file."
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
if (!await host.services.databaseEvents.onDatabaseInitialised(showingNotice)) {
|
|
39
|
+
return reportPreparationFailure(
|
|
40
|
+
log,
|
|
41
|
+
"Database preparation stopped because an initialisation handler failed."
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
if (!await host.services.fileProcessing.commitPendingFileEvents()) {
|
|
45
|
+
return reportPreparationFailure(
|
|
46
|
+
log,
|
|
47
|
+
"Database preparation stopped because pending file events could not be committed."
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
appLifecycle.markIsReady();
|
|
51
|
+
return scanResult;
|
|
52
|
+
} catch (error) {
|
|
53
|
+
log(error, LOG_LEVEL_VERBOSE);
|
|
54
|
+
throw error;
|
|
27
55
|
}
|
|
28
|
-
appLifecycle.markIsReady();
|
|
29
|
-
return true;
|
|
30
56
|
}
|
|
31
57
|
function usePrepareDatabaseForUse(host) {
|
|
32
58
|
const log = createInstanceLogFunction("SF:prepareDatabaseForUse", host.services.API);
|
|
33
59
|
const errorManager = new UnresolvedErrorManager(host.services.appLifecycle, host.services.context.events);
|
|
34
|
-
const initialiseDatabaseHandler = async (showingNotice = false, reopenDatabase = true, ignoreSuspending = false) => {
|
|
35
|
-
return await prepareDatabaseForUse(
|
|
60
|
+
const initialiseDatabaseHandler = async (showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false) => {
|
|
61
|
+
return await prepareDatabaseForUse(
|
|
62
|
+
host,
|
|
63
|
+
log,
|
|
64
|
+
errorManager,
|
|
65
|
+
showingNotice,
|
|
66
|
+
reopenDatabase,
|
|
67
|
+
ignoreSuspending,
|
|
68
|
+
continueOnFileFailure
|
|
69
|
+
);
|
|
36
70
|
};
|
|
37
71
|
host.services.databaseEvents.initialiseDatabase.addHandler(initialiseDatabaseHandler);
|
|
38
72
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type UXFileInfoStub } from "../common/types.js";
|
|
2
2
|
import { type LogFunction } from "../services/lib/logUtils.js";
|
|
3
3
|
import type { NecessaryServices } from "../interfaces/ServiceModule.js";
|
|
4
|
+
import type { IVaultService } from "../services/base/IService.js";
|
|
4
5
|
/**
|
|
5
6
|
* This is a simple handler that accepts all files.
|
|
6
7
|
*/
|
|
@@ -8,7 +9,7 @@ export declare function isAcceptedAlwaysFactory(host: NecessaryServices<never, n
|
|
|
8
9
|
/**
|
|
9
10
|
* Check if a file is accepted based on filename duplication in the vault.
|
|
10
11
|
*/
|
|
11
|
-
export declare function isAcceptedInFilenameDuplicationFactory(host: NecessaryServices<"vault" | "fileProcessing", "storageAccess">, log: LogFunction): (file: string | UXFileInfoStub) => Promise<boolean>;
|
|
12
|
+
export declare function isAcceptedInFilenameDuplicationFactory(host: NecessaryServices<"vault" | "fileProcessing", "storageAccess">, log: LogFunction): (file: string | UXFileInfoStub, options?: Parameters<IVaultService["isTargetFile"]>[1]) => Promise<boolean>;
|
|
12
13
|
/**
|
|
13
14
|
* Check if a file is accepted by the local database (e.g., not rejected by the local DB's target file check).
|
|
14
15
|
* Local database responsible for non-internal files, syncOnlyRegEx, syncIgnoreRegEx
|
|
@@ -35,7 +35,8 @@ function isAcceptedInFilenameDuplicationFactory(host, log) {
|
|
|
35
35
|
return true;
|
|
36
36
|
}
|
|
37
37
|
});
|
|
38
|
-
return async function isAcceptedInFilenameDuplication(file) {
|
|
38
|
+
return async function isAcceptedInFilenameDuplication(file, options) {
|
|
39
|
+
if (options?.skipCaseCollisionCheck) return true;
|
|
39
40
|
const fileCountMap = (await fileCountMapComputed.update(host.services.fileProcessing.totalStorageFileEventCount)).value;
|
|
40
41
|
const filepath = getStoragePathFromUXFileInfo(file);
|
|
41
42
|
const lc = filepath.toLowerCase();
|
|
@@ -68,6 +68,7 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
68
68
|
private applyDatabaseEntryToStorage;
|
|
69
69
|
private preserveUnsyncedStorageAsConflict;
|
|
70
70
|
private _anyHandlerProcessesFileEvent;
|
|
71
|
+
private canRecordStorageDeletion;
|
|
71
72
|
private serializedByFileEventPaths;
|
|
72
73
|
/**
|
|
73
74
|
* Revalidate a persisted storage operation against the current storage state.
|
|
@@ -657,13 +657,15 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
657
657
|
this._log(`Rename event for ${path} has no source path`, LOG_LEVEL_VERBOSE);
|
|
658
658
|
return false;
|
|
659
659
|
}
|
|
660
|
-
const
|
|
660
|
+
const relatedPath = type === "RENAME" ? eventItem.oldPath : type === "DELETE" ? eventItem.renameTarget : void 0;
|
|
661
|
+
const eventPaths = relatedPath === void 0 ? [path] : [path, relatedPath];
|
|
661
662
|
return await this.serializedByFileEventPaths(eventPaths, async () => {
|
|
662
663
|
switch (type) {
|
|
663
664
|
case "CREATE":
|
|
664
665
|
case "CHANGED":
|
|
665
666
|
return await this.storeFileToDB(item.args.file);
|
|
666
667
|
case "DELETE":
|
|
668
|
+
if (!await this.canRecordStorageDeletion(item)) return true;
|
|
667
669
|
return await this.deleteFileFromDB(item.args.file);
|
|
668
670
|
case "RENAME":
|
|
669
671
|
return await this.renameFileInDB(
|
|
@@ -678,6 +680,45 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
678
680
|
}
|
|
679
681
|
});
|
|
680
682
|
}
|
|
683
|
+
async canRecordStorageDeletion(item) {
|
|
684
|
+
const path = item.args.file.path;
|
|
685
|
+
try {
|
|
686
|
+
const current = await this.storage.getStub(path);
|
|
687
|
+
let excludedRenameTarget = null;
|
|
688
|
+
if (item.args.renameTarget !== void 0) {
|
|
689
|
+
const targetPath = item.args.renameTarget;
|
|
690
|
+
const targetItem = await this.storage.getStub(targetPath);
|
|
691
|
+
const target = this.getExactCurrentFile(targetItem, targetPath);
|
|
692
|
+
if (targetItem !== null && !target) {
|
|
693
|
+
this._log(`Preserved ${path}: the rename target no longer resolves to ${targetPath}`, LOG_LEVEL_NOTICE);
|
|
694
|
+
return false;
|
|
695
|
+
}
|
|
696
|
+
const targetIsSelected = !shouldBeIgnored(targetPath) && await this.vault.isTargetFile(targetPath, { skipCaseCollisionCheck: true }) && !(target && this.vault.isFileSizeTooLarge(target.stat.size));
|
|
697
|
+
if (targetIsSelected) {
|
|
698
|
+
this._log(`Preserved ${path}: rename target ${targetPath} is still selected`, LOG_LEVEL_NOTICE);
|
|
699
|
+
return false;
|
|
700
|
+
}
|
|
701
|
+
excludedRenameTarget = target;
|
|
702
|
+
}
|
|
703
|
+
if (current === null) return true;
|
|
704
|
+
if (!isFolderInfo(current)) {
|
|
705
|
+
const [currentId, deletedId] = await Promise.all([
|
|
706
|
+
this.path.path2id(current.path),
|
|
707
|
+
this.path.path2id(path)
|
|
708
|
+
]);
|
|
709
|
+
if (currentId !== deletedId) return true;
|
|
710
|
+
if (excludedRenameTarget && this.storage.normalisePath(current.path) !== this.storage.normalisePath(path) && this.storage.normalisePath(current.path) === this.storage.normalisePath(excludedRenameTarget.path)) {
|
|
711
|
+
return true;
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
this._log(`Preserved ${path}: a current storage item contradicts its deletion event`, LOG_LEVEL_NOTICE);
|
|
715
|
+
return false;
|
|
716
|
+
} catch (ex) {
|
|
717
|
+
this._log(`Could not validate deletion of ${path}; preserved the database entry`, LOG_LEVEL_NOTICE);
|
|
718
|
+
this._log(ex, LOG_LEVEL_VERBOSE);
|
|
719
|
+
return false;
|
|
720
|
+
}
|
|
721
|
+
}
|
|
681
722
|
async serializedByFileEventPaths(eventPaths, callback) {
|
|
682
723
|
const documentIds = await Promise.all(eventPaths.map((eventPath) => this.path.path2id(eventPath)));
|
|
683
724
|
const lockKeys = [...new Set(documentIds)].sort().map((documentId) => `processFileEvent-${documentId}`);
|
|
@@ -702,7 +743,8 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
702
743
|
this._log(`Rename event for ${path} has no source path`, LOG_LEVEL_VERBOSE);
|
|
703
744
|
return true;
|
|
704
745
|
}
|
|
705
|
-
const
|
|
746
|
+
const relatedPath = type === "RENAME" ? eventItem.oldPath : type === "DELETE" ? eventItem.renameTarget : void 0;
|
|
747
|
+
const eventPaths = relatedPath === void 0 ? [path] : [path, relatedPath];
|
|
706
748
|
return await this.serializedByFileEventPaths(eventPaths, async (isSameDocument) => {
|
|
707
749
|
let action;
|
|
708
750
|
try {
|
|
@@ -732,6 +774,9 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
|
|
|
732
774
|
return current && await this.isCurrentFileSelected(current) ? { kind: "store", file: current } : { kind: "none" };
|
|
733
775
|
}
|
|
734
776
|
case "DELETE": {
|
|
777
|
+
if (item.args.renameTarget !== void 0) {
|
|
778
|
+
return await this.canRecordStorageDeletion(item) && await this.canApplyRestoredDeletion(path) ? { kind: "delete", path } : { kind: "none" };
|
|
779
|
+
}
|
|
735
780
|
const current = await this.storage.getStub(path);
|
|
736
781
|
return current === null && await this.canApplyRestoredDeletion(path) ? { kind: "delete", path } : { kind: "none" };
|
|
737
782
|
}
|
|
@@ -34,5 +34,5 @@ export declare abstract class DatabaseEventService<T extends ServiceContext = Se
|
|
|
34
34
|
* @param reopenDatabase Whether to reopen the database if it is already open.
|
|
35
35
|
* @param ignoreSuspending Whether to ignore any suspending state.
|
|
36
36
|
*/
|
|
37
|
-
readonly initialiseDatabase: import("../lib/HandlerUtils.js").
|
|
37
|
+
readonly initialiseDatabase: import("../lib/HandlerUtils.js").MultipleHandlerFunction<(showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean) => Promise<import("./VaultScanResult.js").VaultScanResult>, unknown>;
|
|
38
38
|
}
|
|
@@ -31,7 +31,7 @@ class DatabaseEventService extends ServiceBase {
|
|
|
31
31
|
* @param reopenDatabase Whether to reopen the database if it is already open.
|
|
32
32
|
* @param ignoreSuspending Whether to ignore any suspending state.
|
|
33
33
|
*/
|
|
34
|
-
initialiseDatabase = handlers().
|
|
34
|
+
initialiseDatabase = handlers().bailFirstFailureWithResult("initialiseDatabase");
|
|
35
35
|
}
|
|
36
36
|
export {
|
|
37
37
|
DatabaseEventService
|
|
@@ -16,6 +16,7 @@ import type { ActiveReplicatorContext, ContinuousReplicationRequest, Replication
|
|
|
16
16
|
import type { RemoteResourceKind, RemoteResourceMap } from "../../replication/RemoteResource.js";
|
|
17
17
|
import type { CentralRemoteAdministrationRequest, CentralRemoteAdministrationResult } from "../../replication/CentralRemoteAdministration.js";
|
|
18
18
|
import type { MultipleHandlerFunction } from "../lib/HandlerUtils.js";
|
|
19
|
+
import type { VaultScanResult } from "./VaultScanResult.js";
|
|
19
20
|
declare global {
|
|
20
21
|
interface OPTIONAL_SYNC_FEATURES {
|
|
21
22
|
DISABLE: "DISABLE";
|
|
@@ -86,7 +87,7 @@ export interface IDatabaseEventService {
|
|
|
86
87
|
onDatabaseInitialised(showNotice: boolean): Promise<boolean>;
|
|
87
88
|
onDatabaseHasReady(): Promise<boolean>;
|
|
88
89
|
onResetDatabase(db: LiveSyncLocalDB): Promise<boolean>;
|
|
89
|
-
initialiseDatabase(showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean): Promise<
|
|
90
|
+
initialiseDatabase(showingNotice?: boolean, reopenDatabase?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean): Promise<VaultScanResult>;
|
|
90
91
|
}
|
|
91
92
|
export interface IKeyValueDBService {
|
|
92
93
|
openSimpleStore<T>(kind: string): SimpleStore<T>;
|
|
@@ -348,9 +349,13 @@ export interface ITweakValueService {
|
|
|
348
349
|
export interface IVaultService {
|
|
349
350
|
vaultName(): string;
|
|
350
351
|
getVaultName(): string;
|
|
351
|
-
scanVault(showingNotice?: boolean, ignoreSuspending?: boolean): Promise<
|
|
352
|
+
scanVault(showingNotice?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean): Promise<VaultScanResult>;
|
|
352
353
|
isIgnoredByIgnoreFile(file: string | UXFileInfoStub): Promise<boolean>;
|
|
353
|
-
isTargetFile(file: string | UXFileInfoStub
|
|
354
|
+
isTargetFile(file: string | UXFileInfoStub,
|
|
355
|
+
/** Inspect selection policy without treating a filename collision as an exclusion. */
|
|
356
|
+
options?: {
|
|
357
|
+
skipCaseCollisionCheck?: boolean;
|
|
358
|
+
}): Promise<boolean>;
|
|
354
359
|
isTargetFileInExtra(file: string | UXFileInfoStub): Promise<boolean>;
|
|
355
360
|
isFileSizeTooLarge(size: number): boolean;
|
|
356
361
|
getActiveFilePath(): FilePath | undefined;
|
|
@@ -20,7 +20,7 @@ function createReplicationReadinessEvaluator(dependencies) {
|
|
|
20
20
|
name: "application-ready",
|
|
21
21
|
evaluate: () => {
|
|
22
22
|
if (gates.isApplicationReady()) return true;
|
|
23
|
-
diagnostics.log("
|
|
23
|
+
diagnostics.log("Replication is not ready because application initialisation has not completed.");
|
|
24
24
|
return false;
|
|
25
25
|
}
|
|
26
26
|
},
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Outcome of scanning the Vault against the local database.
|
|
3
|
+
*
|
|
4
|
+
* Boolean values preserve the existing success and failure contract. The
|
|
5
|
+
* additional value lets an ordinary start-up remain ready while reporting
|
|
6
|
+
* that one or more individual files still require attention.
|
|
7
|
+
*/
|
|
8
|
+
export declare const VaultScanResults: {
|
|
9
|
+
readonly FAILED: false;
|
|
10
|
+
readonly COMPLETED: true;
|
|
11
|
+
readonly COMPLETED_WITH_FILE_FAILURES: "completed-with-file-failures";
|
|
12
|
+
};
|
|
13
|
+
export type VaultScanResult = (typeof VaultScanResults)[keyof typeof VaultScanResults];
|
|
@@ -26,7 +26,7 @@ export declare abstract class VaultService<T extends ServiceContext = ServiceCon
|
|
|
26
26
|
* @param showingNotice Whether to show a notice to the user.
|
|
27
27
|
* @param ignoreSuspending Whether to ignore any suspending state.
|
|
28
28
|
*/
|
|
29
|
-
readonly scanVault: import("../lib/HandlerUtils.js").
|
|
29
|
+
readonly scanVault: import("../lib/HandlerUtils.js").MultipleHandlerFunction<(showingNotice?: boolean, ignoreSuspending?: boolean, continueOnFileFailure?: boolean) => Promise<import("./VaultScanResult.js").VaultScanResult>, unknown>;
|
|
30
30
|
/**
|
|
31
31
|
* Check if a file is ignored by the ignore file (e.g., .gitignore, .obsidianignore).
|
|
32
32
|
* @param file The file path or file info stub to check.
|
|
@@ -35,9 +35,11 @@ export declare abstract class VaultService<T extends ServiceContext = ServiceCon
|
|
|
35
35
|
/**
|
|
36
36
|
* Check if a file is a target file for synchronisation.
|
|
37
37
|
* @param file The file path or file info stub to check.
|
|
38
|
-
* @param
|
|
38
|
+
* @param options Optional selection inspection without filename collision rejection.
|
|
39
39
|
*/
|
|
40
|
-
readonly isTargetFile: import("../lib/HandlerUtils.js").BooleanMultipleHandlerFunction<(file: string | import("../../common/types.js").UXFileInfoStub
|
|
40
|
+
readonly isTargetFile: import("../lib/HandlerUtils.js").BooleanMultipleHandlerFunction<(file: string | import("../../common/types.js").UXFileInfoStub, options?: {
|
|
41
|
+
skipCaseCollisionCheck?: boolean;
|
|
42
|
+
}) => Promise<boolean>>;
|
|
41
43
|
/**
|
|
42
44
|
* Check if a file is a target file for some extra feature
|
|
43
45
|
*/
|
|
@@ -28,7 +28,7 @@ class VaultService extends ServiceBase {
|
|
|
28
28
|
* @param showingNotice Whether to show a notice to the user.
|
|
29
29
|
* @param ignoreSuspending Whether to ignore any suspending state.
|
|
30
30
|
*/
|
|
31
|
-
scanVault = handlers().
|
|
31
|
+
scanVault = handlers().bailFirstFailureWithResult("scanVault");
|
|
32
32
|
/**
|
|
33
33
|
* Check if a file is ignored by the ignore file (e.g., .gitignore, .obsidianignore).
|
|
34
34
|
* @param file The file path or file info stub to check.
|
|
@@ -37,7 +37,7 @@ class VaultService extends ServiceBase {
|
|
|
37
37
|
/**
|
|
38
38
|
* Check if a file is a target file for synchronisation.
|
|
39
39
|
* @param file The file path or file info stub to check.
|
|
40
|
-
* @param
|
|
40
|
+
* @param options Optional selection inspection without filename collision rejection.
|
|
41
41
|
*/
|
|
42
42
|
isTargetFile = handlers().bailFirstFailure("isTargetFile");
|
|
43
43
|
/**
|
|
@@ -182,6 +182,19 @@ export declare class AllHandler<T extends unknown[]> extends BooleanHandlerBase<
|
|
|
182
182
|
*/
|
|
183
183
|
invoke(...args: T): Promise<boolean>;
|
|
184
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* A handler that preserves a non-boolean continuation result while continuing
|
|
187
|
+
* to give false results and exceptions veto power.
|
|
188
|
+
*/
|
|
189
|
+
export declare class AllWithResultHandler<T extends unknown[], U extends boolean | string> extends MultiBinder<BooleanHandlerFunc<T, U>> implements InvokableHandler<T, U> {
|
|
190
|
+
/**
|
|
191
|
+
* Invokes every handler in order. The first non-boolean continuation result
|
|
192
|
+
* is retained, but every remaining handler can still reject the operation.
|
|
193
|
+
* @param args The arguments to pass to the handlers.
|
|
194
|
+
* @returns False after any failure, otherwise the retained result or true.
|
|
195
|
+
*/
|
|
196
|
+
invoke(...args: T): Promise<U>;
|
|
197
|
+
}
|
|
185
198
|
/**
|
|
186
199
|
* A handler that invokes every added handler sequentially and reports whether
|
|
187
200
|
* all of them succeeded.
|
|
@@ -276,7 +289,7 @@ export interface MultipleHandlerFunction<TFunc extends (...args: any[]) => U | P
|
|
|
276
289
|
* @param callback The handler function to add.
|
|
277
290
|
* @returns A function to remove the added handler.
|
|
278
291
|
*/
|
|
279
|
-
addHandler: (callback: TFunc) => () => void;
|
|
292
|
+
addHandler: (callback: TFunc, priority?: number) => () => void;
|
|
280
293
|
/**
|
|
281
294
|
* Removes a handler function.
|
|
282
295
|
* @param callback The handler function to remove.
|
|
@@ -334,6 +347,7 @@ export interface BooleanMultiBinderInstance<T extends unknown[]> extends Invokab
|
|
|
334
347
|
export declare function allFunction<TFunc extends (...args: any[]) => Promise<boolean>>(name?: string): BooleanMultipleHandlerFunction<TFunc>;
|
|
335
348
|
export declare function allSettledFunction<TFunc extends (...args: any[]) => Promise<boolean>>(name?: string): BooleanMultipleHandlerFunction<TFunc>;
|
|
336
349
|
export declare function bailFirstFailureFunction<TFunc extends (...args: any[]) => Promise<boolean>>(name?: string): BooleanMultipleHandlerFunction<TFunc>;
|
|
350
|
+
export declare function bailFirstFailureWithResultFunction<TFunc extends (...args: any[]) => Promise<boolean | string>>(name?: string): MultipleHandlerFunction<TFunc>;
|
|
337
351
|
export declare function allParallelFunction<TFunc extends (...args: any[]) => Promise<boolean>>(name?: string): BooleanMultipleHandlerFunction<TFunc>;
|
|
338
352
|
export declare function anySuccessFunction<TFunc extends (...args: any[]) => Promise<boolean>>(name?: string): BooleanMultipleHandlerFunction<TFunc>;
|
|
339
353
|
export declare function firstResultFunction<TFunc extends (...args: any[]) => Promise<unknown>>(name?: string): MultipleHandlerFunction<TFunc>;
|
|
@@ -369,6 +383,14 @@ export declare function handlers<T extends object>(): {
|
|
|
369
383
|
* @returns
|
|
370
384
|
*/
|
|
371
385
|
bailFirstFailure<K extends FunctionKeys<T>>(name: K): BooleanMultipleHandlerFunction<Extract<T[K], (...args: any[]) => Promise<boolean>>>;
|
|
386
|
+
/**
|
|
387
|
+
* Create a handler that preserves a non-boolean continuation result, but
|
|
388
|
+
* still invokes later handlers so that false results and exceptions
|
|
389
|
+
* can reject the operation.
|
|
390
|
+
* @param name
|
|
391
|
+
* @returns
|
|
392
|
+
*/
|
|
393
|
+
bailFirstFailureWithResult<K extends FunctionKeys<T>>(name: K): MultipleHandlerFunction<Extract<T[K], (...args: any[]) => Promise<boolean | string>>>;
|
|
372
394
|
/**
|
|
373
395
|
* Create a handler that invokes all added handler functions sequentially until one returns true.
|
|
374
396
|
* @param name
|
|
@@ -207,6 +207,34 @@ class AllHandler extends BooleanHandlerBase {
|
|
|
207
207
|
return true;
|
|
208
208
|
}
|
|
209
209
|
}
|
|
210
|
+
class AllWithResultHandler extends MultiBinder {
|
|
211
|
+
/**
|
|
212
|
+
* Invokes every handler in order. The first non-boolean continuation result
|
|
213
|
+
* is retained, but every remaining handler can still reject the operation.
|
|
214
|
+
* @param args The arguments to pass to the handlers.
|
|
215
|
+
* @returns False after any failure, otherwise the retained result or true.
|
|
216
|
+
*/
|
|
217
|
+
async invoke(...args) {
|
|
218
|
+
let aggregateResult = true;
|
|
219
|
+
const callbacks = [...this._callbacks];
|
|
220
|
+
for (const callback of callbacks) {
|
|
221
|
+
try {
|
|
222
|
+
const result = await Promise.resolve(callback(...args));
|
|
223
|
+
if (result === false) {
|
|
224
|
+
return false;
|
|
225
|
+
}
|
|
226
|
+
if (aggregateResult === true && result !== true) {
|
|
227
|
+
aggregateResult = result;
|
|
228
|
+
}
|
|
229
|
+
} catch (error) {
|
|
230
|
+
Logger(`AllWithResultHandler ${this._name} treated error as failure!`, LOG_LEVEL_VERBOSE);
|
|
231
|
+
Logger(error, LOG_LEVEL_VERBOSE);
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return aggregateResult;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
210
238
|
class SettledAllHandler extends BooleanHandlerBase {
|
|
211
239
|
/**
|
|
212
240
|
* Invokes every handler in order, treating a false result or an exception
|
|
@@ -321,6 +349,12 @@ function bailFirstFailureFunction(name) {
|
|
|
321
349
|
const handler = new AllHandler(name ?? "bailFirstFailureFunction");
|
|
322
350
|
return getMultipleBound(handler);
|
|
323
351
|
}
|
|
352
|
+
function bailFirstFailureWithResultFunction(name) {
|
|
353
|
+
const handler = new AllWithResultHandler(
|
|
354
|
+
name ?? "bailFirstFailureWithResultFunction"
|
|
355
|
+
);
|
|
356
|
+
return getMultipleBound(handler);
|
|
357
|
+
}
|
|
324
358
|
function allParallelFunction(name) {
|
|
325
359
|
const handler = new ParallelAllHandler(name ?? "allParallelFunction");
|
|
326
360
|
return getMultipleBound(handler);
|
|
@@ -390,6 +424,18 @@ function handlers() {
|
|
|
390
424
|
bailFirstFailure(name) {
|
|
391
425
|
return bailFirstFailureFunction(String(name));
|
|
392
426
|
},
|
|
427
|
+
/**
|
|
428
|
+
* Create a handler that preserves a non-boolean continuation result, but
|
|
429
|
+
* still invokes later handlers so that false results and exceptions
|
|
430
|
+
* can reject the operation.
|
|
431
|
+
* @param name
|
|
432
|
+
* @returns
|
|
433
|
+
*/
|
|
434
|
+
bailFirstFailureWithResult(name) {
|
|
435
|
+
return bailFirstFailureWithResultFunction(
|
|
436
|
+
String(name)
|
|
437
|
+
);
|
|
438
|
+
},
|
|
393
439
|
/**
|
|
394
440
|
* Create a handler that invokes all added handler functions sequentially until one returns true.
|
|
395
441
|
* @param name
|
|
@@ -429,6 +475,7 @@ function handlers() {
|
|
|
429
475
|
}
|
|
430
476
|
export {
|
|
431
477
|
AllHandler,
|
|
478
|
+
AllWithResultHandler,
|
|
432
479
|
AnySuccessHandler,
|
|
433
480
|
Binder,
|
|
434
481
|
BooleanHandlerBase,
|
|
@@ -444,6 +491,7 @@ export {
|
|
|
444
491
|
allSettledFunction,
|
|
445
492
|
anySuccessFunction,
|
|
446
493
|
bailFirstFailureFunction,
|
|
494
|
+
bailFirstFailureWithResultFunction,
|
|
447
495
|
bindableFunction,
|
|
448
496
|
dispatchParallelFunction,
|
|
449
497
|
firstResultFunction,
|
package/docs/development.md
CHANGED
|
@@ -83,6 +83,8 @@ The multiple-remote profile contract has focused unit coverage in `src/serviceFe
|
|
|
83
83
|
|
|
84
84
|
The full offline scanner applies both the host's target-file policy and Commonlib's built-in `shouldBeIgnored` policy before constructing storage/database pairs. This keeps legacy LiveSync log files and flag files outside ordinary create, restore, and delete decisions even when a host accepts their file extension. Lower-level write guards remain a defence for direct calls; they are not the normal full-scan filter. The complete pair-state decision matrix and database-to-storage reflection result contract are documented in [Full offline scan](offline-scanner.md). Extend that document, the Cartesian action test, and both collection-side tests whenever the policy changes.
|
|
85
85
|
|
|
86
|
+
The unreleased watcher deletion checks and the separate database-to-storage reflection path are recorded in [Storage events and database-to-storage reflection](storage-events-and-reflection.md).
|
|
87
|
+
|
|
86
88
|
The revision-tree safety rules are documented in [Conflict resolution and file provenance](conflict-resolution.md). `ConflictManager.unit.spec.ts` owns common-ancestor and conservative merge behaviour, deterministic pair selection for multiple live leaves, and reconstruction from the committed tree between stages. `EntryManagerImpls.unit.spec.ts` owns the distinction between force-writing below an exact revision and conditionally advancing a current live leaf through PouchDB's ordinary revision check. `ServiceDatabaseFileAccessBase.unit.spec.ts` owns all-branch content discovery, short-circuiting history existence checks after an exact match, exact-revision logical deletion, and the public live-base result and event boundary. `ServiceFileHandlerBase.unit.spec.ts` owns the injected provenance policy for edit, deletion, and rename operations, together with user-selected reflection, child creation, strict matching, matching-provenance cleanup after exact branch deletion, and stale-revision refusal. `LiveSyncLocalDB.unit.spec.ts` owns chunk reachability across current winners, shared chunks, live conflicts, available merge ancestry, conflict resolution, and PouchDB replication of collection and recreation. Extend the relevant sides when changing conflict resolution, compaction handling, provenance, chunk collection, or database-to-storage overwrite protection.
|
|
87
89
|
|
|
88
90
|
`LiveSyncReplicator.unit.spec.ts` owns the CouchDB remote-chunk batch contract: a mixed `_all_docs` result must return each available document even when another keyed row reports an error. `ChunkFetcher.unit.spec.ts` owns the consumer behaviour: it writes valid returned chunks and emits `missingChunkRemote` only for requested IDs which are absent from the returned set. Extend both sides when changing remote chunk acquisition or missing-chunk delivery.
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
---
|
|
2
|
+
date: 2026-09-04
|
|
3
|
+
commonlib-version: "0.1.21"
|
|
4
|
+
self-hosted-livesync-version: "1.0.24"
|
|
5
|
+
status: unreleased
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Full offline scan
|
|
9
|
+
|
|
10
|
+
This document defines the developer contract for `synchroniseAllFilesBetweenDBandStorage`. It describes how the scanner classifies each selected path, chooses an action, and records a database-to-storage reflection. Host-specific setup dialogue and recovery choices remain outside this contract.
|
|
11
|
+
|
|
12
|
+
Before applying the host's target-file policy, the scanner validates each decoded Metadata document against its actual local document ID. Consistent internal, customisation, and plug-in storage namespaces remain owned by their dedicated features. A namespace disagreement, or a normal-file ID which does not match the ID derived from its recorded path, is quarantined.
|
|
13
|
+
|
|
14
|
+
The malformed Metadata entry never enters pair processing. If consistent, selected Metadata represents the same case-normalised path, that entry and its storage file continue through the established path-based flow. Otherwise, the storage path is also quarantined. The scanner performs no storage write, database deletion, or last-seen update for an unresolved path, and it retains expired logical deletion history whose identity is inconsistent. This does not add a fourth pair result: quarantined entries do not enter pair processing, and the complete-scan result retains its meaning.
|
|
15
|
+
|
|
16
|
+
`inspectMetadataDocumentIdentities` provides the separate read-only, actual-ID-first report used by a host repair interface. `repairMetadataDocumentIdentity` revalidates one exact source revision, clears its last-seen state, writes and verifies the expected target, and only then tombstones the obsolete ID. It does not provide batch repair. An exact target left by an interrupted attempt permits the same one-entry repair to finish safely.
|
|
17
|
+
|
|
18
|
+
The tables below cover only paths which remain after identity validation, the host's target-file policy, and Commonlib's built-in exclusions.
|
|
19
|
+
|
|
20
|
+
## Full-scan lifecycle
|
|
21
|
+
|
|
22
|
+
`performFullScan` owns the lifecycle around the pair processor:
|
|
23
|
+
|
|
24
|
+
1. confirm that settings permit the scan;
|
|
25
|
+
2. read the scanner's `initialized` marker;
|
|
26
|
+
3. when the marker is present, load and complete the stored storage-event operations from the previous runtime;
|
|
27
|
+
4. process expired logical deletion history;
|
|
28
|
+
5. collect and process the selected storage and database entries;
|
|
29
|
+
6. after the first permitted invocation reaches its aggregate-result boundary, record `initialized = true`; and
|
|
30
|
+
7. return the aggregate pair result under the caller's selected handling of file failures.
|
|
31
|
+
|
|
32
|
+
The `initialized` marker records that a full-scan invocation has reached its aggregate-result boundary and that later invocations may restore stored storage-event operations. It is not a successful-scan marker: the first invocation records it after an aggregate `false` result, `true`, or an accepted partial result. An exception before the aggregate result is produced does not reach that write.
|
|
33
|
+
|
|
34
|
+
Storage-event restoration is a bounded start-up recovery step. Restored events bypass their former batch delay, but `restoreState()` waits for their actual file operations to finish and preserves snapshot sentinel ordering before the file collections are compared. Saved events run first because a deletion or rename can carry operation intent which is no longer visible in the current storage listing. An individual restored operation failure is logged and does not prevent the scan from examining the resulting current state. The following full scan then makes the final reconciliation decision.
|
|
35
|
+
|
|
36
|
+
This ordering does not mean that storage unconditionally overwrites the replay result. The action matrix remains authoritative: when a replayed operation leaves an ambiguous pair, such as a storage file alongside a deleted database entry, the selected policy may deliberately return `skipped` rather than guess which side is intended. This boundary does not include events which are buffered or received later, and it must not be confused with a host's separate replication-result queue.
|
|
37
|
+
|
|
38
|
+
### Stored event interpretation
|
|
39
|
+
|
|
40
|
+
The storage-event snapshot contains operations which the previous runtime observed but had not begun or which were still waiting for their batch boundary. It is not a file-content journal and does not discover changes made while the application was stopped. An operation which had already been removed from the queue and entered file processing is also outside the persisted boundary. The following full scan remains responsible for observing the current storage state across all of these gaps.
|
|
41
|
+
|
|
42
|
+
A restored event is an operation intent which must be revalidated against the current exact storage path before it can cause a database side effect. Its saved file stub, timestamps, and optional editor cache describe the earlier observation; they are not authoritative file content after a restart. Whenever an inclusion is still applicable, the handler reads a fresh stub and content from current storage. Whenever an ordinary deletion's expected absence is contradicted by a current storage item, that deletion is suppressed and the full scan evaluates that current item instead.
|
|
43
|
+
|
|
44
|
+
For this check, an exact path means that the canonical path returned by storage has the saved spelling after ordinary separator normalisation. A file returned only through a case-insensitive lookup does not prove the saved inclusion path. A same-ID rename remains the explicit case-change path described below.
|
|
45
|
+
|
|
46
|
+
The standard-file revalidation contract is:
|
|
47
|
+
|
|
48
|
+
| Saved operation | Current exact storage state | Operation admitted before the full scan |
|
|
49
|
+
| --------------------------------------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
|
50
|
+
| `CREATE` or `CHANGED` | The path is still a selected regular file | Store the current stub and content; do not use saved content as a historical version |
|
|
51
|
+
| `CREATE` or `CHANGED` | The path is absent, no longer selected, or no longer a regular file | Do not replay the inclusion |
|
|
52
|
+
| `DELETE` without `renameTarget` | The selected path remains absent | Apply the saved path's deletion intent |
|
|
53
|
+
| `DELETE` without `renameTarget` | Any current storage item occupies the path | Suppress the stale deletion |
|
|
54
|
+
| `DELETE` retaining `renameTarget` | Current source selection and rename-exclusion checks permit deletion | Apply the old document's deletion, including a deliberately excluded same-ID destination |
|
|
55
|
+
| `RENAME` between distinct document IDs | The new selected path exists | Admit the inclusion half using the new path's current stub and content |
|
|
56
|
+
| `RENAME` between distinct document IDs | The old selected path is absent | Admit the deletion half for the old path |
|
|
57
|
+
| `RENAME` between distinct document IDs | Either half is contradicted by current storage | Suppress only that half; the other independently validated half may proceed |
|
|
58
|
+
| Case-only or otherwise same-ID `RENAME` | A current selected file resolves to the renamed canonical path | Update that single document's path and current content without a separate deletion |
|
|
59
|
+
| Case-only or otherwise same-ID `RENAME` | The current identity does not support the saved rename | Do not replay the rename; let the full scan evaluate the current path |
|
|
60
|
+
|
|
61
|
+
An inability to inspect the current path is not proof of absence. Revalidation therefore suppresses a destructive half when storage inspection fails, records the failure, and leaves the following scan to report whether reconciliation can proceed. Target-file policy and file-size limits are evaluated from current settings rather than inherited from the saved event.
|
|
62
|
+
|
|
63
|
+
The unreleased rename-exclusion guard retains `renameTarget` when a rename is converted to deletion during target filtering. Unlike an ordinary deletion, this intent can remove a file from synchronisation without removing it from storage. The guard distinguishes deliberate exclusion from transient case-collision rejection, preserves a recreated source, and permits a deliberately excluded same-ID destination without requiring that destination to disappear. Older snapshots without this optional field retain the absence-only rule. See [Storage events and database-to-storage reflection](storage-events-and-reflection.md) for the exact admission conditions and the separate ordinary watcher boundary.
|
|
64
|
+
|
|
65
|
+
Independent validation of rename halves does not remove target-first execution. When the new path currently exists, its inclusion must finish successfully before the old-path deletion may run. A failed inclusion preserves the old database entry. When the new path is currently absent, there is no inclusion to publish, and a separately validated absence at the old path may be applied as deletion intent.
|
|
66
|
+
|
|
67
|
+
`INTERNAL` operations remain owned by the host feature which registered the optional-file handler. That owner must read its current internal storage state; the ordinary file-pair scan does not make a recovery guarantee for an internal path. `SENTINEL_FLUSH` carries ordering only and never represents file state.
|
|
68
|
+
|
|
69
|
+
The full scan is not a transaction. Expired deletion history may be updated before a later pair fails, and the device-local last-seen map is saved asynchronously after pair processing. The aggregate result does not promise rollback of those earlier side effects or durable completion of that deferred map save.
|
|
70
|
+
|
|
71
|
+
## Pair states
|
|
72
|
+
|
|
73
|
+
| State | Storage | Local database |
|
|
74
|
+
| ----------------- | ------------------- | -------------------------------- |
|
|
75
|
+
| `both` | File exists | Live metadata entry exists |
|
|
76
|
+
| `storage-only` | File exists | No metadata entry exists |
|
|
77
|
+
| `db-only` | File does not exist | Live metadata entry exists |
|
|
78
|
+
| `both-db-deleted` | File exists | Entry records a logical deletion |
|
|
79
|
+
| `db-only-deleted` | File does not exist | Entry records a logical deletion |
|
|
80
|
+
|
|
81
|
+
A metadata entry with live conflicts is skipped before the ordinary action is selected. It must not be applied to storage by a full scan.
|
|
82
|
+
|
|
83
|
+
## Option predicates
|
|
84
|
+
|
|
85
|
+
The action matrix uses these predicates:
|
|
86
|
+
|
|
87
|
+
- `delete-when-remote-missing` is enabled by `ExtraOnRemote.DELETE_LOCAL_MISSING` or `ExtraOnLocal.DELETE_DB_MISSING`;
|
|
88
|
+
- `delete-when-remote-deleted` is enabled by `ExtraOnRemote.DELETE_LOCAL_MISSING`, `ExtraOnLocal.DELETE_DB_DELETED`, or `ExtraOnLocal.DELETE_DB_MISSING`; and
|
|
89
|
+
- `append-storage-only` is enabled by `ExtraOnLocal.APPEND_STORAGE_ONLY`.
|
|
90
|
+
|
|
91
|
+
These names explain the table only. The source constants remain the API.
|
|
92
|
+
|
|
93
|
+
## Action matrix
|
|
94
|
+
|
|
95
|
+
| Pair state | `DB_APPLY` | `NEWER_WINS` |
|
|
96
|
+
| ----------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- |
|
|
97
|
+
| `both` | `update-storage` | `sync-newer` |
|
|
98
|
+
| `storage-only` | `delete-local` when `delete-when-remote-missing`; otherwise `skip` | `delete-local` when `delete-when-remote-missing`; otherwise `update-db` |
|
|
99
|
+
| `db-only` | `update-storage` | `update-storage`, subject to the device-local offline-deletion override described below |
|
|
100
|
+
| `both-db-deleted` | `delete-local` when `delete-when-remote-deleted`; otherwise `skip` | `delete-local` when `delete-when-remote-deleted`; otherwise `update-db` when `append-storage-only`, or `skip` |
|
|
101
|
+
| `db-only-deleted` | `skip` | `skip` |
|
|
102
|
+
|
|
103
|
+
`NEWER_WINS` changes a `db-only` action to `delete-db` only when the device-local last-seen map proves that the file existed on an earlier scan and that observed storage mtime is not older than the database mtime under `compareMTime`. This represents a file deleted while the scanner was offline. A database mtime copied from a failed reflection is not such evidence.
|
|
104
|
+
|
|
105
|
+
The Cartesian `resolveFilePairAction` unit test is the executable counterpart of this matrix. Change the implementation, table, and test together.
|
|
106
|
+
|
|
107
|
+
## Pair processing result
|
|
108
|
+
|
|
109
|
+
Each selected pair finishes in one of three states:
|
|
110
|
+
|
|
111
|
+
| Result | Meaning | Complete-scan result |
|
|
112
|
+
| ----------- | ------------------------------------------------------------------------------------- | -------------------- |
|
|
113
|
+
| `completed` | The selected action completed, or storage and the database were already equal | Success |
|
|
114
|
+
| `skipped` | Scanner policy deliberately left the pair unchanged | Success |
|
|
115
|
+
| `failed` | The selected action could not complete, returned a failure result, or raised an error | Failure |
|
|
116
|
+
|
|
117
|
+
Conflict guards, size limits, and the explicit `skip` action produce `skipped`. They allow the scan to finish successfully, but do not establish that database content was reflected to storage. In particular, skipping a `db-only` entry must not copy its database mtime into the device-local last-seen map.
|
|
118
|
+
|
|
119
|
+
An actual storage file remains valid evidence of its own observed mtime even when another action for that pair is skipped. This is distinct from recording the database mtime after a successful database-to-storage reflection.
|
|
120
|
+
|
|
121
|
+
## Database-to-storage reflection result
|
|
122
|
+
|
|
123
|
+
When `update-storage` invokes `dbToStorage`, its result controls every success side effect:
|
|
124
|
+
|
|
125
|
+
| `dbToStorage` outcome | Pair result | Success event and log | Last-seen entry for the database mtime | Later `NEWER_WINS` scan |
|
|
126
|
+
| --------------------- | ----------- | --------------------- | -------------------------------------- | ------------------------------------------------- |
|
|
127
|
+
| `true` | `completed` | Emitted | Recorded | May recognise a later absence as a local deletion |
|
|
128
|
+
| `false` | `failed` | Not emitted | Not recorded | Retries `update-storage` while the file is absent |
|
|
129
|
+
| Throws | `failed` | Not emitted | Not recorded | Retries after the failed scan can be run again |
|
|
130
|
+
|
|
131
|
+
The same result applies when `sync-newer` finds an existing storage file and attempts to reflect a newer database entry. A `false` result must not be converted into `completed`, emit a success event, or replace the observed storage mtime with the database mtime.
|
|
132
|
+
|
|
133
|
+
By default, the complete scan returns `false` when any pair is `failed`, and returns `true` when every pair is either `completed` or `skipped`. Quarantined entries do not enter this aggregate, so a strict `true` means that no selected pair failed rather than that every discovered entry was resolved or that storage and the database fully converge.
|
|
134
|
+
|
|
135
|
+
`continueOnFileFailure` is an explicit host completion policy. When it is `true`, the scanner still processes every selected pair, logs each failed path at verbose level, preserves every failed pair and its retry state, and then returns `completed-with-file-failures` from the aggregate boundary. The existing completion log records the total failed count. The option does not turn a failed pair into `completed`, create success events or last-seen evidence, bypass a rejected scan precondition, or accept an exception raised before the aggregate boundary.
|
|
136
|
+
|
|
137
|
+
`VaultScanResult` therefore has three outcomes: `false` for a rejected or strict failure, `true` for a complete scan without pair failures, and `completed-with-file-failures` when the caller explicitly accepts readiness after one or more pair failures. The non-boolean outcome lets the host present an actionable warning without making Commonlib responsible for user-facing notices.
|
|
138
|
+
|
|
139
|
+
Self-hosted LiveSync enables this policy only for ordinary Obsidian start-up, where one unavailable path must not prevent unaffected files from synchronising. Fast Setup, Fetch, Rebuild, direct scans, and CLI mirror or daemon paths retain the strict default.
|
|
140
|
+
|
|
141
|
+
## Result-state verification
|
|
142
|
+
|
|
143
|
+
Focused two-pass coverage starts with a `DB_APPLY` reflection whose file handler returns `false`, persists the resulting scanner state, and then runs `NEWER_WINS`. It verifies that:
|
|
144
|
+
|
|
145
|
+
- both scans report failure;
|
|
146
|
+
- the failed path is retried through `dbToStorage`;
|
|
147
|
+
- no success event or success log is emitted;
|
|
148
|
+
- the database mtime does not enter the last-seen map; and
|
|
149
|
+
- `deleteFileFromDB` is not called.
|
|
150
|
+
|
|
151
|
+
Full-scan coverage also verifies that the strict default returns `false`, an ordinary-start-up scan which explicitly continues returns `completed-with-file-failures` while logging the affected path at verbose level, and a later clean scan returns `true`.
|
|
152
|
+
|
|
153
|
+
A second two-pass case starts with a `db-only` entry skipped by the size limit. It then removes that limit and verifies that the entry is reflected to storage rather than misclassified as an offline local deletion. Coverage for an existing storage file verifies that a failed `sync-newer` reflection retains the observed storage mtime, and full-scan coverage verifies that the aggregate failure reaches its caller after scanner initialisation completes.
|
|
154
|
+
|
|
155
|
+
Identity coverage verifies early quarantine when no consistent target exists, continued processing when consistent Metadata represents the same logical path, expired-deletion retention, special-namespace routing, actual-ID-first inspection, stale and ambiguous repair rejection, target-first ordering, exact-target retry, source preservation after failure, and last-seen clearing.
|
|
156
|
+
|
|
157
|
+
Stored-event coverage verifies that current storage content replaces saved observations, a recreated path suppresses a stale deletion, distinct-ID rename halves are admitted independently from current path state, and a same-ID rename never performs a separate deletion. It also verifies that a storage-inspection failure does not become evidence for a destructive operation and that an individual replay failure does not prevent the full scan from running.
|
|
158
|
+
|
|
159
|
+
Maintain that interaction boundary when changing file-handler results, last-seen persistence, Fast Setup reflection, or offline deletion detection.
|
package/docs/proven-in-use.md
CHANGED
|
@@ -24,9 +24,11 @@ Commonlib's injected remote-service tests verify that an owning operation can ab
|
|
|
24
24
|
|
|
25
25
|
Commonlib's remote-preference result contract distinguishes available synchronisation settings, a remote which has not stored those settings yet, a failed read, and a remote type which does not support central preferences. Focused CouchDB and Object Storage tests cover absent milestone data separately from transport failures, while Self-hosted LiveSync owns the initialisation choices, cancellation policy, and user-facing explanation.
|
|
26
26
|
|
|
27
|
-
Commonlib's full offline scanner distinguishes completed, deliberately skipped, and failed storage/database pairs. Before pair construction, it withholds Metadata whose actual document ID disagrees with its recorded path, without reflection, deletion, or last-seen side effects. A consistently addressed entry for the same logical path continues through the established path-based flow; only a path without such an entry also withholds storage processing. The read-only actual-ID Inspector and separately invoked one-entry repair retain ambiguous states for review, revalidate an exact source revision, verify a target-first copy, and preserve the source on failure. Snapshot-restoration tests additionally verify that restored inclusions use current storage content, stale deletions are suppressed, rename halves are admitted from their current path states, and an inspection failure does not become evidence of absence. The following full scan remains the final reconciliation step.
|
|
27
|
+
Commonlib's full offline scanner distinguishes completed, deliberately skipped, and failed storage/database pairs. Before pair construction, it withholds Metadata whose actual document ID disagrees with its recorded path, without reflection, deletion, or last-seen side effects. A consistently addressed entry for the same logical path continues through the established path-based flow; only a path without such an entry also withholds storage processing. The read-only actual-ID Inspector and separately invoked one-entry repair retain ambiguous states for review, revalidate an exact source revision, verify a target-first copy, and preserve the source on failure. Snapshot-restoration tests additionally verify that restored inclusions use current storage content, stale deletions are suppressed, rename halves are admitted from their current path states, and an inspection failure does not become evidence of absence. The following full scan remains the final reconciliation step.
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
The strict aggregate remains the default for Fast Setup, Fetch, Rebuild, direct scans, and CLI mirror or daemon paths. Self-hosted LiveSync opts only ordinary Obsidian start-up into continuing after individual pair failures. Commonlib returns the distinct `completed-with-file-failures` result, retains each failed result and retry state, records each affected path at verbose level, and leaves any user-facing notice to the host. Focused Commonlib tests cover the default and an explicitly continuing scan; downstream type checks, unit tests, and builds exercise the exact packed implementation and host call. Android-specific storage behaviour remains real-device and reporter-environment evidence rather than a Commonlib guarantee.
|
|
30
|
+
|
|
31
|
+
Commonlib's local-database lifecycle tests distinguish the settings-selected database from the active wrapper. They verify replacement before a changed-suffix reset, fail-closed cleanup after rejected physical preparation, application readiness after the requested scan and completion hooks, verbose diagnostics for rejected preparation stages, and the prevention of rebuilt events and later remote mutation after a failed local reset. Replication readiness tests verify that a descriptive application-initialisation diagnostic replaces the former bare diagnostic. Self-hosted LiveSync owns settings persistence, setup flags, user-facing notices, recovery dialogue, and real-Obsidian rebuild confirmation; those host results remain downstream evidence rather than Commonlib guarantees.
|
|
30
32
|
|
|
31
33
|
The focused `/replication` entry defines the provider capability, configuration identity, same-kind reconciliation, interaction authority, and truthful outcome contracts used when a host composes replication. A host selects its fixed provider subset and registers each definition during composition; an omitted role remains an explicit unsupported result rather than falling back to another provider. Commonlib serialises active replacement, rejects a candidate whose configuration changes before publication, and provides a transition-aware context acquisition boundary. The entry does not own user dialogue, P2P room policy, or selection of unknown providers.
|
|
32
34
|
|
|
@@ -38,6 +40,10 @@ Commonlib's confirmation contract can also convey a host-neutral vertical action
|
|
|
38
40
|
|
|
39
41
|
Self-hosted LiveSync's Obsidian and browser dialogue hosts now share an application-owned `SvelteDialogSession` lifecycle. They continue to consume Commonlib's dialogue types and manager base, but no maintained LiveSync path uses `SvelteDialogMixIn`. The dynamic-inheritance helper remains available through the compatibility boundary and is deprecated in favour of host-owned composition.
|
|
40
42
|
|
|
43
|
+
Commonlib's storage-event tests verify that a current file with the same canonical document ID suppresses a watcher deletion, while observed absence still permits deletion and inspection failure preserves Metadata. Rename-derived deletions retain their destination and distinguish deliberate target exclusion from transient filename collision rejection, including after snapshot restoration. A separate reflection test preserves incoming logical-deletion behaviour. These injected-boundary tests establish the [storage-event admission contract](storage-events-and-reflection.md), not the native event ordering of an external folder rename in a particular Obsidian version.
|
|
44
|
+
|
|
45
|
+
The downstream `E2E_OBSIDIAN_ONLY_PARENT_CASE_DELETION` scenario additionally exercises an external parent-directory case change in real Linux Obsidian 1.12.7. With released Commonlib 0.1.21, native `create` events for the new paths followed by `delete` events for the old paths produce logically deleted Metadata. The `delete` notification carries no `renameTarget`; the guard preserves current same-ID storage without inferring a causal link from the earlier `create`. With the implementation included in Commonlib 0.1.23 installed from its checked package artefact, the same sequence preserves Metadata, Chunks, and unchanged content locally, in CouchDB, on a second Vault, and after restart. This establishes the tested build's deletion protection, not full directory rename support or confirmation in the reporter's Obsidian 1.13.7 environment.
|
|
46
|
+
|
|
41
47
|
## CLI
|
|
42
48
|
|
|
43
49
|
The CLI extends the same neutral context with its database path and injected `StandardIo` in `src/apps/cli/services/NodeServiceContext.ts`. Its entry point creates the Node standard-I/O adapter once and passes it to command composition, while unit tests provide stream-independent fakes for parsing, prompts, standard output, and standard error.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
---
|
|
2
|
-
date: 2026-
|
|
3
|
-
commonlib-version: "0.1.
|
|
4
|
-
self-hosted-livesync-version: "1.0.
|
|
5
|
-
status:
|
|
2
|
+
date: 2026-09-04
|
|
3
|
+
commonlib-version: "0.1.21"
|
|
4
|
+
self-hosted-livesync-version: "1.0.24"
|
|
5
|
+
status: unreleased
|
|
6
6
|
---
|
|
7
7
|
|
|
8
8
|
# Service feature composition
|
|
@@ -76,8 +76,9 @@ export async function prepareDatabaseForUse(
|
|
|
76
76
|
errorManager: UnresolvedErrorManager,
|
|
77
77
|
showingNotice = false,
|
|
78
78
|
reopenDatabase = true,
|
|
79
|
-
ignoreSuspending = false
|
|
80
|
-
|
|
79
|
+
ignoreSuspending = false,
|
|
80
|
+
continueOnFileFailure = false
|
|
81
|
+
): Promise<VaultScanResult> {
|
|
81
82
|
// Open, scan, run completion handlers, commit pending events, and mark ready.
|
|
82
83
|
}
|
|
83
84
|
```
|
|
@@ -90,14 +91,26 @@ export function usePrepareDatabaseForUse(host: PrepareDatabaseHost): void {
|
|
|
90
91
|
const errorManager = new UnresolvedErrorManager(host.services.appLifecycle, host.services.context.events);
|
|
91
92
|
|
|
92
93
|
host.services.databaseEvents.initialiseDatabase.addHandler(
|
|
93
|
-
(showingNotice = false, reopenDatabase = true, ignoreSuspending = false) =>
|
|
94
|
-
prepareDatabaseForUse(
|
|
94
|
+
(showingNotice = false, reopenDatabase = true, ignoreSuspending = false, continueOnFileFailure = false) =>
|
|
95
|
+
prepareDatabaseForUse(
|
|
96
|
+
host,
|
|
97
|
+
log,
|
|
98
|
+
errorManager,
|
|
99
|
+
showingNotice,
|
|
100
|
+
reopenDatabase,
|
|
101
|
+
ignoreSuspending,
|
|
102
|
+
continueOnFileFailure
|
|
103
|
+
)
|
|
95
104
|
);
|
|
96
105
|
}
|
|
97
106
|
```
|
|
98
107
|
|
|
99
108
|
Test the operation's sequencing and failure short-circuiting separately from the handler registration. This keeps a lifecycle rename from forcing every domain test through an application-shaped fixture.
|
|
100
109
|
|
|
110
|
+
Database preparation and full scanning remain strict by default. A host may set `continueOnFileFailure` only when its ordinary start-up policy deliberately permits readiness after individual file-pair failures. In that case, the operation returns `completed-with-file-failures` after all later initialisation steps succeed; it does not collapse the result into `true`. That option does not admit a failed scan precondition or a later initialisation step, and explicit recovery workflows retain the default.
|
|
111
|
+
|
|
112
|
+
The `scanVault` and `initialiseDatabase` services use a failure-veto result aggregator for this contract. It preserves the first non-boolean continuation result but still invokes later handlers, so a later `false` result or exception rejects the operation. `firstResult` is not suitable here because it would return the partial result before those handlers can veto it.
|
|
113
|
+
|
|
101
114
|
## Keep feature-local state private
|
|
102
115
|
|
|
103
116
|
State which exists for one predicate, cache, queue, or scheduling policy can remain in a factory or private context. It does not need to become a property on the application core.
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
---
|
|
2
|
+
date: 2026-09-05
|
|
3
|
+
commonlib-version: "0.1.23"
|
|
4
|
+
self-hosted-livesync-version: "1.0.25"
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# Storage events and database-to-storage reflection
|
|
8
|
+
|
|
9
|
+
This note separates storage-to-database event handling from
|
|
10
|
+
database-to-storage reflection. It describes the Commonlib 0.1.23 guard against
|
|
11
|
+
a stale `DELETE` notification logically deleting Metadata while the same file
|
|
12
|
+
is still present in storage. The version fields identify the reviewed source
|
|
13
|
+
versions.
|
|
14
|
+
|
|
15
|
+
## Processing directions
|
|
16
|
+
|
|
17
|
+
`StorageEventManager.appendQueue` receives watcher events, applies target and
|
|
18
|
+
ignore policies, and queues `CREATE`, `CHANGED`, `DELETE`, or `RENAME` work. A
|
|
19
|
+
rename from a selected path to a rejected path becomes a `DELETE` with an
|
|
20
|
+
old-path deleted stub and the destination in `FileEventArgs.renameTarget`.
|
|
21
|
+
A rename from an excluded path to a selected path becomes a `CREATE`.
|
|
22
|
+
|
|
23
|
+
`renameTarget` is recorded only when one incoming `RENAME` is converted in
|
|
24
|
+
this way. It is not inferred by pairing independent `CREATE` and `DELETE`
|
|
25
|
+
notifications. A `DELETE` without it cannot establish that a current same-ID
|
|
26
|
+
file is the deliberately excluded destination of a rename. The ordinary
|
|
27
|
+
current-storage check therefore suppresses that deletion, even when the
|
|
28
|
+
current file's path would itself be excluded from synchronisation.
|
|
29
|
+
|
|
30
|
+
`ServiceFileHandlerBase` serialises this work by the canonical document ID
|
|
31
|
+
derived from each path. Before a watcher `DELETE` reaches `deleteFileFromDB`,
|
|
32
|
+
the handler rechecks current storage under that lock. Rename-derived deletions
|
|
33
|
+
lock both source and destination document IDs. This direction updates database
|
|
34
|
+
Metadata; it does not remove the physical file from storage.
|
|
35
|
+
|
|
36
|
+
An event marked `restoredFromPreviousRuntime` is separately revalidated against
|
|
37
|
+
current storage and selection before the full offline scan. An ordinary
|
|
38
|
+
restored deletion requires current absence; one retaining `renameTarget` uses
|
|
39
|
+
the rename-exclusion checks below as well as current source selection. Older
|
|
40
|
+
snapshots without that field keep their conservative absence-only rule.
|
|
41
|
+
|
|
42
|
+
Database-to-storage work follows the opposite direction. Replicated Metadata
|
|
43
|
+
reaches `_anyProcessReplicatedDoc`, which applies target and size checks,
|
|
44
|
+
obtains a storage stub, and calls `dbToStorage`. Logical deletion, conflicts,
|
|
45
|
+
and unsynchronised storage content retain their existing reflection rules.
|
|
46
|
+
The watcher guard does not intercept this path or general explicit database
|
|
47
|
+
deletion operations.
|
|
48
|
+
|
|
49
|
+
## Event intent and current storage state
|
|
50
|
+
|
|
51
|
+
| Event intent | Current storage observation | Admitted operation |
|
|
52
|
+
| --- | --- | --- |
|
|
53
|
+
| Watcher `CREATE` or `CHANGED` | A selected regular file is present | Keep the existing storage-to-database write. |
|
|
54
|
+
| Watcher `DELETE` without `renameTarget` | A current file resolves to the same canonical document ID, including at an excluded path, or a folder occupies the path | Suppress the database deletion; no rename destination is known. |
|
|
55
|
+
| Watcher `DELETE` without `renameTarget` | The host lookup reports no current item, or a file belonging to a distinct document ID | Use the existing deletion operation. |
|
|
56
|
+
| Watcher `DELETE` without `renameTarget` | Lookup fails | Leave the database unchanged; failure is unknown, not absence. |
|
|
57
|
+
| `RENAME` converted to `DELETE` | The destination is still selected when case-collision rejection is omitted | Suppress the database deletion. |
|
|
58
|
+
| `RENAME` converted to `DELETE` | The destination is deliberately excluded, and the source is absent | Apply the old document's deletion. |
|
|
59
|
+
| Case-only `RENAME` converted to `DELETE` | Source lookup resolves to the exact deliberately excluded destination, not the original path | Apply the old document's deletion despite the shared ID. |
|
|
60
|
+
| Restored `DELETE` without `renameTarget` | A current item occupies the saved path | Keep the existing restored-event suppression. |
|
|
61
|
+
| Replicated logical deletion | Storage contains a file | Keep the existing database-to-storage conflict and preservation rules. |
|
|
62
|
+
|
|
63
|
+
## Rename exclusion and preservation
|
|
64
|
+
|
|
65
|
+
The retained destination distinguishes deliberate exclusion from temporary
|
|
66
|
+
rejection by the filename collision check, which counts the host's indexed
|
|
67
|
+
paths. The handler queries `isTargetFile` with `skipCaseCollisionCheck` only
|
|
68
|
+
for that classification; other target policies, built-in exclusions, and the
|
|
69
|
+
current file-size limit still apply. This option does not authorise storing a
|
|
70
|
+
duplicate file.
|
|
71
|
+
|
|
72
|
+
A target lookup which resolves to another spelling or a folder is ambiguous
|
|
73
|
+
and preserves the database entry. A file recreated at the exact original path
|
|
74
|
+
also prevents deletion. The same-ID exception in the table applies only when
|
|
75
|
+
source lookup resolves to the exact excluded destination instead.
|
|
76
|
+
|
|
77
|
+
Suppression leaves storage and Metadata unchanged and records a notice. It
|
|
78
|
+
does not automatically write content, schedule a retry, or converge path
|
|
79
|
+
spelling. A later scan or event may reconcile the state, subject to its own
|
|
80
|
+
policy. The storage lookup may be a host index rather than an operating-system
|
|
81
|
+
existence check: the observation is made under the application lock, not in
|
|
82
|
+
an atomic filesystem transaction.
|
|
83
|
+
|
|
84
|
+
## Verification and scope
|
|
85
|
+
|
|
86
|
+
Focused unit tests cover a current canonical file, absence, lookup failure,
|
|
87
|
+
distinct document IDs, an excluded same-ID file without inferred rename
|
|
88
|
+
intent, retained rename destinations, transient collision rejection,
|
|
89
|
+
deliberate exclusion, and source recreation. Live and restored deletions are
|
|
90
|
+
exercised separately. A database-to-storage test also verifies that an
|
|
91
|
+
incoming logical deletion can still remove unchanged storage content.
|
|
92
|
+
|
|
93
|
+
These tests do not establish the native watcher event sequence reported in
|
|
94
|
+
Issue 1168. Real-runtime verification belongs to the host and must distinguish
|
|
95
|
+
observed events from inferred causes. This change does not promise path-case
|
|
96
|
+
convergence or full folder-rename support. It does not rewrite the queue,
|
|
97
|
+
introduce a retry subsystem, or change the database schema.
|
|
98
|
+
|
|
99
|
+
The maintained host's real-runtime comparison is recorded in
|
|
100
|
+
[Proven in maintained hosts](proven-in-use.md#obsidian-plug-in), separately
|
|
101
|
+
from these unit-test guarantees.
|
|
102
|
+
|
|
103
|
+
## Related documents
|
|
104
|
+
|
|
105
|
+
- [Full offline scan](offline-scanner.md), especially [Stored event interpretation](offline-scanner.md#stored-event-interpretation).
|
|
106
|
+
- [Conflict resolution and file provenance](conflict-resolution.md).
|
|
107
|
+
- [Commonlib development](development.md).
|
package/package.json
CHANGED