@vrtmrz/livesync-commonlib 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/API/DirectFileManipulatorV2.js +16 -1
- package/dist/API/processSetting.d.ts +1 -0
- package/dist/API/processSetting.js +6 -1
- package/dist/common/fileContentHash.d.ts +2 -0
- package/dist/common/fileContentHash.js +18 -0
- package/dist/common/models/setting.const.defaults.js +1 -0
- package/dist/common/models/setting.const.qr.js +1 -0
- package/dist/common/models/setting.policy.d.ts +4 -0
- package/dist/common/models/setting.policy.js +1 -0
- package/dist/common/models/setting.type.d.ts +8 -1
- package/dist/common/models/shared.definition.configNames.js +5 -0
- package/dist/common/models/tweak.definition.d.ts +6 -0
- package/dist/common/models/tweak.definition.js +1 -0
- package/dist/common/pathIdentity.d.ts +10 -0
- package/dist/common/pathIdentity.js +34 -0
- package/dist/common/pathNormalization.d.ts +7 -0
- package/dist/common/pathNormalization.js +13 -0
- package/dist/common/settingConstants.d.ts +1 -0
- package/dist/hashing.d.ts +1 -0
- package/dist/hashing.js +3 -1
- package/dist/interfaces/FileReflectionProvenance.d.ts +4 -0
- package/dist/managers/EntryManager/EntryManager.d.ts +1 -0
- package/dist/managers/EntryManager/EntryManager.js +9 -0
- package/dist/managers/EntryManager/EntryManagerImpls.js +22 -5
- package/dist/pouchdb/StreamingFetch.d.ts +3 -1
- package/dist/pouchdb/StreamingFetch.js +3 -1
- package/dist/pouchdb/pathIdentityCompatibility.d.ts +8 -0
- package/dist/pouchdb/pathIdentityCompatibility.js +64 -0
- package/dist/pouchdb/remoteFeatureCompatibility.d.ts +2 -0
- package/dist/pouchdb/remoteFeatureCompatibility.js +9 -1
- package/dist/replication/couchdb/LiveSyncReplicator.js +11 -0
- package/dist/replication/index.d.ts +1 -1
- package/dist/replication/index.js +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.js +28 -6
- package/dist/replication/trystero/P2PRoomSessionOwner.js +18 -5
- package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
- package/dist/replication/trystero/useP2PReplicatorFeature.js +2 -1
- package/dist/serviceFeatures/offlineScanner.d.ts +1 -1
- package/dist/serviceFeatures/offlineScanner.js +4 -2
- package/dist/serviceFeatures/targetFilter.d.ts +1 -1
- package/dist/serviceFeatures/targetFilter.js +28 -17
- package/dist/serviceModules/Rebuilder.d.ts +5 -1
- package/dist/serviceModules/Rebuilder.js +85 -3
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +5 -2
- package/dist/serviceModules/ServiceFileHandlerBase.js +110 -49
- package/dist/services/base/IService.d.ts +2 -1
- package/dist/services/base/PathService.d.ts +3 -3
- package/dist/services/base/PathService.js +6 -3
- package/dist/settings.d.ts +2 -0
- package/dist/settings.js +3 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/conflict-resolution.md +19 -8
- package/docs/proven-in-use.md +8 -0
- package/package.json +1 -1
|
@@ -39,6 +39,7 @@ import {
|
|
|
39
39
|
assessRemoteFeatureDocument,
|
|
40
40
|
describeRemoteFeatureRejection,
|
|
41
41
|
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
42
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
42
43
|
REMOTE_FEATURE_GENERATION,
|
|
43
44
|
usesEncryptedInternalMetadata
|
|
44
45
|
} from "../pouchdb/remoteFeatureCompatibility.js";
|
|
@@ -54,6 +55,7 @@ export {
|
|
|
54
55
|
CENTRAL_REMOTE_ADMINISTRATION_RESULT_STATUSES,
|
|
55
56
|
CENTRAL_REMOTE_REPLICATION_READINESS,
|
|
56
57
|
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
58
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
57
59
|
NO_INTERACTION,
|
|
58
60
|
NO_REMOTE_RESOURCE_CAPABILITIES,
|
|
59
61
|
PEER_REPLICATION_READINESS,
|
|
@@ -61,6 +61,8 @@ export declare class LiveSyncJournalReplicator extends LiveSyncAbstractReplicato
|
|
|
61
61
|
setPreferredRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
|
|
62
62
|
getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
|
|
63
63
|
getRemoteStatus(setting: RemoteDBSettings): Promise<false | RemoteDBStatus>;
|
|
64
|
+
private supportsLegacyPathIdentity;
|
|
65
|
+
private assertLegacyPathIdentity;
|
|
64
66
|
getConnectedDeviceList(setting?: RemoteDBSettings): Promise<false | {
|
|
65
67
|
node_info: Record<string, NodeData>;
|
|
66
68
|
accepted_nodes: string[];
|
|
@@ -11,6 +11,7 @@ import { MinioStorageAdapter } from "./objectstore/MinioStorageAdapter.js";
|
|
|
11
11
|
import { LiveSyncAbstractReplicator } from "../LiveSyncAbstractReplicator.js";
|
|
12
12
|
import { ensureRemoteIsCompatible } from "../../pouchdb/LiveSyncDBFunctions.js";
|
|
13
13
|
import { getEffectiveTweakValues } from "../../pouchdb/LiveSyncDBFunctions.js";
|
|
14
|
+
import { getPathNormalizationVersion } from "../../common/pathNormalization.js";
|
|
14
15
|
import { JournalStorageReadStatuses } from "./objectstore/JournalStorageAdapter.js";
|
|
15
16
|
import { computeKeyedId, configuredIdKey } from "../../common/idDerivation.js";
|
|
16
17
|
import { decryptString, encryptString } from "../../encryption/stringEncryption.js";
|
|
@@ -60,6 +61,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
60
61
|
return await this.setupJournalSyncClient(setting).getReplicationPBKDF2Salt(refresh);
|
|
61
62
|
}
|
|
62
63
|
setupJournalSyncClient(settings = this.currentSettings) {
|
|
64
|
+
this.assertLegacyPathIdentity(settings);
|
|
63
65
|
if (this._client) {
|
|
64
66
|
this._client.applyNewConfig(settings, this.simpleStore, this.env);
|
|
65
67
|
} else {
|
|
@@ -281,13 +283,15 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
281
283
|
(client) => client.receiveRemoteJournal(showingNotice)
|
|
282
284
|
);
|
|
283
285
|
}
|
|
284
|
-
async checkReplicationConnectivity(skipCheck, ignoreCleanLock = false, showMessage = false, setting = this.currentSettings, client
|
|
286
|
+
async checkReplicationConnectivity(skipCheck, ignoreCleanLock = false, showMessage = false, setting = this.currentSettings, client, recordCompatibilityDecision) {
|
|
285
287
|
recordCompatibilityDecision?.(CENTRAL_COMPATIBILITY_NOT_ASSESSED);
|
|
286
|
-
if (!
|
|
288
|
+
if (!this.supportsLegacyPathIdentity(setting)) return false;
|
|
289
|
+
const activeClient = client ?? this.setupJournalSyncClient(setting);
|
|
290
|
+
if (!await activeClient.isAvailable()) {
|
|
287
291
|
return false;
|
|
288
292
|
}
|
|
289
293
|
if (!skipCheck) {
|
|
290
|
-
await
|
|
294
|
+
await activeClient.ensureCheckpointCachesAreFresh();
|
|
291
295
|
this.remoteCleaned = false;
|
|
292
296
|
this.remoteLocked = false;
|
|
293
297
|
this.remoteLockedAndDeviceNotAccepted = false;
|
|
@@ -298,7 +302,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
298
302
|
this.nodeid,
|
|
299
303
|
journalVersionRange(setting),
|
|
300
304
|
setting,
|
|
301
|
-
|
|
305
|
+
activeClient,
|
|
302
306
|
(assessment) => {
|
|
303
307
|
tweakAssessment = assessment;
|
|
304
308
|
}
|
|
@@ -378,6 +382,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
378
382
|
this.updateInfo();
|
|
379
383
|
}
|
|
380
384
|
async tryResetRemoteDatabase(setting) {
|
|
385
|
+
this.assertLegacyPathIdentity(setting);
|
|
381
386
|
this.closeReplication();
|
|
382
387
|
try {
|
|
383
388
|
if (!await this.client.resetBucket()) {
|
|
@@ -392,6 +397,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
392
397
|
}
|
|
393
398
|
}
|
|
394
399
|
async tryCreateRemoteDatabase(setting) {
|
|
400
|
+
this.assertLegacyPathIdentity(setting);
|
|
395
401
|
this.closeReplication();
|
|
396
402
|
Logger("Remote Database Created or Connected", LOG_LEVEL_NOTICE);
|
|
397
403
|
if (!await this.ensurePBKDF2Salt(setting, true, false)) {
|
|
@@ -406,6 +412,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
406
412
|
return await encryptString(proof, setting.passphrase);
|
|
407
413
|
}
|
|
408
414
|
async markRemoteLocked(setting, locked, lockByClean) {
|
|
415
|
+
this.assertLegacyPathIdentity(setting);
|
|
409
416
|
const defInitPoint = {
|
|
410
417
|
_id: MILSTONE_DOCID,
|
|
411
418
|
type: "milestoneinfo",
|
|
@@ -439,6 +446,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
439
446
|
}
|
|
440
447
|
}
|
|
441
448
|
async markRemoteResolved(setting) {
|
|
449
|
+
this.assertLegacyPathIdentity(setting);
|
|
442
450
|
const defInitPoint = {
|
|
443
451
|
_id: MILSTONE_DOCID,
|
|
444
452
|
type: "milestoneinfo",
|
|
@@ -466,8 +474,10 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
466
474
|
}
|
|
467
475
|
async tryConnectRemote(setting, showResult = true) {
|
|
468
476
|
const endpoint = setting.endpoint;
|
|
469
|
-
|
|
477
|
+
let testClient;
|
|
470
478
|
try {
|
|
479
|
+
this.assertLegacyPathIdentity(setting);
|
|
480
|
+
testClient = new MinioStorageAdapter(setting, this.env);
|
|
471
481
|
await testClient.listFiles("", 1);
|
|
472
482
|
Logger(`Connected to ${endpoint} successfully!`, LOG_LEVEL_NOTICE);
|
|
473
483
|
return true;
|
|
@@ -477,10 +487,11 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
477
487
|
Logger(ex, LOG_LEVEL_NOTICE);
|
|
478
488
|
return false;
|
|
479
489
|
} finally {
|
|
480
|
-
testClient
|
|
490
|
+
testClient?.dispose();
|
|
481
491
|
}
|
|
482
492
|
}
|
|
483
493
|
async resetRemoteTweakSettings(setting) {
|
|
494
|
+
this.assertLegacyPathIdentity(setting);
|
|
484
495
|
try {
|
|
485
496
|
const remoteMilestone = await this.client.downloadJson(MILSTONE_DOCID);
|
|
486
497
|
if (!remoteMilestone) {
|
|
@@ -495,6 +506,7 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
495
506
|
}
|
|
496
507
|
}
|
|
497
508
|
async setPreferredRemoteTweakSettings(setting) {
|
|
509
|
+
this.assertLegacyPathIdentity(setting);
|
|
498
510
|
const trialClient = new JournalSyncCore(
|
|
499
511
|
setting,
|
|
500
512
|
this.simpleStore,
|
|
@@ -519,6 +531,7 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
519
531
|
}
|
|
520
532
|
}
|
|
521
533
|
async getRemotePreferredTweakValues(setting) {
|
|
534
|
+
this.assertLegacyPathIdentity(setting);
|
|
522
535
|
const trialClient = new JournalSyncCore(
|
|
523
536
|
setting,
|
|
524
537
|
this.simpleStore,
|
|
@@ -555,6 +568,7 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
555
568
|
}
|
|
556
569
|
}
|
|
557
570
|
async getRemoteStatus(setting) {
|
|
571
|
+
this.assertLegacyPathIdentity(setting);
|
|
558
572
|
const testClient = new MinioStorageAdapter(setting, this.env);
|
|
559
573
|
try {
|
|
560
574
|
return await testClient.getUsage();
|
|
@@ -562,6 +576,14 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
562
576
|
testClient.dispose();
|
|
563
577
|
}
|
|
564
578
|
}
|
|
579
|
+
supportsLegacyPathIdentity(setting) {
|
|
580
|
+
return getPathNormalizationVersion(setting) === 0;
|
|
581
|
+
}
|
|
582
|
+
assertLegacyPathIdentity(setting) {
|
|
583
|
+
if (!this.supportsLegacyPathIdentity(setting)) {
|
|
584
|
+
throw new Error("Object Storage does not support NFC path identity.");
|
|
585
|
+
}
|
|
586
|
+
}
|
|
565
587
|
getConnectedDeviceList(setting) {
|
|
566
588
|
return Promise.resolve(false);
|
|
567
589
|
}
|
|
@@ -12,6 +12,7 @@ import { P2PRoomSession } from "./P2PRoomSession.js";
|
|
|
12
12
|
import { P2PAutomationCoordinator } from "./P2PAutomationCoordinator.js";
|
|
13
13
|
import { PEER_REPLICATION_READINESS } from "../ReplicatorProvider.js";
|
|
14
14
|
import { getP2PReplicatorConfigurationIdentity } from "./p2pReplicatorConfigurationIdentity.js";
|
|
15
|
+
import { getPathNormalizationVersion } from "../../common/pathNormalization.js";
|
|
15
16
|
import {
|
|
16
17
|
ACTIVE_P2P_RELAY_BINDING_CONFLICT,
|
|
17
18
|
activeP2PRelayBindingCovers
|
|
@@ -56,14 +57,18 @@ class P2PRoomSessionOwner {
|
|
|
56
57
|
}
|
|
57
58
|
/** Reconcile one long-lived reason for keeping the room available. */
|
|
58
59
|
async setPersistentDemand(demand, active) {
|
|
59
|
-
const
|
|
60
|
+
const settings = this.env.services.setting.currentSettings();
|
|
61
|
+
const supportsPathIdentity = getPathNormalizationVersion(settings) === 0;
|
|
62
|
+
const enabled = settings.P2P_Enabled && supportsPathIdentity;
|
|
60
63
|
if (active && enabled) {
|
|
61
64
|
this.persistentDemands.add(demand);
|
|
62
65
|
} else {
|
|
63
66
|
this.persistentDemands.delete(demand);
|
|
64
67
|
}
|
|
65
|
-
if (active && !
|
|
68
|
+
if (active && !settings.P2P_Enabled) {
|
|
66
69
|
Logger(this.env.services.context.translate("P2P.NotEnabled"), LOG_LEVEL_NOTICE);
|
|
70
|
+
} else if (active && !supportsPathIdentity) {
|
|
71
|
+
Logger("P2P does not support NFC path identity.", LOG_LEVEL_NOTICE);
|
|
67
72
|
}
|
|
68
73
|
this.fencePreparationForCurrentSettings(enabled);
|
|
69
74
|
await this.reconcileTransport();
|
|
@@ -75,9 +80,13 @@ class P2PRoomSessionOwner {
|
|
|
75
80
|
* retained by another finite operation or by a persistent policy demand.
|
|
76
81
|
*/
|
|
77
82
|
async runWithFiniteDemand(task) {
|
|
78
|
-
|
|
83
|
+
const settings = this.env.services.setting.currentSettings();
|
|
84
|
+
if (!settings.P2P_Enabled) {
|
|
79
85
|
throw new Error("P2P is not enabled.");
|
|
80
86
|
}
|
|
87
|
+
if (getPathNormalizationVersion(settings) !== 0) {
|
|
88
|
+
throw new Error("P2P does not support NFC path identity.");
|
|
89
|
+
}
|
|
81
90
|
const demand = /* @__PURE__ */ Symbol("p2p-finite-room-demand");
|
|
82
91
|
this.finiteDemands.add(demand);
|
|
83
92
|
this.fencePreparationForCurrentSettings(true);
|
|
@@ -99,6 +108,9 @@ class P2PRoomSessionOwner {
|
|
|
99
108
|
}
|
|
100
109
|
/** Arbitrate a complete signalling probe on the room lifecycle queue. */
|
|
101
110
|
runConnectionProbe(trialSettings, runOwnedTrial) {
|
|
111
|
+
if (getPathNormalizationVersion(this.env.services.setting.currentSettings()) !== 0) {
|
|
112
|
+
return Promise.reject(new Error("P2P does not support NFC path identity."));
|
|
113
|
+
}
|
|
102
114
|
return this.enqueueLifecycleOperation(async () => {
|
|
103
115
|
const current = this.current;
|
|
104
116
|
const activeBinding = this.activeBinding;
|
|
@@ -122,7 +134,8 @@ class P2PRoomSessionOwner {
|
|
|
122
134
|
}
|
|
123
135
|
async reconcileTransport() {
|
|
124
136
|
await this.enqueueLifecycleOperation(async () => {
|
|
125
|
-
|
|
137
|
+
const settings = this.env.services.setting.currentSettings();
|
|
138
|
+
if (!settings.P2P_Enabled || getPathNormalizationVersion(settings) !== 0) {
|
|
126
139
|
this.persistentDemands.clear();
|
|
127
140
|
this.finiteDemands.clear();
|
|
128
141
|
}
|
|
@@ -374,7 +387,7 @@ class P2PRoomSessionOwner {
|
|
|
374
387
|
);
|
|
375
388
|
return {
|
|
376
389
|
database,
|
|
377
|
-
enabled: settings.P2P_Enabled,
|
|
390
|
+
enabled: settings.P2P_Enabled && getPathNormalizationVersion(settings) === 0,
|
|
378
391
|
settings,
|
|
379
392
|
deviceName,
|
|
380
393
|
signature: JSON.stringify([getP2PReplicatorConfigurationIdentity(settings), deviceName])
|
|
@@ -249,6 +249,7 @@ export declare class TrysteroReplicator {
|
|
|
249
249
|
bucketPrefix?: string;
|
|
250
250
|
forcePathStyle?: boolean;
|
|
251
251
|
remoteType?: import("../../common/types.js").RemoteType;
|
|
252
|
+
pathNormalizationVersion?: import("../../settings.js").PathNormalizationVersion;
|
|
252
253
|
encrypt?: boolean;
|
|
253
254
|
passphrase?: string;
|
|
254
255
|
idDerivationKey?: string;
|
|
@@ -436,6 +437,7 @@ export declare class TrysteroReplicator {
|
|
|
436
437
|
bucketPrefix?: string;
|
|
437
438
|
forcePathStyle?: boolean;
|
|
438
439
|
remoteType?: import("../../common/types.js").RemoteType;
|
|
440
|
+
pathNormalizationVersion?: import("../../settings.js").PathNormalizationVersion;
|
|
439
441
|
encrypt?: boolean;
|
|
440
442
|
passphrase?: string;
|
|
441
443
|
idDerivationKey?: string;
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
supportedStopActiveTransfer
|
|
16
16
|
} from "../index.js";
|
|
17
17
|
import { getP2PReplicatorConfigurationIdentity } from "./p2pReplicatorConfigurationIdentity.js";
|
|
18
|
+
import { getPathNormalizationVersion } from "../../common/pathNormalization.js";
|
|
18
19
|
function useP2PReplicatorFeature(host, openReplicationUIFactory, openRebuildUIFactory, options = {}) {
|
|
19
20
|
const service = createP2PService(
|
|
20
21
|
{
|
|
@@ -62,7 +63,7 @@ function useP2PReplicatorFeature(host, openReplicationUIFactory, openRebuildUIFa
|
|
|
62
63
|
kind: REMOTE_P2P,
|
|
63
64
|
diagnosticName: "P2P",
|
|
64
65
|
readiness: PEER_REPLICATION_READINESS,
|
|
65
|
-
isConfigured: (settings) => settings.remoteType === REMOTE_P2P && settings.P2P_Enabled,
|
|
66
|
+
isConfigured: (settings) => settings.remoteType === REMOTE_P2P && settings.P2P_Enabled && getPathNormalizationVersion(settings) === 0,
|
|
66
67
|
configurationIdentity: getP2PReplicatorConfigurationIdentity,
|
|
67
68
|
create: createP2PReplicator,
|
|
68
69
|
remoteResources: NO_REMOTE_RESOURCE_CAPABILITIES,
|
|
@@ -150,7 +150,7 @@ export declare function getPathFromEntry(host: NecessaryServices<"path", never>,
|
|
|
150
150
|
export declare function syncFileBetweenDBandStorage(host: NecessaryServices<"setting" | "vault" | "path", "storageAccess" | "fileHandler">, log: LogFunction, file: UXFileInfoStub, doc: MetaEntry): Promise<FilePairProcessResult>;
|
|
151
151
|
export declare function canProceedScan(host: NecessaryServices<"keyValueDB" | "setting", never>, errorManager: UnresolvedErrorManager, log: LogFunction, showingNotice?: boolean, ignoreSuspending?: boolean): boolean;
|
|
152
152
|
/**
|
|
153
|
-
*
|
|
153
|
+
* Normalise the logical comparison key and lower case it when filename case should be handled insensitively.
|
|
154
154
|
* @param settings
|
|
155
155
|
* @param path
|
|
156
156
|
* @returns
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from "../common/types.js";
|
|
10
10
|
import { compareMTime, isAnyNote, isRemediationModeActive } from "../common/utils.js";
|
|
11
11
|
import { shouldBeIgnored, stripAllPrefixes } from "../string_and_binary/path.js";
|
|
12
|
+
import { getPathNormalizationVersion, normalisePathForIdentity } from "../common/pathNormalization.js";
|
|
12
13
|
import { createInstanceLogFunction } from "../services/lib/logUtils.js";
|
|
13
14
|
import { BASE_IS_NEW, EVEN, TARGET_IS_NEW } from "../common/models/shared.const.symbols.js";
|
|
14
15
|
import { UnresolvedErrorManager } from "../services/base/UnresolvedErrorManager.js";
|
|
@@ -422,10 +423,11 @@ function canProceedScan(host, errorManager, log, showingNotice = false, ignoreSu
|
|
|
422
423
|
return true;
|
|
423
424
|
}
|
|
424
425
|
function convertCase(settings, path) {
|
|
426
|
+
const normalisedPath = normalisePathForIdentity(path, getPathNormalizationVersion(settings));
|
|
425
427
|
if (settings.handleFilenameCaseSensitive) {
|
|
426
|
-
return
|
|
428
|
+
return normalisedPath;
|
|
427
429
|
}
|
|
428
|
-
return
|
|
430
|
+
return normalisedPath.toLowerCase();
|
|
429
431
|
}
|
|
430
432
|
async function collectFilesOnStorage(host, settings, log) {
|
|
431
433
|
log("Collecting local files on the storage", LOG_LEVEL_VERBOSE);
|
|
@@ -9,7 +9,7 @@ export declare function isAcceptedAlwaysFactory(host: NecessaryServices<never, n
|
|
|
9
9
|
/**
|
|
10
10
|
* Check if a file is accepted based on filename duplication in the vault.
|
|
11
11
|
*/
|
|
12
|
-
export declare function isAcceptedInFilenameDuplicationFactory(host: NecessaryServices<"vault" | "fileProcessing", "storageAccess">, log: LogFunction): (file: string | UXFileInfoStub, options?: Parameters<IVaultService["isTargetFile"]>[1]) => Promise<boolean>;
|
|
12
|
+
export declare function isAcceptedInFilenameDuplicationFactory(host: NecessaryServices<"vault" | "fileProcessing" | "setting", "storageAccess">, log: LogFunction): (file: string | UXFileInfoStub, options?: Parameters<IVaultService["isTargetFile"]>[1]) => Promise<boolean>;
|
|
13
13
|
/**
|
|
14
14
|
* Check if a file is accepted by the local database (e.g., not rejected by the local DB's target file check).
|
|
15
15
|
* Local database responsible for non-internal files, syncOnlyRegEx, syncIgnoreRegEx
|
|
@@ -4,6 +4,7 @@ import { getStoragePathFromUXFileInfo } from "../common/typeUtils.js";
|
|
|
4
4
|
import { isAcceptedAll } from "../string_and_binary/path.js";
|
|
5
5
|
import { Computed } from "octagonal-wheels/dataobject/Computed";
|
|
6
6
|
import { promiseWithResolvers } from "octagonal-wheels/promises";
|
|
7
|
+
import { getPathNormalizationVersion, normalisePathForIdentity } from "../common/pathNormalization.js";
|
|
7
8
|
function isAcceptedAlwaysFactory(host, log) {
|
|
8
9
|
return (file) => {
|
|
9
10
|
log("File is target finally: " + getStoragePathFromUXFileInfo(file), LOG_LEVEL_DEBUG);
|
|
@@ -12,16 +13,15 @@ function isAcceptedAlwaysFactory(host, log) {
|
|
|
12
13
|
}
|
|
13
14
|
function isAcceptedInFilenameDuplicationFactory(host, log) {
|
|
14
15
|
const fileCountMapComputed = new Computed({
|
|
15
|
-
evaluation: async (fileEventCount) => {
|
|
16
|
+
evaluation: async (fileEventCount, version, caseInsensitive) => {
|
|
16
17
|
const vaultFiles = (await host.serviceModules.storageAccess.getFileNames()).sort();
|
|
17
|
-
const fileCountMap =
|
|
18
|
+
const fileCountMap = /* @__PURE__ */ new Map();
|
|
18
19
|
for (const file of vaultFiles) {
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
}
|
|
20
|
+
const normalised = normalisePathForIdentity(file, version);
|
|
21
|
+
const key = caseInsensitive ? normalised.toLowerCase() : normalised;
|
|
22
|
+
const paths = fileCountMap.get(key) ?? [];
|
|
23
|
+
paths.push(file);
|
|
24
|
+
fileCountMap.set(key, paths);
|
|
25
25
|
}
|
|
26
26
|
return fileCountMap;
|
|
27
27
|
},
|
|
@@ -29,7 +29,7 @@ function isAcceptedInFilenameDuplicationFactory(host, log) {
|
|
|
29
29
|
if (!previousResult) return true;
|
|
30
30
|
if (previousResult instanceof Error) return true;
|
|
31
31
|
if (!previousArgs) return true;
|
|
32
|
-
if (args
|
|
32
|
+
if (args.every((value, index) => value === previousArgs[index])) {
|
|
33
33
|
return false;
|
|
34
34
|
}
|
|
35
35
|
return true;
|
|
@@ -37,14 +37,20 @@ function isAcceptedInFilenameDuplicationFactory(host, log) {
|
|
|
37
37
|
});
|
|
38
38
|
return async function isAcceptedInFilenameDuplication(file, options) {
|
|
39
39
|
if (options?.skipCaseCollisionCheck) return true;
|
|
40
|
-
const
|
|
40
|
+
const version = getPathNormalizationVersion(host.services.setting.currentSettings());
|
|
41
|
+
const caseInsensitive = host.services.vault.shouldCheckCaseInsensitively();
|
|
42
|
+
const fileCountMap = (await fileCountMapComputed.update(
|
|
43
|
+
host.services.fileProcessing.totalStorageFileEventCount,
|
|
44
|
+
version,
|
|
45
|
+
caseInsensitive
|
|
46
|
+
)).value;
|
|
41
47
|
const filepath = getStoragePathFromUXFileInfo(file);
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
+
const normalised = normalisePathForIdentity(filepath, version);
|
|
49
|
+
const key = caseInsensitive ? normalised.toLowerCase() : normalised;
|
|
50
|
+
const duplicates = fileCountMap.get(key);
|
|
51
|
+
if (duplicates && duplicates.length > 1) {
|
|
52
|
+
log("File path identity collision: " + duplicates.map((path) => JSON.stringify(path)).join(", "));
|
|
53
|
+
return false;
|
|
48
54
|
}
|
|
49
55
|
log("File is not duplicated: " + filepath, LOG_LEVEL_DEBUG);
|
|
50
56
|
return true;
|
|
@@ -134,7 +140,12 @@ function useTargetFilters(host) {
|
|
|
134
140
|
const serviceModules = host.serviceModules;
|
|
135
141
|
const _isAcceptedFilenameDuplication = isAcceptedInFilenameDuplicationFactory(
|
|
136
142
|
{
|
|
137
|
-
services: {
|
|
143
|
+
services: {
|
|
144
|
+
context: services.context,
|
|
145
|
+
vault: services.vault,
|
|
146
|
+
fileProcessing: services.fileProcessing,
|
|
147
|
+
setting: services.setting
|
|
148
|
+
},
|
|
138
149
|
serviceModules: { storageAccess: serviceModules.storageAccess }
|
|
139
150
|
},
|
|
140
151
|
logger
|
|
@@ -14,7 +14,7 @@ import type { StorageAccess } from "../interfaces/StorageAccess.js";
|
|
|
14
14
|
import type { LiveSyncEventHub } from "../hub/hub.js";
|
|
15
15
|
import { ServiceModuleBase } from "./ServiceModuleBase.js";
|
|
16
16
|
import type { ControlService } from "../services/base/ControlService.js";
|
|
17
|
-
import type { IFileProcessingService } from "../services/base/IService.js";
|
|
17
|
+
import type { IFileProcessingService, IPathService } from "../services/base/IService.js";
|
|
18
18
|
export interface ServiceRebuilderDependencies {
|
|
19
19
|
events: LiveSyncEventHub;
|
|
20
20
|
appLifecycle: AppLifecycleService;
|
|
@@ -31,6 +31,7 @@ export interface ServiceRebuilderDependencies {
|
|
|
31
31
|
fileHandler: IFileHandler;
|
|
32
32
|
fileProcessing: IFileProcessingService;
|
|
33
33
|
control: ControlService;
|
|
34
|
+
path: IPathService;
|
|
34
35
|
}
|
|
35
36
|
export declare class ServiceRebuilder extends ServiceModuleBase<ServiceRebuilderDependencies> implements Rebuilder {
|
|
36
37
|
private events;
|
|
@@ -48,6 +49,7 @@ export declare class ServiceRebuilder extends ServiceModuleBase<ServiceRebuilder
|
|
|
48
49
|
private fileHandler;
|
|
49
50
|
private fileProcessing;
|
|
50
51
|
private control;
|
|
52
|
+
private path;
|
|
51
53
|
constructor(services: ServiceRebuilderDependencies);
|
|
52
54
|
$performRebuildDB(method: "localOnly" | "remoteOnly" | "rebuildBothByThisDevice" | "localOnlyWithChunks"): Promise<void>;
|
|
53
55
|
informOptionalFeatures(): Promise<void>;
|
|
@@ -57,6 +59,8 @@ export declare class ServiceRebuilder extends ServiceModuleBase<ServiceRebuilder
|
|
|
57
59
|
rebuildEverything(): Promise<void>;
|
|
58
60
|
private performRebuildEverything;
|
|
59
61
|
$rebuildEverything(): Promise<void>;
|
|
62
|
+
private checkPathIdentitiesBeforeReconstruction;
|
|
63
|
+
private checkRemotePathIdentityBeforeFetch;
|
|
60
64
|
$fetchLocal(makeLocalChunkBeforeSync?: boolean, preventMakeLocalFilesBeforeSync?: boolean): Promise<void>;
|
|
61
65
|
$fetchLocalDBFast(autoResume: boolean): Promise<void>;
|
|
62
66
|
private scheduleInitialisation;
|
|
@@ -5,6 +5,12 @@ import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/log
|
|
|
5
5
|
import { delay } from "octagonal-wheels/promises";
|
|
6
6
|
import { EVENT_DATABASE_REBUILT } from "../events/coreEvents.js";
|
|
7
7
|
import { ServiceModuleBase } from "./ServiceModuleBase.js";
|
|
8
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
9
|
+
import { assertMetadataPathIdentities, assertUniqueStoragePathIdentities } from "../common/pathIdentity.js";
|
|
10
|
+
import { VERSIONING_DOCID } from "../common/types.js";
|
|
11
|
+
import { assertDatabasePathIdentity, assertMatchingPathIdentityFeature } from "../pouchdb/pathIdentityCompatibility.js";
|
|
12
|
+
import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
13
|
+
import { isErrorOfMissingDoc } from "../pouchdb/utils_couchdb.js";
|
|
8
14
|
import {
|
|
9
15
|
checkRemoteFeaturesForInitialSync,
|
|
10
16
|
fetchChangesForInitialSync,
|
|
@@ -36,6 +42,7 @@ class ServiceRebuilder extends ServiceModuleBase {
|
|
|
36
42
|
fileHandler;
|
|
37
43
|
fileProcessing;
|
|
38
44
|
control;
|
|
45
|
+
path;
|
|
39
46
|
constructor(services) {
|
|
40
47
|
super(services);
|
|
41
48
|
this.events = services.events;
|
|
@@ -53,6 +60,7 @@ class ServiceRebuilder extends ServiceModuleBase {
|
|
|
53
60
|
this.fileHandler = services.fileHandler;
|
|
54
61
|
this.fileProcessing = services.fileProcessing;
|
|
55
62
|
this.control = services.control;
|
|
63
|
+
this.path = services.path;
|
|
56
64
|
services.database.onDatabaseReset.addHandler(this._onResetLocalDatabase.bind(this));
|
|
57
65
|
services.setting.suspendAllSync.addHandler(this._allSuspendAllSync.bind(this));
|
|
58
66
|
}
|
|
@@ -79,6 +87,7 @@ Please enable them from the settings screen after setup is complete.`,
|
|
|
79
87
|
);
|
|
80
88
|
}
|
|
81
89
|
async rebuildRemote() {
|
|
90
|
+
await this.checkPathIdentitiesBeforeReconstruction(true);
|
|
82
91
|
await this.replicator.runBoundedRemoteActivity(() => this.performRemoteRebuild(), {
|
|
83
92
|
label: "rebuild-remote"
|
|
84
93
|
});
|
|
@@ -104,6 +113,7 @@ Please enable them from the settings screen after setup is complete.`,
|
|
|
104
113
|
return this.rebuildRemote();
|
|
105
114
|
}
|
|
106
115
|
async rebuildEverything() {
|
|
116
|
+
await this.checkPathIdentitiesBeforeReconstruction(false);
|
|
107
117
|
await this.replicator.runBoundedRemoteActivity(() => this.performRebuildEverything(), {
|
|
108
118
|
label: "rebuild-everything"
|
|
109
119
|
});
|
|
@@ -154,6 +164,68 @@ Please enable them from the settings screen after setup is complete.`,
|
|
|
154
164
|
$rebuildEverything() {
|
|
155
165
|
return this.rebuildEverything();
|
|
156
166
|
}
|
|
167
|
+
async checkPathIdentitiesBeforeReconstruction(requireMatchingIds) {
|
|
168
|
+
const settings = this.setting.currentSettings();
|
|
169
|
+
if (requireMatchingIds && this.database.isDatabaseReady()) {
|
|
170
|
+
await assertDatabasePathIdentity(this.database.localDatabase.localDatabase, settings);
|
|
171
|
+
}
|
|
172
|
+
if (getPathNormalizationVersion(settings) === 0) return;
|
|
173
|
+
if (settings.remoteType !== REMOTE_COUCHDB) {
|
|
174
|
+
throw new Error("NFC path identity is supported only with CouchDB. No database has been reset.");
|
|
175
|
+
}
|
|
176
|
+
const paths = await this.storageAccess.getFileNames();
|
|
177
|
+
const selected = [];
|
|
178
|
+
const databaseReady = this.database.isDatabaseReady();
|
|
179
|
+
for (const path of paths) {
|
|
180
|
+
if (!databaseReady || await this.vault.isTargetFile(path, { skipCaseCollisionCheck: true }))
|
|
181
|
+
selected.push(path);
|
|
182
|
+
}
|
|
183
|
+
await assertUniqueStoragePathIdentities(selected, this.path);
|
|
184
|
+
if (databaseReady) {
|
|
185
|
+
await assertMetadataPathIdentities(
|
|
186
|
+
this.database.localDatabase.findAllDocs(),
|
|
187
|
+
this.path,
|
|
188
|
+
requireMatchingIds
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async checkRemotePathIdentityBeforeFetch() {
|
|
193
|
+
const settings = this.setting.currentSettings();
|
|
194
|
+
if (settings.remoteType !== REMOTE_COUCHDB) return;
|
|
195
|
+
const remote = settings.couchDB_URI.replace(/\/+$/, "") + (settings.couchDB_DBNAME === "" ? "" : "/" + settings.couchDB_DBNAME);
|
|
196
|
+
const connection = await this.remote.connect(
|
|
197
|
+
remote,
|
|
198
|
+
generateCredentialObject(settings),
|
|
199
|
+
settings.disableRequestURI || this.API.isMobile(),
|
|
200
|
+
false,
|
|
201
|
+
false,
|
|
202
|
+
false,
|
|
203
|
+
true,
|
|
204
|
+
false,
|
|
205
|
+
parseHeaderValues(settings.couchDB_CustomHeaders),
|
|
206
|
+
settings.useRequestAPI,
|
|
207
|
+
async () => {
|
|
208
|
+
throw new Error("A version document does not require decryption.");
|
|
209
|
+
}
|
|
210
|
+
);
|
|
211
|
+
if (typeof connection === "string") throw new Error(connection);
|
|
212
|
+
try {
|
|
213
|
+
let document;
|
|
214
|
+
try {
|
|
215
|
+
document = await connection.db.get(VERSIONING_DOCID);
|
|
216
|
+
} catch (error) {
|
|
217
|
+
if (!isErrorOfMissingDoc(error) || (await connection.db.info()).doc_count !== 0) throw error;
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
const assessment = assessRemoteFeatureDocument(document);
|
|
221
|
+
if (assessment.status !== "supported" && assessment.status !== "older-generation") {
|
|
222
|
+
throw new Error(describeRemoteFeatureRejection(assessment));
|
|
223
|
+
}
|
|
224
|
+
assertMatchingPathIdentityFeature(document, settings);
|
|
225
|
+
} finally {
|
|
226
|
+
await connection.close();
|
|
227
|
+
}
|
|
228
|
+
}
|
|
157
229
|
$fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync) {
|
|
158
230
|
return this.fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync);
|
|
159
231
|
}
|
|
@@ -289,6 +361,8 @@ Please enable them from the settings screen after setup is complete.`,
|
|
|
289
361
|
return true;
|
|
290
362
|
}
|
|
291
363
|
async fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync, autoResume = true) {
|
|
364
|
+
await this.checkPathIdentitiesBeforeReconstruction(false);
|
|
365
|
+
await this.checkRemotePathIdentityBeforeFetch();
|
|
292
366
|
await this.setting.suspendExtraSync();
|
|
293
367
|
await this.setting.applyPartial({
|
|
294
368
|
isConfigured: true,
|
|
@@ -393,7 +467,8 @@ Are you sure you wish to proceed?`;
|
|
|
393
467
|
generateCredentialObject(settings)
|
|
394
468
|
);
|
|
395
469
|
const customHeaders = parseHeaderValues(settings.couchDB_CustomHeaders);
|
|
396
|
-
await
|
|
470
|
+
await this.checkPathIdentitiesBeforeReconstruction(false);
|
|
471
|
+
await checkRemoteFeaturesForInitialSync(remote, authHeader, customHeaders, settings);
|
|
397
472
|
this.appLifecycle.resetIsReady();
|
|
398
473
|
let checkpoint = this.getFastFetchCheckpoint(remote);
|
|
399
474
|
await this.suspendReflectingDatabase();
|
|
@@ -557,7 +632,7 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
|
|
|
557
632
|
if (!rawCheckpoint) return void 0;
|
|
558
633
|
try {
|
|
559
634
|
const checkpoint = JSON.parse(rawCheckpoint);
|
|
560
|
-
if (checkpoint.remote === remote && checkpoint.sequence !== void 0) {
|
|
635
|
+
if (checkpoint.remote === remote && checkpoint.sequence !== void 0 && getPathNormalizationVersion(checkpoint) === getPathNormalizationVersion(this.setting.currentSettings())) {
|
|
561
636
|
return {
|
|
562
637
|
remote,
|
|
563
638
|
sequence: checkpoint.sequence
|
|
@@ -569,7 +644,14 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
|
|
|
569
644
|
return void 0;
|
|
570
645
|
}
|
|
571
646
|
saveFastFetchCheckpoint(remote, sequence) {
|
|
572
|
-
this.setting.setSmallConfig(
|
|
647
|
+
this.setting.setSmallConfig(
|
|
648
|
+
FAST_FETCH_CHECKPOINT_KEY,
|
|
649
|
+
JSON.stringify({
|
|
650
|
+
remote,
|
|
651
|
+
sequence,
|
|
652
|
+
pathNormalizationVersion: getPathNormalizationVersion(this.setting.currentSettings())
|
|
653
|
+
})
|
|
654
|
+
);
|
|
573
655
|
}
|
|
574
656
|
clearFastFetchCheckpoint() {
|
|
575
657
|
this.setting.deleteSmallConfig(FAST_FETCH_CHECKPOINT_KEY);
|
|
@@ -44,11 +44,14 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
44
44
|
private setting;
|
|
45
45
|
private vault;
|
|
46
46
|
private fileReflectionProvenance?;
|
|
47
|
+
private readonly contentHashes;
|
|
47
48
|
private readonly deferredConflictChecks;
|
|
48
49
|
constructor(services: ServiceFileHandlerDependencies);
|
|
49
50
|
get db(): DatabaseFileAccess;
|
|
50
51
|
get storage(): StorageAccess;
|
|
52
|
+
private getProvenancePath;
|
|
51
53
|
private getProvenance;
|
|
54
|
+
private contentEvidence;
|
|
52
55
|
private setProvenance;
|
|
53
56
|
private deleteProvenance;
|
|
54
57
|
private findUniqueContentRevision;
|
|
@@ -62,8 +65,8 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
|
|
|
62
65
|
* identifies provenance; no match is `unknown`.
|
|
63
66
|
* `preferredPath` selects the provenance key during a rename, and
|
|
64
67
|
* `currentLoaded` supplies an already decoded candidate body when available.
|
|
65
|
-
*
|
|
66
|
-
*
|
|
68
|
+
* A valid saved content hash can prove unchanged bytes after revision
|
|
69
|
+
* pruning. It cannot reconstruct ancestry for an edited file.
|
|
67
70
|
*/
|
|
68
71
|
private classifyStorageContent;
|
|
69
72
|
/** Record provenance only when storage matches one unambiguous live leaf. */
|