@vrtmrz/livesync-commonlib 0.1.40 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/dist/API/DirectFileManipulatorV2.js +16 -1
- package/dist/API/processSetting.d.ts +1 -0
- package/dist/API/processSetting.js +6 -1
- package/dist/common/fileContentHash.d.ts +2 -0
- package/dist/common/fileContentHash.js +18 -0
- package/dist/common/models/setting.const.defaults.js +1 -0
- package/dist/common/models/setting.const.qr.js +1 -0
- package/dist/common/models/setting.policy.d.ts +4 -0
- package/dist/common/models/setting.policy.js +1 -0
- package/dist/common/models/setting.type.d.ts +8 -1
- package/dist/common/models/shared.definition.configNames.js +5 -0
- package/dist/common/models/tweak.definition.d.ts +6 -0
- package/dist/common/models/tweak.definition.js +1 -0
- package/dist/common/pathIdentity.d.ts +10 -0
- package/dist/common/pathIdentity.js +34 -0
- package/dist/common/pathNormalization.d.ts +7 -0
- package/dist/common/pathNormalization.js +13 -0
- package/dist/common/settingConstants.d.ts +1 -0
- package/dist/hashing.d.ts +1 -0
- package/dist/hashing.js +3 -1
- package/dist/interfaces/FileReflectionProvenance.d.ts +4 -0
- package/dist/managers/EntryManager/EntryManager.d.ts +1 -0
- package/dist/managers/EntryManager/EntryManager.js +9 -0
- package/dist/managers/EntryManager/EntryManagerImpls.js +22 -5
- package/dist/pouchdb/StreamingFetch.d.ts +3 -1
- package/dist/pouchdb/StreamingFetch.js +3 -1
- package/dist/pouchdb/pathIdentityCompatibility.d.ts +8 -0
- package/dist/pouchdb/pathIdentityCompatibility.js +64 -0
- package/dist/pouchdb/remoteFeatureCompatibility.d.ts +2 -0
- package/dist/pouchdb/remoteFeatureCompatibility.js +9 -1
- package/dist/replication/couchdb/LiveSyncReplicator.js +11 -0
- package/dist/replication/index.d.ts +1 -1
- package/dist/replication/index.js +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.js +28 -6
- package/dist/replication/trystero/P2PRoomSessionOwner.js +18 -5
- package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
- package/dist/replication/trystero/useP2PReplicatorFeature.js +2 -1
- package/dist/serviceFeatures/offlineScanner.d.ts +1 -1
- package/dist/serviceFeatures/offlineScanner.js +4 -2
- package/dist/serviceFeatures/targetFilter.d.ts +1 -1
- package/dist/serviceFeatures/targetFilter.js +28 -17
- package/dist/serviceModules/Rebuilder.d.ts +5 -1
- package/dist/serviceModules/Rebuilder.js +85 -3
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +5 -2
- package/dist/serviceModules/ServiceFileHandlerBase.js +110 -49
- package/dist/services/base/IService.d.ts +2 -1
- package/dist/services/base/PathService.d.ts +3 -3
- package/dist/services/base/PathService.js +6 -3
- package/dist/settings.d.ts +2 -0
- package/dist/settings.js +3 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/conflict-resolution.md +19 -8
- package/docs/proven-in-use.md +8 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -26,7 +26,7 @@ The package is ESM-only and declares Node 20 or later. Browser entry points are
|
|
|
26
26
|
| -------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
|
|
27
27
|
| `@vrtmrz/livesync-commonlib` | `DirectFileManipulator` for integrations which access CouchDB directly | Deliberately small, but its high-level lifecycle and file-operation semantics are not final |
|
|
28
28
|
| `@vrtmrz/livesync-commonlib/context` | Instance-owned events, translation injection, and host-neutral standard-I/O types | Focused, package-tested pre-1.0 contract |
|
|
29
|
-
| `@vrtmrz/livesync-commonlib/hashing` | Chunk ID calculation
|
|
29
|
+
| `@vrtmrz/livesync-commonlib/hashing` | Chunk ID calculation and exact file-content hashing | Focused, package-tested pre-1.0 contract |
|
|
30
30
|
| `@vrtmrz/livesync-commonlib/browser` | Rooted File System Access API storage | Focused, package-tested pre-1.0 contract |
|
|
31
31
|
| `@vrtmrz/livesync-commonlib/node` | Rooted Node storage, Node standard I/O, and selected Node capabilities | Supported platform façade; keeps Node-only dependencies behind an explicit boundary |
|
|
32
32
|
| `@vrtmrz/livesync-commonlib/p2p` | Host-composed P2P service views and composition helpers | Focused, package-tested pre-1.0 contract with a temporary compatibility façade |
|
|
@@ -125,6 +125,10 @@ The `/hashing` entry exports `HashManager` and its `HashManagerCoreOptions` type
|
|
|
125
125
|
|
|
126
126
|
`computeHash` returns the Chunk hash, including the `+` prefix when encryption is enabled; the database adds its `h:` namespace separately. Reuse a manager for repeated calculations with the same configuration, create a new manager when changing the legacy algorithm or passphrase, and call `clearCaches()` when retiring it. This releases retained key preparation; it does not erase existing settings or in-flight operations. The entry performs ID calculation and does not write Chunks to a database.
|
|
127
127
|
|
|
128
|
+
## File content hashing
|
|
129
|
+
|
|
130
|
+
`computeFileContentHash(content: Blob)` returns a `sha256:<lowercase hexadecimal>` digest of the exact bytes. It includes Blob reading and serialises concurrent hashes to limit whole-file buffer overlap. File-reflection provenance uses this helper to recognise unchanged content independently of retained revision history; it does not replace Chunk ID calculation.
|
|
131
|
+
|
|
128
132
|
## Remote connection profiles
|
|
129
133
|
|
|
130
134
|
New profile-management code can create or update a CouchDB, Object Storage, or P2P connection through the focused remote-configurations entry. Profile IDs provide identity, display names remain editable presentation, and the active selection is explicit. Older flat connection fields remain a runtime projection and an import-compatibility boundary rather than the preferred persistence model.
|
|
@@ -135,6 +139,8 @@ See [the remote configuration profile guide](docs/remote-configurations.md) for
|
|
|
135
139
|
|
|
136
140
|
CouchDB stores the representations already in use as feature identifiers in its version document. Assess that document before synchronisation or direct file operations, and retain unknown identifiers for host diagnostics. See [the remote feature compatibility contract](docs/remote-feature-compatibility.md) for the wire format, update rules, and recovery boundaries.
|
|
137
141
|
|
|
142
|
+
The maintained CLI and Obsidian hosts can opt into [Unicode NFC path identity](docs/unicode-path-identity.md) with CouchDB. Existing databases require an explicit Rebuild or Fetch migration before using it.
|
|
143
|
+
|
|
138
144
|
## Contract scope
|
|
139
145
|
|
|
140
146
|
The package is currently an infrastructure and compatibility boundary. The context, rooted-storage, standard-I/O, and revision-tree safety contracts have focused cross-platform, instance-isolation, or real-PouchDB tests. Platform details which cannot be shared, such as file timestamp fidelity and browser permission handling, remain host concerns and are documented separately.
|
|
@@ -3,11 +3,17 @@ import {
|
|
|
3
3
|
DEFAULT_SYNC_PARAMETERS,
|
|
4
4
|
ProtocolVersions,
|
|
5
5
|
DOCID_SYNC_PARAMETERS,
|
|
6
|
+
VERSIONING_DOCID,
|
|
6
7
|
E2EEAlgorithms,
|
|
7
8
|
VER
|
|
8
9
|
} from "../common/types.js";
|
|
9
10
|
import { checkRemoteVersion } from "../pouchdb/negotiation.js";
|
|
10
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
assessRemoteFeatureDocument,
|
|
13
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
14
|
+
requiredRemoteFeatures,
|
|
15
|
+
usesEncryptedInternalMetadata
|
|
16
|
+
} from "../pouchdb/remoteFeatureCompatibility.js";
|
|
11
17
|
import { assessRemoteDocumentIds } from "../pouchdb/remoteIdCompatibility.js";
|
|
12
18
|
import { PouchDB } from "../pouchdb/pouchdb-http.js";
|
|
13
19
|
import { LiveSyncLocalDB } from "../pouchdb/LiveSyncLocalDB.js";
|
|
@@ -67,6 +73,15 @@ class DirectFileManipulator {
|
|
|
67
73
|
if (idCompatibility === "mismatched") {
|
|
68
74
|
throw new Error("Direct database document IDs do not match the configured ID key.");
|
|
69
75
|
}
|
|
76
|
+
try {
|
|
77
|
+
const versionInfo = await this.liveSyncLocalDB.localDatabase.get(VERSIONING_DOCID);
|
|
78
|
+
const assessment = assessRemoteFeatureDocument(versionInfo);
|
|
79
|
+
if (assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE)) {
|
|
80
|
+
throw new Error("Direct database version or features are not compatible.");
|
|
81
|
+
}
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (!isErrorOfMissingDoc(error)) throw error;
|
|
84
|
+
}
|
|
70
85
|
const requiredFeatures = requiredRemoteFeatures(this.settings);
|
|
71
86
|
if (!await checkRemoteVersion(
|
|
72
87
|
this.liveSyncLocalDB.localDatabase,
|
|
@@ -57,6 +57,7 @@ export declare function decodeSettingsFromSetupURI(uri: string, passphrase: stri
|
|
|
57
57
|
encryptInternalMetadata: boolean;
|
|
58
58
|
idDerivationVersion: 0 | 1;
|
|
59
59
|
idDerivationKey: string;
|
|
60
|
+
pathNormalizationVersion: import("../settings.js").PathNormalizationVersion;
|
|
60
61
|
liveSync: boolean;
|
|
61
62
|
syncOnSave: boolean;
|
|
62
63
|
syncOnStart: boolean;
|
|
@@ -10,9 +10,11 @@ import {
|
|
|
10
10
|
import { decryptString, encryptString } from "../encryption/stringEncryption.js";
|
|
11
11
|
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
|
12
12
|
import { configuredIdKey } from "../common/idDerivation.js";
|
|
13
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
13
14
|
function encodeSettingsToQRCodeData(settings) {
|
|
14
15
|
settings = omitP2PRuntimeSettings(settings);
|
|
15
16
|
configuredIdKey(settings);
|
|
17
|
+
getPathNormalizationVersion(settings);
|
|
16
18
|
const fullIndexes = Object.entries(KeyIndexOfSettings);
|
|
17
19
|
let maxIndex = 0;
|
|
18
20
|
for (const [, index] of fullIndexes) {
|
|
@@ -53,6 +55,7 @@ function decodeSettingsFromQRCodeData(qr) {
|
|
|
53
55
|
);
|
|
54
56
|
}
|
|
55
57
|
configuredIdKey(newSettings);
|
|
58
|
+
newSettings.pathNormalizationVersion = getPathNormalizationVersion(newSettings);
|
|
56
59
|
return omitP2PRuntimeSettings(newSettings);
|
|
57
60
|
}
|
|
58
61
|
var OutputFormat = /* @__PURE__ */ ((OutputFormat2) => {
|
|
@@ -134,6 +137,7 @@ async function encodeSettingsToSetupURI(settingString, passphrase, removePropert
|
|
|
134
137
|
const setting = {
|
|
135
138
|
...omitP2PRuntimeSettings(settingString)
|
|
136
139
|
};
|
|
140
|
+
getPathNormalizationVersion(setting);
|
|
137
141
|
configuredIdKey({
|
|
138
142
|
idDerivationVersion: setting.idDerivationVersion ?? 0,
|
|
139
143
|
idDerivationKey: setting.idDerivationKey ?? ""
|
|
@@ -211,7 +215,8 @@ async function decodeSettingsFromSetupURI(uri, passphrase) {
|
|
|
211
215
|
...imported,
|
|
212
216
|
encryptInternalMetadata: typeof imported.encryptInternalMetadata === "boolean" ? imported.encryptInternalMetadata : false,
|
|
213
217
|
idDerivationVersion: imported.idDerivationVersion ?? 0,
|
|
214
|
-
idDerivationKey: imported.idDerivationKey ?? ""
|
|
218
|
+
idDerivationKey: imported.idDerivationKey ?? "",
|
|
219
|
+
pathNormalizationVersion: getPathNormalizationVersion(imported)
|
|
215
220
|
};
|
|
216
221
|
configuredIdKey(settings);
|
|
217
222
|
return omitP2PRuntimeSettings(settings);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Semaphore } from "octagonal-wheels/concurrency/semaphore";
|
|
2
|
+
import { getWebCrypto } from "../mods.js";
|
|
3
|
+
import { uint8ArrayToHexString } from "../string_and_binary/convert.js";
|
|
4
|
+
const hashing = Semaphore(1);
|
|
5
|
+
async function computeFileContentHash(content) {
|
|
6
|
+
const release = await hashing.acquire();
|
|
7
|
+
try {
|
|
8
|
+
const crypto = await getWebCrypto();
|
|
9
|
+
const bytes = await content.arrayBuffer();
|
|
10
|
+
const digest = await crypto.subtle.digest("SHA-256", bytes);
|
|
11
|
+
return `sha256:${uint8ArrayToHexString(new Uint8Array(digest))}`;
|
|
12
|
+
} finally {
|
|
13
|
+
release();
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export {
|
|
17
|
+
computeFileContentHash
|
|
18
|
+
};
|
|
@@ -485,6 +485,10 @@ export declare const SETTINGS_PROPERTY_POLICY: {
|
|
|
485
485
|
readonly markdown: "credentials";
|
|
486
486
|
readonly persistence: "include";
|
|
487
487
|
};
|
|
488
|
+
readonly pathNormalizationVersion: {
|
|
489
|
+
readonly markdown: "credentials";
|
|
490
|
+
readonly persistence: "include";
|
|
491
|
+
};
|
|
488
492
|
readonly idDerivationKey: {
|
|
489
493
|
readonly markdown: "credentials";
|
|
490
494
|
readonly persistence: "id-key";
|
|
@@ -122,6 +122,7 @@ const SETTINGS_PROPERTY_POLICY = {
|
|
|
122
122
|
encrypt: SettingPolicies.Default,
|
|
123
123
|
passphrase: SettingPolicies.E2EEPassphrase,
|
|
124
124
|
idDerivationVersion: SettingPolicies.MarkdownCredentials,
|
|
125
|
+
pathNormalizationVersion: SettingPolicies.MarkdownCredentials,
|
|
125
126
|
idDerivationKey: SettingPolicies.IdDerivationKey,
|
|
126
127
|
usePathObfuscation: SettingPolicies.Default,
|
|
127
128
|
encryptInternalMetadata: SettingPolicies.Default,
|
|
@@ -575,6 +575,13 @@ export interface RemoteTypeSettings {
|
|
|
575
575
|
remoteType: RemoteType;
|
|
576
576
|
}
|
|
577
577
|
export type E2EEAlgorithm = (typeof E2EEAlgorithms)[keyof typeof E2EEAlgorithms] | "";
|
|
578
|
+
/** Path identity modes retained in persisted settings and shared tweak snapshots. */
|
|
579
|
+
export type PathNormalizationVersion = 0 | 1;
|
|
580
|
+
/** Identity settings for mapping logical paths to remote document IDs. */
|
|
581
|
+
export interface PathIdentitySettings {
|
|
582
|
+
/** Zero preserves legacy path identity; one applies Unicode NFC before ID derivation. */
|
|
583
|
+
pathNormalizationVersion: PathNormalizationVersion;
|
|
584
|
+
}
|
|
578
585
|
/**
|
|
579
586
|
* Represents the settings used for End-to-End encryption.
|
|
580
587
|
*/
|
|
@@ -937,7 +944,7 @@ export interface RemoteConfigurations {
|
|
|
937
944
|
}
|
|
938
945
|
interface ObsidianLiveSyncSettings_PluginSetting extends SyncMethodSettings, UISettings, FileHandlingSettings, MergeBehaviourSettings, EncryptedUserSettings, PeriodicReplicationSettings, InternalFileSettings, PluginSyncSettings, ModeSettings, ExtraTweakSettings, BetaTweakSettings, ObsoleteSettings, DebugModeSettings, SettingSyncSettings, SafetyValveSettings, DataOnSettings, RemoteConfigurations {
|
|
939
946
|
}
|
|
940
|
-
export type RemoteDBSettings = CouchDBConnection & BucketSyncSetting & RemoteTypeSettings & EncryptionSettings & ChunkSettings & EdenSettings & DataOnRemoteDBSettings & ObsoleteRemoteDBSettings & OnDemandChunkSettings & BetaRemoteDBSettings & ReplicationSetting & RemoteDBTweakSettings & FileHandlingSettings & ProcessingBehaviourSettings & OptionalAndNotExposedRemoteDBSettings & CrossPlatformInteroperabilitySettings & ConflictHandlingSettings & EdgeCaseHandlingSettings & DeletedFileMetadataSettings & P2PSyncSetting & RemoteConfigurations;
|
|
947
|
+
export type RemoteDBSettings = CouchDBConnection & BucketSyncSetting & RemoteTypeSettings & PathIdentitySettings & EncryptionSettings & ChunkSettings & EdenSettings & DataOnRemoteDBSettings & ObsoleteRemoteDBSettings & OnDemandChunkSettings & BetaRemoteDBSettings & ReplicationSetting & RemoteDBTweakSettings & FileHandlingSettings & ProcessingBehaviourSettings & OptionalAndNotExposedRemoteDBSettings & CrossPlatformInteroperabilitySettings & ConflictHandlingSettings & EdgeCaseHandlingSettings & DeletedFileMetadataSettings & P2PSyncSetting & RemoteConfigurations;
|
|
941
948
|
export type ObsidianLiveSyncSettings = ObsidianLiveSyncSettings_PluginSetting & RemoteDBSettings & LocalDBSettings;
|
|
942
949
|
export interface HasSettings<T extends Partial<ObsidianLiveSyncSettings>> {
|
|
943
950
|
settings: T;
|
|
@@ -64,6 +64,11 @@ const configurationNames = {
|
|
|
64
64
|
name: "Handle files as Case-Sensitive",
|
|
65
65
|
desc: "If this enabled, All files are handled as case-Sensitive (Previous behaviour)."
|
|
66
66
|
},
|
|
67
|
+
pathNormalizationVersion: {
|
|
68
|
+
name: "Path identity mode",
|
|
69
|
+
desc: "CouchDB only. Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild both databases from a complete authoritative Vault and Fetch on the other devices. Resolve colliding file names first. P2P and Object Storage do not support version 1.",
|
|
70
|
+
isAdvanced: true
|
|
71
|
+
},
|
|
67
72
|
doNotUseFixedRevisionForChunks: {
|
|
68
73
|
name: "Content-derived chunk revisions (obsolete setting)",
|
|
69
74
|
desc: "Chunk revisions are always derived from their content. This stored key is retained only for compatibility."
|
|
@@ -25,6 +25,11 @@ export declare const TweakCompatibilityRules: readonly [{
|
|
|
25
25
|
readonly templateValue: 0;
|
|
26
26
|
readonly reconstruction: "required";
|
|
27
27
|
readonly knownDefault: 0;
|
|
28
|
+
}, {
|
|
29
|
+
readonly key: "pathNormalizationVersion";
|
|
30
|
+
readonly templateValue: 0;
|
|
31
|
+
readonly reconstruction: "required";
|
|
32
|
+
readonly knownDefault: 0;
|
|
28
33
|
}, {
|
|
29
34
|
readonly key: "encryptInternalMetadata";
|
|
30
35
|
readonly templateValue: false;
|
|
@@ -116,6 +121,7 @@ export declare const TweakValuesDefault: Partial<ObsidianLiveSyncSettings>;
|
|
|
116
121
|
export declare const TweakValuesTemplate: {
|
|
117
122
|
tweakModified: number;
|
|
118
123
|
usePluginSyncV2: boolean;
|
|
124
|
+
pathNormalizationVersion: import("./setting.type.js").PathNormalizationVersion;
|
|
119
125
|
encrypt: boolean;
|
|
120
126
|
idDerivationVersion: 0 | 1;
|
|
121
127
|
usePathObfuscation: boolean;
|
|
@@ -6,6 +6,7 @@ const TweakCompatibilityRules = Object.freeze([
|
|
|
6
6
|
{ key: "encrypt", templateValue: false, reconstruction: "required" },
|
|
7
7
|
{ key: "usePathObfuscation", templateValue: false, reconstruction: "required" },
|
|
8
8
|
{ key: "idDerivationVersion", templateValue: 0, reconstruction: "required", knownDefault: 0 },
|
|
9
|
+
{ key: "pathNormalizationVersion", templateValue: 0, reconstruction: "required", knownDefault: 0 },
|
|
9
10
|
{ key: "encryptInternalMetadata", templateValue: false, reconstruction: "none", knownDefault: false },
|
|
10
11
|
{ key: "enableCompression", templateValue: false, reconstruction: "none" },
|
|
11
12
|
{ key: "useEden", templateValue: false, reconstruction: "none" },
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EntryHasPath, FilePathWithPrefix } from "./types.js";
|
|
2
|
+
import type { IPathService } from "../services/base/IService.js";
|
|
3
|
+
/** Check the complete source listing before an operation can discard either database. */
|
|
4
|
+
export declare function assertUniqueStoragePathIdentities(paths: Iterable<FilePathWithPrefix>, pathService: Pick<IPathService, "path2id">): Promise<void>;
|
|
5
|
+
/** Distinct Metadata documents must not be merged by a change of path identity. */
|
|
6
|
+
export declare function assertMetadataPathIdentities(entries: AsyncIterable<EntryHasPath & {
|
|
7
|
+
_id: string;
|
|
8
|
+
}> | Iterable<EntryHasPath & {
|
|
9
|
+
_id: string;
|
|
10
|
+
}>, pathService: Pick<IPathService, "path2id">, requireMatchingIds: boolean): Promise<void>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { id2path_base } from "../string_and_binary/path.js";
|
|
2
|
+
function collision(first, second) {
|
|
3
|
+
return new Error(
|
|
4
|
+
`File path identity collision: ${JSON.stringify(first)} and ${JSON.stringify(second)}. Rename or exclude the distinct files before continuing.`
|
|
5
|
+
);
|
|
6
|
+
}
|
|
7
|
+
async function assertUniqueStoragePathIdentities(paths, pathService) {
|
|
8
|
+
const identities = /* @__PURE__ */ new Map();
|
|
9
|
+
for (const path of paths) {
|
|
10
|
+
const id = await pathService.path2id(path);
|
|
11
|
+
const previous = identities.get(id);
|
|
12
|
+
if (previous !== void 0 && previous !== path) throw collision(previous, path);
|
|
13
|
+
identities.set(id, path);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
async function assertMetadataPathIdentities(entries, pathService, requireMatchingIds) {
|
|
17
|
+
const identities = /* @__PURE__ */ new Map();
|
|
18
|
+
for await (const entry of entries) {
|
|
19
|
+
const path = id2path_base(entry._id, entry);
|
|
20
|
+
const id = await pathService.path2id(path);
|
|
21
|
+
const previous = identities.get(id);
|
|
22
|
+
if (previous && previous.id !== entry._id) throw collision(previous.path, path);
|
|
23
|
+
identities.set(id, { path, id: entry._id });
|
|
24
|
+
if (requireMatchingIds && entry._id !== id) {
|
|
25
|
+
throw new Error(
|
|
26
|
+
`The stored document ID for ${JSON.stringify(path)} uses a different path identity. Rebuild from the authoritative Vault or Fetch the migrated database before synchronising.`
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export {
|
|
32
|
+
assertMetadataPathIdentities,
|
|
33
|
+
assertUniqueStoragePathIdentities
|
|
34
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { PathNormalizationVersion } from "./models/setting.type.js";
|
|
2
|
+
/** Resolve the stored path identity version, treating an absent legacy value as version zero. */
|
|
3
|
+
export declare function getPathNormalizationVersion(settings: {
|
|
4
|
+
pathNormalizationVersion?: number;
|
|
5
|
+
}): PathNormalizationVersion;
|
|
6
|
+
/** Normalise a logical path for document ID derivation without changing its stored path. */
|
|
7
|
+
export declare function normalisePathForIdentity(path: string, version?: number): string;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
function getPathNormalizationVersion(settings) {
|
|
2
|
+
const version = settings.pathNormalizationVersion === void 0 ? 0 : settings.pathNormalizationVersion;
|
|
3
|
+
if (version === 0 || version === 1) return version;
|
|
4
|
+
throw new Error(`Unsupported path normalisation version: ${version}`);
|
|
5
|
+
}
|
|
6
|
+
function normalisePathForIdentity(path, version) {
|
|
7
|
+
const resolvedVersion = getPathNormalizationVersion({ pathNormalizationVersion: version });
|
|
8
|
+
return resolvedVersion === 1 ? path.normalize("NFC") : path;
|
|
9
|
+
}
|
|
10
|
+
export {
|
|
11
|
+
getPathNormalizationVersion,
|
|
12
|
+
normalisePathForIdentity
|
|
13
|
+
};
|
|
@@ -134,6 +134,7 @@ export declare const AllSettingDefault: {
|
|
|
134
134
|
bucketPrefix: string;
|
|
135
135
|
forcePathStyle: boolean;
|
|
136
136
|
remoteType: import("./types.js").RemoteType;
|
|
137
|
+
pathNormalizationVersion: import("./models/setting.type.js").PathNormalizationVersion;
|
|
137
138
|
encrypt: boolean;
|
|
138
139
|
passphrase: string;
|
|
139
140
|
idDerivationVersion: 0 | 1;
|
package/dist/hashing.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
/** Chunk ID calculation using the same hash manager as database writes. */
|
|
2
2
|
export { HashManager } from "./managers/HashManager/HashManager.js";
|
|
3
3
|
export type { HashManagerCoreOptions } from "./managers/HashManager/HashManagerCore.js";
|
|
4
|
+
export { computeFileContentHash } from "./common/fileContentHash.js";
|
package/dist/hashing.js
CHANGED
|
@@ -5,6 +5,10 @@ export type FileReflectionProvenanceRecord = {
|
|
|
5
5
|
revision: string;
|
|
6
6
|
/** Raw modification time of the saved snapshot or storage observed after reflection. */
|
|
7
7
|
observedStorageMtime?: number;
|
|
8
|
+
/** Algorithm-tagged hash of the exact saved or reflected bytes, independent of revision retention. */
|
|
9
|
+
contentHash?: string;
|
|
10
|
+
/** Byte length of the hashed snapshot. Old records may omit both content fields. */
|
|
11
|
+
contentSize?: number;
|
|
8
12
|
};
|
|
9
13
|
export interface FileReflectionProvenance {
|
|
10
14
|
get(path: FilePathWithPrefix): Promise<FileReflectionProvenanceRecord | undefined>;
|
|
@@ -38,4 +38,5 @@ export declare class EntryManager {
|
|
|
38
38
|
putDBEntryWithLiveBaseRevision(note: SavingEntry, baseRevision: string, onlyChunks?: boolean): Promise<false | PouchDB.Core.Response>;
|
|
39
39
|
/** Store unknown ancestry as a fresh root; return the Metadata write result or `false`. */
|
|
40
40
|
putDBEntryAsIndependentRoot(note: SavingEntry): Promise<false | PouchDB.Core.Response>;
|
|
41
|
+
private checkPathIdentity;
|
|
41
42
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
REMOTE_COUCHDB
|
|
3
3
|
} from "../../common/types.js";
|
|
4
|
+
import { assertDatabasePathIdentity } from "../../pouchdb/pathIdentityCompatibility.js";
|
|
4
5
|
import {
|
|
5
6
|
deleteDBEntryByPath,
|
|
6
7
|
getDBEntryByPath,
|
|
@@ -65,21 +66,29 @@ class EntryManager {
|
|
|
65
66
|
return await getDBEntryFromMeta(this.serviceHost, this, meta, dump, waitForReady, localOnly);
|
|
66
67
|
}
|
|
67
68
|
async deleteDBEntry(path, opt) {
|
|
69
|
+
await this.checkPathIdentity();
|
|
68
70
|
return await deleteDBEntryByPath(this.serviceHost, this, path, opt);
|
|
69
71
|
}
|
|
70
72
|
async storeDeletionAtRevision(path, baseRevision) {
|
|
73
|
+
await this.checkPathIdentity();
|
|
71
74
|
return await storeDeletionByPathAtRevision(this.serviceHost, this, path, baseRevision);
|
|
72
75
|
}
|
|
73
76
|
async putDBEntry(note, onlyChunks, conflictBaseRev) {
|
|
77
|
+
await this.checkPathIdentity();
|
|
74
78
|
return await putDBEntry(this.serviceHost, this, note, onlyChunks, conflictBaseRev);
|
|
75
79
|
}
|
|
76
80
|
async putDBEntryWithLiveBaseRevision(note, baseRevision, onlyChunks) {
|
|
81
|
+
await this.checkPathIdentity();
|
|
77
82
|
return await putDBEntryWithLiveBaseRevision(this.serviceHost, this, note, baseRevision, onlyChunks);
|
|
78
83
|
}
|
|
79
84
|
/** Store unknown ancestry as a fresh root; return the Metadata write result or `false`. */
|
|
80
85
|
async putDBEntryAsIndependentRoot(note) {
|
|
86
|
+
await this.checkPathIdentity();
|
|
81
87
|
return await putDBEntryAsIndependentRoot(this.serviceHost, this, note);
|
|
82
88
|
}
|
|
89
|
+
async checkPathIdentity() {
|
|
90
|
+
await assertDatabasePathIdentity(this.localDatabase, this.options.settingService.currentSettings(), true);
|
|
91
|
+
}
|
|
83
92
|
}
|
|
84
93
|
export {
|
|
85
94
|
EntryManager
|
|
@@ -247,12 +247,29 @@ async function getDBEntryMetaByPath(host, { localDatabase }, path, opt, includeD
|
|
|
247
247
|
const id = await host.services.path.path2id(path);
|
|
248
248
|
try {
|
|
249
249
|
let obj = null;
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
250
|
+
try {
|
|
251
|
+
if (opt) {
|
|
252
|
+
obj = await localDatabase.get(id, opt);
|
|
253
|
+
} else {
|
|
254
|
+
obj = await localDatabase.get(id);
|
|
255
|
+
}
|
|
256
|
+
} catch (ex) {
|
|
257
|
+
if (!includeDeleted || opt?.rev !== void 0 || !isErrorOfMissingDoc(ex)) throw ex;
|
|
258
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
259
|
+
const result = await localDatabase.allDocs({ keys: [id] });
|
|
260
|
+
const row = result.rows[0];
|
|
261
|
+
if (!row || !("value" in row) || !row.value.rev) return false;
|
|
262
|
+
const candidate = await localDatabase.get(id, { ...opt, rev: row.value.rev });
|
|
263
|
+
const current = (await localDatabase.allDocs({ keys: [id] })).rows[0];
|
|
264
|
+
if (!current || !("value" in current) || !current.value.rev) return false;
|
|
265
|
+
if (current.value.rev === candidate._rev) {
|
|
266
|
+
obj = candidate;
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (!obj) return false;
|
|
254
271
|
}
|
|
255
|
-
const deleted = obj
|
|
272
|
+
const deleted = obj._deleted || obj.deleted;
|
|
256
273
|
if (!includeDeleted && deleted) return false;
|
|
257
274
|
if (obj.type && obj.type == "leaf") {
|
|
258
275
|
return false;
|
|
@@ -36,7 +36,9 @@ export declare class StreamingFetchFailure extends Error {
|
|
|
36
36
|
}
|
|
37
37
|
export declare function isRetryableStreamingFetchFailure(error: unknown): error is StreamingFetchFailure;
|
|
38
38
|
/** Check remote feature requirements before Fast Fetch opens or resets the local database. */
|
|
39
|
-
export declare function checkRemoteFeaturesForInitialSync(remoteDbUrl: string, authHeader: string, customHeaders?: Record<string, string
|
|
39
|
+
export declare function checkRemoteFeaturesForInitialSync(remoteDbUrl: string, authHeader: string, customHeaders?: Record<string, string>, settings?: {
|
|
40
|
+
pathNormalizationVersion?: number;
|
|
41
|
+
}): Promise<void>;
|
|
40
42
|
export type FetchChangesForInitialSyncProgress = {
|
|
41
43
|
totalFetched: number;
|
|
42
44
|
totalValidFetched: number;
|
|
@@ -2,6 +2,7 @@ import { _fetch } from "../common/coreEnvFunctions.js";
|
|
|
2
2
|
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
|
3
3
|
import { VERSIONING_DOCID } from "../common/types.js";
|
|
4
4
|
import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "./remoteFeatureCompatibility.js";
|
|
5
|
+
import { assertMatchingPathIdentityFeature } from "./pathIdentityCompatibility.js";
|
|
5
6
|
const FAST_FETCH_CHANGES_PAGE_LIMIT = 1e4;
|
|
6
7
|
const FAST_FETCH_CHANGES_PAGE_TIMEOUT_MS = 1e3;
|
|
7
8
|
class StreamingFetchFailure extends Error {
|
|
@@ -75,7 +76,7 @@ async function readResponseText(response, operation) {
|
|
|
75
76
|
throw transportFailure(operation, error);
|
|
76
77
|
}
|
|
77
78
|
}
|
|
78
|
-
async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, customHeaders) {
|
|
79
|
+
async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, customHeaders, settings = {}) {
|
|
79
80
|
const headers = new Headers(customHeaders);
|
|
80
81
|
headers.set("Accept", "application/json");
|
|
81
82
|
headers.set("Authorization", authHeader);
|
|
@@ -97,6 +98,7 @@ async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, custom
|
|
|
97
98
|
const row = result.rows[0];
|
|
98
99
|
if (result.total_rows === 0 && row.error === "not_found") return;
|
|
99
100
|
const assessment = assessRemoteFeatureDocument(row.doc);
|
|
101
|
+
assertMatchingPathIdentityFeature(row.doc, settings);
|
|
100
102
|
if (assessment.status === "supported" || assessment.status === "older-generation") return;
|
|
101
103
|
throw new StreamingFetchFailure("protocol", describeRemoteFeatureRejection(assessment), false);
|
|
102
104
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type EntryDoc } from "../common/types.js";
|
|
2
|
+
type PathIdentitySettings = {
|
|
3
|
+
pathNormalizationVersion?: number;
|
|
4
|
+
};
|
|
5
|
+
export declare function assertMatchingPathIdentityFeature(document: unknown, settings: PathIdentitySettings): void;
|
|
6
|
+
/** Refuse an implicit migration. A fresh database may declare its identity before its first Metadata write. */
|
|
7
|
+
export declare function assertDatabasePathIdentity(database: PouchDB.Database<EntryDoc>, settings: PathIdentitySettings, declare?: boolean): Promise<void>;
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { VERSIONING_DOCID } from "../common/types.js";
|
|
2
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
3
|
+
import {
|
|
4
|
+
assessRemoteFeatureDocument,
|
|
5
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
6
|
+
REMOTE_FEATURE_GENERATION
|
|
7
|
+
} from "./remoteFeatureCompatibility.js";
|
|
8
|
+
import { isErrorOfMissingDoc } from "./utils_couchdb.js";
|
|
9
|
+
import { bumpRemoteVersion } from "./negotiation.js";
|
|
10
|
+
function assertMatchingPathIdentityFeature(document, settings) {
|
|
11
|
+
const assessment = assessRemoteFeatureDocument(document);
|
|
12
|
+
const declared = assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE);
|
|
13
|
+
if (declared !== (getPathNormalizationVersion(settings) === 1)) {
|
|
14
|
+
throw new Error(
|
|
15
|
+
"The database and settings use different path identity versions. Rebuild from the authoritative Vault or Fetch with matching settings before synchronising."
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
async function hasMetadata(database) {
|
|
20
|
+
let startkey;
|
|
21
|
+
for (; ; ) {
|
|
22
|
+
const result = await database.allDocs({
|
|
23
|
+
include_docs: true,
|
|
24
|
+
limit: 100,
|
|
25
|
+
...startkey ? { startkey, skip: 1 } : {}
|
|
26
|
+
});
|
|
27
|
+
for (const row of result.rows) {
|
|
28
|
+
if (row.doc?.type === "plain" || row.doc?.type === "newnote") return true;
|
|
29
|
+
}
|
|
30
|
+
if (result.rows.length < 100) return false;
|
|
31
|
+
startkey = result.rows[result.rows.length - 1].id;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function assertDatabasePathIdentity(database, settings, declare = false) {
|
|
35
|
+
let document;
|
|
36
|
+
try {
|
|
37
|
+
document = await database.get(VERSIONING_DOCID);
|
|
38
|
+
} catch (error) {
|
|
39
|
+
if (!isErrorOfMissingDoc(error)) throw error;
|
|
40
|
+
}
|
|
41
|
+
const assessment = assessRemoteFeatureDocument(document);
|
|
42
|
+
const declared = assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE);
|
|
43
|
+
if (getPathNormalizationVersion(settings) === 0) {
|
|
44
|
+
if (declared) assertMatchingPathIdentityFeature(document, settings);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (declared) return;
|
|
48
|
+
if (await hasMetadata(database)) {
|
|
49
|
+
try {
|
|
50
|
+
document = await database.get(VERSIONING_DOCID);
|
|
51
|
+
} catch (error) {
|
|
52
|
+
if (!isErrorOfMissingDoc(error)) throw error;
|
|
53
|
+
}
|
|
54
|
+
assertMatchingPathIdentityFeature(document, settings);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (declare && !await bumpRemoteVersion(database, REMOTE_FEATURE_GENERATION, [NFC_PATH_IDENTITY_FEATURE])) {
|
|
58
|
+
throw new Error("The database path identity could not be declared. Synchronisation has been stopped.");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export {
|
|
62
|
+
assertDatabasePathIdentity,
|
|
63
|
+
assertMatchingPathIdentityFeature
|
|
64
|
+
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export declare const REMOTE_FEATURE_GENERATION = 13;
|
|
2
2
|
export declare const ENCRYPTED_INTERNAL_METADATA_FEATURE = "encrypted-internal-metadata-v1";
|
|
3
3
|
export declare const INDEPENDENT_ID_DERIVATION_FEATURE = "independent-id-derivation-v1";
|
|
4
|
+
export declare const NFC_PATH_IDENTITY_FEATURE = "nfc-path-identity-v1";
|
|
4
5
|
export declare const supportedFeatures: Set<string>;
|
|
5
6
|
export declare function requiredRemoteFeatures(setting: {
|
|
6
7
|
encrypt?: boolean;
|
|
@@ -9,6 +10,7 @@ export declare function requiredRemoteFeatures(setting: {
|
|
|
9
10
|
encryptInternalMetadata?: boolean;
|
|
10
11
|
idDerivationVersion: 0 | 1;
|
|
11
12
|
idDerivationKey: string;
|
|
13
|
+
pathNormalizationVersion?: number;
|
|
12
14
|
}): string[];
|
|
13
15
|
export declare function usesEncryptedInternalMetadata(setting: {
|
|
14
16
|
encrypt?: boolean;
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { E2EEAlgorithms, VERSIONING_DOCID } from "../common/types.js";
|
|
2
2
|
import { configuredIdKey } from "../common/idDerivation.js";
|
|
3
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
3
4
|
const REMOTE_FEATURE_GENERATION = 13;
|
|
4
5
|
const ENCRYPTED_INTERNAL_METADATA_FEATURE = "encrypted-internal-metadata-v1";
|
|
5
6
|
const INDEPENDENT_ID_DERIVATION_FEATURE = "independent-id-derivation-v1";
|
|
6
|
-
const
|
|
7
|
+
const NFC_PATH_IDENTITY_FEATURE = "nfc-path-identity-v1";
|
|
8
|
+
const supportedFeatures = /* @__PURE__ */ new Set([
|
|
9
|
+
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
10
|
+
INDEPENDENT_ID_DERIVATION_FEATURE,
|
|
11
|
+
NFC_PATH_IDENTITY_FEATURE
|
|
12
|
+
]);
|
|
7
13
|
function requiredRemoteFeatures(setting) {
|
|
8
14
|
const features = [];
|
|
9
15
|
if (usesEncryptedInternalMetadata(setting)) features.push(ENCRYPTED_INTERNAL_METADATA_FEATURE);
|
|
16
|
+
if (getPathNormalizationVersion(setting) === 1) features.push(NFC_PATH_IDENTITY_FEATURE);
|
|
10
17
|
if (setting.encrypt && configuredIdKey(setting)) features.push(INDEPENDENT_ID_DERIVATION_FEATURE);
|
|
11
18
|
return features;
|
|
12
19
|
}
|
|
@@ -54,6 +61,7 @@ function describeRemoteFeatureRejection(assessment) {
|
|
|
54
61
|
export {
|
|
55
62
|
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
56
63
|
INDEPENDENT_ID_DERIVATION_FEATURE,
|
|
64
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
57
65
|
REMOTE_FEATURE_GENERATION,
|
|
58
66
|
assessRemoteFeatureDocument,
|
|
59
67
|
describeRemoteFeatureRejection,
|
|
@@ -881,6 +881,16 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
881
881
|
}
|
|
882
882
|
let ownershipTransferred = false;
|
|
883
883
|
try {
|
|
884
|
+
try {
|
|
885
|
+
await assertDatabasePathIdentity(dbRet.db, setting);
|
|
886
|
+
await assertDatabasePathIdentity(this.env.services.database.localDatabase.localDatabase, setting);
|
|
887
|
+
} catch (error) {
|
|
888
|
+
recordCompatibilityDecision?.(
|
|
889
|
+
centralCompatibilityRejected(CENTRAL_COMPATIBILITY_REJECTION_REASONS.INCOMPATIBLE_VERSION)
|
|
890
|
+
);
|
|
891
|
+
Logger(error, LOG_LEVEL_NOTICE);
|
|
892
|
+
return false;
|
|
893
|
+
}
|
|
884
894
|
if (!skipCheck) {
|
|
885
895
|
this.remoteCleaned = false;
|
|
886
896
|
this.remoteLocked = false;
|
|
@@ -1617,6 +1627,7 @@ ${sendingDocs}/(${sentDocsCount} done)`,
|
|
|
1617
1627
|
});
|
|
1618
1628
|
}
|
|
1619
1629
|
}
|
|
1630
|
+
import { assertDatabasePathIdentity } from "../../pouchdb/pathIdentityCompatibility.js";
|
|
1620
1631
|
export {
|
|
1621
1632
|
LiveSyncCouchDBReplicator
|
|
1622
1633
|
};
|
|
@@ -8,5 +8,5 @@ export { CENTRAL_REMOTE_ADMINISTRATION_ACTIONS, CENTRAL_REMOTE_ADMINISTRATION_FA
|
|
|
8
8
|
export type { MilestoneCentralRemoteAdministrationObservation, CentralRemoteAdministrationAction, CentralRemoteAdministrationFailureReason, CentralRemoteAdministrationObservation, CentralRemoteAdministrationRequest, CentralRemoteAdministrationReplicator, CentralRemoteAdministrationResult, CentralRemoteAdministrationRunner, CentralRemoteAdministrationVerificationFailed, CentralRemoteAdministrationVerified, } from "./CentralRemoteAdministration.js";
|
|
9
9
|
export { CENTRAL_COMPATIBILITY_REJECTION_REASONS } from "./CentralCompatibility.js";
|
|
10
10
|
export type { CentralCompatibilityRecoveryHint, CentralCompatibilityRejectionReason } from "./CentralCompatibility.js";
|
|
11
|
-
export { assessRemoteFeatureDocument, describeRemoteFeatureRejection, ENCRYPTED_INTERNAL_METADATA_FEATURE, REMOTE_FEATURE_GENERATION, usesEncryptedInternalMetadata, } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
11
|
+
export { assessRemoteFeatureDocument, describeRemoteFeatureRejection, ENCRYPTED_INTERNAL_METADATA_FEATURE, NFC_PATH_IDENTITY_FEATURE, REMOTE_FEATURE_GENERATION, usesEncryptedInternalMetadata, } from "../pouchdb/remoteFeatureCompatibility.js";
|
|
12
12
|
export type { RemoteFeatureAssessment } from "../pouchdb/remoteFeatureCompatibility.js";
|