@vrtmrz/livesync-commonlib 0.1.41 → 0.1.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -1
- package/dist/API/DirectFileManipulatorV2.js +16 -1
- package/dist/API/processSetting.d.ts +1 -0
- package/dist/API/processSetting.js +6 -1
- package/dist/common/fileContentHash.d.ts +2 -0
- package/dist/common/fileContentHash.js +18 -0
- package/dist/common/models/setting.const.defaults.js +1 -0
- package/dist/common/models/setting.const.qr.js +1 -0
- package/dist/common/models/setting.policy.d.ts +4 -0
- package/dist/common/models/setting.policy.js +1 -0
- package/dist/common/models/setting.type.d.ts +8 -1
- package/dist/common/models/shared.definition.configNames.js +5 -0
- package/dist/common/models/tweak.definition.d.ts +6 -0
- package/dist/common/models/tweak.definition.js +1 -0
- package/dist/common/pathIdentity.d.ts +19 -0
- package/dist/common/pathIdentity.js +35 -0
- package/dist/common/pathNormalization.d.ts +7 -0
- package/dist/common/pathNormalization.js +13 -0
- package/dist/common/settingConstants.d.ts +1 -0
- package/dist/diagnostics/couchdb.d.ts +15 -0
- package/dist/diagnostics/couchdb.js +129 -0
- package/dist/diagnostics/index.d.ts +10 -0
- package/dist/diagnostics/index.js +26 -0
- package/dist/diagnostics/integrity.d.ts +5 -0
- package/dist/diagnostics/integrity.js +674 -0
- package/dist/diagnostics/local.d.ts +4 -0
- package/dist/diagnostics/local.js +25 -0
- package/dist/diagnostics/reports.d.ts +108 -0
- package/dist/diagnostics/reports.js +353 -0
- package/dist/diagnostics/types.d.ts +130 -0
- package/dist/diagnostics/types.js +13 -0
- package/dist/diagnostics/usage.d.ts +51 -0
- package/dist/diagnostics/usage.js +241 -0
- package/dist/hashing.d.ts +1 -0
- package/dist/hashing.js +3 -1
- package/dist/interfaces/FileReflectionProvenance.d.ts +4 -0
- package/dist/managers/EntryManager/EntryManager.d.ts +1 -0
- package/dist/managers/EntryManager/EntryManager.js +9 -0
- package/dist/pouchdb/StreamingFetch.d.ts +3 -1
- package/dist/pouchdb/StreamingFetch.js +3 -1
- package/dist/pouchdb/encryption.d.ts +3 -1
- package/dist/pouchdb/encryption.js +38 -30
- package/dist/pouchdb/pathIdentityCompatibility.d.ts +8 -0
- package/dist/pouchdb/pathIdentityCompatibility.js +64 -0
- package/dist/pouchdb/remoteFeatureCompatibility.d.ts +2 -0
- package/dist/pouchdb/remoteFeatureCompatibility.js +9 -1
- package/dist/replication/couchdb/LiveSyncReplicator.js +11 -0
- package/dist/replication/index.d.ts +1 -1
- package/dist/replication/index.js +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +2 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.js +28 -6
- package/dist/replication/trystero/P2PRoomSessionOwner.js +18 -5
- package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
- package/dist/replication/trystero/useP2PReplicatorFeature.js +2 -1
- package/dist/serviceFeatures/offlineScanner.d.ts +12 -3
- package/dist/serviceFeatures/offlineScanner.js +4 -2
- package/dist/serviceFeatures/targetFilter.d.ts +1 -1
- package/dist/serviceFeatures/targetFilter.js +28 -17
- package/dist/serviceModules/Rebuilder.d.ts +5 -1
- package/dist/serviceModules/Rebuilder.js +85 -3
- package/dist/serviceModules/ServiceFileHandlerBase.d.ts +5 -2
- package/dist/serviceModules/ServiceFileHandlerBase.js +110 -49
- package/dist/services/base/IService.d.ts +2 -1
- package/dist/services/base/PathService.d.ts +3 -3
- package/dist/services/base/PathService.js +6 -3
- package/dist/settings.d.ts +2 -0
- package/dist/settings.js +3 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/conflict-resolution.md +17 -8
- package/docs/database-diagnostics.md +24 -0
- package/docs/development.md +1 -0
- package/docs/proven-in-use.md +11 -1
- package/docs/unicode-path-identity.md +98 -0
- package/package.json +9 -1
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
function asRecord(value) {
|
|
2
|
+
return value;
|
|
3
|
+
}
|
|
4
|
+
function getRevisionNumber(revision) {
|
|
5
|
+
const generation = revision.split("-")[0];
|
|
6
|
+
return `${generation ? Number.parseInt(generation) : 0}`;
|
|
7
|
+
}
|
|
8
|
+
function getRevisionHash(revision) {
|
|
9
|
+
const hash = revision.split("-")[1];
|
|
10
|
+
if (hash === void 0) throw new Error("invalid revision");
|
|
11
|
+
return hash.substring(0, 6);
|
|
12
|
+
}
|
|
13
|
+
function getRevisionInfos(value) {
|
|
14
|
+
if (!Array.isArray(value)) return [];
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
function getChildren(value) {
|
|
18
|
+
const children = value ?? [];
|
|
19
|
+
if (!Array.isArray(children) || !children.every((child) => typeof child === "string")) {
|
|
20
|
+
throw new Error("invalid chunk references");
|
|
21
|
+
}
|
|
22
|
+
return new Set(children);
|
|
23
|
+
}
|
|
24
|
+
function getStringField(value) {
|
|
25
|
+
if (typeof value !== "string") throw new Error("invalid document field");
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
async function analyseDatabaseUsage(options) {
|
|
29
|
+
const { source, signal, onProgress } = options;
|
|
30
|
+
const errors = [];
|
|
31
|
+
const documentMap = /* @__PURE__ */ new Map();
|
|
32
|
+
const chunkMap = /* @__PURE__ */ new Map();
|
|
33
|
+
const progress = { idsProcessed: 0, revisionsProcessed: 0 };
|
|
34
|
+
let startUpdateSeq;
|
|
35
|
+
let endUpdateSeq;
|
|
36
|
+
let enumerationFailed = false;
|
|
37
|
+
const reportProgress = () => onProgress?.({ ...progress });
|
|
38
|
+
const isCancelled = () => signal?.aborted === true;
|
|
39
|
+
let cancelled = isCancelled();
|
|
40
|
+
if (!cancelled) {
|
|
41
|
+
try {
|
|
42
|
+
const info = await source.info();
|
|
43
|
+
startUpdateSeq = info.update_seq;
|
|
44
|
+
if (startUpdateSeq === void 0) errors.push({ code: "start_update_seq_unavailable" });
|
|
45
|
+
} catch {
|
|
46
|
+
errors.push({ code: "start_info_read_failed" });
|
|
47
|
+
}
|
|
48
|
+
if (isCancelled()) cancelled = true;
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
if (!cancelled && !isCancelled()) {
|
|
52
|
+
for await (const id of source.enumerateIds()) {
|
|
53
|
+
if (isCancelled()) {
|
|
54
|
+
cancelled = true;
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
let revisionInfos = [];
|
|
58
|
+
let revisionList;
|
|
59
|
+
let revisionListReadFailed = false;
|
|
60
|
+
try {
|
|
61
|
+
revisionList = await source.get(id, { revs: true, revs_info: true, conflicts: true });
|
|
62
|
+
} catch {
|
|
63
|
+
if (!isCancelled()) {
|
|
64
|
+
errors.push({ code: "revision_list_read_failed", id });
|
|
65
|
+
revisionListReadFailed = true;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (isCancelled()) {
|
|
69
|
+
cancelled = true;
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
if (revisionList) revisionInfos = getRevisionInfos(asRecord(revisionList)._revs_info);
|
|
73
|
+
else if (!revisionListReadFailed) {
|
|
74
|
+
errors.push({ code: "revision_list_read_failed", id });
|
|
75
|
+
}
|
|
76
|
+
progress.idsProcessed++;
|
|
77
|
+
reportProgress();
|
|
78
|
+
for (const revisionInfo of revisionInfos) {
|
|
79
|
+
if (isCancelled()) {
|
|
80
|
+
cancelled = true;
|
|
81
|
+
break;
|
|
82
|
+
}
|
|
83
|
+
if (revisionInfo.status !== "available") continue;
|
|
84
|
+
const revision = typeof revisionInfo.rev === "string" ? revisionInfo.rev : void 0;
|
|
85
|
+
if (revision === void 0) {
|
|
86
|
+
errors.push({ code: "revision_document_invalid", id });
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
progress.revisionsProcessed++;
|
|
90
|
+
let rawDocument;
|
|
91
|
+
try {
|
|
92
|
+
rawDocument = await source.get(id, { rev: revision });
|
|
93
|
+
} catch {
|
|
94
|
+
if (isCancelled()) {
|
|
95
|
+
cancelled = true;
|
|
96
|
+
reportProgress();
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
errors.push({ code: "revision_read_failed", id, revision });
|
|
100
|
+
reportProgress();
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (isCancelled()) {
|
|
104
|
+
cancelled = true;
|
|
105
|
+
reportProgress();
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
if (!rawDocument) {
|
|
109
|
+
errors.push({ code: "revision_not_found", id, revision });
|
|
110
|
+
reportProgress();
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
const document = asRecord(rawDocument);
|
|
115
|
+
if ("children" in document) {
|
|
116
|
+
const documentId = getStringField(document._id);
|
|
117
|
+
const documentRevision = getStringField(document._rev);
|
|
118
|
+
const path = getStringField(document.path);
|
|
119
|
+
const chunks = getChildren(document.children);
|
|
120
|
+
const revisions = documentMap.get(documentId) ?? [];
|
|
121
|
+
revisions.push({
|
|
122
|
+
id: documentId,
|
|
123
|
+
revision: documentRevision,
|
|
124
|
+
path,
|
|
125
|
+
chunks,
|
|
126
|
+
uniqueChunks: /* @__PURE__ */ new Set(),
|
|
127
|
+
sharedChunks: /* @__PURE__ */ new Set()
|
|
128
|
+
});
|
|
129
|
+
documentMap.set(documentId, revisions);
|
|
130
|
+
} else if (document.type === "leaf") {
|
|
131
|
+
const chunkId = getStringField(document._id);
|
|
132
|
+
if (!chunkMap.has(chunkId) && !document._deleted) {
|
|
133
|
+
if (typeof document.data !== "string") throw new Error("invalid chunk data");
|
|
134
|
+
chunkMap.set(chunkId, { length: document.data.length, refCount: 0 });
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
} catch {
|
|
138
|
+
errors.push({ code: "revision_document_invalid", id, revision });
|
|
139
|
+
}
|
|
140
|
+
reportProgress();
|
|
141
|
+
}
|
|
142
|
+
if (isCancelled()) cancelled = true;
|
|
143
|
+
if (cancelled) break;
|
|
144
|
+
}
|
|
145
|
+
} else if (isCancelled()) {
|
|
146
|
+
cancelled = true;
|
|
147
|
+
}
|
|
148
|
+
} catch {
|
|
149
|
+
if (isCancelled()) cancelled = true;
|
|
150
|
+
else {
|
|
151
|
+
enumerationFailed = true;
|
|
152
|
+
errors.push({ code: "id_enumeration_failed" });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
for (const revisions of documentMap.values()) {
|
|
156
|
+
for (const document of revisions) {
|
|
157
|
+
for (const chunkId of document.chunks) {
|
|
158
|
+
const chunk = chunkMap.get(chunkId);
|
|
159
|
+
if (!chunk) continue;
|
|
160
|
+
if (chunk.refCount === 0) document.uniqueChunks.add(chunkId);
|
|
161
|
+
else document.sharedChunks.add(chunkId);
|
|
162
|
+
chunk.refCount++;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const totalSize = (ids) => {
|
|
167
|
+
let size = 0;
|
|
168
|
+
for (const id of ids) size += chunkMap.get(id)?.length ?? 0;
|
|
169
|
+
return size;
|
|
170
|
+
};
|
|
171
|
+
const rows = [];
|
|
172
|
+
for (const revisions of documentMap.values()) {
|
|
173
|
+
for (const document of revisions) {
|
|
174
|
+
const revisionNumber = getRevisionNumber(document.revision);
|
|
175
|
+
let revisionHash;
|
|
176
|
+
try {
|
|
177
|
+
revisionHash = getRevisionHash(document.revision);
|
|
178
|
+
} catch {
|
|
179
|
+
errors.push({ code: "revision_document_invalid", id: document.id, revision: document.revision });
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
rows.push({
|
|
183
|
+
title: `${document.path} (${document.revision})`,
|
|
184
|
+
id: document.id,
|
|
185
|
+
path: document.path,
|
|
186
|
+
revision: document.revision,
|
|
187
|
+
rev: revisionNumber,
|
|
188
|
+
revHash: revisionHash,
|
|
189
|
+
uniqueChunkCount: document.uniqueChunks.size,
|
|
190
|
+
sharedChunkCount: document.sharedChunks.size,
|
|
191
|
+
uniqueChunkSize: totalSize(document.uniqueChunks),
|
|
192
|
+
sharedChunkSize: totalSize(document.sharedChunks)
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const orphanChunks = [...chunkMap.entries()].filter(([, chunk]) => chunk.refCount === 0);
|
|
197
|
+
rows.push({
|
|
198
|
+
title: "__orphan",
|
|
199
|
+
id: "__orphan",
|
|
200
|
+
path: "__orphan",
|
|
201
|
+
revision: "1-xxxxx",
|
|
202
|
+
rev: "1",
|
|
203
|
+
revHash: "xxxxx",
|
|
204
|
+
uniqueChunkCount: orphanChunks.length,
|
|
205
|
+
sharedChunkCount: 0,
|
|
206
|
+
uniqueChunkSize: orphanChunks.reduce((size, [, chunk]) => size + chunk.length, 0),
|
|
207
|
+
sharedChunkSize: 0
|
|
208
|
+
});
|
|
209
|
+
if (!cancelled && !isCancelled()) {
|
|
210
|
+
try {
|
|
211
|
+
const info = await source.info();
|
|
212
|
+
endUpdateSeq = info.update_seq;
|
|
213
|
+
if (endUpdateSeq === void 0) errors.push({ code: "end_update_seq_unavailable" });
|
|
214
|
+
} catch {
|
|
215
|
+
errors.push({ code: "end_info_read_failed" });
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (isCancelled()) cancelled = true;
|
|
219
|
+
const changedDuringRun = startUpdateSeq === void 0 || endUpdateSeq === void 0 ? void 0 : startUpdateSeq !== endUpdateSeq;
|
|
220
|
+
const snapshotUncertain = startUpdateSeq === void 0 || endUpdateSeq === void 0;
|
|
221
|
+
const hasDocumentRows = rows.length > 1;
|
|
222
|
+
let status;
|
|
223
|
+
if (cancelled) status = "cancelled";
|
|
224
|
+
else if (changedDuringRun === true || snapshotUncertain) status = "partial";
|
|
225
|
+
else if (enumerationFailed && !hasDocumentRows) status = "failed";
|
|
226
|
+
else if (errors.length > 0) status = "partial";
|
|
227
|
+
else status = "complete";
|
|
228
|
+
return {
|
|
229
|
+
version: 1,
|
|
230
|
+
kind: "usage",
|
|
231
|
+
status,
|
|
232
|
+
...startUpdateSeq === void 0 ? {} : { startUpdateSeq },
|
|
233
|
+
...endUpdateSeq === void 0 ? {} : { endUpdateSeq },
|
|
234
|
+
...changedDuringRun === void 0 ? {} : { changedDuringRun },
|
|
235
|
+
rows,
|
|
236
|
+
errors
|
|
237
|
+
};
|
|
238
|
+
}
|
|
239
|
+
export {
|
|
240
|
+
analyseDatabaseUsage
|
|
241
|
+
};
|
package/dist/hashing.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
/** Chunk ID calculation using the same hash manager as database writes. */
|
|
2
2
|
export { HashManager } from "./managers/HashManager/HashManager.js";
|
|
3
3
|
export type { HashManagerCoreOptions } from "./managers/HashManager/HashManagerCore.js";
|
|
4
|
+
export { computeFileContentHash } from "./common/fileContentHash.js";
|
package/dist/hashing.js
CHANGED
|
@@ -5,6 +5,10 @@ export type FileReflectionProvenanceRecord = {
|
|
|
5
5
|
revision: string;
|
|
6
6
|
/** Raw modification time of the saved snapshot or storage observed after reflection. */
|
|
7
7
|
observedStorageMtime?: number;
|
|
8
|
+
/** Algorithm-tagged hash of the exact saved or reflected bytes, independent of revision retention. */
|
|
9
|
+
contentHash?: string;
|
|
10
|
+
/** Byte length of the hashed snapshot. Old records may omit both content fields. */
|
|
11
|
+
contentSize?: number;
|
|
8
12
|
};
|
|
9
13
|
export interface FileReflectionProvenance {
|
|
10
14
|
get(path: FilePathWithPrefix): Promise<FileReflectionProvenanceRecord | undefined>;
|
|
@@ -38,4 +38,5 @@ export declare class EntryManager {
|
|
|
38
38
|
putDBEntryWithLiveBaseRevision(note: SavingEntry, baseRevision: string, onlyChunks?: boolean): Promise<false | PouchDB.Core.Response>;
|
|
39
39
|
/** Store unknown ancestry as a fresh root; return the Metadata write result or `false`. */
|
|
40
40
|
putDBEntryAsIndependentRoot(note: SavingEntry): Promise<false | PouchDB.Core.Response>;
|
|
41
|
+
private checkPathIdentity;
|
|
41
42
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
REMOTE_COUCHDB
|
|
3
3
|
} from "../../common/types.js";
|
|
4
|
+
import { assertDatabasePathIdentity } from "../../pouchdb/pathIdentityCompatibility.js";
|
|
4
5
|
import {
|
|
5
6
|
deleteDBEntryByPath,
|
|
6
7
|
getDBEntryByPath,
|
|
@@ -65,21 +66,29 @@ class EntryManager {
|
|
|
65
66
|
return await getDBEntryFromMeta(this.serviceHost, this, meta, dump, waitForReady, localOnly);
|
|
66
67
|
}
|
|
67
68
|
async deleteDBEntry(path, opt) {
|
|
69
|
+
await this.checkPathIdentity();
|
|
68
70
|
return await deleteDBEntryByPath(this.serviceHost, this, path, opt);
|
|
69
71
|
}
|
|
70
72
|
async storeDeletionAtRevision(path, baseRevision) {
|
|
73
|
+
await this.checkPathIdentity();
|
|
71
74
|
return await storeDeletionByPathAtRevision(this.serviceHost, this, path, baseRevision);
|
|
72
75
|
}
|
|
73
76
|
async putDBEntry(note, onlyChunks, conflictBaseRev) {
|
|
77
|
+
await this.checkPathIdentity();
|
|
74
78
|
return await putDBEntry(this.serviceHost, this, note, onlyChunks, conflictBaseRev);
|
|
75
79
|
}
|
|
76
80
|
async putDBEntryWithLiveBaseRevision(note, baseRevision, onlyChunks) {
|
|
81
|
+
await this.checkPathIdentity();
|
|
77
82
|
return await putDBEntryWithLiveBaseRevision(this.serviceHost, this, note, baseRevision, onlyChunks);
|
|
78
83
|
}
|
|
79
84
|
/** Store unknown ancestry as a fresh root; return the Metadata write result or `false`. */
|
|
80
85
|
async putDBEntryAsIndependentRoot(note) {
|
|
86
|
+
await this.checkPathIdentity();
|
|
81
87
|
return await putDBEntryAsIndependentRoot(this.serviceHost, this, note);
|
|
82
88
|
}
|
|
89
|
+
async checkPathIdentity() {
|
|
90
|
+
await assertDatabasePathIdentity(this.localDatabase, this.options.settingService.currentSettings(), true);
|
|
91
|
+
}
|
|
83
92
|
}
|
|
84
93
|
export {
|
|
85
94
|
EntryManager
|
|
@@ -36,7 +36,9 @@ export declare class StreamingFetchFailure extends Error {
|
|
|
36
36
|
}
|
|
37
37
|
export declare function isRetryableStreamingFetchFailure(error: unknown): error is StreamingFetchFailure;
|
|
38
38
|
/** Check remote feature requirements before Fast Fetch opens or resets the local database. */
|
|
39
|
-
export declare function checkRemoteFeaturesForInitialSync(remoteDbUrl: string, authHeader: string, customHeaders?: Record<string, string
|
|
39
|
+
export declare function checkRemoteFeaturesForInitialSync(remoteDbUrl: string, authHeader: string, customHeaders?: Record<string, string>, settings?: {
|
|
40
|
+
pathNormalizationVersion?: number;
|
|
41
|
+
}): Promise<void>;
|
|
40
42
|
export type FetchChangesForInitialSyncProgress = {
|
|
41
43
|
totalFetched: number;
|
|
42
44
|
totalValidFetched: number;
|
|
@@ -2,6 +2,7 @@ import { _fetch } from "../common/coreEnvFunctions.js";
|
|
|
2
2
|
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
|
3
3
|
import { VERSIONING_DOCID } from "../common/types.js";
|
|
4
4
|
import { assessRemoteFeatureDocument, describeRemoteFeatureRejection } from "./remoteFeatureCompatibility.js";
|
|
5
|
+
import { assertMatchingPathIdentityFeature } from "./pathIdentityCompatibility.js";
|
|
5
6
|
const FAST_FETCH_CHANGES_PAGE_LIMIT = 1e4;
|
|
6
7
|
const FAST_FETCH_CHANGES_PAGE_TIMEOUT_MS = 1e3;
|
|
7
8
|
class StreamingFetchFailure extends Error {
|
|
@@ -75,7 +76,7 @@ async function readResponseText(response, operation) {
|
|
|
75
76
|
throw transportFailure(operation, error);
|
|
76
77
|
}
|
|
77
78
|
}
|
|
78
|
-
async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, customHeaders) {
|
|
79
|
+
async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, customHeaders, settings = {}) {
|
|
79
80
|
const headers = new Headers(customHeaders);
|
|
80
81
|
headers.set("Accept", "application/json");
|
|
81
82
|
headers.set("Authorization", authHeader);
|
|
@@ -97,6 +98,7 @@ async function checkRemoteFeaturesForInitialSync(remoteDbUrl, authHeader, custom
|
|
|
97
98
|
const row = result.rows[0];
|
|
98
99
|
if (result.total_rows === 0 && row.error === "not_found") return;
|
|
99
100
|
const assessment = assessRemoteFeatureDocument(row.doc);
|
|
101
|
+
assertMatchingPathIdentityFeature(row.doc, settings);
|
|
100
102
|
if (assessment.status === "supported" || assessment.status === "older-generation") return;
|
|
101
103
|
throw new StreamingFetchFailure("protocol", describeRemoteFeatureRejection(assessment), false);
|
|
102
104
|
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
|
+
import { Logger } from "../common/logger.js";
|
|
1
2
|
import { type EntryDoc, type AnyEntry, type EntryLeaf, type DocumentID, type E2EEAlgorithm } from "../common/types.js";
|
|
2
3
|
import { encryptWorker, decryptWorker, encryptHKDFWorker, decryptHKDFWorker } from "#worker";
|
|
3
4
|
export declare const encrypt: typeof encryptWorker;
|
|
4
5
|
export declare const decrypt: typeof decryptWorker;
|
|
5
6
|
export declare const encryptHKDF: typeof encryptHKDFWorker;
|
|
6
7
|
export declare const decryptHKDF: typeof decryptHKDFWorker;
|
|
8
|
+
export declare function isEncryptedMeta<T extends AnyEntry>(doc: T): boolean;
|
|
7
9
|
export declare let preprocessOutgoing: (doc: AnyEntry | EntryLeaf) => Promise<AnyEntry | EntryLeaf>;
|
|
8
10
|
export declare let preprocessIncoming: (doc: EntryDoc) => Promise<EntryDoc>;
|
|
9
|
-
export declare function getConfiguredFunctionsForEncryption(passphrase: string, useDynamicIterationCount: boolean, migrationDecrypt: boolean, getPBKDF2Salt: () => Promise<Uint8Array<ArrayBuffer>>, algorithm: E2EEAlgorithm, encryptInternalMetadata?: boolean): {
|
|
11
|
+
export declare function getConfiguredFunctionsForEncryption(passphrase: string, useDynamicIterationCount: boolean, migrationDecrypt: boolean, getPBKDF2Salt: () => Promise<Uint8Array<ArrayBuffer>>, algorithm: E2EEAlgorithm, encryptInternalMetadata?: boolean, log?: typeof Logger): {
|
|
10
12
|
incoming: (doc: AnyEntry | EntryLeaf) => Promise<AnyEntry | EntryLeaf>;
|
|
11
13
|
outgoing: (doc: EntryDoc) => Promise<AnyEntry | EntryLeaf>;
|
|
12
14
|
};
|
|
@@ -32,20 +32,20 @@ function getEncryptionVersion(data) {
|
|
|
32
32
|
}
|
|
33
33
|
return EncryptionVersions.UNENCRYPTED;
|
|
34
34
|
}
|
|
35
|
-
async function tryDecryptV1AsFallback(encryptedData, passphrase, useDynamicIterationCount) {
|
|
35
|
+
async function tryDecryptV1AsFallback(encryptedData, passphrase, useDynamicIterationCount, log = Logger) {
|
|
36
36
|
try {
|
|
37
37
|
return await decrypt(encryptedData, passphrase, useDynamicIterationCount);
|
|
38
38
|
} catch (ex) {
|
|
39
39
|
try {
|
|
40
|
-
|
|
40
|
+
log(
|
|
41
41
|
"Failed to decrypt with V1 method. Fallback to disable useDynamicIterationCount.",
|
|
42
42
|
LOG_LEVEL_VERBOSE
|
|
43
43
|
);
|
|
44
|
-
|
|
44
|
+
log(ex, LOG_LEVEL_VERBOSE);
|
|
45
45
|
return await decrypt(encryptedData, passphrase, false);
|
|
46
46
|
} catch (ex2) {
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
log("Completely failed to decrypt with V1 method.", LOG_LEVEL_VERBOSE);
|
|
48
|
+
log(ex2, LOG_LEVEL_VERBOSE);
|
|
49
49
|
return false;
|
|
50
50
|
}
|
|
51
51
|
}
|
|
@@ -153,7 +153,7 @@ async function incomingEncryptHKDF(doc, passphrase, useDynamicIterationCount, ge
|
|
|
153
153
|
}
|
|
154
154
|
return saveDoc;
|
|
155
155
|
}
|
|
156
|
-
async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase, useDynamicIterationCount, getPBKDF2Salt) {
|
|
156
|
+
async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase, useDynamicIterationCount, getPBKDF2Salt, log) {
|
|
157
157
|
const loadDoc = {
|
|
158
158
|
...doc
|
|
159
159
|
};
|
|
@@ -165,21 +165,26 @@ async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase,
|
|
|
165
165
|
loadDoc.data = await decryptHKDF(loadDoc.data, passphrase, pbkdf2salt);
|
|
166
166
|
delete loadDoc.e_;
|
|
167
167
|
} else if (encryptionVersion === EncryptionVersions.ENCRYPTED) {
|
|
168
|
-
const decryptedData = await tryDecryptV1AsFallback(
|
|
168
|
+
const decryptedData = await tryDecryptV1AsFallback(
|
|
169
|
+
loadDoc.data,
|
|
170
|
+
passphrase,
|
|
171
|
+
useDynamicIterationCount,
|
|
172
|
+
log
|
|
173
|
+
);
|
|
169
174
|
if (decryptedData === false) {
|
|
170
|
-
|
|
175
|
+
log(MESSAGE_FALLBACK_DECRYPT_FAILED, LOG_LEVEL_NOTICE);
|
|
171
176
|
throw new Error(MESSAGE_FALLBACK_DECRYPT_FAILED);
|
|
172
177
|
}
|
|
173
178
|
loadDoc.data = decryptedData;
|
|
174
179
|
delete loadDoc.e_;
|
|
175
180
|
} else if (encryptionVersion === EncryptionVersions.UNENCRYPTED) {
|
|
176
181
|
} else {
|
|
177
|
-
|
|
182
|
+
log("Unknown encryption version. Cannot decrypt.", LOG_LEVEL_NOTICE);
|
|
178
183
|
throw new Error("Unknown encryption version. Cannot decrypt.");
|
|
179
184
|
}
|
|
180
185
|
} catch (ex) {
|
|
181
|
-
|
|
182
|
-
|
|
186
|
+
log(DECRYPTION_HKDF_FAILED, LOG_LEVEL_NOTICE);
|
|
187
|
+
log(ex);
|
|
183
188
|
throw ex;
|
|
184
189
|
}
|
|
185
190
|
}
|
|
@@ -192,16 +197,16 @@ async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase,
|
|
|
192
197
|
loadDoc[key] = metadata[key];
|
|
193
198
|
}
|
|
194
199
|
} catch (ex) {
|
|
195
|
-
|
|
196
|
-
|
|
200
|
+
log(`${DECRYPTION_HKDF_FAILED} on Path`, LOG_LEVEL_NOTICE);
|
|
201
|
+
log(ex);
|
|
197
202
|
throw ex;
|
|
198
203
|
}
|
|
199
204
|
} else {
|
|
200
205
|
const path = getPath(loadDoc);
|
|
201
206
|
if (isPathProbablyObfuscated(path)) {
|
|
202
|
-
const decryptedPath = await tryDecryptV1AsFallback(path, passphrase, useDynamicIterationCount);
|
|
207
|
+
const decryptedPath = await tryDecryptV1AsFallback(path, passphrase, useDynamicIterationCount, log);
|
|
203
208
|
if (decryptedPath === false) {
|
|
204
|
-
|
|
209
|
+
log(`${MESSAGE_FALLBACK_DECRYPT_FAILED} on Path`, LOG_LEVEL_NOTICE);
|
|
205
210
|
throw new Error(MESSAGE_FALLBACK_DECRYPT_FAILED);
|
|
206
211
|
}
|
|
207
212
|
loadDoc.path = decryptedPath;
|
|
@@ -215,7 +220,8 @@ async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase,
|
|
|
215
220
|
const decryptedEden = await tryDecryptV1AsFallback(
|
|
216
221
|
loadDoc.eden[EDEN_ENCRYPTED_KEY].data,
|
|
217
222
|
passphrase,
|
|
218
|
-
useDynamicIterationCount
|
|
223
|
+
useDynamicIterationCount,
|
|
224
|
+
log
|
|
219
225
|
);
|
|
220
226
|
if (decryptedEden === false) throw new Error(MESSAGE_FALLBACK_DECRYPT_FAILED);
|
|
221
227
|
readEden = {
|
|
@@ -224,8 +230,8 @@ async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase,
|
|
|
224
230
|
};
|
|
225
231
|
edenDecrypted = true;
|
|
226
232
|
} catch (ex) {
|
|
227
|
-
|
|
228
|
-
|
|
233
|
+
log(`${DECRYPTION_FALLBACK_FAILED} on Eden`, LOG_LEVEL_NOTICE);
|
|
234
|
+
log(ex);
|
|
229
235
|
throw ex;
|
|
230
236
|
}
|
|
231
237
|
}
|
|
@@ -243,8 +249,8 @@ async function outgoingDecryptHKDF(doc, migrationDecrypt, decrypted, passphrase,
|
|
|
243
249
|
};
|
|
244
250
|
edenDecrypted = true;
|
|
245
251
|
} catch (ex) {
|
|
246
|
-
|
|
247
|
-
|
|
252
|
+
log(`${DECRYPTION_HKDF_FAILED} on Eden`, LOG_LEVEL_NOTICE);
|
|
253
|
+
log(ex);
|
|
248
254
|
throw ex;
|
|
249
255
|
}
|
|
250
256
|
}
|
|
@@ -296,7 +302,7 @@ async function incomingEncryptV1(doc, passphrase, useDynamicIterationCount) {
|
|
|
296
302
|
}
|
|
297
303
|
return saveDoc;
|
|
298
304
|
}
|
|
299
|
-
async function outgoingDecryptV1(doc, migrationDecrypt, decrypted, passphrase, useDynamicIterationCount) {
|
|
305
|
+
async function outgoingDecryptV1(doc, migrationDecrypt, decrypted, passphrase, useDynamicIterationCount, log) {
|
|
300
306
|
const loadDoc = {
|
|
301
307
|
...doc
|
|
302
308
|
};
|
|
@@ -361,15 +367,15 @@ async function outgoingDecryptV1(doc, migrationDecrypt, decrypted, passphrase, u
|
|
|
361
367
|
if (migrationDecrypt && ex2 instanceof SyntaxError) {
|
|
362
368
|
return loadDoc;
|
|
363
369
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
370
|
+
log("Decryption failed.", LOG_LEVEL_NOTICE);
|
|
371
|
+
log(ex2, LOG_LEVEL_VERBOSE);
|
|
372
|
+
log(`id:${loadDoc._id}-${loadDoc._rev?.substring(0, 10)}`, LOG_LEVEL_VERBOSE);
|
|
367
373
|
throw ex2;
|
|
368
374
|
}
|
|
369
375
|
} else {
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
376
|
+
log("Decryption failed.", LOG_LEVEL_NOTICE);
|
|
377
|
+
log(ex, LOG_LEVEL_VERBOSE);
|
|
378
|
+
log(`id:${loadDoc._id}-${loadDoc._rev?.substring(0, 10)}`, LOG_LEVEL_VERBOSE);
|
|
373
379
|
throw ex;
|
|
374
380
|
}
|
|
375
381
|
}
|
|
@@ -382,7 +388,7 @@ let preprocessOutgoing = async (doc) => {
|
|
|
382
388
|
let preprocessIncoming = async (doc) => {
|
|
383
389
|
return await Promise.resolve(doc);
|
|
384
390
|
};
|
|
385
|
-
function getConfiguredFunctionsForEncryption(passphrase, useDynamicIterationCount, migrationDecrypt, getPBKDF2Salt, algorithm, encryptInternalMetadata = false) {
|
|
391
|
+
function getConfiguredFunctionsForEncryption(passphrase, useDynamicIterationCount, migrationDecrypt, getPBKDF2Salt, algorithm, encryptInternalMetadata = false, log = Logger) {
|
|
386
392
|
const decryptedCache = /* @__PURE__ */ new Map();
|
|
387
393
|
const incoming = (doc) => algorithm === E2EEAlgorithms.V2 ? incomingEncryptHKDF(doc, passphrase, useDynamicIterationCount, getPBKDF2Salt, encryptInternalMetadata) : incomingEncryptV1(doc, passphrase, useDynamicIterationCount);
|
|
388
394
|
const outgoing = (doc) => algorithm !== E2EEAlgorithms.ForceV1 ? outgoingDecryptHKDF(
|
|
@@ -391,8 +397,9 @@ function getConfiguredFunctionsForEncryption(passphrase, useDynamicIterationCoun
|
|
|
391
397
|
decryptedCache,
|
|
392
398
|
passphrase,
|
|
393
399
|
useDynamicIterationCount,
|
|
394
|
-
getPBKDF2Salt
|
|
395
|
-
|
|
400
|
+
getPBKDF2Salt,
|
|
401
|
+
log
|
|
402
|
+
) : outgoingDecryptV1(doc, migrationDecrypt, decryptedCache, passphrase, useDynamicIterationCount, log);
|
|
396
403
|
return {
|
|
397
404
|
incoming,
|
|
398
405
|
outgoing
|
|
@@ -459,6 +466,7 @@ export {
|
|
|
459
466
|
encrypt,
|
|
460
467
|
encryptHKDF,
|
|
461
468
|
getConfiguredFunctionsForEncryption,
|
|
469
|
+
isEncryptedMeta,
|
|
462
470
|
preprocessIncoming,
|
|
463
471
|
preprocessOutgoing,
|
|
464
472
|
shouldDecryptEden,
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type EntryDoc } from "../common/types.js";
|
|
2
|
+
type PathIdentitySettings = {
|
|
3
|
+
pathNormalizationVersion?: number;
|
|
4
|
+
};
|
|
5
|
+
export declare function assertMatchingPathIdentityFeature(document: unknown, settings: PathIdentitySettings): void;
|
|
6
|
+
/** Refuse an implicit migration. A fresh database may declare its identity before its first Metadata write. */
|
|
7
|
+
export declare function assertDatabasePathIdentity(database: PouchDB.Database<EntryDoc>, settings: PathIdentitySettings, declare?: boolean): Promise<void>;
|
|
8
|
+
export {};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { VERSIONING_DOCID } from "../common/types.js";
|
|
2
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
3
|
+
import {
|
|
4
|
+
assessRemoteFeatureDocument,
|
|
5
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
6
|
+
REMOTE_FEATURE_GENERATION
|
|
7
|
+
} from "./remoteFeatureCompatibility.js";
|
|
8
|
+
import { isErrorOfMissingDoc } from "./utils_couchdb.js";
|
|
9
|
+
import { bumpRemoteVersion } from "./negotiation.js";
|
|
10
|
+
function assertMatchingPathIdentityFeature(document, settings) {
|
|
11
|
+
const assessment = assessRemoteFeatureDocument(document);
|
|
12
|
+
const declared = assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE);
|
|
13
|
+
if (declared !== (getPathNormalizationVersion(settings) === 1)) {
|
|
14
|
+
throw new Error(
|
|
15
|
+
"The database and settings use different path identity versions. Rebuild from the authoritative Vault or Fetch with matching settings before synchronising."
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
async function hasMetadata(database) {
|
|
20
|
+
let startkey;
|
|
21
|
+
for (; ; ) {
|
|
22
|
+
const result = await database.allDocs({
|
|
23
|
+
include_docs: true,
|
|
24
|
+
limit: 100,
|
|
25
|
+
...startkey ? { startkey, skip: 1 } : {}
|
|
26
|
+
});
|
|
27
|
+
for (const row of result.rows) {
|
|
28
|
+
if (row.doc?.type === "plain" || row.doc?.type === "newnote") return true;
|
|
29
|
+
}
|
|
30
|
+
if (result.rows.length < 100) return false;
|
|
31
|
+
startkey = result.rows[result.rows.length - 1].id;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
async function assertDatabasePathIdentity(database, settings, declare = false) {
|
|
35
|
+
let document;
|
|
36
|
+
try {
|
|
37
|
+
document = await database.get(VERSIONING_DOCID);
|
|
38
|
+
} catch (error) {
|
|
39
|
+
if (!isErrorOfMissingDoc(error)) throw error;
|
|
40
|
+
}
|
|
41
|
+
const assessment = assessRemoteFeatureDocument(document);
|
|
42
|
+
const declared = assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE);
|
|
43
|
+
if (getPathNormalizationVersion(settings) === 0) {
|
|
44
|
+
if (declared) assertMatchingPathIdentityFeature(document, settings);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
if (declared) return;
|
|
48
|
+
if (await hasMetadata(database)) {
|
|
49
|
+
try {
|
|
50
|
+
document = await database.get(VERSIONING_DOCID);
|
|
51
|
+
} catch (error) {
|
|
52
|
+
if (!isErrorOfMissingDoc(error)) throw error;
|
|
53
|
+
}
|
|
54
|
+
assertMatchingPathIdentityFeature(document, settings);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
if (declare && !await bumpRemoteVersion(database, REMOTE_FEATURE_GENERATION, [NFC_PATH_IDENTITY_FEATURE])) {
|
|
58
|
+
throw new Error("The database path identity could not be declared. Synchronisation has been stopped.");
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
export {
|
|
62
|
+
assertDatabasePathIdentity,
|
|
63
|
+
assertMatchingPathIdentityFeature
|
|
64
|
+
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export declare const REMOTE_FEATURE_GENERATION = 13;
|
|
2
2
|
export declare const ENCRYPTED_INTERNAL_METADATA_FEATURE = "encrypted-internal-metadata-v1";
|
|
3
3
|
export declare const INDEPENDENT_ID_DERIVATION_FEATURE = "independent-id-derivation-v1";
|
|
4
|
+
export declare const NFC_PATH_IDENTITY_FEATURE = "nfc-path-identity-v1";
|
|
4
5
|
export declare const supportedFeatures: Set<string>;
|
|
5
6
|
export declare function requiredRemoteFeatures(setting: {
|
|
6
7
|
encrypt?: boolean;
|
|
@@ -9,6 +10,7 @@ export declare function requiredRemoteFeatures(setting: {
|
|
|
9
10
|
encryptInternalMetadata?: boolean;
|
|
10
11
|
idDerivationVersion: 0 | 1;
|
|
11
12
|
idDerivationKey: string;
|
|
13
|
+
pathNormalizationVersion?: number;
|
|
12
14
|
}): string[];
|
|
13
15
|
export declare function usesEncryptedInternalMetadata(setting: {
|
|
14
16
|
encrypt?: boolean;
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { E2EEAlgorithms, VERSIONING_DOCID } from "../common/types.js";
|
|
2
2
|
import { configuredIdKey } from "../common/idDerivation.js";
|
|
3
|
+
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
3
4
|
const REMOTE_FEATURE_GENERATION = 13;
|
|
4
5
|
const ENCRYPTED_INTERNAL_METADATA_FEATURE = "encrypted-internal-metadata-v1";
|
|
5
6
|
const INDEPENDENT_ID_DERIVATION_FEATURE = "independent-id-derivation-v1";
|
|
6
|
-
const
|
|
7
|
+
const NFC_PATH_IDENTITY_FEATURE = "nfc-path-identity-v1";
|
|
8
|
+
const supportedFeatures = /* @__PURE__ */ new Set([
|
|
9
|
+
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
10
|
+
INDEPENDENT_ID_DERIVATION_FEATURE,
|
|
11
|
+
NFC_PATH_IDENTITY_FEATURE
|
|
12
|
+
]);
|
|
7
13
|
function requiredRemoteFeatures(setting) {
|
|
8
14
|
const features = [];
|
|
9
15
|
if (usesEncryptedInternalMetadata(setting)) features.push(ENCRYPTED_INTERNAL_METADATA_FEATURE);
|
|
16
|
+
if (getPathNormalizationVersion(setting) === 1) features.push(NFC_PATH_IDENTITY_FEATURE);
|
|
10
17
|
if (setting.encrypt && configuredIdKey(setting)) features.push(INDEPENDENT_ID_DERIVATION_FEATURE);
|
|
11
18
|
return features;
|
|
12
19
|
}
|
|
@@ -54,6 +61,7 @@ function describeRemoteFeatureRejection(assessment) {
|
|
|
54
61
|
export {
|
|
55
62
|
ENCRYPTED_INTERNAL_METADATA_FEATURE,
|
|
56
63
|
INDEPENDENT_ID_DERIVATION_FEATURE,
|
|
64
|
+
NFC_PATH_IDENTITY_FEATURE,
|
|
57
65
|
REMOTE_FEATURE_GENERATION,
|
|
58
66
|
assessRemoteFeatureDocument,
|
|
59
67
|
describeRemoteFeatureRejection,
|