@vrtmrz/livesync-commonlib 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -4
- package/dist/API/DirectFileManipulatorV2.d.ts +1 -0
- package/dist/API/DirectFileManipulatorV2.js +6 -12
- package/dist/API/processSetting.d.ts +1 -0
- package/dist/common/models/setting.const.defaults.js +1 -0
- package/dist/common/models/setting.const.qr.js +1 -0
- package/dist/common/models/setting.policy.d.ts +4 -0
- package/dist/common/models/setting.policy.js +1 -0
- package/dist/common/models/setting.type.d.ts +2 -0
- package/dist/common/models/tweak.definition.d.ts +1 -0
- package/dist/common/pathNamespace.d.ts +4 -0
- package/dist/common/pathNamespace.js +10 -0
- package/dist/common/settingConstants.d.ts +1 -0
- package/dist/common/settingConstants.js +4 -0
- package/dist/platform/AmbiguousNormalisedPathError.d.ts +16 -0
- package/dist/platform/AmbiguousNormalisedPathError.js +21 -0
- package/dist/platform/StoragePathResolver.d.ts +15 -0
- package/dist/platform/StoragePathResolver.js +113 -0
- package/dist/platform/browser/UnicodeResolvingFileSystemAccessStorageAdapter.d.ts +74 -0
- package/dist/platform/browser/UnicodeResolvingFileSystemAccessStorageAdapter.js +169 -0
- package/dist/platform/browser/index.d.ts +1 -0
- package/dist/platform/browser/index.js +2 -0
- package/dist/platform/browser/storage.d.ts +23 -0
- package/dist/platform/browser/storage.js +138 -0
- package/dist/platform/node/UnicodeResolvingNodeStorageAdapter.d.ts +79 -0
- package/dist/platform/node/UnicodeResolvingNodeStorageAdapter.js +218 -0
- package/dist/platform/node/index.d.ts +1 -0
- package/dist/platform/node/index.js +2 -0
- package/dist/platform/node/storage.d.ts +30 -1
- package/dist/platform/node/storage.js +88 -0
- package/dist/platform/pathComparison.d.ts +53 -0
- package/dist/platform/pathComparison.js +39 -0
- package/dist/platform/pathTypes.d.ts +154 -0
- package/dist/platform/pathTypes.js +0 -0
- package/dist/platform/storagePathResolution.d.ts +13 -0
- package/dist/platform/storagePathResolution.js +9 -0
- package/dist/replication/SyncParamsHandler.d.ts +1 -0
- package/dist/replication/SyncParamsHandler.js +5 -1
- package/dist/replication/couchdb/LiveSyncReplicator.js +5 -3
- package/dist/replication/journal/JournalReceiveFailure.d.ts +7 -0
- package/dist/replication/journal/JournalReceiveFailure.js +16 -0
- package/dist/replication/journal/JournalSyncCore.d.ts +22 -3
- package/dist/replication/journal/JournalSyncCore.js +288 -166
- package/dist/replication/journal/JournalSyncTypes.d.ts +13 -0
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +6 -1
- package/dist/replication/journal/LiveSyncJournalReplicator.js +31 -44
- package/dist/replication/journal/journalCompatibility.d.ts +9 -0
- package/dist/replication/journal/journalCompatibility.js +58 -0
- package/dist/replication/journal/journalReceive.d.ts +23 -0
- package/dist/replication/journal/journalReceive.js +44 -0
- package/dist/replication/journal/journalRecords.d.ts +10 -0
- package/dist/replication/journal/journalRecords.js +96 -0
- package/dist/replication/trystero/TrysteroReplicator.d.ts +2 -0
- package/dist/serviceModules/Rebuilder.js +2 -1
- package/dist/services/base/FileProcessingService.d.ts +2 -2
- package/dist/services/base/RemoteConnection.d.ts +2 -0
- package/dist/services/base/RemoteService.d.ts +1 -1
- package/dist/services/base/RemoteService.js +13 -5
- package/dist/services/implements/injectable/InjectableVaultService.d.ts +1 -1
- package/dist/services/lib/couchdbHttpCache.d.ts +5 -0
- package/dist/services/lib/couchdbHttpCache.js +36 -0
- package/dist/storage.d.ts +4 -0
- package/dist/storage.js +19 -0
- package/dist/string_and_binary/path.js +4 -7
- package/dist/worker/bgWorker.js +1 -1
- package/docs/adr/2026_11_shared_storage_path_resolution.md +75 -0
- package/docs/design_docs/storage_path_resolution.md +306 -0
- package/docs/development.md +11 -1
- package/docs/journal-receive-recovery.md +66 -0
- package/docs/platform-storage.md +61 -0
- package/docs/releasing.md +25 -0
- package/docs/unicode-path-identity.md +49 -17
- package/package.json +9 -1
|
@@ -12,7 +12,7 @@ import { MinioStorageAdapter } from "./objectstore/MinioStorageAdapter.js";
|
|
|
12
12
|
import { LiveSyncAbstractReplicator } from "../LiveSyncAbstractReplicator.js";
|
|
13
13
|
import { ensureRemoteIsCompatible } from "../../pouchdb/LiveSyncDBFunctions.js";
|
|
14
14
|
import { getEffectiveTweakValues } from "../../pouchdb/LiveSyncDBFunctions.js";
|
|
15
|
-
import { assertSupportedPathNormalizationVersion
|
|
15
|
+
import { assertSupportedPathNormalizationVersion } from "../../common/pathNormalization.js";
|
|
16
16
|
import { JournalStorageReadStatuses } from "./objectstore/JournalStorageAdapter.js";
|
|
17
17
|
import { computeKeyedId, configuredIdKey } from "../../common/idDerivation.js";
|
|
18
18
|
import { decryptString, encryptString } from "../../encryption/stringEncryption.js";
|
|
@@ -31,6 +31,12 @@ import {
|
|
|
31
31
|
forgetJoinedDevice as forgetJoinedDeviceFromStore,
|
|
32
32
|
projectJoinedDevices
|
|
33
33
|
} from "../joinedDevices.js";
|
|
34
|
+
import {
|
|
35
|
+
journalVersionRange,
|
|
36
|
+
hasJournalNodeDeclarations,
|
|
37
|
+
matchesJournalPathIdentity,
|
|
38
|
+
inspectJournalPathIdentity
|
|
39
|
+
} from "./journalCompatibility.js";
|
|
34
40
|
const MILSTONE_DOCID = "_00000000-milestone.json";
|
|
35
41
|
async function readRemoteMilestone(client) {
|
|
36
42
|
const result = await client.downloadJsonWithResult(MILSTONE_DOCID);
|
|
@@ -38,23 +44,6 @@ async function readRemoteMilestone(client) {
|
|
|
38
44
|
if (result.status === JournalStorageReadStatuses.NOT_FOUND) return void 0;
|
|
39
45
|
throw result.error;
|
|
40
46
|
}
|
|
41
|
-
const currentVersionRange = {
|
|
42
|
-
min: 0,
|
|
43
|
-
max: 2,
|
|
44
|
-
current: 2
|
|
45
|
-
};
|
|
46
|
-
function journalVersionRange(setting) {
|
|
47
|
-
if (getPathNormalizationVersion(setting) === 1) return { min: 4, max: 4, current: 4 };
|
|
48
|
-
return configuredIdKey(setting) && setting.encrypt && setting.usePathObfuscation ? { min: 3, max: 3, current: 3 } : currentVersionRange;
|
|
49
|
-
}
|
|
50
|
-
function pathIdentityFromJournalRange(range) {
|
|
51
|
-
if (range?.min === 4 && range?.max === 4) return 1;
|
|
52
|
-
if (!range) return 0;
|
|
53
|
-
if (Number.isSafeInteger(range.min) && Number.isSafeInteger(range.max) && range.min >= 0 && range.max <= 3 && range.min <= range.max) {
|
|
54
|
-
return 0;
|
|
55
|
-
}
|
|
56
|
-
throw new Error("The Journal declares an unsupported path identity range.");
|
|
57
|
-
}
|
|
58
47
|
class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
59
48
|
get client() {
|
|
60
49
|
return this.setupJournalSyncClient();
|
|
@@ -89,17 +78,8 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
89
78
|
return this._client;
|
|
90
79
|
}
|
|
91
80
|
async isBucketPathIdentityCompatible(setting, milestone, client) {
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const nodes = /* @__PURE__ */ new Set([...milestone?.accepted_nodes ?? [], ...Object.keys(ranges)]);
|
|
95
|
-
for (const node of nodes) {
|
|
96
|
-
const range = ranges[node];
|
|
97
|
-
const usesNFC = range?.min === 4 && range?.max === 4;
|
|
98
|
-
if (usesNFC !== (pathVersion === 1)) return false;
|
|
99
|
-
}
|
|
100
|
-
const preferred = milestone?.tweak_values?.[DEVICE_ID_PREFERRED];
|
|
101
|
-
if (preferred && getPathNormalizationVersion(preferred) !== pathVersion) return false;
|
|
102
|
-
if (nodes.size === 0 && await client.hasRemoteJournals()) {
|
|
81
|
+
if (!matchesJournalPathIdentity(setting, milestone)) return false;
|
|
82
|
+
if (!hasJournalNodeDeclarations(milestone) && await client.hasRemoteJournals()) {
|
|
103
83
|
throw new Error(
|
|
104
84
|
"The Journal compatibility milestone is missing. Rebuild from the authoritative Vault before synchronising."
|
|
105
85
|
);
|
|
@@ -144,22 +124,13 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
144
124
|
async inspectRemotePathIdentity(setting) {
|
|
145
125
|
const client = this.setupJournalSyncClient(setting);
|
|
146
126
|
const milestone = await readRemoteMilestone(client);
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
const nodes = /* @__PURE__ */ new Set([...milestone?.accepted_nodes ?? [], ...Object.keys(ranges)]);
|
|
150
|
-
for (const node of nodes) {
|
|
151
|
-
versions.add(pathIdentityFromJournalRange(ranges[node]));
|
|
152
|
-
}
|
|
153
|
-
const preferred = milestone?.tweak_values?.[DEVICE_ID_PREFERRED];
|
|
154
|
-
if (preferred) versions.add(getPathNormalizationVersion(preferred));
|
|
155
|
-
if (versions.size > 1) throw new Error("The Journal has conflicting path identity declarations.");
|
|
156
|
-
if (nodes.size === 0 && await client.hasRemoteJournals()) {
|
|
127
|
+
const identity = inspectJournalPathIdentity(milestone);
|
|
128
|
+
if (!hasJournalNodeDeclarations(milestone) && await client.hasRemoteJournals()) {
|
|
157
129
|
throw new Error(
|
|
158
130
|
"The Journal path identity declaration is missing. Use matching settings or Rebuild from the authoritative Vault."
|
|
159
131
|
);
|
|
160
132
|
}
|
|
161
|
-
|
|
162
|
-
return version === void 0 ? { status: "empty" } : { status: "configured", pathNormalizationVersion: version };
|
|
133
|
+
return identity;
|
|
163
134
|
}
|
|
164
135
|
/** Lock existing journals for replacement without declaring the replacement's identity. */
|
|
165
136
|
async markRemoteLockedForRebuild(setting) {
|
|
@@ -178,7 +149,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
178
149
|
throw new Error("Could not upload remote milestone");
|
|
179
150
|
}
|
|
180
151
|
}
|
|
181
|
-
async ensureBucketIsCompatible(deviceNodeID,
|
|
152
|
+
async ensureBucketIsCompatible(deviceNodeID, currentVersionRange, setting = this.currentSettings, client = this.setupJournalSyncClient(setting), recordTweakAssessment) {
|
|
182
153
|
const downloadedMilestone = await readRemoteMilestone(client) ?? false;
|
|
183
154
|
if (!await this.isBucketPathIdentityCompatible(setting, downloadedMilestone || void 0, client)) {
|
|
184
155
|
return "INCOMPATIBLE";
|
|
@@ -193,7 +164,7 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
193
164
|
downloadedMilestone,
|
|
194
165
|
setting,
|
|
195
166
|
deviceNodeID,
|
|
196
|
-
|
|
167
|
+
currentVersionRange,
|
|
197
168
|
{
|
|
198
169
|
app_version: this.env.services.API.getAppVersion(),
|
|
199
170
|
plugin_version: this.env.services.API.getPluginVersion(),
|
|
@@ -356,6 +327,18 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
|
|
|
356
327
|
(client) => client.sendLocalJournal(showingNotice)
|
|
357
328
|
);
|
|
358
329
|
}
|
|
330
|
+
/** Return the stopped Journal from the current transfer client without creating another client. */
|
|
331
|
+
getStoppedJournal() {
|
|
332
|
+
return this._client?.getStoppedJournal();
|
|
333
|
+
}
|
|
334
|
+
/** Recheck connectivity and compatibility, skip the confirmed Journal, and resume receipt. */
|
|
335
|
+
async skipStoppedJournalAndResume(setting, stopped, showingNotice = true) {
|
|
336
|
+
return await this.runDirectionalReplication(
|
|
337
|
+
setting,
|
|
338
|
+
showingNotice,
|
|
339
|
+
(client) => client.skipStoppedJournalAndResume(stopped, showingNotice)
|
|
340
|
+
);
|
|
341
|
+
}
|
|
359
342
|
async replicateAllFromServer(setting, showingNotice) {
|
|
360
343
|
return await this.runDirectionalReplication(
|
|
361
344
|
setting,
|
|
@@ -721,6 +704,10 @@ ${ex.message}`, LOG_LEVEL_NOTICE);
|
|
|
721
704
|
return Promise.resolve(false);
|
|
722
705
|
}
|
|
723
706
|
}
|
|
707
|
+
function isLiveSyncJournalReplicator(value) {
|
|
708
|
+
return value instanceof LiveSyncJournalReplicator;
|
|
709
|
+
}
|
|
724
710
|
export {
|
|
725
|
-
LiveSyncJournalReplicator
|
|
711
|
+
LiveSyncJournalReplicator,
|
|
712
|
+
isLiveSyncJournalReplicator
|
|
726
713
|
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type ChunkVersionRange, type EntryMilestoneInfo, type RemoteDBSettings } from "../../common/types.js";
|
|
2
|
+
import type { PathIdentityInspection } from "../../pouchdb/pathIdentityCompatibility.js";
|
|
3
|
+
export declare function journalVersionRange(setting: RemoteDBSettings): ChunkVersionRange;
|
|
4
|
+
export declare function pathIdentityFromJournalRange(range: ChunkVersionRange | undefined): 0 | 1;
|
|
5
|
+
/** Without a node declaration, the caller must inspect storage before treating the bucket as empty. */
|
|
6
|
+
export declare function hasJournalNodeDeclarations(milestone: EntryMilestoneInfo | undefined): boolean;
|
|
7
|
+
export declare function matchesJournalPathIdentity(setting: RemoteDBSettings, milestone: EntryMilestoneInfo | undefined): boolean;
|
|
8
|
+
/** Project declarations only; an empty result does not establish that remote storage is empty. */
|
|
9
|
+
export declare function inspectJournalPathIdentity(milestone: EntryMilestoneInfo | undefined): PathIdentityInspection;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEVICE_ID_PREFERRED
|
|
3
|
+
} from "../../common/types.js";
|
|
4
|
+
import { configuredIdKey } from "../../common/idDerivation.js";
|
|
5
|
+
import { getPathNormalizationVersion } from "../../common/pathNormalization.js";
|
|
6
|
+
const currentVersionRange = {
|
|
7
|
+
min: 0,
|
|
8
|
+
max: 2,
|
|
9
|
+
current: 2
|
|
10
|
+
};
|
|
11
|
+
function journalVersionRange(setting) {
|
|
12
|
+
if (getPathNormalizationVersion(setting) === 1) return { min: 4, max: 4, current: 4 };
|
|
13
|
+
return configuredIdKey(setting) && setting.encrypt && setting.usePathObfuscation ? { min: 3, max: 3, current: 3 } : currentVersionRange;
|
|
14
|
+
}
|
|
15
|
+
function pathIdentityFromJournalRange(range) {
|
|
16
|
+
if (range?.min === 4 && range?.max === 4) return 1;
|
|
17
|
+
if (!range) return 0;
|
|
18
|
+
if (Number.isSafeInteger(range.min) && Number.isSafeInteger(range.max) && range.min >= 0 && range.max <= 3 && range.min <= range.max) {
|
|
19
|
+
return 0;
|
|
20
|
+
}
|
|
21
|
+
throw new Error("The Journal declares an unsupported path identity range.");
|
|
22
|
+
}
|
|
23
|
+
function journalPathIdentityNodes(milestone) {
|
|
24
|
+
return /* @__PURE__ */ new Set([...milestone?.accepted_nodes ?? [], ...Object.keys(milestone?.node_chunk_info ?? {})]);
|
|
25
|
+
}
|
|
26
|
+
function hasJournalNodeDeclarations(milestone) {
|
|
27
|
+
return journalPathIdentityNodes(milestone).size > 0;
|
|
28
|
+
}
|
|
29
|
+
function matchesJournalPathIdentity(setting, milestone) {
|
|
30
|
+
const pathVersion = getPathNormalizationVersion(setting);
|
|
31
|
+
const ranges = milestone?.node_chunk_info ?? {};
|
|
32
|
+
for (const node of journalPathIdentityNodes(milestone)) {
|
|
33
|
+
const range = ranges[node];
|
|
34
|
+
const usesNFC = range?.min === 4 && range?.max === 4;
|
|
35
|
+
if (usesNFC !== (pathVersion === 1)) return false;
|
|
36
|
+
}
|
|
37
|
+
const preferred = milestone?.tweak_values?.[DEVICE_ID_PREFERRED];
|
|
38
|
+
return !preferred || getPathNormalizationVersion(preferred) === pathVersion;
|
|
39
|
+
}
|
|
40
|
+
function inspectJournalPathIdentity(milestone) {
|
|
41
|
+
const versions = /* @__PURE__ */ new Set();
|
|
42
|
+
const ranges = milestone?.node_chunk_info ?? {};
|
|
43
|
+
for (const node of journalPathIdentityNodes(milestone)) {
|
|
44
|
+
versions.add(pathIdentityFromJournalRange(ranges[node]));
|
|
45
|
+
}
|
|
46
|
+
const preferred = milestone?.tweak_values?.[DEVICE_ID_PREFERRED];
|
|
47
|
+
if (preferred) versions.add(getPathNormalizationVersion(preferred));
|
|
48
|
+
if (versions.size > 1) throw new Error("The Journal has conflicting path identity declarations.");
|
|
49
|
+
const version = [...versions][0];
|
|
50
|
+
return version === void 0 ? { status: "empty" } : { status: "configured", pathNormalizationVersion: version };
|
|
51
|
+
}
|
|
52
|
+
export {
|
|
53
|
+
hasJournalNodeDeclarations,
|
|
54
|
+
inspectJournalPathIdentity,
|
|
55
|
+
journalVersionRange,
|
|
56
|
+
matchesJournalPathIdentity,
|
|
57
|
+
pathIdentityFromJournalRange
|
|
58
|
+
};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EntryDoc } from "../../common/types.js";
|
|
2
|
+
import type { ProcessingEntry } from "./journalRecords.js";
|
|
3
|
+
type ChunkReadRow = {
|
|
4
|
+
id?: string;
|
|
5
|
+
error?: string;
|
|
6
|
+
};
|
|
7
|
+
type WriteResult = PouchDB.Core.Error | PouchDB.Core.Response;
|
|
8
|
+
export declare function partitionJournalDocuments(entries: readonly ProcessingEntry[]): {
|
|
9
|
+
chunks: ProcessingEntry[];
|
|
10
|
+
docs: ProcessingEntry[];
|
|
11
|
+
};
|
|
12
|
+
/** Chunks have stable content, so only absent identifiers need a new local revision. */
|
|
13
|
+
export declare function selectChunksToSave(chunks: readonly ProcessingEntry[], rows: readonly ChunkReadRow[]): PouchDB.Core.PutDocument<EntryDoc>[];
|
|
14
|
+
export declare function hasJournalWriteErrors(results: readonly WriteResult[]): boolean;
|
|
15
|
+
/** Report persisted Chunks even when another row failed, but require every requested Chunk for completion. */
|
|
16
|
+
export declare function assessChunkWrites<T extends PouchDB.Core.PutDocument<EntryDoc>>(chunks: readonly T[], results: readonly WriteResult[]): {
|
|
17
|
+
persisted: T[];
|
|
18
|
+
failed: boolean;
|
|
19
|
+
};
|
|
20
|
+
export declare function journalMetadataRevisions(docs: readonly ProcessingEntry[]): PouchDB.Core.RevisionDiffOptions;
|
|
21
|
+
/** Retain complete transferred Metadata, including revision ancestry, for each missing revision. */
|
|
22
|
+
export declare function selectMetadataToSave(docs: readonly ProcessingEntry[], diff: PouchDB.Core.RevisionDiffResponse): ProcessingEntry[];
|
|
23
|
+
export {};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
function partitionJournalDocuments(entries) {
|
|
2
|
+
const chunks = [];
|
|
3
|
+
const docs = [];
|
|
4
|
+
for (const entry of entries) {
|
|
5
|
+
(entry._id.startsWith("h:") ? chunks : docs).push(entry);
|
|
6
|
+
}
|
|
7
|
+
return { chunks, docs };
|
|
8
|
+
}
|
|
9
|
+
function selectChunksToSave(chunks, rows) {
|
|
10
|
+
if (rows.some((row) => "error" in row && row.error !== "not_found")) {
|
|
11
|
+
throw new Error("Could not inspect existing Chunks");
|
|
12
|
+
}
|
|
13
|
+
const existingIDs = new Set(rows.map((row) => row.id).filter((id) => id !== void 0));
|
|
14
|
+
return chunks.filter((entry) => !existingIDs.has(entry._id)).map((entry) => ({ ...entry, _rev: void 0 }));
|
|
15
|
+
}
|
|
16
|
+
function hasJournalWriteErrors(results) {
|
|
17
|
+
return results.some((row) => "error" in row);
|
|
18
|
+
}
|
|
19
|
+
function assessChunkWrites(chunks, results) {
|
|
20
|
+
const persistedIDs = new Set(results.filter((row) => !("error" in row)).map((row) => row.id));
|
|
21
|
+
return {
|
|
22
|
+
persisted: chunks.filter((entry) => persistedIDs.has(entry._id)),
|
|
23
|
+
failed: hasJournalWriteErrors(results) || chunks.some((entry) => !persistedIDs.has(entry._id))
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
function journalMetadataRevisions(docs) {
|
|
27
|
+
return docs.reduce(
|
|
28
|
+
(revisions, entry) => ({ ...revisions, [entry._id]: [...revisions[entry._id] ?? [], entry._rev] }),
|
|
29
|
+
{}
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
function selectMetadataToSave(docs, diff) {
|
|
33
|
+
return docs.filter(
|
|
34
|
+
(entry) => entry._id in diff && "missing" in diff[entry._id] && (diff[entry._id].missing?.indexOf(entry._rev) ?? 0) !== -1
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
export {
|
|
38
|
+
assessChunkWrites,
|
|
39
|
+
hasJournalWriteErrors,
|
|
40
|
+
journalMetadataRevisions,
|
|
41
|
+
partitionJournalDocuments,
|
|
42
|
+
selectChunksToSave,
|
|
43
|
+
selectMetadataToSave
|
|
44
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type EntryDoc } from "../../common/types.js";
|
|
2
|
+
export type ProcessingEntry = PouchDB.Core.PutDocument<EntryDoc> & PouchDB.Core.GetMeta;
|
|
3
|
+
export declare function serializeDoc(doc: EntryDoc): Uint8Array;
|
|
4
|
+
export declare function decodeJournalRecords(data: Uint8Array): {
|
|
5
|
+
ok: true;
|
|
6
|
+
docs: ProcessingEntry[];
|
|
7
|
+
} | {
|
|
8
|
+
ok: false;
|
|
9
|
+
stage: "decode" | "parse";
|
|
10
|
+
};
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import {} from "../../common/types.js";
|
|
2
|
+
import { escapeNewLineFromString, unescapeNewLineFromString } from "../../common/utils.js";
|
|
3
|
+
const RECORD_SPLIT = `
|
|
4
|
+
`;
|
|
5
|
+
const UNIT_SPLIT = ``;
|
|
6
|
+
const te = new TextEncoder();
|
|
7
|
+
function parseJournalRecord(value) {
|
|
8
|
+
const parsed = JSON.parse(value);
|
|
9
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
10
|
+
throw new TypeError("Invalid Journal record");
|
|
11
|
+
}
|
|
12
|
+
const record = parsed;
|
|
13
|
+
if (typeof record._id !== "string" || record._id.length === 0) {
|
|
14
|
+
throw new TypeError("Invalid Journal document identifier");
|
|
15
|
+
}
|
|
16
|
+
if (record._id.startsWith("h:")) {
|
|
17
|
+
if (record._id.length <= 2 || typeof record._rev !== "string" || typeof record.data !== "string") {
|
|
18
|
+
throw new TypeError("Invalid Journal Chunk record");
|
|
19
|
+
}
|
|
20
|
+
} else if (typeof record._rev !== "string" || record._rev.length === 0) {
|
|
21
|
+
throw new TypeError("Invalid Journal Metadata revision");
|
|
22
|
+
}
|
|
23
|
+
return parsed;
|
|
24
|
+
}
|
|
25
|
+
function decodeCompactJournalRecord(value) {
|
|
26
|
+
const separatorIndex = value.indexOf(UNIT_SPLIT);
|
|
27
|
+
if (separatorIndex === -1) {
|
|
28
|
+
return { ok: false, stage: "parse" };
|
|
29
|
+
}
|
|
30
|
+
const idPart = value.substring(1, separatorIndex);
|
|
31
|
+
if (!idPart.startsWith("h:") || idPart.length <= 2) {
|
|
32
|
+
return { ok: false, stage: "parse" };
|
|
33
|
+
}
|
|
34
|
+
const dataPart = value.substring(separatorIndex + 1);
|
|
35
|
+
return {
|
|
36
|
+
ok: true,
|
|
37
|
+
doc: {
|
|
38
|
+
_id: idPart,
|
|
39
|
+
data: unescapeNewLineFromString(dataPart),
|
|
40
|
+
type: "leaf",
|
|
41
|
+
_rev: ""
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function decodeJournalRecordFrame(frame, decoder) {
|
|
46
|
+
let value;
|
|
47
|
+
try {
|
|
48
|
+
value = decoder.decode(frame);
|
|
49
|
+
} catch {
|
|
50
|
+
return { ok: false, stage: "decode" };
|
|
51
|
+
}
|
|
52
|
+
if (value.startsWith("~")) {
|
|
53
|
+
return decodeCompactJournalRecord(value);
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
return { ok: true, doc: parseJournalRecord(value) };
|
|
57
|
+
} catch {
|
|
58
|
+
return { ok: false, stage: "parse" };
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
function serializeDoc(doc) {
|
|
62
|
+
if (doc._id.startsWith("h:")) {
|
|
63
|
+
const data = doc.data;
|
|
64
|
+
const writeData = escapeNewLineFromString(data);
|
|
65
|
+
return te.encode(`~${doc._id}${UNIT_SPLIT}${writeData}${RECORD_SPLIT}`);
|
|
66
|
+
}
|
|
67
|
+
return te.encode(JSON.stringify(doc) + RECORD_SPLIT);
|
|
68
|
+
}
|
|
69
|
+
function decodeJournalRecords(data) {
|
|
70
|
+
if (data.length === 0) {
|
|
71
|
+
return { ok: true, docs: [] };
|
|
72
|
+
}
|
|
73
|
+
if (data[data.length - 1] !== 10) {
|
|
74
|
+
return { ok: false, stage: "parse" };
|
|
75
|
+
}
|
|
76
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
77
|
+
let idxFrom = 0;
|
|
78
|
+
const result = [];
|
|
79
|
+
while (idxFrom < data.length) {
|
|
80
|
+
const idxTo = data.indexOf(10, idxFrom);
|
|
81
|
+
if (idxTo === -1 || idxTo === idxFrom) {
|
|
82
|
+
return { ok: false, stage: "parse" };
|
|
83
|
+
}
|
|
84
|
+
const record = decodeJournalRecordFrame(data.slice(idxFrom, idxTo), decoder);
|
|
85
|
+
if (record.ok === false) {
|
|
86
|
+
return record;
|
|
87
|
+
}
|
|
88
|
+
result.push(record.doc);
|
|
89
|
+
idxFrom = idxTo + 1;
|
|
90
|
+
}
|
|
91
|
+
return { ok: true, docs: result };
|
|
92
|
+
}
|
|
93
|
+
export {
|
|
94
|
+
decodeJournalRecords,
|
|
95
|
+
serializeDoc
|
|
96
|
+
};
|
|
@@ -242,6 +242,7 @@ export declare class TrysteroReplicator {
|
|
|
242
242
|
jwtSub?: string;
|
|
243
243
|
jwtExpDuration?: number;
|
|
244
244
|
useRequestAPI?: boolean;
|
|
245
|
+
useCouchDBCacheControlHeadersOnly?: boolean;
|
|
245
246
|
accessKey?: string;
|
|
246
247
|
secretKey?: string;
|
|
247
248
|
bucket?: string;
|
|
@@ -432,6 +433,7 @@ export declare class TrysteroReplicator {
|
|
|
432
433
|
jwtSub?: string;
|
|
433
434
|
jwtExpDuration?: number;
|
|
434
435
|
useRequestAPI?: boolean;
|
|
436
|
+
useCouchDBCacheControlHeadersOnly?: boolean;
|
|
435
437
|
accessKey?: string;
|
|
436
438
|
secretKey?: string;
|
|
437
439
|
bucket?: string;
|
|
@@ -222,7 +222,8 @@ Please enable them from the settings screen after setup is complete.`,
|
|
|
222
222
|
settings.useRequestAPI,
|
|
223
223
|
async () => {
|
|
224
224
|
throw new Error("A version document does not require decryption.");
|
|
225
|
-
}
|
|
225
|
+
},
|
|
226
|
+
{ useCouchDBCacheControlHeadersOnly: settings.useCouchDBCacheControlHeadersOnly === true }
|
|
226
227
|
);
|
|
227
228
|
if (typeof connection === "string") throw new Error(connection);
|
|
228
229
|
try {
|
|
@@ -12,13 +12,13 @@ export declare class FileProcessingService<T extends ServiceContext = ServiceCon
|
|
|
12
12
|
* Dispatch a legacy optional notification, without confirming persistence.
|
|
13
13
|
* i.e., hidden files synchronisation or customisation sync.
|
|
14
14
|
*/
|
|
15
|
-
readonly processOptionalFileEvent: import("../lib/HandlerUtils.js").BooleanMultipleHandlerFunction<(path: import("../../
|
|
15
|
+
readonly processOptionalFileEvent: import("../lib/HandlerUtils.js").BooleanMultipleHandlerFunction<(path: import("../../storage.js").FilePath) => Promise<boolean>>;
|
|
16
16
|
/**
|
|
17
17
|
* Complete optional file operations through the first applicable handler.
|
|
18
18
|
* Hosts migrate all optional providers together. Once any typed provider is
|
|
19
19
|
* registered, dispatch uses this contract exclusively.
|
|
20
20
|
*/
|
|
21
|
-
readonly processOptionalFileEventWithResult?: import("../lib/HandlerUtils.js").FirstHandledFunction<(path: import("../../
|
|
21
|
+
readonly processOptionalFileEventWithResult?: import("../lib/HandlerUtils.js").FirstHandledFunction<(path: import("../../storage.js").FilePath, options?: import("./IService.js").OptionalFileEventOptions) => Promise<import("../lib/HandlerUtils.js").HandlerOutcome<void, string>>>;
|
|
22
22
|
/**
|
|
23
23
|
* Commit any pending file events that have been queued for processing.
|
|
24
24
|
*/
|
|
@@ -8,6 +8,8 @@ export interface RemoteConnectionOpenOptions {
|
|
|
8
8
|
readonly encryptionAlgorithm?: E2EEAlgorithm;
|
|
9
9
|
/** Apply HKDF Metadata encryption to obfuscated internal entries on writes. */
|
|
10
10
|
readonly encryptInternalMetadata?: boolean;
|
|
11
|
+
/** Use HTTP headers alone for cache control; an absent value is false. */
|
|
12
|
+
readonly useCouchDBCacheControlHeadersOnly?: boolean;
|
|
11
13
|
/**
|
|
12
14
|
* Cancels abort-capable requests made through the connection when the
|
|
13
15
|
* owning operation is cancelled.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type CouchDBCredentials, type EntryDoc } from "../../common/types.js";
|
|
2
2
|
import type { IRemoteService } from "./IService.js";
|
|
3
3
|
import { ServiceBase, type ServiceContext } from "./ServiceBase.js";
|
|
4
4
|
import { type LOG_LEVEL } from "../../common/types.js";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { E2EEAlgorithms } from "../../common/types.js";
|
|
1
2
|
import { ServiceBase } from "./ServiceBase.js";
|
|
2
3
|
import {
|
|
3
4
|
LOG_LEVEL_DEBUG,
|
|
@@ -12,6 +13,7 @@ import { AuthorizationHeaderGenerator } from "../../replication/httplib.js";
|
|
|
12
13
|
import { UnresolvedErrorManager } from "./UnresolvedErrorManager.js";
|
|
13
14
|
import { createInstanceLogFunction, MARK_LOG_NETWORK_ERROR } from "../lib/logUtils.js";
|
|
14
15
|
import { runWithTrackedPhysicalRequest } from "../lib/remoteActivity.js";
|
|
16
|
+
import { createCouchDBHttpCache } from "../lib/couchdbHttpCache.js";
|
|
15
17
|
import { LiveSyncError } from "../../common/LSError.js";
|
|
16
18
|
function createRemoteConnectionScope(ownerSignal) {
|
|
17
19
|
const controller = new AbortController();
|
|
@@ -188,6 +190,12 @@ class RemoteService extends ServiceBase {
|
|
|
188
190
|
return "Network is offline";
|
|
189
191
|
}
|
|
190
192
|
const connectionScope = createRemoteConnectionScope(connectionOptions.signal);
|
|
193
|
+
const encryptionAlgorithm = connectionOptions.encryptionAlgorithm ?? this._settingService.currentSettings().E2EEAlgorithm;
|
|
194
|
+
const prepareCachedRequest = createCouchDBHttpCache(
|
|
195
|
+
uri,
|
|
196
|
+
typeof passphrase === "string" && passphrase !== "false" && encryptionAlgorithm !== E2EEAlgorithms.ForceV1 ? getPBKDF2Salt : void 0,
|
|
197
|
+
connectionOptions.useCouchDBCacheControlHeadersOnly === true
|
|
198
|
+
);
|
|
191
199
|
const conf = {
|
|
192
200
|
adapter: "http",
|
|
193
201
|
auth: "username" in auth ? auth : void 0,
|
|
@@ -222,11 +230,12 @@ class RemoteService extends ServiceBase {
|
|
|
222
230
|
if (!("username" in auth)) {
|
|
223
231
|
headers.append("authorization", authHeader);
|
|
224
232
|
}
|
|
233
|
+
const cachedRequest = await prepareCachedRequest(requestSrc, opts);
|
|
225
234
|
try {
|
|
226
235
|
const signal = connectionScope.combine(opts?.signal);
|
|
227
236
|
const response = await this.performFetch(
|
|
228
|
-
|
|
229
|
-
{ ...opts, headers, signal, cache:
|
|
237
|
+
cachedRequest.request,
|
|
238
|
+
{ ...opts, headers, signal, cache: cachedRequest.cache },
|
|
230
239
|
useRequestAPI ? FetchMethod.native : FetchMethod.webCompat
|
|
231
240
|
);
|
|
232
241
|
return response;
|
|
@@ -243,8 +252,8 @@ class RemoteService extends ServiceBase {
|
|
|
243
252
|
"Failed to fetch by native fetch API. Trying to fetch by API to get more information."
|
|
244
253
|
);
|
|
245
254
|
const resp2 = await this.performFetch(
|
|
246
|
-
|
|
247
|
-
{ ...opts, headers, cache:
|
|
255
|
+
cachedRequest.request,
|
|
256
|
+
{ ...opts, headers, cache: cachedRequest.cache },
|
|
248
257
|
FetchMethod.native
|
|
249
258
|
);
|
|
250
259
|
if (resp2.status / 100 == 2) {
|
|
@@ -279,7 +288,6 @@ class RemoteService extends ServiceBase {
|
|
|
279
288
|
}
|
|
280
289
|
}
|
|
281
290
|
};
|
|
282
|
-
const encryptionAlgorithm = connectionOptions.encryptionAlgorithm ?? this._settingService.currentSettings().E2EEAlgorithm;
|
|
283
291
|
let db;
|
|
284
292
|
try {
|
|
285
293
|
db = new this._pouchDB(uri, conf);
|
|
@@ -4,6 +4,6 @@ export declare abstract class InjectableVaultService<T extends ServiceContext> e
|
|
|
4
4
|
}
|
|
5
5
|
export declare class InjectableVaultServiceCompat<T extends ServiceContext> extends InjectableVaultService<T> {
|
|
6
6
|
isStorageInsensitive: import("../../lib/HandlerUtils.js").HandlerFunction<() => boolean, unknown>;
|
|
7
|
-
getActiveFilePath: import("../../lib/HandlerUtils.js").HandlerFunction<() => import("../../../
|
|
7
|
+
getActiveFilePath: import("../../lib/HandlerUtils.js").HandlerFunction<() => import("../../../storage.js").FilePath | undefined, unknown>;
|
|
8
8
|
isValidPath(path: string): boolean;
|
|
9
9
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Keep encrypted HTTP responses within the Security Seed which produced them. */
|
|
2
|
+
export declare function createCouchDBHttpCache(uri: string, getPBKDF2Salt?: () => Promise<Uint8Array<ArrayBuffer>>, headersOnly?: boolean): (request: string | Request, options?: RequestInit) => Promise<{
|
|
3
|
+
request: string | Request;
|
|
4
|
+
cache: RequestCache;
|
|
5
|
+
}>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { uint8ArrayToHexString } from "../../string_and_binary/convert.js";
|
|
2
|
+
const processEpoch = Math.floor(Date.now() / 1e3);
|
|
3
|
+
function createCouchDBHttpCache(uri, getPBKDF2Salt, headersOnly = false) {
|
|
4
|
+
const databasePath = new URL(uri).pathname.replace(/\/$/, "");
|
|
5
|
+
let previousSalt;
|
|
6
|
+
let namespace;
|
|
7
|
+
const getNamespace = async () => {
|
|
8
|
+
const salt = await getPBKDF2Salt();
|
|
9
|
+
if (!previousSalt || previousSalt.length !== salt.length || previousSalt.some((byte, i) => byte !== salt[i])) {
|
|
10
|
+
previousSalt = salt.slice();
|
|
11
|
+
const epoch = new TextEncoder().encode(`${processEpoch}:`);
|
|
12
|
+
const input = new Uint8Array(epoch.length + previousSalt.length);
|
|
13
|
+
input.set(epoch);
|
|
14
|
+
input.set(previousSalt, epoch.length);
|
|
15
|
+
namespace = crypto.subtle.digest("SHA-256", input).then((digest) => uint8ArrayToHexString(new Uint8Array(digest)));
|
|
16
|
+
}
|
|
17
|
+
return namespace;
|
|
18
|
+
};
|
|
19
|
+
return async (request, options) => {
|
|
20
|
+
const method = (options?.method ?? (typeof request === "string" ? "GET" : request.method)).toUpperCase();
|
|
21
|
+
if (method !== "GET" && method !== "HEAD") {
|
|
22
|
+
return { request, cache: options?.cache };
|
|
23
|
+
}
|
|
24
|
+
if (headersOnly) return { request, cache: "no-store" };
|
|
25
|
+
const url = new URL(typeof request === "string" ? request : request.url);
|
|
26
|
+
const relativePath = url.pathname.startsWith(`${databasePath}/`) ? decodeURIComponent(url.pathname.slice(databasePath.length + 1)) : "";
|
|
27
|
+
if (getPBKDF2Salt && relativePath && !relativePath.startsWith("_local/")) {
|
|
28
|
+
url.searchParams.set("_livesync_epoch", await getNamespace());
|
|
29
|
+
request = typeof request === "string" ? url.href : new Request(url.href, request);
|
|
30
|
+
}
|
|
31
|
+
return { request, cache: "no-cache" };
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
export {
|
|
35
|
+
createCouchDBHttpCache
|
|
36
|
+
};
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Host-neutral path identity, namespace, and storage resolution contracts. */
|
|
2
|
+
export type { FilePath, DocumentPath, StoragePath, PathComparisonKey, PathComparisonSettings, DocumentPathParts, DirectoryEntry, ReadDirectory, ResolvedRename, StorageEntry, StoragePathResolver, StoragePathResolverOptions, } from "./platform/pathTypes.js";
|
|
3
|
+
export { parseFilePath, parseDocumentPath, splitDocumentPath, joinDocumentPath, getStoragePathComparisonKey, getDocumentPathComparisonKey, } from "./platform/pathComparison.js";
|
|
4
|
+
export { createStoragePathResolver, AmbiguousNormalisedPathError } from "./platform/storagePathResolution.js";
|
package/dist/storage.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import {
|
|
2
|
+
parseFilePath,
|
|
3
|
+
parseDocumentPath,
|
|
4
|
+
splitDocumentPath,
|
|
5
|
+
joinDocumentPath,
|
|
6
|
+
getStoragePathComparisonKey,
|
|
7
|
+
getDocumentPathComparisonKey
|
|
8
|
+
} from "./platform/pathComparison.js";
|
|
9
|
+
import { createStoragePathResolver, AmbiguousNormalisedPathError } from "./platform/storagePathResolution.js";
|
|
10
|
+
export {
|
|
11
|
+
AmbiguousNormalisedPathError,
|
|
12
|
+
createStoragePathResolver,
|
|
13
|
+
getDocumentPathComparisonKey,
|
|
14
|
+
getStoragePathComparisonKey,
|
|
15
|
+
joinDocumentPath,
|
|
16
|
+
parseDocumentPath,
|
|
17
|
+
parseFilePath,
|
|
18
|
+
splitDocumentPath
|
|
19
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { splitPathNamespace } from "../common/pathNamespace.js";
|
|
1
2
|
import { Minimatch } from "minimatch";
|
|
2
3
|
import { getWebCrypto } from "../mods.js";
|
|
3
4
|
import {
|
|
@@ -13,7 +14,7 @@ import {
|
|
|
13
14
|
import { memorizeFuncWithLRUCache } from "../common/utils.js";
|
|
14
15
|
import { uint8ArrayToHexString, writeString } from "./convert.js";
|
|
15
16
|
import { unique } from "octagonal-wheels/collection.js";
|
|
16
|
-
import { CHeader
|
|
17
|
+
import { CHeader } from "../common/models/fileaccess.const.js";
|
|
17
18
|
import { computeKeyedId } from "../common/idDerivation.js";
|
|
18
19
|
function isValidFilenameInWidows(filename) {
|
|
19
20
|
const regex = /[\u0000-\u001f]|[\\":?<>|*#]/g;
|
|
@@ -45,12 +46,8 @@ function addPrefix(path, prefix) {
|
|
|
45
46
|
return `${prefix ?? ""}${path}`;
|
|
46
47
|
}
|
|
47
48
|
function expandFilePathPrefix(path) {
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
return [prefix, path.substring(prefix.length)];
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
return ["", path];
|
|
49
|
+
const [prefix, body] = splitPathNamespace(path);
|
|
50
|
+
return [prefix, body];
|
|
54
51
|
}
|
|
55
52
|
function expandDocumentIDPrefix(id) {
|
|
56
53
|
for (const prefix of [CHeader, PREFIX_OBFUSCATED]) {
|