@vrtmrz/livesync-commonlib 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -1
- package/dist/API/DirectFileManipulatorV2.d.ts +7 -1
- package/dist/API/DirectFileManipulatorV2.js +46 -67
- package/dist/API/DirectFileWatchOwner.d.ts +24 -0
- package/dist/API/DirectFileWatchOwner.js +256 -0
- package/dist/API/processSetting.js +3 -3
- package/dist/ContentSplitter/ContentSplitterRabinKarp.js +2 -17
- package/dist/ContentSplitter/ContentSplitterV1.js +9 -19
- package/dist/ContentSplitter/ContentSplitterV2.js +9 -19
- package/dist/common/models/setting.type.d.ts +1 -1
- package/dist/common/models/shared.definition.configNames.js +1 -2
- package/dist/common/models/tweak.definition.d.ts +1 -1
- package/dist/common/pathNormalization.d.ts +2 -0
- package/dist/common/pathNormalization.js +7 -2
- package/dist/common/utils.d.ts +33 -3
- package/dist/common/utils.js +12 -44
- package/dist/common/utils.object.js +2 -1
- package/dist/concurrency/keyedPendingWork.d.ts +13 -0
- package/dist/concurrency/keyedPendingWork.js +27 -0
- package/dist/conflicts/conflictCheckQueue.d.ts +45 -0
- package/dist/conflicts/conflictCheckQueue.js +108 -0
- package/dist/conflicts.d.ts +3 -0
- package/dist/conflicts.js +4 -0
- package/dist/diagnostics/integrity.js +11 -12
- package/dist/interfaces/ServiceModule.d.ts +42 -24
- package/dist/interfaces/ServiceModule.js +4 -3
- package/dist/managers/ChunkFetcher.d.ts +3 -0
- package/dist/managers/ChunkFetcher.js +20 -13
- package/dist/managers/LiveSyncManagers.d.ts +2 -0
- package/dist/managers/LiveSyncManagers.js +1 -0
- package/dist/managers/StorageEventManager.d.ts +3 -3
- package/dist/managers/StorageEventManager.js +15 -6
- package/dist/managers/chunkFetchSource.d.ts +14 -0
- package/dist/managers/chunkFetchSource.js +12 -0
- package/dist/pouchdb/LiveSyncDBFunctions.js +5 -0
- package/dist/pouchdb/LiveSyncLocalDB.d.ts +7 -1
- package/dist/pouchdb/LiveSyncLocalDB.js +14 -10
- package/dist/pouchdb/chunks.d.ts +16 -0
- package/dist/pouchdb/pathIdentityCompatibility.d.ts +8 -0
- package/dist/pouchdb/pathIdentityCompatibility.js +19 -1
- package/dist/pouchdb/pouchdb-browser.js +2 -84
- package/dist/pouchdb/pouchdb-http.js +2 -84
- package/dist/pouchdb/purgeMulti.d.ts +4 -0
- package/dist/pouchdb/purgeMulti.js +89 -0
- package/dist/replication/RemoteResource.d.ts +3 -0
- package/dist/replication/index.d.ts +2 -0
- package/dist/replication/index.js +2 -0
- package/dist/replication/journal/JournalSyncCore.d.ts +1 -0
- package/dist/replication/journal/JournalSyncCore.js +41 -20
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +11 -3
- package/dist/replication/journal/LiveSyncJournalReplicator.js +123 -45
- package/dist/replication/trystero/P2PRoomSessionOwner.js +14 -13
- package/dist/replication/trystero/TrysteroReplicator.d.ts +7 -5
- package/dist/replication/trystero/TrysteroReplicator.js +101 -74
- package/dist/replication/trystero/TrysteroReplicator.pathIdentity.d.ts +11 -0
- package/dist/replication/trystero/TrysteroReplicator.pathIdentity.js +72 -0
- package/dist/replication/trystero/TrysteroReplicatorP2PServer.js +3 -2
- package/dist/replication/trystero/p2pReplicatorConfigurationIdentity.js +2 -0
- package/dist/replication/trystero/peerPatterns.d.ts +1 -0
- package/dist/replication/trystero/peerPatterns.js +6 -0
- package/dist/replication/trystero/useP2PReplicatorFeature.js +6 -2
- package/dist/rpc/RpcRoom.d.ts +5 -1
- package/dist/rpc/RpcRoom.js +80 -105
- package/dist/serviceFeatures/offlineScanner.d.ts +4 -4
- package/dist/serviceFeatures/offlineScanner.js +14 -15
- package/dist/serviceFeatures/prepareDatabaseForUse.d.ts +2 -2
- package/dist/serviceFeatures/remoteConfig.d.ts +10 -5
- package/dist/serviceModules/Rebuilder.d.ts +1 -0
- package/dist/serviceModules/Rebuilder.js +33 -5
- package/dist/serviceModules/ServiceFileHandlerBase.js +2 -7
- package/dist/services/implements/injectable/InjectableConflictService.d.ts +5 -5
- package/dist/services/lib/HandlerUtils.d.ts +8 -6
- package/dist/services/lib/HandlerUtils.js +14 -0
- package/dist/worker/bgWorker.js +1 -1
- package/docs/conflict-resolution.md +30 -0
- package/docs/development.md +50 -0
- package/docs/proven-in-use.md +80 -0
- package/docs/releasing.md +4 -0
- package/docs/service-feature-composition.md +63 -10
- package/docs/unicode-path-identity.md +64 -3
- package/package.json +11 -3
|
@@ -8,25 +8,15 @@ class ContentSplitterV2 extends ContentSplitterBase {
|
|
|
8
8
|
return settings.chunkSplitterVersion === ChunkAlgorithms.V2 || settings.chunkSplitterVersion === ChunkAlgorithms.V2Segmenter;
|
|
9
9
|
}
|
|
10
10
|
async processSplit(options) {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
} else {
|
|
21
|
-
return (await splitPieces2V2(
|
|
22
|
-
options.blob,
|
|
23
|
-
options.pieceSize,
|
|
24
|
-
options.plainSplit,
|
|
25
|
-
options.minimumChunkSize,
|
|
26
|
-
options.path,
|
|
27
|
-
options.useSegmenter
|
|
28
|
-
))();
|
|
29
|
-
}
|
|
11
|
+
const split = options.useWorker ? splitPieces2WorkerV2 : splitPieces2V2;
|
|
12
|
+
return (await split(
|
|
13
|
+
options.blob,
|
|
14
|
+
options.pieceSize,
|
|
15
|
+
options.plainSplit,
|
|
16
|
+
options.minimumChunkSize,
|
|
17
|
+
options.path,
|
|
18
|
+
options.useSegmenter
|
|
19
|
+
))();
|
|
30
20
|
}
|
|
31
21
|
}
|
|
32
22
|
export {
|
|
@@ -557,7 +557,7 @@ export interface P2PConnectionInfo {
|
|
|
557
557
|
*/
|
|
558
558
|
P2P_useDiagRTC?: boolean;
|
|
559
559
|
}
|
|
560
|
-
export interface P2PSyncSetting extends P2PConnectionInfo {
|
|
560
|
+
export interface P2PSyncSetting extends P2PConnectionInfo, Partial<PathIdentitySettings> {
|
|
561
561
|
P2P_AutoAccepting: AutoAccepting;
|
|
562
562
|
P2P_AutoSyncPeers: string;
|
|
563
563
|
P2P_AutoWatchPeers: string;
|
|
@@ -66,8 +66,7 @@ const configurationNames = {
|
|
|
66
66
|
},
|
|
67
67
|
pathNormalizationVersion: {
|
|
68
68
|
name: "Path identity mode",
|
|
69
|
-
desc: "
|
|
70
|
-
isAdvanced: true
|
|
69
|
+
desc: "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first."
|
|
71
70
|
},
|
|
72
71
|
doNotUseFixedRevisionForChunks: {
|
|
73
72
|
name: "Content-derived chunk revisions (obsolete setting)",
|
|
@@ -120,8 +120,8 @@ export declare const TweakValuesRecommendedTemplate: Partial<ObsidianLiveSyncSet
|
|
|
120
120
|
export declare const TweakValuesDefault: Partial<ObsidianLiveSyncSettings>;
|
|
121
121
|
export declare const TweakValuesTemplate: {
|
|
122
122
|
tweakModified: number;
|
|
123
|
-
usePluginSyncV2: boolean;
|
|
124
123
|
pathNormalizationVersion: import("./setting.type.js").PathNormalizationVersion;
|
|
124
|
+
usePluginSyncV2: boolean;
|
|
125
125
|
encrypt: boolean;
|
|
126
126
|
idDerivationVersion: 0 | 1;
|
|
127
127
|
usePathObfuscation: boolean;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { PathNormalizationVersion } from "./models/setting.type.js";
|
|
2
|
+
/** Reject unsupported path identity versions while allowing an absent legacy value. */
|
|
3
|
+
export declare function assertSupportedPathNormalizationVersion(version: number | undefined): asserts version is PathNormalizationVersion | undefined;
|
|
2
4
|
/** Resolve the stored path identity version, treating an absent legacy value as version zero. */
|
|
3
5
|
export declare function getPathNormalizationVersion(settings: {
|
|
4
6
|
pathNormalizationVersion?: number;
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
function assertSupportedPathNormalizationVersion(version) {
|
|
2
|
+
if (version === void 0 || version === 0 || version === 1) return;
|
|
3
|
+
throw new Error(`Unsupported path normalisation version: ${version}`);
|
|
4
|
+
}
|
|
1
5
|
function getPathNormalizationVersion(settings) {
|
|
2
6
|
const version = settings.pathNormalizationVersion === void 0 ? 0 : settings.pathNormalizationVersion;
|
|
3
|
-
|
|
4
|
-
|
|
7
|
+
assertSupportedPathNormalizationVersion(version);
|
|
8
|
+
return version;
|
|
5
9
|
}
|
|
6
10
|
function normalisePathForIdentity(path, version) {
|
|
7
11
|
const resolvedVersion = getPathNormalizationVersion({ pathNormalizationVersion: version });
|
|
8
12
|
return resolvedVersion === 1 ? path.normalize("NFC") : path;
|
|
9
13
|
}
|
|
10
14
|
export {
|
|
15
|
+
assertSupportedPathNormalizationVersion,
|
|
11
16
|
getPathNormalizationVersion,
|
|
12
17
|
normalisePathForIdentity
|
|
13
18
|
};
|
package/dist/common/utils.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ import { replaceAll, replaceAllPairs } from "octagonal-wheels/string";
|
|
|
3
3
|
export { replaceAll, replaceAllPairs };
|
|
4
4
|
import { concatUInt8Array } from "octagonal-wheels/binary";
|
|
5
5
|
export { concatUInt8Array };
|
|
6
|
+
import { createBinaryBlob, createTextBlob } from "octagonal-wheels/binary/blob";
|
|
7
|
+
export { createBinaryBlob, createTextBlob };
|
|
6
8
|
import { delay, fireAndForget } from "octagonal-wheels/promises";
|
|
7
9
|
export { delay, fireAndForget };
|
|
8
10
|
import { arrayToChunkedArray, unique } from "octagonal-wheels/collection";
|
|
@@ -28,12 +30,40 @@ export declare function getDocData(doc: string | string[]): string;
|
|
|
28
30
|
export declare function getDocDataAsArray(doc: string | string[]): string[];
|
|
29
31
|
export declare function getDocDataAsArrayBuffer(doc: string | string[] | ArrayBuffer): Uint8Array<ArrayBuffer>;
|
|
30
32
|
export declare function isTextBlob(blob: Blob): boolean;
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
/**
|
|
34
|
+
* Normalises content to a Blob through the legacy Commonlib entry point.
|
|
35
|
+
*
|
|
36
|
+
* @param data - Text, ordered text parts, binary bytes, or an existing Blob.
|
|
37
|
+
* @returns The original Blob, or a new text or binary Blob for other inputs.
|
|
38
|
+
* @remarks
|
|
39
|
+
* This compatibility wrapper retains the `createBlob` name and delegates to
|
|
40
|
+
* `toBlob` from `octagonal-wheels/binary/blob`. Existing Blob instances retain
|
|
41
|
+
* their identity and MIME type. Text retains transparent line endings; binary
|
|
42
|
+
* views contribute only their visible bytes. Calls use the current realm's
|
|
43
|
+
* Blob constructor. Document classification and decoding remain in
|
|
44
|
+
* {@link readAsBlob}; importing this legacy module retains its existing
|
|
45
|
+
* environment initialisation.
|
|
46
|
+
*/
|
|
33
47
|
export declare function createBlob(data: string | string[] | Uint8Array<ArrayBuffer> | ArrayBuffer | Blob): Blob;
|
|
34
48
|
export declare function isTextDocument(doc: LoadedEntry): boolean;
|
|
35
49
|
export declare function readAsBlob(doc: LoadedEntry): Blob;
|
|
36
50
|
export declare function readContent(doc: LoadedEntry): string | ArrayBuffer;
|
|
51
|
+
/**
|
|
52
|
+
* Tests whether legacy document-content inputs represent equal bytes.
|
|
53
|
+
*
|
|
54
|
+
* @param docA - Text, ordered text parts, a Blob, or binary bytes.
|
|
55
|
+
* @param docB - The second content value in any supported representation.
|
|
56
|
+
* @returns Whether the normalised inputs contain equal bytes, ignoring MIME types.
|
|
57
|
+
* @remarks
|
|
58
|
+
* Inputs are normalised with {@link createBlob}. Different sizes return false
|
|
59
|
+
* without reading content. IndexedDB comparison availability is selected at
|
|
60
|
+
* module import; that path resolves the current method and receiver before
|
|
61
|
+
* reading both complete Blobs sequentially, including empty inputs.
|
|
62
|
+
* Otherwise, `isBlobContentEqual` from `octagonal-wheels/binary/blob` reads
|
|
63
|
+
* pairs of parts of at most 10,000 bytes, using Commonlib's Base64 encoder and
|
|
64
|
+
* its existing host Buffer fallback. Empty inputs on this path return true
|
|
65
|
+
* without reads. Read and comparison failures propagate unchanged.
|
|
66
|
+
*/
|
|
37
67
|
export declare function isDocContentSame(docA: string | string[] | Blob | ArrayBuffer, docB: string | string[] | Blob | ArrayBuffer): Promise<boolean>;
|
|
38
68
|
export declare function isObfuscatedEntry(doc: DatabaseEntry): doc is AnyEntry;
|
|
39
69
|
export declare function isEncryptedChunkEntry(doc: DatabaseEntry): doc is EntryLeaf;
|
|
@@ -66,7 +96,7 @@ export declare function runWithInterval<T>(key: string, interval: number, task:
|
|
|
66
96
|
* @remarks This function is not designed to be concurrent.
|
|
67
97
|
*/
|
|
68
98
|
export declare function runWithStartInterval<T>(key: string, interval: number, task: () => Promise<T>): Promise<T>;
|
|
69
|
-
export declare const globalConcurrencyController: import("octagonal-wheels/concurrency/
|
|
99
|
+
export declare const globalConcurrencyController: import("octagonal-wheels/concurrency/semaphore").SemaphoreObject;
|
|
70
100
|
export declare function determineTypeFromBlob(data: Blob): "newnote" | "plain";
|
|
71
101
|
/**
|
|
72
102
|
* Whether remediation mode is active.
|
package/dist/common/utils.js
CHANGED
|
@@ -9,8 +9,10 @@ import {
|
|
|
9
9
|
} from "./types.js";
|
|
10
10
|
import { normaliseP2PConnectionPath, normaliseP2PMaxWirePayloadBytes } from "./models/setting.p2p.js";
|
|
11
11
|
import { isErrorOfMissingDoc } from "../pouchdb/utils_couchdb.js";
|
|
12
|
-
import { replaceAll, replaceAllPairs } from "octagonal-wheels/string";
|
|
12
|
+
import { joinDelimitedList, replaceAll, replaceAllPairs, splitDelimitedList } from "octagonal-wheels/string";
|
|
13
|
+
import { formatDuration } from "octagonal-wheels/notation/duration";
|
|
13
14
|
import { concatUInt8Array } from "octagonal-wheels/binary";
|
|
15
|
+
import { createBinaryBlob, createTextBlob, isBlobContentEqual, toBlob } from "octagonal-wheels/binary/blob";
|
|
14
16
|
import { delay, fireAndForget } from "octagonal-wheels/promises";
|
|
15
17
|
import { arrayToChunkedArray, unique } from "octagonal-wheels/collection";
|
|
16
18
|
import { extractObject, isObjectDifferent } from "octagonal-wheels/object";
|
|
@@ -55,17 +57,8 @@ function getDocDataAsArrayBuffer(doc) {
|
|
|
55
57
|
function isTextBlob(blob) {
|
|
56
58
|
return blob.type === "text/plain";
|
|
57
59
|
}
|
|
58
|
-
function createTextBlob(data) {
|
|
59
|
-
const d = Array.isArray(data) ? data : [data];
|
|
60
|
-
return new Blob(d, { endings: "transparent", type: "text/plain" });
|
|
61
|
-
}
|
|
62
|
-
function createBinaryBlob(data) {
|
|
63
|
-
return new Blob([data], { endings: "transparent", type: "application/octet-stream" });
|
|
64
|
-
}
|
|
65
60
|
function createBlob(data) {
|
|
66
|
-
|
|
67
|
-
if (data instanceof Uint8Array || data instanceof ArrayBuffer) return createBinaryBlob(data);
|
|
68
|
-
return createTextBlob(data);
|
|
61
|
+
return toBlob(data);
|
|
69
62
|
}
|
|
70
63
|
function isTextDocument(doc) {
|
|
71
64
|
if (doc.type == "plain") return true;
|
|
@@ -88,23 +81,17 @@ function readContent(doc) {
|
|
|
88
81
|
}
|
|
89
82
|
}
|
|
90
83
|
const isIndexDBCmpExist = typeof compatGlobal?.indexedDB?.cmp !== "undefined";
|
|
84
|
+
async function isArrayBufferContentEqual(left, right) {
|
|
85
|
+
return await arrayBufferToBase64Single(left) === await arrayBufferToBase64Single(right);
|
|
86
|
+
}
|
|
91
87
|
async function isDocContentSame(docA, docB) {
|
|
92
88
|
const blob1 = createBlob(docA);
|
|
93
89
|
const blob2 = createBlob(docB);
|
|
94
|
-
if (blob1.size != blob2.size) return false;
|
|
95
90
|
if (isIndexDBCmpExist) {
|
|
91
|
+
if (blob1.size != blob2.size) return false;
|
|
96
92
|
return compatGlobal.indexedDB.cmp(await blob1.arrayBuffer(), await blob2.arrayBuffer()) === 0;
|
|
97
93
|
}
|
|
98
|
-
|
|
99
|
-
const length = blob1.size;
|
|
100
|
-
let i = 0;
|
|
101
|
-
while (i < length) {
|
|
102
|
-
const ab1 = await blob1.slice(i, i + checkQuantum).arrayBuffer();
|
|
103
|
-
const ab2 = await blob2.slice(i, i + checkQuantum).arrayBuffer();
|
|
104
|
-
i += checkQuantum;
|
|
105
|
-
if (await arrayBufferToBase64Single(ab1) != await arrayBufferToBase64Single(ab2)) return false;
|
|
106
|
-
}
|
|
107
|
-
return true;
|
|
94
|
+
return isBlobContentEqual(blob1, blob2, isArrayBufferContentEqual);
|
|
108
95
|
}
|
|
109
96
|
function isObfuscatedEntry(doc) {
|
|
110
97
|
if (doc._id.startsWith(PREFIX_OBFUSCATED)) {
|
|
@@ -267,24 +254,7 @@ function escapeMarkdownValue(value) {
|
|
|
267
254
|
}
|
|
268
255
|
}
|
|
269
256
|
function timeDeltaToHumanReadable(delta) {
|
|
270
|
-
|
|
271
|
-
if (sec < 60) {
|
|
272
|
-
return `${sec.toFixed(2)}s`;
|
|
273
|
-
}
|
|
274
|
-
const min = sec / 60;
|
|
275
|
-
if (min < 60) {
|
|
276
|
-
return `${min.toFixed(2)}m`;
|
|
277
|
-
}
|
|
278
|
-
const hour = min / 60;
|
|
279
|
-
if (hour < 24) {
|
|
280
|
-
return `${hour.toFixed(2)}h`;
|
|
281
|
-
}
|
|
282
|
-
const day = hour / 24;
|
|
283
|
-
if (day < 365) {
|
|
284
|
-
return `${day.toFixed(2)}d`;
|
|
285
|
-
}
|
|
286
|
-
const year = day / 365;
|
|
287
|
-
return `${year.toFixed(2)}y`;
|
|
257
|
+
return formatDuration(delta);
|
|
288
258
|
}
|
|
289
259
|
async function wrapException(func) {
|
|
290
260
|
try {
|
|
@@ -378,12 +348,10 @@ function parseCustomRegExpList(list, flags, delimiter) {
|
|
|
378
348
|
return items.map((e) => new CustomRegExp(e, flags));
|
|
379
349
|
}
|
|
380
350
|
function constructCustomRegExpList(items, delimiter) {
|
|
381
|
-
return items.map((e) => `${e}`)
|
|
351
|
+
return joinDelimitedList(items.map((e) => `${e}`), `${delimiter}`);
|
|
382
352
|
}
|
|
383
353
|
function splitCustomRegExpList(list, delimiter) {
|
|
384
|
-
|
|
385
|
-
const source = `${list ?? ""}`;
|
|
386
|
-
return source.split(d).filter((e) => e);
|
|
354
|
+
return splitDelimitedList(`${list ?? ""}`, delimiter);
|
|
387
355
|
}
|
|
388
356
|
class CustomRegExp {
|
|
389
357
|
regexp;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { LiveSyncError } from "./LSError.js";
|
|
2
|
+
import { cloneViaJSON } from "octagonal-wheels/encoding/json";
|
|
2
3
|
function asCopy(obj) {
|
|
3
|
-
return
|
|
4
|
+
return cloneViaJSON(obj);
|
|
4
5
|
}
|
|
5
6
|
function ensureError(error) {
|
|
6
7
|
if (error instanceof Error) {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** An unclaimed value; its identity distinguishes replacements for the same key. */
|
|
2
|
+
export interface PendingWork<K, T> {
|
|
3
|
+
readonly key: K;
|
|
4
|
+
readonly value: T;
|
|
5
|
+
}
|
|
6
|
+
/** Keep only the latest unclaimed value for each key, in replacement order. */
|
|
7
|
+
export declare function createKeyedPendingWork<K, T>(keyOf: (value: T) => K): {
|
|
8
|
+
add(value: T): void;
|
|
9
|
+
peek(): PendingWork<K, T> | undefined;
|
|
10
|
+
claim(entry: PendingWork<K, T>): boolean;
|
|
11
|
+
values(): readonly T[];
|
|
12
|
+
readonly size: number;
|
|
13
|
+
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
function createKeyedPendingWork(keyOf) {
|
|
2
|
+
const pending = /* @__PURE__ */ new Map();
|
|
3
|
+
return {
|
|
4
|
+
add(value) {
|
|
5
|
+
const key = keyOf(value);
|
|
6
|
+
pending.delete(key);
|
|
7
|
+
pending.set(key, { key, value });
|
|
8
|
+
},
|
|
9
|
+
peek() {
|
|
10
|
+
return pending.values().next().value;
|
|
11
|
+
},
|
|
12
|
+
claim(entry) {
|
|
13
|
+
if (pending.get(entry.key) !== entry) return false;
|
|
14
|
+
pending.delete(entry.key);
|
|
15
|
+
return true;
|
|
16
|
+
},
|
|
17
|
+
values() {
|
|
18
|
+
return Array.from(pending.values(), (entry) => entry.value);
|
|
19
|
+
},
|
|
20
|
+
get size() {
|
|
21
|
+
return pending.size;
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
export {
|
|
26
|
+
createKeyedPendingWork
|
|
27
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { FilePathWithPrefix } from "../common/types.js";
|
|
2
|
+
import { type Awaitable, type Scope } from "octagonal-wheels/resource/scope";
|
|
3
|
+
/** Host-owned conflict processing and observation for one queue. */
|
|
4
|
+
export interface ConflictCheckQueueOptions {
|
|
5
|
+
/** Process one admitted path. Deferred cleanup occupies the same execution slot. */
|
|
6
|
+
readonly process: (path: FilePathWithPrefix, scope: Scope<undefined>) => Awaitable<unknown>;
|
|
7
|
+
/** Positive safe integer; defaults to ten. Pending paths have no fixed capacity. */
|
|
8
|
+
readonly concurrency?: number;
|
|
9
|
+
/** Observe pending plus active checks. A thrown error stops the queue after active work settles. */
|
|
10
|
+
readonly onProgress?: (remaining: number) => void;
|
|
11
|
+
}
|
|
12
|
+
/** One queue owns pending checks; the host owns conflict decisions and recovery policy. */
|
|
13
|
+
export interface ConflictCheckQueue {
|
|
14
|
+
/** Accept a path without waiting for its check; a pending duplicate moves to the tail. */
|
|
15
|
+
enqueue(path: FilePathWithPrefix): void;
|
|
16
|
+
/** Wait for quiescence, including requeues and cleanup; false means the queue was aborted. Do not await from its processor. */
|
|
17
|
+
waitForIdle(): Promise<boolean>;
|
|
18
|
+
/** Stop admission and drain accepted work. A processor must not await its own queue's closure. */
|
|
19
|
+
close(): Promise<boolean>;
|
|
20
|
+
/** Stop admission and dispatch, join active cleanup, and return the retained, unstarted paths. */
|
|
21
|
+
abort(): Promise<readonly FilePathWithPrefix[]>;
|
|
22
|
+
/** Snapshot of unstarted paths; retained after abort or failure for the host to inspect. */
|
|
23
|
+
readonly pending: readonly FilePathWithPrefix[];
|
|
24
|
+
/** Pending and active checks, including item cleanup. */
|
|
25
|
+
readonly remaining: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Compose keyed pending checks with bounded scoped execution.
|
|
29
|
+
*
|
|
30
|
+
* Input is accepted synchronously without a fixed pending-count limit. Each
|
|
31
|
+
* execution consumes a synchronous iterable which ends when no pending path
|
|
32
|
+
* remains; later arrivals join it or start the next execution. An empty queue has no input read,
|
|
33
|
+
* timer, or worker waiting in the background.
|
|
34
|
+
*
|
|
35
|
+
* Replacement affects unclaimed entries only. A running check may enqueue its
|
|
36
|
+
* own path without waiting for an execution slot. It is not cancelled or replaced
|
|
37
|
+
* by that later input. The host remains responsible for serialising any shared
|
|
38
|
+
* resource or interaction inside the processor.
|
|
39
|
+
*
|
|
40
|
+
* Unhandled processing, cleanup, or observation errors stop further dispatch.
|
|
41
|
+
* Started work and cleanup settle before completion waiters reject; unstarted
|
|
42
|
+
* paths remain inspectable. Catch an item error inside process when the host's
|
|
43
|
+
* explicit policy is to log and continue. There is no implicit retry or logging.
|
|
44
|
+
*/
|
|
45
|
+
export declare function createConflictCheckQueue(options: ConflictCheckQueueOptions): ConflictCheckQueue;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { createKeyedPendingWork } from "../concurrency/keyedPendingWork.js";
|
|
2
|
+
import { createNotifier } from "octagonal-wheels/concurrency/notifier";
|
|
3
|
+
import { forEachInScope } from "octagonal-wheels/concurrency/scoped.iteration";
|
|
4
|
+
import { withScope } from "octagonal-wheels/resource/scope";
|
|
5
|
+
function createConflictCheckQueue(options) {
|
|
6
|
+
const concurrency = options.concurrency ?? 10;
|
|
7
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
|
|
8
|
+
throw new RangeError("Conflict check concurrency must be a positive safe integer.");
|
|
9
|
+
}
|
|
10
|
+
const pending = createKeyedPendingWork((path) => path);
|
|
11
|
+
const changed = createNotifier();
|
|
12
|
+
const failures = [];
|
|
13
|
+
let accepting = true;
|
|
14
|
+
let aborted = false;
|
|
15
|
+
let stoppingAfterFailure = false;
|
|
16
|
+
let active = 0;
|
|
17
|
+
let running;
|
|
18
|
+
const remaining = () => pending.size + active;
|
|
19
|
+
const canDispatch = () => !aborted && !stoppingAfterFailure && failures.length === 0;
|
|
20
|
+
const recordFailure = (error) => {
|
|
21
|
+
accepting = false;
|
|
22
|
+
stoppingAfterFailure = true;
|
|
23
|
+
failures.push(error);
|
|
24
|
+
changed.notify();
|
|
25
|
+
};
|
|
26
|
+
const reportProgress = () => {
|
|
27
|
+
if (failures.length === 0) {
|
|
28
|
+
try {
|
|
29
|
+
options.onProgress?.(remaining());
|
|
30
|
+
} catch (error) {
|
|
31
|
+
recordFailure(error);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
changed.notify();
|
|
35
|
+
};
|
|
36
|
+
function* availableWork() {
|
|
37
|
+
while (canDispatch()) {
|
|
38
|
+
const entry = pending.peek();
|
|
39
|
+
if (!entry) return;
|
|
40
|
+
yield entry;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function schedule() {
|
|
44
|
+
if (running || !canDispatch() || pending.size === 0) return;
|
|
45
|
+
running = Promise.resolve().then(
|
|
46
|
+
() => withScope(
|
|
47
|
+
void 0,
|
|
48
|
+
(parent) => forEachInScope(parent, availableWork(), { concurrency }, async (entry, child) => {
|
|
49
|
+
if (!canDispatch() || !pending.claim(entry)) return;
|
|
50
|
+
active++;
|
|
51
|
+
child.defer(() => {
|
|
52
|
+
active--;
|
|
53
|
+
reportProgress();
|
|
54
|
+
});
|
|
55
|
+
reportProgress();
|
|
56
|
+
try {
|
|
57
|
+
await options.process(entry.value, child);
|
|
58
|
+
} catch (error) {
|
|
59
|
+
accepting = false;
|
|
60
|
+
stoppingAfterFailure = true;
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
})
|
|
64
|
+
)
|
|
65
|
+
).then(() => void 0, recordFailure).finally(() => {
|
|
66
|
+
running = void 0;
|
|
67
|
+
schedule();
|
|
68
|
+
changed.notify();
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
async function waitForIdle() {
|
|
72
|
+
schedule();
|
|
73
|
+
while (running) await changed.wait();
|
|
74
|
+
if (failures.length === 1) throw failures[0];
|
|
75
|
+
if (failures.length > 1) throw new AggregateError(failures, "Conflict checks failed.");
|
|
76
|
+
return !aborted;
|
|
77
|
+
}
|
|
78
|
+
return {
|
|
79
|
+
enqueue(path) {
|
|
80
|
+
if (!accepting) throw new Error("The conflict check queue is closed.");
|
|
81
|
+
pending.add(path);
|
|
82
|
+
reportProgress();
|
|
83
|
+
schedule();
|
|
84
|
+
},
|
|
85
|
+
waitForIdle,
|
|
86
|
+
close() {
|
|
87
|
+
accepting = false;
|
|
88
|
+
schedule();
|
|
89
|
+
return waitForIdle();
|
|
90
|
+
},
|
|
91
|
+
async abort() {
|
|
92
|
+
accepting = false;
|
|
93
|
+
aborted = true;
|
|
94
|
+
changed.notify();
|
|
95
|
+
await waitForIdle();
|
|
96
|
+
return pending.values();
|
|
97
|
+
},
|
|
98
|
+
get pending() {
|
|
99
|
+
return pending.values();
|
|
100
|
+
},
|
|
101
|
+
get remaining() {
|
|
102
|
+
return remaining();
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
export {
|
|
107
|
+
createConflictCheckQueue
|
|
108
|
+
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withScope } from "octagonal-wheels/resource/scope";
|
|
1
2
|
import {
|
|
2
3
|
DOCID_SYNC_PARAMETERS,
|
|
3
4
|
EntryTypes,
|
|
@@ -749,12 +750,11 @@ async function analyseDatabaseIntegrity(options) {
|
|
|
749
750
|
let wasCancelled = cancelled(options);
|
|
750
751
|
let configurationChanged = false;
|
|
751
752
|
let pathEquivalenceGroups = [];
|
|
752
|
-
let ownedRemote;
|
|
753
753
|
if (!wasCancelled && !isConfigurationCurrent(options)) {
|
|
754
754
|
configurationChanged = true;
|
|
755
755
|
appendError(errors, "configuration", "configuration-changed");
|
|
756
756
|
}
|
|
757
|
-
|
|
757
|
+
await withScope(void 0, async (scope) => {
|
|
758
758
|
if (!wasCancelled && !configurationChanged) {
|
|
759
759
|
const localRun = makeSourceRun("local", options.local);
|
|
760
760
|
runs.push(localRun);
|
|
@@ -768,7 +768,14 @@ async function analyseDatabaseIntegrity(options) {
|
|
|
768
768
|
runs.push(remoteRun);
|
|
769
769
|
} else {
|
|
770
770
|
try {
|
|
771
|
-
ownedRemote = await options.remote();
|
|
771
|
+
const ownedRemote = await options.remote();
|
|
772
|
+
scope.defer(async () => {
|
|
773
|
+
try {
|
|
774
|
+
await ownedRemote.close();
|
|
775
|
+
} catch {
|
|
776
|
+
appendError(errors, "remote", "source-unavailable");
|
|
777
|
+
}
|
|
778
|
+
});
|
|
772
779
|
if (cancelled(options)) {
|
|
773
780
|
wasCancelled = true;
|
|
774
781
|
} else {
|
|
@@ -851,15 +858,7 @@ async function analyseDatabaseIntegrity(options) {
|
|
|
851
858
|
}
|
|
852
859
|
}
|
|
853
860
|
}
|
|
854
|
-
}
|
|
855
|
-
if (ownedRemote) {
|
|
856
|
-
try {
|
|
857
|
-
await ownedRemote.close();
|
|
858
|
-
} catch {
|
|
859
|
-
appendError(errors, "remote", "source-unavailable");
|
|
860
|
-
}
|
|
861
|
-
}
|
|
862
|
-
}
|
|
861
|
+
});
|
|
863
862
|
if (cancelled(options)) wasCancelled = true;
|
|
864
863
|
if (!wasCancelled && !configurationChanged && !isConfigurationCurrent(options)) {
|
|
865
864
|
configurationChanged = true;
|
|
@@ -20,29 +20,49 @@ export interface ServiceModules {
|
|
|
20
20
|
*/
|
|
21
21
|
rebuilder: Rebuilder;
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/**
|
|
24
|
+
* Selects service dependencies from the host's service type.
|
|
25
|
+
* A context is retained when the selected source exposes one. The default
|
|
26
|
+
* source preserves the existing ServiceHub dependency contract.
|
|
27
|
+
*/
|
|
28
|
+
export type RequiredServices<T extends keyof THub, THub = ServiceHub> = Pick<THub, T | Extract<"context", keyof THub>>;
|
|
24
29
|
export type RequiredServiceModules<T extends keyof ServiceModules> = Pick<ServiceModules, T>;
|
|
25
|
-
export type RequiredServicesInterfaces<T extends keyof IServiceHub> =
|
|
30
|
+
export type RequiredServicesInterfaces<T extends keyof IServiceHub> = RequiredServices<T, IServiceHub>;
|
|
26
31
|
export type RequiredServiceModulesInterfaces<T extends keyof ServiceModules> = Pick<ServiceModules, T>;
|
|
27
|
-
|
|
28
|
-
|
|
32
|
+
/**
|
|
33
|
+
* Declares the Services and ServiceModules consumed by a feature.
|
|
34
|
+
* Select a more specific source to retain injectable handlers or an extended
|
|
35
|
+
* host context, or use IServiceHub for operation-only interfaces. Additional
|
|
36
|
+
* collaborators, such as a borrowed LogFunction, belong in the feature's own
|
|
37
|
+
* dependency type. This type neither selects runtime values nor owns them.
|
|
38
|
+
*/
|
|
39
|
+
export type NecessaryServices<T extends keyof THub, U extends keyof ServiceModules, THub = ServiceHub> = {
|
|
40
|
+
services: RequiredServices<T, THub>;
|
|
29
41
|
serviceModules: RequiredServiceModules<U>;
|
|
30
42
|
};
|
|
31
|
-
export type NecessaryServicesInterfaces<T extends keyof IServiceHub, U extends keyof ServiceModules> =
|
|
32
|
-
|
|
33
|
-
serviceModules: RequiredServiceModulesInterfaces<U>;
|
|
34
|
-
};
|
|
35
|
-
export type ServiceFeatureFunction<T extends keyof ServiceHub, U extends keyof ServiceModules, TR> = (host: NecessaryServices<T, U>) => TR;
|
|
43
|
+
export type NecessaryServicesInterfaces<T extends keyof IServiceHub, U extends keyof ServiceModules> = NecessaryServices<T, U, IServiceHub>;
|
|
44
|
+
export type ServiceFeatureFunction<T extends keyof THub, U extends keyof ServiceModules, TR, THub = ServiceHub> = (host: NecessaryServices<T, U, THub>) => TR;
|
|
36
45
|
type ServiceFeatureContext<T> = T & {
|
|
37
46
|
_log: LogFunction;
|
|
38
47
|
};
|
|
39
|
-
export type ServiceFeatureFunctionWithContext<T extends keyof
|
|
48
|
+
export type ServiceFeatureFunctionWithContext<T extends keyof THub, U extends keyof ServiceModules, C, TR, THub = ServiceHub> = (host: NecessaryServices<T, U, THub>, context: ServiceFeatureContext<C>) => TR;
|
|
49
|
+
type AnyServiceFeature = (...args: never[]) => unknown;
|
|
50
|
+
type ServiceFeatureWithArguments<T extends keyof THub, U extends keyof ServiceModules, THub = ServiceHub> = (host: NecessaryServices<T, U, THub>, ...args: never[]) => unknown;
|
|
40
51
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
52
|
+
* Retains a service feature's complete function type and identity.
|
|
53
|
+
* Declare the dependency type on the feature parameter so that required
|
|
54
|
+
* collaborators, concrete service types, and returned views remain explicit.
|
|
55
|
+
* Optional arguments, overloads, generic signatures, and asynchronous results
|
|
56
|
+
* are preserved. Older unannotated callbacks retain their ServiceHub context;
|
|
57
|
+
* their dependency keys cannot be inferred from the callback body.
|
|
58
|
+
*
|
|
59
|
+
* This helper does not invoke the feature, acquire or release its dependencies,
|
|
60
|
+
* or wrap its return value or errors. The host owns composition and disposal.
|
|
61
|
+
* @param featureFunction The feature function to retain.
|
|
62
|
+
* @returns The supplied function without a runtime wrapper.
|
|
44
63
|
* @example
|
|
45
|
-
*
|
|
64
|
+
* type IndexHost = NecessaryServices<"appLifecycle", "storageAccess">;
|
|
65
|
+
* const useIndexFeature = createServiceFeature((host: IndexHost) => {
|
|
46
66
|
* const { appLifecycle } = host.services;
|
|
47
67
|
* const { storageAccess } = host.serviceModules;
|
|
48
68
|
*
|
|
@@ -51,18 +71,16 @@ export type ServiceFeatureFunctionWithContext<T extends keyof ServiceHub, U exte
|
|
|
51
71
|
* return true;
|
|
52
72
|
* });
|
|
53
73
|
* });
|
|
54
|
-
*
|
|
55
|
-
* useIndexFeature({
|
|
56
|
-
* services: { context: serviceHub.context, appLifecycle: serviceHub.appLifecycle },
|
|
57
|
-
* serviceModules: { storageAccess },
|
|
58
|
-
* });
|
|
59
74
|
*/
|
|
75
|
+
export declare function createServiceFeature<TFeature extends ServiceFeatureWithArguments<keyof ServiceHub, keyof ServiceModules>>(featureFunction: TFeature): TFeature;
|
|
76
|
+
export declare function createServiceFeature<TFeature extends AnyServiceFeature>(featureFunction: TFeature): TFeature;
|
|
60
77
|
export declare function createServiceFeature<T extends keyof ServiceHub, U extends keyof ServiceModules, TR>(featureFunction: ServiceFeatureFunction<T, U, TR>): ServiceFeatureFunction<T, U, TR>;
|
|
61
|
-
type ContextFactory<T extends keyof
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
78
|
+
type ContextFactory<T extends keyof THub, U extends keyof ServiceModules, C, THub = ServiceHub> = (host: NecessaryServices<T, U, THub>) => ServiceFeatureContext<C>;
|
|
79
|
+
/** Selects the host type for the existing service-feature builder. */
|
|
80
|
+
export declare function serviceFeature<T extends keyof THub, U extends keyof ServiceModules, THub = ServiceHub>(): {
|
|
81
|
+
create: <TR, TFeature extends (host: NecessaryServices<T, U, THub>, ...args: never[]) => TR = ServiceFeatureFunction<T, U, TR, THub>>(featureFunction: TFeature) => TFeature;
|
|
82
|
+
withContext<C extends object = object>(ContextFactory: ContextFactory<T, U, C, THub>): {
|
|
83
|
+
create: <TR>(featureFunction: ServiceFeatureFunctionWithContext<T, U, C, TR, THub>) => (host: NecessaryServices<T, U, THub>, context: ServiceFeatureContext<C>) => TR;
|
|
66
84
|
};
|
|
67
85
|
};
|
|
68
86
|
export {};
|
|
@@ -2,10 +2,11 @@ function createServiceFeature(featureFunction) {
|
|
|
2
2
|
return featureFunction;
|
|
3
3
|
}
|
|
4
4
|
function serviceFeature() {
|
|
5
|
+
function create(featureFunction) {
|
|
6
|
+
return featureFunction;
|
|
7
|
+
}
|
|
5
8
|
return {
|
|
6
|
-
create
|
|
7
|
-
return featureFunction;
|
|
8
|
-
},
|
|
9
|
+
create,
|
|
9
10
|
withContext(ContextFactory) {
|
|
10
11
|
return {
|
|
11
12
|
create: (featureFunction) => (host, context) => featureFunction(host, ContextFactory(host))
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type DocumentID } from "../common/types.js";
|
|
2
2
|
import { type ChunkManager } from "./ChunkManager.js";
|
|
3
|
+
import type { ChunkFetchSource } from "./chunkFetchSource.js";
|
|
3
4
|
import type { IReplicatorService, ISettingService } from "../services/base/IService.js";
|
|
4
5
|
export declare const EVENT_MISSING_CHUNKS = "missingChunks";
|
|
5
6
|
export declare const EVENT_MISSING_CHUNK_REMOTE = "missingChunkRemote";
|
|
@@ -9,6 +10,8 @@ export type ChunkFetcherOptions = {
|
|
|
9
10
|
chunkManager: ChunkManager;
|
|
10
11
|
replicatorService: IReplicatorService;
|
|
11
12
|
deliveryStallTimeoutMs?: number;
|
|
13
|
+
/** Direct hosts replace both retrieval and acceptance; the default uses the active replicator. */
|
|
14
|
+
source?: ChunkFetchSource;
|
|
12
15
|
};
|
|
13
16
|
export declare class ChunkFetcher {
|
|
14
17
|
options: ChunkFetcherOptions;
|