@vrtmrz/livesync-commonlib 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -8
- package/dist/API/DirectFileManipulatorV2.d.ts +7 -1
- package/dist/API/DirectFileManipulatorV2.js +46 -67
- package/dist/API/DirectFileWatchOwner.d.ts +24 -0
- package/dist/API/DirectFileWatchOwner.js +256 -0
- package/dist/API/processSetting.js +3 -3
- package/dist/common/models/setting.type.d.ts +1 -1
- package/dist/common/models/shared.definition.configNames.js +1 -2
- package/dist/common/models/tweak.definition.d.ts +1 -1
- package/dist/common/pathNormalization.d.ts +2 -0
- package/dist/common/pathNormalization.js +7 -2
- package/dist/managers/ChunkFetcher.d.ts +3 -0
- package/dist/managers/ChunkFetcher.js +20 -13
- package/dist/managers/LiveSyncManagers.d.ts +2 -0
- package/dist/managers/LiveSyncManagers.js +1 -0
- package/dist/managers/chunkFetchSource.d.ts +14 -0
- package/dist/managers/chunkFetchSource.js +12 -0
- package/dist/pouchdb/LiveSyncDBFunctions.js +5 -0
- package/dist/pouchdb/LiveSyncLocalDB.d.ts +7 -1
- package/dist/pouchdb/LiveSyncLocalDB.js +13 -10
- package/dist/pouchdb/pathIdentityCompatibility.d.ts +8 -0
- package/dist/pouchdb/pathIdentityCompatibility.js +19 -1
- package/dist/replication/RemoteResource.d.ts +3 -0
- package/dist/replication/index.d.ts +2 -0
- package/dist/replication/index.js +2 -0
- package/dist/replication/journal/JournalSyncCore.d.ts +1 -0
- package/dist/replication/journal/JournalSyncCore.js +35 -19
- package/dist/replication/journal/LiveSyncJournalReplicator.d.ts +11 -3
- package/dist/replication/journal/LiveSyncJournalReplicator.js +123 -45
- package/dist/replication/trystero/P2PRoomSessionOwner.js +14 -13
- package/dist/replication/trystero/TrysteroReplicator.d.ts +7 -5
- package/dist/replication/trystero/TrysteroReplicator.js +98 -70
- package/dist/replication/trystero/TrysteroReplicator.pathIdentity.d.ts +11 -0
- package/dist/replication/trystero/TrysteroReplicator.pathIdentity.js +72 -0
- package/dist/replication/trystero/p2pReplicatorConfigurationIdentity.js +2 -0
- package/dist/replication/trystero/useP2PReplicatorFeature.js +6 -2
- package/dist/serviceModules/Rebuilder.d.ts +1 -0
- package/dist/serviceModules/Rebuilder.js +33 -5
- package/dist/worker/bgWorker.js +1 -1
- package/docs/conflict-resolution.md +1 -1
- package/docs/development.md +4 -0
- package/docs/proven-in-use.md +30 -13
- package/docs/releasing.md +2 -0
- package/docs/service-feature-composition.md +7 -1
- package/docs/unicode-path-identity.md +64 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -14,6 +14,19 @@ This package is primarily for maintainers of those clients and for integrations
|
|
|
14
14
|
|
|
15
15
|
## Status and versioning
|
|
16
16
|
|
|
17
|
+
Commonlib [0.2.3](https://github.com/vrtmrz/livesync-commonlib/releases/tag/0.2.3)
|
|
18
|
+
is published on npm through both `latest` and `next`. Self-hosted LiveSync
|
|
19
|
+
[1.0.44](https://github.com/vrtmrz/obsidian-livesync/releases/tag/1.0.44)
|
|
20
|
+
uses that exact version. See the [0.2.3 release notes](updates.md#023) for the
|
|
21
|
+
handler-setter source compatibility change and positive-infinity formatting
|
|
22
|
+
exception.
|
|
23
|
+
|
|
24
|
+
Version 0.3.0 requires custom clients passed to
|
|
25
|
+
`LiveSyncJournalReplicator.ensureBucketIsCompatible` to implement
|
|
26
|
+
`hasRemoteJournals(): Promise<boolean>`. The standard Journal client already
|
|
27
|
+
provides this operation. See the [0.3.0 release notes](updates.md#030) for the
|
|
28
|
+
custom-client requirement and path identity migration conditions.
|
|
29
|
+
|
|
17
30
|
Commonlib is pre-1.0. The compatibility policy described here begins with 0.2.0 and does not retroactively apply to published 0.1.x versions. Within 0.2.x, Commonlib preserves published APIs, types, behaviour, supported persisted data, and exported `compat/*` paths; intentional breaking changes use 0.3.0. Consumers may declare ranges such as `^0.2.0`; retain a lockfile and use `npm ci` for reproducible installs. An entry in the package export map means that the import is deliberate and package-tested; it does not make every semantic detail of that entry a final 1.0 guarantee.
|
|
18
31
|
|
|
19
32
|
Treat every `compat/*` subpath as a migration-only interface. A source-directory import, Git submodule, or consumer-defined TypeScript alias is not a supported package boundary.
|
|
@@ -52,8 +65,8 @@ The existing `compat/services/lib` boundary retains `bindableFunction` and `lazy
|
|
|
52
65
|
|
|
53
66
|
The `compat/pouchdb/chunks` entry retains the obsolete `transferChunks`,
|
|
54
67
|
`balanceChunkPurgedDBs`, and `fetchAllUsedChunks` helpers for existing consumers.
|
|
55
|
-
|
|
56
|
-
|
|
68
|
+
Self-hosted LiveSync 1.0.44 uses the standard locked-remote recovery dialogue
|
|
69
|
+
in place of its purge-based cleanup recovery. Garbage
|
|
57
70
|
Collection V3 propagates ordinary deletion revisions and does not use these
|
|
58
71
|
helpers. Their compatibility exports and existing transfer behaviour remain
|
|
59
72
|
available; new integrations should not adopt this obsolete cleanup path.
|
|
@@ -68,7 +81,7 @@ The P2P entry exposes eight focused views over one service owner, together with
|
|
|
68
81
|
|
|
69
82
|
## Blob helper migration
|
|
70
83
|
|
|
71
|
-
|
|
84
|
+
Commonlib 0.2.3 retains `createBlob`, `createTextBlob`, and
|
|
72
85
|
`createBinaryBlob` through the existing `compat/common/utils` entry. Existing
|
|
73
86
|
callers can keep those imports. `createBlob` preserves an existing Blob's
|
|
74
87
|
identity and MIME type; text and binary inputs produce new Blobs with
|
|
@@ -81,8 +94,8 @@ Its normalisation function is named `toBlob`; its creation functions retain
|
|
|
81
94
|
compares two Blobs in bounded parts and accepts an optional synchronous or
|
|
82
95
|
asynchronous ArrayBuffer equality predicate. See the
|
|
83
96
|
[Octagonal Wheels Blob guide](https://github.com/vrtmrz/fancy-kit/blob/main/packages/octagonal-wheels/guides/imports-and-runtime.md#blob-creation-and-normalisation)
|
|
84
|
-
for input and runtime contracts.
|
|
85
|
-
|
|
97
|
+
for input and runtime contracts. Commonlib 0.2.3 pins the published
|
|
98
|
+
Octagonal Wheels 0.1.56 dependency.
|
|
86
99
|
|
|
87
100
|
Commonlib continues to own document classification, decoding in `readAsBlob`,
|
|
88
101
|
and content-comparison policy. The legacy `isDocContentSame` retains its
|
|
@@ -292,7 +305,19 @@ See [the remote configuration profile guide](docs/remote-configurations.md) for
|
|
|
292
305
|
|
|
293
306
|
CouchDB stores the representations already in use as feature identifiers in its version document. Assess that document before synchronisation or direct file operations, and retain unknown identifiers for host diagnostics. See [the remote feature compatibility contract](docs/remote-feature-compatibility.md) for the wire format, update rules, and recovery boundaries.
|
|
294
307
|
|
|
295
|
-
The maintained CLI and Obsidian hosts can opt into [Unicode NFC path identity](docs/unicode-path-identity.md) with CouchDB. Existing databases require an explicit Rebuild or Fetch migration before using it.
|
|
308
|
+
The maintained CLI and Obsidian hosts can opt into [Unicode NFC path identity](docs/unicode-path-identity.md) with CouchDB, Object Storage, and P2P in the unreleased transport extension. Existing databases require an explicit Rebuild or Fetch migration before using it. P2P participants must all be updated and migrated.
|
|
309
|
+
|
|
310
|
+
## Direct database watches
|
|
311
|
+
|
|
312
|
+
Direct CouchDB reads use the same Chunk arrival waiting and bounded retry processing as application replication. They retrieve missing Chunks through the prepared source connection, including its decryption transforms, without writing fetched Chunks back to that source.
|
|
313
|
+
|
|
314
|
+
Existing integrations can use `DirectFileManipulator.beginWatch(callback, checkIsInterested?, onCheckpoint?)`. The file callback retains its `(doc: ReadyEntry, seq?: string | number)` signature. Resolving it acknowledges delivery. An incomplete read or rejected delivery remains pending and retries after delays from one second up to 30 seconds, while other documents continue. At most four file-processing operations run concurrently. A newer change for the same document supersedes its pending read, including deletion, rename, or exclusion by `checkIsInterested`.
|
|
315
|
+
|
|
316
|
+
The optional third callback, `(seq: string | number) => Promise<void> | void`, receives the completed feed position before the first unfinished change. Persist that checkpoint before resolving the callback. Commonlib retries rejected checkpoint notifications and advances `since` only after acknowledgement. A successful file callback can have a later sequence than this safe checkpoint; do not save its sequence independently. Sequence tokens are opaque. Without a checkpoint callback, `since` advances in memory.
|
|
317
|
+
|
|
318
|
+
Reconnects replay from `since`; a new process must restore the host's saved checkpoint before starting its watch. There is no persistent retry queue, so consumers must tolerate repeated delivery after reconnect or restart. `endWatch()` cancels watch retry and reconnect timers and prevents pending reads from delivering into a stopped or replacement watch. `close()` also retires the Chunk fetcher. Already-invoked host callbacks finish their own work, and callbacks for the same document remain serialised across stop and restart. Outstanding transport reads occupy a processing slot until they settle. A disconnected watch still owns its scheduled reconnect and rejects a duplicate `beginWatch()` call.
|
|
319
|
+
|
|
320
|
+
Logical deletions and note-shaped tombstones are delivered with empty `data` without loading deleted Chunks. A bare tombstone can reuse pending note metadata; without known metadata it remains ignored because an obfuscated document ID does not supply a file path.
|
|
296
321
|
|
|
297
322
|
## Contract scope
|
|
298
323
|
|
|
@@ -319,14 +344,14 @@ MIT
|
|
|
319
344
|
|
|
320
345
|
The shared formatter displays positive infinity as `∞y`, including through the existing `timeDeltaToHumanReadable` entry. Other duration formatting cases retain their previous behaviour.
|
|
321
346
|
|
|
322
|
-
|
|
347
|
+
Commonlib 0.2.3 keeps `timeDeltaToHumanReadable`,
|
|
323
348
|
`splitCustomRegExpList`, and `constructCustomRegExpList` at the existing
|
|
324
349
|
`compat/common/utils` entry. They delegate to `formatDuration` in
|
|
325
350
|
`octagonal-wheels/notation/duration` and the delimiter helpers in
|
|
326
351
|
`octagonal-wheels/string`. Commonlib's internal `asCopy` helper delegates to
|
|
327
352
|
`cloneViaJSON` in `octagonal-wheels/encoding/json`; it does not add a Commonlib
|
|
328
353
|
public entry. New general-purpose callers can use the focused Octagonal Wheels
|
|
329
|
-
entries
|
|
354
|
+
entries in Octagonal Wheels 0.1.56 or later.
|
|
330
355
|
|
|
331
356
|
Duration units and rounding remain unchanged. The delimiter adapters retain
|
|
332
357
|
their tagged types and string coercion; splitting removes empty parts and
|
|
@@ -137,7 +137,13 @@ export declare class DirectFileManipulator implements LiveSyncLocalDBEnv {
|
|
|
137
137
|
watching: boolean;
|
|
138
138
|
changes: PouchDB.Core.Changes<EntryDoc> | undefined;
|
|
139
139
|
since: string;
|
|
140
|
-
|
|
140
|
+
private watchOwner?;
|
|
141
|
+
/**
|
|
142
|
+
* Watch file changes, retrying incomplete reads and rejected consumer callbacks.
|
|
143
|
+
* Resolving the file callback acknowledges processing. The optional checkpoint
|
|
144
|
+
* callback must resolve after the host has persisted its safe resume position.
|
|
145
|
+
*/
|
|
146
|
+
beginWatch(callback: (doc: ReadyEntry, seq?: string | number) => Promise<any> | void, checkIsInterested?: (doc: MetaEntry) => boolean, onCheckpoint?: (seq: string | number) => Promise<void> | void): boolean;
|
|
141
147
|
endWatch(): void;
|
|
142
148
|
followUpdates(callback: (doc: ReadyEntry, seq?: string | number) => Promise<any> | void, checkIsInterested?: (doc: MetaEntry) => boolean): Promise<string | number>;
|
|
143
149
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { MemoryValueStore } from "@vrtmrz/value-store";
|
|
2
|
+
import { createDirectChunkFetchSource } from "../managers/chunkFetchSource.js";
|
|
2
3
|
import {
|
|
3
4
|
DEFAULT_SETTINGS,
|
|
4
5
|
DEFAULT_SYNC_PARAMETERS,
|
|
@@ -41,11 +42,7 @@ import {
|
|
|
41
42
|
import { HeadlessServiceHub } from "../services/HeadlessServices.js";
|
|
42
43
|
import { HeadlessDatabaseService } from "../services/implements/headless/HeadlessDatabaseService.js";
|
|
43
44
|
import { ServiceContext } from "../services/base/ServiceBase.js";
|
|
44
|
-
import {
|
|
45
|
-
function isNoteEntry(doc) {
|
|
46
|
-
if (!doc) return false;
|
|
47
|
-
return doc.type == "newnote" || doc.type == "plain";
|
|
48
|
-
}
|
|
45
|
+
import { DirectFileWatchOwner, isNoteEntry } from "./DirectFileWatchOwner.js";
|
|
49
46
|
function isReadyEntry(doc) {
|
|
50
47
|
if (!doc) return false;
|
|
51
48
|
return "data" in doc;
|
|
@@ -136,7 +133,9 @@ class DirectFileManipulator {
|
|
|
136
133
|
});
|
|
137
134
|
this.services.setting.settings = getSettings();
|
|
138
135
|
this.services.databaseEvents.onDatabaseInitialisation.addHandler(this.$everyOnInitializeDatabase.bind(this));
|
|
139
|
-
this.liveSyncLocalDB = new LiveSyncLocalDB(this.options.url, this
|
|
136
|
+
this.liveSyncLocalDB = new LiveSyncLocalDB(this.options.url, this, {
|
|
137
|
+
createChunkFetchSource: createDirectChunkFetchSource
|
|
138
|
+
});
|
|
140
139
|
void this.ready.promise.catch(() => {
|
|
141
140
|
});
|
|
142
141
|
void this.init();
|
|
@@ -214,6 +213,7 @@ class DirectFileManipulator {
|
|
|
214
213
|
return this.settings;
|
|
215
214
|
}
|
|
216
215
|
async close() {
|
|
216
|
+
this.endWatch();
|
|
217
217
|
await this.liveSyncLocalDB.close();
|
|
218
218
|
return this.liveSyncLocalDB.onunload();
|
|
219
219
|
}
|
|
@@ -375,74 +375,53 @@ class DirectFileManipulator {
|
|
|
375
375
|
}
|
|
376
376
|
}
|
|
377
377
|
watching = false;
|
|
378
|
-
// _abortController?: AbortController;
|
|
379
378
|
changes;
|
|
380
379
|
since = "";
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
380
|
+
watchOwner;
|
|
381
|
+
/**
|
|
382
|
+
* Watch file changes, retrying incomplete reads and rejected consumer callbacks.
|
|
383
|
+
* Resolving the file callback acknowledges processing. The optional checkpoint
|
|
384
|
+
* callback must resolve after the host has persisted its safe resume position.
|
|
385
|
+
*/
|
|
386
|
+
beginWatch(callback, checkIsInterested, onCheckpoint) {
|
|
387
|
+
const manipulator = this;
|
|
388
|
+
this.watchOwner ??= new DirectFileWatchOwner({
|
|
389
|
+
state: {
|
|
390
|
+
get watching() {
|
|
391
|
+
return manipulator.watching;
|
|
392
|
+
},
|
|
393
|
+
set watching(value) {
|
|
394
|
+
manipulator.watching = value;
|
|
395
|
+
},
|
|
396
|
+
get changes() {
|
|
397
|
+
return manipulator.changes;
|
|
398
|
+
},
|
|
399
|
+
set changes(value) {
|
|
400
|
+
manipulator.changes = value;
|
|
401
|
+
},
|
|
402
|
+
get since() {
|
|
403
|
+
return manipulator.since;
|
|
404
|
+
},
|
|
405
|
+
set since(value) {
|
|
406
|
+
manipulator.since = value;
|
|
403
407
|
}
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
await callback(docX, change.seq);
|
|
416
|
-
Logger(`WATCH: PROCESS DONE: ${doc.path}`, LEVEL_INFO, "watch");
|
|
417
|
-
} catch (ex) {
|
|
418
|
-
Logger(`WATCH: PROCESS FAILED`, LEVEL_INFO, "watch");
|
|
419
|
-
Logger(ex, LEVEL_VERBOSE, "watch");
|
|
420
|
-
}
|
|
421
|
-
}).on("complete", () => {
|
|
422
|
-
Logger(`WATCH: FINISHED`, LEVEL_INFO, "watch");
|
|
423
|
-
this.watching = false;
|
|
424
|
-
this.changes = void 0;
|
|
425
|
-
}).on("error", (err) => {
|
|
426
|
-
Logger(`WATCH: ERROR: `, LEVEL_INFO, "watch");
|
|
427
|
-
Logger(err, LEVEL_VERBOSE, "watch");
|
|
428
|
-
if (this.watching) {
|
|
429
|
-
Logger(`WATCH: CONNECTION HAS BEEN CLOSED, RECONNECTING...`, LEVEL_INFO, "watch");
|
|
430
|
-
this.watching = false;
|
|
431
|
-
this.changes = void 0;
|
|
432
|
-
compatGlobal.setTimeout(() => {
|
|
433
|
-
this.beginWatch(callback, checkIsInterested);
|
|
434
|
-
}, 1e4);
|
|
435
|
-
} else {
|
|
436
|
-
Logger(`WATCH: CONNECTION HAS BEEN CLOSED.`, LEVEL_INFO, "watch");
|
|
408
|
+
},
|
|
409
|
+
openChanges(since) {
|
|
410
|
+
return manipulator.liveSyncLocalDB.localDatabase.changes({
|
|
411
|
+
include_docs: true,
|
|
412
|
+
since,
|
|
413
|
+
selector: { type: { $ne: "leaf" } },
|
|
414
|
+
live: true
|
|
415
|
+
});
|
|
416
|
+
},
|
|
417
|
+
readDocument(doc) {
|
|
418
|
+
return manipulator.getByMeta(doc);
|
|
437
419
|
}
|
|
438
420
|
});
|
|
421
|
+
return this.watchOwner.beginWatch(callback, checkIsInterested, onCheckpoint);
|
|
439
422
|
}
|
|
440
423
|
endWatch() {
|
|
441
|
-
|
|
442
|
-
Logger(`WATCH: CANCELLING PROCESS.`, LEVEL_INFO, "watch");
|
|
443
|
-
this.changes.cancel();
|
|
444
|
-
Logger(`WATCH: CANCELLING SIGNAL HAS BEEN SENT.`, LEVEL_INFO, "watch");
|
|
445
|
-
}
|
|
424
|
+
this.watchOwner?.endWatch();
|
|
446
425
|
}
|
|
447
426
|
async followUpdates(callback, checkIsInterested) {
|
|
448
427
|
try {
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { EntryDoc, NewEntry, PlainEntry } from "../common/types.js";
|
|
2
|
+
import type { MetaEntry, ReadyEntry } from "./DirectFileManipulatorV2.js";
|
|
3
|
+
export interface DirectFileWatchState {
|
|
4
|
+
watching: boolean;
|
|
5
|
+
changes: PouchDB.Core.Changes<EntryDoc> | undefined;
|
|
6
|
+
since: string;
|
|
7
|
+
}
|
|
8
|
+
export interface DirectFileWatchHost {
|
|
9
|
+
readonly state: DirectFileWatchState;
|
|
10
|
+
openChanges(since: string): PouchDB.Core.Changes<EntryDoc>;
|
|
11
|
+
readDocument(doc: MetaEntry): Promise<ReadyEntry>;
|
|
12
|
+
}
|
|
13
|
+
type FileCallback = (doc: ReadyEntry, seq?: string | number) => Promise<unknown> | void;
|
|
14
|
+
type SelectionCallback = (doc: MetaEntry) => boolean;
|
|
15
|
+
type CheckpointCallback = (seq: string | number) => Promise<void> | void;
|
|
16
|
+
export declare function isNoteEntry(doc: EntryDoc | false | undefined): doc is NewEntry | PlainEntry;
|
|
17
|
+
/** Own watch sessions and work which remains in flight across stop and restart. */
|
|
18
|
+
export declare class DirectFileWatchOwner {
|
|
19
|
+
private readonly context;
|
|
20
|
+
constructor(host: DirectFileWatchHost);
|
|
21
|
+
beginWatch(callback: FileCallback, checkIsInterested?: SelectionCallback, onCheckpoint?: CheckpointCallback): boolean;
|
|
22
|
+
endWatch(): void;
|
|
23
|
+
}
|
|
24
|
+
export {};
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { compatGlobal } from "../common/coreEnvFunctions.js";
|
|
2
|
+
import { LEVEL_INFO, LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
|
3
|
+
import { bindContext } from "octagonal-wheels/context";
|
|
4
|
+
import { withScope } from "octagonal-wheels/resource/scope";
|
|
5
|
+
function isNoteEntry(doc) {
|
|
6
|
+
return !!doc && (doc.type == "newnote" || doc.type == "plain");
|
|
7
|
+
}
|
|
8
|
+
function isActive(session) {
|
|
9
|
+
return session.owner.session === session;
|
|
10
|
+
}
|
|
11
|
+
function isCurrent({ session, item, epoch }) {
|
|
12
|
+
return isActive(session) && session.connected && session.connection === epoch && session.pending.get(item.id) === item;
|
|
13
|
+
}
|
|
14
|
+
function retryDelay(failures) {
|
|
15
|
+
return Math.min(3e4, 1e3 * 2 ** Math.min(failures - 1, 5));
|
|
16
|
+
}
|
|
17
|
+
function documentPath(doc) {
|
|
18
|
+
return doc && "path" in doc ? doc.path : void 0;
|
|
19
|
+
}
|
|
20
|
+
function safePosition(session) {
|
|
21
|
+
return session.pending.values().next().value?.before ?? session.latest;
|
|
22
|
+
}
|
|
23
|
+
function cancelRetry(session) {
|
|
24
|
+
if (session.retryTimer !== void 0) clearTimeout(session.retryTimer);
|
|
25
|
+
session.retryTimer = void 0;
|
|
26
|
+
}
|
|
27
|
+
function cancelFeed(session) {
|
|
28
|
+
const state = session.owner.host.state;
|
|
29
|
+
const changes = state.changes;
|
|
30
|
+
state.changes = void 0;
|
|
31
|
+
changes?.cancel();
|
|
32
|
+
}
|
|
33
|
+
function stop(session) {
|
|
34
|
+
session.owner.host.state.watching = false;
|
|
35
|
+
session.connected = false;
|
|
36
|
+
session.connection++;
|
|
37
|
+
cancelRetry(session);
|
|
38
|
+
if (session.reconnectTimer !== void 0) clearTimeout(session.reconnectTimer);
|
|
39
|
+
session.reconnectTimer = void 0;
|
|
40
|
+
session.pending.clear();
|
|
41
|
+
session.owner.session = void 0;
|
|
42
|
+
cancelFeed(session);
|
|
43
|
+
}
|
|
44
|
+
function schedule(session) {
|
|
45
|
+
cancelRetry(session);
|
|
46
|
+
if (!isActive(session) || !session.connected) return;
|
|
47
|
+
let next = Infinity;
|
|
48
|
+
if (session.owner.inFlight < 4) {
|
|
49
|
+
for (const item of session.pending.values()) {
|
|
50
|
+
if (!item.running && !session.owner.delivering.has(item.id)) next = Math.min(next, item.due);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (!session.checkpointTask && safePosition(session).order > session.durableOrder) {
|
|
54
|
+
next = Math.min(next, session.checkpointDue);
|
|
55
|
+
}
|
|
56
|
+
if (next !== Infinity) {
|
|
57
|
+
session.retryTimer = compatGlobal.setTimeout(bindContext(session, retry), Math.max(0, next - Date.now()));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function retry(session) {
|
|
61
|
+
session.retryTimer = void 0;
|
|
62
|
+
void pump(session);
|
|
63
|
+
}
|
|
64
|
+
async function persistCheckpoint({ session, position, epoch, previous }) {
|
|
65
|
+
if (previous) await previous;
|
|
66
|
+
if (!isActive(session) || session.connection !== epoch) return;
|
|
67
|
+
try {
|
|
68
|
+
await session.onCheckpoint?.(position.seq);
|
|
69
|
+
if (!isActive(session)) return;
|
|
70
|
+
session.owner.host.state.since = String(position.seq);
|
|
71
|
+
if (session.connection === epoch) session.durableOrder = position.order;
|
|
72
|
+
session.checkpointFailures = 0;
|
|
73
|
+
session.checkpointDue = 0;
|
|
74
|
+
} catch (ex) {
|
|
75
|
+
Logger(`WATCH: CHECKPOINT FAILED`, LEVEL_INFO, "watch");
|
|
76
|
+
Logger(ex, LEVEL_VERBOSE, "watch");
|
|
77
|
+
if (isActive(session) && session.connection === epoch) {
|
|
78
|
+
session.checkpointDue = Date.now() + retryDelay(++session.checkpointFailures);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
function finishCheckpoint({ session, task }) {
|
|
83
|
+
session.checkpointTask = void 0;
|
|
84
|
+
if (session.owner.checkpointPending === task) session.owner.checkpointPending = void 0;
|
|
85
|
+
if (isActive(session)) {
|
|
86
|
+
void flushCheckpoint(session);
|
|
87
|
+
schedule(session);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
function flushCheckpoint(session) {
|
|
91
|
+
if (session.checkpointTask) return session.checkpointTask;
|
|
92
|
+
if (!isActive(session) || !session.connected || Date.now() < session.checkpointDue) return Promise.resolve();
|
|
93
|
+
const position = safePosition(session);
|
|
94
|
+
if (position.order <= session.durableOrder) return Promise.resolve();
|
|
95
|
+
const context = {
|
|
96
|
+
session,
|
|
97
|
+
position,
|
|
98
|
+
epoch: session.connection,
|
|
99
|
+
previous: session.owner.checkpointPending
|
|
100
|
+
};
|
|
101
|
+
const task = persistCheckpoint(context).finally(bindContext(context, finishCheckpoint));
|
|
102
|
+
context.task = task;
|
|
103
|
+
session.checkpointTask = task;
|
|
104
|
+
session.owner.checkpointPending = task;
|
|
105
|
+
return task;
|
|
106
|
+
}
|
|
107
|
+
function releaseDelivery({ session, item }) {
|
|
108
|
+
session.owner.delivering.delete(item.id);
|
|
109
|
+
}
|
|
110
|
+
async function deliver(scope) {
|
|
111
|
+
const { session, item, ready } = scope.value;
|
|
112
|
+
session.owner.delivering.add(item.id);
|
|
113
|
+
scope.defer(bindContext(scope.value, releaseDelivery));
|
|
114
|
+
await session.callback(ready, item.seq);
|
|
115
|
+
}
|
|
116
|
+
async function processDocument(context) {
|
|
117
|
+
const { session, item } = context;
|
|
118
|
+
let loading = true;
|
|
119
|
+
try {
|
|
120
|
+
const doc = item.doc;
|
|
121
|
+
if (isNoteEntry(doc) && (!session.checkIsInterested || session.checkIsInterested(doc))) {
|
|
122
|
+
Logger(`WATCH: PROCESSING: ${doc.path}`, LEVEL_VERBOSE, "watch");
|
|
123
|
+
const ready = doc.deleted || doc._deleted ? { ...doc, data: [] } : await session.owner.host.readDocument(doc);
|
|
124
|
+
if (!isCurrent(context)) return;
|
|
125
|
+
loading = false;
|
|
126
|
+
await withScope({ ...context, ready }, deliver);
|
|
127
|
+
if (!isCurrent(context)) return;
|
|
128
|
+
Logger(`WATCH: PROCESS DONE: ${doc.path}`, LEVEL_INFO, "watch");
|
|
129
|
+
}
|
|
130
|
+
if (!isCurrent(context)) return;
|
|
131
|
+
session.pending.delete(item.id);
|
|
132
|
+
void flushCheckpoint(session);
|
|
133
|
+
} catch (ex) {
|
|
134
|
+
if (!isCurrent(context)) return;
|
|
135
|
+
Logger(
|
|
136
|
+
loading ? `WATCH: DOCUMENT LOAD FAILED: ${documentPath(item.doc)}` : `WATCH: PROCESS FAILED`,
|
|
137
|
+
LEVEL_INFO,
|
|
138
|
+
"watch"
|
|
139
|
+
);
|
|
140
|
+
Logger(ex, LEVEL_VERBOSE, "watch");
|
|
141
|
+
item.due = Date.now() + retryDelay(++item.failures);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function releaseFileOperation({ session, item }) {
|
|
145
|
+
item.running = false;
|
|
146
|
+
session.owner.inFlight--;
|
|
147
|
+
if (session.owner.session) void pump(session.owner.session);
|
|
148
|
+
}
|
|
149
|
+
async function processFileOperation(scope) {
|
|
150
|
+
scope.defer(bindContext(scope.value, releaseFileOperation));
|
|
151
|
+
await processDocument(scope.value);
|
|
152
|
+
}
|
|
153
|
+
function pump(session) {
|
|
154
|
+
if (!isActive(session) || !session.connected) return Promise.resolve();
|
|
155
|
+
const tasks = [];
|
|
156
|
+
for (const item of session.pending.values()) {
|
|
157
|
+
if (session.owner.inFlight >= 4) break;
|
|
158
|
+
if (item.running || item.due > Date.now() || session.owner.delivering.has(item.id)) continue;
|
|
159
|
+
item.running = true;
|
|
160
|
+
session.owner.inFlight++;
|
|
161
|
+
tasks.push(withScope({ session, item, epoch: session.connection }, processFileOperation));
|
|
162
|
+
}
|
|
163
|
+
void flushCheckpoint(session);
|
|
164
|
+
schedule(session);
|
|
165
|
+
return Promise.all(tasks).then(() => {
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
async function reconnectAfterCheckpoint(session) {
|
|
169
|
+
session.reconnectTimer = void 0;
|
|
170
|
+
if (session.checkpointTask) await session.checkpointTask;
|
|
171
|
+
if (isActive(session)) connect(session);
|
|
172
|
+
}
|
|
173
|
+
function reconnect({ session, epoch }) {
|
|
174
|
+
if (!isActive(session) || !session.connected || session.connection !== epoch) return;
|
|
175
|
+
session.owner.host.state.watching = false;
|
|
176
|
+
session.connected = false;
|
|
177
|
+
session.connection++;
|
|
178
|
+
cancelRetry(session);
|
|
179
|
+
session.pending.clear();
|
|
180
|
+
cancelFeed(session);
|
|
181
|
+
Logger(`WATCH: CONNECTION HAS BEEN CLOSED, RECONNECTING...`, LEVEL_INFO, "watch");
|
|
182
|
+
session.reconnectTimer = compatGlobal.setTimeout(bindContext(session, reconnectAfterCheckpoint), 1e4);
|
|
183
|
+
}
|
|
184
|
+
async function receiveChange({ session, epoch }, change) {
|
|
185
|
+
if (!isActive(session) || !session.connected || session.connection !== epoch) return;
|
|
186
|
+
const before = session.latest;
|
|
187
|
+
session.latest = { order: before.order + 1, seq: change.seq };
|
|
188
|
+
const id = change.id ?? change.doc?._id ?? documentPath(change.doc) ?? `row:${session.latest.order}`;
|
|
189
|
+
const previous = session.pending.get(id);
|
|
190
|
+
const doc = change.doc?._deleted && previous?.doc ? { ...previous.doc, ...change.doc } : change.doc;
|
|
191
|
+
session.pending.set(id, {
|
|
192
|
+
id,
|
|
193
|
+
doc,
|
|
194
|
+
seq: change.seq,
|
|
195
|
+
before: previous?.before ?? before,
|
|
196
|
+
running: false,
|
|
197
|
+
failures: 0,
|
|
198
|
+
due: 0
|
|
199
|
+
});
|
|
200
|
+
await pump(session);
|
|
201
|
+
}
|
|
202
|
+
function receiveError(context, error) {
|
|
203
|
+
if (!isActive(context.session) || context.session.connection !== context.epoch) return;
|
|
204
|
+
Logger(`WATCH: ERROR: `, LEVEL_INFO, "watch");
|
|
205
|
+
Logger(error, LEVEL_VERBOSE, "watch");
|
|
206
|
+
reconnect(context);
|
|
207
|
+
}
|
|
208
|
+
function connect(session) {
|
|
209
|
+
if (!isActive(session)) return;
|
|
210
|
+
const context = { session, epoch: ++session.connection };
|
|
211
|
+
const { state } = session.owner.host;
|
|
212
|
+
state.watching = true;
|
|
213
|
+
session.connected = true;
|
|
214
|
+
session.latest = { order: 0, seq: state.since };
|
|
215
|
+
session.durableOrder = 0;
|
|
216
|
+
session.checkpointFailures = 0;
|
|
217
|
+
session.checkpointDue = 0;
|
|
218
|
+
try {
|
|
219
|
+
state.changes = session.owner.host.openChanges(state.since).on("change", bindContext(context, receiveChange)).on("complete", bindContext(context, reconnect)).on("error", bindContext(context, receiveError));
|
|
220
|
+
} catch (ex) {
|
|
221
|
+
Logger(`WATCH: ERROR: `, LEVEL_INFO, "watch");
|
|
222
|
+
Logger(ex, LEVEL_VERBOSE, "watch");
|
|
223
|
+
reconnect(context);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
class DirectFileWatchOwner {
|
|
227
|
+
context;
|
|
228
|
+
constructor(host) {
|
|
229
|
+
this.context = { host, inFlight: 0, delivering: /* @__PURE__ */ new Set() };
|
|
230
|
+
}
|
|
231
|
+
beginWatch(callback, checkIsInterested, onCheckpoint) {
|
|
232
|
+
if (this.context.session || this.context.host.state.watching) return false;
|
|
233
|
+
const session = {
|
|
234
|
+
owner: this.context,
|
|
235
|
+
callback,
|
|
236
|
+
checkIsInterested,
|
|
237
|
+
onCheckpoint,
|
|
238
|
+
pending: /* @__PURE__ */ new Map(),
|
|
239
|
+
latest: { order: 0, seq: this.context.host.state.since },
|
|
240
|
+
durableOrder: 0,
|
|
241
|
+
connection: 0,
|
|
242
|
+
connected: false,
|
|
243
|
+
checkpointFailures: 0,
|
|
244
|
+
checkpointDue: 0
|
|
245
|
+
};
|
|
246
|
+
this.context.session = session;
|
|
247
|
+
connect(session);
|
|
248
|
+
}
|
|
249
|
+
endWatch() {
|
|
250
|
+
if (this.context.session) stop(this.context.session);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
export {
|
|
254
|
+
DirectFileWatchOwner,
|
|
255
|
+
isNoteEntry
|
|
256
|
+
};
|
|
@@ -10,11 +10,11 @@ import {
|
|
|
10
10
|
import { decryptString, encryptString } from "../encryption/stringEncryption.js";
|
|
11
11
|
import { LOG_LEVEL_VERBOSE, Logger } from "octagonal-wheels/common/logger";
|
|
12
12
|
import { configuredIdKey } from "../common/idDerivation.js";
|
|
13
|
-
import { getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
13
|
+
import { assertSupportedPathNormalizationVersion, getPathNormalizationVersion } from "../common/pathNormalization.js";
|
|
14
14
|
function encodeSettingsToQRCodeData(settings) {
|
|
15
15
|
settings = omitP2PRuntimeSettings(settings);
|
|
16
16
|
configuredIdKey(settings);
|
|
17
|
-
|
|
17
|
+
assertSupportedPathNormalizationVersion(settings.pathNormalizationVersion);
|
|
18
18
|
const fullIndexes = Object.entries(KeyIndexOfSettings);
|
|
19
19
|
let maxIndex = 0;
|
|
20
20
|
for (const [, index] of fullIndexes) {
|
|
@@ -137,7 +137,7 @@ async function encodeSettingsToSetupURI(settingString, passphrase, removePropert
|
|
|
137
137
|
const setting = {
|
|
138
138
|
...omitP2PRuntimeSettings(settingString)
|
|
139
139
|
};
|
|
140
|
-
|
|
140
|
+
assertSupportedPathNormalizationVersion(setting.pathNormalizationVersion);
|
|
141
141
|
configuredIdKey({
|
|
142
142
|
idDerivationVersion: setting.idDerivationVersion ?? 0,
|
|
143
143
|
idDerivationKey: setting.idDerivationKey ?? ""
|
|
@@ -557,7 +557,7 @@ export interface P2PConnectionInfo {
|
|
|
557
557
|
*/
|
|
558
558
|
P2P_useDiagRTC?: boolean;
|
|
559
559
|
}
|
|
560
|
-
export interface P2PSyncSetting extends P2PConnectionInfo {
|
|
560
|
+
export interface P2PSyncSetting extends P2PConnectionInfo, Partial<PathIdentitySettings> {
|
|
561
561
|
P2P_AutoAccepting: AutoAccepting;
|
|
562
562
|
P2P_AutoSyncPeers: string;
|
|
563
563
|
P2P_AutoWatchPeers: string;
|
|
@@ -66,8 +66,7 @@ const configurationNames = {
|
|
|
66
66
|
},
|
|
67
67
|
pathNormalizationVersion: {
|
|
68
68
|
name: "Path identity mode",
|
|
69
|
-
desc: "
|
|
70
|
-
isAdvanced: true
|
|
69
|
+
desc: "Version 0 preserves legacy path identity; version 1 uses Unicode NFC. Physical file names stay unchanged. Update every device and stop synchronisation, then Rebuild from a complete authoritative Vault and Fetch on the other devices. With P2P, Rebuild prepares only the source device's local database. Resolve colliding file names first."
|
|
71
70
|
},
|
|
72
71
|
doNotUseFixedRevisionForChunks: {
|
|
73
72
|
name: "Content-derived chunk revisions (obsolete setting)",
|
|
@@ -120,8 +120,8 @@ export declare const TweakValuesRecommendedTemplate: Partial<ObsidianLiveSyncSet
|
|
|
120
120
|
export declare const TweakValuesDefault: Partial<ObsidianLiveSyncSettings>;
|
|
121
121
|
export declare const TweakValuesTemplate: {
|
|
122
122
|
tweakModified: number;
|
|
123
|
-
usePluginSyncV2: boolean;
|
|
124
123
|
pathNormalizationVersion: import("./setting.type.js").PathNormalizationVersion;
|
|
124
|
+
usePluginSyncV2: boolean;
|
|
125
125
|
encrypt: boolean;
|
|
126
126
|
idDerivationVersion: 0 | 1;
|
|
127
127
|
usePathObfuscation: boolean;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { PathNormalizationVersion } from "./models/setting.type.js";
|
|
2
|
+
/** Reject unsupported path identity versions while allowing an absent legacy value. */
|
|
3
|
+
export declare function assertSupportedPathNormalizationVersion(version: number | undefined): asserts version is PathNormalizationVersion | undefined;
|
|
2
4
|
/** Resolve the stored path identity version, treating an absent legacy value as version zero. */
|
|
3
5
|
export declare function getPathNormalizationVersion(settings: {
|
|
4
6
|
pathNormalizationVersion?: number;
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
function assertSupportedPathNormalizationVersion(version) {
|
|
2
|
+
if (version === void 0 || version === 0 || version === 1) return;
|
|
3
|
+
throw new Error(`Unsupported path normalisation version: ${version}`);
|
|
4
|
+
}
|
|
1
5
|
function getPathNormalizationVersion(settings) {
|
|
2
6
|
const version = settings.pathNormalizationVersion === void 0 ? 0 : settings.pathNormalizationVersion;
|
|
3
|
-
|
|
4
|
-
|
|
7
|
+
assertSupportedPathNormalizationVersion(version);
|
|
8
|
+
return version;
|
|
5
9
|
}
|
|
6
10
|
function normalisePathForIdentity(path, version) {
|
|
7
11
|
const resolvedVersion = getPathNormalizationVersion({ pathNormalizationVersion: version });
|
|
8
12
|
return resolvedVersion === 1 ? path.normalize("NFC") : path;
|
|
9
13
|
}
|
|
10
14
|
export {
|
|
15
|
+
assertSupportedPathNormalizationVersion,
|
|
11
16
|
getPathNormalizationVersion,
|
|
12
17
|
normalisePathForIdentity
|
|
13
18
|
};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type DocumentID } from "../common/types.js";
|
|
2
2
|
import { type ChunkManager } from "./ChunkManager.js";
|
|
3
|
+
import type { ChunkFetchSource } from "./chunkFetchSource.js";
|
|
3
4
|
import type { IReplicatorService, ISettingService } from "../services/base/IService.js";
|
|
4
5
|
export declare const EVENT_MISSING_CHUNKS = "missingChunks";
|
|
5
6
|
export declare const EVENT_MISSING_CHUNK_REMOTE = "missingChunkRemote";
|
|
@@ -9,6 +10,8 @@ export type ChunkFetcherOptions = {
|
|
|
9
10
|
chunkManager: ChunkManager;
|
|
10
11
|
replicatorService: IReplicatorService;
|
|
11
12
|
deliveryStallTimeoutMs?: number;
|
|
13
|
+
/** Direct hosts replace both retrieval and acceptance; the default uses the active replicator. */
|
|
14
|
+
source?: ChunkFetchSource;
|
|
12
15
|
};
|
|
13
16
|
export declare class ChunkFetcher {
|
|
14
17
|
options: ChunkFetcherOptions;
|