@vrtmrz/livesync-commonlib 0.2.3 → 0.3.0

package/dist/managers/ChunkFetcher.js

@@ -327,20 +327,25 @@ class ChunkFetcher {
       await this.resolveLocallyAvailableRetries(requestClaims);
       let pendingIDs = await this.waitBeforeRemoteRequest(requestIDs, requestClaims, localCheckCompletionVersion);
       if (pendingIDs.length === 0) return;
-      const replicator = this.options.replicatorService.getActiveReplicator();
-      if (!replicator) {
-        Logger("No active replicator was found to request missing chunks.");
-        return;
-      }
-      if (!canFetchRemoteChunks(replicator)) {
-        Logger("The active replicator does not support fetching individual remote chunks.");
-        for (const chunkID of pendingIDs) {
-          this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
+      const requestCompletionVersion = this.finiteCompletionVersion;
+      let fetched;
+      if (this.options.source) {
+        fetched = await this.options.source.fetchChunks(pendingIDs);
+      } else {
+        const replicator = this.options.replicatorService.getActiveReplicator();
+        if (!replicator) {
+          Logger("No active replicator was found to request missing chunks.");
+          return;
         }
-        return;
+        if (!canFetchRemoteChunks(replicator)) {
+          Logger("The active replicator does not support fetching individual remote chunks.");
+          for (const chunkID of pendingIDs) {
+            this.chunkManager.emitEvent(EVENT_MISSING_CHUNK_REMOTE, chunkID);
+          }
+          return;
+        }
+        fetched = await replicator.fetchRemoteChunks(pendingIDs, false);
       }
-      const requestCompletionVersion = this.finiteCompletionVersion;
-      const fetched = await replicator.fetchRemoteChunks(pendingIDs, false);
       pendingIDs = this.getActiveClaimIds(pendingIDs, requestClaims);
       this.touchClaims(pendingIDs, requestClaims);
       if (!fetched) {
@@ -354,7 +359,9 @@ class ChunkFetcher {
       this.logFetchedChunks(pendingIDs, classification);
       const { chunks, missingIds, invalidIds } = classification;
       try {
-        if (chunks.length > 0) {
+        if (chunks.length > 0 && this.options.source) {
+          await this.options.source.storeChunks(chunks);
+        } else if (chunks.length > 0) {
           Logger(`Writing fetched chunks (${chunks.length}) to the database...`);
           const result = await this.chunkManager.write(
             chunks,

package/dist/managers/LiveSyncManagers.d.ts

@@ -2,6 +2,7 @@ import { type EntryDoc } from "../common/types.js";
 import { ContentSplitter } from "../ContentSplitter/ContentSplitters.js";
 import { ChangeManager } from "./ChangeManager.js";
 import { ChunkFetcher } from "./ChunkFetcher.js";
+import type { ChunkFetchSource } from "./chunkFetchSource.js";
 import { ChunkManager } from "./ChunkManager.js";
 import { ConflictManager } from "./ConflictManager.js";
 import { EntryManager } from "./EntryManager/EntryManager.js";
@@ -15,6 +16,7 @@ export interface LiveSyncManagersOptions<TSettingService extends ISettingService
     pathService: IPathService;
     replicatorService: IReplicatorService;
     APIService: IAPIService;
+    chunkFetchSource?: ChunkFetchSource;
 }
 export declare class LiveSyncManagers {
     protected _pathService: IPathService;

package/dist/managers/LiveSyncManagers.js

@@ -76,6 +76,7 @@ class LiveSyncManagers {
     });
     const chunkFetcher = new ChunkFetcher({
       chunkManager,
+      source: this.options.chunkFetchSource,
       replicatorService: this.options.replicatorService,
       settingService: this.options.settingService
     });

package/dist/managers/chunkFetchSource.d.ts

@@ -0,0 +1,14 @@
+import type { DocumentID, EntryDoc, EntryLeaf } from "../common/types.js";
+/** IO supplied by a host whose chunk source does not use the active replicator. */
+export interface ChunkFetchSource {
+    /** Missing identifiers are omitted; false means that this request could not retrieve chunks. */
+    fetchChunks(ids: DocumentID[]): Promise<EntryLeaf[] | false>;
+    /** Accept fetched chunks before notifying waiting readers. */
+    storeChunks(chunks: EntryLeaf[]): Promise<void>;
+}
+/**
+ * Read through the prepared direct database, including its decryption transforms.
+ * The source already owns these documents, so accepting them performs no write.
+ * Construct a new source for each database lifetime.
+ */
+export declare function createDirectChunkFetchSource(database: PouchDB.Database<EntryDoc>): ChunkFetchSource;

package/dist/managers/chunkFetchSource.js

@@ -0,0 +1,12 @@
+function createDirectChunkFetchSource(database) {
+  return {
+    async fetchChunks(ids) {
+      const result = await database.allDocs({ keys: ids, include_docs: true });
+      return result.rows.flatMap((row) => "doc" in row && row.doc ? [row.doc] : []);
+    },
+    storeChunks: () => Promise.resolve()
+  };
+}
+export {
+  createDirectChunkFetchSource
+};

package/dist/pouchdb/LiveSyncDBFunctions.js

@@ -6,6 +6,7 @@ import {
 } from "../common/types.js";
 import { extractObject, isObjectDifferent, resolveWithIgnoreKnownError } from "../common/utils.js";
 import { assessTweakCompatibility } from "../common/models/tweak.compatibility.js";
+import { getPathNormalizationVersion } from "../common/pathNormalization.js";
 import { usesEncryptedInternalMetadata } from "./remoteFeatureCompatibility.js";
 function getEffectiveTweakValues(setting) {
   return {
@@ -56,6 +57,10 @@ async function ensureRemoteIsCompatible(infoSrc, setting, deviceNodeID, currentV
   if (tweakAssessment.entries.some(({ key, relation }) => key === "idDerivationVersion" && relation === "different")) {
     return "ID_KEY_MISMATCH";
   }
+  if (getPathNormalizationVersion(currentTweakValues) !== getPathNormalizationVersion(preferred_tweak)) {
+    recordTweakAssessment?.(tweakAssessment);
+    return ["MISMATCHED", preferred_tweak];
+  }
   if (!setting.disableCheckingConfigMismatch) {
     recordTweakAssessment?.(tweakAssessment);
     if (tweakAssessment.alignment === "mismatched") {

package/dist/pouchdb/LiveSyncLocalDB.d.ts

@@ -1,6 +1,7 @@
 import { type EntryDoc, type EntryLeaf, type Credential, type RemoteDBSettings, type DocumentID, type FilePathWithPrefix, type FilePath, type DatabaseEntry, type LoadedEntry, type MetaEntry, type SavingEntry, type diff_result_leaf } from "../common/types.js";
 import type { LiveSyncEventHub } from "../hub/hub.js";
 import { LiveSyncManagers } from "../managers/LiveSyncManagers.js";
+import type { ChunkFetchSource } from "../managers/chunkFetchSource.js";
 import type { AutoMergeResult } from "../managers/ConflictManager.js";
 import type { IServiceHub } from "../services/base/IService.js";
 import type { ServiceContext } from "../services/base/ServiceBase.js";
@@ -33,7 +34,12 @@ export type GeneratedChunk = {
     id: DocumentID;
     piece: string;
 };
+/** Optional host IO bound afresh to each prepared database instance. */
+export interface LiveSyncLocalDBRuntimeOptions {
+    createChunkFetchSource?: (database: PouchDB.Database<EntryDoc>) => ChunkFetchSource;
+}
 export declare class LiveSyncLocalDB {
+    private readonly runtimeOptions;
     auth: Credential;
     dbname: string;
     settings: RemoteDBSettings;
@@ -52,7 +58,7 @@ export declare class LiveSyncLocalDB {
     /** Share owner retirement across every close path for one physical database lifetime. */
     private activeReplicationClose?;
     private databaseUnloadNotification?;
-    constructor(dbname: string, env: LiveSyncLocalDBEnv);
+    constructor(dbname: string, env: LiveSyncLocalDBEnv, runtimeOptions?: LiveSyncLocalDBRuntimeOptions);
     /** Notify dependent services at most once for each physical database lifecycle. */
     private notifyDatabaseUnload;
     onunload(): void;

package/dist/pouchdb/LiveSyncLocalDB.js

@@ -16,6 +16,18 @@ function getNoFromRev(rev) {
   return parseInt(rev.split("-")[0]);
 }
 class LiveSyncLocalDB {
+  constructor(dbname, env, runtimeOptions = {}) {
+    this.runtimeOptions = runtimeOptions;
+    this.auth = {
+      username: "",
+      password: ""
+    };
+    this.dbname = dbname;
+    this.env = env;
+    this._log = createInstanceLogFunction("LiveSyncLocalDB", this.env.services.API);
+    this.refreshSettings();
+  }
+  runtimeOptions;
   auth;
   dbname;
   settings;
@@ -47,16 +59,6 @@ class LiveSyncLocalDB {
   /** Share owner retirement across every close path for one physical database lifetime. */
   activeReplicationClose;
   databaseUnloadNotification;
-  constructor(dbname, env) {
-    this.auth = {
-      username: "",
-      password: ""
-    };
-    this.dbname = dbname;
-    this.env = env;
-    this._log = createInstanceLogFunction("LiveSyncLocalDB", this.env.services.API);
-    this.refreshSettings();
-  }
   /** Notify dependent services at most once for each physical database lifecycle. */
   notifyDatabaseUnload() {
     if (!this.databaseUnloadNotification) {
@@ -170,6 +172,7 @@ class LiveSyncLocalDB {
     this.databaseUnloadNotification = void 0;
     const manager = new LiveSyncManagers({
       database: this.localDatabase,
+      chunkFetchSource: this.runtimeOptions?.createChunkFetchSource?.(this.localDatabase),
       APIService: this.env.services.API,
       pathService: this.env.services.path,
       replicatorService: this.env.services.replicator,

package/dist/pouchdb/pathIdentityCompatibility.d.ts

@@ -2,6 +2,14 @@ import { type EntryDoc } from "../common/types.js";
 type PathIdentitySettings = {
     pathNormalizationVersion?: number;
 };
+export type PathIdentityInspection = {
+    readonly status: "empty";
+} | {
+    readonly status: "configured";
+    readonly pathNormalizationVersion: 0 | 1;
+};
+/** Read the existing declaration without changing a remote or declaring a new identity. */
+export declare function inspectDatabasePathIdentity(database: PouchDB.Database<EntryDoc>): Promise<PathIdentityInspection>;
 export declare function assertMatchingPathIdentityFeature(document: unknown, settings: PathIdentitySettings): void;
 /** Refuse an implicit migration. A fresh database may declare its identity before its first Metadata write. */
 export declare function assertDatabasePathIdentity(database: PouchDB.Database<EntryDoc>, settings: PathIdentitySettings, declare?: boolean): Promise<void>;

package/dist/pouchdb/pathIdentityCompatibility.js

@@ -2,11 +2,28 @@ import { VERSIONING_DOCID } from "../common/types.js";
 import { getPathNormalizationVersion } from "../common/pathNormalization.js";
 import {
   assessRemoteFeatureDocument,
+  describeRemoteFeatureRejection,
   NFC_PATH_IDENTITY_FEATURE,
   REMOTE_FEATURE_GENERATION
 } from "./remoteFeatureCompatibility.js";
 import { isErrorOfMissingDoc } from "./utils_couchdb.js";
 import { bumpRemoteVersion } from "./negotiation.js";
+async function inspectDatabasePathIdentity(database) {
+  let document;
+  try {
+    document = await database.get(VERSIONING_DOCID);
+  } catch (error) {
+    if (!isErrorOfMissingDoc(error)) throw error;
+    return (await database.info()).doc_count === 0 ? { status: "empty" } : { status: "configured", pathNormalizationVersion: 0 };
+  }
+  const assessment = assessRemoteFeatureDocument(document);
+  if (assessment.status === "older-generation") return { status: "configured", pathNormalizationVersion: 0 };
+  if (assessment.status !== "supported") throw new Error(describeRemoteFeatureRejection(assessment));
+  return {
+    status: "configured",
+    pathNormalizationVersion: assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE) ? 1 : 0
+  };
+}
 function assertMatchingPathIdentityFeature(document, settings) {
   const assessment = assessRemoteFeatureDocument(document);
   const declared = assessment.status === "supported" && assessment.usedFeatures.includes(NFC_PATH_IDENTITY_FEATURE);
@@ -60,5 +77,6 @@ async function assertDatabasePathIdentity(database, settings, declare = false) {
 }
 export {
   assertDatabasePathIdentity,
-  assertMatchingPathIdentityFeature
+  assertMatchingPathIdentityFeature,
+  inspectDatabasePathIdentity
 };

package/dist/replication/RemoteResource.d.ts

@@ -1,3 +1,4 @@
+import type { PathIdentityInspection } from "../pouchdb/pathIdentityCompatibility.js";
 import type { RemoteDBSettings, RemotePreferredTweakResult } from "../common/types.js";
 import type { JoinedDevicesSnapshot, ForgetJoinedDeviceResult } from "./joinedDevices.js";
 import type { RemoteDBStatus } from "./LiveSyncAbstractReplicator.js";
@@ -36,6 +37,8 @@ export type RemoteConnectionProbeResult = {
 export interface RemoteConnectionProbe {
     check(options?: RemoteConnectionProbeOptions): Promise<RemoteConnectionProbeResult>;
     getStatus(): Promise<false | RemoteDBStatus>;
+    /** Inspect a central remote before choosing initial host settings. */
+    inspectPathIdentity?(): Promise<PathIdentityInspection>;
     dispose(): Promise<void>;
 }
 /** A finite preferred-tweak reader bound to one trial configuration. */

package/dist/replication/index.d.ts

@@ -13,3 +13,5 @@ export { assessRemoteFeatureDocument, describeRemoteFeatureRejection, ENCRYPTED_
 export type { RemoteFeatureAssessment } from "../pouchdb/remoteFeatureCompatibility.js";
 export type { JoinedDevice, JoinedDevicesSnapshot, ForgetJoinedDeviceResult } from "./joinedDevices.js";
 export type { ReplicationStatics, CouchDBReplicationPhase, CouchDBReplicationProgress, ReplicationCheckpointProgress, } from "../common/models/shared.definition.js";
+export { inspectDatabasePathIdentity } from "../pouchdb/pathIdentityCompatibility.js";
+export type { PathIdentityInspection } from "../pouchdb/pathIdentityCompatibility.js";

package/dist/replication/index.js

@@ -43,6 +43,7 @@ import {
   REMOTE_FEATURE_GENERATION,
   usesEncryptedInternalMetadata
 } from "../pouchdb/remoteFeatureCompatibility.js";
+import { inspectDatabasePathIdentity } from "../pouchdb/pathIdentityCompatibility.js";
 export {
   CAPABILITY_NOT_APPLICABLE,
   CAPABILITY_NOT_IMPLEMENTED,
@@ -72,6 +73,7 @@ export {
   centralRemoteAdministrationVerified,
   defineReplicatorProviderDefinitions,
   describeRemoteFeatureRejection,
+  inspectDatabasePathIdentity,
   isCentralRemoteAdministrationVerified,
   isReplicationCompleted,
   milestoneSatisfiesCentralRemoteAdministration,

package/dist/replication/journal/JournalSyncCore.d.ts

@@ -62,6 +62,7 @@ export declare class JournalSyncCore {
     private _createSendCompressTransformStream;
     private _createSendUploadWritableStream;
     sendLocalJournal(showMessage?: boolean): Promise<boolean>;
+    hasRemoteJournals(): Promise<boolean>;
     _getRemoteJournals(): Promise<string[]>;
     processDocuments(allDocs: ProcessingEntry[]): Promise<boolean>;
     private _createReceiveReadableStream;

package/dist/replication/journal/JournalSyncCore.js

@@ -33,6 +33,7 @@ import {
   encryptBinary as encryptBinaryHKDF,
   decryptBinary as decryptBinaryHKDF
 } from "octagonal-wheels/encryption/hkdf";
+import { assertDatabasePathIdentity } from "../../pouchdb/pathIdentityCompatibility.js";
 const RECORD_SPLIT = `
 `;
 const UNIT_SPLIT = ``;
@@ -45,6 +46,9 @@ function serializeDoc(doc) {
   }
   return te.encode(JSON.stringify(doc) + RECORD_SPLIT);
 }
+function isJournalKey(key) {
+  return !key.startsWith("_");
+}
 class JournalSyncCore {
   _settings;
   storage;
@@ -217,7 +221,13 @@ class JournalSyncCore {
     this.syncParamsHandler = void 0;
   }
   async resetCheckpointInfo() {
-    await this.updateCheckPointInfo((info) => ({ ...CheckPointInfoDefault }));
+    await this.updateCheckPointInfo(() => ({
+      ...CheckPointInfoDefault,
+      knownIDs: /* @__PURE__ */ new Set(),
+      sentIDs: /* @__PURE__ */ new Set(),
+      receivedFiles: /* @__PURE__ */ new Set(),
+      sentFiles: /* @__PURE__ */ new Set()
+    }));
   }
   getJournalEpochFromSyncParams(params) {
     return `${params.protocolVersion}:${params.pbkdf2salt}`;
@@ -262,32 +272,33 @@ class JournalSyncCore {
   }
   async resetBucket() {
     this.resetAllCaches();
-    let files = [];
     try {
+      const skippedFiles = [];
+      let from = "";
+      let files;
       do {
-        files = await this.storage.listFiles("", 100);
-        if (files.length == 0) {
-          break;
-        }
-        if (!await this.storage.deleteFiles(files)) {
-          return false;
+        files = await this.storage.listFiles(from, 100);
+        if (files.length > 0) {
+          const nextFrom = files[files.length - 1];
+          if (nextFrom <= from) {
+            throw new Error("Journal reset listing did not advance. Control documents have been retained.");
+          }
+          skippedFiles.push(...files.filter((key) => !isJournalKey(key)));
+          const journals = files.filter(isJournalKey);
+          if (journals.length > 0 && !await this.storage.deleteFiles(journals)) return false;
+          from = nextFrom;
         }
-      } while (files.length != 0);
+      } while (files.length > 0);
+      while (skippedFiles.length > 0) {
+        if (!await this.storage.deleteFiles(skippedFiles.splice(0, 100))) return false;
+      }
     } catch (ex) {
       Logger(`WARNING! Could not delete files.`, LOG_LEVEL_NOTICE, "reset-bucket");
       Logger(ex, LOG_LEVEL_VERBOSE);
       return false;
     }
-    const journals = await this._getRemoteJournals();
-    if (journals.length == 0) {
-      Logger("Nothing to delete!", LOG_LEVEL_NOTICE);
-      return true;
-    }
-    if (!await this.storage.deleteFiles(journals)) {
-      return false;
-    }
-    Logger(`${journals.length} items has been deleted!`, LOG_LEVEL_NOTICE);
     await this.resetCheckpointInfo();
+    Logger("Remote bucket cleared", LOG_LEVEL_NOTICE);
     return true;
   }
   getRemoteKey() {
@@ -502,6 +513,7 @@ class JournalSyncCore {
     });
   }
   async sendLocalJournal(showMessage = false) {
+    await assertDatabasePathIdentity(this.db, this.currentSettings);
     this.updateInfo({ syncStatus: "JOURNAL_SEND" });
     return await shareRunningResult("send_journal_stream", async () => {
       this.requestedStop = false;
@@ -544,12 +556,15 @@ class JournalSyncCore {
       }
     });
   }
+  async hasRemoteJournals() {
+    return (await this.storage.listFiles("", 1)).some(isJournalKey);
+  }
   async _getRemoteJournals() {
     const checkPointInfo = await this.getCheckpointInfo();
     const StartAfter = [...checkPointInfo.receivedFiles.keys()].sort(
       (a, b) => b.localeCompare(a, void 0, { numeric: true })
     )[0];
-    const files = (await this.storage.listFiles(StartAfter)).filter((e) => !e.startsWith("_"));
+    const files = (await this.storage.listFiles(StartAfter)).filter(isJournalKey);
     if (!files) return [];
     return files.sort((a, b) => a.localeCompare(b, void 0, { numeric: true }));
   }
@@ -723,6 +738,7 @@ class JournalSyncCore {
     });
   }
   async receiveRemoteJournal(showMessage = false) {
+    await assertDatabasePathIdentity(this.db, this.currentSettings);
     this.updateInfo({ syncStatus: "JOURNAL_RECEIVE" });
     return await shareRunningResult("receive_journal_stream", async () => {
       this.requestedStop = false;

package/dist/replication/journal/LiveSyncJournalReplicator.d.ts

@@ -1,3 +1,4 @@
+import type { PathIdentityInspection } from "../../pouchdb/pathIdentityCompatibility.js";
 import { type RemoteDBSettings, type ChunkVersionRange, type NodeData, type RemotePreferredTweakResult, type TweakAssessment } from "../../common/types.js";
 import { JournalSyncCore } from "./JournalSyncCore.js";
 import { LiveSyncAbstractReplicator, type RemoteDBStatus } from "../LiveSyncAbstractReplicator.js";
@@ -23,7 +24,16 @@ export declare class LiveSyncJournalReplicator extends LiveSyncAbstractReplicato
     private journalTransferStopGeneration;
     getReplicationPBKDF2Salt(setting: RemoteDBSettings, refresh?: boolean): Promise<Uint8Array<ArrayBuffer>>;
     setupJournalSyncClient(settings?: RemoteDBSettings): JournalSyncCore;
-    ensureBucketIsCompatible(deviceNodeID: string, currentVersionRange: ChunkVersionRange, setting?: RemoteDBSettings, client?: Pick<JournalSyncCore, "downloadJsonWithResult" | "getCheckpointInfo" | "uploadJson">, recordTweakAssessment?: (assessment: TweakAssessment) => void): Promise<ENSURE_DB_RESULT>;
+    private isBucketPathIdentityCompatible;
+    private isBucketIdDerivationCompatible;
+    private assertBucketIdentity;
+    /** Inspect identity without publishing this node, preparing a seed, or changing checkpoints. */
+    assertRemotePathIdentity(setting: RemoteDBSettings): Promise<void>;
+    /** Read path identity for initial setup without publishing a node or preparing a seed. */
+    inspectRemotePathIdentity(setting: RemoteDBSettings): Promise<PathIdentityInspection>;
+    /** Lock existing journals for replacement without declaring the replacement's identity. */
+    markRemoteLockedForRebuild(setting: RemoteDBSettings): Promise<void>;
+    ensureBucketIsCompatible(deviceNodeID: string, currentVersionRange: ChunkVersionRange, setting?: RemoteDBSettings, client?: Pick<JournalSyncCore, "downloadJsonWithResult" | "getCheckpointInfo" | "uploadJson" | "hasRemoteJournals">, recordTweakAssessment?: (assessment: TweakAssessment) => void): Promise<ENSURE_DB_RESULT>;
     constructor(env: LiveSyncJournalReplicatorEnv);
     migrate(from: number, to: number): Promise<boolean>;
     /**
@@ -62,8 +72,6 @@ export declare class LiveSyncJournalReplicator extends LiveSyncAbstractReplicato
     setPreferredRemoteTweakSettings(setting: RemoteDBSettings): Promise<void>;
     getRemotePreferredTweakValues(setting: RemoteDBSettings): Promise<RemotePreferredTweakResult>;
     getRemoteStatus(setting: RemoteDBSettings): Promise<false | RemoteDBStatus>;
-    private supportsLegacyPathIdentity;
-    private assertLegacyPathIdentity;
     private getOwnJoinedDeviceId;
     getJoinedDevices(setting: RemoteDBSettings): Promise<JoinedDevicesSnapshot>;
     forgetJoinedDevice(setting: RemoteDBSettings, deviceId: string): Promise<ForgetJoinedDeviceResult>;