@vrtmrz/livesync-commonlib 0.1.0-rc.0

package/dist/serviceModules/Rebuilder.js

@@ -0,0 +1,449 @@
+import { FlagFilesHumanReadable } from "../common/models/redflag.const.js";
+import { REMOTE_COUCHDB, REMOTE_MINIO } from "../common/models/setting.const.js";
+import { DEFAULT_SETTINGS } from "../common/models/setting.const.defaults.js";
+import { LOG_LEVEL_NOTICE, LOG_LEVEL_VERBOSE } from "octagonal-wheels/common/logger";
+import { delay } from "octagonal-wheels/promises";
+import { EVENT_DATABASE_REBUILT } from "../events/coreEvents.js";
+import { ServiceModuleBase } from "./ServiceModuleBase.js";
+import { fetchChangesForInitialSync } from "../pouchdb/StreamingFetch.js";
+import { getConfiguredFunctionsForEncryption } from "../pouchdb/encryption.js";
+import { AuthorizationHeaderGenerator, generateCredentialObject } from "../replication/httplib.js";
+import { sizeToHumanReadable } from "octagonal-wheels/number";
+const FAST_FETCH_CHECKPOINT_KEY = "fast-fetch-checkpoint";
+const FAST_FETCH_RETRY_DELAYS = [2e3, 5e3, 1e4, 2e4];
+class ServiceRebuilder extends ServiceModuleBase {
+  events;
+  appLifecycle;
+  API;
+  UI;
+  setting;
+  remote;
+  databaseEvents;
+  storageAccess;
+  replicator;
+  vault;
+  replication;
+  database;
+  fileHandler;
+  control;
+  constructor(services) {
+    super(services);
+    this.events = services.events;
+    this.appLifecycle = services.appLifecycle;
+    this.API = services.API;
+    this.UI = services.UI;
+    this.setting = services.setting;
+    this.remote = services.remote;
+    this.databaseEvents = services.databaseEvents;
+    this.storageAccess = services.storageAccess;
+    this.replicator = services.replicator;
+    this.vault = services.vault;
+    this.replication = services.replication;
+    this.database = services.database;
+    this.fileHandler = services.fileHandler;
+    this.control = services.control;
+    services.database.onDatabaseReset.addHandler(this._onResetLocalDatabase.bind(this));
+    services.setting.suspendAllSync.addHandler(this._allSuspendAllSync.bind(this));
+  }
+  async $performRebuildDB(method) {
+    if (method == "localOnly") {
+      await this.$fetchLocal();
+    }
+    if (method == "localOnlyWithChunks") {
+      await this.$fetchLocal(true);
+    }
+    if (method == "remoteOnly") {
+      await this.$rebuildRemote();
+    }
+    if (method == "rebuildBothByThisDevice") {
+      await this.$rebuildEverything();
+    }
+  }
+  async informOptionalFeatures() {
+    await this.UI.showMarkdownDialog(
+      "All optional features are disabled",
+      `Customisation Sync and Hidden File Sync will all be disabled.
+Please enable them from the settings screen after setup is complete.`,
+      ["OK"]
+    );
+  }
+  async askUsingOptionalFeature(opt) {
+    if (await this.UI.confirm.askYesNoDialog(
+      "Do you want to enable extra features? If you are new to Self-hosted LiveSync, try the core feature first!",
+      { title: "Enable extra features", defaultOption: "No", timeout: 15 }
+    ) == "yes") {
+      await this.setting.suggestOptionalFeatures(opt);
+    }
+  }
+  async rebuildRemote() {
+    await this.replicator.runBoundedRemoteActivity(() => this.performRemoteRebuild(), {
+      label: "rebuild-remote"
+    });
+    await this.informOptionalFeatures();
+  }
+  async performRemoteRebuild() {
+    await this.setting.suspendExtraSync();
+    await this.setting.applyPartial({
+      isConfigured: true,
+      notifyThresholdOfRemoteStorageSize: DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize
+    });
+    await this.control.applySettings();
+    await this.replication.markLocked();
+    await this._tryResetRemoteDatabase();
+    await this.replication.markLocked();
+    await delay(500);
+    await delay(1e3);
+    await this.replication.replicateAllToRemote(true);
+    await delay(1e3);
+    await this.replication.replicateAllToRemote(true, true);
+  }
+  $rebuildRemote() {
+    return this.rebuildRemote();
+  }
+  async rebuildEverything() {
+    await this.replicator.runBoundedRemoteActivity(() => this.performRebuildEverything(), {
+      label: "rebuild-everything"
+    });
+    await this.informOptionalFeatures();
+  }
+  async performRebuildEverything() {
+    await this.setting.suspendExtraSync();
+    await this.setting.applyPartial({
+      isConfigured: true,
+      notifyThresholdOfRemoteStorageSize: DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize
+    });
+    await this.control.applySettings();
+    await this.resetLocalDatabase();
+    await delay(1e3);
+    await this.databaseEvents.initialiseDatabase(true, true, true);
+    await this.replication.markLocked();
+    await this._tryResetRemoteDatabase();
+    await this.replication.markLocked();
+    await delay(500);
+    await delay(1e3);
+    await this.replication.replicateAllToRemote(true);
+    await delay(1e3);
+    await this.replication.replicateAllToRemote(true, true);
+  }
+  $rebuildEverything() {
+    return this.rebuildEverything();
+  }
+  $fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync) {
+    return this.fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync);
+  }
+  $fetchLocalDBFast(autoResume) {
+    return this.fetchLocalDBFast(autoResume);
+  }
+  async scheduleRebuild() {
+    try {
+      await this.storageAccess.writeFileAuto(FlagFilesHumanReadable.REBUILD_ALL, "");
+    } catch (ex) {
+      this._log(`Could not create ${FlagFilesHumanReadable.REBUILD_ALL}`, LOG_LEVEL_NOTICE);
+      this._log(ex, LOG_LEVEL_VERBOSE);
+    }
+    this.appLifecycle.performRestart();
+  }
+  async scheduleFetch() {
+    try {
+      await this.storageAccess.writeFileAuto(FlagFilesHumanReadable.FETCH_ALL, "");
+    } catch (ex) {
+      this._log(`Could not create ${FlagFilesHumanReadable.FETCH_ALL}`, LOG_LEVEL_NOTICE);
+      this._log(ex, LOG_LEVEL_VERBOSE);
+    }
+    this.appLifecycle.performRestart();
+  }
+  async _tryResetRemoteDatabase() {
+    const currentReplicator = this.replicator.getActiveReplicator();
+    const settings = this.setting.currentSettings();
+    if (!currentReplicator) {
+      this._log("No active replicator found when trying to reset remote database.", LOG_LEVEL_NOTICE);
+      return;
+    }
+    await currentReplicator.tryResetRemoteDatabase(settings);
+  }
+  // private async _tryCreateRemoteDatabase(): Promise<void> {
+  //     const currentReplicator = this.replicator.getActiveReplicator();
+  //     const settings = this.setting.currentSettings();
+  //     if (!currentReplicator) {
+  //         this._log("No active replicator found when trying to create remote database.", LOG_LEVEL_NOTICE);
+  //         return;
+  //     }
+  //     await currentReplicator.tryCreateRemoteDatabase(settings);
+  // }
+  _onResetLocalDatabase() {
+    this.storageAccess.clearTouched();
+    return Promise.resolve(true);
+  }
+  async suspendAllSync() {
+    await this.setting.applyPartial({
+      liveSync: false,
+      periodicReplication: false,
+      syncOnSave: false,
+      syncOnEditorSave: false,
+      syncOnStart: false,
+      syncOnFileOpen: false,
+      syncAfterMerge: false
+    });
+    await this.setting.suspendExtraSync();
+  }
+  async suspendReflectingDatabase(ignoreMinIO = false) {
+    const settings = this.setting.currentSettings();
+    if (settings.doNotSuspendOnFetching) return;
+    if (!ignoreMinIO && settings.remoteType == REMOTE_MINIO) return;
+    this._log(
+      `Suspending reflection: Database and storage changes will not be reflected in each other until completely finished the fetching.`,
+      LOG_LEVEL_NOTICE
+    );
+    await this.setting.applyPartial({
+      suspendParseReplicationResult: true,
+      suspendFileWatching: true
+    });
+    await this.setting.saveSettingData();
+  }
+  async resumeReflectingDatabase(ignoreMinIO = false) {
+    const settings = this.setting.currentSettings();
+    if (settings.doNotSuspendOnFetching) return;
+    if (!ignoreMinIO && settings.remoteType == REMOTE_MINIO) return;
+    this._log(`Database and storage reflection has been resumed!`, LOG_LEVEL_NOTICE);
+    await this.setting.applyPartial({
+      suspendParseReplicationResult: false,
+      suspendFileWatching: false
+    });
+    await this.vault.scanVault(true);
+    await this.replication.onBeforeReplicate(false);
+    await this.setting.saveSettingData();
+  }
+  async fetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync, autoResume = true) {
+    await this.setting.suspendExtraSync();
+    await this.setting.applyPartial({
+      isConfigured: true,
+      notifyThresholdOfRemoteStorageSize: DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize
+    });
+    const settings = this.setting.currentSettings();
+    if (settings.maxMTimeForReflectEvents > 0) {
+      const date = new Date(settings.maxMTimeForReflectEvents);
+      const ask = `Your settings restrict file reflection times to no later than ${date.toLocaleString()}.
+
+**This is a recovery configuration.**
+
+This operation should only be performed on an empty vault.
+Are you sure you wish to proceed?`;
+      const PROCEED = "I understand, proceed";
+      const CANCEL = "Cancel operation";
+      const CLEARANDPROCEED = "Clear restriction and proceed";
+      const choices = [PROCEED, CLEARANDPROCEED, CANCEL];
+      const ret = await this.UI.confirm.askSelectStringDialogue(ask, choices, {
+        title: "Confirm restricted fetch",
+        defaultAction: CANCEL,
+        timeout: 0
+      });
+      if (ret == CLEARANDPROCEED) {
+        await this.setting.applyPartial({ maxMTimeForReflectEvents: 0 });
+        await this.setting.saveSettingData();
+      }
+      if (ret == CANCEL) {
+        return;
+      }
+    }
+    await this.replicator.runBoundedRemoteActivity(
+      () => this.performFetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync, autoResume),
+      { label: "rebuild-fetch" }
+    );
+  }
+  async performFetchLocal(makeLocalChunkBeforeSync, preventMakeLocalFilesBeforeSync, autoResume = true) {
+    await this.suspendReflectingDatabase(!autoResume);
+    await this.control.applySettings();
+    await this.resetLocalDatabase();
+    await delay(1e3);
+    await this.database.openDatabase({
+      databaseEvents: this.databaseEvents,
+      replicator: this.replicator
+    });
+    this.appLifecycle.markIsReady();
+    if (makeLocalChunkBeforeSync) {
+      await this.fileHandler.createAllChunks(true);
+    } else if (!preventMakeLocalFilesBeforeSync) {
+      await this.databaseEvents.initialiseDatabase(true, true, true);
+    } else {
+    }
+    await this.replication.markResolved();
+    await delay(500);
+    await this.replication.replicateAllFromRemote(true);
+    await delay(1e3);
+    await this.replication.replicateAllFromRemote(true);
+    if (autoResume) {
+      await this.finishRebuild();
+    }
+  }
+  async fetchLocalDBFast(autoResume) {
+    await this.setting.suspendExtraSync();
+    await this.setting.applyPartial({
+      isConfigured: true,
+      notifyThresholdOfRemoteStorageSize: DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize
+    });
+    const settings = this.setting.currentSettings();
+    if (settings.remoteType !== REMOTE_COUCHDB) {
+      this._log(
+        "Fast database fetch is available only for CouchDB remote. Falling back to standard fetch.",
+        LOG_LEVEL_NOTICE
+      );
+      await this.fetchLocal(false, true, autoResume);
+      return;
+    }
+    await this.replicator.runBoundedRemoteActivity(() => this.performFetchLocalDBFast(settings, autoResume), {
+      label: "fast-fetch"
+    });
+  }
+  async performFetchLocalDBFast(settings, autoResume) {
+    const remote = settings.couchDB_URI.replace(/\/+$/, "") + (settings.couchDB_DBNAME == "" ? "" : "/" + settings.couchDB_DBNAME);
+    let checkpoint = this.getFastFetchCheckpoint(remote);
+    await this.suspendReflectingDatabase();
+    await this.control.applySettings();
+    let since = checkpoint?.sequence ?? "0";
+    if (checkpoint) {
+      this._log(
+        `Resuming fast database fetch from sequence: ${checkpoint.sequence}`,
+        LOG_LEVEL_NOTICE,
+        "fetch-init-resume"
+      );
+    } else {
+      await this.resetLocalDatabase();
+      await delay(1e3);
+    }
+    await this.database.openDatabase({
+      databaseEvents: this.databaseEvents,
+      replicator: this.replicator
+    });
+    this.appLifecycle.markIsReady();
+    let localDB = this.database.localDatabase.localDatabase;
+    if (checkpoint && (await localDB.info()).doc_count == 0) {
+      this._log(
+        "Fast fetch checkpoint found, but the local database is empty. Starting from the beginning.",
+        LOG_LEVEL_NOTICE,
+        "fetch-init-resume"
+      );
+      this.clearFastFetchCheckpoint();
+      await this.resetLocalDatabase();
+      await delay(1e3);
+      await this.database.openDatabase({
+        databaseEvents: this.databaseEvents,
+        replicator: this.replicator
+      });
+      localDB = this.database.localDatabase.localDatabase;
+      since = "0";
+    }
+    const replicator = this.replicator.getActiveReplicator() ?? await this.replicator.getNewReplicator();
+    if (!replicator) {
+      throw new Error("No active replicator found for fast fetch.");
+    }
+    const salt = () => replicator.getReplicationPBKDF2Salt(settings);
+    const enc = getConfiguredFunctionsForEncryption(
+      settings.passphrase,
+      false,
+      false,
+      salt,
+      settings.E2EEAlgorithm
+    );
+    const authHeader = await new AuthorizationHeaderGenerator().getAuthorizationHeader(
+      generateCredentialObject(settings)
+    );
+    for (let attempt = 0; ; attempt++) {
+      try {
+        await fetchChangesForInitialSync(
+          localDB,
+          remote,
+          authHeader,
+          enc.outgoing,
+          since,
+          (progress) => {
+            this._log(
+              `Fast fetch progress: ${progress.totalValidFetched} / ${progress.docsToFetch}
+Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
+              LOG_LEVEL_NOTICE,
+              "fetch-init-progress"
+            );
+          },
+          (sequence) => this.saveFastFetchCheckpoint(remote, sequence)
+        );
+        break;
+      } catch (ex) {
+        if (attempt >= FAST_FETCH_RETRY_DELAYS.length) throw ex;
+        checkpoint = this.getFastFetchCheckpoint(remote);
+        since = checkpoint?.sequence ?? since;
+        this._log(
+          `Fast fetch interrupted. Retrying from sequence: ${since}`,
+          LOG_LEVEL_NOTICE,
+          "fetch-init-resume"
+        );
+        await delay(FAST_FETCH_RETRY_DELAYS[attempt]);
+      }
+    }
+    const allDocs = await localDB.allDocs({ include_docs: false });
+    this._log(
+      `Fast database fetch completed. Total documents in local database: ${allDocs.total_rows}`,
+      LOG_LEVEL_NOTICE,
+      "fetch-init-complete"
+    );
+    await this.replication.markResolved();
+    if (autoResume) {
+      await this.resumeReflectingDatabase(true);
+    }
+    this.clearFastFetchCheckpoint();
+  }
+  /**
+   * Finish rebuild process with resuming the reflection.
+   *
+   * @param ignoreMinIO Whether to ignore minio for resuming the reflection.
+   */
+  async finishRebuild(ignoreMinIO = true) {
+    await this.resumeReflectingDatabase(ignoreMinIO);
+  }
+  /**
+   * Fetch local database with making all chunks.
+   * This is a wrapper for {@link fetchLocal} with makeLocalChunkBeforeSync = true.
+   *
+   * @returns
+   */
+  async fetchLocalWithRebuild() {
+    return await this.fetchLocal(true);
+  }
+  async _allSuspendAllSync() {
+    await this.suspendAllSync();
+    return true;
+  }
+  async resetLocalDatabase() {
+    const settings = this.setting.currentSettings();
+    if (settings.isConfigured && settings.additionalSuffixOfDatabaseName == "") {
+      await this.database.resetDatabase();
+    }
+    const suffix = this.API.getAppID() || "";
+    await this.setting.applyPartial({ additionalSuffixOfDatabaseName: suffix });
+    await this.database.resetDatabase();
+    this.events.emitEvent(EVENT_DATABASE_REBUILT);
+  }
+  getFastFetchCheckpoint(remote) {
+    const rawCheckpoint = this.setting.getSmallConfig(FAST_FETCH_CHECKPOINT_KEY);
+    if (!rawCheckpoint) return void 0;
+    try {
+      const checkpoint = JSON.parse(rawCheckpoint);
+      if (checkpoint.remote === remote && checkpoint.sequence !== void 0) {
+        return {
+          remote,
+          sequence: checkpoint.sequence
+        };
+      }
+    } catch {
+    }
+    this.clearFastFetchCheckpoint();
+    return void 0;
+  }
+  saveFastFetchCheckpoint(remote, sequence) {
+    this.setting.setSmallConfig(FAST_FETCH_CHECKPOINT_KEY, JSON.stringify({ remote, sequence }));
+  }
+  clearFastFetchCheckpoint() {
+    this.setting.deleteSmallConfig(FAST_FETCH_CHECKPOINT_KEY);
+  }
+}
+export {
+  ServiceRebuilder
+};

package/dist/serviceModules/ServiceDatabaseFileAccessBase.d.ts

@@ -0,0 +1,40 @@
+import type { UXFileInfoStub, FilePathWithPrefix, UXFileInfo, MetaEntry, LoadedEntry, FilePath } from "../common/types.js";
+import type { DatabaseFileAccess } from "../interfaces/DatabaseFileAccess.js";
+import type { StorageAccess } from "../interfaces/StorageAccess.js";
+import type { APIService } from "../services/base/APIService.js";
+import type { DatabaseService } from "../services/base/DatabaseService.js";
+import type { PathService } from "../services/base/PathService.js";
+import type { VaultService } from "../services/base/VaultService.js";
+import { ServiceModuleBase } from "./ServiceModuleBase.js";
+import type { LiveSyncEventHub } from "../hub/hub.js";
+export interface ServiceDatabaseFileAccessDependencies {
+    events: LiveSyncEventHub;
+    API: APIService;
+    vault: VaultService;
+    storageAccess: StorageAccess;
+    path: PathService;
+    database: DatabaseService;
+}
+export declare class ServiceDatabaseFileAccessBase extends ServiceModuleBase<ServiceDatabaseFileAccessDependencies> implements DatabaseFileAccess {
+    private events;
+    private vault;
+    private storageAccess;
+    private path;
+    private database;
+    constructor(services: ServiceDatabaseFileAccessDependencies);
+    checkIsTargetFile(file: UXFileInfoStub | FilePathWithPrefix): Promise<boolean>;
+    delete(file: UXFileInfoStub | FilePathWithPrefix, rev?: string): Promise<boolean>;
+    createChunks(file: UXFileInfo, force?: boolean, skipCheck?: boolean): Promise<boolean>;
+    store(file: UXFileInfo, force?: boolean, skipCheck?: boolean): Promise<boolean>;
+    storeAsConflictedRevision(file: UXFileInfo, currentRev: string, skipCheck?: boolean): Promise<boolean>;
+    storeContent(path: FilePathWithPrefix, content: string): Promise<boolean>;
+    private __store;
+    private getParentRev;
+    hasContentInRevisionHistory(file: UXFileInfoStub | FilePathWithPrefix, content: string | string[] | Blob | ArrayBuffer, currentRev?: string): Promise<boolean>;
+    getConflictedRevs(file: UXFileInfoStub | FilePathWithPrefix): Promise<string[]>;
+    fetch(file: UXFileInfoStub | FilePathWithPrefix, rev?: string, waitForReady?: boolean, skipCheck?: boolean): Promise<UXFileInfo | false>;
+    fetchEntryMeta(file: UXFileInfoStub | FilePathWithPrefix, rev?: string, skipCheck?: boolean): Promise<MetaEntry | false>;
+    fetchEntryFromMeta(meta: MetaEntry, waitForReady?: boolean, skipCheck?: boolean): Promise<LoadedEntry | false>;
+    fetchEntry(file: UXFileInfoStub | FilePathWithPrefix, rev?: string, waitForReady?: boolean, skipCheck?: boolean): Promise<LoadedEntry | false>;
+    deleteFromDBbyPath(fullPath: FilePath | FilePathWithPrefix, rev?: string): Promise<boolean>;
+}