@vrtmrz/livesync-commonlib 0.1.26 → 0.1.28

package/dist/common/utils.d.ts

@@ -68,6 +68,18 @@ export declare function runWithInterval<T>(key: string, interval: number, task:
 export declare function runWithStartInterval<T>(key: string, interval: number, task: () => Promise<T>): Promise<T>;
 export declare const globalConcurrencyController: import("octagonal-wheels/concurrency/semaphore_v2").SemaphoreObject;
 export declare function determineTypeFromBlob(data: Blob): "newnote" | "plain";
+/**
+ * Whether remediation mode is active.
+ *
+ * While a modification-time limit is configured, Commonlib refuses every reconciliation scan
+ * between the storage and the local database and keeps storage events unqueued, so that a state
+ * being recovered is not overwritten by the current one. A host therefore stays unready while the
+ * limit is configured, because readiness depends upon that refused scan. Applying the limit to the
+ * documents which arrive is the responsibility of the host.
+ * @param settings Settings to inspect
+ * @returns `true` when a modification-time limit is configured.
+ */
+export declare function isRemediationModeActive(settings: Pick<ObsidianLiveSyncSettings, "maxMTimeForReflectEvents">): boolean;
 export declare function determineType(path: string, data: string | string[] | Uint8Array | ArrayBuffer | Blob): "newnote" | "plain";
 export declare function isAnyNote(doc: DatabaseEntry): doc is NewEntry | PlainEntry;
 export declare function isLoadedEntry(doc: DatabaseEntry): doc is LoadedEntry;

package/dist/common/utils.js

@@ -199,6 +199,9 @@ const globalConcurrencyController = Semaphore(50);
 function determineTypeFromBlob(data) {
   return isTextBlob(data) ? "plain" : "newnote";
 }
+function isRemediationModeActive(settings) {
+  return (settings.maxMTimeForReflectEvents ?? 0) > 0;
+}
 function determineType(path, data) {
   if (data instanceof Blob) {
     return determineTypeFromBlob(data);
@@ -548,6 +551,7 @@ export {
   isObfuscatedEntry,
   isObjectDifferent,
   isObjectMargeApplicable,
+  isRemediationModeActive,
   isSensibleMargeApplicable,
   isSyncInfoEntry,
   isTextBlob,

package/dist/interfaces/FileHandler.d.ts

@@ -2,6 +2,12 @@ import type { FilePath, FilePathWithPrefix, MetaEntry } from "../common/models/d
 import type { UXFileInfo, UXFileInfoStub, UXInternalFileInfoStub } from "../common/models/fileaccess.type.js";
 export interface IFileHandler {
     readFileFromStub(file: UXFileInfoStub | UXFileInfo): Promise<UXFileInfo>;
+    /**
+     * Record an untracked, unchanged file during an equal-time scan.
+     * Return true only when the exact current revision was recorded. This
+     * operation never stores a document revision or writes to storage.
+     */
+    tryRecordUntrackedFileRevision(info: UXFileInfoStub | UXFileInfo, expectedRevision: string): Promise<boolean>;
     storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean): Promise<boolean>;
     /**
      * Store the current storage content as a child of an exact live database revision.

package/dist/managers/EntryManager/EntryManagerImpls.js

@@ -209,7 +209,7 @@ function isTargetFile(host, filenameSrc) {
   const file = filenameSrc.startsWith(ICHeader) ? filenameSrc.substring(ICHeader.length) : filenameSrc;
   if (file.startsWith(ICXHeader)) return true;
   if (file.startsWith(PSCHeader)) return true;
-  if (file.includes(":")) {
+  if (file.startsWith(IDPrefixes.Chunk) || file.startsWith(IDPrefixes.Obfuscated)) {
     return false;
   }
   if (!settings.syncInternalFiles) {

package/dist/managers/StorageEventManager.js

@@ -6,7 +6,7 @@ import {
   LOG_LEVEL_NOTICE,
   LOG_LEVEL_VERBOSE
 } from "../common/types.js";
-import { delay, fireAndForget, throttle } from "../common/utils.js";
+import { delay, fireAndForget, isRemediationModeActive, throttle } from "../common/utils.js";
 import {} from "../common/types.js";
 import { serialized, skipIfDuplicated } from "octagonal-wheels/concurrency/lock";
 import { isWaitingForTimeout } from "octagonal-wheels/concurrency/task";
@@ -86,7 +86,7 @@ class StorageEventManagerBase extends StorageEventManager {
     const settings = this.settings;
     if (!settings.isConfigured) return;
     if (settings.suspendFileWatching) return;
-    if (settings.maxMTimeForReflectEvents > 0) {
+    if (isRemediationModeActive(settings)) {
       return;
     }
     this.fileProcessing.onStorageFileEvent();

package/dist/replication/ReplicatorProvider.d.ts

@@ -131,12 +131,16 @@ export type ReplicationFailed = {
 export type ReplicationOutcome = ReplicationCompleted | ReplicationCancelled | ReplicationBlocked | ReplicationPartial | ReplicationFailed;
 /** Preparation required before an ordinary operation may use a provider. */
 export interface ReplicationReadinessRequirements {
-    readonly centralRemotePreparation: "required" | "not-applicable";
+    readonly centralRemotePreparation: "required" | "provider-owned" | "not-applicable";
 }
 /** Shared readiness contract for providers backed by the central remote. */
 export declare const CENTRAL_REMOTE_REPLICATION_READINESS: Readonly<{
     readonly centralRemotePreparation: "required";
 }>;
+/** Central provider which prepares its Security Seed within each transfer. */
+export declare const PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS: Readonly<{
+    readonly centralRemotePreparation: "provider-owned";
+}>;
 /** Shared readiness contract for peer-to-peer providers without a central remote. */
 export declare const PEER_REPLICATION_READINESS: Readonly<{
     readonly centralRemotePreparation: "not-applicable";

package/dist/replication/ReplicatorProvider.js

@@ -25,6 +25,9 @@ const REPLICATION_CANCELLED = Object.freeze({ status: "cancelled" });
 const CENTRAL_REMOTE_REPLICATION_READINESS = Object.freeze({
   centralRemotePreparation: "required"
 });
+const PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS = Object.freeze({
+  centralRemotePreparation: "provider-owned"
+});
 const PEER_REPLICATION_READINESS = Object.freeze({
   centralRemotePreparation: "not-applicable"
 });
@@ -131,6 +134,7 @@ export {
   CENTRAL_REMOTE_REPLICATION_READINESS,
   NO_INTERACTION,
   PEER_REPLICATION_READINESS,
+  PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS,
   REPLICATION_CANCELLED,
   REPLICATION_COMPLETED,
   REPLICATION_PROGRESS_PRESENTATIONS,

package/dist/replication/SyncParamsHandler.d.ts

@@ -32,4 +32,6 @@ export declare class SyncParamsNotFoundError extends SyncParamsHandlerError {
 }
 export declare class SyncParamsUpdateError extends SyncParamsHandlerError {
 }
+/** Create a handler owned by one operation rather than the process-wide server cache. */
+export declare function createSyncParamsHandler({ put, get, create }: CreateSyncParamsHanderOptions): SyncParamsHandler;
 export {};

package/dist/replication/SyncParamsHandler.js

@@ -114,5 +114,6 @@ export {
   SyncParamsNotFoundError,
   SyncParamsUpdateError,
   clearHandlers,
-  createSyncParamsHanderForServer
+  createSyncParamsHanderForServer,
+  createSyncParamsHandler
 };

package/dist/replication/index.d.ts

@@ -1,4 +1,4 @@
-export { CAPABILITY_NOT_APPLICABLE, CAPABILITY_NOT_IMPLEMENTED, CENTRAL_REMOTE_REPLICATION_READINESS, NO_INTERACTION, PEER_REPLICATION_READINESS, REPLICATION_CANCELLED, REPLICATION_COMPLETED, REPLICATION_PROGRESS_PRESENTATIONS, USER_INITIATED_REPLICATION_AUTHORITY, defineReplicatorProviderDefinitions, isReplicationCompleted, outcomeFromContinuousOpenReplication, outcomeFromFiniteOpenReplication, replicationBlocked, replicationFailed, supportedOpenReplicationContinuous, supportedOpenReplicationOneShot, supportedOpenReplicationUnattended, supportedStopActiveTransfer, supportedCapability, } from "./ReplicatorProvider.js";
+export { CAPABILITY_NOT_APPLICABLE, CAPABILITY_NOT_IMPLEMENTED, CENTRAL_REMOTE_REPLICATION_READINESS, NO_INTERACTION, PEER_REPLICATION_READINESS, PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS, REPLICATION_CANCELLED, REPLICATION_COMPLETED, REPLICATION_PROGRESS_PRESENTATIONS, USER_INITIATED_REPLICATION_AUTHORITY, defineReplicatorProviderDefinitions, isReplicationCompleted, outcomeFromContinuousOpenReplication, outcomeFromFiniteOpenReplication, replicationBlocked, replicationFailed, supportedOpenReplicationContinuous, supportedOpenReplicationOneShot, supportedOpenReplicationUnattended, supportedStopActiveTransfer, supportedCapability, } from "./ReplicatorProvider.js";
 export type { ReplicatorInstance } from "./ReplicatorInstance.js";
 export type { ActiveReplicatorContext, CapabilitySupport, ContinuousReplicationRequest, ContinuousReplicationTrigger, ContinuousRunner, InteractionAuthority, InteractionPermissions, NotApplicableCapability, NotImplementedCapability, NoInteraction, PermittedInteractionPermissions, ReplicationBlockReason, ReplicationAttemptFailure, ReplicationCancelled, ReplicationCompleted, ReplicationFailed, ReplicationInteraction, ReplicationOutcome, ReplicationProgressPresentation, ReplicationReadinessRequirements, ReplicationPartial, ReplicationFailureRequest, ReplicatorConfigurationIdentity, ReplicatorProviderDefinition, ReplicatorProviderDefinitionMap, StopActiveTransferRunner, SupportedCapability, UnattendedOneShotRequest, UnattendedReplicationTrigger, UnattendedOneShotRunner, UserInitiatedOneShotRequest, UserInitiatedReplicationAuthority, UserInitiatedOneShotRunner, } from "./ReplicatorProvider.js";
 export { CAPABILITY_SUPPORT_KINDS, CAPABILITY_UNAVAILABLE_REASONS } from "./ProviderCapability.js";

package/dist/replication/index.js

@@ -4,6 +4,7 @@ import {
   CENTRAL_REMOTE_REPLICATION_READINESS,
   NO_INTERACTION,
   PEER_REPLICATION_READINESS,
+  PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS,
   REPLICATION_CANCELLED,
   REPLICATION_COMPLETED,
   REPLICATION_PROGRESS_PRESENTATIONS,
@@ -48,6 +49,7 @@ export {
   NO_INTERACTION,
   NO_REMOTE_RESOURCE_CAPABILITIES,
   PEER_REPLICATION_READINESS,
+  PROVIDER_OWNED_CENTRAL_REMOTE_REPLICATION_READINESS,
   REMOTE_RESOURCE_KINDS,
   REPLICATION_CANCELLED,
   REPLICATION_COMPLETED,

package/dist/replication/journal/JournalSyncCore.d.ts

@@ -16,6 +16,7 @@ export declare class JournalSyncCore {
     env: LiveSyncJournalReplicatorEnv;
     store: SimpleStore<CheckPointInfo>;
     requestedStop: boolean;
+    private syncParamsHandler?;
     getInitialSyncParameters(): Promise<SyncParameters>;
     getSyncParameters(): Promise<SyncParameters>;
     putSyncParameters(params: SyncParameters): Promise<boolean>;
@@ -44,6 +45,7 @@ export declare class JournalSyncCore {
     resetBucket(): Promise<boolean>;
     getRemoteKey(): string;
     getReplicationPBKDF2Salt(refresh?: boolean): Promise<Uint8Array<ArrayBuffer>>;
+    private getSyncParamsHandler;
     isEncryptionPrevented(fileName: string): boolean;
     private decryptDataV2;
     private decryptDataV1;

package/dist/replication/journal/JournalSyncCore.js

@@ -22,8 +22,7 @@ import {
   JournalStorageReadStatuses
 } from "./objectstore/JournalStorageAdapter.js";
 import {
-  clearHandlers,
-  createSyncParamsHanderForServer,
+  createSyncParamsHandler,
   SyncParamsFetchError,
   SyncParamsNotFoundError,
   SyncParamsUpdateError
@@ -61,6 +60,7 @@ class JournalSyncCore {
   env;
   store;
   requestedStop = false;
+  syncParamsHandler;
   getInitialSyncParameters() {
     return Promise.resolve({
       ...DEFAULT_SYNC_PARAMETERS,
@@ -109,7 +109,6 @@ class JournalSyncCore {
     this.store = store;
     this.hash = this.getHash(settings);
     this.storage = storage;
-    clearHandlers();
   }
   async downloadJson(key) {
     try {
@@ -156,7 +155,7 @@ class JournalSyncCore {
     this.store = store;
     this.hash = this.getHash(settings);
     this.storage.applyNewConfig(settings);
-    clearHandlers();
+    this.resetAllCaches();
   }
   updateInfo(info) {
     const old = this.env.services.replicator.replicationStatics.value;
@@ -205,23 +204,18 @@ class JournalSyncCore {
     return this._currentCheckPointInfo;
   }
   resetAllCaches() {
-    clearHandlers();
+    this.syncParamsHandler = void 0;
   }
   async resetCheckpointInfo() {
     await this.updateCheckPointInfo((info) => ({ ...CheckPointInfoDefault }));
-    clearHandlers();
   }
   getJournalEpochFromSyncParams(params) {
     return `${params.protocolVersion}:${params.pbkdf2salt}`;
   }
   async ensureCheckpointCachesAreFresh() {
-    let journalEpoch = "";
-    try {
-      const params = await this.getSyncParameters();
-      journalEpoch = this.getJournalEpochFromSyncParams(params);
-    } catch {
-      return;
-    }
+    const params = await this.getSyncParamsHandler().fetch(true);
+    if (!params) throw new SyncParamsFetchError("Could not prepare Journal synchronisation parameters");
+    const journalEpoch = this.getJournalEpochFromSyncParams(params);
     const current = await this.getCheckpointInfo();
     if (current.journalEpoch === journalEpoch) {
       return;
@@ -252,12 +246,12 @@ class JournalSyncCore {
       receivedFiles: /* @__PURE__ */ new Set(),
       sentFiles: /* @__PURE__ */ new Set()
     }));
-    clearHandlers();
   }
   async isAvailable() {
     return await this.storage.isAvailable();
   }
   async resetBucket() {
+    this.resetAllCaches();
     let files = [];
     try {
       do {
@@ -269,7 +263,6 @@ class JournalSyncCore {
           return false;
         }
       } while (files.length != 0);
-      clearHandlers();
     } catch (ex) {
       Logger(`WARNING! Could not delete files.`, LOG_LEVEL_NOTICE, "reset-bucket");
       Logger(ex, LOG_LEVEL_VERBOSE);
@@ -291,13 +284,14 @@ class JournalSyncCore {
     return this.getHash(this._settings);
   }
   async getReplicationPBKDF2Salt(refresh) {
-    const server = this.getRemoteKey();
-    const manager = createSyncParamsHanderForServer(server, {
+    return await this.getSyncParamsHandler().getPBKDF2Salt(refresh);
+  }
+  getSyncParamsHandler() {
+    return this.syncParamsHandler ??= createSyncParamsHandler({
       put: (params) => this.putSyncParameters(params),
       get: () => this.getSyncParameters(),
       create: () => this.getInitialSyncParameters()
     });
-    return await manager.getPBKDF2Salt(refresh);
   }
   isEncryptionPrevented(fileName) {
     if (fileName.endsWith(DOCID_JOURNAL_SYNC_PARAMETERS)) return true;
@@ -762,6 +756,7 @@ class JournalSyncCore {
   /** Stop this Journal operation owner and release its storage transport. */
   dispose() {
     this.requestStop();
+    this.resetAllCaches();
     this.storage.dispose?.();
   }
 }

package/dist/replication/journal/LiveSyncJournalReplicator.js

@@ -12,7 +12,6 @@ import { MinioStorageAdapter } from "./objectstore/MinioStorageAdapter.js";
 import { LiveSyncAbstractReplicator } from "../LiveSyncAbstractReplicator.js";
 import { ensureRemoteIsCompatible } from "../../pouchdb/LiveSyncDBFunctions.js";
 import { extractObject } from "../../common/utils.js";
-import { clearHandlers } from "../SyncParamsHandler.js";
 import { JournalStorageReadStatuses } from "./objectstore/JournalStorageAdapter.js";
 import {
   CENTRAL_COMPATIBILITY_ACCEPTED,
@@ -154,20 +153,24 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
   async openReplication(setting, _, showResult, ignoreCleanLock = false, recordCompatibilityDecision) {
     return await this.runJournalTransfer(async (stopGeneration) => {
       const client = this.setupJournalSyncClient(setting);
-      stopGeneration = this.journalTransferStopGeneration;
-      if (!await this.checkReplicationConnectivity(
-        false,
-        ignoreCleanLock,
-        showResult,
-        setting,
-        client,
-        recordCompatibilityDecision
-      )) {
-        return false;
+      try {
+        stopGeneration = this.journalTransferStopGeneration;
+        if (!await this.checkReplicationConnectivity(
+          false,
+          ignoreCleanLock,
+          showResult,
+          setting,
+          client,
+          recordCompatibilityDecision
+        )) {
+          return false;
+        }
+        if (stopGeneration !== this.journalTransferStopGeneration) return false;
+        this.hasEnteredReplication = true;
+        return await client.sync(showResult);
+      } finally {
+        client.resetAllCaches();
       }
-      if (stopGeneration !== this.journalTransferStopGeneration) return false;
-      this.hasEnteredReplication = true;
-      return await client.sync(showResult);
     });
   }
   /**
@@ -191,20 +194,24 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
   async runDirectionalReplication(setting, showingNotice, transfer, recordCompatibilityDecision) {
     return await this.runJournalTransfer(async (stopGeneration) => {
       const client = this.setupJournalSyncClient(setting);
-      stopGeneration = this.journalTransferStopGeneration;
-      if (!await this.checkReplicationConnectivity(
-        false,
-        false,
-        !!showingNotice,
-        setting,
-        client,
-        recordCompatibilityDecision
-      )) {
-        return false;
+      try {
+        stopGeneration = this.journalTransferStopGeneration;
+        if (!await this.checkReplicationConnectivity(
+          false,
+          false,
+          !!showingNotice,
+          setting,
+          client,
+          recordCompatibilityDecision
+        )) {
+          return false;
+        }
+        if (stopGeneration !== this.journalTransferStopGeneration) return false;
+        this.hasEnteredReplication = true;
+        return await transfer(client);
+      } finally {
+        client.resetAllCaches();
       }
-      if (stopGeneration !== this.journalTransferStopGeneration) return false;
-      this.hasEnteredReplication = true;
-      return await transfer(client);
     });
   }
   /** Capture only the compatibility decision made by this exact borrowed client. */
@@ -344,7 +351,6 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
       if (!await this.client.resetBucket()) {
         throw new Error("Could not reset remote bucket");
       }
-      clearHandlers();
       Logger("Remote Bucket Cleared", LOG_LEVEL_NOTICE);
       await this.tryCreateRemoteDatabase(setting);
     } catch (ex) {
@@ -356,7 +362,6 @@ class LiveSyncJournalReplicator extends LiveSyncAbstractReplicator {
   async tryCreateRemoteDatabase(setting) {
     this.closeReplication();
     Logger("Remote Database Created or Connected", LOG_LEVEL_NOTICE);
-    clearHandlers();
     if (!await this.ensurePBKDF2Salt(setting, true, false)) {
       throw new Error("Could not ensure PBKDF2 salt (Security Seed)");
     }

package/dist/serviceFeatures/offlineScanner.js

@@ -7,7 +7,7 @@ import {
   LOG_LEVEL_VERBOSE,
   isMetaEntry
 } from "../common/types.js";
-import { compareMTime, isAnyNote } from "../common/utils.js";
+import { compareMTime, isAnyNote, isRemediationModeActive } from "../common/utils.js";
 import { shouldBeIgnored, stripAllPrefixes } from "../string_and_binary/path.js";
 import { createInstanceLogFunction } from "../services/lib/logUtils.js";
 import { BASE_IS_NEW, EVEN, TARGET_IS_NEW } from "../common/models/shared.const.symbols.js";
@@ -389,6 +389,9 @@ async function syncFileBetweenDBandStorage(host, log, file, doc) {
         return FilePairProcessResults.SKIPPED;
       }
     case EVEN:
+      if (doc._rev) {
+        await host.serviceModules.fileHandler.tryRecordUntrackedFileRevision(file, doc._rev);
+      }
       log("STORAGE == DB :" + file.path, LOG_LEVEL_DEBUG);
       return FilePairProcessResults.COMPLETED;
     default:
@@ -411,7 +414,7 @@ function canProceedScan(host, errorManager, log, showingNotice = false, ignoreSu
   }
   errorManager.clearError(ERR_SUSPENDING);
   const MSG_IN_REMEDIATION = `Started in remediation Mode! (Max mtime for reflect events is set). Synchronising between the storage and the local database is now prevented.`;
-  if (settings.maxMTimeForReflectEvents > 0) {
+  if (isRemediationModeActive(settings)) {
     errorManager.showError(MSG_IN_REMEDIATION, LOG_LEVEL_NOTICE);
     return false;
   }

package/dist/serviceModules/Rebuilder.js

@@ -8,7 +8,7 @@ import { ServiceModuleBase } from "./ServiceModuleBase.js";
 import { fetchChangesForInitialSync, isRetryableStreamingFetchFailure } from "../pouchdb/StreamingFetch.js";
 import { getConfiguredFunctionsForEncryption } from "../pouchdb/encryption.js";
 import { AuthorizationHeaderGenerator, generateCredentialObject } from "../replication/httplib.js";
-import { parseHeaderValues } from "../common/utils.js";
+import { isRemediationModeActive, parseHeaderValues } from "../common/utils.js";
 import { sizeToHumanReadable } from "octagonal-wheels/number";
 import { REMOTE_RESOURCE_KINDS } from "../replication/index.js";
 const FAST_FETCH_CHECKPOINT_KEY = "fast-fetch-checkpoint";
@@ -106,6 +106,15 @@ Please enable them from the settings screen after setup is complete.`,
     await this.informOptionalFeatures();
   }
   async performRebuildEverything() {
+    if (isRemediationModeActive(this.setting.currentSettings())) {
+      this._log(
+        `Rebuilding has been refused: remediation mode is active. Clear the modification-time limit to rebuild.`,
+        LOG_LEVEL_NOTICE
+      );
+      throw new Error(
+        "Rebuilding is not available while remediation mode is active. Clear the modification-time limit first."
+      );
+    }
     this.appLifecycle.resetIsReady();
     await this.setting.suspendExtraSync();
     await this.setting.applyPartial({
@@ -249,7 +258,8 @@ Please enable them from the settings screen after setup is complete.`,
       suspendParseReplicationResult: false,
       suspendFileWatching: false
     });
-    if (!await this.vault.scanVault(true)) return false;
+    const inRemediationMode = isRemediationModeActive(this.setting.currentSettings());
+    if (!inRemediationMode && !await this.vault.scanVault(true)) return false;
     if (!await this.replication.onBeforeReplicate(false)) return false;
     return true;
   }
@@ -281,7 +291,7 @@ Please enable them from the settings screen after setup is complete.`,
       notifyThresholdOfRemoteStorageSize: DEFAULT_SETTINGS.notifyThresholdOfRemoteStorageSize
     });
     const settings = this.setting.currentSettings();
-    if (settings.maxMTimeForReflectEvents > 0) {
+    if (isRemediationModeActive(settings)) {
       const date = new Date(settings.maxMTimeForReflectEvents);
       const ask = `Your settings restrict file reflection times to no later than ${date.toLocaleString()}.
 
@@ -318,7 +328,12 @@ Are you sure you wish to proceed?`;
     await this.resetLocalDatabase();
     this.clearFastFetchCheckpoint();
     await delay(1e3);
-    if (makeLocalChunkBeforeSync) {
+    if (isRemediationModeActive(this.setting.currentSettings())) {
+      this._log(
+        `Remediation mode: the files in storage are not stored in the database before fetching.`,
+        LOG_LEVEL_NOTICE
+      );
+    } else if (makeLocalChunkBeforeSync) {
       await this.fileHandler.createAllChunks(true);
     } else if (!preventMakeLocalFilesBeforeSync) {
       await this.prepareLocalDatabaseForRebuild();
@@ -496,7 +511,9 @@ Total bytes fetched: ${sizeToHumanReadable(progress.totalBytes)}`,
       if (controlsReflection) {
         await this.setting.saveSettingData();
       }
-      this.appLifecycle.markIsReady();
+      if (!isRemediationModeActive(this.setting.currentSettings())) {
+        this.appLifecycle.markIsReady();
+      }
       completed = true;
       return true;
     } finally {

package/dist/serviceModules/ServiceFileHandlerBase.d.ts

@@ -94,6 +94,7 @@ export declare abstract class ServiceFileHandlerBase extends ServiceModuleBase<S
     getPath(entry: AnyEntry): FilePathWithPrefix;
     getPathWithoutPrefix(entry: AnyEntry): FilePathWithPrefix;
     readFileFromStub(file: UXFileInfoStub | UXFileInfo): Promise<UXFileInfo>;
+    tryRecordUntrackedFileRevision(info: UXFileInfoStub | UXFileInfo, expectedRevision: string): Promise<boolean>;
     private infoToStub;
     storeFileToDB(info: UXFileInfoStub | UXFileInfo | UXInternalFileInfoStub | FilePathWithPrefix, force?: boolean, onlyChunks?: boolean): Promise<boolean>;
     storeFileToDBWithBaseRevision(info: UXFileInfoStub | UXFileInfo | FilePathWithPrefix, baseRevision: string, createIfDifferent?: boolean): Promise<boolean>;

package/dist/serviceModules/ServiceFileHandlerBase.js

@@ -215,6 +215,36 @@ class ServiceFileHandlerBase extends ServiceModuleBase {
     }
     return readFile;
   }
+  async tryRecordUntrackedFileRevision(info, expectedRevision) {
+    if (!this.fileReflectionProvenance || !expectedRevision) return false;
+    const path = getStoragePathFromUXFileInfo(info);
+    try {
+      return await this.serializedByFileEventPaths([path], async () => {
+        const provenance = this.fileReflectionProvenance;
+        if (await provenance.get(path)) return false;
+        const stub = await this.storage.getStub(path);
+        if (!stub || isFolderInfo(stub) || stub.path !== path) return false;
+        const file = await this.readFileFromStub(stub);
+        const current = await this.db.fetchEntryMeta(path, void 0, true);
+        if (!current || current._rev !== expectedRevision || current.deleted || current._deleted) return false;
+        if ((await this.db.getConflictedRevs(path)).length > 0) return false;
+        const entry = await this.db.fetchEntry(path, expectedRevision, true, true, true);
+        if (!entry || entry._rev !== expectedRevision || entry.deleted || entry._deleted) return false;
+        if (!await isDocContentSame(readContent(entry), file.body)) return false;
+        const latest = await this.db.fetchEntryMeta(path, void 0, true);
+        if (!latest || latest._rev !== expectedRevision || latest.deleted || latest._deleted) return false;
+        if ((await this.db.getConflictedRevs(path)).length > 0) return false;
+        if (!await this.currentStorageSnapshotMatches(file, void 0)) return false;
+        if (await provenance.get(path)) return false;
+        await provenance.set(path, { revision: expectedRevision, observedStorageMtime: file.stat.mtime });
+        return true;
+      });
+    } catch (ex) {
+      this._log(`Could not record unchanged file provenance for ${path}`, LOG_LEVEL_VERBOSE);
+      this._log(ex, LOG_LEVEL_VERBOSE);
+      return false;
+    }
+  }
   async infoToStub(info) {
     if (info == null) return null;
     const file = typeof info === "string" ? await this.storage.getFileStub(info) : info;

package/dist/services/base/PathService.js

@@ -1,5 +1,5 @@
 import { ServiceBase } from "./ServiceBase.js";
-import { addPrefix, id2path_base, path2id_base } from "../../string_and_binary/path.js";
+import { addPrefix, expandFilePathPrefix, id2path_base, path2id_base } from "../../string_and_binary/path.js";
 import { isInternalMetadata, stripInternalMetadataPrefix } from "../../common/typeUtils.js";
 class PathService extends ServiceBase {
   settingService;
@@ -14,19 +14,12 @@ class PathService extends ServiceBase {
   }
   _id2path(id, entry) {
     const filename = id2path_base(id, entry);
-    const temp = filename.split(":");
-    const path = temp.pop();
-    const normalizedPath = this.normalizePath(path);
-    temp.push(normalizedPath);
-    const fixedPath = temp.join(":");
-    return fixedPath;
+    const [prefix, path] = expandFilePathPrefix(filename);
+    return prefix + this.normalizePath(path);
   }
   async _path2id(filename, obfuscatePassphrase, caseInsensitive) {
-    const temp = filename.split(":");
-    const path = temp.pop();
-    const normalizedPath = this.normalizePath(path);
-    temp.push(normalizedPath);
-    const fixedPath = temp.join(":");
+    const [prefix, path] = expandFilePathPrefix(filename);
+    const fixedPath = prefix + this.normalizePath(path);
     const out = await path2id_base(fixedPath, obfuscatePassphrase, caseInsensitive);
     return out;
   }

package/dist/string_and_binary/path.js

@@ -13,6 +13,7 @@ import {
 import { memorizeFuncWithLRUCache } from "../common/utils.js";
 import { uint8ArrayToHexString, writeString } from "./convert.js";
 import { unique } from "octagonal-wheels/collection.js";
+import { CHeader, ICHeader, ICXHeader, PSCHeader } from "../common/models/fileaccess.const.js";
 function isValidFilenameInWidows(filename) {
   const regex = /[\u0000-\u001f]|[\\":?<>|*#]/g;
   if (regex.test(filename)) return false;
@@ -21,11 +22,11 @@ function isValidFilenameInWidows(filename) {
   return true;
 }
 function isValidFilenameInDarwin(filename) {
-  const regex = /[\u0000-\u001f]|[:]/g;
+  const regex = /[\u0000-\u001f]/g;
   return !regex.test(filename);
 }
 function isValidFilenameInLinux(filename) {
-  const regex = /[\u0000-\u001f]|[:]/g;
+  const regex = /[\u0000-\u001f]/g;
   return !regex.test(filename);
 }
 function isValidFilenameInAndroid(filename) {
@@ -33,37 +34,30 @@ function isValidFilenameInAndroid(filename) {
   return !regex.test(filename);
 }
 function isFilePath(path) {
-  if (path.indexOf(":") === -1) return true;
-  return false;
+  return expandFilePathPrefix(path)[0] === "";
 }
 function stripAllPrefixes(prefixedPath) {
-  if (isFilePath(prefixedPath)) return prefixedPath;
-  const [, body] = expandFilePathPrefix(prefixedPath);
-  return stripAllPrefixes(body);
+  return stripPrefix(prefixedPath);
 }
 function addPrefix(path, prefix) {
   if (prefix && path.startsWith(prefix)) return path;
   return `${prefix ?? ""}${path}`;
 }
 function expandFilePathPrefix(path) {
-  let [prefix, body] = path.split(":", 2);
-  if (!body) {
-    body = prefix;
-    prefix = "";
-  } else {
-    prefix = prefix + ":";
+  for (const prefix of [ICHeader, ICXHeader, PSCHeader]) {
+    if (path.startsWith(prefix)) {
+      return [prefix, path.substring(prefix.length)];
+    }
   }
-  return [prefix, body];
+  return ["", path];
 }
 function expandDocumentIDPrefix(id) {
-  let [prefix, body] = id.split(":", 2);
-  if (!body) {
-    body = prefix;
-    prefix = "";
-  } else {
-    prefix = prefix + ":";
+  for (const prefix of [CHeader, PREFIX_OBFUSCATED]) {
+    if (id.startsWith(prefix)) {
+      return [prefix, id.substring(prefix.length)];
+    }
   }
-  return [prefix, body];
+  return expandFilePathPrefix(id);
 }
 const _hashString = memorizeFuncWithLRUCache(async (key) => {
   const buff = writeString(key);
@@ -112,11 +106,7 @@ function getPathWithoutPrefix(entry) {
   return stripAllPrefixes(f);
 }
 function stripPrefix(prefixedPath) {
-  const [prefix, body] = prefixedPath.split(":", 2);
-  if (!body) {
-    return prefix;
-  }
-  return body;
+  return expandFilePathPrefix(prefixedPath)[1];
 }
 function shouldBeIgnored(filename) {
   if (filename == FLAGMD_REDFLAG) {