@vrtmrz/livesync-commonlib 0.1.43 → 0.2.0

package/dist/diagnostics/usage.js

@@ -1,3 +1,4 @@
+const USAGE_ACCOUNTING_YIELD_EVERY = 64;
 function asRecord(value) {
   return value;
 }
@@ -30,13 +31,29 @@ async function analyseDatabaseUsage(options) {
   const errors = [];
   const documentMap = /* @__PURE__ */ new Map();
   const chunkMap = /* @__PURE__ */ new Map();
-  const progress = { idsProcessed: 0, revisionsProcessed: 0 };
+  const progress = {
+    phase: "reading",
+    idsProcessed: 0,
+    revisionsProcessed: 0,
+    processed: 0,
+    total: 0,
+    totalIsFinal: false
+  };
   let startUpdateSeq;
   let endUpdateSeq;
   let enumerationFailed = false;
-  const reportProgress = () => onProgress?.({ ...progress });
+  const reportProgress = () => {
+    try {
+      onProgress?.({ ...progress });
+    } catch {
+    }
+  };
   const isCancelled = () => signal?.aborted === true;
   let cancelled = isCancelled();
+  let documentRevisionCount = 0;
+  let referenceCount = 0;
+  reportProgress();
+  if (isCancelled()) cancelled = true;
   if (!cancelled) {
     try {
       const info = await source.info();
@@ -47,6 +64,7 @@ async function analyseDatabaseUsage(options) {
     }
     if (isCancelled()) cancelled = true;
   }
+  let enumerationFinished = false;
   try {
     if (!cancelled && !isCancelled()) {
       for await (const id of source.enumerateIds()) {
@@ -74,7 +92,16 @@ async function analyseDatabaseUsage(options) {
           errors.push({ code: "revision_list_read_failed", id });
         }
         progress.idsProcessed++;
+        for (const revisionInfo of revisionInfos) {
+          if (revisionInfo.status === "available" && typeof revisionInfo.rev === "string") {
+            progress.total++;
+          }
+        }
         reportProgress();
+        if (isCancelled()) {
+          cancelled = true;
+          break;
+        }
         for (const revisionInfo of revisionInfos) {
           if (isCancelled()) {
             cancelled = true;
@@ -86,27 +113,28 @@ async function analyseDatabaseUsage(options) {
             errors.push({ code: "revision_document_invalid", id });
             continue;
           }
-          progress.revisionsProcessed++;
           let rawDocument;
           try {
             rawDocument = await source.get(id, { rev: revision });
           } catch {
             if (isCancelled()) {
               cancelled = true;
-              reportProgress();
               break;
             }
             errors.push({ code: "revision_read_failed", id, revision });
+            progress.revisionsProcessed++;
+            progress.processed++;
             reportProgress();
             continue;
           }
           if (isCancelled()) {
             cancelled = true;
-            reportProgress();
             break;
           }
           if (!rawDocument) {
             errors.push({ code: "revision_not_found", id, revision });
+            progress.revisionsProcessed++;
+            progress.processed++;
             reportProgress();
             continue;
           }
@@ -124,9 +152,13 @@ async function analyseDatabaseUsage(options) {
                 path,
                 chunks,
                 uniqueChunks: /* @__PURE__ */ new Set(),
-                sharedChunks: /* @__PURE__ */ new Set()
+                sharedChunks: /* @__PURE__ */ new Set(),
+                uniqueChunkSize: 0,
+                sharedChunkSize: 0
               });
               documentMap.set(documentId, revisions);
+              documentRevisionCount++;
+              referenceCount += chunks.size;
             } else if (document.type === "leaf") {
               const chunkId = getStringField(document._id);
               if (!chunkMap.has(chunkId) && !document._deleted) {
@@ -137,11 +169,14 @@ async function analyseDatabaseUsage(options) {
           } catch {
             errors.push({ code: "revision_document_invalid", id, revision });
           }
+          progress.revisionsProcessed++;
+          progress.processed++;
           reportProgress();
         }
         if (isCancelled()) cancelled = true;
         if (cancelled) break;
       }
+      if (!cancelled && !isCancelled()) enumerationFinished = true;
     } else if (isCancelled()) {
       cancelled = true;
     }
@@ -152,60 +187,127 @@ async function analyseDatabaseUsage(options) {
       errors.push({ code: "id_enumeration_failed" });
     }
   }
-  for (const revisions of documentMap.values()) {
-    for (const document of revisions) {
-      for (const chunkId of document.chunks) {
-        const chunk = chunkMap.get(chunkId);
-        if (!chunk) continue;
-        if (chunk.refCount === 0) document.uniqueChunks.add(chunkId);
-        else document.sharedChunks.add(chunkId);
-        chunk.refCount++;
+  if (isCancelled()) cancelled = true;
+  if (enumerationFinished) {
+    progress.totalIsFinal = true;
+    reportProgress();
+    if (isCancelled()) cancelled = true;
+  }
+  const readingWasCancelled = cancelled;
+  progress.phase = "accounting";
+  progress.processed = 0;
+  progress.total = referenceCount + documentRevisionCount + chunkMap.size;
+  progress.totalIsFinal = true;
+  reportProgress();
+  let accountingCancelled = !readingWasCancelled && isCancelled();
+  const rows = [];
+  const accountingErrors = [];
+  const recordAccountingWork = () => {
+    progress.processed++;
+    return progress.processed % USAGE_ACCOUNTING_YIELD_EVERY === 0;
+  };
+  const yieldAfterAccountingBatch = async () => {
+    reportProgress();
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    return readingWasCancelled || !isCancelled();
+  };
+  if (!readingWasCancelled && isCancelled()) accountingCancelled = true;
+  if (!accountingCancelled) {
+    for (const revisions of documentMap.values()) {
+      for (const document of revisions) {
+        for (const chunkId of document.chunks) {
+          const chunk = chunkMap.get(chunkId);
+          if (chunk) {
+            if (chunk.refCount === 0) {
+              document.uniqueChunks.add(chunkId);
+              document.uniqueChunkSize += chunk.length;
+            } else {
+              document.sharedChunks.add(chunkId);
+              document.sharedChunkSize += chunk.length;
+            }
+            chunk.refCount++;
+          }
+          if (recordAccountingWork() && !await yieldAfterAccountingBatch()) {
+            accountingCancelled = true;
+            break;
+          }
+        }
+        if (accountingCancelled) break;
+      }
+      if (accountingCancelled) break;
+    }
+  }
+  if (!accountingCancelled) {
+    for (const revisions of documentMap.values()) {
+      for (const document of revisions) {
+        const revisionNumber = getRevisionNumber(document.revision);
+        let revisionHash;
+        try {
+          revisionHash = getRevisionHash(document.revision);
+        } catch {
+          accountingErrors.push({
+            code: "revision_document_invalid",
+            id: document.id,
+            revision: document.revision
+          });
+          if (recordAccountingWork() && !await yieldAfterAccountingBatch()) {
+            accountingCancelled = true;
+            break;
+          }
+          continue;
+        }
+        rows.push({
+          title: `${document.path} (${document.revision})`,
+          id: document.id,
+          path: document.path,
+          revision: document.revision,
+          rev: revisionNumber,
+          revHash: revisionHash,
+          uniqueChunkCount: document.uniqueChunks.size,
+          sharedChunkCount: document.sharedChunks.size,
+          uniqueChunkSize: document.uniqueChunkSize,
+          sharedChunkSize: document.sharedChunkSize
+        });
+        if (recordAccountingWork() && !await yieldAfterAccountingBatch()) {
+          accountingCancelled = true;
+          break;
+        }
       }
+      if (accountingCancelled) break;
     }
   }
-  const totalSize = (ids) => {
-    let size = 0;
-    for (const id of ids) size += chunkMap.get(id)?.length ?? 0;
-    return size;
-  };
-  const rows = [];
-  for (const revisions of documentMap.values()) {
-    for (const document of revisions) {
-      const revisionNumber = getRevisionNumber(document.revision);
-      let revisionHash;
-      try {
-        revisionHash = getRevisionHash(document.revision);
-      } catch {
-        errors.push({ code: "revision_document_invalid", id: document.id, revision: document.revision });
-        continue;
+  let orphanChunkCount = 0;
+  let orphanChunkSize = 0;
+  if (!accountingCancelled) {
+    for (const chunk of chunkMap.values()) {
+      if (chunk.refCount === 0) {
+        orphanChunkCount++;
+        orphanChunkSize += chunk.length;
+      }
+      if (recordAccountingWork() && !await yieldAfterAccountingBatch()) {
+        accountingCancelled = true;
+        break;
       }
-      rows.push({
-        title: `${document.path} (${document.revision})`,
-        id: document.id,
-        path: document.path,
-        revision: document.revision,
-        rev: revisionNumber,
-        revHash: revisionHash,
-        uniqueChunkCount: document.uniqueChunks.size,
-        sharedChunkCount: document.sharedChunks.size,
-        uniqueChunkSize: totalSize(document.uniqueChunks),
-        sharedChunkSize: totalSize(document.sharedChunks)
-      });
     }
   }
-  const orphanChunks = [...chunkMap.entries()].filter(([, chunk]) => chunk.refCount === 0);
-  rows.push({
-    title: "__orphan",
-    id: "__orphan",
-    path: "__orphan",
-    revision: "1-xxxxx",
-    rev: "1",
-    revHash: "xxxxx",
-    uniqueChunkCount: orphanChunks.length,
-    sharedChunkCount: 0,
-    uniqueChunkSize: orphanChunks.reduce((size, [, chunk]) => size + chunk.length, 0),
-    sharedChunkSize: 0
-  });
+  if (accountingCancelled) {
+    cancelled = true;
+    rows.length = 0;
+  } else {
+    errors.push(...accountingErrors);
+    rows.push({
+      title: "__orphan",
+      id: "__orphan",
+      path: "__orphan",
+      revision: "1-xxxxx",
+      rev: "1",
+      revHash: "xxxxx",
+      uniqueChunkCount: orphanChunkCount,
+      sharedChunkCount: 0,
+      uniqueChunkSize: orphanChunkSize,
+      sharedChunkSize: 0
+    });
+  }
   if (!cancelled && !isCancelled()) {
     try {
       const info = await source.info();
@@ -216,6 +318,8 @@ async function analyseDatabaseUsage(options) {
     }
   }
   if (isCancelled()) cancelled = true;
+  reportProgress();
+  if (isCancelled()) cancelled = true;
   const changedDuringRun = startUpdateSeq === void 0 || endUpdateSeq === void 0 ? void 0 : startUpdateSeq !== endUpdateSeq;
   const snapshotUncertain = startUpdateSeq === void 0 || endUpdateSeq === void 0;
   const hasDocumentRows = rows.length > 1;

package/dist/managers/EntryManager/EntryManagerImpls.js

@@ -206,7 +206,8 @@ async function putDBEntryInternal(host, managers, note, onlyChunks, revisionTarg
 }
 function isTargetFile(host, filenameSrc) {
   const settings = host.services.setting.currentSettings();
-  const file = filenameSrc.startsWith(ICHeader) ? filenameSrc.substring(ICHeader.length) : filenameSrc;
+  const isInternalFile = filenameSrc.startsWith(ICHeader);
+  const file = isInternalFile ? filenameSrc.substring(ICHeader.length) : filenameSrc;
   if (file.startsWith(ICXHeader)) return true;
   if (file.startsWith(PSCHeader)) return true;
   if (file.startsWith(IDPrefixes.Chunk) || file.startsWith(IDPrefixes.Obfuscated)) {
@@ -217,7 +218,7 @@ function isTargetFile(host, filenameSrc) {
       return false;
     }
   }
-  if (settings.syncOnlyRegEx) {
+  if (!isInternalFile && settings.syncOnlyRegEx) {
     const syncOnly = getFileRegExp(settings, "syncOnlyRegEx");
     if (syncOnly.length > 0 && !syncOnly.some((e) => e.test(file))) return false;
   }

package/dist/pouchdb/LiveSyncDBFunctions.js

@@ -86,6 +86,10 @@ async function ensureRemoteIsCompatible(infoSrc, setting, deviceNodeID, currentV
       writeMilestone = true;
     }
   }
+  if (nodeDeviceInfo.platform !== void 0 && info.platform !== nodeDeviceInfo.platform) {
+    remoteMilestone.node_info[deviceNodeID].platform = nodeDeviceInfo.platform;
+    writeMilestone = true;
+  }
   const diffLastConnected = now - (remoteMilestone.node_info[deviceNodeID].last_connected || 0);
   if (diffLastConnected > 6e4) {
     remoteMilestone.node_info[deviceNodeID].last_connected = now;

package/dist/pouchdb/StreamingFetch.d.ts

@@ -54,6 +54,10 @@ export type FetchChangesForInitialSyncProgress = {
  * @param decryptFunction Function to decrypt each document.
  * @param since Sequence ID to start fetching changes from (default is '0').
  * @param customHeaders Additional request headers required by the CouchDB endpoint or its reverse proxy.
+ * @param onStoredRevisions Observes revision identities after their complete local batch is persisted.
  */
-export declare function fetchChangesForInitialSync(downloadToDB: PouchDB.Database, remoteDbUrl: string, authHeader: string, decryptFunction: (doc: EntryDoc) => Promise<AnyEntry | EntryLeaf>, since?: number | string, onProgress?: (progress: FetchChangesForInitialSyncProgress) => void, onCheckpoint?: (sequence: DBSequence) => void | Promise<void>, customHeaders?: Record<string, string>): Promise<void>;
+export declare function fetchChangesForInitialSync(downloadToDB: PouchDB.Database, remoteDbUrl: string, authHeader: string, decryptFunction: (doc: EntryDoc) => Promise<AnyEntry | EntryLeaf>, since?: number | string, onProgress?: (progress: FetchChangesForInitialSyncProgress) => void, onCheckpoint?: (sequence: DBSequence) => void | Promise<void>, customHeaders?: Record<string, string>, onStoredRevisions?: (revisions: readonly {
+    id: string;
+    rev: string;
+}[]) => void | Promise<void>): Promise<void>;
 export {};