@vrtmrz/livesync-commonlib 0.1.43 → 0.1.44

package/dist/diagnostics/integrity.js

@@ -32,6 +32,12 @@ const INITIAL_COVERAGE = {
 function isRecord(value) {
   return value !== null && typeof value === "object";
 }
+function isFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+function isRecordedSize(value) {
+  return isFiniteNumber(value) && value >= 0;
+}
 function isCurrentMetadata(value) {
   return isRecord(value) && (value.type === EntryTypes.NOTE_BINARY || value.type === EntryTypes.NOTE_PLAIN);
 }
@@ -52,10 +58,140 @@ function hasUpdateSequence(value) {
 function appendError(errors, source, reason, details = {}) {
   errors.push({ source, reason, ...details });
 }
-function progress(options, phase, processed) {
+function concurrencyLimit(options) {
+  return Number.isFinite(options.concurrency) ? Math.max(1, Math.min(16, Math.floor(options.concurrency))) : 4;
+}
+class InspectionPhase {
+  constructor(options, phase, source) {
+    this.options = options;
+    this.phase = phase;
+    this.source = source;
+    this.log("started");
+  }
+  options;
+  phase;
+  source;
+  processed = 0;
+  total;
+  started = performance.now();
+  lastLog = this.started;
+  activeReads = 0;
+  peakActiveReads = 0;
+  readOperations = 0;
+  readDurationMs = 0;
+  async read(read, documents = 1) {
+    const started = performance.now();
+    this.readOperations += documents;
+    this.peakActiveReads = Math.max(this.peakActiveReads, ++this.activeReads);
+    try {
+      return await read();
+    } finally {
+      this.activeReads--;
+      this.readDurationMs += performance.now() - started;
+    }
+  }
+  update() {
+    if (this.total !== void 0) this.total = Math.max(this.total, this.processed);
+    if (this.phase !== "grouping") {
+      try {
+        this.options.onProgress?.({
+          phase: this.phase,
+          source: this.source,
+          processed: this.processed,
+          total: this.total,
+          elapsedMs: Math.round(performance.now() - this.started)
+        });
+      } catch {
+      }
+    }
+    if (performance.now() - this.lastLog >= 5e3) this.log("progress");
+  }
+  log(event) {
+    this.lastLog = performance.now();
+    try {
+      this.options.onLog?.({
+        phase: this.phase,
+        source: this.source,
+        event,
+        processed: this.processed,
+        total: this.total,
+        elapsedMs: Math.round(this.lastLog - this.started),
+        concurrency: concurrencyLimit(this.options),
+        activeReads: this.activeReads,
+        peakActiveReads: this.peakActiveReads,
+        readOperations: this.readOperations,
+        readDurationMs: Math.round(this.readDurationMs)
+      });
+    } catch {
+    }
+  }
+}
+async function inspectBatches(items, options, inspect, merge) {
+  const pending = [];
+  const drain = async () => {
+    const results = await Promise.allSettled(pending.splice(0));
+    let failure;
+    for (const result of results) {
+      if (result.status === "fulfilled") merge(result.value);
+      else failure ??= result;
+    }
+    if (failure) throw failure.reason;
+  };
   try {
-    options.onProgress?.({ phase, processed });
-  } catch {
+    for await (const item of items) {
+      if (cancelled(options)) return false;
+      const task = inspect(item);
+      void task.catch(() => {
+      });
+      pending.push(task);
+      if (pending.length >= concurrencyLimit(options)) await drain();
+      if (cancelled(options)) return false;
+    }
+    return !cancelled(options);
+  } finally {
+    await drain();
+  }
+}
+const DOCUMENT_BATCH_SIZE = 64;
+async function* documentBatches(ids, options) {
+  let pending = [];
+  let failure;
+  try {
+    for await (const id of ids) {
+      if (cancelled(options)) return;
+      pending.push(id);
+      if (pending.length === DOCUMENT_BATCH_SIZE) {
+        yield pending;
+        pending = [];
+      }
+    }
+  } catch (error) {
+    failure = { error };
+  }
+  if (pending.length && !cancelled(options)) yield pending;
+  if (failure) throw failure.error;
+}
+async function readMany(source, ids, phase, options) {
+  if (!ids.length) return /* @__PURE__ */ new Map();
+  try {
+    const results = await phase.read(() => source.getMany(ids, options), ids.length);
+    const byId = new Map(results.map((result) => [result.id, result]));
+    return new Map(
+      ids.map((id) => [id, byId.get(id) ?? { id, error: new DiagnosticReadError("document-unreadable") }])
+    );
+  } catch (error) {
+    return new Map(ids.map((id) => [id, { id, error }]));
+  }
+}
+function readResult(result) {
+  if ("error" in result) throw result.error;
+  return result.document;
+}
+function mergeSourceWork(target, work) {
+  target.errors.push(...work.errors);
+  target.observations.push(...work.observations);
+  for (const key of ["documents", "metadata", "excluded", "conflictBranches", "contentReads"]) {
+    target.coverage[key] += work.coverage[key];
   }
 }
 function cancelled(options) {
@@ -98,6 +234,7 @@ async function classifyObservation(options, run, document, branch, conflictCount
       return void 0;
     }
     const rawPath = document.path;
+    const recordedSize = isRecordedSize(document.size) ? document.size : void 0;
     const mismatch = identity.status === MetadataDocumentIdentityStatuses.UNRESOLVED ? identity.diagnostic.reason : void 0;
     const report = {
       reference,
@@ -113,6 +250,16 @@ async function classifyObservation(options, run, document, branch, conflictCount
       pathIsNFC: rawPath === rawPath.normalize("NFC"),
       caseEquivalentPaths: false,
       nfcEquivalentPaths: false,
+      metadata: {
+        type: document.type,
+        ...recordedSize !== void 0 ? { recordedSize } : {},
+        chunkReferences: document.children.length,
+        uniqueChunkReferences: new Set(document.children).size,
+        embeddedChunkReferences: document.children.reduce(
+          (count, child) => count + (getEdenChunk(document, child) !== void 0 ? 1 : 0),
+          0
+        )
+      },
       content: {
         status: "not-checked",
         chunksRead: 0,
@@ -136,106 +283,121 @@ async function classifyObservation(options, run, document, branch, conflictCount
     return void 0;
   }
 }
-async function scanSource(options, run, nextReference, metadataProgress) {
+async function inspectMetadataDocument(options, run, id, phase, prefetched) {
+  if (cancelled(options)) return;
+  run.coverage.documents++;
+  if (isSkippedDocumentId(id)) {
+    run.coverage.excluded++;
+    return;
+  }
+  let winnerValue;
+  try {
+    winnerValue = prefetched === void 0 ? await phase.read(() => run.source.get(id, { conflicts: true })) : readResult(prefetched);
+  } catch (error) {
+    if (cancelled(options)) return;
+    appendError(run.errors, run.name, reasonFromReadError(error, "document-unreadable"), { id });
+    return;
+  }
+  if (cancelled(options)) return;
+  if (!isRecord(winnerValue) || winnerValue._id !== id) {
+    appendError(run.errors, run.name, "invalid-metadata", { id });
+    return;
+  }
+  if (!isCurrentMetadata(winnerValue)) {
+    run.coverage.excluded++;
+    return;
+  }
+  const winner = winnerValue;
+  if (!isValidCurrentMetadata(winner)) {
+    appendError(run.errors, run.name, "invalid-metadata", {
+      id,
+      revision: typeof winner._rev === "string" ? winner._rev : void 0
+    });
+    return;
+  }
+  const conflictRevisions = Array.isArray(winner._conflicts) ? winner._conflicts.filter((revision) => typeof revision === "string") : [];
+  await classifyObservation(options, run, winner, "winner", conflictRevisions.length, "pending");
+  if (cancelled(options)) return;
+  for (const revision of conflictRevisions) {
+    if (cancelled(options)) return;
+    let branchValue;
+    try {
+      branchValue = await phase.read(() => run.source.get(id, { rev: revision, conflicts: true }));
+    } catch (error) {
+      if (cancelled(options)) return;
+      appendError(run.errors, run.name, reasonFromReadError(error, "revision-unavailable"), {
+        id,
+        revision
+      });
+      continue;
+    }
+    if (cancelled(options)) return;
+    if (!isRecord(branchValue) || branchValue._id !== id || branchValue._rev !== revision) {
+      appendError(run.errors, run.name, "revision-unavailable", { id, revision });
+      continue;
+    }
+    if (!isCurrentMetadata(branchValue)) {
+      run.coverage.excluded++;
+      continue;
+    }
+    await classifyObservation(options, run, branchValue, "conflict", conflictRevisions.length, "pending");
+  }
+}
+async function scanSource(options, run, nextReference) {
   if (cancelled(options)) return;
+  const phase = new InspectionPhase(options, "metadata", run.name);
   try {
     const info = await run.source.info();
     run.startSeq = info.update_seq;
+    if (Number.isSafeInteger(info.doc_count) && info.doc_count >= 0) phase.total = info.doc_count;
     if (!hasUpdateSequence(run.startSeq)) appendError(run.errors, run.name, "source-unavailable");
   } catch {
     appendError(run.errors, run.name, "source-unavailable");
   }
-  if (cancelled(options)) return;
+  phase.update();
   let enumerationComplete = false;
+  const merge = (work) => {
+    for (const observation of work.observations) {
+      observation.report.reference = nextReference(run.name);
+      observation.report.vault.reference = `vault:${observation.report.reference}`;
+    }
+    mergeSourceWork(run, work);
+  };
+  const inspect = async (id, prefetched) => {
+    const work = makeSourceRun(run.name, run.source);
+    if (typeof id !== "string") appendError(work.errors, run.name, "source-unavailable");
+    else await inspectMetadataDocument(options, work, id, phase, prefetched);
+    if (!cancelled(options)) {
+      phase.processed++;
+      phase.update();
+    }
+    return work;
+  };
   try {
-    for await (const id of run.source.enumerateIds()) {
-      if (cancelled(options)) return;
-      if (typeof id !== "string") {
-        appendError(run.errors, run.name, "source-unavailable");
-        continue;
-      }
-      run.coverage.documents++;
-      metadataProgress.processed++;
-      progress(options, "metadata", metadataProgress.processed);
-      if (isSkippedDocumentId(id)) {
-        run.coverage.excluded++;
-        continue;
-      }
-      if (cancelled(options)) return;
-      let winnerValue;
-      try {
-        winnerValue = await run.source.get(id, { conflicts: true });
-      } catch (error) {
-        if (cancelled(options)) return;
-        appendError(run.errors, run.name, reasonFromReadError(error, "document-unreadable"), { id });
-        continue;
-      }
-      if (cancelled(options)) return;
-      if (!isRecord(winnerValue) || winnerValue._id !== id) {
-        appendError(run.errors, run.name, "invalid-metadata", { id });
-        continue;
-      }
-      if (!isCurrentMetadata(winnerValue)) {
-        run.coverage.excluded++;
-        continue;
-      }
-      const winner = winnerValue;
-      if (!isValidCurrentMetadata(winner)) {
-        appendError(run.errors, run.name, "invalid-metadata", {
-          id,
-          revision: typeof winner._rev === "string" ? winner._rev : void 0
-        });
-        continue;
-      }
-      const conflictRevisions = Array.isArray(winner._conflicts) ? winner._conflicts.filter((revision) => typeof revision === "string") : [];
-      await classifyObservation(
-        options,
-        run,
-        winner,
-        "winner",
-        conflictRevisions.length,
-        nextReference(run.name)
-      );
-      if (cancelled(options)) return;
-      for (const revision of conflictRevisions) {
-        if (cancelled(options)) return;
-        let branchValue;
-        try {
-          branchValue = await run.source.get(id, { rev: revision, conflicts: true });
-        } catch (error) {
-          if (cancelled(options)) return;
-          appendError(run.errors, run.name, reasonFromReadError(error, "revision-unavailable"), {
-            id,
-            revision
-          });
-          continue;
-        }
-        if (cancelled(options)) return;
-        if (!isRecord(branchValue) || branchValue._id !== id || branchValue._rev !== revision) {
-          appendError(run.errors, run.name, "revision-unavailable", { id, revision });
-          continue;
-        }
-        if (!isCurrentMetadata(branchValue)) {
-          run.coverage.excluded++;
-          continue;
+    if (!cancelled(options)) {
+      if (run.source.getMany) {
+        for await (const ids of documentBatches(run.source.enumerateIds(), options)) {
+          if (cancelled(options)) break;
+          const targets = ids.filter((id) => typeof id === "string" && !isSkippedDocumentId(id));
+          const prefetched = await readMany(run.source, targets, phase, { conflicts: true });
+          if (cancelled(options)) break;
+          await inspectBatches(ids, options, (id) => inspect(id, prefetched.get(id)), merge);
         }
-        await classifyObservation(
-          options,
-          run,
-          branchValue,
-          "conflict",
-          conflictRevisions.length,
-          nextReference(run.name)
-        );
+        enumerationComplete = !cancelled(options);
+      } else {
+        enumerationComplete = await inspectBatches(run.source.enumerateIds(), options, inspect, merge);
       }
     }
-    enumerationComplete = true;
   } catch {
     if (!cancelled(options)) appendError(run.errors, run.name, "source-unavailable");
   }
-  if (cancelled(options)) return;
-  run.metadataScanComplete = enumerationComplete && run.errors.length === 0;
-  run.enumerationComplete = enumerationComplete;
+  if (!cancelled(options)) {
+    run.metadataScanComplete = enumerationComplete && run.errors.length === 0;
+    run.enumerationComplete = enumerationComplete;
+    if (enumerationComplete) phase.total = phase.processed;
+  }
+  phase.update();
+  phase.log(cancelled(options) ? "cancelled" : run.metadataScanComplete ? "completed" : "partial");
 }
 function groupObservations(runs) {
   const observations = runs.flatMap((run) => run.observations);
@@ -277,19 +439,42 @@ function markPathEquivalences(observations) {
   for (const observation of observations) {
     const path = observation.report.path;
     const caseFold = path.toLowerCase();
-    const casePaths = pathsByCaseFold.get(caseFold) ?? /* @__PURE__ */ new Set();
-    casePaths.add(path);
+    const casePaths = pathsByCaseFold.get(caseFold) ?? [];
+    casePaths.push(observation);
     pathsByCaseFold.set(caseFold, casePaths);
     const nfc = path.normalize("NFC");
-    const nfcPaths = pathsByNfc.get(nfc) ?? /* @__PURE__ */ new Set();
-    nfcPaths.add(path);
+    const nfcPaths = pathsByNfc.get(nfc) ?? [];
+    nfcPaths.push(observation);
     pathsByNfc.set(nfc, nfcPaths);
   }
-  for (const observation of observations) {
-    const path = observation.report.path;
-    observation.report.caseEquivalentPaths = (pathsByCaseFold.get(path.toLowerCase())?.size ?? 0) > 1;
-    observation.report.nfcEquivalentPaths = (pathsByNfc.get(path.normalize("NFC"))?.size ?? 0) > 1;
+  const equivalenceGroups = [];
+  for (const [kind, groups] of [
+    ["case", pathsByCaseFold],
+    ["nfc", pathsByNfc]
+  ]) {
+    let groupNumber = 0;
+    for (const members of groups.values()) {
+      const hasDifferentSpellings = new Set(members.map((observation) => observation.report.path)).size > 1;
+      if (kind === "case") {
+        for (const observation of members) observation.report.caseEquivalentPaths = hasDifferentSpellings;
+      } else {
+        for (const observation of members) observation.report.nfcEquivalentPaths = hasDifferentSpellings;
+      }
+      if (!hasDifferentSpellings) continue;
+      const reference = `${kind}:${++groupNumber}`;
+      if (kind === "case") {
+        for (const observation of members) observation.report.caseEquivalentGroup = reference;
+      } else {
+        for (const observation of members) observation.report.nfcEquivalentGroup = reference;
+      }
+      equivalenceGroups.push({
+        reference,
+        kind,
+        members: members.map((observation) => observation.report.reference)
+      });
+    }
   }
+  return equivalenceGroups;
 }
 function markSuspectGroups(groups, runs) {
   const crossSourceComparisonAvailable = runs.some((run) => run.name === "local" && run.metadataScanComplete) && runs.some((run) => run.name === "remote" && run.metadataScanComplete);
@@ -336,7 +521,7 @@ function getEdenChunk(document, id) {
   if (!isRecord(chunk) || typeof chunk.data !== "string") return void 0;
   return chunk.data;
 }
-async function readObservationContent(options, observation, run) {
+async function readObservationContent(options, observation, run, phase) {
   run.coverage.contentReads++;
   const unavailableChunks = [];
   const data = [];
@@ -353,54 +538,63 @@ async function readObservationContent(options, observation, run) {
       unavailableChunks
     };
   }
-  for (const child of observation.document.children) {
+  for (let offset = 0; offset < observation.document.children.length; offset += DOCUMENT_BATCH_SIZE) {
     if (cancelled(options)) return { content: { status: "not-checked" }, chunksRead, unavailableChunks };
-    const eden = getEdenChunk(observation.document, child);
-    if (eden !== void 0) {
-      data.push(eden);
-      chunksRead++;
-      continue;
-    }
-    try {
-      const chunk = await observation.source.get(child);
+    const children = observation.document.children.slice(offset, offset + DOCUMENT_BATCH_SIZE);
+    const prefetched = observation.source.getMany ? await readMany(
+      observation.source,
+      [...new Set(children.filter((child) => getEdenChunk(observation.document, child) === void 0))],
+      phase
+    ) : void 0;
+    for (const child of children) {
       if (cancelled(options)) return { content: { status: "not-checked" }, chunksRead, unavailableChunks };
-      if (!isValidChunk(chunk, child) || chunk._deleted === true) {
-        unavailableChunks.push({ id: child, status: "unreadable" });
-        appendError(run.errors, run.name, "chunk-unreadable", {
-          id: observation.report.id,
-          revision: observation.report.revision,
-          path: observation.report.path,
-          chunkId: child
-        });
+      const eden = getEdenChunk(observation.document, child);
+      if (eden !== void 0) {
+        data.push(eden);
+        chunksRead++;
         continue;
       }
-      data.push(chunk.data);
-      chunksRead++;
-    } catch (error) {
-      if (cancelled(options)) return { content: { status: "not-checked" }, chunksRead, unavailableChunks };
-      if (isMissingRead(error)) {
-        const eden2 = getEdenChunk(observation.document, child);
-        if (eden2 !== void 0) {
-          data.push(eden2);
-          chunksRead++;
+      try {
+        const chunk = prefetched ? readResult(prefetched.get(child)) : await phase.read(() => observation.source.get(child));
+        if (cancelled(options)) return { content: { status: "not-checked" }, chunksRead, unavailableChunks };
+        if (!isValidChunk(chunk, child) || chunk._deleted === true) {
+          unavailableChunks.push({ id: child, status: "unreadable" });
+          appendError(run.errors, run.name, "chunk-unreadable", {
+            id: observation.report.id,
+            revision: observation.report.revision,
+            path: observation.report.path,
+            chunkId: child
+          });
+          continue;
+        }
+        data.push(chunk.data);
+        chunksRead++;
+      } catch (error) {
+        if (cancelled(options)) return { content: { status: "not-checked" }, chunksRead, unavailableChunks };
+        if (isMissingRead(error)) {
+          const eden2 = getEdenChunk(observation.document, child);
+          if (eden2 !== void 0) {
+            data.push(eden2);
+            chunksRead++;
+            continue;
+          }
+          unavailableChunks.push({ id: child, status: "missing" });
+          appendError(run.errors, run.name, reasonFromReadError(error, "chunk-missing"), {
+            id: observation.report.id,
+            revision: observation.report.revision,
+            path: observation.report.path,
+            chunkId: child
+          });
           continue;
         }
-        unavailableChunks.push({ id: child, status: "missing" });
-        appendError(run.errors, run.name, reasonFromReadError(error, "chunk-missing"), {
+        unavailableChunks.push({ id: child, status: "unreadable" });
+        appendError(run.errors, run.name, reasonFromReadError(error, "chunk-unreadable"), {
           id: observation.report.id,
           revision: observation.report.revision,
           path: observation.report.path,
           chunkId: child
         });
-        continue;
       }
-      unavailableChunks.push({ id: child, status: "unreadable" });
-      appendError(run.errors, run.name, reasonFromReadError(error, "chunk-unreadable"), {
-        id: observation.report.id,
-        revision: observation.report.revision,
-        path: observation.report.path,
-        chunkId: child
-      });
     }
   }
   if (unavailableChunks.length > 0) {
@@ -451,10 +645,10 @@ function relation(left, right) {
   }
   return "equal";
 }
-async function readVaultContent(options, path, errors) {
+async function readVaultContent(options, path, errors, phase) {
   if (cancelled(options)) return { status: "not-checked" };
   try {
-    const content = await options.readStorage(path);
+    const content = await phase.read(() => options.readStorage(path));
     if (cancelled(options)) return { status: "not-checked" };
     if (content.status === "unreadable" || content.reason) {
       appendError(errors, "storage", content.reason ?? "content-unreadable", { path });
@@ -467,7 +661,7 @@ async function readVaultContent(options, path, errors) {
     return { status: "unreadable", reason };
   }
 }
-async function inspectGroupContent(options, group, runsByName, errors, contentProgress) {
+async function inspectGroupContent(options, group, runsByName, errors, phase) {
   if (!group.suspect) return;
   const byRawPath = /* @__PURE__ */ new Map();
   for (const observation of group.observations) {
@@ -489,32 +683,42 @@ async function inspectGroupContent(options, group, runsByName, errors, contentPr
     if (cancelled(options)) return;
     const run = runsByName.get(observation.report.source);
     if (!run) continue;
-    const outcome = await readObservationContent(options, observation, run);
+    const outcome = await readObservationContent(options, observation, run, phase);
+    const decodedSize = outcome.content.status === "available" ? outcome.content.bytes.length : void 0;
+    const recordedSize = observation.report.metadata?.recordedSize;
     observation.report.content = {
       status: outcome.content.status,
       ...outcome.content.reason ? { reason: outcome.content.reason } : {},
       chunksRead: outcome.chunksRead,
-      unavailableChunks: outcome.unavailableChunks
+      unavailableChunks: outcome.unavailableChunks,
+      ...decodedSize !== void 0 ? {
+        size: decodedSize,
+        ...recordedSize !== void 0 ? { recordedSizeDifference: decodedSize - recordedSize } : {}
+      } : {}
     };
     contentByObservation.set(observation, outcome.content);
     contentByReference.set(observation.report.reference, outcome.content);
-    contentProgress.processed++;
-    progress(options, "content", contentProgress.processed);
     if (cancelled(options)) return;
   }
   const vaultByPath = /* @__PURE__ */ new Map();
   for (const [path] of byRawPath) {
     if (cancelled(options)) return;
-    const vaultContent = await readVaultContent(options, path, errors);
+    const vaultContent = await readVaultContent(options, path, errors, phase);
     vaultByPath.set(path, vaultContent);
     const vaultReference = vaultReferences.get(path);
     if (vaultReference) contentByReference.set(vaultReference, vaultContent);
     for (const observation of group.observations) {
       if (observation.report.path !== path) continue;
+      const observationContent = contentByObservation.get(observation);
       observation.report.vault = {
         reference: vaultReference ?? `vault:${observation.report.reference}`,
         status: vaultContent.status,
-        ...vaultContent.reason ? { reason: vaultContent.reason } : {}
+        ...vaultContent.reason ? { reason: vaultContent.reason } : {},
+        ...vaultContent.status === "available" ? {
+          size: vaultContent.bytes.length,
+          ...observationContent?.status === "available" ? { contentSizeDifference: vaultContent.bytes.length - observationContent.bytes.length } : {},
+          ...isFiniteNumber(vaultContent.mtime) && isFiniteNumber(observation.document.mtime) ? { mtimeDifferenceMs: vaultContent.mtime - observation.document.mtime } : {}
+        } : {}
       };
     }
     if (cancelled(options)) return;
@@ -540,12 +744,11 @@ function determineStatus(wasCancelled, errors, runs, configurationChanged) {
 async function analyseDatabaseIntegrity(options) {
   const errors = [];
   const runs = [];
-  const metadataProgress = { processed: 0 };
-  const contentProgress = { processed: 0 };
   const referenceNumbers = { local: 0, remote: 0 };
   const nextReference = (source) => `${source}:${++referenceNumbers[source]}`;
   let wasCancelled = cancelled(options);
   let configurationChanged = false;
+  let pathEquivalenceGroups = [];
   let ownedRemote;
   if (!wasCancelled && !isConfigurationCurrent(options)) {
     configurationChanged = true;
@@ -555,8 +758,8 @@ async function analyseDatabaseIntegrity(options) {
     if (!wasCancelled && !configurationChanged) {
       const localRun = makeSourceRun("local", options.local);
       runs.push(localRun);
-      await scanSource(options, localRun, nextReference, metadataProgress);
-      if (!cancelled(options)) {
+      await scanSource(options, localRun, nextReference);
+      if (!cancelled(options) && options.includeRemote !== false) {
         if (!options.remote) {
           const remoteRun = { ...makeSourceRun("remote", options.local), source: options.local };
           remoteRun.coverage.status = "failed";
@@ -571,7 +774,7 @@ async function analyseDatabaseIntegrity(options) {
             } else {
               const remoteRun = makeSourceRun("remote", ownedRemote);
               runs.push(remoteRun);
-              await scanSource(options, remoteRun, nextReference, metadataProgress);
+              await scanSource(options, remoteRun, nextReference);
             }
           } catch {
             if (cancelled(options)) {
@@ -588,21 +791,43 @@ async function analyseDatabaseIntegrity(options) {
       }
       if (cancelled(options)) wasCancelled = true;
       if (!wasCancelled) {
+        const grouping = new InspectionPhase(options, "grouping");
         const groups = groupObservations(runs);
-        markPathEquivalences(runs.flatMap((run) => run.observations));
+        pathEquivalenceGroups = markPathEquivalences(runs.flatMap((run) => run.observations));
         markSuspectGroups(groups, runs);
-        const runsByName = new Map(runs.map((run) => [run.name, run]));
-        for (const group of groups) {
-          if (cancelled(options)) {
-            wasCancelled = true;
-            break;
-          }
-          await inspectGroupContent(options, group, runsByName, errors, contentProgress);
-          if (cancelled(options)) {
-            wasCancelled = true;
-            break;
+        grouping.processed = groups.length;
+        grouping.total = groups.length;
+        grouping.log("completed");
+        const suspects = groups.filter((group) => group.suspect);
+        const content = new InspectionPhase(options, "content");
+        content.total = suspects.length;
+        content.update();
+        await inspectBatches(
+          suspects,
+          options,
+          async (group) => {
+            const work = runs.map((run) => makeSourceRun(run.name, run.source));
+            const groupErrors = [];
+            await inspectGroupContent(
+              options,
+              group,
+              new Map(work.map((run) => [run.name, run])),
+              groupErrors,
+              content
+            );
+            if (!cancelled(options)) {
+              content.processed++;
+              content.update();
+            }
+            return { work, groupErrors };
+          },
+          ({ work, groupErrors }) => {
+            for (let index = 0; index < runs.length; index++) mergeSourceWork(runs[index], work[index]);
+            errors.push(...groupErrors);
           }
-        }
+        );
+        if (cancelled(options)) wasCancelled = true;
+        content.log(wasCancelled ? "cancelled" : "completed");
       }
     }
     if (!wasCancelled && !cancelled(options)) {
@@ -660,8 +885,10 @@ async function analyseDatabaseIntegrity(options) {
     settings: { ...options.settings },
     coverage,
     observations: allObservations,
+    ...pathEquivalenceGroups.length > 0 ? { pathEquivalenceGroups } : {},
     errors,
     scope: {
+      remote: options.includeRemote === false ? "excluded" : "included",
       metadata: "normal-files-and-current-conflicts",
       content: "suspect-groups-only",
       history: "not-checked",