@indigoai-us/hq-cloud 6.16.64 → 6.16.66

package/dist/cli/sync.js

@@ -6,11 +6,13 @@
  */
 import * as fs from "fs";
 import * as path from "path";
+import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
 import { VaultAuthError, VaultClient } from "../vault-client.js";
 import { readlinkOrNull } from "../lib/readlink-safe.js";
 import { describeError } from "../lib/describe-error.js";
 import { isRetryableIdentityRefusal, isTerminalIdentityRefusal, } from "../cognito-auth.js";
 import { RateLimitedError } from "../object-io.js";
+import { recoverHqRootLocalState } from "../hq-root-recovery.js";
 import { isTransientHttpStatusError } from "../lib/net-errors.js";
 import { emitCloudTelemetry, } from "../telemetry-events.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
@@ -18,7 +20,7 @@ import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
 import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
 import { SKILLS_KEY_PREFIX } from "./doctor.js";
-import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
+import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
 import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
 import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
 import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
@@ -187,6 +189,11 @@ export function targetedJournalSessionCensusSizes() {
  * dirty entries through a run-scoped journal writer.
  */
 const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
+/**
+ * A full-pull hand-off unit. Every member has finished its transfer and the
+ * resulting journal delta is durable before realtime can acquire the writer.
+ */
+export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
 /**
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -320,36 +327,59 @@ export async function sync(options) {
     return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
 }
 async function syncWithOperationLockHeld(options) {
-    const run = await buildPullContext(options);
     try {
-        const plan = planPull(run);
-        emitPullPlan(run.emit, plan);
-        reportInvalidRemoteKeys(run, plan);
-        const scopeRun = run.keyedJournal
-            ? emptyTargetedScopeRun(run)
-            : executeScopeShrink(run, planScopeShrink(run));
-        const counters = createPullCounters();
-        const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
-        const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
-        if (conflictRun.abortResult) {
-            return conflictRun.abortResult;
-        }
-        await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
-        // The following added-by and tombstone phases may still make authenticated
-        // vault calls. Make a completed sub-threshold download tail durable before
-        // they begin, so their failure cannot turn a successful bulk transfer into
-        // a full replay on the next pass.
-        flushDownloadedJournalTail(run);
-        await emitAndReportNewFiles(run, plan);
-        await verifyPlannedJournalTombstones(run, plan);
-        executeJournalTombstoneDeletes(run, plan, counters);
-        emitScopeMaterializationGap(run, plan);
-        return finalizePullRun(run, plan, scopeRun, counters);
+        recoverHqRootLocalState(options.hqRoot);
     }
-    finally {
-        // Release a pass-local lease even on conflict, transfer, or auth failure.
-        // The one bounded hot lease remains available only for its same vault.
-        run.releaseKeyedJournal?.();
+    catch (err) {
+        console.warn("hq-root-recovery: local HQ recovery failed:", err);
+    }
+    let carried;
+    for (;;) {
+        const run = await buildPullContext(options);
+        if (carried)
+            run.carriedProgress = carried;
+        try {
+            const plan = await planPull(run);
+            emitPullPlan(run.emit, plan);
+            reportInvalidRemoteKeys(run, plan);
+            const scopeRun = run.keyedJournal
+                ? emptyTargetedScopeRun(run)
+                : executeScopeShrink(run, planScopeShrink(run));
+            const counters = createPullCounters();
+            // A streaming all-mode planner retains only executor-relevant items. Its
+            // ordinary skips have already been counted while the page was live.
+            counters.filesSkipped = plan.preplannedFilesSkipped;
+            counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
+            seedCarriedPullProgress(counters, carried);
+            const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
+            const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
+            if (conflictRun.abortResult) {
+                return conflictRun.abortResult;
+            }
+            await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
+            // The following added-by and tombstone phases may still make authenticated
+            // vault calls. Make a completed sub-threshold download tail durable before
+            // they begin, so their failure cannot turn a successful bulk transfer into
+            // a full replay on the next pass.
+            flushDownloadedJournalTail(run);
+            await emitAndReportNewFiles(run, plan);
+            await verifyPlannedJournalTombstones(run, plan);
+            executeJournalTombstoneDeletes(run, plan, counters);
+            emitScopeMaterializationGap(run, plan);
+            return finalizePullRun(run, plan, scopeRun, counters);
+        }
+        catch (error) {
+            if (error instanceof CooperativePullHandoff) {
+                carried = error.completed;
+                continue;
+            }
+            throw error;
+        }
+        finally {
+            // Release a pass-local lease even on conflict, transfer, or auth failure.
+            // The one bounded hot lease remains available only for its same vault.
+            run.releaseKeyedJournal?.();
+        }
     }
 }
 async function buildPullContext(options) {
@@ -438,8 +468,9 @@ async function buildPullContext(options) {
         ? targetedJournalSession(journalSlug)
         : undefined;
     const keyedJournal = keyedJournalLease?.store;
+    let remoteFiles;
     try {
-        const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
+        remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
         const fileTombstones = options.fileTombstones ??
             await fetchFileTombstones(vaultConfig, options.personalMode === true
                 ? { personal: true }
@@ -487,43 +518,27 @@ async function buildPullContext(options) {
     catch (error) {
         // `buildPullContext` can fail during LIST/freshness. Do not leave the
         // leased projection marked in use when no PullRunContext is returned.
+        if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
+            remoteFiles.dispose();
         keyedJournalLease?.release();
         throw error;
     }
 }
-/**
- * List the remote objects the current pull can actually materialize.
- *
- * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
- * a `shared`/`custom` membership whose effective scope is a handful of
- * prefixes — so every pull materialized the entire remote listing (tens of
- * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
- * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
- * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
- * bounded parallel, deduped), which bounds the listing's memory to the scope's
- * actual footprint.
- *
- * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
- * the single broad listing: fanning `all` out by top-level prefix would need a
- * delimiter LIST to discover the top-level names first and still union the
- * same total object count in one plan, so it bounds nothing today. (A
- * streaming planner is the real fix for `all` and is out of scope here.)
- *
- * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
- * issue zero LIST calls. Consumers already treat an empty listing correctly:
- * the deletion sweep in `computePullPlan` only considers journal keys covered
- * by the LISTED scope, so keys absent because they were never listed can never
- * be mistaken for remote deletes.
- */
+function isStreamingRemoteFileSource(source) {
+    return !Array.isArray(source);
+}
+function allModeRemoteFileStream(ctx) {
+    return spoolRemoteFiles(streamRemoteFiles(ctx));
+}
 async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
     const entries = toScopePrefixEntries(listedPrefixSet);
     if (entries.some((e) => e.prefix === ""))
-        return listRemoteFiles(ctx);
+        return allModeRemoteFileStream(ctx);
     if (entries.length === 0) {
         // `all` mode always carries `[""]`, so an empty set only occurs for a
         // scoped mode (empty grants, or a targeted intersection that excluded
         // everything). Defensive `all` branch: never silently narrow a full pull.
-        return syncMode === "all" ? listRemoteFiles(ctx) : [];
+        return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
     }
     return listRemoteForScope({
         ctx,
@@ -544,8 +559,14 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
         },
     });
 }
-function planPull(run) {
-    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
+async function planPull(run) {
+    try {
+        return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
+    }
+    finally {
+        if (isStreamingRemoteFileSource(run.remoteFiles))
+            run.remoteFiles.dispose();
+    }
 }
 /** Build only the rows a targeted remote LIST can observe. */
 function journalForTargetedPull(store, prefixes) {
@@ -658,9 +679,55 @@ function createPullCounters() {
         conflictPaths: [],
         changedPaths: [],
         appliedUpserts: [],
+        newFiles: [],
         hasRetryableOutcome: false,
     };
 }
+/** Fold a previous attempt's completed work into a restarted run's counters. */
+function seedCarriedPullProgress(counters, carried) {
+    if (!carried)
+        return;
+    counters.filesDownloaded += carried.filesDownloaded;
+    counters.bytesDownloaded += carried.bytesDownloaded;
+    counters.conflicts += carried.conflicts;
+    counters.filesTombstoned += carried.filesTombstoned;
+    counters.conflictPaths.push(...carried.conflictPaths);
+    counters.changedPaths.push(...carried.changedPaths);
+    counters.appliedUpserts.push(...carried.appliedUpserts);
+    if (carried.hasRetryableOutcome)
+        counters.hasRetryableOutcome = true;
+}
+/**
+ * Snapshot what survives the hand-off. The numeric and path axes already
+ * include everything carried into this attempt; `newFiles` is accumulated
+ * separately so it stays a record of pre-restart downloads only, and cannot
+ * double up with the current plan's own new files.
+ */
+function carryPullProgress(previous, counters) {
+    return {
+        filesDownloaded: counters.filesDownloaded,
+        bytesDownloaded: counters.bytesDownloaded,
+        conflicts: counters.conflicts,
+        conflictPaths: [...counters.conflictPaths],
+        filesTombstoned: counters.filesTombstoned,
+        changedPaths: [...counters.changedPaths],
+        appliedUpserts: [...counters.appliedUpserts],
+        newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
+        hasRetryableOutcome: counters.hasRetryableOutcome,
+    };
+}
+/**
+ * Every new file this pull is responsible for reporting: the ones a previous
+ * attempt downloaded before handing off, then this plan's own. A carried path
+ * that the fresh plan also lists is reported once.
+ */
+function reportableNewFiles(run, plan) {
+    const carried = run.carriedProgress?.newFiles;
+    if (carried === undefined || carried.length === 0)
+        return plan.newFiles;
+    const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
+    return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
+}
 function planScopeShrink(run) {
     const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
     const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
@@ -1121,6 +1188,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         if (resolution === "abort") {
             run.emit({ type: "new-files", files: [] });
             persistPullJournal(run);
+            const abortedNewFiles = reportableNewFiles(run, plan);
             return {
                 filesDownloaded: counters.filesDownloaded,
                 bytesDownloaded: counters.bytesDownloaded,
@@ -1128,8 +1196,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 conflicts: counters.conflicts,
                 conflictPaths: counters.conflictPaths,
                 aborted: true,
-                newFiles: plan.newFiles,
-                newFilesCount: plan.newFilesCount,
+                newFiles: abortedNewFiles,
+                newFilesCount: abortedNewFiles.length,
                 filesExcludedByPolicy: plan.filesExcludedByPolicy,
                 filesTombstoned: 0,
                 filesOutOfScope: counters.filesOutOfScope,
@@ -1255,10 +1323,21 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             discardProbe();
     }
 }
-async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
-    if (downloadItems.length === 0)
-        return;
-    await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
+/**
+ * The old plan may have tombstone evidence that predates a realtime hand-off.
+ * Rebuilding from disk after a hand-off makes the continuation observe the
+ * realtime journal and a fresh LIST before it can perform another mutation.
+ */
+class CooperativePullHandoff extends Error {
+    /** Completed work the restarted pull must keep reporting. */
+    completed;
+    constructor(completed) {
+        super("full pull handed off to realtime work");
+        this.name = "CooperativePullHandoff";
+        this.completed = completed;
+    }
+}
+async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
     const queue = [...downloadItems];
     const inFlight = new Set();
     const workerErrors = [];
@@ -1273,7 +1352,7 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
             inFlight.size < transferConcurrency &&
             queue.length > 0) {
             const downloadItem = queue.shift();
-            const download = () => downloadOne(run, downloadItem, counters);
+            const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
             const p = (run.options.transferSemaphore
                 ? run.options.transferSemaphore.withPermit(download)
                 : download())
@@ -1297,16 +1376,44 @@ async function executeDownloadExecutor(run, downloadItems, transferConcurrency,
             break;
         }
     }
-    if (workerErrors.length > 0) {
-        persistPullJournal(run);
-        const first = workerErrors[0];
-        if (workerErrors.length > 1) {
-            first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
+    return workerErrors;
+}
+function throwDownloadWorkerErrors(run, workerErrors) {
+    persistPullJournal(run);
+    const first = workerErrors[0];
+    if (workerErrors.length > 1) {
+        first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
+    }
+    throw first;
+}
+async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
+    if (downloadItems.length === 0)
+        return;
+    await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
+    // Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
+    // Only a watch chore that supplied the cooperative seam pays for drained
+    // units and can return its root lock to queued realtime work.
+    if (!run.options.cooperativeCheckpoint) {
+        const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
+        if (workerErrors.length > 0)
+            throwDownloadWorkerErrors(run, workerErrors);
+        return;
+    }
+    for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
+        const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
+        const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
+        // `executeDownloadQueue` does not return until every member of this chunk
+        // has settled. Persist before the callback so the suspended chore owns no
+        // mutable pass state while realtime uses the root lock.
+        flushDownloadedJournalTail(run);
+        if (workerErrors.length > 0)
+            throwDownloadWorkerErrors(run, workerErrors);
+        if (await run.options.cooperativeCheckpoint() === true) {
+            throw new CooperativePullHandoff(carryPullProgress(run.carriedProgress, counters));
         }
-        throw first;
     }
 }
-async function downloadOne(run, downloadItem, counters) {
+async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
     const { remoteFile, localPath } = downloadItem;
     await refreshRunContextIfExpiring(run);
     if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
@@ -1375,6 +1482,9 @@ async function downloadOne(run, downloadItem, counters) {
         counters.filesDownloaded++;
         counters.bytesDownloaded += size;
         counters.changedPaths.push(remoteFile.key);
+        if (downloadItem.isNew) {
+            counters.newFiles.push({ path: remoteFile.key, bytes: remoteFile.size });
+        }
         // The event-push emitter hashes regular file bodies. Symlink journal hashes
         // describe link targets instead, and the startup baseline excludes them for
         // the same reason, so only hand regular-file downloads to that ledger.
@@ -1483,7 +1593,8 @@ async function downloadOne(run, downloadItem, counters) {
         return;
     }
     run.downloadsSinceJournalCheckpoint++;
-    if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
+    if (!deferJournalCheckpoint &&
+        run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
         flushDownloadedJournalTail(run);
     }
 }
@@ -1516,15 +1627,19 @@ function persistPullJournal(run) {
 }
 async function emitAndReportNewFiles(run, plan) {
     const enrichedNewFiles = [];
+    // New files a cooperative hand-off already downloaded are reported here too:
+    // the restarted plan cannot see them as new, so this is their only
+    // notification.
+    const newFiles = reportableNewFiles(run, plan);
     // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
     // created-by HEADs below reuse the cache instead of each minting its own
     // presign. Without this, a big catch-up pull (hundreds of new files) bursts
     // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
     // then fails. Mirrors the tombstone HEAD-verify pre-prime.
-    await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
+    await primeObjectTransport(run.ctx, "get", newFiles.map((nf) => nf.path));
     const HEAD_CONCURRENCY = 5;
-    for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
-        const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
+    for (let i = 0; i < newFiles.length; i += HEAD_CONCURRENCY) {
+        const batch = newFiles.slice(i, i + HEAD_CONCURRENCY);
         const results = await Promise.all(batch.map(async (nf) => {
             let addedBy = null;
             try {
@@ -1705,6 +1820,7 @@ function finalizePullRun(run, plan, scopeRun, counters) {
             // best-effort: a post-sync refresh failure never fails the sync
         }
     }
+    const newFiles = reportableNewFiles(run, plan);
     const result = {
         filesDownloaded: counters.filesDownloaded,
         bytesDownloaded: counters.bytesDownloaded,
@@ -1712,8 +1828,8 @@ function finalizePullRun(run, plan, scopeRun, counters) {
         conflicts: counters.conflicts,
         conflictPaths: counters.conflictPaths,
         aborted: false,
-        newFiles: plan.newFiles,
-        newFilesCount: plan.newFilesCount,
+        newFiles,
+        newFilesCount: newFiles.length,
         filesExcludedByPolicy: plan.filesExcludedByPolicy,
         filesTombstoned: counters.filesTombstoned,
         filesOutOfScope: counters.filesOutOfScope,
@@ -1942,7 +2058,8 @@ class LocalSnapshotChangedError extends Error {
  * the real outcome of the upcoming Stage-2 execution rather than an
  * upper-bound guess.
  *
- * Pure function: no S3 calls, no journal writes, no event emission. The
+ * It performs no writes, journal mutations, or event emission. An all-mode
+ * source performs its paged classification LIST while it is consumed; the
  * caller (`sync()`) is responsible for emitting the resulting plan event
  * before iterating `items`.
  */
@@ -2010,6 +2127,36 @@ excludePrefixes = [],
 // callers preserve the pre-policy, watcher-intent-only behavior.
 propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
     const items = [];
+    const streaming = isStreamingRemoteFileSource(remoteFiles);
+    const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
+    let preplannedFilesSkipped = 0;
+    let preplannedFilesOutOfScope = 0;
+    let preplannedFilesExcludedByPolicy = 0;
+    if (streaming) {
+        const retain = items.push.bind(items);
+        items.push = (...next) => {
+            for (const item of next) {
+                // These actions still need the executor for an event or a state change.
+                if (item.action === "download" ||
+                    item.action === "conflict" ||
+                    item.action === "tombstone-delete" ||
+                    item.action === "skip-junk-key-spelling" ||
+                    item.action === "skip-unreadable-link") {
+                    retain(item);
+                }
+                else if (item.action === "skip-out-of-scope") {
+                    preplannedFilesOutOfScope++;
+                }
+                else if (item.action === "skip-excluded-policy") {
+                    preplannedFilesExcludedByPolicy++;
+                }
+                else {
+                    preplannedFilesSkipped++;
+                }
+            }
+            return items.length;
+        };
+    }
     // Remote keys the server will never presign — collected so the caller can
     // warn + emit telemetry without allowing one poison object to fail a pull.
     const invalidRemoteKeys = [];
@@ -2025,8 +2172,35 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // never gated by this map (see the guard below), so pre-existing families
     // keep syncing untouched until the vault doctor (US-003) collapses them.
     const journaledKeyByCanonical = new Map();
+    // The delete sweep starts with journaled keys and clears them as the streamed
+    // LIST proves each one present. This retains existing journal-key references
+    // instead of a second, whole-vault population of long remote key strings.
+    const unseenJournalKeys = new Set();
+    // Most journal keys have already been normalized to POSIX and can be cleared
+    // directly. Keep the legacy Windows spelling defense-in-depth without
+    // imposing a second map entry on every current journal row.
+    const legacyJournalKeysByPosix = new Map();
+    // The junk-spelling delete guard needs all journal twins, while
+    // `journaledKeyByCanonical` above deliberately retains only the preferred
+    // representative for classification.
+    const journaledSkillKeysByCanonical = new Map();
     for (const journaledKey of journalRows(journal).keys()) {
         onHeartbeat?.();
+        const posixJournaledKey = toPosixKey(journaledKey);
+        // A scoped LIST cannot prove anything about keys outside its prefixes.
+        // Do not copy the entire vault's journal into its presence bookkeeping.
+        const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
+        if (inListedScope)
+            unseenJournalKeys.add(journaledKey);
+        if (inListedScope && posixJournaledKey !== journaledKey) {
+            const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
+            if (legacyKeys === undefined) {
+                legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
+            }
+            else {
+                legacyKeys.push(journaledKey);
+            }
+        }
         if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
             continue;
         const canonical = canonicalVaultKeySpelling(journaledKey);
@@ -2034,6 +2208,15 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
         if (existing === undefined || journaledKey === canonical) {
             journaledKeyByCanonical.set(canonical, journaledKey);
         }
+        if (!inListedScope)
+            continue;
+        const skillKeys = journaledSkillKeysByCanonical.get(canonical);
+        if (skillKeys === undefined) {
+            journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
+        }
+        else {
+            skillKeys.push(journaledKey);
+        }
     }
     // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
     // keys that carry child objects under `${key}/…` in this same LIST. The
@@ -2043,22 +2226,40 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // objects beneath it. A REAL directory, by contrast, only ever syncs as
     // those child objects — never as a single object AT its own key. So a single
     // remote object sitting at a key that also has children is unambiguously a
-    // directory-overlay marker, not a regular file. Precomputed once (pure path
-    // derivation, no I/O — the planner is synchronous) so the dir-vs-object
-    // collision branch can tell an inert overlay marker apart from a genuine
-    // file-vs-directory structural collision. feedback_d2082110.
-    const remoteKeysWithChildren = new Set();
-    for (const rf of remoteFiles) {
+    // directory-overlay marker, not a regular file. Evidence is complete before
+    // classification for both the spooled all-mode LIST and scoped arrays.
+    // Build scoped evidence incrementally: flatMap would transiently retain an
+    // ancestor string for every row/depth before Set deduplication.
+    const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
+        ? remoteFiles.remoteKeysWithChildren
+        : collectRemoteAncestors(remoteFiles);
+    // Both sources replay synchronously. for-await on a scoped array introduces
+    // promise/microtask work per object even though no I/O is being awaited.
+    for (const remoteFile of listedFiles) {
         onHeartbeat?.();
-        const key = toPosixKey(rf.key);
-        let slash = key.indexOf("/");
-        while (slash !== -1) {
-            remoteKeysWithChildren.add(key.slice(0, slash));
-            slash = key.indexOf("/", slash + 1);
+        const posixRemoteKey = toPosixKey(remoteFile.key);
+        // S3 object keys are POSIX. Match the old delete-sweep membership check:
+        // a legacy backslash journal entry counts as present only when its POSIX
+        // form appears in the LIST, while a malformed backslash remote key does
+        // not accidentally change that behavior.
+        if (remoteFile.key === posixRemoteKey) {
+            unseenJournalKeys.delete(remoteFile.key);
+            for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
+                unseenJournalKeys.delete(legacyKey);
+            }
+        }
+        if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
+            // A remote junk spelling means its canonical journal twin is still
+            // remotely present. Keep that twin out of the delete sweep unless an
+            // exact FILE_TOMBSTONE is authoritative, precisely matching the old
+            // remoteCanonicalSkillKeys protection without retaining remote keys.
+            const canonical = canonicalVaultKeySpelling(remoteFile.key);
+            for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
+                if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
+                    unseenJournalKeys.delete(journaledKey);
+                }
+            }
         }
-    }
-    for (const remoteFile of remoteFiles) {
-        onHeartbeat?.();
         const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
         if (invalidKey !== null) {
             invalidRemoteKeys.push({
@@ -2711,6 +2912,9 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
             filesToSkip++;
         }
     }
+    filesToSkip += preplannedFilesSkipped;
+    filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
+    filesOutOfScope += preplannedFilesOutOfScope;
     // Bug #9 — cross-machine delete propagation. The peer's push leg
     // already removed the file from S3 (verified in the deep-test
     // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
@@ -2719,38 +2923,21 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // that-was-there-before" — so the file stayed forever on every
     // receiver and `filesTombstoned: 0` showed up on every pull.
     //
-    // Closing the loop: walk the journal, find every entry whose key is
-    // NOT in the remote LIST set, AND whose path passes the current
+    // Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
+    // whose path passes the current
     // ignore filter (paths newly excluded by .hqignore must not trigger
     // mass-delete), AND — in personalMode — survives the same companies/*
     // gating the download branch applies. The executor will apply each
     // as a local delete + journal removal. Symmetric to the push side's
     // `propagateDeletes` plan in share.ts.
-    const remoteKeySet = new Set();
-    // Skill-farm twin protection (companion to the junk-spelling pull guard):
-    // when the LIST holds only a junk spelling of a journaled canonical key,
-    // the logical object is still remotely present — a raw has() would class
-    // the canonical journal entry as remote-deleted, unlink the good local
-    // wrapper, and drop the journal entry (after which the next pull would
-    // materialize the junk spelling the guard just refused). Track canonical
-    // forms of listed skill keys and consult them below.
-    const remoteCanonicalSkillKeys = new Set();
-    for (const rf of remoteFiles) {
-        remoteKeySet.add(rf.key);
-        if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
-            remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
-        }
-    }
     const tombstones = [];
-    for (const key of journalRows(journal).keys()) {
+    for (const key of unseenJournalKeys) {
         // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
         // carry backslash separators; the remote LIST is always forward-slash, so a
         // raw `has(key)` would miss the match and class a live local file as
         // remote-deleted. normalizeJournalKeys() defuses this on load, but the
         // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
         const posixKey = toPosixKey(key);
-        if (remoteKeySet.has(posixKey))
-            continue;
         // Listing-scope guard (targeted sync / scoped listing). The remote LIST
         // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
         // pull narrows further), so "absent from the LIST" is evidence of a remote
@@ -2762,16 +2949,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
         // `all` mode passes `[""]`, which covers every key — no behavior change.
         if (!isCoveredByAny(posixKey, prefixSet))
             continue;
-        if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
-            remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
-            // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
-            // delete and wins over the twin protection — a lingering junk
-            // spelling in the LIST must not keep a deliberately-deleted wrapper
-            // alive forever. (The key is absent from the LIST here, so there is
-            // no newer re-create to defer to.)
-            fileTombstones.get(posixKey) === undefined) {
-            continue;
-        }
         // Fast-path ignore, BEFORE containment canonicalization. The tombstone
         // sweep runs for every journaled key, and a stale journal can carry
         // hundreds of thousands of entries under an ignored runtime tree (e.g.
@@ -2880,6 +3057,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     }
     return {
         items,
+        preplannedFilesSkipped,
+        preplannedFilesOutOfScope,
         filesToDownload,
         bytesToDownload,
         filesToSkip,