@indigoai-us/hq-cloud 6.16.58-alpha.3 → 6.16.59

package/dist/cli/sync.js

@@ -6,18 +6,19 @@
  */
 import * as fs from "fs";
 import * as path from "path";
-import { spoolRemoteFiles, collectRemoteAncestors } from "../sync/remote-file-spool.js";
 import { VaultAuthError, VaultClient } from "../vault-client.js";
 import { readlinkOrNull } from "../lib/readlink-safe.js";
 import { describeError } from "../lib/describe-error.js";
 import { isRetryableIdentityRefusal, isTerminalIdentityRefusal, } from "../cognito-auth.js";
+import { RateLimitedError } from "../object-io.js";
+import { isTransientHttpStatusError } from "../lib/net-errors.js";
 import { emitCloudTelemetry, } from "../telemetry-events.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { ensureCompanyVaultTransport } from "../company-vault-transport.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
 import { canonicalVaultKeySpelling, localPathForVaultKey, vaultKeyForLocalPath, } from "../local-path-codec.js";
 import { SKILLS_KEY_PREFIX } from "./doctor.js";
-import { downloadFile, streamRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
+import { downloadFile, listRemoteFiles, headRemoteFile, primeObjectTransport, toPosixKey, classifyVaultKey, DanglingSymlinkParentError, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
 import { readJournalForPass, writeJournal, getStateDir, openJournalStoreSession, buildJournalBaseline, journalDeltaFromBaseline, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, updateEntry, removeEntry, getEntry, normalizeEtag, migrateToV2, gcTombstones, isTombstone, tombstoneEntry, lastPullRecord, appendPullRecord, generatePullId, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, } from "../journal.js";
 import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
 import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
@@ -186,11 +187,6 @@ export function targetedJournalSessionCensusSizes() {
  * dirty entries through a run-scoped journal writer.
  */
 const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 10_000;
-/**
- * A full-pull hand-off unit. Every member has finished its transfer and the
- * resulting journal delta is durable before realtime can acquire the writer.
- */
-export const FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE = 500;
 /**
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
  * an oversized batch wholesale, so the client MUST split a large report into
@@ -324,53 +320,36 @@ export async function sync(options) {
     return withOperationLock(options.hqRoot, "sync", () => syncWithOperationLockHeld(options));
 }
 async function syncWithOperationLockHeld(options) {
-    let carried;
-    for (;;) {
-        const run = await buildPullContext(options);
-        if (carried)
-            run.carriedProgress = carried;
-        try {
-            const plan = await planPull(run);
-            emitPullPlan(run.emit, plan);
-            reportInvalidRemoteKeys(run, plan);
-            const scopeRun = run.keyedJournal
-                ? emptyTargetedScopeRun(run)
-                : executeScopeShrink(run, planScopeShrink(run));
-            const counters = createPullCounters();
-            // A streaming all-mode planner retains only executor-relevant items. Its
-            // ordinary skips have already been counted while the page was live.
-            counters.filesSkipped = plan.preplannedFilesSkipped;
-            counters.filesOutOfScope = plan.preplannedFilesOutOfScope;
-            seedCarriedPullProgress(counters, carried);
-            const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
-            const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
-            if (conflictRun.abortResult) {
-                return conflictRun.abortResult;
-            }
-            await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
-            // The following added-by and tombstone phases may still make authenticated
-            // vault calls. Make a completed sub-threshold download tail durable before
-            // they begin, so their failure cannot turn a successful bulk transfer into
-            // a full replay on the next pass.
-            flushDownloadedJournalTail(run);
-            await emitAndReportNewFiles(run, plan);
-            await verifyPlannedJournalTombstones(run, plan);
-            executeJournalTombstoneDeletes(run, plan, counters);
-            emitScopeMaterializationGap(run, plan);
-            return finalizePullRun(run, plan, scopeRun, counters);
-        }
-        catch (error) {
-            if (error instanceof CooperativePullHandoff) {
-                carried = error.completed;
-                continue;
-            }
-            throw error;
-        }
-        finally {
-            // Release a pass-local lease even on conflict, transfer, or auth failure.
-            // The one bounded hot lease remains available only for its same vault.
-            run.releaseKeyedJournal?.();
-        }
+    const run = await buildPullContext(options);
+    try {
+        const plan = planPull(run);
+        emitPullPlan(run.emit, plan);
+        reportInvalidRemoteKeys(run, plan);
+        const scopeRun = run.keyedJournal
+            ? emptyTargetedScopeRun(run)
+            : executeScopeShrink(run, planScopeShrink(run));
+        const counters = createPullCounters();
+        const transferConcurrency = resolveTransferConcurrency(run.hostLoadAtStart);
+        const conflictRun = await executeConflictExecutor(run, plan, scopeRun, counters);
+        if (conflictRun.abortResult) {
+            return conflictRun.abortResult;
+        }
+        await executeDownloadExecutor(run, conflictRun.downloadItems, transferConcurrency, counters);
+        // The following added-by and tombstone phases may still make authenticated
+        // vault calls. Make a completed sub-threshold download tail durable before
+        // they begin, so their failure cannot turn a successful bulk transfer into
+        // a full replay on the next pass.
+        flushDownloadedJournalTail(run);
+        await emitAndReportNewFiles(run, plan);
+        await verifyPlannedJournalTombstones(run, plan);
+        executeJournalTombstoneDeletes(run, plan, counters);
+        emitScopeMaterializationGap(run, plan);
+        return finalizePullRun(run, plan, scopeRun, counters);
+    }
+    finally {
+        // Release a pass-local lease even on conflict, transfer, or auth failure.
+        // The one bounded hot lease remains available only for its same vault.
+        run.releaseKeyedJournal?.();
     }
 }
 async function buildPullContext(options) {
@@ -459,9 +438,8 @@ async function buildPullContext(options) {
         ? targetedJournalSession(journalSlug)
         : undefined;
     const keyedJournal = keyedJournalLease?.store;
-    let remoteFiles;
     try {
-        remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
+        const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, requestedPullPrefixes.length > 0);
         const fileTombstones = options.fileTombstones ??
             await fetchFileTombstones(vaultConfig, options.personalMode === true
                 ? { personal: true }
@@ -509,27 +487,43 @@ async function buildPullContext(options) {
     catch (error) {
         // `buildPullContext` can fail during LIST/freshness. Do not leave the
         // leased projection marked in use when no PullRunContext is returned.
-        if (remoteFiles && isStreamingRemoteFileSource(remoteFiles))
-            remoteFiles.dispose();
         keyedJournalLease?.release();
         throw error;
     }
 }
-function isStreamingRemoteFileSource(source) {
-    return !Array.isArray(source);
-}
-function allModeRemoteFileStream(ctx) {
-    return spoolRemoteFiles(streamRemoteFiles(ctx));
-}
+/**
+ * List the remote objects the current pull can actually materialize.
+ *
+ * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
+ * a `shared`/`custom` membership whose effective scope is a handful of
+ * prefixes — so every pull materialized the entire remote listing (tens of
+ * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
+ * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
+ * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
+ * bounded parallel, deduped), which bounds the listing's memory to the scope's
+ * actual footprint.
+ *
+ * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
+ * the single broad listing: fanning `all` out by top-level prefix would need a
+ * delimiter LIST to discover the top-level names first and still union the
+ * same total object count in one plan, so it bounds nothing today. (A
+ * streaming planner is the real fix for `all` and is out of scope here.)
+ *
+ * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
+ * issue zero LIST calls. Consumers already treat an empty listing correctly:
+ * the deletion sweep in `computePullPlan` only considers journal keys covered
+ * by the LISTED scope, so keys absent because they were never listed can never
+ * be mistaken for remote deletes.
+ */
 async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargetedPull) {
     const entries = toScopePrefixEntries(listedPrefixSet);
     if (entries.some((e) => e.prefix === ""))
-        return allModeRemoteFileStream(ctx);
+        return listRemoteFiles(ctx);
     if (entries.length === 0) {
         // `all` mode always carries `[""]`, so an empty set only occurs for a
         // scoped mode (empty grants, or a targeted intersection that excluded
         // everything). Defensive `all` branch: never silently narrow a full pull.
-        return syncMode === "all" ? allModeRemoteFileStream(ctx) : [];
+        return syncMode === "all" ? listRemoteFiles(ctx) : [];
     }
     return listRemoteForScope({
         ctx,
@@ -550,14 +544,8 @@ async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet, isTargete
         },
     });
 }
-async function planPull(run) {
-    try {
-        return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
-    }
-    finally {
-        if (isStreamingRemoteFileSource(run.remoteFiles))
-            run.remoteFiles.dispose();
-    }
+function planPull(run) {
+    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.propagateDeletePolicy ?? "owned-only", run.options.onHeartbeat, run.journalRowCount);
 }
 /** Build only the rows a targeted remote LIST can observe. */
 function journalForTargetedPull(store, prefixes) {
@@ -670,55 +658,9 @@ function createPullCounters() {
         conflictPaths: [],
         changedPaths: [],
         appliedUpserts: [],
-        newFiles: [],
         hasRetryableOutcome: false,
     };
 }
-/** Fold a previous attempt's completed work into a restarted run's counters. */
-function seedCarriedPullProgress(counters, carried) {
-    if (!carried)
-        return;
-    counters.filesDownloaded += carried.filesDownloaded;
-    counters.bytesDownloaded += carried.bytesDownloaded;
-    counters.conflicts += carried.conflicts;
-    counters.filesTombstoned += carried.filesTombstoned;
-    counters.conflictPaths.push(...carried.conflictPaths);
-    counters.changedPaths.push(...carried.changedPaths);
-    counters.appliedUpserts.push(...carried.appliedUpserts);
-    if (carried.hasRetryableOutcome)
-        counters.hasRetryableOutcome = true;
-}
-/**
- * Snapshot what survives the hand-off. The numeric and path axes already
- * include everything carried into this attempt; `newFiles` is accumulated
- * separately so it stays a record of pre-restart downloads only, and cannot
- * double up with the current plan's own new files.
- */
-function carryPullProgress(previous, counters) {
-    return {
-        filesDownloaded: counters.filesDownloaded,
-        bytesDownloaded: counters.bytesDownloaded,
-        conflicts: counters.conflicts,
-        conflictPaths: [...counters.conflictPaths],
-        filesTombstoned: counters.filesTombstoned,
-        changedPaths: [...counters.changedPaths],
-        appliedUpserts: [...counters.appliedUpserts],
-        newFiles: [...(previous?.newFiles ?? []), ...counters.newFiles],
-        hasRetryableOutcome: counters.hasRetryableOutcome,
-    };
-}
-/**
- * Every new file this pull is responsible for reporting: the ones a previous
- * attempt downloaded before handing off, then this plan's own. A carried path
- * that the fresh plan also lists is reported once.
- */
-function reportableNewFiles(run, plan) {
-    const carried = run.carriedProgress?.newFiles;
-    if (carried === undefined || carried.length === 0)
-        return plan.newFiles;
-    const planned = new Set(plan.newFiles.map((newFile) => newFile.path));
-    return [...carried.filter((newFile) => !planned.has(newFile.path)), ...plan.newFiles];
-}
 function planScopeShrink(run) {
     const lastRecord = lastPullRecord(run.journal, run.ctx.uid);
     const lastPrefixSet = lastRecord && lastRecord.prefixSet.length > 0
@@ -1179,7 +1121,6 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         if (resolution === "abort") {
             run.emit({ type: "new-files", files: [] });
             persistPullJournal(run);
-            const abortedNewFiles = reportableNewFiles(run, plan);
             return {
                 filesDownloaded: counters.filesDownloaded,
                 bytesDownloaded: counters.bytesDownloaded,
@@ -1187,8 +1128,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 conflicts: counters.conflicts,
                 conflictPaths: counters.conflictPaths,
                 aborted: true,
-                newFiles: abortedNewFiles,
-                newFilesCount: abortedNewFiles.length,
+                newFiles: plan.newFiles,
+                newFilesCount: plan.newFilesCount,
                 filesExcludedByPolicy: plan.filesExcludedByPolicy,
                 filesTombstoned: 0,
                 filesOutOfScope: counters.filesOutOfScope,
@@ -1314,21 +1255,10 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             discardProbe();
     }
 }
-/**
- * The old plan may have tombstone evidence that predates a realtime hand-off.
- * Rebuilding from disk after a hand-off makes the continuation observe the
- * realtime journal and a fresh LIST before it can perform another mutation.
- */
-class CooperativePullHandoff extends Error {
-    /** Completed work the restarted pull must keep reporting. */
-    completed;
-    constructor(completed) {
-        super("full pull handed off to realtime work");
-        this.name = "CooperativePullHandoff";
-        this.completed = completed;
-    }
-}
-async function executeDownloadQueue(run, downloadItems, transferConcurrency, counters, deferJournalCheckpoint) {
+async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
+    if (downloadItems.length === 0)
+        return;
+    await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
     const queue = [...downloadItems];
     const inFlight = new Set();
     const workerErrors = [];
@@ -1343,7 +1273,7 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
             inFlight.size < transferConcurrency &&
             queue.length > 0) {
             const downloadItem = queue.shift();
-            const download = () => downloadOne(run, downloadItem, counters, deferJournalCheckpoint);
+            const download = () => downloadOne(run, downloadItem, counters);
             const p = (run.options.transferSemaphore
                 ? run.options.transferSemaphore.withPermit(download)
                 : download())
@@ -1367,44 +1297,16 @@ async function executeDownloadQueue(run, downloadItems, transferConcurrency, cou
             break;
         }
     }
-    return workerErrors;
-}
-function throwDownloadWorkerErrors(run, workerErrors) {
-    persistPullJournal(run);
-    const first = workerErrors[0];
-    if (workerErrors.length > 1) {
-        first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
-    }
-    throw first;
-}
-async function executeDownloadExecutor(run, downloadItems, transferConcurrency, counters) {
-    if (downloadItems.length === 0)
-        return;
-    await primeObjectTransport(run.ctx, "get", downloadItems.map((d) => d.remoteFile.key));
-    // Ordinary and targeted pulls retain the existing 10k crash-recovery cadence.
-    // Only a watch chore that supplied the cooperative seam pays for drained
-    // units and can return its root lock to queued realtime work.
-    if (!run.options.cooperativeCheckpoint) {
-        const workerErrors = await executeDownloadQueue(run, downloadItems, transferConcurrency, counters, false);
-        if (workerErrors.length > 0)
-            throwDownloadWorkerErrors(run, workerErrors);
-        return;
-    }
-    for (let start = 0; start < downloadItems.length; start += FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE) {
-        const chunk = downloadItems.slice(start, start + FULL_PULL_COOPERATIVE_CHECKPOINT_BATCH_SIZE);
-        const workerErrors = await executeDownloadQueue(run, chunk, transferConcurrency, counters, true);
-        // `executeDownloadQueue` does not return until every member of this chunk
-        // has settled. Persist before the callback so the suspended chore owns no
-        // mutable pass state while realtime uses the root lock.
-        flushDownloadedJournalTail(run);
-        if (workerErrors.length > 0)
-            throwDownloadWorkerErrors(run, workerErrors);
-        if (await run.options.cooperativeCheckpoint() === true) {
-            throw new CooperativePullHandoff(carryPullProgress(run.carriedProgress, counters));
+    if (workerErrors.length > 0) {
+        persistPullJournal(run);
+        const first = workerErrors[0];
+        if (workerErrors.length > 1) {
+            first.message = `${first.message} (and ${workerErrors.length - 1} more download-worker errors)`;
         }
+        throw first;
     }
 }
-async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint = false) {
+async function downloadOne(run, downloadItem, counters) {
     const { remoteFile, localPath } = downloadItem;
     await refreshRunContextIfExpiring(run);
     if (!isDownloadWritePathStillContained(run.companyRoot, remoteFile.key, localPath)) {
@@ -1473,9 +1375,6 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
         counters.filesDownloaded++;
         counters.bytesDownloaded += size;
         counters.changedPaths.push(remoteFile.key);
-        if (downloadItem.isNew) {
-            counters.newFiles.push({ path: remoteFile.key, bytes: remoteFile.size });
-        }
         // The event-push emitter hashes regular file bodies. Symlink journal hashes
         // describe link targets instead, and the startup baseline excludes them for
         // the same reason, so only hand regular-file downloads to that ledger.
@@ -1564,8 +1463,7 @@ async function downloadOne(run, downloadItem, counters, deferJournalCheckpoint =
         return;
     }
     run.downloadsSinceJournalCheckpoint++;
-    if (!deferJournalCheckpoint &&
-        run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
+    if (run.downloadsSinceJournalCheckpoint >= DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE) {
         flushDownloadedJournalTail(run);
     }
 }
@@ -1598,19 +1496,15 @@ function persistPullJournal(run) {
 }
 async function emitAndReportNewFiles(run, plan) {
     const enrichedNewFiles = [];
-    // New files a cooperative hand-off already downloaded are reported here too:
-    // the restarted plan cannot see them as new, so this is their only
-    // notification.
-    const newFiles = reportableNewFiles(run, plan);
     // Batch-mint the GET presigns once (chunked, breaker-aware) so the per-file
     // created-by HEADs below reuse the cache instead of each minting its own
     // presign. Without this, a big catch-up pull (hundreds of new files) bursts
     // the presign endpoint, trips the circuit breaker, and every enrichment HEAD
     // then fails. Mirrors the tombstone HEAD-verify pre-prime.
-    await primeObjectTransport(run.ctx, "get", newFiles.map((nf) => nf.path));
+    await primeObjectTransport(run.ctx, "get", plan.newFiles.map((nf) => nf.path));
     const HEAD_CONCURRENCY = 5;
-    for (let i = 0; i < newFiles.length; i += HEAD_CONCURRENCY) {
-        const batch = newFiles.slice(i, i + HEAD_CONCURRENCY);
+    for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
+        const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
         const results = await Promise.all(batch.map(async (nf) => {
             let addedBy = null;
             try {
@@ -1668,11 +1562,31 @@ async function verifyPlannedJournalTombstones(run, plan) {
                 }
                 if (isAccessDenied(err))
                     return null;
-                run.emit({
-                    type: "error",
-                    path: tombstone.key,
-                    message: `tombstone HEAD verify failed (deferring): ${describeError(err)}`,
-                });
+                const message = `tombstone HEAD verify failed (deferring): ${describeError(err)}`;
+                // A vault 100/hr throttle (or 429/502/503/504) is a deferral the
+                // next sync will retry. Emitting a hard `type: "error"` here used
+                // to land the company in the runner's `errors[]` and force exit 2
+                // (`degraded partial`) even though the tombstone was already
+                // dropped from this pass. Reuse the existing diagnostic marker
+                // so the company stays `complete` and the runner exits 0.
+                const transient = err instanceof RateLimitedError ||
+                    (err instanceof Error && err.name === "RateLimited") ||
+                    isTransientHttpStatusError(err);
+                if (transient) {
+                    run.emit({
+                        type: "error",
+                        diagnostic: true,
+                        path: tombstone.key,
+                        message,
+                    });
+                }
+                else {
+                    run.emit({
+                        type: "error",
+                        path: tombstone.key,
+                        message,
+                    });
+                }
                 return null;
             }
         }));
@@ -1771,7 +1685,6 @@ function finalizePullRun(run, plan, scopeRun, counters) {
             // best-effort: a post-sync refresh failure never fails the sync
         }
     }
-    const newFiles = reportableNewFiles(run, plan);
     const result = {
         filesDownloaded: counters.filesDownloaded,
         bytesDownloaded: counters.bytesDownloaded,
@@ -1779,8 +1692,8 @@ function finalizePullRun(run, plan, scopeRun, counters) {
         conflicts: counters.conflicts,
         conflictPaths: counters.conflictPaths,
         aborted: false,
-        newFiles,
-        newFilesCount: newFiles.length,
+        newFiles: plan.newFiles,
+        newFilesCount: plan.newFilesCount,
         filesExcludedByPolicy: plan.filesExcludedByPolicy,
         filesTombstoned: counters.filesTombstoned,
         filesOutOfScope: counters.filesOutOfScope,
@@ -2009,8 +1922,7 @@ class LocalSnapshotChangedError extends Error {
  * the real outcome of the upcoming Stage-2 execution rather than an
  * upper-bound guess.
  *
- * It performs no writes, journal mutations, or event emission. An all-mode
- * source performs its paged classification LIST while it is consumed; the
+ * Pure function: no S3 calls, no journal writes, no event emission. The
  * caller (`sync()`) is responsible for emitting the resulting plan event
  * before iterating `items`.
  */
@@ -2078,36 +1990,6 @@ excludePrefixes = [],
 // callers preserve the pre-policy, watcher-intent-only behavior.
 propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows(journal).size) {
     const items = [];
-    const streaming = isStreamingRemoteFileSource(remoteFiles);
-    const listedFiles = isStreamingRemoteFileSource(remoteFiles) ? remoteFiles.files : remoteFiles;
-    let preplannedFilesSkipped = 0;
-    let preplannedFilesOutOfScope = 0;
-    let preplannedFilesExcludedByPolicy = 0;
-    if (streaming) {
-        const retain = items.push.bind(items);
-        items.push = (...next) => {
-            for (const item of next) {
-                // These actions still need the executor for an event or a state change.
-                if (item.action === "download" ||
-                    item.action === "conflict" ||
-                    item.action === "tombstone-delete" ||
-                    item.action === "skip-junk-key-spelling" ||
-                    item.action === "skip-unreadable-link") {
-                    retain(item);
-                }
-                else if (item.action === "skip-out-of-scope") {
-                    preplannedFilesOutOfScope++;
-                }
-                else if (item.action === "skip-excluded-policy") {
-                    preplannedFilesExcludedByPolicy++;
-                }
-                else {
-                    preplannedFilesSkipped++;
-                }
-            }
-            return items.length;
-        };
-    }
     // Remote keys the server will never presign — collected so the caller can
     // warn + emit telemetry without allowing one poison object to fail a pull.
     const invalidRemoteKeys = [];
@@ -2123,35 +2005,8 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // never gated by this map (see the guard below), so pre-existing families
     // keep syncing untouched until the vault doctor (US-003) collapses them.
     const journaledKeyByCanonical = new Map();
-    // The delete sweep starts with journaled keys and clears them as the streamed
-    // LIST proves each one present. This retains existing journal-key references
-    // instead of a second, whole-vault population of long remote key strings.
-    const unseenJournalKeys = new Set();
-    // Most journal keys have already been normalized to POSIX and can be cleared
-    // directly. Keep the legacy Windows spelling defense-in-depth without
-    // imposing a second map entry on every current journal row.
-    const legacyJournalKeysByPosix = new Map();
-    // The junk-spelling delete guard needs all journal twins, while
-    // `journaledKeyByCanonical` above deliberately retains only the preferred
-    // representative for classification.
-    const journaledSkillKeysByCanonical = new Map();
     for (const journaledKey of journalRows(journal).keys()) {
         onHeartbeat?.();
-        const posixJournaledKey = toPosixKey(journaledKey);
-        // A scoped LIST cannot prove anything about keys outside its prefixes.
-        // Do not copy the entire vault's journal into its presence bookkeeping.
-        const inListedScope = isCoveredByAny(posixJournaledKey, prefixSet);
-        if (inListedScope)
-            unseenJournalKeys.add(journaledKey);
-        if (inListedScope && posixJournaledKey !== journaledKey) {
-            const legacyKeys = legacyJournalKeysByPosix.get(posixJournaledKey);
-            if (legacyKeys === undefined) {
-                legacyJournalKeysByPosix.set(posixJournaledKey, [journaledKey]);
-            }
-            else {
-                legacyKeys.push(journaledKey);
-            }
-        }
         if (!journaledKey.startsWith(SKILLS_KEY_PREFIX))
             continue;
         const canonical = canonicalVaultKeySpelling(journaledKey);
@@ -2159,15 +2014,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
         if (existing === undefined || journaledKey === canonical) {
             journaledKeyByCanonical.set(canonical, journaledKey);
         }
-        if (!inListedScope)
-            continue;
-        const skillKeys = journaledSkillKeysByCanonical.get(canonical);
-        if (skillKeys === undefined) {
-            journaledSkillKeysByCanonical.set(canonical, [journaledKey]);
-        }
-        else {
-            skillKeys.push(journaledKey);
-        }
     }
     // Remote keys that ALSO appear as an ancestor of another remote key — i.e.
     // keys that carry child objects under `${key}/…` in this same LIST. The
@@ -2177,40 +2023,22 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // objects beneath it. A REAL directory, by contrast, only ever syncs as
     // those child objects — never as a single object AT its own key. So a single
     // remote object sitting at a key that also has children is unambiguously a
-    // directory-overlay marker, not a regular file. Evidence is complete before
-    // classification for both the spooled all-mode LIST and scoped arrays.
-    // Build scoped evidence incrementally: flatMap would transiently retain an
-    // ancestor string for every row/depth before Set deduplication.
-    const remoteKeysWithChildren = isStreamingRemoteFileSource(remoteFiles)
-        ? remoteFiles.remoteKeysWithChildren
-        : collectRemoteAncestors(remoteFiles);
-    // Both sources replay synchronously. for-await on a scoped array introduces
-    // promise/microtask work per object even though no I/O is being awaited.
-    for (const remoteFile of listedFiles) {
+    // directory-overlay marker, not a regular file. Precomputed once (pure path
+    // derivation, no I/O — the planner is synchronous) so the dir-vs-object
+    // collision branch can tell an inert overlay marker apart from a genuine
+    // file-vs-directory structural collision. feedback_d2082110.
+    const remoteKeysWithChildren = new Set();
+    for (const rf of remoteFiles) {
         onHeartbeat?.();
-        const posixRemoteKey = toPosixKey(remoteFile.key);
-        // S3 object keys are POSIX. Match the old delete-sweep membership check:
-        // a legacy backslash journal entry counts as present only when its POSIX
-        // form appears in the LIST, while a malformed backslash remote key does
-        // not accidentally change that behavior.
-        if (remoteFile.key === posixRemoteKey) {
-            unseenJournalKeys.delete(remoteFile.key);
-            for (const legacyKey of legacyJournalKeysByPosix.get(remoteFile.key) ?? []) {
-                unseenJournalKeys.delete(legacyKey);
-            }
-        }
-        if (remoteFile.key.startsWith(SKILLS_KEY_PREFIX)) {
-            // A remote junk spelling means its canonical journal twin is still
-            // remotely present. Keep that twin out of the delete sweep unless an
-            // exact FILE_TOMBSTONE is authoritative, precisely matching the old
-            // remoteCanonicalSkillKeys protection without retaining remote keys.
-            const canonical = canonicalVaultKeySpelling(remoteFile.key);
-            for (const journaledKey of journaledSkillKeysByCanonical.get(canonical) ?? []) {
-                if (fileTombstones.get(toPosixKey(journaledKey)) === undefined) {
-                    unseenJournalKeys.delete(journaledKey);
-                }
-            }
+        const key = toPosixKey(rf.key);
+        let slash = key.indexOf("/");
+        while (slash !== -1) {
+            remoteKeysWithChildren.add(key.slice(0, slash));
+            slash = key.indexOf("/", slash + 1);
         }
+    }
+    for (const remoteFile of remoteFiles) {
+        onHeartbeat?.();
         const invalidKey = classifyVaultKey(remoteFile.key, personalMode ? "personal" : "company");
         if (invalidKey !== null) {
             invalidRemoteKeys.push({
@@ -2863,9 +2691,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
             filesToSkip++;
         }
     }
-    filesToSkip += preplannedFilesSkipped;
-    filesExcludedByPolicy += preplannedFilesExcludedByPolicy;
-    filesOutOfScope += preplannedFilesOutOfScope;
     // Bug #9 — cross-machine delete propagation. The peer's push leg
     // already removed the file from S3 (verified in the deep-test
     // addendum: `aws s3 ls` showed the keys absent post-push). Pre-fix
@@ -2874,21 +2699,38 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     // that-was-there-before" — so the file stayed forever on every
     // receiver and `filesTombstoned: 0` showed up on every pull.
     //
-    // Closing the loop: walk journal keys the streamed LIST did NOT clear, AND
-    // whose path passes the current
+    // Closing the loop: walk the journal, find every entry whose key is
+    // NOT in the remote LIST set, AND whose path passes the current
     // ignore filter (paths newly excluded by .hqignore must not trigger
     // mass-delete), AND — in personalMode — survives the same companies/*
     // gating the download branch applies. The executor will apply each
     // as a local delete + journal removal. Symmetric to the push side's
     // `propagateDeletes` plan in share.ts.
+    const remoteKeySet = new Set();
+    // Skill-farm twin protection (companion to the junk-spelling pull guard):
+    // when the LIST holds only a junk spelling of a journaled canonical key,
+    // the logical object is still remotely present — a raw has() would class
+    // the canonical journal entry as remote-deleted, unlink the good local
+    // wrapper, and drop the journal entry (after which the next pull would
+    // materialize the junk spelling the guard just refused). Track canonical
+    // forms of listed skill keys and consult them below.
+    const remoteCanonicalSkillKeys = new Set();
+    for (const rf of remoteFiles) {
+        remoteKeySet.add(rf.key);
+        if (rf.key.startsWith(SKILLS_KEY_PREFIX)) {
+            remoteCanonicalSkillKeys.add(canonicalVaultKeySpelling(rf.key));
+        }
+    }
     const tombstones = [];
-    for (const key of unseenJournalKeys) {
+    for (const key of journalRows(journal).keys()) {
         // Compare membership in POSIX space. A pre-5.47.2 Windows journal key can
         // carry backslash separators; the remote LIST is always forward-slash, so a
         // raw `has(key)` would miss the match and class a live local file as
         // remote-deleted. normalizeJournalKeys() defuses this on load, but the
         // POSIX compare is defense-in-depth (ridge data-loss, feedback_b8d09d0f).
         const posixKey = toPosixKey(key);
+        if (remoteKeySet.has(posixKey))
+            continue;
         // Listing-scope guard (targeted sync / scoped listing). The remote LIST
         // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
         // pull narrows further), so "absent from the LIST" is evidence of a remote
@@ -2900,6 +2742,16 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
         // `all` mode passes `[""]`, which covers every key — no behavior change.
         if (!isCoveredByAny(posixKey, prefixSet))
             continue;
+        if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
+            remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
+            // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative
+            // delete and wins over the twin protection — a lingering junk
+            // spelling in the LIST must not keep a deliberately-deleted wrapper
+            // alive forever. (The key is absent from the LIST here, so there is
+            // no newer re-create to defer to.)
+            fileTombstones.get(posixKey) === undefined) {
+            continue;
+        }
         // Fast-path ignore, BEFORE containment canonicalization. The tombstone
         // sweep runs for every journaled key, and a stale journal can carry
         // hundreds of thousands of entries under an ignored runtime tree (e.g.
@@ -3008,8 +2860,6 @@ propagateDeletePolicy = "owned-only", onHeartbeat, journalRowCount = journalRows
     }
     return {
         items,
-        preplannedFilesSkipped,
-        preplannedFilesOutOfScope,
         filesToDownload,
         bytesToDownload,
         filesToSkip,