@indigoai-us/hq-cloud 6.15.31 → 6.15.32

package/dist/cli/sync.js

@@ -8,6 +8,7 @@ import * as fs from "fs";
 import * as path from "path";
 import { VaultAuthError, VaultClient } from "../vault-client.js";
 import { readlinkOrNull } from "../lib/readlink-safe.js";
+import { describeError } from "../lib/describe-error.js";
 import { emitCloudTelemetry, } from "../telemetry-events.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
@@ -18,7 +19,8 @@ import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, re
 import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
 import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
 import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
-import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
+import { coalescePrefixes, intersectPrefixSets, isCoveredByAny, pathToScopePrefix, toScopePrefixEntries, } from "../prefix-coalesce.js";
+import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
 import { createIgnoreFilter } from "../ignore.js";
 import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
 import { isEphemeralPath, isForbiddenCompanyVaultKey, isMalformedVaultKey } from "./share.js";
@@ -326,7 +328,19 @@ async function buildPullContext(options) {
     // of the scope-shrink inputs (see `planScopeShrink`). Empty when the caller
     // passes nothing (legacy behavior: no exclusions).
     const currentExcludeSet = coalescePrefixes(options.excludePrefixes ?? []);
-    const remoteFiles = await listRemoteFiles(ctx);
+    // Targeted-pull narrowing (`--scope-path`): intersect the requested paths
+    // with the durable scope so the LISTED scope can only shrink, never widen.
+    // A path may be a file or a directory — we can't know without I/O — so each
+    // requested path contributes both its exact and its directory spelling,
+    // mirroring `effectivePushPrefixSet` on the push side.
+    const requestedPullPrefixes = coalescePrefixes((options.pullPrefixes ?? []).flatMap((p) => [
+        pathToScopePrefix(p),
+        pathToScopePrefix(p.endsWith("/") ? p : p + "/"),
+    ]));
+    const listedPrefixSet = requestedPullPrefixes.length === 0
+        ? currentPrefixSet
+        : intersectPrefixSets(currentPrefixSet, requestedPullPrefixes);
+    const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet);
     const fileTombstones = options.personalMode === true
         ? new Map()
         : await fetchCompanyTombstones(vaultConfig, ctx.uid);
@@ -346,13 +360,60 @@ async function buildPullContext(options) {
         remoteFiles,
         syncMode,
         currentPrefixSet,
+        listedPrefixSet,
         currentExcludeSet,
         fileTombstones,
         downloadsSinceJournalCheckpoint: 0,
     };
 }
+/**
+ * List the remote objects the current pull can actually materialize.
+ *
+ * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
+ * a `shared`/`custom` membership whose effective scope is a handful of
+ * prefixes — so every pull materialized the entire remote listing (tens of
+ * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
+ * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
+ * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
+ * bounded parallel, deduped), which bounds the listing's memory to the scope's
+ * actual footprint.
+ *
+ * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
+ * the single broad listing: fanning `all` out by top-level prefix would need a
+ * delimiter LIST to discover the top-level names first and still union the
+ * same total object count in one plan, so it bounds nothing today. (A
+ * streaming planner is the real fix for `all` and is out of scope here.)
+ *
+ * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
+ * issue zero LIST calls. Consumers already treat an empty listing correctly:
+ * the deletion sweep in `computePullPlan` only considers journal keys covered
+ * by the LISTED scope, so keys absent because they were never listed can never
+ * be mistaken for remote deletes.
+ */
+async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet) {
+    const entries = toScopePrefixEntries(listedPrefixSet);
+    if (entries.some((e) => e.prefix === ""))
+        return listRemoteFiles(ctx);
+    if (entries.length === 0) {
+        // `all` mode always carries `[""]`, so an empty set only occurs for a
+        // scoped mode (empty grants, or a targeted intersection that excluded
+        // everything). Defensive `all` branch: never silently narrow a full pull.
+        return syncMode === "all" ? listRemoteFiles(ctx) : [];
+    }
+    return listRemoteForScope({
+        ctx,
+        scope: {
+            companyUid: ctx.uid,
+            syncMode,
+            prefixSet: listedPrefixSet,
+            strategy: listedPrefixSet.length > POST_FILTER_THRESHOLD
+                ? "broad-postfilter"
+                : "vend-fanout",
+        },
+    });
+}
 function planPull(run) {
-    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.currentPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.onHeartbeat);
+    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.onHeartbeat);
 }
 function emitPullPlan(emit, plan) {
     emit({
@@ -626,12 +687,13 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     // the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
     // divergent local content) that the keep path produced for these files.
     if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
+        const currentSnapshot = snapshotLocalState(localPath);
         downloadItems.push({
             action: "download",
             remoteFile,
             localPath,
             isNew: false,
-            localSnapshot: item.localSnapshot,
+            localSnapshot: currentSnapshot ?? item.localSnapshot,
         });
         run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
         return null;
@@ -654,9 +716,24 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     }
     let remoteFetched = false;
     let converged = false;
+    let executionLocal = null;
     try {
         const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
         remoteFetched = true;
+        executionLocal = readCurrentLocalFileState(localPath);
+        if (executionLocal === null) {
+            // The path changed kind, disappeared, or kept moving while the remote
+            // probe was in flight. Leave both sides untouched and let the next
+            // per-file pass classify that new state; a stale conflict is not useful.
+            try {
+                fs.rmSync(conflictAbs, { force: true });
+            }
+            catch {
+                /* best-effort cleanup */
+            }
+            counters.filesSkipped++;
+            return null;
+        }
         if (fs.lstatSync(conflictAbs).isSymbolicLink()) {
             const target = readlinkOrNull(conflictAbs);
             if (target === null) {
@@ -670,11 +747,12 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 return null;
             }
             else {
-                converged = hashSymlinkTarget(target) === item.localHash;
+                converged = hashSymlinkTarget(target) === executionLocal.hash;
             }
         }
         else {
-            converged = (downloaded.contentHash ?? hashFile(conflictAbs)) === item.localHash;
+            converged =
+                (downloaded.contentHash ?? hashFile(conflictAbs)) === executionLocal.hash;
         }
     }
     catch (probeErr) {
@@ -702,6 +780,10 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         });
     }
     if (converged) {
+        if (executionLocal === null) {
+            counters.filesSkipped++;
+            return null;
+        }
         if (remoteFetched) {
             try {
                 fs.rmSync(conflictAbs, { force: true });
@@ -710,10 +792,11 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 /* best-effort cleanup; a stray identical mirror is harmless */
             }
         }
-        updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", localPath, {
+        updateEntry(run.journal, remoteFile.key, executionLocal.hash, executionLocal.size, "down", localPath, {
             remoteEtag: remoteFile.etag,
-            mtimeMs: item.localMtime.getTime(),
-            kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
+            mtimeMs: executionLocal.mtimeMs,
+            ctimeMs: executionLocal.ctimeMs,
+            kind: executionLocal.kind,
         });
         run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
         counters.filesSkipped++;
@@ -721,11 +804,20 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     }
     counters.conflicts++;
     counters.conflictPaths.push(remoteFile.key);
+    // A successful probe always decides from the local bytes captured after the
+    // GET. Only a failed probe falls back to the preview snapshot so the existing
+    // error-path behavior remains conservative.
+    const localHash = executionLocal?.hash ?? item.localHash;
+    const localSize = executionLocal?.size ?? item.localSize;
+    const localMtime = executionLocal?.mtime ?? item.localMtime;
+    const localSnapshot = executionLocal?.snapshot ?? item.localSnapshot;
+    const localKind = executionLocal?.kind ??
+        (item.localSnapshot.kind === "symlink" ? "symlink" : "file");
     const resolution = await resolveConflict({
         path: remoteFile.key,
-        localHash: item.localHash,
+        localHash,
         remoteModified: remoteFile.lastModified,
-        localModified: item.localMtime,
+        localModified: localMtime,
         direction: "pull",
     }, run.options.onConflict);
     run.emit({
@@ -734,7 +826,9 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         direction: "pull",
         resolution,
     });
-    if (resolution !== "abort" && resolution !== "overwrite") {
+    if (resolution !== "abort" &&
+        resolution !== "overwrite" &&
+        resolution !== "keep") {
         if (remoteFetched) {
             try {
                 appendConflictEntry(run.hqRoot, {
@@ -744,8 +838,12 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                     detectedAt,
                     side: "pull",
                     machineId,
-                    localHash: item.localHash,
+                    localHash,
                     remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
+                    // The holding resolutions leave local alone and this sidecar keeps
+                    // the cloud copy. `keep` is indexed only after its local rename
+                    // succeeds below, so this row can never describe the wrong side.
+                    preserved: "remote",
                 });
             }
             catch (mirrorErr) {
@@ -757,7 +855,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             }
         }
     }
-    else if (remoteFetched) {
+    else if (resolution !== "keep" && remoteFetched) {
         try {
             fs.rmSync(conflictAbs, { force: true });
         }
@@ -785,22 +883,103 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
         };
     }
-    if (resolution === "keep" || resolution === "skip") {
+    // Hold the local body where it is, recording the remote etag so the conflict
+    // cannot re-fire (#137) and arming the journal-honesty flag because the held
+    // copy never matched that etag. The flag stops the currency-gated delete
+    // planner propagating a delete whose currency would falsely match on HEAD.
+    // Any genuine future download clears it by replacing the entry.
+    const holdLocal = () => {
         counters.filesSkipped++;
-        updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", localPath, {
+        updateEntry(run.journal, remoteFile.key, localHash, localSize, "down", localPath, {
             remoteEtag: remoteFile.etag,
-            mtimeMs: item.localMtime.getTime(),
-            kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
+            mtimeMs: localMtime.getTime(),
+            kind: localKind,
+        });
+        const heldEntry = getEntry(run.journal, remoteFile.key);
+        if (heldEntry)
+            heldEntry.localDiverges = true;
+        return null;
+    };
+    // `publish-local` is a PUSH-leg strategy: it means "send my body up". On the
+    // pull leg there is nothing to publish, so it holds the local body and arms
+    // the honesty flag, which is what the push leg then promotes under an
+    // If-Match fence. Falling through to the download branch instead would
+    // overwrite the very local copy the operator elected, which is the data loss
+    // this strategy exists to prevent.
+    //
+    // `skip` is the interactive "leave this one, I will deal with it later". A
+    // resolution that says leave it alone may not rename the operator's file, so
+    // it holds too — it is deliberately NOT bundled with `keep` any more.
+    if (resolution === "skip" || resolution === "publish-local") {
+        return holdLocal();
+    }
+    // `keep` ADOPTS the cloud body as the working copy and preserves the local
+    // one beside it under the `.conflict-` name.
+    //
+    // It used to mean the opposite — hold local, stamp the remote etag, arm
+    // `localDiverges` — and that combination was terminal. The push leg refuses
+    // to send a body the journal calls divergent; the pull leg sees a remote
+    // whose etag already matches and leaves it alone; and every later `keep`
+    // re-armed the flag. The file converged in neither direction, and because the
+    // desktop app hardcodes `--on-conflict keep` that was the default outcome for
+    // every real conflict. Adopting the remote clears the divergence by
+    // construction: the entry then describes the body that is actually on disk.
+    // The local bytes are not destroyed, only renamed.
+    if (resolution === "keep") {
+        if (!remoteFetched) {
+            // The probe never produced a body (its failure is already emitted above).
+            // Hold rather than move aside a file we have nothing to replace it with.
+            return holdLocal();
+        }
+        try {
+            // The sidecar currently holds the fetched probe copy of the remote.
+            // Remove it explicitly before renaming: POSIX rename replaces a file at
+            // the destination, but Windows rename rejects an occupied destination.
+            // The download queued below re-fetches it. One extra GET buys the entire
+            // tested download path —
+            // symlink materialization, path codec, mtime and etag journalling —
+            // instead of a hand-rolled three-way swap with its own crash window.
+            fs.rmSync(conflictAbs, { force: true });
+            fs.renameSync(localPath, conflictAbs);
+        }
+        catch (preserveErr) {
+            run.emit({
+                type: "error",
+                path: remoteFile.key,
+                message: `conflict keep could not preserve the local copy: ${preserveErr instanceof Error ? preserveErr.message : String(preserveErr)}`,
+            });
+            return holdLocal();
+        }
+        try {
+            appendConflictEntry(run.hqRoot, {
+                id: buildConflictId(originalRelative, detectedAt),
+                originalPath: originalRelative,
+                conflictPath: conflictRelative,
+                detectedAt,
+                side: "pull",
+                machineId,
+                localHash,
+                remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
+                preserved: "local",
+            });
+        }
+        catch (mirrorErr) {
+            run.emit({
+                type: "error",
+                path: remoteFile.key,
+                message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
+            });
+        }
+        downloadItems.push({
+            action: "download",
+            remoteFile,
+            localPath,
+            isNew: false,
+            // The local body was just moved aside, so the pre-replace guard must
+            // expect absence. Passing the original snapshot would make that guard
+            // abort the very download it exists to protect.
+            localSnapshot: { kind: "absent" },
         });
-        // Journal-honesty: we recorded the remote etag so this conflict can't
-        // re-fire (#137), but the KEPT local copy diverges from that remote — it
-        // never matched it. Flag the entry so the currency-gated delete planner
-        // refuses to propagate a delete for it (its currency would falsely match on
-        // HEAD, and the delete would destroy the divergent remote version). Any
-        // genuine future download clears the flag by replacing the entry.
-        const keptEntry = getEntry(run.journal, remoteFile.key);
-        if (keptEntry)
-            keptEntry.localDiverges = true;
         return null;
     }
     downloadItems.push({
@@ -808,7 +987,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         remoteFile,
         localPath,
         isNew: false,
-        localSnapshot: item.localSnapshot,
+        localSnapshot,
     });
     return null;
 }
@@ -891,6 +1070,12 @@ async function downloadOne(run, downloadItem, counters) {
         updateEntry(run.journal, remoteFile.key, hash, size, "down", localPath, {
             remoteEtag: remoteFile.etag,
             mtimeMs: localLstat.mtimeMs,
+            // Stamped from the lstat already in hand so the next push can skip this
+            // file on stat alone. The two conflict-resolution `updateEntry` calls
+            // above deliberately do NOT stamp ctime: neither has a post-write stat
+            // available, and a `keep`/`publish-local` entry is marked
+            // `localDiverges` immediately after, which the fast path refuses anyway.
+            ctimeMs: localLstat.ctimeMs,
             ...(createdBySub !== undefined ? { createdBySub } : {}),
             kind: isLocalSymlink ? "symlink" : "file",
         });
@@ -952,7 +1137,7 @@ async function downloadOne(run, downloadItem, counters) {
             run.emit({
                 type: "error",
                 path: remoteFile.key,
-                message: err instanceof Error ? err.message : String(err),
+                message: describeError(err),
             });
         }
         return;
@@ -1185,6 +1370,12 @@ function finalizePullRun(run, plan, scopeRun, counters) {
  * -files loop this event exists to end. `SyncResult.filesOutOfScope` keeps its
  * original both-causes meaning; only this advisory is narrowed.
  */
+// NOTE: since scoped pulls list only their in-scope prefixes (see
+// `listRemoteFilesForScope`), out-of-scope remote keys usually never enter the
+// plan and this event fires only for keys a broad-postfilter listing (or a
+// legacy caller passing a full listing) still surfaced. The withheld-count
+// telemetry traded away here is the cost of not materializing the full remote
+// listing on every scoped pull.
 function emitScopeMaterializationGap(run, plan) {
     if (run.syncMode === "all")
         return;
@@ -1314,6 +1505,36 @@ function matchesLocalSnapshot(localPath, expected) {
     return ((actual.kind !== "file" && actual.kind !== "symlink") ||
         actual.hash === expected.hash);
 }
+/**
+ * Read the local side at the execution seam, not from the whole-run preview.
+ * A second hash check closes the read-to-stat window: when an active writer
+ * moves the file while we inspect it, the caller defers this path instead of
+ * manufacturing a decision from a mixed snapshot.
+ */
+function readCurrentLocalFileState(localPath) {
+    const snapshot = snapshotLocalState(localPath);
+    if (snapshot === null || (snapshot.kind !== "file" && snapshot.kind !== "symlink")) {
+        return null;
+    }
+    let lstat;
+    try {
+        lstat = fs.lstatSync(localPath);
+    }
+    catch {
+        return null;
+    }
+    if (!matchesLocalSnapshot(localPath, snapshot))
+        return null;
+    return {
+        hash: snapshot.hash,
+        size: snapshot.kind === "symlink" ? 0 : lstat.size,
+        mtime: lstat.mtime,
+        mtimeMs: lstat.mtimeMs,
+        ctimeMs: lstat.ctimeMs,
+        kind: snapshot.kind,
+        snapshot,
+    };
+}
 function hasCurrentLocalDeleteIntent(entry) {
     const intent = entry?.localDeleteIntent;
     return !!(intent &&
@@ -2133,6 +2354,17 @@ excludePrefixes = [], onHeartbeat) {
         const posixKey = toPosixKey(key);
         if (remoteKeySet.has(posixKey))
             continue;
+        // Listing-scope guard (targeted sync / scoped listing). The remote LIST
+        // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
+        // pull narrows further), so "absent from the LIST" is evidence of a remote
+        // delete ONLY for keys the LIST could have contained. A journal key
+        // outside the listed scope was simply never asked about — treating it as
+        // remote-deleted would mass-delete every out-of-scope local file on the
+        // first scoped pull. Out-of-scope journal entries stay the scope-shrink
+        // pass's responsibility (quarantine semantics), never this sweep's.
+        // `all` mode passes `[""]`, which covers every key — no behavior change.
+        if (!isCoveredByAny(posixKey, prefixSet))
+            continue;
         if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
             remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
             // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative