@indigoai-us/hq-cloud 6.15.30 → 6.15.31

package/dist/cli/sync.js

@@ -8,7 +8,6 @@ import * as fs from "fs";
 import * as path from "path";
 import { VaultAuthError, VaultClient } from "../vault-client.js";
 import { readlinkOrNull } from "../lib/readlink-safe.js";
-import { describeError } from "../lib/describe-error.js";
 import { emitCloudTelemetry, } from "../telemetry-events.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
@@ -19,8 +18,7 @@ import { readJournal, writeJournal, hashFile, hashSymlinkTarget, updateEntry, re
 import { isGeneratedCoreMirrorKey, PERSONAL_VAULT_MANIFEST_KEY, } from "../personal-vault.js";
 import { isPersonalVaultExcluded } from "../personal-vault-exclusions.js";
 import { buildScopeShrinkPlan, applyScopeShrink, ScopeShrinkBlockedError, ScopeShrinkLargePruneError, } from "../scope-shrink.js";
-import { coalescePrefixes, intersectPrefixSets, isCoveredByAny, pathToScopePrefix, toScopePrefixEntries, } from "../prefix-coalesce.js";
-import { listRemoteForScope, POST_FILTER_THRESHOLD } from "../remote-pull.js";
+import { coalescePrefixes, isCoveredByAny, } from "../prefix-coalesce.js";
 import { createIgnoreFilter } from "../ignore.js";
 import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, readHostLoad, } from "../sync-core.js";
 import { isEphemeralPath, isForbiddenCompanyVaultKey, isMalformedVaultKey } from "./share.js";
@@ -328,19 +326,7 @@ async function buildPullContext(options) {
     // of the scope-shrink inputs (see `planScopeShrink`). Empty when the caller
     // passes nothing (legacy behavior: no exclusions).
     const currentExcludeSet = coalescePrefixes(options.excludePrefixes ?? []);
-    // Targeted-pull narrowing (`--scope-path`): intersect the requested paths
-    // with the durable scope so the LISTED scope can only shrink, never widen.
-    // A path may be a file or a directory — we can't know without I/O — so each
-    // requested path contributes both its exact and its directory spelling,
-    // mirroring `effectivePushPrefixSet` on the push side.
-    const requestedPullPrefixes = coalescePrefixes((options.pullPrefixes ?? []).flatMap((p) => [
-        pathToScopePrefix(p),
-        pathToScopePrefix(p.endsWith("/") ? p : p + "/"),
-    ]));
-    const listedPrefixSet = requestedPullPrefixes.length === 0
-        ? currentPrefixSet
-        : intersectPrefixSets(currentPrefixSet, requestedPullPrefixes);
-    const remoteFiles = await listRemoteFilesForScope(ctx, syncMode, listedPrefixSet);
+    const remoteFiles = await listRemoteFiles(ctx);
     const fileTombstones = options.personalMode === true
         ? new Map()
         : await fetchCompanyTombstones(vaultConfig, ctx.uid);
@@ -360,60 +346,13 @@ async function buildPullContext(options) {
         remoteFiles,
         syncMode,
         currentPrefixSet,
-        listedPrefixSet,
         currentExcludeSet,
         fileTombstones,
         downloadsSinceJournalCheckpoint: 0,
     };
 }
-/**
- * List the remote objects the current pull can actually materialize.
- *
- * Historically this was ALWAYS a full-bucket `listRemoteFiles(ctx)` — even for
- * a `shared`/`custom` membership whose effective scope is a handful of
- * prefixes — so every pull materialized the entire remote listing (tens of
- * thousands of RemoteFile objects on large vaults) only for `computePullPlan`
- * to discard most of it as `skip-out-of-scope`. Scoped pulls now list ONLY the
- * in-scope prefixes via `listRemoteForScope` (per-prefix ListObjectsV2 calls,
- * bounded parallel, deduped), which bounds the listing's memory to the scope's
- * actual footprint.
- *
- * `all`-mode (an unnarrowed full-vault pull, `listedPrefixSet` `[""]`) keeps
- * the single broad listing: fanning `all` out by top-level prefix would need a
- * delimiter LIST to discover the top-level names first and still union the
- * same total object count in one plan, so it bounds nothing today. (A
- * streaming planner is the real fix for `all` and is out of scope here.)
- *
- * An empty `listedPrefixSet` in a scoped mode means "nothing in scope" —
- * issue zero LIST calls. Consumers already treat an empty listing correctly:
- * the deletion sweep in `computePullPlan` only considers journal keys covered
- * by the LISTED scope, so keys absent because they were never listed can never
- * be mistaken for remote deletes.
- */
-async function listRemoteFilesForScope(ctx, syncMode, listedPrefixSet) {
-    const entries = toScopePrefixEntries(listedPrefixSet);
-    if (entries.some((e) => e.prefix === ""))
-        return listRemoteFiles(ctx);
-    if (entries.length === 0) {
-        // `all` mode always carries `[""]`, so an empty set only occurs for a
-        // scoped mode (empty grants, or a targeted intersection that excluded
-        // everything). Defensive `all` branch: never silently narrow a full pull.
-        return syncMode === "all" ? listRemoteFiles(ctx) : [];
-    }
-    return listRemoteForScope({
-        ctx,
-        scope: {
-            companyUid: ctx.uid,
-            syncMode,
-            prefixSet: listedPrefixSet,
-            strategy: listedPrefixSet.length > POST_FILTER_THRESHOLD
-                ? "broad-postfilter"
-                : "vend-fanout",
-        },
-    });
-}
 function planPull(run) {
-    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.listedPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.onHeartbeat);
+    return computePullPlan(run.remoteFiles, run.journal, run.companyRoot, run.shouldSync, run.options.personalMode === true, run.options.teamSyncedSlugs ?? null, run.currentPrefixSet, run.fileTombstones, run.currentExcludeSet, run.options.onHeartbeat);
 }
 function emitPullPlan(emit, plan) {
     emit({
@@ -687,13 +626,12 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     // the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
     // divergent local content) that the keep path produced for these files.
     if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
-        const currentSnapshot = snapshotLocalState(localPath);
         downloadItems.push({
             action: "download",
             remoteFile,
             localPath,
             isNew: false,
-            localSnapshot: currentSnapshot ?? item.localSnapshot,
+            localSnapshot: item.localSnapshot,
         });
         run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
         return null;
@@ -716,24 +654,9 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     }
     let remoteFetched = false;
     let converged = false;
-    let executionLocal = null;
     try {
         const downloaded = await downloadFile(run.ctx, remoteFile.key, conflictAbs);
         remoteFetched = true;
-        executionLocal = readCurrentLocalFileState(localPath);
-        if (executionLocal === null) {
-            // The path changed kind, disappeared, or kept moving while the remote
-            // probe was in flight. Leave both sides untouched and let the next
-            // per-file pass classify that new state; a stale conflict is not useful.
-            try {
-                fs.rmSync(conflictAbs, { force: true });
-            }
-            catch {
-                /* best-effort cleanup */
-            }
-            counters.filesSkipped++;
-            return null;
-        }
         if (fs.lstatSync(conflictAbs).isSymbolicLink()) {
             const target = readlinkOrNull(conflictAbs);
             if (target === null) {
@@ -747,12 +670,11 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 return null;
             }
             else {
-                converged = hashSymlinkTarget(target) === executionLocal.hash;
+                converged = hashSymlinkTarget(target) === item.localHash;
             }
         }
         else {
-            converged =
-                (downloaded.contentHash ?? hashFile(conflictAbs)) === executionLocal.hash;
+            converged = (downloaded.contentHash ?? hashFile(conflictAbs)) === item.localHash;
         }
     }
     catch (probeErr) {
@@ -780,10 +702,6 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         });
     }
     if (converged) {
-        if (executionLocal === null) {
-            counters.filesSkipped++;
-            return null;
-        }
         if (remoteFetched) {
             try {
                 fs.rmSync(conflictAbs, { force: true });
@@ -792,11 +710,10 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                 /* best-effort cleanup; a stray identical mirror is harmless */
             }
         }
-        updateEntry(run.journal, remoteFile.key, executionLocal.hash, executionLocal.size, "down", localPath, {
+        updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", localPath, {
             remoteEtag: remoteFile.etag,
-            mtimeMs: executionLocal.mtimeMs,
-            ctimeMs: executionLocal.ctimeMs,
-            kind: executionLocal.kind,
+            mtimeMs: item.localMtime.getTime(),
+            kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
         });
         run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
         counters.filesSkipped++;
@@ -804,20 +721,11 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
     }
     counters.conflicts++;
     counters.conflictPaths.push(remoteFile.key);
-    // A successful probe always decides from the local bytes captured after the
-    // GET. Only a failed probe falls back to the preview snapshot so the existing
-    // error-path behavior remains conservative.
-    const localHash = executionLocal?.hash ?? item.localHash;
-    const localSize = executionLocal?.size ?? item.localSize;
-    const localMtime = executionLocal?.mtime ?? item.localMtime;
-    const localSnapshot = executionLocal?.snapshot ?? item.localSnapshot;
-    const localKind = executionLocal?.kind ??
-        (item.localSnapshot.kind === "symlink" ? "symlink" : "file");
     const resolution = await resolveConflict({
         path: remoteFile.key,
-        localHash,
+        localHash: item.localHash,
         remoteModified: remoteFile.lastModified,
-        localModified: localMtime,
+        localModified: item.localMtime,
         direction: "pull",
     }, run.options.onConflict);
     run.emit({
@@ -826,9 +734,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         direction: "pull",
         resolution,
     });
-    if (resolution !== "abort" &&
-        resolution !== "overwrite" &&
-        resolution !== "keep") {
+    if (resolution !== "abort" && resolution !== "overwrite") {
         if (remoteFetched) {
             try {
                 appendConflictEntry(run.hqRoot, {
@@ -838,12 +744,8 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
                     detectedAt,
                     side: "pull",
                     machineId,
-                    localHash,
+                    localHash: item.localHash,
                     remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
-                    // The holding resolutions leave local alone and this sidecar keeps
-                    // the cloud copy. `keep` is indexed only after its local rename
-                    // succeeds below, so this row can never describe the wrong side.
-                    preserved: "remote",
                 });
             }
             catch (mirrorErr) {
@@ -855,7 +757,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             }
         }
     }
-    else if (resolution !== "keep" && remoteFetched) {
+    else if (remoteFetched) {
         try {
             fs.rmSync(conflictAbs, { force: true });
         }
@@ -883,103 +785,22 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
             changedPaths: [...counters.changedPaths, ...scopeRun.changedPaths],
         };
     }
-    // Hold the local body where it is, recording the remote etag so the conflict
-    // cannot re-fire (#137) and arming the journal-honesty flag because the held
-    // copy never matched that etag. The flag stops the currency-gated delete
-    // planner propagating a delete whose currency would falsely match on HEAD.
-    // Any genuine future download clears it by replacing the entry.
-    const holdLocal = () => {
+    if (resolution === "keep" || resolution === "skip") {
         counters.filesSkipped++;
-        updateEntry(run.journal, remoteFile.key, localHash, localSize, "down", localPath, {
+        updateEntry(run.journal, remoteFile.key, item.localHash, item.localSize, "down", localPath, {
             remoteEtag: remoteFile.etag,
-            mtimeMs: localMtime.getTime(),
-            kind: localKind,
-        });
-        const heldEntry = getEntry(run.journal, remoteFile.key);
-        if (heldEntry)
-            heldEntry.localDiverges = true;
-        return null;
-    };
-    // `publish-local` is a PUSH-leg strategy: it means "send my body up". On the
-    // pull leg there is nothing to publish, so it holds the local body and arms
-    // the honesty flag, which is what the push leg then promotes under an
-    // If-Match fence. Falling through to the download branch instead would
-    // overwrite the very local copy the operator elected, which is the data loss
-    // this strategy exists to prevent.
-    //
-    // `skip` is the interactive "leave this one, I will deal with it later". A
-    // resolution that says leave it alone may not rename the operator's file, so
-    // it holds too — it is deliberately NOT bundled with `keep` any more.
-    if (resolution === "skip" || resolution === "publish-local") {
-        return holdLocal();
-    }
-    // `keep` ADOPTS the cloud body as the working copy and preserves the local
-    // one beside it under the `.conflict-` name.
-    //
-    // It used to mean the opposite — hold local, stamp the remote etag, arm
-    // `localDiverges` — and that combination was terminal. The push leg refuses
-    // to send a body the journal calls divergent; the pull leg sees a remote
-    // whose etag already matches and leaves it alone; and every later `keep`
-    // re-armed the flag. The file converged in neither direction, and because the
-    // desktop app hardcodes `--on-conflict keep` that was the default outcome for
-    // every real conflict. Adopting the remote clears the divergence by
-    // construction: the entry then describes the body that is actually on disk.
-    // The local bytes are not destroyed, only renamed.
-    if (resolution === "keep") {
-        if (!remoteFetched) {
-            // The probe never produced a body (its failure is already emitted above).
-            // Hold rather than move aside a file we have nothing to replace it with.
-            return holdLocal();
-        }
-        try {
-            // The sidecar currently holds the fetched probe copy of the remote.
-            // Remove it explicitly before renaming: POSIX rename replaces a file at
-            // the destination, but Windows rename rejects an occupied destination.
-            // The download queued below re-fetches it. One extra GET buys the entire
-            // tested download path —
-            // symlink materialization, path codec, mtime and etag journalling —
-            // instead of a hand-rolled three-way swap with its own crash window.
-            fs.rmSync(conflictAbs, { force: true });
-            fs.renameSync(localPath, conflictAbs);
-        }
-        catch (preserveErr) {
-            run.emit({
-                type: "error",
-                path: remoteFile.key,
-                message: `conflict keep could not preserve the local copy: ${preserveErr instanceof Error ? preserveErr.message : String(preserveErr)}`,
-            });
-            return holdLocal();
-        }
-        try {
-            appendConflictEntry(run.hqRoot, {
-                id: buildConflictId(originalRelative, detectedAt),
-                originalPath: originalRelative,
-                conflictPath: conflictRelative,
-                detectedAt,
-                side: "pull",
-                machineId,
-                localHash,
-                remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
-                preserved: "local",
-            });
-        }
-        catch (mirrorErr) {
-            run.emit({
-                type: "error",
-                path: remoteFile.key,
-                message: `conflict mirror index write failed: ${mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)}`,
-            });
-        }
-        downloadItems.push({
-            action: "download",
-            remoteFile,
-            localPath,
-            isNew: false,
-            // The local body was just moved aside, so the pre-replace guard must
-            // expect absence. Passing the original snapshot would make that guard
-            // abort the very download it exists to protect.
-            localSnapshot: { kind: "absent" },
+            mtimeMs: item.localMtime.getTime(),
+            kind: item.localSnapshot.kind === "symlink" ? "symlink" : "file",
         });
+        // Journal-honesty: we recorded the remote etag so this conflict can't
+        // re-fire (#137), but the KEPT local copy diverges from that remote — it
+        // never matched it. Flag the entry so the currency-gated delete planner
+        // refuses to propagate a delete for it (its currency would falsely match on
+        // HEAD, and the delete would destroy the divergent remote version). Any
+        // genuine future download clears the flag by replacing the entry.
+        const keptEntry = getEntry(run.journal, remoteFile.key);
+        if (keptEntry)
+            keptEntry.localDiverges = true;
         return null;
     }
     downloadItems.push({
@@ -987,7 +808,7 @@ async function executeConflictItem(run, plan, scopeRun, counters, downloadItems,
         remoteFile,
         localPath,
         isNew: false,
-        localSnapshot,
+        localSnapshot: item.localSnapshot,
     });
     return null;
 }
@@ -1070,12 +891,6 @@ async function downloadOne(run, downloadItem, counters) {
         updateEntry(run.journal, remoteFile.key, hash, size, "down", localPath, {
             remoteEtag: remoteFile.etag,
             mtimeMs: localLstat.mtimeMs,
-            // Stamped from the lstat already in hand so the next push can skip this
-            // file on stat alone. The two conflict-resolution `updateEntry` calls
-            // above deliberately do NOT stamp ctime: neither has a post-write stat
-            // available, and a `keep`/`publish-local` entry is marked
-            // `localDiverges` immediately after, which the fast path refuses anyway.
-            ctimeMs: localLstat.ctimeMs,
             ...(createdBySub !== undefined ? { createdBySub } : {}),
             kind: isLocalSymlink ? "symlink" : "file",
         });
@@ -1137,7 +952,7 @@ async function downloadOne(run, downloadItem, counters) {
             run.emit({
                 type: "error",
                 path: remoteFile.key,
-                message: describeError(err),
+                message: err instanceof Error ? err.message : String(err),
             });
         }
         return;
@@ -1370,12 +1185,6 @@ function finalizePullRun(run, plan, scopeRun, counters) {
  * -files loop this event exists to end. `SyncResult.filesOutOfScope` keeps its
  * original both-causes meaning; only this advisory is narrowed.
  */
-// NOTE: since scoped pulls list only their in-scope prefixes (see
-// `listRemoteFilesForScope`), out-of-scope remote keys usually never enter the
-// plan and this event fires only for keys a broad-postfilter listing (or a
-// legacy caller passing a full listing) still surfaced. The withheld-count
-// telemetry traded away here is the cost of not materializing the full remote
-// listing on every scoped pull.
 function emitScopeMaterializationGap(run, plan) {
     if (run.syncMode === "all")
         return;
@@ -1505,36 +1314,6 @@ function matchesLocalSnapshot(localPath, expected) {
     return ((actual.kind !== "file" && actual.kind !== "symlink") ||
         actual.hash === expected.hash);
 }
-/**
- * Read the local side at the execution seam, not from the whole-run preview.
- * A second hash check closes the read-to-stat window: when an active writer
- * moves the file while we inspect it, the caller defers this path instead of
- * manufacturing a decision from a mixed snapshot.
- */
-function readCurrentLocalFileState(localPath) {
-    const snapshot = snapshotLocalState(localPath);
-    if (snapshot === null || (snapshot.kind !== "file" && snapshot.kind !== "symlink")) {
-        return null;
-    }
-    let lstat;
-    try {
-        lstat = fs.lstatSync(localPath);
-    }
-    catch {
-        return null;
-    }
-    if (!matchesLocalSnapshot(localPath, snapshot))
-        return null;
-    return {
-        hash: snapshot.hash,
-        size: snapshot.kind === "symlink" ? 0 : lstat.size,
-        mtime: lstat.mtime,
-        mtimeMs: lstat.mtimeMs,
-        ctimeMs: lstat.ctimeMs,
-        kind: snapshot.kind,
-        snapshot,
-    };
-}
 function hasCurrentLocalDeleteIntent(entry) {
     const intent = entry?.localDeleteIntent;
     return !!(intent &&
@@ -2354,17 +2133,6 @@ excludePrefixes = [], onHeartbeat) {
         const posixKey = toPosixKey(key);
         if (remoteKeySet.has(posixKey))
             continue;
-        // Listing-scope guard (targeted sync / scoped listing). The remote LIST
-        // now covers ONLY `prefixSet` (scoped modes list per-prefix; a targeted
-        // pull narrows further), so "absent from the LIST" is evidence of a remote
-        // delete ONLY for keys the LIST could have contained. A journal key
-        // outside the listed scope was simply never asked about — treating it as
-        // remote-deleted would mass-delete every out-of-scope local file on the
-        // first scoped pull. Out-of-scope journal entries stay the scope-shrink
-        // pass's responsibility (quarantine semantics), never this sweep's.
-        // `all` mode passes `[""]`, which covers every key — no behavior change.
-        if (!isCoveredByAny(posixKey, prefixSet))
-            continue;
         if (posixKey.startsWith(SKILLS_KEY_PREFIX) &&
             remoteCanonicalSkillKeys.has(canonicalVaultKeySpelling(posixKey)) &&
             // An EXACT FILE_TOMBSTONE for the journaled key is an authoritative