@indigoai-us/hq-cloud 6.18.104 → 6.18.106

package/dist/cli/share.js

@@ -24,7 +24,7 @@ import { readPendingConflictNotices, recordConflictParked } from "../conflict-no
 import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
 import { uploadFile, uploadSymlink, toPosixKey, headRemoteFile, readVerifiedRemoteSymlinkTarget, deleteRemoteFile, downloadFile, primeObjectTransport, primeUploads, publishedContentHash, setUploadContentHash, WindowsSymlinkPrivilegeError, WINDOWS_SYMLINK_PRIVILEGE_REMEDY, WindowsRenameBlockedError, WINDOWS_RENAME_BLOCKED_REMEDY, renameOrRaiseWindowsBlocked, } from "../s3.js";
 import * as crypto from "crypto";
-import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, } from "../object-io.js";
+import { bindObjectIOToContext, PresignObjectIO, RateLimitedError, resolveObjectIO, S3SdkObjectIO, isPresignDeletedNewer, } from "../object-io.js";
 import { primePrivateFolderUpload } from "../private-folder-transport.js";
 import { getJournalPath, isUnderAnyRoot, readJournalForPass, bindPersonalVaultJournalToIdentity, readJournalReadOnly, readJournalScoped, readQuarantinedJournalPaths, writeJournal, setJournalSnapshotQuarantineListener, hashFile, hashSymlinkTarget, journalRows, journalEntryMatchesStat, journalFingerprintMatchesStat, dropJournalDecodeWindow, timestampsTooCoarseToTrust, updateEntry, PrematureJournalEntryError, removeEntry, isTombstone, normalizeEtag, PERSONAL_VAULT_JOURNAL_SLUG, migratePersonalVaultJournal, migrateToV2, configureRejectedDeltaRecovery, getJournalRecoveryMarker, getStateDir, personalVaultHasOtherIdentityState, readPersonalVaultOwnerUid, } from "../journal.js";
 import { admitPushIdentity, machineIdentitySwitchRecorded, notePushIdentityHeld, noteCompanyRebind, } from "../push-identity-fence.js";
@@ -38,11 +38,11 @@ import { createHumanSkipLineLog, formatReconcileSkipLine, isUpToDateSkipReason,
 import { fetchFileTombstones, } from "./tombstones.js";
 import { compileScopeMatcher, } from "../prefix-coalesce.js";
 import { hasRemoteChanged, isAccessDenied, resolveActiveCompany, resolveTransferConcurrency, resolveUploadByteBudget, readHostLoad, } from "../sync-core.js";
-import { buildConflictBackupPath, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
+import { buildConflictBackupPath, CONFLICT_BACKUPS_DIR, buildConflictId, buildConflictProbePath, readShortMachineId, } from "../lib/conflict-file.js";
 import { appendConflictEntry, findReusableConflictBackup } from "../lib/conflict-index.js";
 import { appendIfUnchanged, isAppendOnlyPath, mergeAppendOnly } from "../lib/append-only-merge.js";
 import { isBoardJsonPath, mergeBoardJson, replaceIfUnchanged } from "../lib/board-merge.js";
-import { findLandedUpload, followLandedUploads, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, } from "../lib/same-path-upload-lock.js";
+import { findLandedUpload, findUploadIntent, followLandedUploads, recordUploadIntent, SamePathUploadHold, samePathUploadKey, SamePathUploadSupersededError, settleUploadIntent, } from "../lib/same-path-upload-lock.js";
 import { isCloudAuthoritative, isLocallyIneligibleForPush } from "../lib/cloud-authoritative.js";
 import { groupServerOwnedSkips } from "../lib/server-owned-skips.js";
 import { isOntologyEntityKey, ONTOLOGY_MULTI_WRITER_FLAG, resolveOntologyEntityPush, } from "../lib/ontology-entity-push.js";
@@ -114,6 +114,60 @@ async function remoteContentDiffers(ctx, relativePath, localHash, hqRoot, symlin
         }
     }
 }
+/**
+ * A push refused as `deleted-newer`: keep the live local file, copy its bytes
+ * once into `.hq/conflict-backups/` (skipped when a backup with the same bytes
+ * already exists for this key, so repeated passes do not pile up copies), and
+ * drop the journal row so the pull tombstone sweep cannot delete the file.
+ */
+function keepLocalOverDeletedNewer(run, absolutePath, relativePath) {
+    let backedUp = false;
+    try {
+        const localHash = hashFile(absolutePath);
+        const vaultKey = vaultKeyForLocalPath(run.hqRoot, absolutePath);
+        const probe = localPathForVaultKey(run.hqRoot, buildConflictBackupPath(vaultKey, new Date().toISOString(), readShortMachineId(run.hqRoot)));
+        // The backup must stay inside the sync-ignored backups directory. A key
+        // that resolves outside the HQ root skips the backup; the live file and
+        // the journal drop below still protect the bytes.
+        const backupsRoot = path.resolve(run.hqRoot, CONFLICT_BACKUPS_DIR);
+        const relToBackups = path.relative(backupsRoot, path.resolve(probe));
+        if (relToBackups.startsWith("..") || path.isAbsolute(relToBackups)) {
+            throw new Error("backup path escapes the conflict-backups directory");
+        }
+        const dir = path.dirname(probe);
+        const base = path.basename(vaultKey);
+        const existing = fs.existsSync(dir)
+            ? fs.readdirSync(dir).filter((name) => name.startsWith(`${base}.`) && name !== base)
+            : [];
+        const already = existing.some((name) => {
+            try {
+                return hashFile(path.join(dir, name)) === localHash;
+            }
+            catch {
+                return false;
+            }
+        });
+        if (!already) {
+            fs.mkdirSync(dir, { recursive: true });
+            fs.copyFileSync(absolutePath, probe, fs.constants.COPYFILE_EXCL);
+        }
+        backedUp = true;
+    }
+    catch {
+        // A failed backup still leaves the live file in place; the journal drop
+        // below is what keeps the pull from deleting it.
+    }
+    if (journalRows(run.journal).delete(relativePath))
+        flushShareJournal(run);
+    run.emit({
+        type: "error",
+        diagnostic: true,
+        path: relativePath,
+        message: `server has a newer deleted version of '${relativePath}', kept your copy` +
+            (backedUp ? " (backup in .hq/conflict-backups/)" : "") +
+            `. Restore the newer version with 'hq files restore --version-id', or delete your copy.`,
+    });
+}
 /**
  * Local-only ephemeral artifacts: conflict-side files written whenever a
  * 3-way merge preserves the displaced side for inspection. Depending on the
@@ -1180,6 +1234,16 @@ async function runShareOnce(options, run) {
         return buildShareResult(run, counters, filesRefusedStalePaths, uploadResult.abortFlightConflictPaths, pathResults, true);
     }
     await executeDeletes(run, plans.deletePlan, plans.decommissionPlan, counters, filesRefusedStalePaths, pathResults, !uploadResult.hadUploadFailure && uploadResult.workerErrors.length === 0);
+    if (run.writeForbiddenSet.size > REFUSED_STALE_PATH_CAP) {
+        const more = run.writeForbiddenSet.size - REFUSED_STALE_PATH_CAP;
+        run.emit({
+            type: "error",
+            diagnostic: true,
+            path: "(write-forbidden)",
+            message: `${more} more files were refused by the vault (no write access, server put denied: FILES_PRESIGN_FORBIDDEN); ` +
+                `${run.writeForbiddenSet.size} refused in total, kept local-only`,
+        });
+    }
     finalizeShareJournal(run);
     throwUploadWorkerErrors(uploadResult.workerErrors, run.emit);
     // A reconcile cursor is an all-or-nothing boundary for this bounded apply
@@ -1587,6 +1651,7 @@ async function createPushRunContext(options) {
             threshold: checkpointThresholdFor(journal, options.durableFullPassProgress === true),
             baselineLastSync: journal.lastSync,
             error: null,
+            unjournaledIntents: [],
         },
     };
 }
@@ -3056,6 +3121,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
             await restartForChangedBody(afterHead);
             return;
         }
+        // The remote may be an upload this machine started whose journal row
+        // never reached disk (the pass was killed after its PUT). Take it as the
+        // baseline before classifying, so the next local edit is an ordinary
+        // one-sided change instead of a conflict against our own write.
+        if (remoteMeta && item.kind === "file")
+            adoptOwnUploadIntent(run, item, remoteMeta);
         // Journal entry for this key (if any). Used both for 3-way conflict
         // classification and for the write fence below: Prefer last-synced
         // `remoteEtag` over live HEAD so a peer-advanced object cannot be
@@ -3164,8 +3235,10 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
             // Replaying that unchanged baseline is a stale revert, not an edit. A
             // user who deliberately restores some other historic content has a hash
             // different from the journal baseline and still reaches normal conflict
-            // resolution below.
-            if (journalEntry !== undefined &&
+            // resolution below. A remote still at the journaled revision has not
+            // superseded anything, whatever its published lineage says.
+            if (remoteChanged &&
+                journalEntry !== undefined &&
                 journalEntry.localDiverges !== true &&
                 remoteContentHash !== undefined &&
                 journalEntry.hash === localHash &&
@@ -3540,7 +3613,7 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                     ? { ifMatch: journalEntry.remoteEtag }
                     : { ifMatch: remoteMeta.etag }
             : { ifNoneMatch: "*" };
-        const performUpload = async (pc, observedRemote = remoteMeta) => {
+        const performUploadOnce = async (pc, observedRemote) => {
             const isSymlinkUpload = item.kind === "symlink";
             const lstat = fs.lstatSync(absolutePath);
             const size = isSymlinkUpload ? 0 : lstat.size;
@@ -3767,6 +3840,9 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                 else {
                     // Strictly after the PUT returned and the entry was applied, so an
                     // interrupted pass can only ever re-upload a file, never skip one.
+                    // Until that checkpoint the upload's intent stays on disk, so a pass
+                    // killed before it cannot make the next pass take this object for a
+                    // peer's edit (feedback_a1696094).
                     checkpointShareJournal(run);
                 }
                 if (skillContentAtPut !== undefined &&
@@ -3797,6 +3873,36 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                 transfer.complete();
             }
         };
+        // The intent is on disk before the PUT leaves. A landed PUT keeps it until
+        // the next journal write puts its row on disk (see writeShareJournal),
+        // including when this call then fails on that very write. Only a PUT the
+        // server refused (412), or one superseded before it was sent, is known not
+        // to have landed and forgets it at once.
+        // Any other failure (timeout, reset) may have landed, so the intent stays
+        // for this process's life and ages out on disk. A process that dies with
+        // intents outstanding leaves them for the next pass to recognise those
+        // objects as this machine's own writes instead of a peer's edits.
+        const performUpload = async (pc, observedRemote = remoteMeta) => {
+            const intent = item.kind === "file"
+                ? recordUploadIntent(run.journalSlug, relativePath, localHash)
+                : undefined;
+            const landedBefore = uploadCompleted;
+            try {
+                await performUploadOnce(pc, observedRemote);
+            }
+            catch (err) {
+                if (intent) {
+                    if (uploadCompleted && !landedBefore)
+                        run.checkpoint.unjournaledIntents.push(intent);
+                    else if (isPreconditionFailed(err) || err instanceof SamePathUploadSupersededError) {
+                        settleUploadIntent(intent);
+                    }
+                }
+                throw err;
+            }
+            if (intent)
+                run.checkpoint.unjournaledIntents.push(intent);
+        };
         try {
             await performUpload(precondition);
         }
@@ -3962,6 +4068,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                                 throw retryErr;
                             if (emitPlanLimitSkip(retryErr, relativePath))
                                 return;
+                            if (isPresignDeletedNewer(retryErr)) {
+                                keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
+                                counters.filesSkipped++;
+                                counters.filesSkippedConflict++;
+                                return;
+                            }
                             if (isPreconditionFailed(retryErr)) {
                                 counters.filesSkipped++;
                                 counters.filesSkippedConflict++;
@@ -4073,6 +4185,12 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                         }
                         if (emitPlanLimitSkip(retryErr, relativePath))
                             return;
+                        if (isPresignDeletedNewer(retryErr)) {
+                            keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
+                            counters.filesSkipped++;
+                            counters.filesSkippedConflict++;
+                            return;
+                        }
                         hadUploadFailure = true;
                         run.uploadFailedSet.add(relativePath);
                         run.emit({
@@ -4093,18 +4211,35 @@ async function executeUploads(run, pushPlan, counters, conflictPaths) {
                 counters.filesSkippedConflict++;
                 return;
             }
+            if (isPresignDeletedNewer(err)) {
+                // The server holds a NEWER version of this key behind a delete marker
+                // (hq-pro refusalReason "deleted-newer", feedback_7b30e522). Keep the
+                // local file, park one backup copy out of tree, and drop the journal
+                // row so the pull leg sees an untracked local file instead of a
+                // remote delete to apply. Not an upload failure: nothing was lost.
+                keepLocalOverDeletedNewer(run, item.absolutePath, relativePath);
+                counters.filesSkipped++;
+                counters.filesSkippedConflict++;
+                return;
+            }
             if (isAccessDenied(err)) {
                 hadUploadFailure = true;
                 if (isPresignWriteForbidden(err)) {
                     // A write denial on a key in this push: report it instead of
                     // folding it into the silent scope-excluded count (feedback_682e9187).
+                    // Name the first REFUSED_STALE_PATH_CAP paths; the rest are folded
+                    // into one summary event after the uploads finish, so a member
+                    // without write access to a large folder does not get one line per
+                    // file on every pass (feedback_f84671f4).
                     run.writeForbiddenSet.add(relativePath);
-                    run.emit({
-                        type: "error",
-                        diagnostic: true,
-                        path: relativePath,
-                        message: describeError(err),
-                    });
+                    if (run.writeForbiddenSet.size <= REFUSED_STALE_PATH_CAP) {
+                        run.emit({
+                            type: "error",
+                            diagnostic: true,
+                            path: relativePath,
+                            message: describeError(err),
+                        });
+                    }
                     return;
                 }
                 run.scopeExcludedSet.add(relativePath);
@@ -4375,6 +4510,43 @@ function adoptLandedSamePathUpload(run, item, landed) {
     }
     return true;
 }
+/**
+ * Take a remote revision as this run's baseline when its published content
+ * hash matches an upload this machine recorded before a PUT whose journal row
+ * never reached disk (the pass was killed in between). The object then holds
+ * our own bytes, so the decision below sees an ordinary local edit, fenced on
+ * the remote ETag, instead of a two-sided conflict that lets the cloud copy
+ * replace the newer local file (feedback_a1696094).
+ *
+ * The row carries no local stat stamps: they would claim the local body still
+ * hashes to the adopted value. Returns true when the row was replaced.
+ */
+function adoptOwnUploadIntent(run, item, remoteMeta) {
+    const remoteHash = publishedContentHash(remoteMeta.metadata);
+    if (remoteHash === undefined || remoteHash === item.localHash)
+        return false;
+    const row = journalRows(run.journal).get(item.relativePath);
+    if (row?.localDiverges === true || row?.hash === remoteHash)
+        return false;
+    const remoteEtag = normalizeEtag(remoteMeta.etag);
+    if (row?.remoteEtag !== undefined && normalizeEtag(row.remoteEtag) === remoteEtag)
+        return false;
+    const rowWrittenAt = row ? Date.parse(row.syncedAt) : Number.NaN;
+    const intent = findUploadIntent(run.journalSlug, item.relativePath, remoteHash, Number.isFinite(rowWrittenAt) ? rowWrittenAt : undefined);
+    if (!intent)
+        return false;
+    try {
+        updateEntry(run.journal, item.relativePath, remoteHash, remoteMeta.size, "up", item.absolutePath, {
+            remoteEtag,
+            kind: "file",
+        });
+    }
+    catch {
+        // The local path vanished; the normal replan settles it.
+        return false;
+    }
+    return true;
+}
 /**
  * Exact-prefix and storage-class-specific, matching the pull planner: only
  * session transcripts in DEEP_ARCHIVE are frozen. Other archived objects keep
@@ -4694,7 +4866,11 @@ function recordPushConflictNotice(run, item, backupPath, sideKept, decision, rem
             companySlug: run.options.personalMode === true ? null : run.ctx.slug,
             relativePath: vaultKeyForLocalPath(run.hqRoot, item.absolutePath),
             backupPath,
-            winnerReason: remoteChangedSinceJournal
+            // A remote kept over a NEWER local body is not "remote-newer"; the
+            // decision names why it was kept.
+            winnerReason: remoteChangedSinceJournal ||
+                decision?.reason === "remote-changed" ||
+                decision?.reason === "no-baseline"
                 ? "remote-changed-since-journal"
                 : sideKept === "local" ? "local-newer" : "remote-newer",
             sideKept,
@@ -4896,7 +5072,7 @@ async function adoptRemoteOnPush(run, item, remoteHash, decision, probe) {
         // Persist the converged remote-backed entry immediately. A conflict
         // resolution is a completed transfer even though the push counter is
         // recorded as skipped, and an interrupted pass must not revive it.
-        checkpointShareJournal(run);
+        checkpointShareJournal(run, { persistRow: true });
     }
     catch (adoptErr) {
         if (parkAttempted && (!backupHash || backupHash !== item.localHash)) {
@@ -5310,25 +5486,34 @@ function checkpointThresholdFor(journal, durableFullPassProgress = false) {
  * Called after each result is applied to the in-memory journal. The entry is
  * written only AFTER its object PUT has returned, so a checkpoint can at worst
  * cause a file to be re-uploaded — never mark an unshipped file as shipped.
+ *
+ * `persistRow` writes the journal even between checkpoints, for a result that
+ * changed the vault (an upload, an adopted conflict). The write appends only
+ * the changed rows; the object-event flush and the checkpoint counters keep
+ * their cadence.
  */
-function checkpointShareJournal(run) {
+function checkpointShareJournal(run, { persistRow = false } = {}) {
     run.checkpoint.appliedSinceFlush += 1;
     const override = checkpointCadenceOverride();
     if (override !== null) {
-        if (run.checkpoint.appliedSinceFlush >= override)
+        if (run.checkpoint.appliedSinceFlush >= override) {
             flushShareJournal(run);
-        return;
+            return;
+        }
     }
-    if (run.checkpoint.appliedSinceFlush < run.checkpoint.threshold)
-        return;
-    if (run.options.durableFullPassProgress === true) {
-        flushShareJournal(run);
-        return;
+    else if (run.checkpoint.appliedSinceFlush >= run.checkpoint.threshold) {
+        if (run.options.durableFullPassProgress === true) {
+            flushShareJournal(run);
+            return;
+        }
+        const now = Date.now();
+        if (now - run.checkpoint.lastFlushAt >= CHECKPOINT_MIN_INTERVAL_MS) {
+            flushShareJournal(run, now);
+            return;
+        }
     }
-    const now = Date.now();
-    if (now - run.checkpoint.lastFlushAt < CHECKPOINT_MIN_INTERVAL_MS)
-        return;
-    flushShareJournal(run, now);
+    if (persistRow)
+        writeShareJournal(run);
 }
 /**
  * Publish the journal as it currently stands, WITHOUT advancing `lastSync`.
@@ -5336,6 +5521,14 @@ function checkpointShareJournal(run) {
  * "these objects are shipped", not "this sync finished".
  */
 function flushShareJournal(run, now = Date.now()) {
+    writeShareJournal(run);
+    run.checkpoint.appliedSinceFlush = 0;
+    run.checkpoint.lastFlushAt = now;
+    void run.objectEvents.flush();
+    run.checkpoint.threshold = checkpointThresholdFor(run.journal, run.options.durableFullPassProgress === true);
+}
+/** The journal write of {@link flushShareJournal}, without its bookkeeping. */
+function writeShareJournal(run) {
     const liveLastSync = run.journal.lastSync;
     run.journal.lastSync = run.checkpoint.baselineLastSync;
     const release = setJournalSnapshotQuarantineListener(run.journalSlug, (diagnostic) => run.emit({ type: "journal-quarantine", ...diagnostic }));
@@ -5350,10 +5543,16 @@ function flushShareJournal(run, now = Date.now()) {
         release();
         run.journal.lastSync = liveLastSync;
     }
-    run.checkpoint.appliedSinceFlush = 0;
-    run.checkpoint.lastFlushAt = now;
-    void run.objectEvents.flush();
-    run.checkpoint.threshold = checkpointThresholdFor(run.journal, run.options.durableFullPassProgress === true);
+    settleJournaledIntents(run);
+}
+/** The rows of every landed upload are on disk now; their intents can go. */
+function settleJournaledIntents(run) {
+    const settled = run.checkpoint.unjournaledIntents;
+    if (settled.length === 0)
+        return;
+    run.checkpoint.unjournaledIntents = [];
+    for (const intent of settled)
+        settleUploadIntent(intent);
 }
 function finalizeShareJournal(run) {
     run.journal.lastSync = new Date().toISOString();
@@ -5364,6 +5563,7 @@ function finalizeShareJournal(run) {
     finally {
         release();
     }
+    settleJournaledIntents(run);
     if (run.excludedSet.size > 0) {
         const samplePaths = [];
         for (const p of run.excludedSet) {
@@ -5608,6 +5808,7 @@ function buildShareResult(run, counters, filesRefusedStalePaths, conflictPaths,
         uploadFailedPaths: [...run.uploadFailedSet].slice(0, REFUSED_STALE_PATH_CAP),
         filesWriteForbidden: run.writeForbiddenSet.size,
         writeForbiddenPaths: [...run.writeForbiddenSet].slice(0, REFUSED_STALE_PATH_CAP),
+        writeForbiddenHoldPaths: [...run.writeForbiddenSet],
         filesSuppressedByTombstone: counters.filesSuppressedByTombstone,
         ...(run.skillRegisterLandedBytes
             ? { skillRegistrationFallbacks: run.skillRegistrationFallbacks }