@indigoai-us/hq-cloud 5.33.0 → 5.35.0

package/src/cli/share.ts

@@ -9,7 +9,8 @@ import * as fs from "fs";
 import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
-import { uploadFile, uploadSymlink, headRemoteFile, deleteRemoteFile } from "../s3.js";
+import { uploadFile, uploadSymlink, headRemoteFile, deleteRemoteFile, downloadFile } from "../s3.js";
+import * as crypto from "crypto";
 import type { UploadAuthor } from "../s3.js";
 import {
   readJournal,
@@ -28,6 +29,12 @@ import {
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy } from "./conflict.js";
 import type { SyncProgressEvent } from "./sync.js";
+import {
+  buildConflictId,
+  buildConflictPath,
+  readShortMachineId,
+} from "../lib/conflict-file.js";
+import { appendConflictEntry } from "../lib/conflict-index.js";
 
 /**
  * Local-only ephemeral artifacts: conflict-mirror files written by the pull
@@ -71,16 +78,20 @@ import type { SyncProgressEvent } from "./sync.js";
  * journaled mirror that's been deleted locally doesn't get included in the
  * regular delete plan (the dedicated reconcile path handles existing litter).
  */
-const EPHEMERAL_PATH_PATTERN =
+export const EPHEMERAL_PATH_PATTERN =
   /\.conflict-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z-(?:[a-f0-9]+|unknown)(?:\.[^/]*)?$/;
 
 /**
  * Cheap pure check — pass the relative key OR a basename; either works. Used
  * in both the file walker (basename matching) and the delete-plan walker
- * (relative-key matching). The regex matches anywhere in the string, which is
- * fine: the `.conflict-<ISO>-<hash>.` token is unambiguous.
+ * (relative-key matching), and also by the pull walker in sync.ts to refuse
+ * downloading legacy conflict-mirror files that still live in cloud staging
+ * (Bug #2 in the 5.33.0 deep-test report — push-side filtered them since
+ * 5.33.0 but pull-side downloaded them freely until this export). The regex
+ * matches anywhere in the string, which is fine: the
+ * `.conflict-<ISO>-<hash>.` token is unambiguous.
  */
-function isEphemeralPath(p: string): boolean {
+export function isEphemeralPath(p: string): boolean {
   return EPHEMERAL_PATH_PATTERN.test(p);
 }
 
@@ -440,6 +451,16 @@ export interface ShareResult {
    * `currency-gated`.
    */
   filesRefusedStale: number;
+  /**
+   * Paths corresponding to `filesRefusedStale`, capped at 50 to keep the
+   * event payload bounded (mirrors `newFiles` capping). Surfaces *which*
+   * paths were refused so operators can triage the recurring
+   * \`filesRefusedStale: 205\` signal flagged in the 5.33.0 deep-test —
+   * the count alone is impossible to investigate because the per-file
+   * \`delete-refused-stale-etag\` events vanish from the event stream
+   * once the runner has folded them into the totals.
+   */
+  filesRefusedStalePaths: string[];
   /**
    * Number of paths blocked by `PERSONAL_VAULT_DEFAULT_EXCLUSIONS` during this
    * run (push leg, personalMode=true). Includes both files that would have
@@ -568,6 +589,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
   // propagateDeletes=false path.
   let filesTombstoned = 0;
   let filesRefusedStale = 0;
+  // Capped at 50 to bound event payload size — `newFiles` uses the same cap.
+  const REFUSED_STALE_PATH_CAP = 50;
+  const filesRefusedStalePaths: string[] = [];
   const conflictPaths: string[] = [];
 
   // Collect all files to share
@@ -692,13 +716,58 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     // the current remote ETag against the one captured at last sync; when
     // missing (legacy entries), we fall back to the same `lastModified >
     // syncedAt` heuristic the pull side uses.
+    //
+    // Bug #7 (data-loss class — see workspace/reports/hq-cloud-5.33.0-
+    // deep-test.md): for a path with NO prior journal entry (first push
+    // from this machine), the localChanged/remoteChanged predicates above
+    // both evaluate FALSE (their guards require `!!journalEntry`). Push
+    // fell through to an unconditional PUT, silently clobbering any
+    // peer's content already at that key. The verification report's V7
+    // isolated this — the bug is independent of \`--on-conflict\` mode;
+    // it's keyed on "do I have a prior journal entry?" not on the flag.
+    //
+    // Fresh-collision branch: when remoteMeta exists and there's no
+    // journal entry, hash the local body (MD5 for parity with S3's
+    // single-part etag) and compare. Match → no conflict, silently skip
+    // the PUT (the bytes are already there). Mismatch → treat as a
+    // conflict in the same shared branch below.
     const remoteMeta = await headRemoteFile(ctx, relativePath);
     if (remoteMeta) {
       const journalEntry = journal.files[relativePath];
       const localChanged = !!journalEntry && journalEntry.hash !== localHash;
       const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
 
-      if (localChanged && remoteChanged) {
+      let isFreshCollision = false;
+      if (!journalEntry && item.kind === "file") {
+        // Single-part S3 PUT etag is MD5 of the body. Multipart uploads
+        // produce \`<md5>-<partCount>\`; we treat any non-single-part etag
+        // as ambiguous and DO classify as a conflict (safer for the
+        // first-time path — false positives prompt the operator, false
+        // negatives lose data). Symlink records (\`kind: "symlink"\`)
+        // skip the check entirely — the wire body shape (\`hq-symlink:\`
+        // prefix + target) isn't a pure byte mirror and would mis-
+        // classify; symlink overwrites are rare and an audit pass after
+        // the broader bug-cleanup wave can extend coverage if needed.
+        const remoteEtagNormalized = normalizeEtag(remoteMeta.etag);
+        const isMultipart = /-\d+$/.test(remoteEtagNormalized);
+        if (!isMultipart) {
+          const localBody = fs.readFileSync(absolutePath);
+          const localMd5 = crypto.createHash("md5").update(localBody).digest("hex");
+          if (localMd5 !== remoteEtagNormalized) {
+            isFreshCollision = true;
+          }
+          // Match → bytes are already there; fall through to upload
+          // path which is idempotent (S3 will overwrite with identical
+          // content + carry our metadata). Cheap, no behavior change.
+        } else {
+          // Multipart object pre-exists with unknown body shape — assume
+          // collision rather than risk a silent overwrite. The operator
+          // can resolve via the standard conflict prompt.
+          isFreshCollision = true;
+        }
+      }
+
+      if ((localChanged && remoteChanged) || isFreshCollision) {
         conflictPaths.push(relativePath);
 
         const resolution = await resolveConflict(
@@ -729,6 +798,10 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
             // ShareResult shape stable for consumers that destructure.
             filesTombstoned,
             filesRefusedStale,
+            // Always present so consumers can destructure without a
+            // defaulting fallback. Empty on the abort path because the
+            // delete-plan execution loop is short-circuited.
+            filesRefusedStalePaths,
             // Exclusions are computed during the upload walk which has
             // already completed by the time we hit a per-file conflict-
             // abort, so the count is meaningful here. No event emit on
@@ -740,6 +813,45 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
           };
         }
         if (resolution === "keep" || resolution === "skip") {
+          // Bug #7 mirror branch: when the resolution is keep/skip on a
+          // FRESH collision (no prior journal entry), download the
+          // remote bytes to \`<orig>.conflict-<ts>-<short>\` so both
+          // versions survive on disk. Mirrors the pull-side mirror-write
+          // routine in sync.ts exactly. Skipped for stale-journal
+          // conflicts (the pre-Bug-#7 codepath) — those already produce
+          // a pull-side mirror on the next sync cycle.
+          if (isFreshCollision) {
+            try {
+              const detectedAt = new Date().toISOString();
+              const machineId = readShortMachineId(hqRoot);
+              const originalRelative = path.relative(hqRoot, absolutePath);
+              const conflictRelative = buildConflictPath(
+                originalRelative,
+                detectedAt,
+                machineId,
+              );
+              const conflictAbs = path.join(hqRoot, conflictRelative);
+              await downloadFile(ctx, relativePath, conflictAbs);
+              appendConflictEntry(hqRoot, {
+                id: buildConflictId(originalRelative, detectedAt),
+                originalPath: originalRelative,
+                conflictPath: conflictRelative,
+                detectedAt,
+                side: "push",
+                machineId,
+                localHash,
+                remoteHash: normalizeEtag(remoteMeta.etag),
+              });
+            } catch (mirrorErr) {
+              emit({
+                type: "error",
+                path: relativePath,
+                message: `conflict mirror write failed: ${
+                  mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
+                }`,
+              });
+            }
+          }
           filesSkipped++;
           continue;
         }
@@ -862,6 +974,9 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
   for (const refused of deletePlan.refusedStale) {
     if (decommissionedSet && decommissionedSet.has(refused.key)) continue;
     filesRefusedStale++;
+    if (filesRefusedStalePaths.length < REFUSED_STALE_PATH_CAP) {
+      filesRefusedStalePaths.push(refused.key);
+    }
     emit({
       type: "delete-refused-stale-etag",
       path: refused.key,
@@ -901,6 +1016,7 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     filesDeleted,
     filesTombstoned,
     filesRefusedStale,
+    filesRefusedStalePaths,
     filesExcludedByPolicy: excludedSet.size,
     conflictPaths,
     aborted: false,