@indigoai-us/hq-cloud 6.2.0 → 6.2.2

package/src/cli/rescue-core.ts

@@ -28,6 +28,13 @@ import { spawnSync } from "child_process";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
+import {
+  readJournal,
+  writeJournal,
+  hashFile,
+  updateEntry,
+  PERSONAL_VAULT_JOURNAL_SLUG,
+} from "../journal.js";
 
 export interface RunRescueResult {
   status: number;
@@ -792,7 +799,7 @@ function doRescue(
   }
   const overlay = run(
     "rsync",
-    ["-a", ...overlayProtect, ...rsyncExcludes, srcDir + "/", hqRoot + "/"],
+    ["-ai", ...overlayProtect, ...rsyncExcludes, srcDir + "/", hqRoot + "/"],
     { env },
   );
   if (overlay.status !== 0) {
@@ -818,6 +825,68 @@ function doRescue(
     }
   }
 
+  // --- Reconcile the sync-journal baseline for re-laid files ---
+  // The overlay rewrote scaffold files from upstream, but their personal-vault
+  // sync-journal entries still carry the PRE-rescue hash. The next sync would
+  // then read localHash != journal.hash ("local changed") and — when the vault
+  // also moved — mint a false `.conflict-*` mirror (the root cause of the
+  // conflict-file litter). Re-stamp the baseline to the freshly laid-down
+  // content so `localChanged` is false: the common single-machine case
+  // converges silently, and a genuinely-divergent vault degrades to a clean
+  // pull instead of a conflict mirror.
+  //
+  // Scoped STRICTLY to the paths rsync reported transferring (`-i` itemize) —
+  // never a blanket journal rewrite — so a user's unsynced pending edit (which
+  // the walk already rescued to personal/ rather than overlaying) is never
+  // silently marked as synced. Best-effort: a reconcile failure must never
+  // fail the rescue.
+  try {
+    const relaid: string[] = [];
+    for (const line of overlay.stdout.split("\n")) {
+      // itemize line: 11-char change code, space, path. 2nd col `f` = file.
+      const m = /^[<>ch.]f\S{9} (.+)$/.exec(line);
+      if (m) relaid.push(m[1]);
+    }
+    if (relaid.length > 0) {
+      const journal = readJournal(PERSONAL_VAULT_JOURNAL_SLUG);
+      let restamped = 0;
+      for (const rel of relaid) {
+        const prev = journal.files[rel];
+        if (!prev) continue; // only re-stamp paths already in the synced baseline
+        const abs = path.join(hqRoot, rel);
+        let st: fs.Stats;
+        try {
+          st = fs.lstatSync(abs);
+        } catch {
+          continue;
+        }
+        if (st.isSymbolicLink() || !st.isFile()) continue;
+        updateEntry(
+          journal,
+          rel,
+          hashFile(abs),
+          st.size,
+          prev.direction,
+          prev.remoteEtag,
+          st.mtimeMs,
+        );
+        restamped++;
+      }
+      if (restamped > 0) {
+        writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, journal);
+        out(
+          `==> Reconciled sync-journal baseline for ${restamped} re-laid file(s)\n`,
+        );
+      }
+    }
+  } catch (e) {
+    out(
+      `    (sync-journal reconcile skipped: ${
+        e instanceof Error ? e.message : String(e)
+      })\n`,
+    );
+  }
+
   // --- Stamp sync-point provenance into core/core.yaml ---
   if (cfg.narrowPaths.length === 0 && isFileFollow(coreYaml)) {
     const nowUtc = utcStamp(new Date(), "colon");

package/src/cli/rescue-journal-reconcile.test.ts

@@ -0,0 +1,156 @@
+/**
+ * Regression for the rescue → sync-journal stale-baseline bug.
+ *
+ * Root cause: the rescue overlay rewrites scaffold files from upstream but
+ * never re-stamps the personal-vault sync journal. The journal keeps the
+ * PRE-rescue hash, so the next sync reads localHash != journal.hash ("local
+ * changed") and — when the vault also moved — mints a false `.conflict-*`
+ * mirror. `runRescue` now re-stamps the journal baseline for the files the
+ * overlay actually re-laid (scoped to rsync `-i` itemize output, so a user's
+ * pending edit is never touched).
+ *
+ * This test seeds a journal whose entry for an overlaid file carries the OLD
+ * hash, runs a real (non-dry-run) rescue, and asserts the entry was re-stamped
+ * to the freshly laid-down content — i.e. `localChanged` would now be false.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { execFileSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { runRescue } from "./rescue-core.js";
+import {
+  hashFile,
+  readJournal,
+  writeJournal,
+  PERSONAL_VAULT_JOURNAL_SLUG,
+} from "../journal.js";
+
+function hasGit(): boolean {
+  try {
+    execFileSync("git", ["--version"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+const gitAvailable = hasGit();
+
+function runRescueCapture(argv: string[], env: NodeJS.ProcessEnv) {
+  let stdout = "";
+  const origOut = process.stdout.write.bind(process.stdout);
+  const origErr = process.stderr.write.bind(process.stderr);
+  process.stdout.write = ((c: unknown) => ((stdout += String(c)), true)) as typeof process.stdout.write;
+  process.stderr.write = (() => true) as typeof process.stderr.write;
+  let status: number;
+  try {
+    status = runRescue(argv, { env }).status;
+  } finally {
+    process.stdout.write = origOut;
+    process.stderr.write = origErr;
+  }
+  return { status, stdout };
+}
+
+describe.skipIf(!gitAvailable)("rescue re-stamps sync-journal baseline for re-laid files", () => {
+  let workDir: string, upstream: string, hqRoot: string, stateDir: string, floorSha: string;
+  let env: NodeJS.ProcessEnv;
+  let savedStateDir: string | undefined;
+
+  const git = (cwd: string, ...args: string[]) =>
+    execFileSync("git", args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@t", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@t" },
+    }).toString().trim();
+
+  beforeAll(() => {
+    workDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-journal-"));
+
+    // upstream: floor (doc.md=v1), then HEAD advances doc.md -> v2.
+    upstream = path.join(workDir, "upstream");
+    fs.mkdirSync(path.join(upstream, "core"), { recursive: true });
+    git(workDir, "init", "-b", "main", "upstream");
+    fs.writeFileSync(path.join(upstream, "core/doc.md"), "v1\n");
+    git(upstream, "add", "-A");
+    git(upstream, "commit", "-m", "floor");
+    floorSha = git(upstream, "rev-parse", "HEAD");
+    fs.writeFileSync(path.join(upstream, "core/doc.md"), "v2\n");
+    git(upstream, "add", "-A");
+    git(upstream, "commit", "-m", "head");
+
+    // local HQ root: doc.md == floor (v1) -> rescue overlays it to HEAD (v2).
+    hqRoot = path.join(workDir, "hq");
+    fs.mkdirSync(path.join(hqRoot, "core"), { recursive: true });
+    fs.mkdirSync(path.join(hqRoot, "personal"), { recursive: true });
+    fs.mkdirSync(path.join(hqRoot, "companies"), { recursive: true });
+    const docAbs = path.join(hqRoot, "core/doc.md");
+    fs.writeFileSync(docAbs, "v1\n");
+
+    // state dir + seeded personal-vault journal carrying the STALE (v1) hash.
+    stateDir = path.join(workDir, "state");
+    fs.mkdirSync(stateDir, { recursive: true });
+    savedStateDir = process.env.HQ_STATE_DIR;
+    process.env.HQ_STATE_DIR = stateDir; // getStateDir() reads process.env
+    const staleHash = hashFile(docAbs); // hash of v1
+    writeJournal(PERSONAL_VAULT_JOURNAL_SLUG, {
+      version: "2",
+      lastSync: new Date(0).toISOString(),
+      files: {
+        "core/doc.md": {
+          hash: staleHash,
+          size: fs.statSync(docAbs).size,
+          syncedAt: new Date(0).toISOString(),
+          direction: "down",
+          remoteEtag: "seed-etag",
+          mtimeMs: fs.statSync(docAbs).mtimeMs,
+        },
+      },
+      pulls: [],
+    } as never);
+
+    // git shim: redirect `git clone <github-url>` to the local fixture.
+    const realGit = execFileSync("bash", ["-lc", "command -v git"]).toString().trim() || "/usr/bin/git";
+    const shimDir = path.join(workDir, "shim");
+    fs.mkdirSync(shimDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(shimDir, "git"),
+      `#!/usr/bin/env bash
+if [ "$1" = "clone" ]; then
+  args=(); for a in "$@"; do case "$a" in https://github.com/*) a=${JSON.stringify(upstream)} ;; esac; args+=("$a"); done
+  exec ${JSON.stringify(realGit)} "\${args[@]}"
+fi
+exec ${JSON.stringify(realGit)} "$@"
+`,
+      { mode: 0o755 },
+    );
+    env = { ...process.env, PATH: `${shimDir}:${process.env.PATH ?? ""}`, HQ_STATE_DIR: stateDir };
+  });
+
+  afterAll(() => {
+    if (savedStateDir === undefined) delete process.env.HQ_STATE_DIR;
+    else process.env.HQ_STATE_DIR = savedStateDir;
+    if (workDir) fs.rmSync(workDir, { recursive: true, force: true });
+  });
+
+  it("re-stamps the journal so the overlaid file is no longer seen as locally changed", () => {
+    const docAbs = path.join(hqRoot, "core/doc.md");
+    const staleHash = readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files["core/doc.md"].hash;
+
+    const r = runRescueCapture(
+      ["--hq-root", hqRoot, "--source", "test/repo", "--ref", "main", "--floor-sha", floorSha, "--yes", "--no-backup"],
+      env,
+    );
+    expect(r.status, r.stdout).toBe(0);
+
+    // overlay actually re-laid the file (v1 -> v2)
+    expect(fs.readFileSync(docAbs, "utf-8")).toBe("v2\n");
+
+    // journal entry was re-stamped to the NEW content's hash
+    const entry = readJournal(PERSONAL_VAULT_JOURNAL_SLUG).files["core/doc.md"];
+    expect(entry.hash).toBe(hashFile(docAbs)); // == hash(v2): localChanged would be FALSE
+    expect(entry.hash).not.toBe(staleHash); // proves it changed from the stale v1 baseline
+    expect(entry.remoteEtag).toBe("seed-etag"); // remote side untouched -> clean push/converge, not conflict
+    expect(r.stdout).toContain("Reconciled sync-journal baseline");
+  });
+});

package/src/cli/share.test.ts

@@ -39,7 +39,7 @@ vi.mock("readline", () => ({
 
 import * as readline from "readline";
 import { share, _testing as shareTesting } from "./share.js";
-import { deleteRemoteFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
+import { deleteRemoteFile, downloadFile, headRemoteFile, uploadFile, uploadSymlink } from "../s3.js";
 import type { EntityContext } from "../types.js";
 
 const mockConfig: VaultServiceConfig = {
@@ -482,6 +482,110 @@ describe("share", () => {
     expect(result.filesUploaded).toBe(0);
   });
 
+  it("fresh-install multipart collision: byte-identical remote is NOT a conflict (reconciled, no mirror)", async () => {
+    // Root cause of "HQ installs with conflicts": on a first push the
+    // journal is empty, so the fresh-collision branch decides conflict-vs-
+    // identical from the remote etag alone. A multipart remote etag
+    // (\`<md5>-<partCount>\`) is NOT a content hash, so the OLD code assumed
+    // a collision for ANY multipart object — minting a false conflict for
+    // the most common fresh-install case: re-pushing a byte-identical
+    // core/ scaffold file whose remote copy happened to be multipart-
+    // uploaded. The fix fetches the remote bytes once and compares content
+    // hashes; identical content reconciles (journal re-stamped, PUT
+    // skipped) instead of producing a conflict + \`.conflict-*\` mirror.
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const testFile = path.join(companyRoot, "core-scaffold.md");
+    const scaffoldBytes = "identical-scaffold-content-shipped-to-every-user";
+    fs.writeFileSync(testFile, scaffoldBytes);
+
+    // Remote exists with a MULTIPART etag and NO journal entry (first push).
+    vi.mocked(headRemoteFile).mockResolvedValueOnce({
+      lastModified: new Date(),
+      etag: '"d41d8cd98f00b204e9800998ecf8427e-2"',
+      size: scaffoldBytes.length,
+    });
+    // The convergence probe downloads the remote bytes — which are
+    // byte-identical to local — so the content hashes match.
+    vi.mocked(downloadFile).mockImplementationOnce(async (_ctx, _key, dest) => {
+      fs.writeFileSync(dest as string, scaffoldBytes);
+      return undefined as never;
+    });
+
+    const events: unknown[] = [];
+    const result = await share({
+      paths: [testFile],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onConflict: "keep",
+      onEvent: (e) => events.push(e),
+    });
+
+    // No conflict, no upload (bytes already there), counted as a skip.
+    expect(result.conflictPaths).toEqual([]);
+    expect(result.filesUploaded).toBe(0);
+    expect(result.filesSkipped).toBe(1);
+    const conflicts = events.filter(
+      (e) => typeof e === "object" && e !== null && (e as { type?: string }).type === "conflict",
+    );
+    expect(conflicts).toHaveLength(0);
+    const reconciled = events.filter(
+      (e) => typeof e === "object" && e !== null && (e as { type?: string }).type === "reconciled",
+    );
+    expect(reconciled).toHaveLength(1);
+    expect(reconciled[0]).toMatchObject({ path: "core-scaffold.md", direction: "push" });
+    // No conflict-mirror litter on disk.
+    const litter = fs
+      .readdirSync(companyRoot)
+      .filter((n) => n.includes(".conflict-"));
+    expect(litter).toEqual([]);
+    // Local file untouched.
+    expect(fs.readFileSync(testFile, "utf-8")).toBe(scaffoldBytes);
+  });
+
+  it("fresh-install multipart collision: genuinely different remote IS a conflict (Bug #7 preserved)", async () => {
+    // The convergence probe must not weaken Bug #7 protection: a multipart
+    // remote whose CONTENT actually differs from local is still a real
+    // fresh collision and must surface a conflict (so a peer's content
+    // isn't silently clobbered).
+    const companyRoot = path.join(tmpDir, "companies", "acme");
+    fs.mkdirSync(companyRoot, { recursive: true });
+    const testFile = path.join(companyRoot, "diverged-multipart.md");
+    fs.writeFileSync(testFile, "my-local-version");
+
+    vi.mocked(headRemoteFile).mockResolvedValueOnce({
+      lastModified: new Date(),
+      etag: '"0123456789abcdef0123456789abcdef-3"',
+      size: 32,
+    });
+    // The probe (and the subsequent keep-mirror download) return DIFFERENT
+    // remote bytes.
+    vi.mocked(downloadFile).mockImplementation(async (_ctx, _key, dest) => {
+      fs.writeFileSync(dest as string, "a-peers-different-version");
+      return undefined as never;
+    });
+
+    const events: unknown[] = [];
+    const result = await share({
+      paths: [testFile],
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onConflict: "keep",
+      onEvent: (e) => events.push(e),
+    });
+
+    expect(result.conflictPaths).toEqual(["diverged-multipart.md"]);
+    expect(result.filesUploaded).toBe(0);
+    const conflicts = events.filter(
+      (e) => typeof e === "object" && e !== null && (e as { type?: string }).type === "conflict",
+    );
+    expect(conflicts).toHaveLength(1);
+    // Local file untouched under keep.
+    expect(fs.readFileSync(testFile, "utf-8")).toBe("my-local-version");
+  });
+
   it("uploads (no conflict) when only the local side changed since last sync", async () => {
     // Regression for hq-cloud#<conflict-detection>: a local edit to a file
     // that exists on S3 used to trigger a push conflict because the

package/src/cli/share.ts

@@ -6,6 +6,7 @@
  */
 
 import * as fs from "fs";
+import * as os from "os";
 import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
@@ -45,6 +46,59 @@ import {
 } from "../lib/conflict-file.js";
 import { appendConflictEntry } from "../lib/conflict-index.js";
 
+/**
+ * Push-side fresh-collision convergence probe.
+ *
+ * For a first push (no journal entry) where the remote object already
+ * exists AND was multipart-uploaded, the remote etag (`<md5>-<partCount>`)
+ * is not a content hash, so we cannot tell "byte-identical" from "genuine
+ * divergence" without looking at the bytes. Fetch the remote object once to
+ * a throwaway temp file, hash it the same symlink-aware way the planner
+ * hashed local, and report whether the two contents DIFFER.
+ *
+ * Returns `true` on a genuine difference (a real fresh collision) and
+ * `false` when the bytes are identical. Fails safe to `true` on any
+ * fetch/hash error — a false positive merely prompts the operator, while a
+ * false negative could silently clobber a peer's content. Mirrors the
+ * pull-side convergence guard in `sync.ts`.
+ *
+ * The temp file lives in the OS temp dir (never under hqRoot) so it can
+ * never round-trip to S3, and is removed in a `finally` regardless of
+ * outcome. The name is derived from the relative path so concurrent probes
+ * for distinct files never collide on disk.
+ */
+async function remoteContentDiffers(
+  ctx: EntityContext,
+  relativePath: string,
+  localHash: string,
+  hqRoot: string,
+): Promise<boolean> {
+  const probeKey = crypto
+    .createHash("sha256")
+    .update(relativePath)
+    .digest("hex")
+    .slice(0, 16);
+  const probePath = path.join(
+    os.tmpdir(),
+    `hq-conflict-probe-${readShortMachineId(hqRoot)}-${probeKey}.tmp`,
+  );
+  try {
+    await downloadFile(ctx, relativePath, probePath);
+    const remoteHash = fs.lstatSync(probePath).isSymbolicLink()
+      ? hashSymlinkTarget(fs.readlinkSync(probePath))
+      : hashFile(probePath);
+    return remoteHash !== localHash;
+  } catch {
+    return true;
+  } finally {
+    try {
+      fs.rmSync(probePath, { force: true });
+    } catch {
+      /* best-effort cleanup; a stray temp probe is harmless */
+    }
+  }
+}
+
 /**
  * Local-only ephemeral artifacts: conflict-mirror files written by the pull
  * leg whenever a 3-way merge keeps local AND wants to preserve the remote
@@ -940,12 +994,10 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
       const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
 
       let isFreshCollision = false;
+      let multipartConverged = false;
       if (!journalEntry && item.kind === "file") {
         // Single-part S3 PUT etag is MD5 of the body. Multipart uploads
-        // produce \`<md5>-<partCount>\`; we treat any non-single-part etag
-        // as ambiguous and DO classify as a conflict (safer for the
-        // first-time path — false positives prompt the operator, false
-        // negatives lose data). Symlink records (\`kind: "symlink"\`)
+        // produce \`<md5>-<partCount>\`. Symlink records (\`kind: "symlink"\`)
         // skip the check entirely — the wire body shape (\`hq-symlink:\`
         // prefix + target) isn't a pure byte mirror and would mis-
         // classify; symlink overwrites are rare and an audit pass after
@@ -962,13 +1014,55 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
           // path which is idempotent (S3 will overwrite with identical
           // content + carry our metadata). Cheap, no behavior change.
         } else {
-          // Multipart object pre-exists with unknown body shape — assume
-          // collision rather than risk a silent overwrite. The operator
-          // can resolve via the standard conflict prompt.
-          isFreshCollision = true;
+          // Multipart remote etag is \`<md5>-<partCount>\`, NOT a usable
+          // content hash, so — unlike the single-part branch — we cannot
+          // decide collision-vs-identical from the etag alone. The old
+          // behavior assumed a collision here, which minted a FALSE
+          // conflict for the most common fresh-install case: re-pushing a
+          // byte-identical \`core/\` scaffold file whose remote copy happened
+          // to be multipart-uploaded. Every fresh install that hit an
+          // already-populated bucket therefore came up "with conflicts".
+          //
+          // Instead, fetch the remote bytes once and compare content
+          // hashes directly — the same convergence guard the pull side
+          // uses (sync.ts). Identical content is NOT a conflict. On any
+          // fetch/hash failure we fail safe to "conflict" (false positives
+          // prompt the operator; false negatives risk clobbering a peer).
+          const remoteDiffers = await remoteContentDiffers(
+            ctx,
+            relativePath,
+            localHash,
+            hqRoot,
+          );
+          if (remoteDiffers) {
+            isFreshCollision = true;
+          } else {
+            // Byte-identical multipart object already present. Seed the
+            // journal baseline from the remote so the next sync sees no
+            // change on either side, and skip the redundant PUT —
+            // re-uploading would needlessly rewrite remote and churn its
+            // etag from multipart to single-part.
+            multipartConverged = true;
+          }
         }
       }
 
+      if (multipartConverged) {
+        const lstat = fs.lstatSync(absolutePath);
+        updateEntry(
+          journal,
+          relativePath,
+          localHash,
+          lstat.size,
+          "up",
+          remoteMeta.etag,
+          lstat.mtimeMs,
+        );
+        emit({ type: "reconciled", path: relativePath, direction: "push" });
+        filesSkipped++;
+        return;
+      }
+
       if ((localChanged && remoteChanged) || isFreshCollision) {
         conflictPaths.push(relativePath);
 

package/src/cli/sync.ts

@@ -149,7 +149,7 @@ export type SyncProgressEvent =
        */
       type: "reconciled";
       path: string;
-      direction: "pull";
+      direction: "pull" | "push";
     }
   | {
       type: "new-files";