@indigoai-us/hq-cloud 5.4.6 → 5.7.1

package/src/cli/sync.ts

@@ -7,13 +7,20 @@
 
 import * as fs from "fs";
 import * as path from "path";
-import type { VaultServiceConfig } from "../types.js";
+import type { VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { downloadFile, listRemoteFiles } from "../s3.js";
+import type { RemoteFile } from "../s3.js";
 import { readJournal, writeJournal, hashFile, updateEntry, getEntry, normalizeEtag } from "../journal.js";
 import { createIgnoreFilter } from "../ignore.js";
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
+import {
+  buildConflictId,
+  buildConflictPath,
+  readShortMachineId,
+} from "../lib/conflict-file.js";
+import { appendConflictEntry } from "../lib/conflict-index.js";
 
 /**
  * Per-file events emitted by `sync()` as it progresses.
@@ -25,8 +32,31 @@ import type { ConflictStrategy, ConflictResolution } from "./conflict.js";
  *
  * The human CLI (`hq sync`) leaves `onEvent` undefined and falls through to
  * `defaultConsoleLogger` below, which preserves the existing tty output.
+ *
+ * A single `plan` event is emitted once at the start of every run, before
+ * any `progress`/`conflict`/`error` events. It carries the totals derived
+ * from a Stage-1 classification pass so consumers can render an accurate
+ * progress denominator before transfers begin (the menubar's "Preparing
+ * sync…" pre-pass becomes obsolete once the runner forwards this).
  */
 export type SyncProgressEvent =
+  | {
+      type: "plan";
+      /** Files this run intends to download (pull-only; 0 from share). */
+      filesToDownload: number;
+      bytesToDownload: number;
+      /** Files this run intends to upload (push-only; 0 from sync). */
+      filesToUpload: number;
+      bytesToUpload: number;
+      /** Files classified as no-op (ignored, unchanged, local-only on pull). */
+      filesToSkip: number;
+      /**
+       * Files known up-front to be conflicts. Pull-side fills this from the
+       * 3-way merge against the journal; push-side leaves it 0 because
+       * conflict detection requires a remote HEAD that runs in Stage 2.
+       */
+      filesToConflict: number;
+    }
   | { type: "progress"; path: string; bytes: number; message?: string }
   | { type: "error"; path: string; message: string }
   | {
@@ -123,90 +153,130 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
   // List all remote files (IAM session policy filters at the AWS layer)
   const remoteFiles = await listRemoteFiles(ctx);
 
-  for (const remoteFile of remoteFiles) {
-    const localPath = path.join(companyRoot, remoteFile.key);
+  // Stage 1: classify every remote file against the journal + local disk.
+  // Hashing happens here (not in the transfer loop) so the plan event below
+  // carries an accurate denominator before any progress events fire.
+  const plan = computePullPlan(
+    remoteFiles,
+    journal,
+    companyRoot,
+    shouldSync,
+    options.personalMode === true,
+  );
 
-    if (options.personalMode === true && remoteFile.key.startsWith("companies/")) {
-      filesSkipped++;
-      continue;
-    }
+  emit({
+    type: "plan",
+    filesToDownload: plan.filesToDownload,
+    bytesToDownload: plan.bytesToDownload,
+    // sync() is pull-only; push counts are sourced from share()'s plan event.
+    filesToUpload: 0,
+    bytesToUpload: 0,
+    filesToSkip: plan.filesToSkip,
+    filesToConflict: plan.filesToConflict,
+  });
 
-    // Apply ignore rules
-    if (!shouldSync(localPath)) {
+  // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
+  // inline loop; the only structural change is that classification has
+  // already happened (so `localHash` is reused instead of re-hashing).
+  for (const item of plan.items) {
+    if (
+      item.action === "skip-ignored" ||
+      item.action === "skip-personal-mode" ||
+      item.action === "skip-unchanged" ||
+      item.action === "skip-local-only"
+    ) {
       filesSkipped++;
       continue;
     }
 
-    // Auto-refresh context if credentials expiring
+    const { remoteFile, localPath } = item;
+
+    // Auto-refresh context if credentials expiring (kept in execute phase
+    // because Stage 1 is fast — no need to refresh just to classify).
     if (isExpiringSoon(ctx.expiresAt)) {
       ctx = await refreshEntityContext(companyRef, vaultConfig);
     }
 
-    // Check for local conflict
-    const journalEntry = getEntry(journal, remoteFile.key);
+    if (item.action === "conflict") {
+      conflicts++;
+      conflictPaths.push(remoteFile.key);
 
-    if (fs.existsSync(localPath)) {
-      const localHash = hashFile(localPath);
-      const localChanged = !!journalEntry && journalEntry.hash !== localHash;
-      const remoteChanged = !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
-
-      // A real conflict requires BOTH sides to have moved since the last
-      // sync. If only local changed, push will handle it; pulling here would
-      // clobber the local edit. If only remote changed, fall through to
-      // download. If neither moved, skip.
-      if (localChanged && remoteChanged) {
-        conflicts++;
-        conflictPaths.push(remoteFile.key);
-
-        const resolution = await resolveConflict(
-          {
-            path: remoteFile.key,
-            localHash,
-            remoteModified: remoteFile.lastModified,
-            localModified: fs.statSync(localPath).mtime,
-            direction: "pull",
-          },
-          onConflict,
-        );
-
-        emit({
-          type: "conflict",
+      const resolution = await resolveConflict(
+        {
           path: remoteFile.key,
+          localHash: item.localHash,
+          remoteModified: remoteFile.lastModified,
+          localModified: fs.statSync(localPath).mtime,
           direction: "pull",
-          resolution,
-        });
+        },
+        onConflict,
+      );
 
-        if (resolution === "abort") {
-          writeJournal(journalSlug, journal);
-          return {
-            filesDownloaded,
-            bytesDownloaded,
-            filesSkipped,
-            conflicts,
-            conflictPaths,
-            aborted: true,
-          };
-        }
-        if (resolution === "keep" || resolution === "skip") {
-          filesSkipped++;
-          continue;
+      emit({
+        type: "conflict",
+        path: remoteFile.key,
+        direction: "pull",
+        resolution,
+      });
+
+      // Write `<original>.conflict-<ts>-<machine>.<ext>` mirror + append to
+      // `<hqRoot>/.hq-conflicts/index.json` so the user can later run
+      // `/resolve-conflicts` to walk pending conflicts. Skipped for "abort"
+      // (user gave up) and "overwrite" (cloud bytes are about to replace
+      // local — mirror would be redundant). Best-effort: failure here only
+      // emits an error, doesn't break the sync.
+      if (resolution !== "abort" && resolution !== "overwrite") {
+        try {
+          const detectedAt = new Date().toISOString();
+          const machineId = readShortMachineId();
+          const originalRelative = path.relative(hqRoot, localPath);
+          const conflictRelative = buildConflictPath(
+            originalRelative,
+            detectedAt,
+            machineId,
+          );
+          const conflictAbs = path.join(hqRoot, conflictRelative);
+          await downloadFile(ctx, remoteFile.key, conflictAbs);
+          appendConflictEntry(hqRoot, {
+            id: buildConflictId(originalRelative, detectedAt),
+            originalPath: originalRelative,
+            conflictPath: conflictRelative,
+            detectedAt,
+            side: "pull",
+            machineId,
+            localHash: item.localHash,
+            remoteHash: remoteFile.etag ? normalizeEtag(remoteFile.etag) : "",
+          });
+        } catch (mirrorErr) {
+          emit({
+            type: "error",
+            path: remoteFile.key,
+            message: `conflict mirror write failed: ${
+              mirrorErr instanceof Error ? mirrorErr.message : String(mirrorErr)
+            }`,
+          });
         }
-        // "overwrite" falls through to download
-      } else if (journalEntry && localChanged && !remoteChanged) {
-        // Local-only edit: leave it for the push phase to upload. Pulling
-        // would silently overwrite the user's work.
-        filesSkipped++;
-        continue;
-      } else if (journalEntry && !localChanged && !remoteChanged) {
-        // Neither side moved — nothing to do.
+      }
+
+      if (resolution === "abort") {
+        writeJournal(journalSlug, journal);
+        return {
+          filesDownloaded,
+          bytesDownloaded,
+          filesSkipped,
+          conflicts,
+          conflictPaths,
+          aborted: true,
+        };
+      }
+      if (resolution === "keep" || resolution === "skip") {
         filesSkipped++;
         continue;
       }
-      // Otherwise (no journal entry, or remote-only changed) fall through
-      // to download.
+      // "overwrite" falls through to download
     }
 
-    // Download
+    // Download (action === "download" or conflict resolved to "overwrite")
     try {
       await downloadFile(ctx, remoteFile.key, localPath);
 
@@ -216,8 +286,10 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
       // drift independently of mtime drift.
       updateEntry(journal, remoteFile.key, hash, stat.size, "down", remoteFile.etag);
 
-      // Attach message from journal entry if present
-      const remoteJournalMessage = (journalEntry as { message?: string } | undefined)?.message;
+      // Attach message from the prior journal entry if present (set by a
+      // previous `share` operation that included a --message).
+      const priorEntry = getEntry(journal, remoteFile.key);
+      const remoteJournalMessage = (priorEntry as { message?: string } | undefined)?.message;
       emit({
         type: "progress",
         path: remoteFile.key,
@@ -287,6 +359,124 @@ function hasRemoteChanged(
   return remote.lastModified.getTime() > syncedAt;
 }
 
+/**
+ * Stage-1 classification for a single remote object. Each remote file falls
+ * into exactly one bucket; the executor in `sync()` switches on `action` to
+ * decide what to do. `localHash` is carried on `conflict` items so the
+ * executor can hand it to `resolveConflict` without re-hashing.
+ */
+type PullPlanItem =
+  | { action: "download"; remoteFile: RemoteFile; localPath: string }
+  | { action: "skip-ignored"; remoteFile: RemoteFile; localPath: string }
+  | { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
+  | { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
+  | { action: "skip-local-only"; remoteFile: RemoteFile; localPath: string }
+  | {
+      action: "conflict";
+      remoteFile: RemoteFile;
+      localPath: string;
+      localHash: string;
+    };
+
+interface PullPlan {
+  items: PullPlanItem[];
+  filesToDownload: number;
+  bytesToDownload: number;
+  filesToSkip: number;
+  filesToConflict: number;
+}
+
+/**
+ * Stage-1 planning pass: classify every remote file into download / skip /
+ * conflict buckets without performing any S3 transfers. Local hashes are
+ * computed here (not in the transfer loop) so the totals returned reflect
+ * the real outcome of the upcoming Stage-2 execution rather than an
+ * upper-bound guess.
+ *
+ * Pure function: no S3 calls, no journal writes, no event emission. The
+ * caller (`sync()`) is responsible for emitting the resulting plan event
+ * before iterating `items`.
+ */
+function computePullPlan(
+  remoteFiles: RemoteFile[],
+  journal: SyncJournal,
+  companyRoot: string,
+  shouldSync: (filePath: string) => boolean,
+  personalMode: boolean,
+): PullPlan {
+  const items: PullPlanItem[] = [];
+
+  for (const remoteFile of remoteFiles) {
+    const localPath = path.join(companyRoot, remoteFile.key);
+
+    if (personalMode && remoteFile.key.startsWith("companies/")) {
+      items.push({ action: "skip-personal-mode", remoteFile, localPath });
+      continue;
+    }
+
+    if (!shouldSync(localPath)) {
+      items.push({ action: "skip-ignored", remoteFile, localPath });
+      continue;
+    }
+
+    const journalEntry = getEntry(journal, remoteFile.key);
+
+    if (fs.existsSync(localPath)) {
+      const localHash = hashFile(localPath);
+      const localChanged = !!journalEntry && journalEntry.hash !== localHash;
+      const remoteChanged =
+        !!journalEntry && hasRemoteChanged(remoteFile, journalEntry);
+
+      // Mirror the original 3-way merge from the inline loop. Tested by
+      // `does NOT flag a pull conflict when only local changed since last
+      // sync` and `detects conflicts with local changes…`.
+      if (localChanged && remoteChanged) {
+        items.push({
+          action: "conflict",
+          remoteFile,
+          localPath,
+          localHash,
+        });
+        continue;
+      }
+      if (journalEntry && localChanged && !remoteChanged) {
+        items.push({ action: "skip-local-only", remoteFile, localPath });
+        continue;
+      }
+      if (journalEntry && !localChanged && !remoteChanged) {
+        items.push({ action: "skip-unchanged", remoteFile, localPath });
+        continue;
+      }
+      // No journal entry, or remote-only changed → fall through to download.
+    }
+
+    items.push({ action: "download", remoteFile, localPath });
+  }
+
+  let filesToDownload = 0;
+  let bytesToDownload = 0;
+  let filesToSkip = 0;
+  let filesToConflict = 0;
+  for (const item of items) {
+    if (item.action === "download") {
+      filesToDownload++;
+      bytesToDownload += item.remoteFile.size;
+    } else if (item.action === "conflict") {
+      filesToConflict++;
+    } else {
+      filesToSkip++;
+    }
+  }
+
+  return {
+    items,
+    filesToDownload,
+    bytesToDownload,
+    filesToSkip,
+    filesToConflict,
+  };
+}
+
 /**
  * Check if an error is an S3 access denied (expected for filtered guests).
  */
@@ -303,7 +493,26 @@ function isAccessDenied(err: unknown): boolean {
  * without an `onEvent` see no behavioral change.
  */
 function defaultConsoleLogger(event: SyncProgressEvent): void {
-  if (event.type === "progress") {
+  if (event.type === "plan") {
+    // Terse single line so humans see what's about to happen without
+    // drowning the per-file output that follows. Skip when there's
+    // nothing to do — no signal, no noise.
+    const movement = event.filesToDownload + event.filesToUpload + event.filesToConflict;
+    if (movement > 0) {
+      const parts: string[] = [];
+      if (event.filesToDownload > 0) {
+        parts.push(`${event.filesToDownload} to download (${event.bytesToDownload} bytes)`);
+      }
+      if (event.filesToUpload > 0) {
+        parts.push(`${event.filesToUpload} to upload (${event.bytesToUpload} bytes)`);
+      }
+      if (event.filesToConflict > 0) {
+        parts.push(`${event.filesToConflict} conflict(s)`);
+      }
+      parts.push(`${event.filesToSkip} unchanged`);
+      console.log(`Plan: ${parts.join(", ")}`);
+    }
+  } else if (event.type === "progress") {
     if (event.message) {
       console.log(`  ✓ ${event.path} — "${event.message}"`);
     } else {

package/src/lib/conflict-file.ts

@@ -0,0 +1,101 @@
+/**
+ * Conflict file naming + writing.
+ *
+ * When share/sync detects divergence, the cloud's version of the file is
+ * written next to the original with a name encoding the timestamp and the
+ * machine that detected the conflict. Lets multiple machines independently
+ * surface their own conflicts without name collisions, and lets the user
+ * (or the `/resolve-conflicts` HQ skill) see local + cloud side-by-side
+ * in their file browser.
+ */
+
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+
+/**
+ * Path to `~/.hq/menubar.json`. Evaluated lazily at call time (not module
+ * load) so that tests overriding `HOME` after import — and any future code
+ * that changes the user's effective home dir at runtime — see the right
+ * file. Going through `os.homedir()` rather than `process.env.HOME` keeps
+ * the Windows USERPROFILE fallback intact.
+ */
+function menubarJsonPath(): string {
+  return path.join(os.homedir(), ".hq", "menubar.json");
+}
+
+/**
+ * Read the short machine ID (first 6 chars) from `~/.hq/menubar.json`.
+ * Falls back to "unknown" if the file is missing/unreadable — conflict
+ * files should still be written even when machine identity is unclear.
+ */
+export function readShortMachineId(): string {
+  try {
+    const raw = fs.readFileSync(menubarJsonPath(), "utf-8");
+    const parsed = JSON.parse(raw);
+    const id = typeof parsed.machineId === "string" ? parsed.machineId : "";
+    return id.slice(0, 6) || "unknown";
+  } catch {
+    return "unknown";
+  }
+}
+
+/**
+ * Build the conflict file path for an original. ISO uses `-` instead of
+ * `:` so the result is filesystem-safe on every OS, and the original
+ * extension is preserved at the end so editors syntax-highlight correctly.
+ *
+ *   knowledge/notes.md, 2026-04-27T22:05:14Z, abc123
+ *     → knowledge/notes.md.conflict-2026-04-27T22-05-14Z-abc123.md
+ *
+ *   projects/foo/prd.json, ..., abc123
+ *     → projects/foo/prd.json.conflict-...-abc123.json
+ *
+ * Files without an extension get the `.conflict-...` suffix appended verbatim.
+ */
+export function buildConflictPath(
+  originalRelative: string,
+  detectedAt: string,
+  shortMachineId: string,
+): string {
+  const safeTs = detectedAt.replace(/:/g, "-").replace(/\.\d+/, "");
+  const ext = path.extname(originalRelative); // ".md" or "" if none
+  // The full original path is preserved (extension and all) so users can
+  // visually pair `notes.md` with `notes.md.conflict-…md` in their file
+  // browser. The trailing `<ext>` after the timestamp keeps the file
+  // syntax-highlighted in editors that key off the final extension.
+  const suffix = `.conflict-${safeTs}-${shortMachineId}${ext}`;
+  return `${originalRelative}${suffix}`;
+}
+
+/**
+ * Write the cloud-side bytes to the conflict path. Creates parent dirs as
+ * needed (the conflict file always lives next to the original, so the
+ * parent already exists in the steady-state — but defense-in-depth).
+ */
+export function writeConflictFile(
+  hqRoot: string,
+  conflictRelative: string,
+  contents: Buffer,
+): void {
+  const abs = path.join(hqRoot, conflictRelative);
+  fs.mkdirSync(path.dirname(abs), { recursive: true });
+  fs.writeFileSync(abs, contents);
+}
+
+/**
+ * Stable conflict ID — used to dedupe re-detections of the same conflict.
+ * Re-running sync after a conflict but before the user has resolved should
+ * NOT pile up duplicate entries. The id is derived from the original path
+ * and the detection timestamp; if the same original conflicts twice with
+ * the user resolving in between, that's a new id (different timestamp),
+ * which is correct.
+ */
+export function buildConflictId(
+  originalRelative: string,
+  detectedAt: string,
+): string {
+  const safeTs = detectedAt.replace(/:/g, "-").replace(/\.\d+/, "");
+  const safePath = originalRelative.replace(/[\/\\.]/g, "-");
+  return `${safePath}-${safeTs}`;
+}

package/src/lib/conflict-index.ts

@@ -0,0 +1,127 @@
+/**
+ * Conflict index — durable record of pending divergences awaiting resolution.
+ *
+ * Lives at `<hq_root>/.hq-conflicts/index.json` (inside HQ content, NOT in
+ * `~/.hq/`). Two reasons it sits in HQ content rather than in the state dir:
+ *   1. The `/resolve-conflicts` HQ skill discovers it relative to the user's
+ *      HQ folder — that's the user's mental anchor for "where my files are."
+ *   2. The conflict-side files themselves live in HQ content, so the index
+ *      and the files it references stay co-located. If the user moves HQ,
+ *      the index moves with it.
+ *
+ * Excluded from cross-machine sync via `.hqignore` — each machine resolves
+ * its own queue. We never propagate conflict files (they'd just create more
+ * conflicts on the other side).
+ *
+ * Writes are atomic (tmp + rename). The resolution skill mutates this file
+ * mid-walk; a torn write would corrupt the only record of pending conflicts
+ * and could lose track of files we'd written to disk. Higher stakes than the
+ * journal, which the next sync can rebuild.
+ */
+
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as path from "path";
+import type { ConflictIndex, ConflictIndexEntry } from "../types.js";
+
+const CONFLICTS_DIR = ".hq-conflicts";
+const INDEX_FILENAME = "index.json";
+
+/**
+ * Absolute path to the conflict index for a given HQ root.
+ */
+export function getConflictIndexPath(hqRoot: string): string {
+  return path.join(hqRoot, CONFLICTS_DIR, INDEX_FILENAME);
+}
+
+/**
+ * Read the conflict index. Returns an empty index if the file doesn't exist
+ * yet (first-conflict-ever case).
+ *
+ * Throws on corrupt JSON — we deliberately don't auto-repair, since the only
+ * record of pending conflicts is too important to silently overwrite. The
+ * `/resolve-conflicts` skill surfaces this case to the user with a manual
+ * inspection prompt.
+ */
+export function readConflictIndex(hqRoot: string): ConflictIndex {
+  const indexPath = getConflictIndexPath(hqRoot);
+  if (!fs.existsSync(indexPath)) {
+    return { version: 1, conflicts: [] };
+  }
+  const raw = fs.readFileSync(indexPath, "utf-8");
+  const parsed = JSON.parse(raw) as ConflictIndex;
+  // Defensive: an empty file or wrong-shape JSON shouldn't crash callers.
+  if (!parsed || typeof parsed !== "object" || !Array.isArray(parsed.conflicts)) {
+    return { version: 1, conflicts: [] };
+  }
+  return parsed;
+}
+
+/**
+ * Atomically write the conflict index. Writes to `<index>.tmp.<random>` then
+ * renames into place — `rename(2)` is atomic on POSIX, so a crash mid-write
+ * leaves either the old file or the new one, never a half-written one.
+ *
+ * Always sorts conflicts by `detectedAt` ascending before writing — keeps
+ * the file diff-friendly across runs and makes "oldest-first walk" the
+ * natural read order in the resolution skill.
+ */
+export function writeConflictIndex(
+  hqRoot: string,
+  index: ConflictIndex,
+): void {
+  const indexPath = getConflictIndexPath(hqRoot);
+  fs.mkdirSync(path.dirname(indexPath), { recursive: true });
+
+  const sorted: ConflictIndex = {
+    version: index.version,
+    conflicts: [...index.conflicts].sort((a, b) =>
+      a.detectedAt.localeCompare(b.detectedAt),
+    ),
+  };
+
+  // Random suffix in tmp name avoids collision if two sync runs ever overlap
+  // (shouldn't happen — the runner serializes — but cheap insurance).
+  const tmpPath = `${indexPath}.tmp.${crypto.randomBytes(6).toString("hex")}`;
+  fs.writeFileSync(tmpPath, JSON.stringify(sorted, null, 2));
+  fs.renameSync(tmpPath, indexPath);
+}
+
+/**
+ * Idempotent append. If an entry with the same `id` already exists (same
+ * original path, same detection timestamp), update it in place rather than
+ * duplicating. This matters because re-running sync after a conflict but
+ * before resolution will re-detect the same divergence — without dedup the
+ * index would grow unboundedly.
+ *
+ * The "update in place" path also covers the case where the cloud advanced
+ * again between detections: we want the latest `remoteVersionId` and
+ * `remoteHash` so the resolution skill shows the user the *current* cloud
+ * state, not stale data from the first detection.
+ */
+export function appendConflictEntry(
+  hqRoot: string,
+  entry: ConflictIndexEntry,
+): void {
+  const index = readConflictIndex(hqRoot);
+  const existingIdx = index.conflicts.findIndex((c) => c.id === entry.id);
+  if (existingIdx >= 0) {
+    index.conflicts[existingIdx] = entry;
+  } else {
+    index.conflicts.push(entry);
+  }
+  writeConflictIndex(hqRoot, index);
+}
+
+/**
+ * Remove an entry by id. Used by the `/resolve-conflicts` skill after the
+ * user picks a resolution and the conflict file is cleaned up. No-op if the
+ * id isn't present (e.g. user manually removed the file then re-ran the
+ * skill — we want that to be a clean exit, not an error).
+ */
+export function removeConflictEntry(hqRoot: string, id: string): void {
+  const index = readConflictIndex(hqRoot);
+  const filtered = index.conflicts.filter((c) => c.id !== id);
+  if (filtered.length === index.conflicts.length) return;
+  writeConflictIndex(hqRoot, { version: index.version, conflicts: filtered });
+}