@indigoai-us/hq-cloud 5.4.5 → 5.7.0

package/src/cli/share.ts

@@ -7,15 +7,109 @@
 
 import * as fs from "fs";
 import * as path from "path";
-import type { VaultServiceConfig } from "../types.js";
+import type { VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { uploadFile, headRemoteFile } from "../s3.js";
-import { readJournal, writeJournal, hashFile, updateEntry } from "../journal.js";
+import { readJournal, writeJournal, hashFile, updateEntry, normalizeEtag } from "../journal.js";
 import { createIgnoreFilter, isWithinSizeLimit } from "../ignore.js";
 import { resolveConflict } from "./conflict.js";
 import type { ConflictStrategy } from "./conflict.js";
 import type { SyncProgressEvent } from "./sync.js";
 
+/**
+ * Stage-1 classification for a single local file in a push run. Pre-HEAD —
+ * only inputs we can evaluate locally (size limit, journal hash, optional
+ * skip-unchanged) determine the action. Files that pass classification as
+ * `upload` are still subject to a per-file HEAD + 3-way conflict check in
+ * Stage 2 before the actual PUT, so the `filesToUpload` count in the plan
+ * event is an upper bound: it includes files that may turn out to be
+ * conflicts. V1.5 follow-up: replace per-file HEAD with a single LIST so
+ * conflicts can be classified up-front and reported in the plan.
+ */
+type PushPlanItem =
+  | {
+      action: "upload";
+      absolutePath: string;
+      relativePath: string;
+      localHash: string;
+      size: number;
+    }
+  | {
+      action: "skip-size-limit";
+      absolutePath: string;
+      relativePath: string;
+    }
+  | {
+      action: "skip-unchanged";
+      absolutePath: string;
+      relativePath: string;
+    };
+
+interface PushPlan {
+  items: PushPlanItem[];
+  filesToUpload: number;
+  bytesToUpload: number;
+  filesToSkip: number;
+}
+
+/**
+ * Pure Stage-1 pass for push: walk the candidate file list, hash each one,
+ * apply the size-limit and skip-unchanged gates, and return a classified
+ * plan plus aggregate counts. No S3 calls, no journal writes, no event
+ * emission.
+ *
+ * The conflict count is intentionally absent from the returned `PushPlan` —
+ * detecting a push conflict requires a remote HEAD that we defer to Stage 2.
+ * Consumers that want a conflict count get it from the `complete` event.
+ */
+function computePushPlan(
+  filesToShare: { absolutePath: string; relativePath: string }[],
+  journal: SyncJournal,
+  skipUnchanged: boolean,
+): PushPlan {
+  const items: PushPlanItem[] = [];
+
+  for (const { absolutePath, relativePath } of filesToShare) {
+    if (!isWithinSizeLimit(absolutePath)) {
+      items.push({ action: "skip-size-limit", absolutePath, relativePath });
+      continue;
+    }
+
+    const localHash = hashFile(absolutePath);
+
+    if (skipUnchanged) {
+      const existing = journal.files[relativePath];
+      if (existing && existing.hash === localHash) {
+        items.push({ action: "skip-unchanged", absolutePath, relativePath });
+        continue;
+      }
+    }
+
+    const size = fs.statSync(absolutePath).size;
+    items.push({
+      action: "upload",
+      absolutePath,
+      relativePath,
+      localHash,
+      size,
+    });
+  }
+
+  let filesToUpload = 0;
+  let bytesToUpload = 0;
+  let filesToSkip = 0;
+  for (const item of items) {
+    if (item.action === "upload") {
+      filesToUpload++;
+      bytesToUpload += item.size;
+    } else {
+      filesToSkip++;
+    }
+  }
+
+  return { items, filesToUpload, bytesToUpload, filesToSkip };
+}
+
 export interface ShareOptions {
   /** Path(s) to share (files or directories) */
   paths: string[];
@@ -94,45 +188,68 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
   // Collect all files to share
   const filesToShare = collectFiles(paths, hqRoot, syncRoot, shouldSync);
 
-  for (const { absolutePath, relativePath } of filesToShare) {
-    if (!isWithinSizeLimit(absolutePath)) {
+  // Stage 1: classify each file. Pre-HEAD — only inputs we can evaluate
+  // locally (size limit, journal hash, optional skip-unchanged) are
+  // considered. The plan event below carries an upper-bound `filesToUpload`
+  // (true conflicts emerge from the per-file HEAD in Stage 2 and aren't
+  // knowable here). The final `complete` event reports authoritative counts.
+  const plan = computePushPlan(filesToShare, journal, skipUnchanged === true);
+
+  emit({
+    type: "plan",
+    // share() is push-only; pull counts are sourced from sync()'s plan event.
+    filesToDownload: 0,
+    bytesToDownload: 0,
+    filesToUpload: plan.filesToUpload,
+    bytesToUpload: plan.bytesToUpload,
+    filesToSkip: plan.filesToSkip,
+    // Push conflicts require a remote HEAD; we don't yet do that in Stage 1,
+    // so this stays 0. V1.5 (single LIST) will let us classify them up-front.
+    filesToConflict: 0,
+  });
+
+  // Stage 2: execute. Skip items pre-classified as no-ops, then for each
+  // upload candidate run the HEAD + 3-way conflict check + actual PUT.
+  for (const item of plan.items) {
+    if (item.action === "skip-size-limit") {
       emit({
         type: "error",
-        path: relativePath,
+        path: item.relativePath,
         message: "file exceeds size limit",
       });
       filesSkipped++;
       continue;
     }
-
-    // Skip-if-unchanged gate: the hot path for bidirectional Sync Now. When
-    // walking an entire company folder, this is what keeps us from re-uploading
-    // every file every tick. Off by default so `hq share <file>` keeps its
-    // explicit-intent semantics (user named it, user wants it sent).
-    const localHash = hashFile(absolutePath);
-    if (skipUnchanged) {
-      const existing = journal.files[relativePath];
-      if (existing && existing.hash === localHash) {
-        filesSkipped++;
-        continue;
-      }
+    if (item.action === "skip-unchanged") {
+      filesSkipped++;
+      continue;
     }
 
+    const { absolutePath, relativePath, localHash } = item;
+
     // Auto-refresh context if credentials expiring
     if (isExpiringSoon(ctx.expiresAt)) {
       ctx = await refreshEntityContext(companyRef, vaultConfig);
     }
 
-    // Check for remote conflict — refuse to overwrite newer remote version
+    // Check for remote conflict — refuse to overwrite newer remote version.
+    //
+    // A real conflict requires BOTH sides to have moved since the last sync.
+    // The previous predicate only checked `journalEntry.hash !== localHash`,
+    // which mislabelled every local edit as a conflict and (combined with
+    // `--on-conflict keep`) silently dropped the user's edit. We now compare
+    // the current remote ETag against the one captured at last sync; when
+    // missing (legacy entries), we fall back to the same `lastModified >
+    // syncedAt` heuristic the pull side uses.
     const remoteMeta = await headRemoteFile(ctx, relativePath);
     if (remoteMeta) {
       const journalEntry = journal.files[relativePath];
+      const localChanged = !!journalEntry && journalEntry.hash !== localHash;
+      const remoteChanged = !!journalEntry && hasRemoteChanged(remoteMeta, journalEntry);
 
-      // If remote has changed since our last sync, it's a conflict
-      if (journalEntry && journalEntry.hash !== localHash) {
+      if (localChanged && remoteChanged) {
         conflictPaths.push(relativePath);
 
-        // Local has changes — check if remote also changed
         const resolution = await resolveConflict(
           {
             path: relativePath,
@@ -171,10 +288,12 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
     try {
       const stat = fs.statSync(absolutePath);
 
-      await uploadFile(ctx, absolutePath, relativePath);
+      const { etag } = await uploadFile(ctx, absolutePath, relativePath);
 
-      // Update journal with optional message
-      updateEntry(journal, relativePath, localHash, stat.size, "up");
+      // Update journal with optional message; capture the post-upload ETag
+      // so the next sync can distinguish "remote moved since we last wrote"
+      // from "user edited locally" without conflating the two.
+      updateEntry(journal, relativePath, localHash, stat.size, "up", etag);
       if (message) {
         journal.files[relativePath] = {
           ...journal.files[relativePath],
@@ -215,7 +334,13 @@ export async function share(options: ShareOptions): Promise<ShareResult> {
  * emitted before `onEvent` was added — tty users see no change.
  */
 function defaultConsoleLogger(event: SyncProgressEvent): void {
-  if (event.type === "progress") {
+  if (event.type === "plan") {
+    if (event.filesToUpload > 0) {
+      console.log(
+        `Plan: ${event.filesToUpload} to upload (${event.bytesToUpload} bytes), ${event.filesToSkip} unchanged`,
+      );
+    }
+  } else if (event.type === "progress") {
     if (event.message) {
       console.log(`  ✓ ${event.path} — "${event.message}"`);
     } else {
@@ -225,7 +350,7 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     console.error(
       `  ⚠ conflict (${event.direction}): ${event.path} — ${event.resolution}`,
     );
-  } else {
+  } else if (event.type === "error") {
     console.error(`  ✗ ${event.path} — ${event.message}`);
   }
 }
@@ -318,3 +443,21 @@ function isWithin(parent: string, child: string): boolean {
   const rel = path.relative(parent, child);
   return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
 }
+
+/**
+ * Returns true when the remote object appears to have moved since the
+ * journal entry's last-recorded sync. Prefers ETag equality; falls back to
+ * `lastModified > syncedAt` for legacy entries written before remoteEtag
+ * was tracked. Conservative on tie (`<=` skews "remote unchanged") so an
+ * S3-side mtime that exactly equals our syncedAt is not treated as drift.
+ */
+function hasRemoteChanged(
+  remote: { lastModified: Date; etag: string },
+  entry: { syncedAt: string; remoteEtag?: string },
+): boolean {
+  if (entry.remoteEtag) {
+    return normalizeEtag(remote.etag) !== entry.remoteEtag;
+  }
+  const syncedAt = new Date(entry.syncedAt).getTime();
+  return remote.lastModified.getTime() > syncedAt;
+}

package/src/cli/sync.test.ts

@@ -330,4 +330,155 @@ describe("sync", () => {
     // File should be overwritten with mock content
     expect(fs.readFileSync(path.join(companyDocs, "handoff.md"), "utf-8")).toBe("mock file content");
   });
+
+  it("does NOT flag a pull conflict when only local changed since last sync", async () => {
+    // Regression: previously, any local edit to a file that also existed on
+    // S3 produced a pull conflict because the predicate only checked
+    // `journalEntry.hash !== localHash`. With `--on-conflict keep` this
+    // silently dropped local edits during the round-trip. With remoteEtag
+    // matching the journal, the remote is known unchanged and the pull
+    // phase should leave the local edit alone for the push phase to upload.
+    const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
+    fs.mkdirSync(companyDocs, { recursive: true });
+    fs.writeFileSync(path.join(companyDocs, "handoff.md"), "local edit");
+
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/handoff.md": {
+            hash: "stale-hash-from-pre-edit",
+            size: 20,
+            syncedAt: new Date(Date.now() - 3600000).toISOString(),
+            direction: "down",
+            // Matches the listRemoteFiles mock's etag for handoff.md.
+            remoteEtag: "abc123",
+          },
+        },
+      }),
+    );
+
+    const result = await sync({
+      company: "acme",
+      onConflict: "keep",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    expect(result.conflicts).toBe(0);
+    expect(result.conflictPaths).toEqual([]);
+    // Local edit must be preserved (not clobbered by download)
+    expect(fs.readFileSync(path.join(companyDocs, "handoff.md"), "utf-8")).toBe("local edit");
+  });
+
+  it("records remoteEtag from listRemoteFiles on the journal entry after download", async () => {
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    const journal = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+    expect(journal.files["docs/handoff.md"].remoteEtag).toBe("abc123");
+    expect(journal.files["knowledge/readme.md"].remoteEtag).toBe("def456");
+  });
+
+  // ── Stage-1 plan event ─────────────────────────────────────────────────
+
+  it("emits a plan event before any progress events", async () => {
+    const events: { type: string }[] = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => events.push({ type: e.type }),
+    });
+
+    // Plan must be the first event so consumers can use its totals as
+    // the progress denominator before any per-file events arrive.
+    expect(events.length).toBeGreaterThan(0);
+    expect(events[0].type).toBe("plan");
+    const planIndex = events.findIndex((e) => e.type === "plan");
+    const firstProgressIndex = events.findIndex((e) => e.type === "progress");
+    expect(firstProgressIndex).toBeGreaterThan(planIndex);
+  });
+
+  it("plan event totals reflect the upcoming Stage-2 work (all-new case)", async () => {
+    // Both mock remote files are new locally → both counted as downloads,
+    // bytes summed from listRemoteFiles, no conflicts, no skips.
+    const planEvents: unknown[] = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "plan") {
+          planEvents.push(e);
+        }
+      },
+    });
+
+    expect(planEvents).toHaveLength(1);
+    expect(planEvents[0]).toMatchObject({
+      type: "plan",
+      filesToDownload: 2,
+      bytesToDownload: 142, // 42 + 100 from the s3 mock
+      filesToUpload: 0, // sync() never plans uploads
+      bytesToUpload: 0,
+      filesToSkip: 0,
+      filesToConflict: 0,
+    });
+  });
+
+  it("plan event counts a 3-way conflict separately from downloads", async () => {
+    // Local edit + journal-tracked + remote ETag drifted → conflict.
+    const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
+    fs.mkdirSync(companyDocs, { recursive: true });
+    fs.writeFileSync(path.join(companyDocs, "handoff.md"), "local edit");
+
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/handoff.md": {
+            hash: "stale-hash-from-pre-edit",
+            size: 20,
+            syncedAt: new Date(Date.now() - 3600000).toISOString(),
+            direction: "down",
+            // Mismatched ETag — listRemoteFiles mock returns "abc123",
+            // we record a stale one so remoteChanged is true.
+            remoteEtag: "stale-remote-etag",
+          },
+        },
+      }),
+    );
+
+    const planEvents: Array<{
+      type: string;
+      filesToDownload?: number;
+      filesToConflict?: number;
+      filesToSkip?: number;
+    }> = [];
+    await sync({
+      company: "acme",
+      onConflict: "keep",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "plan") planEvents.push(e);
+      },
+    });
+
+    expect(planEvents).toHaveLength(1);
+    // Conflict is counted separately; only the new file is in toDownload.
+    expect(planEvents[0]).toMatchObject({
+      filesToDownload: 1,
+      filesToConflict: 1,
+      filesToSkip: 0,
+    });
+  });
 });