@indigoai-us/hq-cloud 5.14.0 → 5.16.0

package/src/cli/sync.test.ts

@@ -448,6 +448,249 @@ describe("sync", () => {
     });
   });
 
+  // ── New-file detection (US-001) ──────────────────────────────────────
+
+  it("classifies all files as new on first sync (clean directory)", async () => {
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    expect(result.newFilesCount).toBe(2);
+    expect(result.newFiles).toEqual(
+      expect.arrayContaining([
+        { path: "docs/handoff.md", bytes: 42 },
+        { path: "knowledge/readme.md", bytes: 100 },
+      ]),
+    );
+    expect(result.newFiles).toHaveLength(2);
+  });
+
+  it("classifies only genuinely new files when some exist locally", async () => {
+    // Pre-create one file + journal entry so it's treated as an update,
+    // while the second file is genuinely new.
+    const companyDocs = path.join(tmpDir, "companies", "acme", "docs");
+    fs.mkdirSync(companyDocs, { recursive: true });
+    fs.writeFileSync(path.join(companyDocs, "handoff.md"), "mock file content");
+
+    // Journal entry with matching remoteEtag but stale hash to force download
+    // (remote-only changed scenario).
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/handoff.md": {
+            hash: "old-hash-before-remote-update",
+            size: 20,
+            syncedAt: new Date(Date.now() - 3600000).toISOString(),
+            direction: "down",
+            // Stale etag — forces remoteChanged=true, localChanged=false
+            // (local hash matches what's on disk because we didn't edit it,
+            // but wait — journal hash != current disk hash because we wrote
+            // "mock file content" but stored "old-hash-before-remote-update".
+            // To avoid conflict we need localChanged=false. Let's compute
+            // the real hash and use it.)
+          },
+        },
+      }),
+    );
+
+    // Re-read the journal to compute the actual hash of the file we wrote,
+    // then update the journal so localChanged is false.
+    const { hashFile: realHashFile } = await import("../journal.js");
+    const actualHash = realHashFile(path.join(companyDocs, "handoff.md"));
+    fs.writeFileSync(
+      journalPath,
+      JSON.stringify({
+        version: "1",
+        lastSync: new Date().toISOString(),
+        files: {
+          "docs/handoff.md": {
+            hash: actualHash,
+            size: 17,
+            syncedAt: new Date(Date.now() - 3600000).toISOString(),
+            direction: "down",
+            remoteEtag: "stale-etag-to-force-download",
+          },
+        },
+      }),
+    );
+
+    const result = await sync({
+      company: "acme",
+      onConflict: "overwrite",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    // handoff.md existed locally → isNew: false (update)
+    // readme.md did not exist → isNew: true (new)
+    expect(result.newFilesCount).toBe(1);
+    expect(result.newFiles).toEqual([
+      { path: "knowledge/readme.md", bytes: 100 },
+    ]);
+  });
+
+  it("returns empty newFiles when all files already exist", async () => {
+    // First sync downloads everything
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    // Second sync — files are now local and journal is up to date.
+    // Nothing downloads, newFiles should be empty.
+    const result = await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    expect(result.newFilesCount).toBe(0);
+    expect(result.newFiles).toEqual([]);
+  });
+
+  // ── New-files event with attribution (US-002) ───────────────────────
+
+  it("emits a new-files event with attribution from HeadObject metadata", async () => {
+    vi.mocked(s3Module.headRemoteFile).mockImplementation(async (_ctx, key) => {
+      if (key === "docs/handoff.md") {
+        return {
+          lastModified: new Date(),
+          etag: '"abc123"',
+          size: 42,
+          metadata: { "created-by": "alice@example.com" },
+        };
+      }
+      if (key === "knowledge/readme.md") {
+        return {
+          lastModified: new Date(),
+          etag: '"def456"',
+          size: 100,
+          metadata: { "created-by": "bob@example.com" },
+        };
+      }
+      return null;
+    });
+
+    const newFilesEvents: Array<{
+      type: string;
+      files: Array<{ path: string; bytes: number; addedBy: string | null }>;
+    }> = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "new-files") newFilesEvents.push(e);
+      },
+    });
+
+    expect(newFilesEvents).toHaveLength(1);
+    expect(newFilesEvents[0].files).toHaveLength(2);
+    expect(newFilesEvents[0].files).toEqual(
+      expect.arrayContaining([
+        { path: "docs/handoff.md", bytes: 42, addedBy: "alice@example.com" },
+        { path: "knowledge/readme.md", bytes: 100, addedBy: "bob@example.com" },
+      ]),
+    );
+  });
+
+  it("sets addedBy to null when HeadObject fails (best-effort)", async () => {
+    vi.mocked(s3Module.headRemoteFile).mockRejectedValue(new Error("S3 transient error"));
+
+    const newFilesEvents: Array<{
+      type: string;
+      files: Array<{ path: string; bytes: number; addedBy: string | null }>;
+    }> = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "new-files") newFilesEvents.push(e);
+      },
+    });
+
+    expect(newFilesEvents).toHaveLength(1);
+    // Both files should still appear, just with null addedBy
+    expect(newFilesEvents[0].files).toHaveLength(2);
+    for (const f of newFilesEvents[0].files) {
+      expect(f.addedBy).toBeNull();
+    }
+  });
+
+  it("sets addedBy to null when metadata has no created-by key", async () => {
+    vi.mocked(s3Module.headRemoteFile).mockResolvedValue({
+      lastModified: new Date(),
+      etag: '"abc"',
+      size: 10,
+      metadata: {},
+    });
+
+    const newFilesEvents: Array<{
+      type: string;
+      files: Array<{ path: string; bytes: number; addedBy: string | null }>;
+    }> = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "new-files") newFilesEvents.push(e);
+      },
+    });
+
+    expect(newFilesEvents).toHaveLength(1);
+    for (const f of newFilesEvents[0].files) {
+      expect(f.addedBy).toBeNull();
+    }
+  });
+
+  it("emits new-files event with empty array when no new files exist", async () => {
+    // First sync downloads everything
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+    });
+
+    // Second sync — files already exist locally
+    const newFilesEvents: Array<{
+      type: string;
+      files: Array<{ path: string; bytes: number; addedBy: string | null }>;
+    }> = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => {
+        if (e.type === "new-files") newFilesEvents.push(e);
+      },
+    });
+
+    expect(newFilesEvents).toHaveLength(1);
+    expect(newFilesEvents[0].files).toEqual([]);
+  });
+
+  it("new-files event is emitted after all progress events", async () => {
+    const eventTypes: string[] = [];
+    await sync({
+      company: "acme",
+      vaultConfig: mockConfig,
+      hqRoot: tmpDir,
+      onEvent: (e) => eventTypes.push(e.type),
+    });
+
+    const lastProgressIdx = eventTypes.lastIndexOf("progress");
+    const newFilesIdx = eventTypes.indexOf("new-files");
+    expect(newFilesIdx).toBeGreaterThan(lastProgressIdx);
+  });
+
   it("plan event counts a 3-way conflict separately from downloads", async () => {
     // Local edit + journal-tracked + remote ETag drifted → conflict.
     const companyDocs = path.join(tmpDir, "companies", "acme", "docs");

package/src/cli/sync.ts

@@ -9,7 +9,7 @@ import * as fs from "fs";
 import * as path from "path";
 import type { VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
-import { downloadFile, listRemoteFiles } from "../s3.js";
+import { downloadFile, listRemoteFiles, headRemoteFile } from "../s3.js";
 import type { RemoteFile } from "../s3.js";
 import { readJournal, writeJournal, hashFile, updateEntry, getEntry, normalizeEtag } from "../journal.js";
 import { createIgnoreFilter } from "../ignore.js";
@@ -80,6 +80,10 @@ export type SyncProgressEvent =
       path: string;
       direction: "pull" | "push";
       resolution: ConflictResolution;
+    }
+  | {
+      type: "new-files";
+      files: Array<{ path: string; bytes: number; addedBy: string | null }>;
     };
 
 export interface SyncOptions {
@@ -125,6 +129,14 @@ export interface SyncResult {
    */
   conflictPaths: string[];
   aborted: boolean;
+  /**
+   * Files classified as "new" during pull — i.e. the remote file had no
+   * local counterpart at classification time. Additive field; empty array
+   * when no new files were detected or on push-only syncs.
+   */
+  newFiles: Array<{ path: string; bytes: number }>;
+  /** Convenience count: `newFiles.length`. */
+  newFilesCount: number;
 }
 
 /**
@@ -276,6 +288,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
       }
 
       if (resolution === "abort") {
+        emit({ type: "new-files", files: [] });
         writeJournal(journalSlug, journal);
         return {
           filesDownloaded,
@@ -284,6 +297,8 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
           conflicts,
           conflictPaths,
           aborted: true,
+          newFiles: plan.newFiles,
+          newFilesCount: plan.newFilesCount,
         };
       }
       if (resolution === "keep" || resolution === "skip") {
@@ -352,6 +367,41 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     }
   }
 
+  // ── New-files attribution (US-002) ─────────────────────────────────────
+  // Enrich plan.newFiles with `addedBy` from S3 user metadata. HeadObject
+  // calls are best-effort and capped at 5 concurrent to avoid hammering S3.
+  const enrichedNewFiles: Array<{ path: string; bytes: number; addedBy: string | null }> = [];
+  const HEAD_CONCURRENCY = 5;
+  for (let i = 0; i < plan.newFiles.length; i += HEAD_CONCURRENCY) {
+    const batch = plan.newFiles.slice(i, i + HEAD_CONCURRENCY);
+    const results = await Promise.all(
+      batch.map(async (nf) => {
+        let addedBy: string | null = null;
+        try {
+          const head = await headRemoteFile(ctx, nf.path);
+          if (head?.metadata?.["created-by"]) {
+            addedBy = head.metadata["created-by"];
+          }
+        } catch (headErr) {
+          // Best-effort: log to console (Sentry captures via global handler)
+          // and fall through with addedBy = null.
+          try {
+            console.error(
+              `[hq-sync] HeadObject failed for ${nf.path}: ${
+                headErr instanceof Error ? headErr.message : String(headErr)
+              }`,
+            );
+          } catch {
+            // Swallow — logging must never break sync.
+          }
+        }
+        return { path: nf.path, bytes: nf.bytes, addedBy };
+      }),
+    );
+    enrichedNewFiles.push(...results);
+  }
+  emit({ type: "new-files", files: enrichedNewFiles });
+
   // Stamp lastSync on every successful run so the menubar's "Last sync · X ago"
   // ticks even when nothing transferred. updateEntry only fires on actual
   // downloads; without this, a no-op sync leaves lastSync at the time of the
@@ -366,6 +416,8 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     conflicts,
     conflictPaths,
     aborted: false,
+    newFiles: plan.newFiles,
+    newFilesCount: plan.newFilesCount,
   };
 }
 
@@ -409,7 +461,7 @@ function hasRemoteChanged(
  * executor can hand it to `resolveConflict` without re-hashing.
  */
 type PullPlanItem =
-  | { action: "download"; remoteFile: RemoteFile; localPath: string }
+  | { action: "download"; remoteFile: RemoteFile; localPath: string; isNew: boolean }
   | { action: "skip-ignored"; remoteFile: RemoteFile; localPath: string }
   | { action: "skip-personal-mode"; remoteFile: RemoteFile; localPath: string }
   | { action: "skip-unchanged"; remoteFile: RemoteFile; localPath: string }
@@ -427,6 +479,9 @@ interface PullPlan {
   bytesToDownload: number;
   filesToSkip: number;
   filesToConflict: number;
+  /** Files classified as new (no local counterpart at classification time). */
+  newFiles: Array<{ path: string; bytes: number }>;
+  newFilesCount: number;
 }
 
 /**
@@ -463,8 +518,9 @@ function computePullPlan(
     }
 
     const journalEntry = getEntry(journal, remoteFile.key);
+    const localExists = fs.existsSync(localPath);
 
-    if (fs.existsSync(localPath)) {
+    if (localExists) {
       const localHash = hashFile(localPath);
       const localChanged = !!journalEntry && journalEntry.hash !== localHash;
       const remoteChanged =
@@ -493,17 +549,21 @@ function computePullPlan(
       // No journal entry, or remote-only changed → fall through to download.
     }
 
-    items.push({ action: "download", remoteFile, localPath });
+    items.push({ action: "download", remoteFile, localPath, isNew: !localExists });
   }
 
   let filesToDownload = 0;
   let bytesToDownload = 0;
   let filesToSkip = 0;
   let filesToConflict = 0;
+  const newFiles: Array<{ path: string; bytes: number }> = [];
   for (const item of items) {
     if (item.action === "download") {
       filesToDownload++;
       bytesToDownload += item.remoteFile.size;
+      if (item.isNew) {
+        newFiles.push({ path: item.remoteFile.key, bytes: item.remoteFile.size });
+      }
     } else if (item.action === "conflict") {
       filesToConflict++;
     } else {
@@ -517,6 +577,8 @@ function computePullPlan(
     bytesToDownload,
     filesToSkip,
     filesToConflict,
+    newFiles,
+    newFilesCount: newFiles.length,
   };
 }
 
@@ -567,5 +629,18 @@ function defaultConsoleLogger(event: SyncProgressEvent): void {
     console.error(
       `  ⚠ conflict (${event.direction}): ${event.path} — ${event.resolution}`,
     );
+  } else if (event.type === "new-files") {
+    if (event.files.length > 0) {
+      console.log(`${event.files.length} new file${event.files.length === 1 ? "" : "s"}`);
+      const MAX_SHOWN = 20;
+      const shown = event.files.slice(0, MAX_SHOWN);
+      for (const f of shown) {
+        const who = f.addedBy ? ` (added by ${f.addedBy})` : "";
+        console.log(`  + ${f.path}${who}`);
+      }
+      if (event.files.length > MAX_SHOWN) {
+        console.log(`  ... and ${event.files.length - MAX_SHOWN} more`);
+      }
+    }
   }
 }

package/src/index.ts

@@ -50,8 +50,14 @@ export {
 } from "./cognito-auth.js";
 export type { CognitoAuthConfig, CognitoTokens } from "./cognito-auth.js";
 
+// Personal-vault scope helpers — shared between hq-sync-runner and `hq sync`
+export {
+  PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,
+  computePersonalVaultPaths,
+} from "./personal-vault.js";
+
 // VaultClient SDK (VLT-7)
-export { VaultClient } from "./vault-client.js";
+export { VaultClient, pickCanonicalPersonEntity } from "./vault-client.js";
 export {
   VaultClientError,
   VaultAuthError,

package/src/personal-vault.ts

@@ -0,0 +1,63 @@
+/**
+ * Personal-vault scope helpers — shared between the menubar runner
+ * (`hq-sync-runner`) and the `hq sync` CLI so every composer of a personal
+ * push uses the same exclusion list.
+ *
+ * The exclusion list mirrors the Rust constant of the same name in
+ * `hq-sync/src-tauri/src/commands/personal.rs` so the Tauri menubar's
+ * first-push and this Node engine's steady-state push enforce identical
+ * scope. Every other top-level entry under hq_root (e.g. `.claude/`,
+ * `knowledge/`, `modules/`, `README.md`, `.codex/`, `core/`, `data/`,
+ * `personal/`) is included, subject to the usual `.hqignore` filter.
+ *
+ * Excluded entries (and why):
+ *   - `.git`: a git repo's own metadata is hostile to multi-machine
+ *     sync; .gitignore alone doesn't cover `.git/` because it's the repo
+ *     itself, not a tracked path.
+ *   - `companies/`: synced separately by the runner's per-membership
+ *     fanout; do not double-write into the personal vault.
+ *   - `repos/`, `workspace/`: per user directive — heavy local-only
+ *     content (cloned remotes, session threads) that has no business in
+ *     the personal vault.
+ *
+ * Note: `core/`, `data/`, and `personal/` were previously excluded but are
+ * INCLUDED as of user directive 2026-05-13. `core/` ships the hq-core
+ * scaffold — policies/, settings/, skills/, workers/, plus the rules
+ * manifest at core/core.yaml. `data/` and `personal/` carry per-user data,
+ * policies, hooks, and skills that follow the user across machines. The
+ * hq-root identity marker `core.yaml` (at hq_root, distinct from
+ * `core/core.yaml`) is filtered separately by the root-anchored
+ * `/core.yaml` DEFAULT_IGNORES rule.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+
+export const PERSONAL_VAULT_EXCLUDED_TOP_LEVEL: readonly string[] = [
+  ".git",
+  "companies",
+  "repos",
+  "workspace",
+];
+
+/**
+ * Compute absolute paths to share for the personal vault: every top-level
+ * entry under `hqRoot` whose basename is NOT in
+ * `PERSONAL_VAULT_EXCLUDED_TOP_LEVEL`. Mirrors the Rust
+ * `is_personal_vault_path` predicate (just hoisted to the top-level step).
+ * Order is whatever `fs.readdirSync` returns — share() doesn't care, and
+ * the per-file walk inside share() handles recursion uniformly. Missing
+ * hqRoot returns []; callers treat that as "no personal content to push"
+ * rather than a hard error.
+ */
+export function computePersonalVaultPaths(hqRoot: string): string[] {
+  let entries: string[];
+  try {
+    entries = fs.readdirSync(hqRoot);
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((name) => !PERSONAL_VAULT_EXCLUDED_TOP_LEVEL.includes(name))
+    .map((name) => path.join(hqRoot, name));
+}

package/src/s3.ts

@@ -214,7 +214,7 @@ export async function deleteRemoteFile(
 export async function headRemoteFile(
   ctx: EntityContext,
   key: string,
-): Promise<{ lastModified: Date; etag: string; size: number } | null> {
+): Promise<{ lastModified: Date; etag: string; size: number; metadata?: Record<string, string> } | null> {
   const client = buildClient(ctx);
   try {
     const response = await client.send(
@@ -227,6 +227,7 @@ export async function headRemoteFile(
       lastModified: response.LastModified || new Date(),
       etag: response.ETag || "",
       size: response.ContentLength || 0,
+      metadata: response.Metadata,
     };
   } catch (err: unknown) {
     if (err && typeof err === "object" && "name" in err && err.name === "NotFound") {