@indigoai-us/hq-cloud 6.14.6 → 6.14.8

package/src/cli/rescue-snapshot.ts

@@ -0,0 +1,51 @@
+/** Filesystem shape captured during rescue classification. */
+export type RescuePathKind = "missing" | "file" | "directory" | "symlink" | "other";
+
+
+/** Raised when a path changes shape between read-only classification and apply. */
+export class RescuePathChangedError extends Error {
+  constructor(
+    public readonly rel: string,
+    public readonly expected: RescuePathKind,
+    public readonly actual: RescuePathKind,
+  ) {
+    super(
+      `rescue path changed after classification: ${rel} (expected ${expected}, found ${actual}); ` +
+        "another process may be rebuilding generated HQ paths, so rescue stopped before mutation",
+    );
+    this.name = "RescuePathChangedError";
+  }
+}
+
+/** Return whether a directory is reproducible state that rescue must not snapshot or classify. */
+export function isRescueTransientDirectory(rel: string): boolean {
+  const normalized = rel.replace(/\\/g, "/").replace(/^\.\//, "").replace(/\/$/, "");
+  if (
+    normalized === "worktrees" ||
+    normalized.startsWith("worktrees/") ||
+    normalized === ".worktrees" ||
+    normalized.startsWith(".worktrees/") ||
+    normalized === ".sst-sandbox-home" ||
+    normalized.startsWith(".sst-sandbox-home/") ||
+    normalized === ".tmp-codex-asar" ||
+    normalized.startsWith(".tmp-codex-asar/") ||
+    normalized === ".claude/worktrees" ||
+    normalized.startsWith(".claude/worktrees/")
+  ) {
+    return true;
+  }
+  return normalized
+    .split("/")
+    .some((part) =>
+      ["node_modules", ".git", ".pnpm-store", ".sst", ".next", "target"].includes(part),
+    );
+}
+
+/** Fail safely when apply observes a different path shape than classification did. */
+export function assertRescuePathTypeUnchanged(
+  rel: string,
+  expected: RescuePathKind,
+  actual: RescuePathKind,
+): void {
+  if (expected !== actual) throw new RescuePathChangedError(rel, expected, actual);
+}

package/src/cli/rescue.test.ts

@@ -1,5 +1,12 @@
 import { describe, it, expect } from "vitest";
 import { buildRescueArgs, extractLockTimeout } from "./rescue.js";
+import { toCygwinRsyncPath } from "./rescue-core.js";
+import {
+  decodeLocalVaultSegment,
+  encodeLocalVaultSegment,
+  localPathForVaultKey,
+  vaultKeyForLocalPath,
+} from "../local-path-codec.js";
 
 describe("extractLockTimeout", () => {
   it("returns the explicit option untouched when extraArgs is empty/absent", () => {
@@ -95,3 +102,30 @@ describe("buildRescueArgs", () => {
     expect(buildRescueArgs({ paths: [] })).toEqual([]);
   });
 });
+
+describe("Windows sync path adapters", () => {
+  it("converts drive-letter and UNC rsync arguments to Cygwin paths", () => {
+    expect(toCygwinRsyncPath("C:\\HQ\\core\\")).toBe("/cygdrive/c/HQ/core/");
+    expect(toCygwinRsyncPath("D:/HQ/workspace")).toBe("/cygdrive/d/HQ/workspace");
+    expect(toCygwinRsyncPath("\\\\fileserver\\hq-share\\HQ\\")).toBe(
+      "//fileserver/hq-share/HQ/",
+    );
+  });
+
+  it("round-trips Windows-safe local names without changing canonical vault keys", () => {
+    const win32 = true;
+    expect(encodeLocalVaultSegment("core:skill", win32)).toBe("core%3Askill");
+    expect(encodeLocalVaultSegment("core%3Askill", win32)).toBe("core%253Askill");
+    expect(encodeLocalVaultSegment("core\u0000skill", win32)).toBe("core%00skill");
+    expect(decodeLocalVaultSegment("core%3Askill", win32)).toBe("core:skill");
+
+    // Pull materializes the key under a safe filename, while the journal and
+    // the following push retain the original canonical S3 key.
+    const canonicalKey = "factory:slack/apps/web/oauth-token.json";
+    const pulledLocalPath = localPathForVaultKey("C:\\HQ\\companies\\acme", canonicalKey, win32);
+    expect(pulledLocalPath).toContain("factory%3Aslack");
+    const journalKey = canonicalKey;
+    const pushedKey = vaultKeyForLocalPath("C:\\HQ\\companies\\acme", pulledLocalPath, win32);
+    expect(pushedKey).toBe(journalKey);
+  });
+});

package/src/cli/share.ts

@@ -11,6 +11,7 @@ import * as path from "path";
 import type { EntityContext, VaultServiceConfig, SyncJournal } from "../types.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
+import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
 import {
   uploadFile,
   uploadSymlink,
@@ -788,7 +789,7 @@ function wrapFilterWithScope(
 ): (absPath: string, isDir?: boolean) => boolean {
   return (absPath: string, isDir?: boolean) => {
     if (!underlying(absPath, isDir)) return false;
-    const rel = path.relative(syncRoot, absPath).split(path.sep).join("/");
+    const rel = vaultKeyForLocalPath(syncRoot, absPath);
     if (rel === "" || rel.startsWith("..")) return true; // root / outside — defer
     if (isDir) {
       if (isDirInScope(rel, prefixSet)) return true;
@@ -819,7 +820,7 @@ export function wrapFilterWithIgnoreVisibility(
     if (allowed) return true;
 
     onAnyExcluded?.();
-    const rel = path.relative(hqRoot, absPath).split(path.sep).join("/");
+    const rel = vaultKeyForLocalPath(hqRoot, absPath);
     if (rel === "" || rel.startsWith("..")) return false;
     if (!isExpectedIgnore(rel)) onIgnoreExcluded(rel);
     return false;
@@ -1456,13 +1457,13 @@ async function writePushConflictMirror(
   try {
     const detectedAt = new Date().toISOString();
     const machineId = readShortMachineId(run.hqRoot);
-    const originalRelative = path.relative(run.hqRoot, item.absolutePath);
+    const originalRelative = vaultKeyForLocalPath(run.hqRoot, item.absolutePath);
     const conflictRelative = buildConflictPath(
       originalRelative,
       detectedAt,
       machineId,
     );
-    const conflictAbs = path.join(run.hqRoot, conflictRelative);
+    const conflictAbs = localPathForVaultKey(run.hqRoot, conflictRelative);
     if (!isMaterializationPathStillContained(run.syncRoot, conflictAbs)) {
       run.emit({
         type: "error",
@@ -1787,7 +1788,7 @@ function collectFiles(
         console.error(`  Warning: ${p} is outside company folder, skipping.`);
         continue;
       }
-      const relativePath = toPosixKey(path.relative(syncRoot, absolutePath));
+      const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
       // Probe the filter with both isDir hints — we don't know whether
       // the link's target is a file or a directory without
       // stat-following the link, which we explicitly avoid (it would
@@ -1817,7 +1818,7 @@ function collectFiles(
       if (!filter(absolutePath, true)) continue;
       results.push(...walkDir(absolutePath, syncRoot, filter));
     } else if (lstat.isFile()) {
-      const relativePath = toPosixKey(path.relative(syncRoot, absolutePath));
+      const relativePath = vaultKeyForLocalPath(syncRoot, absolutePath);
       if (filter(absolutePath)) {
         results.push({ kind: "file", absolutePath, relativePath });
       }
@@ -1867,7 +1868,7 @@ function walkDir(
       results.push({
         kind: "symlink",
         absolutePath,
-        relativePath: toPosixKey(path.relative(syncRoot, absolutePath)),
+        relativePath: vaultKeyForLocalPath(syncRoot, absolutePath),
         target: fs.readlinkSync(absolutePath),
       });
       continue;
@@ -1883,7 +1884,7 @@ function walkDir(
       results.push({
         kind: "file",
         absolutePath,
-        relativePath: toPosixKey(path.relative(syncRoot, absolutePath)),
+        relativePath: vaultKeyForLocalPath(syncRoot, absolutePath),
       });
     }
   }
@@ -2002,11 +2003,11 @@ function resolveDeleteScopeRoots(
     if (!isWithin(syncRoot, absolutePath)) continue;
     const stat = fs.statSync(absolutePath);
     if (!stat.isDirectory()) continue;
-    const rel = path.relative(syncRoot, absolutePath);
+    const rel = vaultKeyForLocalPath(syncRoot, absolutePath);
     if (rel === "" || rel === ".") {
       return [""];
     }
-    prefixes.add(rel.split(path.sep).join("/"));
+    prefixes.add(rel);
   }
   return Array.from(prefixes);
 }
@@ -2244,7 +2245,7 @@ async function computeDeletePlan(
     );
     if (!inScope) continue;
     inScopeJournalEntries++;
-    const localPath = path.join(syncRoot, relativeKey);
+    const localPath = localPathForVaultKey(syncRoot, relativeKey);
     let presentLocally = true;
     try {
       fs.lstatSync(localPath);

package/src/cli/sync.test.ts

@@ -2566,6 +2566,68 @@ describe("sync", () => {
     expect(journal.files["knowledge/readme.md"].remoteEtag).toBe("def456");
   });
 
+  it("checkpoints completed downloads so a failed bulk pass resumes only its tail", async () => {
+    process.env.HQ_SYNC_TRANSFER_CONCURRENCY = "1";
+    const remoteFiles = Array.from({ length: 9 }, (_, index) => ({
+      key: `bulk/file-${index}.md`,
+      size: 10,
+      lastModified: new Date(),
+      etag: `"bulk-${index}"`,
+    }));
+    const defaultDownload = vi.mocked(s3Module.downloadFile).getMockImplementation();
+    if (!defaultDownload) throw new Error("missing default download mock");
+
+    try {
+      vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
+      vi.mocked(s3Module.downloadFile).mockImplementation(
+        async (ctx, key, localPath) => {
+          if (key === "bulk/file-8.md") {
+            // This models the next transfer crashing the process. The first
+            // eight entries must already be durable before the final writer.
+            const checkpoint = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+            expect(Object.keys(checkpoint.files)).toHaveLength(8);
+            throw new Error("simulated crash after checkpoint");
+          }
+          return defaultDownload(ctx, key, localPath);
+        },
+      );
+
+      const first = await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipReindex: true,
+        onEvent: () => undefined,
+      });
+      expect(first.filesDownloaded).toBe(8);
+
+      const checkpoint = JSON.parse(fs.readFileSync(journalPath, "utf-8"));
+      expect(Object.keys(checkpoint.files)).toHaveLength(8);
+
+      vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
+      vi.mocked(s3Module.downloadFile).mockClear();
+      vi.mocked(s3Module.listRemoteFiles).mockResolvedValueOnce(remoteFiles);
+
+      await sync({
+        company: "acme",
+        vaultConfig: mockConfig,
+        hqRoot: tmpDir,
+        skipReindex: true,
+        onEvent: () => undefined,
+      });
+
+      expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledTimes(1);
+      expect(vi.mocked(s3Module.downloadFile)).toHaveBeenCalledWith(
+        expect.anything(),
+        "bulk/file-8.md",
+        expect.any(String),
+      );
+    } finally {
+      vi.mocked(s3Module.downloadFile).mockImplementation(defaultDownload);
+      delete process.env.HQ_SYNC_TRANSFER_CONCURRENCY;
+    }
+  });
+
   // ── Stage-1 plan event ─────────────────────────────────────────────────
 
   it("emits a plan event before any progress events", async () => {

package/src/cli/sync.ts

@@ -19,6 +19,7 @@ import {
 } from "../telemetry-events.js";
 import { resolveEntityContext, isExpiringSoon, refreshEntityContext } from "../context.js";
 import { createSyncProgressRecorder } from "../sync-progress.js";
+import { localPathForVaultKey, vaultKeyForLocalPath } from "../local-path-codec.js";
 import {
   downloadFile,
   listRemoteFiles,
@@ -26,7 +27,7 @@ import {
   primeObjectTransport,
   toPosixKey,
 } from "../s3.js";
-import type { RemoteFile } from "../s3.js";
+import type { DownloadModeWarning, RemoteFile } from "../s3.js";
 import {
   readJournal,
   writeJournal,
@@ -601,6 +602,8 @@ interface PullRunContext {
    */
   currentExcludeSet: string[];
   fileTombstones: ReadonlyMap<string, CompanyTombstone>;
+  /** Successful downloads since the last durable journal checkpoint. */
+  downloadsSinceJournalCheckpoint: number;
 }
 
 interface PullCounters {
@@ -646,6 +649,8 @@ export function resolveAutoPruneCap(): number {
 
 /** Max time to wait on the best-effort new-files notification POST. */
 const NOTIFY_FILE_ADDED_TIMEOUT_MS = 5000;
+/** Bound crash recovery to at most this many completed pull downloads. */
+const DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE = 8;
 
 /**
  * Server cap on files per `/v1/notify/file-added` report. The endpoint rejects
@@ -914,6 +919,7 @@ async function buildPullContext(options: SyncOptions): Promise<PullRunContext> {
     currentPrefixSet,
     currentExcludeSet,
     fileTombstones,
+    downloadsSinceJournalCheckpoint: 0,
   };
 }
 
@@ -971,6 +977,30 @@ function reportInvalidScopedKeys(run: PullRunContext, plan: PullPlan): void {
   }, { claims: run.options.telemetryClaims });
 }
 
+/**
+ * A legacy object without hq-mode can otherwise downgrade a hook to the
+ * receiver umask with no durable signal. downloadFile already prints a local
+ * warning for every caller; the sync path also records a path-free telemetry
+ * event so operators can find vaults that still need the one-time backfill.
+ */
+function reportModeWarnings(
+  run: PullRunContext,
+  warnings: DownloadModeWarning[] | undefined,
+): void {
+  for (const warning of warnings ?? []) {
+    void emitCloudTelemetry(new VaultClient(run.vaultConfig), {
+      eventName: "sync_mode_guardrail_warning",
+      source: "hq-sync",
+      companyUid: run.ctx.uid,
+      properties: {
+        leg: "pull",
+        reason: warning.reason,
+        fallback: warning.fallback ?? null,
+      },
+    }, { claims: run.options.telemetryClaims });
+  }
+}
+
 function createPullCounters(): PullCounters {
   return {
     filesDownloaded: 0,
@@ -1220,7 +1250,7 @@ async function executeConflictItem(
   // the REMOTE hash/etag) and skip the conflict path entirely. This removes both
   // the `.conflict-` mirror loop AND the journal false-stamp (remote etag over
   // divergent local content) that the keep path produced for these files.
-  if (isCloudAuthoritative(path.relative(run.hqRoot, localPath))) {
+  if (isCloudAuthoritative(vaultKeyForLocalPath(run.hqRoot, localPath))) {
     downloadItems.push({ action: "download", remoteFile, localPath, isNew: false });
     run.emit({ type: "reconciled", path: remoteFile.key, direction: "pull" });
     return null;
@@ -1230,14 +1260,14 @@ async function executeConflictItem(
 
   const detectedAt = new Date().toISOString();
   const machineId = readShortMachineId(run.hqRoot);
-  const originalRelative = path.relative(run.hqRoot, localPath);
+  const originalRelative = vaultKeyForLocalPath(run.hqRoot, localPath);
   const conflictRelative = buildConflictPath(
     originalRelative,
     detectedAt,
     machineId,
   );
-  const conflictAbs = path.join(run.hqRoot, conflictRelative);
-  const conflictKey = toPosixKey(path.relative(run.companyRoot, conflictAbs));
+  const conflictAbs = localPathForVaultKey(run.hqRoot, conflictRelative);
+  const conflictKey = vaultKeyForLocalPath(run.companyRoot, conflictAbs);
 
   if (!isDownloadWritePathStillContained(run.companyRoot, conflictKey, conflictAbs)) {
     counters.filesSkipped++;
@@ -1459,11 +1489,12 @@ async function downloadOne(
   }
 
   try {
-    const { metadata, contentHash, contentSize } = await downloadFile(
+    const { metadata, contentHash, contentSize, modeWarnings } = await downloadFile(
       run.ctx,
       remoteFile.key,
       localPath,
     );
+    reportModeWarnings(run, modeWarnings);
     const author = metadata?.["created-by"] ?? null;
     const createdBySub = metadata?.["created-by-sub"];
 
@@ -1508,6 +1539,19 @@ async function downloadOne(
         message: err instanceof Error ? err.message : String(err),
       });
     }
+    return;
+  }
+
+  run.downloadsSinceJournalCheckpoint++;
+  if (
+    run.downloadsSinceJournalCheckpoint >=
+    DOWNLOAD_JOURNAL_CHECKPOINT_BATCH_SIZE
+  ) {
+    // writeJournal is already fsync + rename atomic. Persist completed work
+    // before scheduling more objects so an abrupt process exit only replays a
+    // small tail, not an entire bulk download.
+    writeJournal(run.journalSlug, run.journal);
+    run.downloadsSinceJournalCheckpoint = 0;
   }
 }
 
@@ -1803,7 +1847,7 @@ function resolveContainedVaultPath(root: string, key: string): string | null {
   if (isMalformedVaultKey(key) || hasTraversalSegment(key)) return null;
 
   const resolvedRoot = path.resolve(root);
-  const resolvedLocal = path.resolve(resolvedRoot, key);
+  const resolvedLocal = localPathForVaultKey(resolvedRoot, key);
   if (!isPathWithin(resolvedRoot, resolvedLocal)) return null;
 
   let realRoot: string;

package/src/local-path-codec.ts

@@ -0,0 +1,95 @@
+/**
+ * Reversible local filename adapter for canonical vault keys on Win32.
+ *
+ * Vault keys are POSIX identifiers and may contain ':' (for example
+ * `factory:slack`).  Win32 cannot materialize those strings as filename
+ * segments, so keep the key canonical in S3 and journals while encoding only
+ * the local segment that reaches the filesystem.
+ */
+import * as path from "path";
+
+const WIN32_RESERVED_CHARS = /[<>:"/\\|?*]/;
+const WIN32_DEVICE_NAME = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\..*)?$/i;
+
+function isWin32ReservedCharacter(char: string): boolean {
+  return char.charCodeAt(0) <= 0x1f || WIN32_RESERVED_CHARS.test(char);
+}
+
+function percentEscape(char: string): string {
+  return `%${char.charCodeAt(0).toString(16).toUpperCase().padStart(2, "0")}`;
+}
+
+/**
+ * Encode one canonical vault-key segment for a Win32 filename. Percent is
+ * escaped before every other character so decoding cannot collide with a
+ * literal `%3A` name. The optional flag makes platform-specific behavior
+ * directly testable without mutating `process.platform`.
+ */
+export function encodeLocalVaultSegment(
+  segment: string,
+  win32: boolean = process.platform === "win32",
+): string {
+  if (!win32) return segment;
+
+  let encoded = "";
+  for (let index = 0; index < segment.length; index++) {
+    const char = segment[index]!;
+    const isTrailingDotOrSpace =
+      index === segment.length - 1 && (char === "." || char === " ");
+    const isDotSegment = (segment === "." || segment === "..") && index === 0;
+    const isReservedDeviceFirstChar = WIN32_DEVICE_NAME.test(segment) && index === 0;
+    if (
+      char === "%" ||
+      isWin32ReservedCharacter(char) ||
+      isTrailingDotOrSpace ||
+      isDotSegment ||
+      isReservedDeviceFirstChar
+    ) {
+      encoded += percentEscape(char);
+    } else {
+      encoded += char;
+    }
+  }
+  return encoded;
+}
+
+/** Decode one local filename segment back to its canonical vault-key form. */
+export function decodeLocalVaultSegment(
+  segment: string,
+  win32: boolean = process.platform === "win32",
+): string {
+  if (!win32) return segment;
+  return segment.replace(/%([0-9a-f]{2})/gi, (_whole, hex: string) =>
+    String.fromCharCode(Number.parseInt(hex, 16)),
+  );
+}
+
+/** Materialize a canonical slash-delimited vault key beneath a local root. */
+export function localPathForVaultKey(
+  root: string,
+  key: string,
+  win32: boolean = process.platform === "win32",
+): string {
+  return path.join(
+    root,
+    ...key.split("/").map((segment) => encodeLocalVaultSegment(segment, win32)),
+  );
+}
+
+/**
+ * Turn a local, root-relative path back into its canonical vault key. This
+ * recognizes legacy percent-encoded Windows materializations as well as the
+ * current codec, so they converge on one remote/journal key instead of
+ * creating duplicates.
+ */
+export function vaultKeyForLocalPath(
+  root: string,
+  localPath: string,
+  win32: boolean = process.platform === "win32",
+): string {
+  return path
+    .relative(root, localPath)
+    .split(path.sep)
+    .map((segment) => decodeLocalVaultSegment(segment, win32))
+    .join("/");
+}