@indigoai-us/hq-cloud 6.4.0 → 6.6.0

package/src/cli/sync.ts

@@ -40,6 +40,7 @@ import {
   applyScopeShrink,
   ScopeShrinkBlockedError,
   ScopeShrinkLargePruneError,
+  type ScopeShrinkAdviceContext,
 } from "../scope-shrink.js";
 import { coalescePrefixes, isCoveredByAny } from "../prefix-coalesce.js";
 import { createIgnoreFilter } from "../ignore.js";
@@ -355,6 +356,25 @@ export interface SyncOptions {
    * tombstoned. Mirrors `hq sync narrow --force`.
    */
   forceScopeShrink?: boolean;
+  /**
+   * How `sync()` handles a scope shrink (US-005 / DEV-1768):
+   *
+   *   - `"block"` (default) — a human is present (foreground `hq sync`). Dirty
+   *     out-of-scope orphans, or a clean prune over the safety cap, raise a
+   *     structured error whose advice is followable from a terminal. Clean
+   *     orphans within the cap are QUARANTINED (moved, not deleted).
+   *   - `"auto-recover"` — the background menubar runner, which can take no
+   *     interactive flag. NEVER throws on a shrink: dirty orphans are kept on
+   *     disk + un-tracked, clean orphans are quarantined, and the bulk-prune
+   *     cap is bypassed (quarantine is non-destructive). This is what clears an
+   *     already-wedged journal on the next sync, idempotently and without data
+   *     loss — the recovery seam for the all→shared seed bug.
+   *
+   * Both policies are non-destructive for CLEAN files (quarantine, never
+   * silent delete) — the deliberate `hq sync narrow --apply` ritual is the only
+   * path that hard-deletes, and it confirms first.
+   */
+  scopeShrinkPolicy?: "block" | "auto-recover";
   /**
    * The caller's own Cognito `sub`, used by the scope-shrink authorship guard
    * so a scope shrink never prunes content the caller authored. Injected by the
@@ -655,23 +675,38 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     // explicit `hq sync narrow` ritual opts out of the unknown-author shield.
     protectUnknownAuthors: true,
   });
-  if (shrinkPlan.dirty.length > 0 && options.forceScopeShrink !== true) {
+  // Policy: the background menubar runner ("auto-recover") can take no
+  // interactive flag, so it must never throw on a shrink — it self-heals
+  // non-destructively (dirty kept on disk + un-tracked, clean quarantined).
+  // A foreground `hq sync` ("block", the default) keeps the protective gate
+  // but renders FOLLOWABLE advice. `autoRecover` implies force (proceed) and
+  // bypasses the bulk-prune cap (quarantine is non-destructive, so a large
+  // recovery move is safe). DEV-1768.
+  const scopeShrinkPolicy = options.scopeShrinkPolicy ?? "block";
+  const autoRecover = scopeShrinkPolicy === "auto-recover";
+  const adviceContext: ScopeShrinkAdviceContext = autoRecover ? "runner" : "cli";
+  const effectiveForce = options.forceScopeShrink === true || autoRecover;
+
+  if (shrinkPlan.dirty.length > 0 && !effectiveForce) {
     throw new ScopeShrinkBlockedError(
       ctx.uid,
       lastRecord?.syncMode ?? "unknown",
       syncMode,
       shrinkPlan.dirty,
       shrinkPlan.clean,
+      adviceContext,
     );
   }
-  // Bulk-delete guard: refuse to auto-prune more than the safety cap of CLEAN
-  // files in a single background sync. A deliberate large narrow goes through
-  // `hq sync narrow --apply` (its own confirmation), and `--force-scope-shrink`
-  // (or raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited (opt
-  // out). The engine deletes nothing when it throws here.
+  // Bulk guard: refuse to auto-move more than the safety cap of CLEAN files in
+  // a single foreground sync. A deliberate large narrow goes through
+  // `hq sync narrow --apply` (its own confirmation); `--force-scope-shrink` (or
+  // raising HQ_SYNC_MAX_AUTO_PRUNE) overrides. Cap of 0 = unlimited. Skipped
+  // under auto-recover — quarantine is non-destructive so a big recovery is
+  // safe, and the runner has no way to act on a thrown cap. The engine moves
+  // nothing when it throws here.
   const autoPruneCap = resolveAutoPruneCap();
   if (
-    options.forceScopeShrink !== true &&
+    !effectiveForce &&
     autoPruneCap > 0 &&
     shrinkPlan.clean.length > autoPruneCap
   ) {
@@ -680,27 +715,65 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
       syncMode,
       shrinkPlan.clean.length,
       autoPruneCap,
+      adviceContext,
     );
   }
+  // Clean orphans are QUARANTINED (moved into `.hq/scope-quarantine/<slug>/`,
+  // recoverable), never silently deleted — a background sync purging local
+  // files unannounced was DEV-1768 fix #3. The quarantine root lives under the
+  // real HQ root's `.hq/` (outside `companyRoot` and never pushed), so moved
+  // files don't round-trip back through S3.
+  const scopeQuarantineRoot = path.join(
+    hqRoot,
+    ".hq",
+    "scope-quarantine",
+    journalSlug,
+  );
   const shrinkResult = applyScopeShrink({
     journal,
     plan: shrinkPlan,
     hqRoot: companyRoot,
-    forceScopeShrink: options.forceScopeShrink === true,
+    forceScopeShrink: effectiveForce,
     reason: "scope_shrink",
+    cleanDisposition: "quarantine",
+    quarantineRoot: scopeQuarantineRoot,
   });
-  // Surface each removed clean orphan as a `deleted` progress event so the
-  // menubar stream renders the prune the same way it renders a cross-machine
-  // tombstone (the Rust parser already handles `deleted: true`).
-  for (const orphan of shrinkPlan.clean) {
+  // Surface each affected orphan explicitly (named path) so the prune is never
+  // silent. Quarantined clean files render as `deleted: true` (removed from the
+  // working tree, recoverable in quarantine); dirty files KEPT on disk render
+  // as a non-deletion notice so the operator knows they were un-tracked, not
+  // removed. The Rust menubar parser already handles `deleted: true`.
+  for (const relPath of shrinkResult.quarantinedPaths) {
+    emit({
+      type: "progress",
+      path: relPath,
+      bytes: 0,
+      deleted: true,
+      message: `scope-narrowed: moved out-of-scope copy to ${scopeQuarantineRoot}`,
+    });
+  }
+  for (const relPath of shrinkResult.removedPaths) {
     emit({
       type: "progress",
-      path: orphan.path,
+      path: relPath,
       bytes: 0,
       deleted: true,
       message: "scope-narrowed (removed local copy outside sync scope)",
     });
   }
+  for (const relPath of shrinkResult.dirtyKeptPaths) {
+    emit({
+      type: "progress",
+      path: relPath,
+      bytes: 0,
+      message:
+        "scope-narrowed: locally-modified file KEPT on disk, un-tracked from sync (outside scope)",
+    });
+  }
+  // "Removed from the working tree" = deleted OR quarantined; both vacate the
+  // file's original path. Reported as `scopeOrphansRemoved` for back-compat.
+  const scopeOrphansRemoved =
+    shrinkResult.cleanRemoved + shrinkResult.cleanQuarantined;
 
   // Stage 2: execute the plan. Per-item branching mirrors the pre-refactor
   // inline loop; the only structural change is that classification has
@@ -949,7 +1022,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
           // a conflict abort. `filesOutOfScope` reflects how far the serial
           // pass got before the abort; that's acceptable for an abort result.
           filesOutOfScope,
-          scopeOrphansRemoved: shrinkResult.cleanRemoved,
+          scopeOrphansRemoved,
           scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
         };
         break;
@@ -1328,7 +1401,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     syncMode,
     prefixSet: currentPrefixSet,
     scopeChangeDetected: shrinkPlan.scopeChangeDetected,
-    orphansRemoved: shrinkResult.cleanRemoved,
+    orphansRemoved: scopeOrphansRemoved,
     orphansBlocked: shrinkResult.dirtyTombstoned,
   });
 
@@ -1347,7 +1420,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
   const changedOnDisk =
     filesDownloaded > 0 ||
     filesTombstoned > 0 ||
-    shrinkResult.cleanRemoved > 0;
+    scopeOrphansRemoved > 0;
   if (!options.skipReindex && changedOnDisk) {
     try {
       // skipLock: the surrounding sync run already holds this root's operation
@@ -1370,7 +1443,7 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     filesExcludedByPolicy: plan.filesExcludedByPolicy,
     filesTombstoned,
     filesOutOfScope,
-    scopeOrphansRemoved: shrinkResult.cleanRemoved,
+    scopeOrphansRemoved,
     scopeOrphansBlocked: shrinkResult.dirtyTombstoned,
   };
 }

package/src/index.ts

@@ -69,6 +69,7 @@ export {
   buildScopeShrinkPlan,
   applyScopeShrink,
   ScopeShrinkBlockedError,
+  ScopeShrinkLargePruneError,
 } from "./scope-shrink.js";
 export type {
   OrphanClassification,
@@ -76,6 +77,8 @@ export type {
   BuildScopeShrinkPlanInput,
   ApplyScopeShrinkInput,
   ApplyScopeShrinkResult,
+  ScopeShrinkAdviceContext,
+  CleanOrphanDisposition,
 } from "./scope-shrink.js";
 
 // Engine-layer ACL-aware pull orchestration (US-005)
@@ -118,6 +121,14 @@ export {
 } from "./cognito-auth.js";
 export type { CognitoAuthConfig, CognitoTokens } from "./cognito-auth.js";
 
+// Per-company PULL scope resolver (US-005) — shared between hq-sync-runner and
+// `hq sync pull|now` (hq-cli). Exported so hq-cli's foreground pull paths resolve
+// the SAME effective scope the menubar runner does, instead of defaulting every
+// CLI pull to `syncMode: "all"` (the seed of the all→shared scope-shrink wedge,
+// DEV-1768).
+export { resolvePullScope, readPinnedPrefixes } from "./sync/pull-scope.js";
+export type { PullScope, PullScopeClient } from "./sync/pull-scope.js";
+
 // Personal-vault scope helpers — shared between hq-sync-runner and `hq sync`
 export {
   PERSONAL_VAULT_EXCLUDED_TOP_LEVEL,

package/src/s3.test.ts

@@ -6,7 +6,7 @@
  * `Metadata`, so the listing's HEAD fan-out had nothing to attribute.
  */
 
-import { describe, it, expect, beforeEach, vi } from "vitest";
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
@@ -83,6 +83,7 @@ import {
   uploadFile,
   uploadSymlink,
   toPosixKey,
+  primeUploads,
   downloadFile,
   listRemoteFiles,
   SYMLINK_BODY_PREFIX,
@@ -92,6 +93,12 @@ import {
   FILE_MTIME_META_KEY,
   FILE_BTIME_META_KEY,
 } from "./s3.js";
+import {
+  setObjectIOFactory,
+  presignObjectIOFactory,
+  type PresignTransportClient,
+} from "./object-io.js";
+import type { PresignResultRow } from "./vault-client.js";
 import type { EntityContext } from "./types.js";
 
 function makeCtx(): EntityContext {
@@ -158,6 +165,89 @@ describe("backslash key normalization (Windows client → POSIX S3 key)", () =>
   });
 });
 
+describe("backslash key normalization on the PRESIGNED (GA) transport", () => {
+  // The tests above assert normalization on the S3-SDK transport (sentCommands
+  // capture SDK PutObjectCommands). As of 6.5.0 the presigned-URL transport is
+  // GA for every company vault (cmp_*) — that's the path a real Windows client
+  // like Ridge's now takes. These lock the SAME write-side invariant on that
+  // transport: a backslash key is canonicalized to POSIX before the server is
+  // ever asked to presign it, so a non-POSIX vault key can never be minted.
+  let tmpFile: string;
+  let presignCalls: Array<{ op?: string; keys: Array<{ key?: string }> }>;
+
+  function presignKeysFor(op: string): string[] {
+    return presignCalls
+      .filter((c) => c.op === op)
+      .flatMap((c) => c.keys.map((k) => k.key ?? ""));
+  }
+
+  beforeEach(() => {
+    tmpFile = path.join(
+      os.tmpdir(),
+      `s3-presign-backslash-${Date.now()}-${Math.random()}.md`,
+    );
+    fs.writeFileSync(tmpFile, "hello");
+
+    presignCalls = [];
+    const vault: PresignTransportClient = {
+      // Echo each requested key back as a usable URL so putObject's PUT
+      // resolves; record the op + keys so we can assert what was signed.
+      presign: async (input) => {
+        presignCalls.push({ op: input.op, keys: input.keys as Array<{ key?: string }> });
+        const results: PresignResultRow[] = input.keys.map((k) => ({
+          key: (k as { key: string }).key,
+          op: input.op ?? "put",
+          url: `https://s3.test/${(k as { key: string }).key}`,
+        }));
+        return { results, expiresAt: "2099-01-01T00:00:00.000Z" };
+      },
+      listFiles: async () => ({ objects: [], cursor: null, truncated: false }),
+    };
+    setObjectIOFactory(presignObjectIOFactory(vault));
+    // putObject moves bytes over the presigned URL via global fetch.
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => new Response(null, { status: 200, headers: { etag: '"e"' } })),
+    );
+  });
+
+  afterEach(() => {
+    setObjectIOFactory(null);
+    vi.unstubAllGlobals();
+  });
+
+  it("uploadFile presigns + PUTs a POSIX key for a Windows-style backslash path", async () => {
+    await uploadFile(makeCtx(), tmpFile, "knowledge\\books-eoi.md");
+    expect(presignKeysFor("put")).toContain("knowledge/books-eoi.md");
+    // The malformed backslash key is never presigned (so never PUT).
+    expect(presignKeysFor("put")).not.toContain("knowledge\\books-eoi.md");
+  });
+
+  it("uploadSymlink presigns a POSIX key for a Windows-style backslash path", async () => {
+    await uploadSymlink(makeCtx(), "../target.md", "data\\boots-accounts.json");
+    expect(presignKeysFor("put")).toContain("data/boots-accounts.json");
+    expect(presignKeysFor("put")).not.toContain("data\\boots-accounts.json");
+  });
+
+  it("primeUploads canonicalizes keys so the primed PUT URL matches the upload lookup", async () => {
+    // Prime then upload the SAME Windows-origin key. Both must key the cache
+    // under the POSIX form: prime signs the POSIX PUT URL, and uploadFile's
+    // hasPrimedPut(toPosixKey(...)) reuses it instead of re-presigning — proving
+    // the primed fast-path can't strand a backslash key (and can't store one).
+    await primeUploads(makeCtx(), [
+      { key: "knowledge\\books-eoi.md", localPath: tmpFile, isSymlink: false },
+    ]);
+    // prime("get") + prime("put") both went out under the POSIX key only.
+    expect(presignKeysFor("get")).toEqual(["knowledge/books-eoi.md"]);
+    expect(presignKeysFor("put")).toEqual(["knowledge/books-eoi.md"]);
+
+    const putCountAfterPrime = presignKeysFor("put").length;
+    await uploadFile(makeCtx(), tmpFile, "knowledge\\books-eoi.md");
+    // No NEW put presign: uploadFile reused the primed POSIX URL (cache hit).
+    expect(presignKeysFor("put").length).toBe(putCountAfterPrime);
+  });
+});
+
 describe("uploadFile", () => {
   let tmpFile: string;
 

package/src/s3.ts

@@ -339,9 +339,14 @@ export async function primeUploads(
   if (!io.prime || items.length === 0) return;
 
   // Prime GET first so each item's created-at HEAD reuses a cached URL.
+  // Canonicalize to POSIX here (one-canonical-form, matching the uploadFile /
+  // uploadSymlink boundary): a Windows-origin backslash key must cache under
+  // the SAME key uploadFile later looks up via hasPrimedPut(toPosixKey(...)),
+  // or the primed URL silently misses and the upload re-presigns. It also
+  // keeps the created-at HEAD pointed at the real (POSIX) object.
   await io.prime(
     "get",
-    items.map((i) => ({ key: i.key })),
+    items.map((i) => ({ key: toPosixKey(i.key) })),
   );
 
   // Build per-key PUT metadata with the SAME builders the upload path uses,
@@ -356,10 +361,15 @@ export async function primeUploads(
   const worker = async (): Promise<void> => {
     while (next < items.length) {
       const it = items[next++];
-      const createdAt = await resolveCreatedAt(io, it.key, it.author);
+      // Same boundary guardrail as uploadFile/uploadSymlink: prime under the
+      // canonical POSIX key so the cached PUT URL is keyed identically to the
+      // hasPrimedPut/putObject lookup, and a backslash key can never be primed
+      // (let alone stored) as a non-POSIX vault key.
+      const key = toPosixKey(it.key);
+      const createdAt = await resolveCreatedAt(io, key, it.author);
       if (it.isSymlink) {
         putKeys.push({
-          key: it.key,
+          key,
           contentType: "application/octet-stream",
           metadata: {
             [SYMLINK_TARGET_META_KEY]: SYMLINK_MARKER_META_VALUE,
@@ -374,8 +384,8 @@ export async function primeUploads(
           // raced rm / EPERM — leave stamps off (receiver umask default).
         }
         putKeys.push({
-          key: it.key,
-          contentType: getMimeType(it.key),
+          key,
+          contentType: getMimeType(key),
           metadata: {
             ...(it.author ? buildAuthorMetadata(it.author, createdAt) : {}),
             ...modeTime,

package/src/scope-shrink.test.ts

@@ -498,6 +498,52 @@ describe("applyScopeShrink", () => {
       journal.files["companies/indigo/scratch/clean.md"]?.removedReason,
     ).toBe("narrow_apply");
   });
+
+  // DEV-1768 fix #3: clean orphans are MOVED to quarantine (recoverable),
+  // not unlinked, when `cleanDisposition: "quarantine"`.
+  it("quarantines clean orphans (moves, not deletes) and reports named paths", () => {
+    const rel = "companies/indigo/scratch/clean.md";
+    const abs = path.join(hqRoot, rel);
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "clean");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(abs, past / 1000, past / 1000);
+
+    const quarantineRoot = path.join(hqRoot, ".hq", "scope-quarantine", "indigo");
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        [rel]: { hash: sha256("clean"), size: 5, syncedAt: new Date().toISOString(), direction: "down" },
+      },
+    };
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    const result = applyScopeShrink({
+      journal,
+      plan,
+      hqRoot,
+      forceScopeShrink: false,
+      cleanDisposition: "quarantine",
+      quarantineRoot,
+    });
+
+    expect(result.cleanQuarantined).toBe(1);
+    expect(result.cleanRemoved).toBe(0);
+    expect(result.quarantinedPaths).toEqual([rel]);
+    expect(result.removedPaths).toEqual([]);
+    expect(result.quarantineRoot).toBe(quarantineRoot);
+    // Moved, not deleted: gone from the working tree, present + intact in quarantine.
+    expect(fs.existsSync(abs)).toBe(false);
+    const quarantined = path.join(quarantineRoot, rel);
+    expect(fs.existsSync(quarantined)).toBe(true);
+    expect(fs.readFileSync(quarantined, "utf-8")).toBe("clean");
+    // Journal entry tombstoned so it stops being re-flagged (idempotent).
+    expect(journal.files[rel]?.removedAt).toBeTruthy();
+  });
 });
 
 describe("ScopeShrinkBlockedError", () => {
@@ -527,4 +573,29 @@ describe("ScopeShrinkBlockedError", () => {
     expect(err.dirty).toHaveLength(1);
     expect(err.name).toBe("ScopeShrinkBlockedError");
   });
+
+  // DEV-1768 fix #2: advice must be FOLLOWABLE from the entry point that throws.
+  // The old message always said "pass --force-scope-shrink", which the menubar
+  // runner cannot accept.
+  const mk = (ctx?: "cli" | "runner" | "engine") =>
+    new ScopeShrinkBlockedError("cmp_x", "all", "shared", [], [], ctx);
+
+  it("runner-context advice points at a terminal command, NOT an impossible flag", () => {
+    const msg = mk("runner").message;
+    // The followable path from the menubar is a terminal command.
+    expect(msg).toContain("hq sync narrow --apply");
+    // It must NOT imply the flag is acceptable to the menubar itself.
+    expect(msg).toContain("menubar sync cannot take this flag");
+  });
+
+  it("cli-context advice offers the flag the CLI actually accepts", () => {
+    const msg = mk("cli").message;
+    expect(msg).toContain("--force-scope-shrink");
+    expect(msg).toContain("hq sync narrow --apply");
+  });
+
+  it("defaults to the engine/library advice when no context is given", () => {
+    expect(mk().adviceContext).toBe("engine");
+    expect(mk().message).toContain("hq sync narrow --apply");
+  });
 });