@indigoai-us/hq-cloud 5.22.0 → 5.24.0

package/src/scope-shrink.test.ts

@@ -0,0 +1,402 @@
+/**
+ * Scope-shrink detection + classification + clean removal (US-005).
+ *
+ * Pins the hybrid-contract semantics decided in US-000 Task 3:
+ *   - orphans = (covered by last scope) ∧ ¬(covered by current scope)
+ *   - tombstones are NOT re-flagged on subsequent pulls
+ *   - clean orphan: local missing OR hash matches + mtime ≤ syncedAt
+ *   - dirty orphan: anything else
+ *   - applyScopeShrink deletes clean orphans, leaves dirty files alone
+ *     when `forceScopeShrink: true` but tombstones the journal entry
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import * as crypto from "crypto";
+import {
+  buildScopeShrinkPlan,
+  applyScopeShrink,
+  ScopeShrinkBlockedError,
+} from "./scope-shrink.js";
+import type { SyncJournal } from "./types.js";
+
+function sha256(content: string): string {
+  return crypto.createHash("sha256").update(content).digest("hex");
+}
+
+function emptyJournal(): SyncJournal {
+  return { version: "2", lastSync: "", files: {}, pulls: [] };
+}
+
+describe("buildScopeShrinkPlan", () => {
+  let hqRoot: string;
+
+  beforeEach(() => {
+    hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-scope-shrink-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(hqRoot, { recursive: true, force: true });
+  });
+
+  it("returns empty plan when no orphans exist", () => {
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/meetings/a.md": {
+          hash: "h",
+          size: 1,
+          syncedAt: "2026-05-01T00:00:00.000Z",
+          direction: "down",
+        },
+      },
+    };
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/"],
+    });
+    expect(plan.orphans).toEqual([]);
+    expect(plan.scopeChangeDetected).toBe(false);
+  });
+
+  it("flags files covered by lastPrefixSet but not currentPrefixSet as orphans", () => {
+    const meetingsAbs = path.join(
+      hqRoot,
+      "companies/indigo/meetings/a.md",
+    );
+    const scratchAbs = path.join(
+      hqRoot,
+      "companies/indigo/scratch/jacob/draft.md",
+    );
+    fs.mkdirSync(path.dirname(meetingsAbs), { recursive: true });
+    fs.mkdirSync(path.dirname(scratchAbs), { recursive: true });
+    fs.writeFileSync(meetingsAbs, "meetings");
+    fs.writeFileSync(scratchAbs, "draft");
+
+    const now = new Date().toISOString();
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/meetings/a.md": {
+          hash: sha256("meetings"),
+          size: 8,
+          syncedAt: now,
+          direction: "down",
+        },
+        "companies/indigo/scratch/jacob/draft.md": {
+          hash: sha256("draft"),
+          size: 5,
+          syncedAt: now,
+          direction: "down",
+        },
+      },
+    };
+    // Backdate the scratch file so mtime ≤ syncedAt.
+    const past = Date.now() - 60_000;
+    fs.utimesSync(scratchAbs, past / 1000, past / 1000);
+    fs.utimesSync(meetingsAbs, past / 1000, past / 1000);
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.orphans.map((o) => o.path)).toEqual([
+      "companies/indigo/scratch/jacob/draft.md",
+    ]);
+    expect(plan.clean).toHaveLength(1);
+    expect(plan.dirty).toHaveLength(0);
+    expect(plan.scopeChangeDetected).toBe(true);
+  });
+
+  it("classifies a locally modified orphan as dirty (hash mismatch)", () => {
+    const abs = path.join(hqRoot, "companies/indigo/scratch/notes.md");
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "ORIGINAL");
+
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/notes.md": {
+          hash: sha256("DIFFERENT"), // doesn't match what's on disk
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+    };
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.dirty).toHaveLength(1);
+    expect(plan.dirty[0]?.dirtyReason).toBe("hash-mismatch");
+  });
+
+  it("classifies a recently mtime-touched file as dirty", () => {
+    const abs = path.join(hqRoot, "companies/indigo/scratch/notes.md");
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "ORIGINAL");
+
+    const syncedAt = new Date(Date.now() - 60_000).toISOString();
+    // Touch mtime forward, well past syncedAt + 1s grace.
+    const future = Date.now() + 10_000;
+    fs.utimesSync(abs, future / 1000, future / 1000);
+
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/notes.md": {
+          hash: sha256("ORIGINAL"), // hash matches, but mtime is forward
+          size: 8,
+          syncedAt,
+          direction: "down",
+        },
+      },
+    };
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.dirty).toHaveLength(1);
+    expect(plan.dirty[0]?.dirtyReason).toBe("modified-after-sync");
+  });
+
+  it("treats a locally-missing orphan as clean (already removed by user)", () => {
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/gone.md": {
+          hash: sha256("gone"),
+          size: 4,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+    };
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.clean).toHaveLength(1);
+    expect(plan.clean[0]?.path).toBe("companies/indigo/scratch/gone.md");
+  });
+
+  it("skips tombstoned entries (do not re-flag on subsequent pulls)", () => {
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/already.md": {
+          hash: "h",
+          size: 1,
+          syncedAt: "2026-05-01T00:00:00.000Z",
+          direction: "down",
+          removedAt: "2026-05-02T00:00:00.000Z",
+          removedReason: "scope_shrink",
+        },
+      },
+    };
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.orphans).toEqual([]);
+  });
+
+  it("skips push-only entries (direction: 'up')", () => {
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/local-only.md": {
+          hash: "h",
+          size: 1,
+          syncedAt: "2026-05-01T00:00:00.000Z",
+          direction: "up",
+        },
+      },
+    };
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    expect(plan.orphans).toEqual([]);
+  });
+});
+
+describe("applyScopeShrink", () => {
+  let hqRoot: string;
+
+  beforeEach(() => {
+    hqRoot = fs.mkdtempSync(path.join(os.tmpdir(), "hq-scope-shrink-apply-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(hqRoot, { recursive: true, force: true });
+  });
+
+  it("deletes clean orphans on disk + tombstones their journal entries", () => {
+    const abs = path.join(hqRoot, "companies/indigo/scratch/clean.md");
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "clean");
+
+    const syncedAt = new Date().toISOString();
+    const past = Date.now() - 60_000;
+    fs.utimesSync(abs, past / 1000, past / 1000);
+
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/clean.md": {
+          hash: sha256("clean"),
+          size: 5,
+          syncedAt,
+          direction: "down",
+        },
+      },
+    };
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    const result = applyScopeShrink({
+      journal,
+      plan,
+      hqRoot,
+      forceScopeShrink: false,
+    });
+
+    expect(result.cleanRemoved).toBe(1);
+    expect(result.dirtyTombstoned).toBe(0);
+    expect(fs.existsSync(abs)).toBe(false);
+    expect(
+      journal.files["companies/indigo/scratch/clean.md"]?.removedAt,
+    ).toBeTruthy();
+    expect(
+      journal.files["companies/indigo/scratch/clean.md"]?.removedReason,
+    ).toBe("scope_shrink");
+  });
+
+  it("leaves dirty files on disk when forceScopeShrink: true, but tombstones the entry", () => {
+    const abs = path.join(hqRoot, "companies/indigo/scratch/dirty.md");
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "MODIFIED");
+
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/dirty.md": {
+          hash: sha256("ORIGINAL"), // mismatch — dirty
+          size: 8,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+    };
+
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    const result = applyScopeShrink({
+      journal,
+      plan,
+      hqRoot,
+      forceScopeShrink: true,
+    });
+
+    expect(result.dirtyTombstoned).toBe(1);
+    expect(fs.existsSync(abs)).toBe(true); // PRESERVED
+    expect(fs.readFileSync(abs, "utf-8")).toBe("MODIFIED");
+    expect(
+      journal.files["companies/indigo/scratch/dirty.md"]?.removedAt,
+    ).toBeTruthy();
+  });
+
+  it("accepts a custom reason for the tombstone (e.g. narrow_apply)", () => {
+    const abs = path.join(hqRoot, "companies/indigo/scratch/clean.md");
+    fs.mkdirSync(path.dirname(abs), { recursive: true });
+    fs.writeFileSync(abs, "x");
+    const past = Date.now() - 60_000;
+    fs.utimesSync(abs, past / 1000, past / 1000);
+
+    const journal: SyncJournal = {
+      ...emptyJournal(),
+      files: {
+        "companies/indigo/scratch/clean.md": {
+          hash: sha256("x"),
+          size: 1,
+          syncedAt: new Date().toISOString(),
+          direction: "down",
+        },
+      },
+    };
+    const plan = buildScopeShrinkPlan({
+      journal,
+      hqRoot,
+      lastPrefixSet: ["companies/indigo/"],
+      currentPrefixSet: ["companies/indigo/meetings/"],
+    });
+    applyScopeShrink({
+      journal,
+      plan,
+      hqRoot,
+      forceScopeShrink: false,
+      reason: "narrow_apply",
+    });
+    expect(
+      journal.files["companies/indigo/scratch/clean.md"]?.removedReason,
+    ).toBe("narrow_apply");
+  });
+});
+
+describe("ScopeShrinkBlockedError", () => {
+  it("carries from/to syncMode + dirty/clean orphan lists for the CLI", () => {
+    const err = new ScopeShrinkBlockedError(
+      "cmp_indigo",
+      "all",
+      "shared",
+      [
+        {
+          path: "companies/indigo/scratch/notes.md",
+          entry: {
+            hash: "h",
+            size: 1,
+            syncedAt: "",
+            direction: "down",
+          },
+          clean: false,
+          dirtyReason: "hash-mismatch",
+        },
+      ],
+      [],
+    );
+    expect(err.code).toBe("SCOPE_SHRINK_BLOCKED");
+    expect(err.fromMode).toBe("all");
+    expect(err.toMode).toBe("shared");
+    expect(err.dirty).toHaveLength(1);
+    expect(err.name).toBe("ScopeShrinkBlockedError");
+  });
+});

package/src/scope-shrink.ts

@@ -0,0 +1,264 @@
+/**
+ * Scope-shrink detection + classification + clean removal (US-005).
+ *
+ * Implements the "hybrid scope-change contract" decided in US-000 Task 3
+ * (see companies/indigo/projects/hq-sync-browse-vs-sync/references.md):
+ *
+ *   - Compare the current pull's `prefixSet` against the last `PullRecord`'s
+ *     `prefixSet` for the same company. Files in the journal covered by the
+ *     previous scope but NOT covered by the new scope are **orphans**.
+ *   - Classify each orphan **clean** (safe to silently delete) or **dirty**
+ *     (locally modified — sacred, never silently delete).
+ *   - The remote-pull caller drives:
+ *       * default mode: abort the leg if any dirty orphan exists;
+ *       * `--force-scope-shrink`: continue, leave dirty files on disk,
+ *         tombstone their journal entries.
+ *
+ * The pure-detection layer here intentionally does NOT touch disk for the
+ * tombstone write — that lives in `journal.ts`. It DOES touch disk for the
+ * orphan classification (hash + stat) because cleanliness is a function of
+ * the file's current on-disk state vs the journal.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import type {
+  JournalEntry,
+  PullRecord,
+  SyncJournal,
+} from "./types.js";
+import { hashFile, tombstoneEntry } from "./journal.js";
+import { isCoveredByAny } from "./prefix-coalesce.js";
+
+export interface OrphanClassification {
+  /** Relative path (journal key). */
+  path: string;
+  /** Journal entry as of last sync. */
+  entry: JournalEntry;
+  /** True iff the local file is provably unchanged since last sync. */
+  clean: boolean;
+  /** Why we called it dirty — surfaced in the abort error for operators. */
+  dirtyReason?:
+    | "modified-after-sync"
+    | "hash-mismatch"
+    | "stat-error";
+}
+
+export interface ScopeShrinkPlan {
+  /** Set of files covered by `lastPrefixSet` but not by `currentPrefixSet`. */
+  orphans: OrphanClassification[];
+  /** Subset of `orphans` with `clean === true`. */
+  clean: OrphanClassification[];
+  /** Subset of `orphans` with `clean === false`. */
+  dirty: OrphanClassification[];
+  /** True iff at least one orphan was found. */
+  scopeChangeDetected: boolean;
+}
+
+export interface BuildScopeShrinkPlanInput {
+  journal: SyncJournal;
+  hqRoot: string;
+  /** Coalesced prefixes used by the LAST pull for this company. */
+  lastPrefixSet: string[];
+  /** Coalesced prefixes the CURRENT pull will use. */
+  currentPrefixSet: string[];
+}
+
+/**
+ * Build a scope-shrink plan: find orphans, classify each clean/dirty.
+ * Pure given the journal + filesystem state — no network, no journal
+ * mutation.
+ *
+ * **Tombstone-aware:** journal entries that already carry a `removedAt`
+ * marker are skipped — they represent a prior scope-shrink prune and must
+ * not be re-flagged as orphans on each subsequent pull (that's the whole
+ * point of the tombstone retention window).
+ *
+ * **Direction-aware:** only `direction: "down"` entries (and pre-ETag
+ * legacy entries without an explicit direction marker) participate in
+ * shrink detection. Push-only files (`direction: "up"`) represent local
+ * authorship — they aren't in scope-as-pulled, so a scope change doesn't
+ * orphan them.
+ */
+export function buildScopeShrinkPlan(
+  input: BuildScopeShrinkPlanInput,
+): ScopeShrinkPlan {
+  const { journal, hqRoot, lastPrefixSet, currentPrefixSet } = input;
+  const orphans: OrphanClassification[] = [];
+
+  for (const [relPath, entry] of Object.entries(journal.files)) {
+    if (entry.removedAt) continue; // tombstone — already pruned
+    if (entry.direction !== "down") continue;
+    if (!isCoveredByAny(relPath, lastPrefixSet)) continue;
+    if (isCoveredByAny(relPath, currentPrefixSet)) continue;
+    orphans.push(classifyOrphan(relPath, entry, hqRoot));
+  }
+
+  const clean = orphans.filter((o) => o.clean);
+  const dirty = orphans.filter((o) => !o.clean);
+  return {
+    orphans,
+    clean,
+    dirty,
+    scopeChangeDetected: orphans.length > 0,
+  };
+}
+
+/**
+ * Classify a single orphan:
+ *
+ *   - **Clean** when:
+ *       * the local file is missing (already removed by the user — harmless), OR
+ *       * `sha256(localFile) === entry.hash` AND `stat.mtime ≤ entry.syncedAt`.
+ *   - **Dirty** otherwise.
+ *
+ * Symlinks: we don't re-hash with the symlink-target convention here —
+ * the safer default is to treat any symlink whose lstat exists but whose
+ * target hash doesn't match `entry.hash` (via `hashFile` reading the
+ * target) as dirty. In practice symlinks materialize through the same
+ * code path as files; a stale-target symlink is correctly flagged dirty
+ * and the operator-facing message points at the path either way.
+ */
+function classifyOrphan(
+  relPath: string,
+  entry: JournalEntry,
+  hqRoot: string,
+): OrphanClassification {
+  const absPath = path.join(hqRoot, relPath);
+  let stat: fs.Stats;
+  try {
+    stat = fs.lstatSync(absPath);
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code;
+    if (code === "ENOENT") {
+      return { path: relPath, entry, clean: true };
+    }
+    return {
+      path: relPath,
+      entry,
+      clean: false,
+      dirtyReason: "stat-error",
+    };
+  }
+
+  // mtime guard: a local edit moves mtime past syncedAt. We use ≤ because
+  // a download stamps syncedAt at the close of the write; mtime is set by
+  // the OS before that, so an unmodified pulled file has mtime ≤ syncedAt.
+  const mtimeMs = stat.mtimeMs;
+  const syncedAtMs = Date.parse(entry.syncedAt);
+  if (!Number.isNaN(syncedAtMs) && mtimeMs > syncedAtMs + 1000) {
+    // 1s grace for filesystem clock jitter.
+    return {
+      path: relPath,
+      entry,
+      clean: false,
+      dirtyReason: "modified-after-sync",
+    };
+  }
+
+  // Hash check — final word. If the content matches the journaled hash,
+  // the file is provably what the last pull left there.
+  let actualHash: string;
+  try {
+    actualHash = hashFile(absPath);
+  } catch {
+    return {
+      path: relPath,
+      entry,
+      clean: false,
+      dirtyReason: "stat-error",
+    };
+  }
+  if (actualHash !== entry.hash) {
+    return {
+      path: relPath,
+      entry,
+      clean: false,
+      dirtyReason: "hash-mismatch",
+    };
+  }
+  return { path: relPath, entry, clean: true };
+}
+
+/**
+ * Structured error thrown when the engine refuses to proceed because a scope
+ * shrink would orphan dirty files. The CLI catches this and renders the
+ * operator-facing message; the engine never prints directly.
+ */
+export class ScopeShrinkBlockedError extends Error {
+  readonly code = "SCOPE_SHRINK_BLOCKED";
+  constructor(
+    public readonly companyUid: string,
+    public readonly fromMode: PullRecord["syncMode"] | "unknown",
+    public readonly toMode: PullRecord["syncMode"],
+    public readonly dirty: OrphanClassification[],
+    public readonly clean: OrphanClassification[],
+  ) {
+    super(
+      `Sync scope shrank for ${companyUid} (${fromMode} → ${toMode}); ` +
+        `${dirty.length} dirty file(s) outside the new scope would be ` +
+        `pruned from the journal. Resolve or pass --force-scope-shrink.`,
+    );
+    this.name = "ScopeShrinkBlockedError";
+  }
+}
+
+export interface ApplyScopeShrinkInput {
+  journal: SyncJournal;
+  plan: ScopeShrinkPlan;
+  hqRoot: string;
+  /**
+   * When `true`, dirty files are LEFT ON DISK and their journal entries are
+   * tombstoned anyway. When `false` (default), the caller should have
+   * already aborted on dirty orphans — this function still tombstones any
+   * dirty entries handed to it, on the assumption the caller knows what
+   * it's doing.
+   */
+  forceScopeShrink: boolean;
+  reason?: "scope_shrink" | "narrow_apply" | "manual";
+}
+
+export interface ApplyScopeShrinkResult {
+  cleanRemoved: number;
+  dirtyTombstoned: number;
+}
+
+/**
+ * Apply a scope-shrink plan: delete clean orphans on disk + tombstone their
+ * journal entries. With `forceScopeShrink: true`, dirty orphans are
+ * preserved on disk but their journal entries are also tombstoned.
+ *
+ * Returns counts for the audit log row (`scope_shrink_blocked` /
+ * `scope_shrink_forced`).
+ */
+export function applyScopeShrink(
+  input: ApplyScopeShrinkInput,
+): ApplyScopeShrinkResult {
+  const { journal, plan, hqRoot, forceScopeShrink } = input;
+  const reason = input.reason ?? "scope_shrink";
+  let cleanRemoved = 0;
+  let dirtyTombstoned = 0;
+
+  for (const orphan of plan.clean) {
+    const absPath = path.join(hqRoot, orphan.path);
+    try {
+      fs.unlinkSync(absPath);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      if (code !== "ENOENT") throw err; // missing-on-disk is fine; anything else escalates
+    }
+    tombstoneEntry(journal, orphan.path, reason);
+    cleanRemoved++;
+  }
+
+  if (forceScopeShrink) {
+    for (const orphan of plan.dirty) {
+      // Do NOT delete the file — that's the entire point of the `--force`
+      // contract: keep dirty content on disk, prune only the journal entry.
+      tombstoneEntry(journal, orphan.path, reason);
+      dirtyTombstoned++;
+    }
+  }
+
+  return { cleanRemoved, dirtyTombstoned };
+}