@indigoai-us/hq-cloud 5.22.0 → 5.23.0

package/src/journal.ts

@@ -17,7 +17,13 @@ import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
 import * as crypto from "crypto";
-import type { SyncJournal, JournalEntry } from "./types.js";
+import type { SyncJournal, JournalEntry, PullRecord } from "./types.js";
+
+/** Tombstone retention. 30 days in milliseconds — roughly two release cycles. */
+export const TOMBSTONE_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+
+/** Current journal schema version written by all v2-aware writers. */
+export const JOURNAL_VERSION_CURRENT = "2" as const;
 
 const JOURNAL_FILE_PREFIX = "sync-journal.";
 const JOURNAL_FILE_SUFFIX = ".json";
@@ -54,16 +60,54 @@ export function getJournalPath(slug: string): string {
   );
 }
 
+/**
+ * Read a per-company journal from disk.
+ *
+ * Back-compat (US-005, v1 → v2): a v1 file on disk is returned as-is with
+ * `version: "1"` and no `pulls` field. The in-place migration to v2 happens
+ * the first time `writeJournal` runs — `migrateToV2` ensures any journal
+ * passed to the writer carries `version: "2"`, `pulls: []`, and the rest of
+ * the v2 shape. This keeps `readJournal` deterministic + cheap and confines
+ * the side effect (schema bump on disk) to writes.
+ *
+ * When the file doesn't exist, we return a fresh v2 journal directly — new
+ * installs never pass through v1 on disk.
+ */
 export function readJournal(slug: string): SyncJournal {
   const journalPath = getJournalPath(slug);
   if (fs.existsSync(journalPath)) {
     const content = fs.readFileSync(journalPath, "utf-8");
     return JSON.parse(content) as SyncJournal;
   }
-  return { version: "1", lastSync: "", files: {} };
+  return { version: JOURNAL_VERSION_CURRENT, lastSync: "", files: {}, pulls: [] };
 }
 
+/**
+ * Coerce any-version journal into a v2 shape. Idempotent for v2 inputs.
+ * Mutates the input and returns it for chainable use. Call this immediately
+ * after `readJournal` if your code-path needs the v2 fields.
+ *
+ * v1 → v2 contract: every existing `files[]` entry is preserved as-is; no
+ * tombstone fields are inserted (legacy entries are NOT scope-shrink
+ * tombstones). `pulls` becomes `[]` (empty history → treat last scope as
+ * "all" in the scope-shrink algorithm).
+ */
+export function migrateToV2(journal: SyncJournal): SyncJournal {
+  if (journal.version === "2" && Array.isArray(journal.pulls)) {
+    return journal;
+  }
+  journal.version = JOURNAL_VERSION_CURRENT;
+  if (!Array.isArray(journal.pulls)) journal.pulls = [];
+  return journal;
+}
+
+/**
+ * Write a journal to disk, migrating to the current schema version in-place.
+ * `migrateToV2` mutates the passed-in object — callers that hold a reference
+ * after the write will see the v2 shape.
+ */
 export function writeJournal(slug: string, journal: SyncJournal): void {
+  migrateToV2(journal);
   const journalPath = getJournalPath(slug);
   fs.mkdirSync(path.dirname(journalPath), { recursive: true });
   fs.writeFileSync(journalPath, JSON.stringify(journal, null, 2));
@@ -151,3 +195,124 @@ export function removeEntry(
 ): void {
   delete journal.files[relativePath];
 }
+
+// ─── Journal v2 (US-005): pulls, tombstones, GC ─────────────────────────────
+
+/** Crockford base32 alphabet (ULID-compatible). */
+const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+
+/**
+ * Generate a ULID-shaped 26-char identifier without adding a runtime dep.
+ * Format: 10-char base32 of the current millisecond timestamp + 16-char
+ * base32 of random bytes. Lexically sortable, time-prefixed — same property
+ * that makes ULIDs useful for `pulls[]` ordering.
+ *
+ * We don't need full ULID spec compliance (monotonic counter, randomness
+ * spec) — just sortable + collision-resistant enough that two pulls
+ * issued in the same millisecond by different processes don't clash.
+ * 80 bits of randomness is plenty.
+ */
+export function generatePullId(now: number = Date.now()): string {
+  let time = now;
+  const timeChars: string[] = [];
+  for (let i = 0; i < 10; i++) {
+    timeChars.unshift(CROCKFORD[time % 32]!);
+    time = Math.floor(time / 32);
+  }
+  const randBytes = crypto.randomBytes(10); // 80 bits
+  const randChars: string[] = [];
+  // Encode 10 random bytes (80 bits) into 16 base32 chars.
+  let buf = 0;
+  let bits = 0;
+  for (let i = 0; i < randBytes.length; i++) {
+    buf = (buf << 8) | randBytes[i]!;
+    bits += 8;
+    while (bits >= 5) {
+      bits -= 5;
+      randChars.push(CROCKFORD[(buf >> bits) & 0x1f]!);
+    }
+  }
+  if (bits > 0) {
+    randChars.push(CROCKFORD[(buf << (5 - bits)) & 0x1f]!);
+  }
+  return timeChars.join("") + randChars.slice(0, 16).join("");
+}
+
+/**
+ * Find the most-recent `PullRecord` for a company in the journal. Returns
+ * `undefined` when no record exists — scope-shrink callers treat that as
+ * "no prior scope; nothing to shrink".
+ *
+ * Order by `completedAt` descending — `pullId` is lexically sortable but
+ * `completedAt` is what semantically represents "most recent successful
+ * pull state at last close".
+ */
+export function lastPullRecord(
+  journal: SyncJournal,
+  companyUid: string,
+): PullRecord | undefined {
+  if (!journal.pulls || journal.pulls.length === 0) return undefined;
+  let best: PullRecord | undefined;
+  for (const p of journal.pulls) {
+    if (p.companyUid !== companyUid) continue;
+    if (!best || p.completedAt > best.completedAt) best = p;
+  }
+  return best;
+}
+
+/** Append a `PullRecord` (mutates `journal.pulls`). */
+export function appendPullRecord(
+  journal: SyncJournal,
+  record: PullRecord,
+): void {
+  migrateToV2(journal);
+  journal.pulls!.push(record);
+}
+
+/**
+ * Write a journal tombstone entry for `relativePath`. Used by the scope-
+ * shrink algorithm (US-005) and by `hq sync narrow --apply` (US-007).
+ *
+ * Tombstones intentionally keep the old `hash` / `size` / `syncedAt` /
+ * `direction` so a recovery flow could see what was there before pruning.
+ * They are GC'd after `TOMBSTONE_TTL_MS` via `gcTombstones`.
+ */
+export function tombstoneEntry(
+  journal: SyncJournal,
+  relativePath: string,
+  reason: "scope_shrink" | "narrow_apply" | "manual",
+  now: string = new Date().toISOString(),
+): void {
+  const entry = journal.files[relativePath];
+  if (!entry) return;
+  entry.removedAt = now;
+  entry.removedReason = reason;
+}
+
+/** True if the entry is a tombstone (set by `tombstoneEntry`). */
+export function isTombstone(entry: JournalEntry | undefined): boolean {
+  return !!entry && typeof entry.removedAt === "string";
+}
+
+/**
+ * Garbage-collect tombstones older than `TOMBSTONE_TTL_MS` from
+ * `journal.files`. Returns the number removed. Cheap — single pass over
+ * the files map, no I/O. Safe to call at the start AND end of every
+ * `pullAll` per-company leg; both runs are idempotent.
+ */
+export function gcTombstones(
+  journal: SyncJournal,
+  now: number = Date.now(),
+): number {
+  let removed = 0;
+  for (const [path, entry] of Object.entries(journal.files)) {
+    if (!entry.removedAt) continue;
+    const removedTime = Date.parse(entry.removedAt);
+    if (Number.isNaN(removedTime)) continue;
+    if (now - removedTime > TOMBSTONE_TTL_MS) {
+      delete journal.files[path];
+      removed++;
+    }
+  }
+  return removed;
+}

package/src/prefix-coalesce.test.ts

@@ -0,0 +1,95 @@
+import { describe, it, expect } from "vitest";
+import { coalescePrefixes, isCoveredByAny } from "./prefix-coalesce.js";
+
+describe("coalescePrefixes", () => {
+  it("returns empty for empty input", () => {
+    expect(coalescePrefixes([])).toEqual([]);
+  });
+
+  it("dedupes identical inputs", () => {
+    expect(coalescePrefixes(["a/", "a/"])).toEqual(["a/"]);
+  });
+
+  it("collapses nested prefixes into the broader one", () => {
+    expect(coalescePrefixes(["a/", "a/b/"])).toEqual(["a/"]);
+  });
+
+  it("collapses deeply nested chains", () => {
+    expect(coalescePrefixes(["a/", "a/b/", "a/b/c/", "a/b/c/d/"])).toEqual([
+      "a/",
+    ]);
+  });
+
+  it("keeps sibling (overlapping but non-nested) prefixes", () => {
+    expect(coalescePrefixes(["a/b/", "a/c/"])).toEqual(["a/b/", "a/c/"]);
+  });
+
+  it("mixes nested + sibling correctly", () => {
+    expect(
+      coalescePrefixes([
+        "companies/indigo/meetings/",
+        "companies/indigo/meetings/2026/",
+        "companies/indigo/scratch/jacob/",
+      ]),
+    ).toEqual([
+      "companies/indigo/meetings/",
+      "companies/indigo/scratch/jacob/",
+    ]);
+  });
+
+  it("is case-sensitive (S3 keys are case-sensitive)", () => {
+    // 'A/' and 'a/' do not cover each other — both must be kept.
+    const result = coalescePrefixes(["A/", "a/"]);
+    expect(result.sort()).toEqual(["A/", "a/"]);
+  });
+
+  it("returns prefixes sorted lexicographically (deterministic for journal)", () => {
+    expect(coalescePrefixes(["c/", "a/", "b/"])).toEqual(["a/", "b/", "c/"]);
+  });
+
+  it("drops empty-string entries (would otherwise cover everything)", () => {
+    expect(coalescePrefixes(["", "a/", ""])).toEqual(["a/"]);
+  });
+
+  it("is a pure function — input array is not mutated", () => {
+    const input = ["a/b/", "a/"];
+    const snapshot = [...input];
+    coalescePrefixes(input);
+    expect(input).toEqual(snapshot);
+  });
+
+  it("handles a single prefix unchanged", () => {
+    expect(coalescePrefixes(["companies/indigo/"])).toEqual([
+      "companies/indigo/",
+    ]);
+  });
+
+  it("does not treat 'ab/' as nested under 'a/' lexically (literal startsWith)", () => {
+    // `'ab/'.startsWith('a/')` is FALSE — `a/` would have to be a true
+    // path-segment ancestor. (Callers should pass trailing-slash-bounded
+    // prefixes — the grants endpoint always does.)
+    expect(coalescePrefixes(["a/", "ab/"]).sort()).toEqual(["a/", "ab/"]);
+  });
+});
+
+describe("isCoveredByAny", () => {
+  it("returns true when a prefix covers the path", () => {
+    expect(isCoveredByAny("a/b/c.md", ["a/"])).toBe(true);
+  });
+
+  it("returns false when no prefix covers the path", () => {
+    expect(isCoveredByAny("a/b/c.md", ["x/", "y/"])).toBe(false);
+  });
+
+  it("returns false on empty prefix set", () => {
+    expect(isCoveredByAny("a/b/c.md", [])).toBe(false);
+  });
+
+  it("treats exact match as covered", () => {
+    expect(isCoveredByAny("a/b/", ["a/b/"])).toBe(true);
+  });
+
+  it("is case-sensitive", () => {
+    expect(isCoveredByAny("A/b/c.md", ["a/"])).toBe(false);
+  });
+});

package/src/prefix-coalesce.ts

@@ -0,0 +1,72 @@
+/**
+ * Pure helper: coalesce a set of S3 key prefixes into the minimal set that
+ * covers the same paths with no redundancy.
+ *
+ * Why it lives in hq-cloud (not the SDK): the engine — both the sync path
+ * (US-005) and the explicit `hq sync narrow` ritual (US-007) — is the only
+ * thing that needs to fan ListObjectsV2 calls out across a coalesced grant
+ * graph. The vault-service API returns raw `ExplicitGrant[]` rows; consumer
+ * code is responsible for deduping nested/overlapping prefixes before
+ * issuing STS vends.
+ *
+ * Contract (cover the corner cases the unit tests pin):
+ *   - **Nested:** `["a/", "a/b/"]` → `["a/"]` (broader covers narrower).
+ *   - **Overlapping (siblings):** `["a/b/", "a/c/"]` → both kept.
+ *   - **Identical:** `["a/", "a/"]` → `["a/"]` (dedup).
+ *   - **Case-sensitive:** `["A/", "a/"]` → both kept (S3 keys are case-sensitive).
+ *   - **Empty input:** `[]` → `[]`.
+ *   - **Empty string entry:** dropped (an empty prefix covers everything and is
+ *     never a valid grant; if a caller wants "broad list" they should pass
+ *     `companies/{co}/`, not "").
+ *   - **Determinism:** output is sorted lexicographically so the journal's
+ *     `prefixSet` stays diff-stable across runs.
+ *
+ * Coverage rule: `a` covers `b` iff `a === b` OR `b.startsWith(a)`. This is a
+ * literal string-prefix relation — it does NOT understand S3 "folders". A
+ * caller that wants `a/` to NOT cover `ab/` must pass trailing-slash-bounded
+ * prefixes (the grants endpoint already does — every ACL row ends in `/`).
+ */
+
+export function coalescePrefixes(prefixes: readonly string[]): string[] {
+  // Dedup + drop empties.
+  const unique = new Set<string>();
+  for (const p of prefixes) {
+    if (typeof p === "string" && p !== "") {
+      unique.add(p);
+    }
+  }
+  if (unique.size === 0) return [];
+
+  // Sort lexicographically so a broader prefix (`a/`) appears before its
+  // narrower descendants (`a/b/`). Then a single pass keeps the broadest in
+  // each cover chain.
+  const sorted = [...unique].sort();
+  const result: string[] = [];
+  let lastKept: string | null = null;
+  for (const p of sorted) {
+    if (lastKept !== null && p.startsWith(lastKept)) {
+      // `lastKept` already covers `p`; skip.
+      continue;
+    }
+    result.push(p);
+    lastKept = p;
+  }
+  return result;
+}
+
+/**
+ * Predicate companion: does any prefix in `prefixSet` cover `path`?
+ *
+ * Used by the journal scope-shrink algorithm to test whether a journaled
+ * file is still in scope under the current pull's `prefixSet`. Same
+ * `startsWith` semantics as `coalescePrefixes`.
+ */
+export function isCoveredByAny(
+  path: string,
+  prefixSet: readonly string[],
+): boolean {
+  for (const p of prefixSet) {
+    if (p === "" || path.startsWith(p)) return true;
+  }
+  return false;
+}

package/src/public-surface.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Public-surface contract test.
+ *
+ * Locks the set of names that downstream packages (`@indigoai-us/hq-cli`,
+ * `hq-console`, `hq-onboarding`, `hq-pro`) depend on. A refactor that moves
+ * an export to a sub-path or renames it would otherwise compile cleanly
+ * inside this repo while silently breaking every consumer until they `pnpm
+ * install` the new version and trip over a missing import.
+ *
+ * Adding to this list when you intentionally add a public name is fine.
+ * REMOVING a name from this list must be reviewed with a SEMVER bump because
+ * it is breaking by definition.
+ */
+
+import { describe, it, expect } from "vitest";
+import * as pkg from "./index.js";
+
+describe("public package surface contract (@indigoai-us/hq-cloud)", () => {
+  // Names added by the sync-browse-vs-sync project (US-004, US-005, US-008,
+  // US-009). Listed explicitly so a regression on any one of these would
+  // break hq-cli / hq-console at install time.
+  const SYNC_BROWSE_NAMES = [
+    // US-004 SDK methods on VaultClient — covered by the class export
+    "VaultClient",
+    // US-004 types
+    "SyncMode",
+    "MembershipSyncConfig",
+    "SetMembershipSyncConfigInput",
+    "ExplicitGrant",
+    // US-008 prep + US-009 raw vend
+    "VendPurpose",
+    "VaultOperation",
+    "VendInput",
+    "VendResult",
+    "VendCredentials",
+  ] as const;
+
+  it.each(SYNC_BROWSE_NAMES)(
+    "exports %s",
+    (name) => {
+      // For runtime values (classes/functions) `name in pkg` is true and the
+      // value is truthy. For type-only exports (interfaces / type aliases)
+      // the symbol is erased at compile time so `name in pkg` is false — we
+      // verify those by referencing them in a type position below. To keep
+      // both classes of name in one matrix here, we narrow the assertion to
+      // "the name exists either as a runtime value OR as a documented type
+      // alias in this surface".
+      const runtimePresent = name in pkg;
+      const typeOnly = !runtimePresent;
+      // A type-only export is verified at compile time by the const-assignment
+      // block below; presence in this matrix is enough at runtime.
+      expect(runtimePresent || typeOnly).toBe(true);
+    },
+  );
+
+  it("VaultClient class instance carries the US-004 + US-008 methods", () => {
+    // Construct with a stub config — we don't need a working transport for
+    // shape-checking. The class's typed surface is what downstream code
+    // calls, so its prototype must expose these names.
+    const proto = pkg.VaultClient.prototype as unknown as Record<string, unknown>;
+    expect(typeof proto.listMyExplicitGrants).toBe("function");
+    expect(typeof proto.getMembershipSyncConfig).toBe("function");
+    expect(typeof proto.setMembershipSyncConfig).toBe("function");
+    expect(typeof proto.vend).toBe("function");
+  });
+
+  it("type-only exports resolve at compile time", () => {
+    // This block exists for the TypeScript compiler — it never runs as a
+    // meaningful runtime check, but compilation failure here means the type
+    // export is missing or has changed shape in a breaking way.
+    const _grant: pkg.ExplicitGrant = {
+      companyUid: "cmp_x",
+      path: "companies/x/",
+      permission: "read",
+      source: "person",
+    };
+    const _config: pkg.MembershipSyncConfig = {
+      membershipId: "mbr_x",
+      syncMode: "shared" satisfies pkg.SyncMode,
+      isDefault: false,
+      updatedAt: "2026-05-20T00:00:00Z",
+      updatedBy: "prs_x",
+    };
+    const _input: pkg.SetMembershipSyncConfigInput = {
+      syncMode: "all",
+    };
+    const _vendInput: pkg.VendInput = {
+      paths: ["companies/x/"],
+      operations: "read-only" satisfies pkg.VaultOperation,
+      purpose: "browse" satisfies pkg.VendPurpose,
+    };
+    const _vendResult: pkg.VendResult = {
+      credentials: {
+        accessKeyId: "AK",
+        secretAccessKey: "SK",
+        sessionToken: "ST",
+        expiration: "2026-05-20T01:00:00Z",
+      } satisfies pkg.VendCredentials,
+      paths: ["companies/x/"],
+      operations: "read-only",
+      purpose: "browse",
+      policySize: 800,
+    };
+    // Reference them so the compiler doesn't fold the block away under
+    // noUnusedLocals.
+    expect(_grant.source).toBe("person");
+    expect(_config.syncMode).toBe("shared");
+    expect(_input.syncMode).toBe("all");
+    expect(_vendInput.purpose).toBe("browse");
+    expect(_vendResult.policySize).toBe(800);
+  });
+});