@indigoai-us/hq-cloud 6.15.0 → 6.15.2

package/src/ignore.ts

@@ -1,469 +0,0 @@
-/**
- * Ignore-file parser for cloud sync.
- *
- * Two modes:
- *   - **Permissive (default)**: everything syncs except what ignore layers
- *     subtract. Three layers stack (later overrides earlier):
- *       1. Built-in defaults — VCS, node_modules, build artifacts, caches,
- *          env files. Covers the common stacks so a first-time sync over a
- *          random project folder doesn't push `target/` or `.next/` to S3.
- *       2. Repo `.gitignore` at hqRoot — reuses existing exclusions so we
- *          don't re-list every build directory. Root-level only.
- *       3. `.hqignore` (preferred) or `.hqsyncignore` (legacy) — sync-specific
- *          overrides. Use `!pattern` to re-include something earlier layers
- *          excluded.
- *
- *   - **Allowlist**: triggered when `.hqinclude` exists at hqRoot. Nothing
- *     syncs unless its path matches at least one pattern in `.hqinclude`. The
- *     three exclusion layers still subtract on top — so even allowlisted
- *     subtrees won't push `node_modules/` or `.env`. Privacy-by-default for
- *     HQ trees that contain mixed personal + shareable data.
- *
- * SESSIONS ARE PUSH-ONLY, NOT PUSH-IGNORED (US-006). The company-vault
- * `sessions/{personUid}/...` prefix (session transcripts) is intentionally NOT
- * added to `DEFAULT_IGNORES`: push-only means it MUST still upload on the push
- * walk. Its never-AUTO-pull semantics are enforced ONE layer over, in the pull
- * scope resolver — `resolvePullScope().excludePrefixes` (src/sync/pull-scope.ts)
- * subtracts `sessions/` from the effective pull scope in EVERY mode (including
- * `all`), and `computePullPlan` classifies those keys `skip-out-of-scope`.
- * Ignoring `sessions/` HERE would wrongly block the upload too, so the split is
- * deliberate: this module governs the push walk; pull-scope governs never-pull.
- */
-
-import * as fs from "fs";
-import * as path from "path";
-import ignore from "ignore";
-
-// Patterns that must never sync regardless of project type.
-// Grouped by ecosystem so new stacks are easy to add.
-export const DEFAULT_IGNORES = [
-  // VCS + OS
-  ".git/",
-  ".git",
-  ".DS_Store",
-  "Thumbs.db",
-
-  // Node / JS
-  "node_modules/",
-  // pnpm's content-addressed package store. Large and machine-local like
-  // node_modules/; a synced store writes a `.conflict-` copy of every blob
-  // each cycle, which then re-syncs and amplifies into tens of thousands of
-  // files. Never round-trip it.
-  ".pnpm-store/",
-  "dist/",
-  "build/",
-  ".next/",
-  ".nuxt/",
-  ".svelte-kit/",
-  ".turbo/",
-  ".parcel-cache/",
-  ".vite/",
-  "coverage/",
-
-  // Rust / Tauri
-  "target/",
-
-  // Python
-  "__pycache__/",
-  "*.pyc",
-  ".pytest_cache/",
-  ".mypy_cache/",
-  ".ruff_cache/",
-  ".venv/",
-  "venv/",
-
-  // Go / JVM / other
-  "vendor/",
-  "out/",
-  "*.class",
-
-  // Generic caches / temp
-  ".cache/",
-  // Project-local qmd index (qmd init). Must never round-trip through the vault —
-  // sqlite-vec vec0 tables corrupt when copied across machines (feedback_332c7ccc /
-  // sqlite-vec#297). Default global index at ~/.cache/qmd is already outside hqRoot.
-  ".qmd/",
-  "tmp/",
-  ".tmp/",
-
-  // HQ sync internal state (never round-trip these). The `.hq-*` wildcard
-  // covers `.hq-sync.pid`, `.hq-sync-journal.json`, `.hq-sync-state.json`,
-  // `.hq-embeddings-pending.json`, and any future internal-state file. The
-  // `.hqignore` / `.hqsyncignore` / `.hqinclude` config files don't match
-  // (no hyphen). The `.hq/` directory holds per-host source-of-truth files
-  // (`install-manifest.json`, `machine-id`, `machine.json`, `config.json`)
-  // that MUST NEVER round-trip through the bucket — see Bug #1/#6/#8 in
-  // the 5.33.0 deep-test report. Leaking these between machines makes two
-  // hosts share an installer id / machine-id and breaks the contract the
-  // 5.33.0 machine-id fix relies on.
-  "*.pid",
-  ".hq-*",
-  ".hq/",
-  "modules.lock",
-  // NOTE: the root-anchored `/core.yaml` exclusion is intentionally NOT a
-  // static default — it is layout-dependent and added conditionally in
-  // `createIgnoreFilter` (see the `core/core.yaml` probe there). On v15+ the
-  // scaffold/version manifest lives at `core/core.yaml` and a root-level
-  // `core.yaml` is a stale per-machine duplicate that must not round-trip; on
-  // v12–v14 roots there is NO `core/core.yaml` and the ROOT `core.yaml` IS the
-  // scaffold/version manifest, which MUST sync (else the vault has no version
-  // marker and the file is undetectable across user installs).
-  // hq modules manifest — local module-resolution state, never synced.
-  "modules/modules.yaml",
-  // per-company identity file — written locally on first sync, never round-tripped.
-  "company.yaml",
-  // auto-generated tool index and policy digest — regenerated locally per-machine.
-  "INDEX.md",
-  "policies/_digest.md",
-
-  // Claude Code worktrees — local-only working copies, never synced.
-  "**/.claude/worktrees/",
-
-  // Claude Code per-machine session state — every active session writes
-  // to `.claude/state/active-session-project` and
-  // `.claude/state/auto-session-project-<sessionUuid>` independently on
-  // each host. Syncing them produces a guaranteed conflict per machine
-  // per session (~25 of the 30 mirrors written during the 5.34.0 live-
-  // sync test traced here). The `.claude/audit/` siblings are also
-  // per-host (audit log of hook decisions, suppressions, instructions).
-  // Both classes are session-/host-scoped by design and must never
-  // round-trip. Anchored under `**/.claude/` so the same rule covers
-  // hqRoot, company sub-dirs, and embedded knowledge repos uniformly —
-  // mirrors the `**/.claude/worktrees/` pattern just above.
-  "**/.claude/state/",
-  "**/.claude/audit/",
-
-  // Claude Code settings are machine-local: `env.PATH` is a snapshot of the
-  // host's shell PATH taken at install time, and model/permission prefs are
-  // per-operator-per-machine. Syncing them propagates one machine's PATH
-  // onto every other machine (and overwrites the installer's day-one PATH
-  // fix ~40s after a fresh install pulls existing cloud state) — the root
-  // cause of the recurring "qmd: command not found" reports (see
-  // workspace/e2e-mac/reports/20260701-qmd-path in HQ). Never round-trip
-  // them, in either direction.
-  "**/.claude/settings.json",
-  "**/.claude/settings.local.json",
-
-  // Skill hero images (skills-first-class US-014) — cloud-only artifacts stored
-  // under a skill's `.skill-media/` dir (e.g. `skills/<name>/.skill-media/hero.png`).
-  // They live in the vault/DB and are surfaced by the console, but must NEVER
-  // round-trip through the file sync onto agent machines: agents run the skill
-  // from SKILL.md and have no use for the image, and pulling it would litter
-  // every skill dir with binary blobs. Anchored `**/` so it matches at any depth
-  // (hqRoot skills, company skills, embedded knowledge repos) — excludes them
-  // from BOTH push and pull. Mirrors the `**/.claude/worktrees/` anchor shape.
-  "**/.skill-media/",
-
-  // HQ repos directory (managed separately, not synced). Root-anchored so
-  // only hqRoot/repos/ is excluded; bare `repos/` would match any nested
-  // directory named repos (for example companies/<co>/knowledge/repos/),
-  // silently dropping discover-generated knowledge (DEV-1791).
-  "/repos/",
-
-  // Secrets / env
-  ".env",
-  ".env.*",
-];
-
-function readIgnoreFile(filePath: string): string | null {
-  if (!fs.existsSync(filePath)) return null;
-  try {
-    return fs.readFileSync(filePath, "utf-8");
-  } catch {
-    return null;
-  }
-}
-
-/**
- * Compile a depth-anchored ancestor matcher from .hqinclude content.
- *
- * Git itself never asks "should I descend into this dir?" — it walks its
- * index and checks each file directly. Our walkers/watchers don't have an
- * index, so they have to make a descent decision per directory. To preserve
- * full gitignore semantics in allowlist mode, we must allow descent into
- * every ancestor of any include pattern. We can't reuse the `ignore` lib
- * for this because gitignore's `foo/*\/` would also match `foo/x/y/z/` —
- * recursing into the dir — which would defeat the privacy invariant.
- *
- * The matcher therefore checks an ancestor candidate against each include
- * pattern's prefix segments at the EXACT same depth, with `*` and `?`
- * resolved per-segment. `**` segments are treated as wildcards that match
- * any single segment here — sufficient for descent decisions, since the
- * include matcher itself still gates files.
- */
-function compileAncestorMatcher(
-  includeContent: string,
-): ((relDir: string) => boolean) | null {
-  const prefixes: RegExp[][] = [];
-  for (const raw of includeContent.split(/\r?\n/)) {
-    const line = raw.trim();
-    if (!line || line.startsWith("#") || line.startsWith("!")) continue;
-    const stripped = line.replace(/^\//, "").replace(/\/$/, "");
-    if (!stripped.includes("/")) continue;
-    const segs = stripped.split("/").map(segmentToRegex);
-    for (let i = 1; i < segs.length; i++) {
-      prefixes.push(segs.slice(0, i));
-    }
-  }
-  if (!prefixes.length) return null;
-  return (relDir: string): boolean => {
-    const parts = relDir.split("/");
-    for (const pat of prefixes) {
-      if (pat.length !== parts.length) continue;
-      let ok = true;
-      for (let i = 0; i < pat.length; i++) {
-        if (!pat[i].test(parts[i])) {
-          ok = false;
-          break;
-        }
-      }
-      if (ok) return true;
-    }
-    return false;
-  };
-}
-
-function segmentToRegex(seg: string): RegExp {
-  // Translate a single gitignore path segment to an anchored regex. `*` and
-  // `**` both match any single segment here (segments never contain `/`),
-  // `?` matches one char. Everything else is escaped literal.
-  let body = "";
-  for (let i = 0; i < seg.length; i++) {
-    const ch = seg[i];
-    if (ch === "*") {
-      if (seg[i + 1] === "*") i++;
-      body += "[^/]*";
-    } else if (ch === "?") {
-      body += "[^/]";
-    } else if (/[.+^${}()|[\]\\]/.test(ch)) {
-      body += "\\" + ch;
-    } else {
-      body += ch;
-    }
-  }
-  return new RegExp(`^${body}$`);
-}
-
-/**
- * Match a path whose basename is the macOS Finder custom-icon file: the
- * literal `Icon` followed by a carriage return (U+000D).
- *
- * This CANNOT be expressed as a `DEFAULT_IGNORES` entry, which is why it is a
- * predicate instead of a pattern string. Two independent reasons:
- *
- *   1. A pattern line ending in a literal CR is inert. The `ignore` package
- *      strips the trailing CR, so the line degrades to `Icon` and never
- *      matches — verified against ignore@5.3.2:
- *      `ignore().add("Icon\r\n").ignores("Icon\r") === false`. Git does the
- *      same thing (it reads the CR as part of the line ending), which is why
- *      the GitHub macOS template has to end that line with TWO CRs.
- *   2. The usual workaround pattern `Icon?` DOES match, but `?` is a
- *      single-character wildcard in gitignore syntax, so it also swallows any
- *      real file named `Icons`, `Icon1`, `IconX`. Silently dropping user
- *      content is worse than the cruft we are excluding.
- *
- * Matching the exact byte sequence keeps the exclusion precise. Finder writes
- * these into every directory it renders a custom icon for, and a folder-level
- * cloud-sync agent (iCloud Desktop, backup tooling) propagates them across an
- * entire tree — including into `.git/`. They carry no user content.
- *
- * They are also unsyncable by construction: a CR is a control character, so
- * the vault key validator rejects the key outright
- * (INVALID_KEY_CONTROL_CHARS). Excluding them at the walk is what keeps them
- * from reaching the upload path at all.
- */
-export function isMacFinderIconFile(relPath: string): boolean {
-  const basename = relPath.replace(/\/$/, "").split("/").pop() ?? "";
-  return basename === "Icon\r";
-}
-
-/**
- * True when any segment of `relPath` contains a character the vault key
- * validator rejects (see `KEY_CONTROL_CHARS` in s3.ts — C0 controls plus DEL).
- *
- * Kept in lock-step with that validator on purpose: a key this returns `true`
- * for can never be stored, so planning an upload for it only produces a
- * guaranteed per-file failure.
- */
-export function hasControlCharacters(relPath: string): boolean {
-  // eslint-disable-next-line no-control-regex
-  return /[\x00-\x1F\x7F]/.test(relPath);
-}
-
-export const EXPECTED_IGNORE_SEGMENTS = new Set([
-  ".git",
-  "node_modules",
-  ".pnpm-store",
-  "dist",
-  "build",
-  ".next",
-  ".nuxt",
-  ".svelte-kit",
-  ".turbo",
-  ".parcel-cache",
-  ".vite",
-  "coverage",
-  "target",
-  "__pycache__",
-  ".pytest_cache",
-  ".mypy_cache",
-  ".ruff_cache",
-  ".venv",
-  "venv",
-  "vendor",
-  "out",
-  ".cache",
-  ".qmd",
-  "tmp",
-  ".tmp",
-  ".hq",
-  ".claude",
-  // Cloud-only skill hero images (US-014) — never synced onto agent machines.
-  ".skill-media",
-]);
-
-/**
- * Classify a base-ignore rejection as expected noise vs noteworthy content.
- *
- * `true` means the ignored path belongs to an expected by-design exclusion
- * class: HQ-managed top-level roots, build/VCS/cache/machine-local segments,
- * or known generated/noise basenames. `false` means the rejected path looks
- * like real user content and should be surfaced by push observability.
- *
- * Top-level `repos/` and `workspace/` are expected, but nested segments named
- * `repos` or `workspace` are deliberately noteworthy: that is the DEV-1791
- * regression signal this classifier exists to preserve.
- */
-export function isExpectedIgnore(relPath: string): boolean {
-  if (
-    relPath === "repos" ||
-    relPath.startsWith("repos/") ||
-    relPath === "workspace" ||
-    relPath.startsWith("workspace/")
-  ) {
-    return true;
-  }
-
-  const segments = relPath.split("/");
-  for (const segment of segments) {
-    if (EXPECTED_IGNORE_SEGMENTS.has(segment)) return true;
-  }
-
-  const basename = segments[segments.length - 1] ?? "";
-  return (
-    basename === ".DS_Store" ||
-    basename === "Thumbs.db" ||
-    // Same class as .DS_Store: OS-generated, zero user content. Classified
-    // expected so a tree full of them stays silent instead of firing the
-    // "EXCLUDED from sync — review in case this is unintended" warning once
-    // per file (Finder scatters one into every directory it touches).
-    basename === "Icon\r" ||
-    basename === "company.yaml" ||
-    basename === "INDEX.md" ||
-    basename === "modules.lock" ||
-    basename.endsWith(".pyc") ||
-    basename.endsWith(".class") ||
-    basename.endsWith(".pid") ||
-    basename.startsWith(".hq-") ||
-    basename === ".env" ||
-    basename.startsWith(".env.") ||
-    basename === ".mcp.json"
-  );
-}
-
-export function createIgnoreFilter(
-  hqRoot: string,
-): (filePath: string, isDir?: boolean) => boolean {
-  const ig = ignore();
-
-  // Layer 1: baseline defaults
-  ig.add(DEFAULT_IGNORES);
-
-  // Layer 1b: layout-aware root `core.yaml` exclusion. The hq-core scaffold/
-  // version manifest moved location across releases: v15+ keeps it at
-  // `core/core.yaml` (which always syncs), v12–v14 keep it at the ROOT as
-  // `core.yaml`. We must sync whichever one is the real manifest, on every
-  // layout, so the vault always carries a version marker.
-  //   - v15+ (core/core.yaml present): exclude the root `/core.yaml` — it is a
-  //     stale/duplicate per-machine marker, never the authoritative manifest.
-  //   - v12–v14 (core/core.yaml absent): do NOT exclude root `core.yaml` — it
-  //     IS the manifest and must round-trip, else the file is undetectable
-  //     across user installs (the ~98 "no core.yaml in vault" users were all old-layout).
-  // Root-anchored (`/core.yaml`) so this never affects `core/core.yaml`.
-  if (fs.existsSync(path.join(hqRoot, "core", "core.yaml"))) {
-    ig.add("/core.yaml");
-  }
-
-  // Layer 2: repo's .gitignore (common case — covers most build dirs already)
-  const gitignore = readIgnoreFile(path.join(hqRoot, ".gitignore"));
-  if (gitignore) ig.add(gitignore);
-
-  // Layer 3: sync-specific overrides. .hqignore is the documented name;
-  // .hqsyncignore is the legacy name we still honor.
-  const hqignore =
-    readIgnoreFile(path.join(hqRoot, ".hqignore")) ??
-    readIgnoreFile(path.join(hqRoot, ".hqsyncignore"));
-  if (hqignore) ig.add(hqignore);
-
-  // Allowlist mode: when `.hqinclude` exists, sync is opt-in. The matcher
-  // here treats include patterns as ignore patterns and inverts the verdict —
-  // a path is "allowed" iff its relative path matches at least one entry.
-  // Exclusion layers above still subtract, so build artifacts inside an
-  // allowlisted subtree (e.g. node_modules/ inside companies/x/repos/y/) are
-  // still skipped.
-  const hqinclude = readIgnoreFile(path.join(hqRoot, ".hqinclude"));
-  const includeMatcher = hqinclude ? ignore().add(hqinclude) : null;
-  // Ancestor matcher: matches every directory that lies on the path to an
-  // include pattern. Consulted ONLY for `isDir=true` queries so a walker can
-  // descend through `companies/` and `companies/indigo/` to reach the leaf
-  // `companies/*/knowledge/`. Files directly inside those intermediate dirs
-  // remain excluded — this is the privacy invariant of allowlist mode.
-  const ancestorMatcher = hqinclude ? compileAncestorMatcher(hqinclude) : null;
-
-  return (filePath: string, isDir = false): boolean => {
-    const relative = path.relative(hqRoot, filePath);
-    if (!relative || relative.startsWith("..")) return true; // outside HQ root
-
-    // macOS Finder custom-icon files. Excluded here rather than as a
-    // DEFAULT_IGNORES pattern because neither available spelling works —
-    // see isMacFinderIconFile for the ignore@5 CR-stripping proof and the
-    // `Icon?` over-match hazard.
-    if (isMacFinderIconFile(relative)) return false;
-
-    // Gitignore dir-only patterns (`foo/`) only match candidate paths that
-    // end with `/`. The `ignore` lib has no stat awareness, so when the
-    // caller knows the entry is a directory we hand the matcher the
-    // canonical trailing-slash form. This mirrors how git itself decides:
-    // it knows from the index whether each entry is a tree or a blob.
-    // Applied symmetrically to BOTH layers — exclude and include — to
-    // preserve full gitignore semantics on both sides.
-    const probe = isDir && !relative.endsWith("/") ? relative + "/" : relative;
-
-    if (ig.ignores(probe)) return false;
-    if (!includeMatcher) return true;
-    if (includeMatcher.ignores(probe)) return true;
-    // Directory query that didn't match the include pattern itself — allow
-    // if it's an ancestor of one (so the walker can descend to the leaf).
-    if (isDir && ancestorMatcher) {
-      // Ancestor matching is depth-anchored, so feed it the slashless form.
-      const relDir = relative.replace(/\/$/, "");
-      if (relDir && ancestorMatcher(relDir)) return true;
-    }
-    return false;
-  };
-}
-
-/**
- * Check if a file exceeds the max sync size (50MB default)
- */
-export function isWithinSizeLimit(
-  filePath: string,
-  maxBytes = 50 * 1024 * 1024
-): boolean {
-  try {
-    const stat = fs.statSync(filePath);
-    return stat.size <= maxBytes;
-  } catch {
-    return false;
-  }
-}