@indigoai-us/hq-cli 5.96.0 → 5.97.0

package/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## [Unreleased]
 
+## [5.97.0]
+
+### Added
+
+- `hq reindex` now keeps files over GitHub's 100MB limit out of the HQ repo.
+  The HQ root is a git repo that `git add -A` sweeps wholesale after every
+  sync, and session logs and package stores routinely pass that limit — once
+  such a blob is committed, every later push is rejected with GH001
+  permanently, because the blob is in history. Oversized files now gain an
+  anchored `.gitignore` rule, and any already in the index are dropped with
+  `git rm --cached`, which leaves the working-tree file untouched. Every
+  affected path is named on stdout, since untracking mutates the user's index
+  and must never be silent. The scan runs through git rather than the
+  filesystem, so nested checkouts under `repos/` are excluded automatically,
+  and it is throttled to once an hour per root so the hook-driven reindex path
+  stays cheap. (#346)
+
 ## [5.96.0]
 
 ### Changed

package/dist/commands/reindex.js

@@ -30,6 +30,7 @@ import * as yaml from 'js-yaml';
 import { reindex, rescue } from '@indigoai-us/hq-cloud';
 import { trustHqRuntimeHooks } from '../utils/hook-trust.js';
 import { findHqRoot } from '../utils/manifest.js';
+import { guardLargeFiles } from '../utils/large-file-guard.js';
 const HOOK_EVENTS = ['SessionStart', 'UserPromptSubmit', 'PreToolUse'];
 const HOOK_CHECK_RELATIVE_PATH = path.join('core', 'scripts', 'check-hq-hooks.sh');
 /** Resolve the same root the repair/check commands must operate on. */
@@ -205,6 +206,25 @@ export function repairExtremeHookDrift(hqRoot, allowRepair = true) {
         printHookHealthWarning(hqRoot, 'hook configuration repair could not run');
     }
 }
+/**
+ * Keep files above GitHub's 100MB limit out of the HQ repo, and say so.
+ *
+ * Untracking is a mutation of the user's index, so it is never silent: every
+ * affected path is named on stdout. Throttled internally to one scan per hour
+ * per root, so the hook-driven reindex path stays cheap.
+ */
+function reportLargeFileGuard(hqRoot) {
+    const { scanned, ignored, untracked } = guardLargeFiles(hqRoot);
+    if (!scanned)
+        return;
+    const removed = new Set(untracked);
+    for (const file of untracked) {
+        console.log(`reindex: ${file} exceeds GitHub's 100MB limit — removed from git tracking and ignored (the file itself is untouched)`);
+    }
+    for (const file of ignored.filter((f) => !removed.has(f))) {
+        console.log(`reindex: ${file} exceeds GitHub's 100MB limit — added to .gitignore`);
+    }
+}
 export function registerReindexCommand(program) {
     program
         .command('reindex')
@@ -239,6 +259,9 @@ export function registerReindexCommand(program) {
         repairExtremeHookDrift(hqRoot, status === 0);
         if (status === 0)
             await trustHqRuntimeHooks(hqRoot);
+        // Runs even when reindex failed: a failed reindex does not make an
+        // oversized blob any less likely to wedge the next push.
+        reportLargeFileGuard(hqRoot);
         process.exit(status);
     });
 }

package/dist/utils/large-file-guard.d.ts

@@ -0,0 +1,39 @@
+/** GitHub rejects any single file above this size. */
+export declare const DEFAULT_THRESHOLD_BYTES: number;
+/** At most one scan per hour per root. */
+export declare const SCAN_THROTTLE_MS: number;
+export interface LargeFileGuardResult {
+    /** False when the root is not a git repo, or the scan was throttled. */
+    scanned: boolean;
+    /** Repo-relative paths newly added to `.gitignore`. */
+    ignored: string[];
+    /** Repo-relative paths removed from the index (still on disk). */
+    untracked: string[];
+}
+export interface LargeFileGuardOptions {
+    /** Defaults to {@link DEFAULT_THRESHOLD_BYTES}. */
+    thresholdBytes?: number;
+    /** Injectable clock, in epoch milliseconds. */
+    now?: number;
+    /** Bypass the throttle. */
+    force?: boolean;
+}
+/**
+ * Render a repo-relative path as a literal `.gitignore` pattern.
+ *
+ * The leading slash anchors it to the repo root — without it a bare filename
+ * would also ignore same-named files in every subdirectory. Glob and
+ * character-class metacharacters are escaped so the rule matches the one real
+ * path and nothing else, and a trailing space is escaped because git strips
+ * unescaped ones.
+ */
+export declare function toGitignorePattern(relPath: string): string;
+/**
+ * Scan `hqRoot` for files above the size limit, ignore them, and drop any that
+ * are already in the index.
+ *
+ * Never throws: every failure path degrades to "did nothing", because a guard
+ * that can take `hq reindex` down is worse than the problem it prevents.
+ */
+export declare function guardLargeFiles(hqRoot: string, options?: LargeFileGuardOptions): LargeFileGuardResult;
+//# sourceMappingURL=large-file-guard.d.ts.map

package/dist/utils/large-file-guard.js

@@ -0,0 +1,249 @@
+/**
+ * Large-file guard — keep blobs over GitHub's hard limit out of the HQ repo.
+ *
+ * HQ's root is a git repo that `git add -A` sweeps wholesale (the desktop app's
+ * git-mirror does exactly that after every sync). Session logs and package
+ * stores routinely blow past GitHub's 100 MB per-file limit, and once such a
+ * blob is committed every subsequent push is rejected with GH001 — permanently,
+ * since the blob is in history. Recovering means rewriting history.
+ *
+ * This guard runs from `hq reindex` and closes the door before that happens:
+ *   - oversized files gain a `.gitignore` rule, so `git add -A` skips them;
+ *   - oversized files already in the index are dropped from it with
+ *     `git rm --cached`, which leaves the working-tree file untouched.
+ *
+ * Design constraints that shaped this:
+ *
+ * - **Git-aware, not filesystem-aware.** A naive `find -size +100M` over an HQ
+ *   root takes ~37s and surfaces files inside nested checkouts under `repos/`,
+ *   which the outer repo does not own. Driving everything through git costs
+ *   ~13s and gets nested-repo boundaries right for free.
+ * - **Index, not HEAD.** `git rm --cached` mutates the index and leaves HEAD
+ *   alone until the next commit, so enumerating HEAD would re-report the same
+ *   file on every run.
+ * - **Throttled.** `hq reindex` also fires from a PostToolUse hook shim on
+ *   skill/worker/policy edits. A stamp file in the git dir bounds the scan to
+ *   once an hour so a policy-editing session doesn't pay the cost repeatedly.
+ */
+import { spawnSync } from "node:child_process";
+import * as fs from "node:fs";
+import * as path from "node:path";
+/** GitHub rejects any single file above this size. */
+export const DEFAULT_THRESHOLD_BYTES = 100 * 1024 * 1024;
+/** At most one scan per hour per root. */
+export const SCAN_THROTTLE_MS = 60 * 60 * 1000;
+/** Lives in the git dir, so it is never committed, synced, or seen by status. */
+const STAMP_BASENAME = "hq-large-file-scan.stamp";
+const GITIGNORE_HEADER = "# hq reindex: files over GitHub's 100MB limit (added automatically)";
+/**
+ * git output for a fully-populated HQ root runs to tens of megabytes
+ * (~450k tracked paths). The Node default of 1 MiB would truncate it.
+ */
+const MAX_GIT_BUFFER = 512 * 1024 * 1024;
+const SKIPPED = Object.freeze({
+    scanned: false,
+    ignored: [],
+    untracked: [],
+});
+function git(cwd, args, input) {
+    try {
+        const out = spawnSync("git", args, {
+            cwd,
+            input,
+            encoding: "utf8",
+            maxBuffer: MAX_GIT_BUFFER,
+        });
+        if (out.error || out.status !== 0)
+            return { ok: false, stdout: "" };
+        return { ok: true, stdout: out.stdout ?? "" };
+    }
+    catch {
+        // A missing cwd or an absent git binary must degrade to "guard did
+        // nothing", never take reindex down with it.
+        return { ok: false, stdout: "" };
+    }
+}
+/** Absolute path to the repo's git dir, or undefined when cwd is not a repo. */
+function resolveGitDir(hqRoot) {
+    if (!fs.existsSync(hqRoot))
+        return undefined;
+    const out = git(hqRoot, ["rev-parse", "--git-dir"]);
+    if (!out.ok)
+        return undefined;
+    const raw = out.stdout.trim();
+    if (!raw)
+        return undefined;
+    return path.isAbsolute(raw) ? raw : path.resolve(hqRoot, raw);
+}
+function isThrottled(stampPath, now) {
+    try {
+        const last = Number(fs.readFileSync(stampPath, "utf8").trim());
+        if (!Number.isFinite(last))
+            return false;
+        return now - last < SCAN_THROTTLE_MS;
+    }
+    catch {
+        return false; // no stamp yet, or unreadable — scan.
+    }
+}
+function writeStamp(stampPath, now) {
+    try {
+        fs.writeFileSync(stampPath, `${now}\n`);
+    }
+    catch {
+        // Losing the stamp only costs an extra scan next time.
+    }
+}
+/** Split a NUL-delimited git record stream, dropping the trailing empty field. */
+function splitNul(raw) {
+    return raw.split("\0").filter((entry) => entry.length > 0);
+}
+/**
+ * Oversized blobs in the *index*.
+ *
+ * `git ls-files -s` yields `<mode> <object> <stage>\t<path>`; the object ids are
+ * piped through `cat-file --batch-check` so sizes come from the object database
+ * rather than ~450k filesystem stats.
+ */
+function oversizedInIndex(hqRoot, threshold) {
+    const listed = git(hqRoot, ["ls-files", "-s", "-z"]);
+    if (!listed.ok)
+        return [];
+    const entries = [];
+    for (const record of splitNul(listed.stdout)) {
+        const tab = record.indexOf("\t");
+        if (tab === -1)
+            continue;
+        const fields = record.slice(0, tab).split(" ");
+        if (fields.length < 2)
+            continue;
+        entries.push({ oid: fields[1], file: record.slice(tab + 1) });
+    }
+    if (entries.length === 0)
+        return [];
+    const sizes = git(hqRoot, ["cat-file", "--batch-check=%(objectsize)"], `${entries.map((e) => e.oid).join("\n")}\n`);
+    if (!sizes.ok)
+        return [];
+    // One output line per input line, in order.
+    const lines = sizes.stdout.split("\n");
+    const oversized = [];
+    for (let i = 0; i < entries.length; i += 1) {
+        const size = Number(lines[i]);
+        if (Number.isFinite(size) && size > threshold)
+            oversized.push(entries[i].file);
+    }
+    return oversized;
+}
+/**
+ * Oversized files git would pick up on the next `add -A` — untracked or
+ * modified. `git status` already honours `.gitignore` and stops at nested
+ * repository boundaries, so neither needs handling here.
+ */
+function oversizedInWorktree(hqRoot, threshold) {
+    const status = git(hqRoot, ["status", "--porcelain", "-z", "--untracked-files=all"]);
+    if (!status.ok)
+        return [];
+    const records = splitNul(status.stdout);
+    const oversized = [];
+    for (let i = 0; i < records.length; i += 1) {
+        const record = records[i];
+        if (record.length < 4)
+            continue;
+        const code = record.slice(0, 2);
+        const file = record.slice(3);
+        // Renames and copies emit the source path as a second record; consume it
+        // so it is not parsed as a status line of its own.
+        if (code.startsWith("R") || code.startsWith("C"))
+            i += 1;
+        if (code.includes("D"))
+            continue; // going away — nothing to guard
+        try {
+            const stat = fs.statSync(path.join(hqRoot, file));
+            if (stat.isFile() && stat.size > threshold)
+                oversized.push(file);
+        }
+        catch {
+            // Vanished between status and stat; nothing to do.
+        }
+    }
+    return oversized;
+}
+/**
+ * Render a repo-relative path as a literal `.gitignore` pattern.
+ *
+ * The leading slash anchors it to the repo root — without it a bare filename
+ * would also ignore same-named files in every subdirectory. Glob and
+ * character-class metacharacters are escaped so the rule matches the one real
+ * path and nothing else, and a trailing space is escaped because git strips
+ * unescaped ones.
+ */
+export function toGitignorePattern(relPath) {
+    const escaped = relPath.replace(/([\\*?[\]!#])/g, "\\$1");
+    const anchored = `/${escaped}`;
+    return anchored.endsWith(" ") ? `${anchored.slice(0, -1)}\\ ` : anchored;
+}
+function appendGitignore(hqRoot, patterns) {
+    if (patterns.length === 0)
+        return true;
+    const file = path.join(hqRoot, ".gitignore");
+    let existing = "";
+    try {
+        existing = fs.readFileSync(file, "utf8");
+    }
+    catch {
+        // No .gitignore yet — it gets created below.
+    }
+    const present = new Set(existing.split("\n").map((line) => line.trim()));
+    const fresh = patterns.filter((p) => !present.has(p));
+    if (fresh.length === 0)
+        return true;
+    const needsNewline = existing.length > 0 && !existing.endsWith("\n");
+    const header = existing.includes(GITIGNORE_HEADER) ? "" : `${GITIGNORE_HEADER}\n`;
+    try {
+        fs.appendFileSync(file, `${needsNewline ? "\n" : ""}${header}${fresh.join("\n")}\n`);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Scan `hqRoot` for files above the size limit, ignore them, and drop any that
+ * are already in the index.
+ *
+ * Never throws: every failure path degrades to "did nothing", because a guard
+ * that can take `hq reindex` down is worse than the problem it prevents.
+ */
+export function guardLargeFiles(hqRoot, options = {}) {
+    const threshold = options.thresholdBytes ?? DEFAULT_THRESHOLD_BYTES;
+    const now = options.now ?? Date.now();
+    const gitDir = resolveGitDir(hqRoot);
+    if (!gitDir)
+        return SKIPPED;
+    const stampPath = path.join(gitDir, STAMP_BASENAME);
+    if (!options.force && isThrottled(stampPath, now))
+        return SKIPPED;
+    writeStamp(stampPath, now);
+    const tracked = oversizedInIndex(hqRoot, threshold);
+    const trackedSet = new Set(tracked);
+    const worktree = oversizedInWorktree(hqRoot, threshold).filter((p) => !trackedSet.has(p));
+    const all = [...tracked, ...worktree];
+    if (all.length === 0)
+        return { scanned: true, ignored: [], untracked: [] };
+    // Ignore first, so a concurrent `git add -A` cannot re-stage what we are
+    // about to remove from the index.
+    const patterns = all.map(toGitignorePattern);
+    const wroteIgnore = appendGitignore(hqRoot, patterns);
+    const untracked = [];
+    for (const file of tracked) {
+        const removed = git(hqRoot, ["rm", "--cached", "--quiet", "--ignore-unmatch", "--", file]);
+        if (removed.ok)
+            untracked.push(file);
+    }
+    return {
+        scanned: true,
+        ignored: wroteIgnore ? all : [],
+        untracked,
+    };
+}
+//# sourceMappingURL=large-file-guard.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.96.0",
+  "version": "5.97.0",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {