harnery 0.7.1 → 0.8.0

package/dist/lib/tools/ripgrep.js

@@ -0,0 +1,217 @@
+import { spawn, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, renameSync, rmSync, statSync, writeFileSync, } from "node:fs";
+import { readFile, stat } from "node:fs/promises";
+import { homedir, tmpdir } from "node:os";
+import { join } from "node:path";
+import { resolveBinName, ripgrepAutoInstall } from "../../core/config.js";
+import { coordEnv } from "../env.js";
+/**
+ * Managed ripgrep provisioning for `grep --files` / content search.
+ *
+ * harnery can install a pinned, checksum-verified ripgrep into its own tools
+ * directory (`$XDG_DATA_HOME/harnery/tools`, default `~/.local/share/harnery/
+ * tools`) so `harn grep` gets the fast engine even on machines where nobody
+ * ever installed rg. The download is version-pinned and every artifact's
+ * sha256 is baked in below (cross-checked against the official release
+ * checksums), so the install is reproducible, never "whatever is latest."
+ *
+ * Consent model: the installer only runs automatically when the host project
+ * opts in via `.harnery/config.jsonc` `{ "tools": { "ripgrep": { "autoInstall":
+ * true } } }` — a repo commits that once and every clone self-heals. Without
+ * the opt-in, a missing rg produces a rate-limited stderr hint naming
+ * `harn doctor --fix` (explicit install). Environments that forbid downloads
+ * lose nothing: every failure path falls back to GNU grep.
+ */
+export const RG_VERSION = "14.1.1";
+/** sha256 pins verified against the official ripgrep release .sha256 assets. */
+const ARTIFACTS = {
+    "linux-x64": {
+        name: `ripgrep-${RG_VERSION}-x86_64-unknown-linux-musl.tar.gz`,
+        sha256: "4cf9f2741e6c465ffdb7c26f38056a59e2a2544b51f7cc128ef28337eeae4d8e",
+    },
+    "linux-arm64": {
+        name: `ripgrep-${RG_VERSION}-aarch64-unknown-linux-gnu.tar.gz`,
+        sha256: "c827481c4ff4ea10c9dc7a4022c8de5db34a5737cb74484d62eb94a95841ab2f",
+    },
+    "darwin-x64": {
+        name: `ripgrep-${RG_VERSION}-x86_64-apple-darwin.tar.gz`,
+        sha256: "fc87e78f7cb3fea12d69072e7ef3b21509754717b746368fd40d88963630e2b3",
+    },
+    "darwin-arm64": {
+        name: `ripgrep-${RG_VERSION}-aarch64-apple-darwin.tar.gz`,
+        sha256: "24ad76777745fbff131c8fbc466742b011f925bfa4fffa2ded6def23b5b937be",
+    },
+};
+function platformKey() {
+    return `${process.platform}-${process.arch}`;
+}
+/** True when a pinned artifact exists for this OS/arch (Windows: not yet). */
+export function rgInstallSupported() {
+    return platformKey() in ARTIFACTS;
+}
+/** Machine-level harnery tools dir (NOT the per-project .harnery/ state dir). */
+export function toolsDir() {
+    const xdg = process.env.XDG_DATA_HOME;
+    const base = xdg && xdg.trim() !== "" ? xdg : join(homedir(), ".local", "share");
+    return join(base, "harnery", "tools");
+}
+/** Path the managed rg binary lives at once installed. */
+export function managedRgPath() {
+    return join(toolsDir(), "rg");
+}
+function isExecutable(p) {
+    try {
+        const r = spawnSync(p, ["--version"], { stdio: "ignore" });
+        return r.status === 0;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Locate a usable ripgrep. Precedence:
+ *   1. HARNERY_RG_PATH (explicit override)
+ *   2. the managed tools-dir install (works even when PATH doesn't carry it)
+ *   3. `rg` on PATH
+ * Returns the spawnable path/name, or null when none respond to --version.
+ */
+export function findRg() {
+    const override = coordEnv("RG_PATH");
+    if (override && override.trim() !== "") {
+        return isExecutable(override) ? override : null;
+    }
+    const managed = managedRgPath();
+    if (existsSync(managed) && isExecutable(managed))
+        return managed;
+    if (isExecutable("rg"))
+        return "rg";
+    return null;
+}
+/**
+ * Download + verify + install the pinned ripgrep into the tools dir.
+ * Atomic: extracts to a temp dir and renames the binary into place.
+ * Throws on unsupported platform, network failure, or checksum mismatch —
+ * callers are expected to fall back to grep and say why.
+ */
+export async function installRg(log) {
+    const key = platformKey();
+    const artifact = ARTIFACTS[key];
+    if (!artifact) {
+        throw new Error(`no pinned ripgrep artifact for ${key} — install ripgrep manually (https://github.com/BurntSushi/ripgrep#installation)`);
+    }
+    const url = `https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${artifact.name}`;
+    log?.(`downloading ${artifact.name} (pinned ${RG_VERSION})...`);
+    const res = await fetch(url);
+    if (!res.ok)
+        throw new Error(`download failed: HTTP ${res.status} for ${url}`);
+    const bytes = Buffer.from(await res.arrayBuffer());
+    const digest = createHash("sha256").update(bytes).digest("hex");
+    if (digest !== artifact.sha256) {
+        throw new Error(`sha256 mismatch for ${artifact.name}: expected ${artifact.sha256}, got ${digest} — refusing to install`);
+    }
+    const work = join(tmpdir(), `harnery-rg-${process.pid}`);
+    rmSync(work, { recursive: true, force: true });
+    mkdirSync(work, { recursive: true });
+    try {
+        const tarball = join(work, artifact.name);
+        writeFileSync(tarball, bytes);
+        await extractTar(tarball, work);
+        const extractedRg = join(work, artifact.name.replace(/\.tar\.gz$/, ""), "rg");
+        const rgStat = await stat(extractedRg);
+        if (!rgStat.isFile())
+            throw new Error(`extracted archive missing rg binary at ${extractedRg}`);
+        mkdirSync(toolsDir(), { recursive: true });
+        const dest = managedRgPath();
+        // rename() can't cross filesystems (tmp is often tmpfs); copy + rename
+        // within the destination dir keeps the final placement atomic.
+        const staged = `${dest}.tmp-${process.pid}`;
+        writeFileSync(staged, await readFile(extractedRg));
+        chmodSync(staged, 0o755);
+        renameSync(staged, dest);
+        if (!isExecutable(dest)) {
+            rmSync(dest, { force: true });
+            throw new Error(`installed rg at ${dest} but it failed to execute`);
+        }
+        log?.(`installed ripgrep ${RG_VERSION} -> ${dest}`);
+        return dest;
+    }
+    finally {
+        rmSync(work, { recursive: true, force: true });
+    }
+}
+function extractTar(tarball, cwd) {
+    return new Promise((resolveP, reject) => {
+        const proc = spawn("tar", ["xzf", tarball], { cwd, stdio: ["ignore", "ignore", "pipe"] });
+        let stderr = "";
+        proc.stderr.setEncoding("utf8");
+        proc.stderr.on("data", (c) => {
+            stderr += c;
+        });
+        proc.on("error", reject);
+        proc.on("close", (code) => {
+            if (code === 0)
+                resolveP();
+            else
+                reject(new Error(`tar exited ${code}: ${stderr.trim()}`));
+        });
+    });
+}
+/**
+ * Shared engine resolver for the ripgrep-backed search commands (`grep`,
+ * `callers`). Returns the engine plus the spawnable rg path.
+ *
+ * `HARNERY_GREP_ENGINE=rg|grep` forces one; otherwise findRg() probes
+ * HARNERY_RG_PATH → the managed tools-dir install → PATH. On a miss, the
+ * host's `.harnery/config.jsonc` `tools.ripgrep.autoInstall` consent triggers
+ * a pinned, checksum-verified install into the harnery tools dir (any failure
+ * falls back to grep with a stderr note); without consent, a rate-limited
+ * stderr hint (keyed per `hintKey`) names the explicit install command.
+ */
+export async function resolveSearchEngine(hintKey = "grep") {
+    const forced = coordEnv("GREP_ENGINE");
+    if (forced === "grep")
+        return { engine: "grep", rgBin: "rg" };
+    const found = findRg();
+    if (found)
+        return { engine: "rg", rgBin: found };
+    if (forced === "rg") {
+        // Forced rg with none findable: let the spawn error surface loudly.
+        return { engine: "rg", rgBin: "rg" };
+    }
+    if (rgInstallSupported()) {
+        if (ripgrepAutoInstall()) {
+            try {
+                const installed = await installRg((line) => process.stderr.write(`harnery: ${line}\n`));
+                return { engine: "rg", rgBin: installed };
+            }
+            catch (err) {
+                process.stderr.write(`harnery: ripgrep auto-install failed (${err.message}); using grep fallback\n`);
+                return { engine: "grep", rgBin: "rg" };
+            }
+        }
+        const bin = resolveBinName();
+        hintOncePerDay(`${bin} ${hintKey}: ripgrep not found; using the slower GNU grep fallback. ` +
+            `Run \`${bin} doctor --fix\` to install it (pinned + checksum-verified), ` +
+            `or set { "tools": { "ripgrep": { "autoInstall": true } } } in .harnery/config.jsonc.`);
+    }
+    return { engine: "grep", rgBin: "rg" };
+}
+/**
+ * Emit `line` to stderr at most once per 24h (stamp file in the tools dir).
+ * Keeps the "rg missing" nudge from nagging on every single grep invocation.
+ */
+export function hintOncePerDay(line) {
+    try {
+        const stamp = join(toolsDir(), ".rg-hint-stamp");
+        if (existsSync(stamp) && Date.now() - statSync(stamp).mtimeMs < 24 * 60 * 60 * 1000) {
+            return;
+        }
+        mkdirSync(toolsDir(), { recursive: true });
+        writeFileSync(stamp, String(Date.now()));
+    }
+    catch {
+        // stamping failed; still emit the hint
+    }
+    process.stderr.write(`${line}\n`);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harnery",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Multi-agent coordination for AI coding agents - Claude Code, Cursor, and Codex.",
   "license": "MIT",
   "author": "Ryan Kelly",

package/src/commander.ts

@@ -120,6 +120,15 @@ export interface HarneryProgramContext {
    * carrying provider-specific checks. harn standalone ships none.
    */
   envSections?: Record<string, EnvSection>;
+  /**
+   * Extra directory names for `grep` to exclude by default — host-generated
+   * mirrors, vendored trees, and similar directories that only ever produce
+   * duplicate or noisy matches. Names match at any depth (same semantics as
+   * grep's `--exclude-dir`). Applied alongside the built-in skip list and
+   * disabled together with it by `--no-default-excludes`. harn standalone
+   * ships none.
+   */
+  grepExcludeDirs?: readonly string[];
   /**
    * Optional callback that returns extra HTTP headers to attach to outbound
    * `fetch` calls based on the target URL. Useful for consumers that need

package/src/commands/callers.ts

@@ -1,14 +1,17 @@
 import { spawn } from "node:child_process";
 import type { Command } from "commander";
 import type { EmitContext, HarneryProgramContext } from "../commander.ts";
+import { resolveSearchEngine, type SearchEngine } from "../lib/tools/ripgrep.ts";
 
 /**
  * `harn callers <symbol>`: find references to a symbol across the monorepo with
- * kind classification (call / import / type / decl / ref). Thin wrapper over
- * grep -rnw with post-filtering: declarations + single-line comments are
- * filtered out by default, and lines starting with " * " (multi-line comment
- * continuation) are dropped. Inherent text-based-search limitations: multi-line
- * string literals and block-comment interiors aren't excluded.
+ * kind classification (call / import / type / decl / ref). Whole-word search
+ * (ripgrep when available, GNU grep fallback — same engine + provisioning path
+ * as `harn grep`, honoring HARNERY_GREP_ENGINE) with post-filtering:
+ * declarations + single-line comments are filtered out by default, and lines
+ * starting with " * " (multi-line comment continuation) are dropped. Inherent
+ * text-based-search limitations: multi-line string literals and block-comment
+ * interiors aren't excluded. Repos are searched in parallel.
  */
 
 interface CallersOpts {
@@ -116,21 +119,31 @@ async function runCallers(
   }
 
   const started = Date.now();
+  const { engine, rgBin } = await resolveSearchEngine("callers");
   const repos = resolveRepos(opts, context);
   const limit = opts.limit ? Number.parseInt(opts.limit, 10) : 500;
 
+  // Search every repo concurrently, each capped at the global limit, then
+  // apply the global budget in repo order so truncation stays deterministic.
+  // In --all-repos mode the parent scan prunes submodule dirs (each submodule
+  // is scanned on its own), so a match is attributed to exactly one repo.
+  const perRepo = await Promise.all(
+    repos.map((repo) => {
+      const pruneDirs = opts.allRepos && repo.name === "parent" ? (context?.submodules ?? []) : [];
+      return searchInRepo(symbol, opts, repo, limit, engine, rgBin, pruneDirs);
+    }),
+  );
+
   const allCallers: Caller[] = [];
   let truncated = false;
-
-  for (const repo of repos) {
-    if (allCallers.length >= limit) {
-      truncated = true;
-      break;
-    }
-    const remaining = limit - allCallers.length;
-    const found = await grepInRepo(symbol, opts, repo, remaining);
+  let budget = limit;
+  for (let i = 0; i < repos.length; i++) {
+    const found = perRepo[i] ?? { callers: [], truncated: false };
     if (found.truncated) truncated = true;
-    allCallers.push(...found.callers);
+    const take = Math.min(found.callers.length, Math.max(0, budget));
+    if (found.callers.length > take) truncated = true;
+    allCallers.push(...found.callers.slice(0, take));
+    budget -= take;
   }
 
   const by_kind: Record<string, number> = {};
@@ -184,24 +197,72 @@ function resolveRepos(
   return [{ name: "cwd", cwd: process.cwd() }];
 }
 
-function grepInRepo(
-  symbol: string,
-  opts: CallersOpts,
-  repo: { name: string; cwd: string },
-  limit: number,
-): Promise<{ callers: Caller[]; truncated: boolean }> {
-  const args: string[] = ["-rnw", "--color=never", "-I", "-E"];
-  for (const d of DEFAULT_EXCLUDE_DIRS) args.push(`--exclude-dir=${d}`);
-  for (const f of DEFAULT_EXCLUDE_FILES) args.push(`--exclude=${f}`);
+function resolveLangGlobs(opts: CallersOpts): string[] | undefined {
   const langGlobs = opts.lang ? LANG_GLOBS[opts.lang] : undefined;
   if (opts.lang && !langGlobs) {
     throw new Error(`unknown --lang "${opts.lang}". Valid: ${Object.keys(LANG_GLOBS).join(", ")}`);
   }
+  return langGlobs;
+}
+
+function buildCallersGrepArgs(
+  symbol: string,
+  opts: CallersOpts,
+  extraExcludeDirs: readonly string[],
+): string[] {
+  const args: string[] = ["-rnw", "-H", "--color=never", "-I", "-E"];
+  for (const d of DEFAULT_EXCLUDE_DIRS) args.push(`--exclude-dir=${d}`);
+  for (const f of DEFAULT_EXCLUDE_FILES) args.push(`--exclude=${f}`);
+  for (const d of extraExcludeDirs) args.push(`--exclude-dir=${d}`);
+  const langGlobs = resolveLangGlobs(opts);
   if (langGlobs) for (const g of langGlobs) args.push(`--include=${g}`);
   args.push("--", symbol, ".");
+  return args;
+}
+
+function buildCallersRgArgs(
+  symbol: string,
+  opts: CallersOpts,
+  extraExcludeDirs: readonly string[],
+): string[] {
+  // --hidden --no-ignore --no-config: match grep semantics (dotdirs searched,
+  // .gitignore ignored, user rg config can't skew results). -w = whole word.
+  // Lang globs (positives) go before dir excludes (negatives) since rg globs
+  // are last-match-wins.
+  const args: string[] = [
+    "-nw",
+    "--no-heading",
+    "--with-filename",
+    "--color=never",
+    "--no-config",
+    "--hidden",
+    "--no-ignore",
+  ];
+  const langGlobs = resolveLangGlobs(opts);
+  if (langGlobs) for (const g of langGlobs) args.push(`--glob=${g}`);
+  for (const d of DEFAULT_EXCLUDE_DIRS) args.push(`--glob=!${d}`);
+  for (const f of DEFAULT_EXCLUDE_FILES) args.push(`--glob=!${f}`);
+  for (const d of extraExcludeDirs) args.push(`--glob=!${d}`);
+  args.push("--", symbol, ".");
+  return args;
+}
+
+function searchInRepo(
+  symbol: string,
+  opts: CallersOpts,
+  repo: { name: string; cwd: string },
+  limit: number,
+  engine: SearchEngine,
+  rgBin: string,
+  extraExcludeDirs: readonly string[],
+): Promise<{ callers: Caller[]; truncated: boolean }> {
+  const [bin, args] =
+    engine === "rg"
+      ? ([rgBin, buildCallersRgArgs(symbol, opts, extraExcludeDirs)] as const)
+      : (["grep", buildCallersGrepArgs(symbol, opts, extraExcludeDirs)] as const);
 
   return new Promise((resolveP, reject) => {
-    const proc = spawn("grep", args, { cwd: repo.cwd, stdio: ["ignore", "pipe", "pipe"] });
+    const proc = spawn(bin, args, { cwd: repo.cwd, stdio: ["ignore", "pipe", "pipe"] });
     const callers: Caller[] = [];
     let buffer = "";
     let stderr = "";
@@ -237,8 +298,11 @@ function grepInRepo(
         const parsed = parseCallerLine(buffer, symbol, opts, repo.name);
         if (parsed) callers.push(parsed);
       }
-      if (code !== null && code !== 0 && code !== 1 && !truncated) {
-        reject(new Error(`grep exited ${code}: ${stderr.trim() || "(no stderr)"}`));
+      // Exit 1 = no matches (both engines): normal. Exit 2 with nothing
+      // collected = a real failure; with matches, a partial walk error we
+      // tolerate (surface what was found).
+      if (code !== null && code !== 0 && code !== 1 && !truncated && callers.length === 0) {
+        reject(new Error(`${bin} exited ${code}: ${stderr.trim() || "(no stderr)"}`));
         return;
       }
       resolveP({ callers, truncated });

package/src/commands/doctor.ts

@@ -17,8 +17,9 @@ import os from "node:os";
 import path from "node:path";
 import type { Command } from "commander";
 import type { EmitContext } from "../commander.ts";
-import { resolveBinName } from "../core/config.ts";
+import { resolveBinName, ripgrepAutoInstall } from "../core/config.ts";
 import { loadHarnessWiring } from "../core/hooks/harness/wiring.ts";
+import { findRg, installRg, managedRgPath, rgInstallSupported } from "../lib/tools/ripgrep.ts";
 
 type Severity = "ok" | "warn" | "fail";
 
@@ -31,6 +32,7 @@ interface Check {
 
 interface CheckOpts {
   json?: boolean;
+  fix?: boolean;
 }
 
 export function registerDoctorCommand(program: Command, emit: EmitContext): void {
@@ -41,7 +43,18 @@ export function registerDoctorCommand(program: Command, emit: EmitContext): void
         "unless a required dep (Node, git) is missing.",
     )
     .option("--json", "Machine-readable JSON output")
-    .action((opts: CheckOpts) => {
+    .option(
+      "--fix",
+      "Install missing managed tools (currently: ripgrep, pinned + checksum-verified)",
+    )
+    .action(async (opts: CheckOpts) => {
+      if (opts.fix && !findRg() && rgInstallSupported()) {
+        try {
+          await installRg((line) => emit.text(`${line}\n`));
+        } catch (err) {
+          emit.text(`ripgrep install failed: ${(err as Error).message}\n`);
+        }
+      }
       const checks = runChecks();
       const requiredFailed = checks.some((c) => c.severity === "fail");
 
@@ -80,6 +93,7 @@ export function runChecks(): Check[] {
     checkNode(),
     checkGit(),
     checkBun(),
+    checkRipgrep(),
     checkHarneryDir(),
     checkHarnessHooks(),
     checkRestic(),
@@ -89,6 +103,40 @@ export function runChecks(): Check[] {
   ];
 }
 
+function checkRipgrep(): Check {
+  const found = findRg();
+  if (found) {
+    const r = whichVersion(found);
+    const managed = found === managedRgPath() ? ", managed" : "";
+    return {
+      name: "ripgrep",
+      severity: "ok",
+      detail: `${r.out.replace(/^ripgrep\s*/, "")} (${found === "rg" ? "PATH" : found}${managed})`,
+    };
+  }
+  if (!rgInstallSupported()) {
+    return {
+      name: "ripgrep",
+      severity: "warn",
+      detail: "missing (grep fallback; no pinned artifact for this OS/arch)",
+      hint: "https://github.com/BurntSushi/ripgrep#installation",
+    };
+  }
+  if (ripgrepAutoInstall()) {
+    return {
+      name: "ripgrep",
+      severity: "warn",
+      detail: "missing (autoInstall on: will self-provision on first grep)",
+    };
+  }
+  return {
+    name: "ripgrep",
+    severity: "warn",
+    detail: "missing (grep fallback works, just slower)",
+    hint: `${resolveBinName()} doctor --fix  (pinned + checksum-verified, installs to the harnery tools dir)`,
+  };
+}
+
 function whichVersion(bin: string, args: string[] = ["--version"]): { ok: boolean; out: string } {
   const r = spawnSync(bin, args, { encoding: "utf-8" });
   if (r.status !== 0) return { ok: false, out: "" };