@visual-z/xeno 0.1.0

package/src/store/search.ts

@@ -0,0 +1,644 @@
+/**
+ * Workspace search: grep, find, ls.
+ *
+ * The walk reuses buildFileIndex so ignore rules, the 20000 cap, and the
+ * no-symlink rule stay one implementation. Ripgrep / fd are used only when
+ * they are already on PATH -- never downloaded -- because fetching a binary
+ * at runtime is a supply-chain path the Isolation section of README.md
+ * told the user they would not be taking.
+ *
+ * Both engines format the same way so two machines do not silently disagree.
+ */
+
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import { lstat, readdir, readFile } from "node:fs/promises";
+import { isAbsolute, join, relative, sep } from "node:path";
+import { Worker } from "node:worker_threads";
+import { assertNever } from "../core/exhaustive.ts";
+import { buildFileIndex } from "./file-index.ts";
+import { DEFAULT_IGNORE } from "./workspace.ts";
+
+/**
+ * Match / entry cap. Assumed, not measured: enough for a model to read in
+ * one result; past this it should refine the query. A silent cut would hide
+ * how much evidence was dropped.
+ */
+export const DEFAULT_MATCH_LIMIT = 100;
+
+/** Assumed, not measured: same order of magnitude as pi's find default. */
+export const DEFAULT_FIND_LIMIT = 1_000;
+
+/** Assumed, not measured: a directory listing larger than this needs a filter. */
+export const DEFAULT_LS_LIMIT = 500;
+
+/**
+ * Assumed, not measured: a longer line belongs in `read`, not in a match
+ * list. Cutting it without a cap would let one minified file blow the budget.
+ */
+export const MAX_LINE_CHARS = 500;
+
+/**
+ * Assumed, not measured: same ballpark as the tool-output cap in tools.ts.
+ * Hitting it drops trailing matches and says how many.
+ */
+export const MAX_OUTPUT_BYTES = 30_000;
+
+/**
+ * Assumed, not measured: a 1 MB file is already past what a match snippet
+ * can usefully show; ripgrep is told the same ceiling so the two engines
+ * skip the same files.
+ */
+export const MAX_FILE_BYTES = 1_000_000;
+
+/**
+ * Per-file match ceiling. A linear scan of MAX_FILE_BYTES with a simple
+ * pattern was measured at ~0.07ms/MB on bun 1.3; 100ms is assumed slack on
+ * top of that. `(a+)+$` against 40 a's was measured at ~220ms here -- JSC
+ * aborts the backtrack around there rather than hanging forever -- so a
+ * ceiling above that would never fire on the card's catastrophic case, and
+ * the agent would see "no matches" instead of a cutoff. The pattern comes
+ * from the model, not a person: a hung tool_call has no result in the log.
+ */
+export const MATCH_TIMEOUT_MS = 100;
+
+/** Assumed: an rg/fd process that runs longer than this is stuck. */
+export const PROCESS_TIMEOUT_MS = 30_000;
+
+export type SearchEngine = "auto" | "ripgrep" | "walk" | "fd";
+
+export interface SearchResult {
+  content: string;
+  ok: boolean;
+}
+
+export interface GrepOpts {
+  readonly root: string;
+  /** Absolute path, already confined to root. */
+  readonly path: string;
+  readonly pattern: string;
+  readonly glob?: string;
+  readonly limit?: number;
+  readonly engine?: "auto" | "ripgrep" | "walk";
+  readonly timeoutMs?: number;
+}
+
+export interface FindOpts {
+  readonly root: string;
+  readonly path: string;
+  readonly pattern: string;
+  readonly limit?: number;
+  readonly engine?: "auto" | "fd" | "walk";
+  readonly timeoutMs?: number;
+}
+
+export interface LsOpts {
+  readonly root: string;
+  readonly path: string;
+  readonly limit?: number;
+}
+
+export interface MatchLine {
+  readonly path: string;
+  readonly line: number;
+  readonly text: string;
+}
+
+const MATCH_WORKER_SOURCE = `
+const { parentPort } = require("node:worker_threads");
+parentPort.on("message", (msg) => {
+  const { pattern, flags, lines } = msg;
+  try {
+    const re = new RegExp(pattern, flags);
+    const hits = [];
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      re.lastIndex = 0;
+      if (re.test(line)) hits.push({ line: i + 1, text: line });
+    }
+    parentPort.postMessage({ ok: true, hits });
+  } catch (err) {
+    parentPort.postMessage({
+      ok: false,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+});
+`;
+
+export function findOnPath(name: string): string | null {
+  const pathEnv = process.env["PATH"] ?? "";
+  const delim = process.platform === "win32" ? ";" : ":";
+  for (const dir of pathEnv.split(delim)) {
+    if (dir.length === 0) continue;
+    const candidate = join(dir, name);
+    if (existsSync(candidate)) return candidate;
+  }
+  return null;
+}
+
+function posixRel(root: string, abs: string): string | null {
+  const rel = relative(root, abs);
+  if (rel.startsWith("..") || isAbsolute(rel)) return null;
+  return rel.split(sep).join("/");
+}
+
+function ignoredRel(rel: string): boolean {
+  if (rel.length === 0) return false;
+  const parts = rel.split("/");
+  return DEFAULT_IGNORE.some((name) => parts.includes(name));
+}
+
+/**
+ * Convert a glob to a fully-anchored regexp. Assumed to cover the shapes
+ * models actually type (basename wildcards and double-star path globs), not
+ * the full gitignore grammar -- a missed edge is a missed file, not a hang.
+ */
+export function globToRegExp(glob: string): RegExp {
+  let i = 0;
+  let out = "";
+  while (i < glob.length) {
+    if (glob.startsWith("**/", i)) {
+      out += "(?:.*/)?";
+      i += 3;
+      continue;
+    }
+    if (glob[i] === "*" && glob[i + 1] === "*") {
+      out += ".*";
+      i += 2;
+      continue;
+    }
+    const c = glob[i]!;
+    if (c === "*") out += "[^/]*";
+    else if (c === "?") out += "[^/]";
+    else if (".+^${}()|[]\\".includes(c)) out += `\\${c}`;
+    else out += c;
+    i += 1;
+  }
+  return new RegExp(`^${out}$`);
+}
+
+export function globMatches(glob: string, relPath: string): boolean {
+  const posix = relPath.split(sep).join("/");
+  const slash = posix.lastIndexOf("/");
+  const base = slash === -1 ? posix : posix.slice(slash + 1);
+  const re = globToRegExp(glob);
+  if (glob.includes("/")) return re.test(posix);
+  return re.test(base);
+}
+
+function splitLines(text: string): string[] {
+  const normalized = text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
+  if (normalized.length === 0) return [];
+  const trimmed = normalized.endsWith("\n") ? normalized.slice(0, -1) : normalized;
+  if (trimmed.length === 0) return [];
+  return trimmed.split("\n");
+}
+
+function truncateLine(text: string): string {
+  const flat = text.replace(/\r/g, "");
+  if (flat.length <= MAX_LINE_CHARS) return flat;
+  return flat.slice(0, MAX_LINE_CHARS) + "…";
+}
+
+function formatMatch(m: MatchLine): string {
+  return `${m.path}:${m.line}: ${truncateLine(m.text)}`;
+}
+
+/**
+ * Apply the match-count cap and the byte cap. Overflow is counted, never
+ * dropped silently: a test that only asserted "at most N" would pass an
+ * implementation that threw the rest away.
+ */
+function renderLines(lines: string[], limit: number, extraNotices: string[]): string {
+  const omittedFromCap = Math.max(0, lines.length - limit);
+  let shown = lines.slice(0, limit);
+  let omitted = omittedFromCap;
+
+  let body = shown.join("\n");
+  while (shown.length > 0 && body.length > MAX_OUTPUT_BYTES) {
+    shown = shown.slice(0, -1);
+    omitted += 1;
+    body = shown.join("\n");
+  }
+
+  const notices = [...extraNotices];
+  if (omitted > 0) notices.push(`${omitted} more matches omitted`);
+  if (notices.length === 0) return body;
+  const note = `[${notices.join(". ")}]`;
+  return body.length > 0 ? `${body}\n${note}` : note;
+}
+
+function noMatches(notices: string[]): SearchResult {
+  if (notices.length === 0) return { content: "No matches found", ok: true };
+  return { content: `[${notices.join(". ")}]`, ok: true };
+}
+
+async function inspectTarget(
+  root: string,
+  abs: string,
+): Promise<{ kind: "file" | "dir"; rel: string } | { error: string }> {
+  const rel = posixRel(root, abs);
+  if (rel === null) return { error: `path escapes the workspace root: ${abs}` };
+  if (ignoredRel(rel)) return { error: `path is ignored: ${rel}` };
+
+  let st;
+  try {
+    st = await lstat(abs);
+  } catch {
+    return { error: `path not found: ${rel || "."}` };
+  }
+  if (st.isSymbolicLink()) {
+    return { error: `refusing to follow a symlink: ${rel || "."}` };
+  }
+  if (st.isDirectory()) return { kind: "dir", rel };
+  if (st.isFile()) return { kind: "file", rel };
+  return { error: `not a file or directory: ${rel || "."}` };
+}
+
+async function listedFiles(abs: string, rel: string, kind: "file" | "dir"): Promise<string[]> {
+  if (kind === "file") return ignoredRel(rel) ? [] : [rel];
+  const index = await buildFileIndex(abs);
+  return index.paths.map((p) => (rel.length === 0 ? p : `${rel}/${p}`));
+}
+
+interface WorkerMatch {
+  readonly ok: boolean;
+  readonly hits?: ReadonlyArray<{ readonly line: number; readonly text: string }>;
+}
+
+class LineMatcher {
+  private worker: Worker | null = null;
+
+  private spawn(): Worker {
+    return new Worker(MATCH_WORKER_SOURCE, { eval: true });
+  }
+
+  async match(
+    pattern: string,
+    flags: string,
+    lines: string[],
+  ): Promise<"timeout" | "invalid" | MatchLine[]> {
+    if (this.worker === null) this.worker = this.spawn();
+    const worker = this.worker;
+    return await new Promise((resolve) => {
+      let settled = false;
+      const finish = (value: "timeout" | "invalid" | MatchLine[]): void => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        worker.off("message", onMessage);
+        worker.off("error", onError);
+        resolve(value);
+      };
+      const timer = setTimeout(() => {
+        void worker.terminate();
+        this.worker = null;
+        finish("timeout");
+      }, MATCH_TIMEOUT_MS);
+      const onMessage = (msg: WorkerMatch) => {
+        if (!msg.ok) {
+          finish("invalid");
+          return;
+        }
+        finish((msg.hits ?? []).map((h) => ({ path: "", line: h.line, text: h.text })));
+      };
+      const onError = () => {
+        void worker.terminate();
+        this.worker = null;
+        finish("invalid");
+      };
+      worker.on("message", onMessage);
+      worker.on("error", onError);
+      worker.postMessage({ pattern, flags, lines });
+    });
+  }
+
+  close(): void {
+    if (this.worker !== null) {
+      void this.worker.terminate();
+      this.worker = null;
+    }
+  }
+}
+
+function runCommand(
+  cmd: string,
+  args: readonly string[],
+  timeoutMs: number,
+): Promise<{ stdout: string; stderr: string; exit: number | null }> {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, [...args], { stdio: ["ignore", "pipe", "pipe"] });
+    let stdout = "";
+    let stderr = "";
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk: string) => {
+      stdout += chunk;
+    });
+    child.stderr.on("data", (chunk: string) => {
+      stderr += chunk;
+    });
+    const timer = setTimeout(() => child.kill(), timeoutMs);
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      resolve({ stdout, stderr, exit: code });
+    });
+    child.on("error", (err) => {
+      clearTimeout(timer);
+      resolve({ stdout, stderr: err.message, exit: null });
+    });
+  });
+}
+
+function rgIgnoreArgs(): string[] {
+  const args: string[] = [];
+  for (const name of DEFAULT_IGNORE) {
+    args.push("--glob", `!${name}`);
+    args.push("--glob", `!${name}/**`);
+    args.push("--glob", `!**/${name}`);
+    args.push("--glob", `!**/${name}/**`);
+  }
+  return args;
+}
+
+function parseRgJson(root: string, stdout: string): MatchLine[] {
+  const hits: MatchLine[] = [];
+  for (const raw of stdout.split("\n")) {
+    if (raw.trim().length === 0) continue;
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch {
+      continue;
+    }
+    if (typeof parsed !== "object" || parsed === null) continue;
+    const rec = parsed as Record<string, unknown>;
+    if (rec["type"] !== "match") continue;
+    const data = rec["data"];
+    if (typeof data !== "object" || data === null) continue;
+    const d = data as Record<string, unknown>;
+    const pathObj = d["path"];
+    const pathText =
+      typeof pathObj === "object" && pathObj !== null
+        ? String((pathObj as Record<string, unknown>)["text"] ?? "")
+        : "";
+    const lineNumber = d["line_number"];
+    const linesObj = d["lines"];
+    const lineText =
+      typeof linesObj === "object" && linesObj !== null
+        ? String((linesObj as Record<string, unknown>)["text"] ?? "")
+        : "";
+    if (pathText.length === 0 || typeof lineNumber !== "number") continue;
+    const rel = posixRel(root, pathText) ?? posixRel(root, join(root, pathText));
+    if (rel === null || ignoredRel(rel)) continue;
+    hits.push({
+      path: rel,
+      line: lineNumber,
+      text: lineText.replace(/\n$/, "").replace(/\r/g, ""),
+    });
+  }
+  return hits;
+}
+
+async function grepRipgrep(opts: GrepOpts, abs: string, limit: number): Promise<SearchResult | null> {
+  const rg = findOnPath("rg");
+  if (rg === null) return null;
+  const args = [
+    "--json",
+    "--color=never",
+    "--hidden",
+    "--no-follow",
+    "--no-ignore",
+    "--max-filesize",
+    String(MAX_FILE_BYTES),
+    ...rgIgnoreArgs(),
+  ];
+  if (opts.glob !== undefined) args.push("--glob", opts.glob);
+  args.push("--", opts.pattern, abs);
+  const ran = await runCommand(rg, args, opts.timeoutMs ?? PROCESS_TIMEOUT_MS);
+  // 0 = matches, 1 = none; anything else is a real failure (bad pattern, etc.).
+  if (ran.exit !== 0 && ran.exit !== 1) {
+    const err = ran.stderr.trim() || `ripgrep exited ${ran.exit ?? "null"}`;
+    return { content: err, ok: false };
+  }
+  const hits = parseRgJson(opts.root, ran.stdout);
+  hits.sort(compareMatch);
+  const lines = hits.map(formatMatch);
+  if (lines.length === 0) return noMatches([]);
+  return { content: renderLines(lines, limit, []), ok: true };
+}
+
+function compareMatch(a: MatchLine, b: MatchLine): number {
+  if (a.path !== b.path) return a.path < b.path ? -1 : 1;
+  return a.line - b.line;
+}
+
+async function grepWalk(opts: GrepOpts, abs: string, kind: "file" | "dir", rel: string, limit: number): Promise<SearchResult> {
+  try {
+    new RegExp(opts.pattern);
+  } catch (err) {
+    return { content: `invalid regex: ${(err as Error).message}`, ok: false };
+  }
+
+  const files = await listedFiles(abs, rel, kind);
+  const glob = opts.glob;
+  const filtered = glob === undefined ? files : files.filter((p) => globMatches(glob, p));
+  const matcher = new LineMatcher();
+  const hits: MatchLine[] = [];
+  const notices: string[] = [];
+  try {
+    for (const fileRel of filtered) {
+      if (ignoredRel(fileRel)) continue;
+      const fileAbs = join(opts.root, fileRel);
+      let st;
+      try {
+        st = await lstat(fileAbs);
+      } catch {
+        continue;
+      }
+      if (st.isSymbolicLink() || !st.isFile()) continue;
+      if (st.size > MAX_FILE_BYTES) continue;
+      let text: string;
+      try {
+        text = await readFile(fileAbs, "utf8");
+      } catch {
+        continue;
+      }
+      if (text.includes("\0")) continue;
+      const lines = splitLines(text);
+      const matched = await matcher.match(opts.pattern, "", lines);
+      if (matched === "timeout") {
+        notices.push(`search cut off on ${fileRel} after ${MATCH_TIMEOUT_MS}ms`);
+        continue;
+      }
+      if (matched === "invalid") {
+        return { content: `invalid regex: ${opts.pattern}`, ok: false };
+      }
+      for (const hit of matched) hits.push({ path: fileRel, line: hit.line, text: hit.text });
+    }
+  } finally {
+    matcher.close();
+  }
+
+  hits.sort(compareMatch);
+  const formatted = hits.map(formatMatch);
+  if (formatted.length === 0) return noMatches(notices);
+  return { content: renderLines(formatted, limit, notices), ok: true };
+}
+
+export async function grep(opts: GrepOpts): Promise<SearchResult> {
+  const pattern = opts.pattern;
+  if (pattern.length === 0) return { content: "grep needs a pattern", ok: false };
+
+  const target = await inspectTarget(opts.root, opts.path);
+  if ("error" in target) return { content: target.error, ok: false };
+
+  const limit = opts.limit !== undefined && opts.limit > 0 ? Math.floor(opts.limit) : DEFAULT_MATCH_LIMIT;
+  const engine = opts.engine ?? "auto";
+
+  switch (engine) {
+    case "walk":
+      return grepWalk(opts, opts.path, target.kind, target.rel, limit);
+    case "ripgrep": {
+      const got = await grepRipgrep(opts, opts.path, limit);
+      if (got === null) return { content: "ripgrep is not on PATH", ok: false };
+      return got;
+    }
+    case "auto": {
+      if (findOnPath("rg") !== null) {
+        const got = await grepRipgrep(opts, opts.path, limit);
+        if (got !== null) return got;
+      }
+      return grepWalk(opts, opts.path, target.kind, target.rel, limit);
+    }
+    default:
+      return assertNever(engine, "grep");
+  }
+}
+
+function fdExcludeArgs(): string[] {
+  const args: string[] = [];
+  for (const name of DEFAULT_IGNORE) args.push("--exclude", name);
+  return args;
+}
+
+async function findFd(opts: FindOpts, abs: string, limit: number): Promise<SearchResult | null> {
+  const fd = findOnPath("fd");
+  if (fd === null) return null;
+  const args = [
+    "--glob",
+    "--color=never",
+    "--hidden",
+    "--no-ignore",
+    "--absolute-path",
+    ...fdExcludeArgs(),
+  ];
+  let pattern = opts.pattern;
+  if (pattern.includes("/")) {
+    args.push("--full-path");
+    if (!pattern.startsWith("**/") && pattern !== "**") pattern = `**/${pattern}`;
+  }
+  args.push("--", pattern, abs);
+  const ran = await runCommand(fd, args, opts.timeoutMs ?? PROCESS_TIMEOUT_MS);
+  if (ran.exit !== 0 && ran.stdout.trim().length === 0) {
+    const err = ran.stderr.trim() || `fd exited ${ran.exit ?? "null"}`;
+    return { content: err, ok: false };
+  }
+  const paths: string[] = [];
+  for (const raw of ran.stdout.split("\n")) {
+    const line = raw.replace(/\r$/, "").trim();
+    if (line.length === 0) continue;
+    const rel = posixRel(opts.root, line);
+    if (rel === null || ignoredRel(rel)) continue;
+    paths.push(rel);
+  }
+  paths.sort((a, b) => (a < b ? -1 : 1));
+  if (paths.length === 0) return { content: "No files found matching pattern", ok: true };
+  return { content: renderLines(paths, limit, []), ok: true };
+}
+
+async function findWalk(opts: FindOpts, abs: string, kind: "file" | "dir", rel: string, limit: number): Promise<SearchResult> {
+  const files = await listedFiles(abs, rel, kind);
+  const paths = files.filter((p) => !ignoredRel(p) && globMatches(opts.pattern, p));
+  paths.sort((a, b) => (a < b ? -1 : 1));
+  if (paths.length === 0) return { content: "No files found matching pattern", ok: true };
+  return { content: renderLines(paths, limit, []), ok: true };
+}
+
+export async function find(opts: FindOpts): Promise<SearchResult> {
+  if (opts.pattern.length === 0) return { content: "find needs a pattern", ok: false };
+  const target = await inspectTarget(opts.root, opts.path);
+  if ("error" in target) return { content: target.error, ok: false };
+  const limit = opts.limit !== undefined && opts.limit > 0 ? Math.floor(opts.limit) : DEFAULT_FIND_LIMIT;
+  const engine = opts.engine ?? "auto";
+  switch (engine) {
+    case "walk":
+      return findWalk(opts, opts.path, target.kind, target.rel, limit);
+    case "fd": {
+      const got = await findFd(opts, opts.path, limit);
+      if (got === null) return { content: "fd is not on PATH", ok: false };
+      return got;
+    }
+    case "auto": {
+      if (findOnPath("fd") !== null) {
+        const got = await findFd(opts, opts.path, limit);
+        if (got !== null) return got;
+      }
+      return findWalk(opts, opts.path, target.kind, target.rel, limit);
+    }
+    default:
+      return assertNever(engine, "find");
+  }
+}
+
+export async function ls(opts: LsOpts): Promise<SearchResult> {
+  const target = await inspectTarget(opts.root, opts.path);
+  if ("error" in target) return { content: target.error, ok: false };
+  if (target.kind !== "dir") return { content: `not a directory: ${target.rel || "."}`, ok: false };
+
+  let entries;
+  try {
+    entries = await readdir(opts.path, { withFileTypes: true });
+  } catch (err) {
+    return { content: `cannot list ${target.rel || "."}: ${(err as Error).message}`, ok: false };
+  }
+
+  const names: string[] = [];
+  for (const entry of entries) {
+    if (DEFAULT_IGNORE.includes(entry.name)) continue;
+    const childRel = target.rel.length === 0 ? entry.name : `${target.rel}/${entry.name}`;
+    if (ignoredRel(childRel)) continue;
+    // Do not follow: a symlink is listed as a name, never as the directory
+    // it points at, so a link to / cannot walk the disk.
+    if (entry.isSymbolicLink()) {
+      names.push(entry.name);
+      continue;
+    }
+    if (entry.isDirectory()) names.push(`${entry.name}/`);
+    else if (entry.isFile()) names.push(entry.name);
+  }
+  names.sort((a, b) => {
+    const al = a.toLowerCase();
+    const bl = b.toLowerCase();
+    if (al === bl) return a < b ? -1 : 1;
+    return al < bl ? -1 : 1;
+  });
+
+  const limit = opts.limit !== undefined && opts.limit > 0 ? Math.floor(opts.limit) : DEFAULT_LS_LIMIT;
+  if (names.length === 0) return { content: "(empty directory)", ok: true };
+  const omitted = Math.max(0, names.length - limit);
+  const shown = names.slice(0, limit);
+  let body = shown.join("\n");
+  let extra = omitted;
+  while (shown.length > 0 && body.length > MAX_OUTPUT_BYTES) {
+    shown.pop();
+    extra += 1;
+    body = shown.join("\n");
+  }
+  if (extra > 0) {
+    const note = `[${extra} more matches omitted]`;
+    body = body.length > 0 ? `${body}\n${note}` : note;
+  }
+  return { content: body, ok: true };
+}

package/src/store/serial.ts

@@ -0,0 +1,13 @@
+/** O.5: one in-process coordinator per journal; callers never nest the same key. */
+import { resolve } from "node:path";
+
+const tails = new Map<string, Promise<void>>();
+export async function serialized<T>(key: string, run: () => Promise<T>): Promise<T> {
+  const id = resolve(key), previous = tails.get(id) ?? Promise.resolve();
+  let release!: () => void;
+  const current = new Promise<void>(done => { release = done; });
+  tails.set(id, current);
+  await previous;
+  try { return await run(); }
+  finally { release(); if (tails.get(id) === current) tails.delete(id); }
+}