little-coder 1.8.0 → 1.8.1

package/.pi/extensions/extra-tools/glob.test.ts

@@ -0,0 +1,89 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { globFiles, renderGlobOutcome, DEFAULT_HEAVY_DIRS } from "./glob.ts";
+
+let dir: string;
+
+beforeAll(() => {
+  dir = mkdtempSync(join(tmpdir(), "glob-test-"));
+  // real source we want to find
+  mkdirSync(join(dir, "src", "sub"), { recursive: true });
+  writeFileSync(join(dir, "src", "a.py"), "");
+  writeFileSync(join(dir, "src", "sub", "b.py"), "");
+  writeFileSync(join(dir, "README.md"), "");
+  // heavy dirs that must be pruned (with files matching the pattern inside)
+  mkdirSync(join(dir, "node_modules", "pkg", "deep"), { recursive: true });
+  writeFileSync(join(dir, "node_modules", "pkg", "deep", "x.py"), "");
+  mkdirSync(join(dir, ".git", "objects"), { recursive: true });
+  writeFileSync(join(dir, ".git", "objects", "y.py"), "");
+  mkdirSync(join(dir, "dist"), { recursive: true });
+  writeFileSync(join(dir, "dist", "z.py"), "");
+});
+
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+describe("globFiles", () => {
+  it("matches real files and prunes heavy dirs (node_modules/.git/dist)", async () => {
+    const { matches, scanTruncated, matchTruncated } = await globFiles("**/*.py", { base: dir });
+    const rel = matches.map((m) => m.slice(dir.length + 1)).sort();
+    expect(rel).toEqual(["src/a.py", "src/sub/b.py"]);
+    expect(matches.some((m) => m.includes("node_modules"))).toBe(false);
+    expect(matches.some((m) => m.includes(".git"))).toBe(false);
+    expect(matches.some((m) => m.includes("/dist/"))).toBe(false);
+    expect(scanTruncated).toBe(false);
+    expect(matchTruncated).toBe(false);
+  });
+
+  it("caps matches at maxMatches and flags matchTruncated", async () => {
+    const many = mkdtempSync(join(tmpdir(), "glob-many-"));
+    for (let i = 0; i < 50; i++) writeFileSync(join(many, `f${i}.txt`), "");
+    try {
+      const { matches, matchTruncated } = await globFiles("*.txt", { base: many, maxMatches: 10 });
+      expect(matches.length).toBe(10);
+      expect(matchTruncated).toBe(true);
+    } finally {
+      rmSync(many, { recursive: true, force: true });
+    }
+  });
+
+  it("stops the walk at maxScan and flags scanTruncated (memory bound)", async () => {
+    // A low budget must halt the walk regardless of how many entries exist.
+    const { scanned, scanTruncated } = await globFiles("**/*", { base: dir, maxScan: 3 });
+    expect(scanTruncated).toBe(true);
+    expect(scanned).toBeLessThanOrEqual(5); // a couple over the budget, not unbounded
+  });
+
+  it("the heavy-dir set covers the usual offenders", () => {
+    for (const d of ["node_modules", ".git", "dist", ".cache", "Library", "venv", "target"]) {
+      expect(DEFAULT_HEAVY_DIRS.has(d)).toBe(true);
+    }
+  });
+});
+
+describe("renderGlobOutcome", () => {
+  it("reports no matches plainly", () => {
+    expect(renderGlobOutcome({ matches: [], scanned: 5, scanTruncated: false, matchTruncated: false }))
+      .toBe("No files matched");
+  });
+  it("notes scan truncation when nothing matched", () => {
+    expect(renderGlobOutcome({ matches: [], scanned: 9, scanTruncated: true, matchTruncated: false }, 200000))
+      .toMatch(/stopped after scanning 200000 entries/);
+  });
+  it("appends a match-cap note", () => {
+    const text = renderGlobOutcome(
+      { matches: ["/a", "/b"], scanned: 2, scanTruncated: false, matchTruncated: true },
+      200000,
+      500,
+    );
+    expect(text).toMatch(/stopped at 500 matches/);
+  });
+  it("appends a scan-cap note when there were partial matches", () => {
+    const text = renderGlobOutcome(
+      { matches: ["/a"], scanned: 9, scanTruncated: true, matchTruncated: false },
+      200000,
+    );
+    expect(text).toMatch(/results may be incomplete/);
+  });
+});

package/.pi/extensions/extra-tools/glob.ts

@@ -0,0 +1,102 @@
+import { glob as fsGlob } from "node:fs/promises";
+
+// Bounded file globbing. The naive `for await (…glob…) { if (len>=500) break }`
+// only caps MATCHES — it does nothing about the WALK. Run from a huge root
+// (e.g. a home directory with macOS Library / caches / node_modules), fs.glob
+// recursively descends everything, and its internal traversal state grows until
+// the Node process OOMs (heap, not the model's context) — long before 500
+// matches are found if matches are sparse. fs.glob exposes no signal/abort and
+// no depth/scan cap, so we bound it through the one hook it does call for every
+// entry: `exclude`. We use it to (a) prune heavy/irrelevant directories so they
+// are never descended, and (b) meter total entries scanned — once the budget is
+// hit, exclude everything, which winds the walk down.
+
+/** Directories never worth descending for a file search — pruned at the dir
+ *  level (returning true from `exclude` on a directory stops descent), which is
+ *  what keeps a home-directory glob from exhausting memory. */
+export const DEFAULT_HEAVY_DIRS: ReadonlySet<string> = new Set([
+  // version control
+  ".git", ".hg", ".svn",
+  // dependencies / language caches
+  "node_modules", ".venv", "venv", "__pycache__", ".tox", ".mypy_cache",
+  ".pytest_cache", ".gradle", ".cargo", "vendor", "Pods",
+  // build output
+  "dist", "build", "out", "target", ".next", ".nuxt", ".output", ".svelte-kit",
+  // tool caches
+  ".cache", ".npm", ".pnpm-store", ".yarn", ".turbo",
+  // macOS / system heavies that blow up a home-dir walk
+  "Library", "Applications", ".Trash", "Photos Library.photoslibrary",
+]);
+
+export interface GlobOptions {
+  base: string;
+  maxScan?: number;
+  maxMatches?: number;
+  heavyDirs?: ReadonlySet<string>;
+}
+
+export interface GlobOutcome {
+  matches: string[];
+  scanned: number;
+  /** the walk was cut short at maxScan entries (results may be incomplete) */
+  scanTruncated: boolean;
+  /** matches were capped at maxMatches */
+  matchTruncated: boolean;
+}
+
+export const DEFAULT_MAX_SCAN = 200_000;
+export const DEFAULT_MAX_MATCHES = 500;
+
+export async function globFiles(pattern: string, opts: GlobOptions): Promise<GlobOutcome> {
+  const maxScan = opts.maxScan ?? DEFAULT_MAX_SCAN;
+  const maxMatches = opts.maxMatches ?? DEFAULT_MAX_MATCHES;
+  const heavy = opts.heavyDirs ?? DEFAULT_HEAVY_DIRS;
+
+  const matches: string[] = [];
+  let scanned = 0;
+  let scanTruncated = false;
+  let matchTruncated = false;
+
+  // Called for every entry the walk visits (files AND directories). Pruning a
+  // directory here stops descent into it. Also our scan meter: once the budget
+  // is spent, exclude everything so fs.glob stops adding work and ends.
+  const exclude = (entry: unknown): boolean => {
+    scanned++;
+    if (scanned > maxScan) {
+      scanTruncated = true;
+      return true;
+    }
+    const name = typeof entry === "string" ? entry : String((entry as { name?: string })?.name ?? entry);
+    return name.split(/[\\/]/).some((seg) => heavy.has(seg));
+  };
+
+  // `exclude` as a predicate isn't in every @types/node version's fs.glob
+  // signature, but it's supported at runtime (Node 22+); cast to pass it through.
+  for await (const m of fsGlob(pattern, { cwd: opts.base, exclude } as Parameters<typeof fsGlob>[1])) {
+    matches.push(`${opts.base}/${m}`);
+    if (matches.length >= maxMatches) {
+      matchTruncated = true;
+      break;
+    }
+  }
+
+  matches.sort();
+  return { matches, scanned, scanTruncated, matchTruncated };
+}
+
+/** Render a globFiles outcome as the tool's text output, with a one-line note
+ *  when results were cut short so the model knows to narrow its search. */
+export function renderGlobOutcome(o: GlobOutcome, maxScan = DEFAULT_MAX_SCAN, maxMatches = DEFAULT_MAX_MATCHES): string {
+  if (o.matches.length === 0) {
+    return o.scanTruncated
+      ? `No files matched (search stopped after scanning ${maxScan} entries — narrow the base path; build/dependency/cache dirs are skipped automatically).`
+      : "No files matched";
+  }
+  let text = o.matches.join("\n");
+  if (o.matchTruncated) {
+    text += `\n… (stopped at ${maxMatches} matches — narrow the pattern for the rest)`;
+  } else if (o.scanTruncated) {
+    text += `\n… (search stopped after scanning ${maxScan} entries — results may be incomplete; narrow the base path)`;
+  }
+  return text;
+}

package/.pi/extensions/extra-tools/index.ts

@@ -1,6 +1,6 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Type } from "@sinclair/typebox";
-import { glob as globSync } from "node:fs/promises";
+import { globFiles, renderGlobOutcome } from "./glob.ts";
 
 // Ports of tools.py::_glob, _webfetch, _websearch. Pi ships its own grep/find,
 // so those are not re-registered here.
@@ -10,7 +10,9 @@ export default function (pi: ExtensionAPI) {
     name: "glob",
     label: "Glob",
     description:
-      "Find files matching a glob pattern. Returns a sorted list of matching paths (up to 500).",
+      "Find files matching a glob pattern. Returns a sorted list of matching paths (up to 500). " +
+      "Common dependency/build/cache dirs (node_modules, .git, dist, …) are skipped, and the walk " +
+      "is bounded — for a focused search, pass a `path` rather than globbing a whole home directory.",
     parameters: Type.Object({
       pattern: Type.String({ description: "Glob pattern e.g. **/*.py" }),
       path: Type.Optional(Type.String({ description: "Base directory (default: cwd)" })),
@@ -18,16 +20,11 @@ export default function (pi: ExtensionAPI) {
     async execute(_id, { pattern, path }) {
       try {
         const base = path || process.cwd();
-        const matches: string[] = [];
-        // Node 22's fs/promises.glob — returns an async iterator
-        for await (const m of globSync(pattern, { cwd: base })) {
-          matches.push(`${base}/${m}`);
-          if (matches.length >= 500) break;
-        }
-        matches.sort();
-        const text = matches.length === 0 ? "No files matched" : matches.join("\n");
+        // Bounded walk: prunes heavy dirs and caps total entries scanned so a
+        // recursive glob from a huge root can't exhaust the process heap.
+        const outcome = await globFiles(pattern, { base });
         return {
-          content: [{ type: "text", text }],
+          content: [{ type: "text", text: renderGlobOutcome(outcome) }],
           details: {},
         };
       } catch (e) {

package/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to little-coder are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and little-coder's public interface (CLI, providers, tools, skills) follows semver starting at `v0.0.1` post-rename.
 
+## [v1.8.1] — 2026-05-23
+
+### Fixed
+- **`glob` no longer exhausts memory on a recursive search from a huge root.** The tool capped *matches* at 500 but never bounded the *walk*: run from a home directory (or any tree with macOS `Library`, caches, or `node_modules`), `fs.glob` recursively descended everything and its internal traversal state grew until the Node **process** ran out of heap — a host-memory crash (`Ineffective mark-compacts near heap limit`), entirely distinct from the model's *context window* (the read-guard / window machinery operates on tool *results* in tokens; this died mid-walk, before any result existed). The walk is now bounded two ways: heavy/irrelevant directories (`node_modules`, `.git`, `dist`, `.cache`, `Library`, `venv`, `target`, …) are **pruned** — never descended — and a hard scan budget (200 000 entries) halts the walk through the one hook `fs.glob` calls per entry (`exclude`), since it exposes no signal/abort. When results are cut short the output says so, so the model narrows its search. New unit-tested `globFiles` / `renderGlobOutcome` helpers (`.pi/extensions/extra-tools/glob.ts`), verified to prune `node_modules` (0 descent) and to halt at the scan budget.
+
+### Notes for upgraders
+- For a focused search, pass a `path` (a project subdirectory) instead of globbing from a home directory. Hidden directories continue to be skipped by `fs.glob` as before.
+
+---
+
 ## [v1.8.0] — 2026-05-23
 
 little-coder now **auto-detects the llama.cpp server's live context window** at startup and registers the model with it, so a `llama-server -c 131072` shows 128k instead of the declared default — no config edit. This completes [v1.7.0](#v170--2026-05-23): the budget already *followed* the registered window; now the registered window itself comes from the running server.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "little-coder",
-  "version": "1.8.0",
+  "version": "1.8.1",
   "description": "A pi-based coding agent optimized for small local language models. Reproduces the whitepaper's scaffold-model-fit adaptations as pi extensions.",
   "homepage": "https://github.com/itayinbarr/little-coder",
   "repository": {