@bevel-software/platform-core-backend 0.32.0 → 0.34.0

package/src/modules/workspace/__tests__/workspace.tools.test.ts

@@ -8,13 +8,14 @@ import AdmZip from 'adm-zip';
 import express from 'express';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { testKbContext } from '../../../__tests__/kb-context.js';
+import { slowRipgrep } from '../../../__tests__/slow-ripgrep.js';
 import { LocalFilesystem } from '@mastra/core/workspace';
 import { ToolRegistry } from '../../tool-registry/tool-registry.js';
 import { createToolHandlerFactory } from '../../tool-helpers/tool-handler.js';
 import { ToolError, type ToolContext } from '../../tool-helpers/tool.contract.js';
 import type { ToolAuth } from '../../tool-auth/tool-auth.middleware.js';
 import { registerWorkspaceTools } from '../workspace.tools.js';
-import { WorkerPatternMatcher, type WorkerPatternMatcherOptions } from '../bounded-matcher.js';
+import { RipgrepPatternMatcher, type RipgrepPatternMatcherOptions } from '../index.js';
 import { sharedFileRules, sharedFileRulesSection } from '../../agent-instructions/shared-file-rules.js';
 import { composeAgentGuide } from '../../agent-guide/agent-guide.js';
 import { GUIDE_FIRST_SENTENCE } from '../../tool-registry/guide-first.js';
@@ -25,6 +26,7 @@ import { WorkflowHooks, type AgentOperationContext } from '../../workflow/workfl
 import { SESSION_ID_DESCRIPTION, ToolDescriptionNotes } from '../agent-access.gate.js';
 import { SpillStore } from '../spill-store.js';
 import { DocExtractService } from '../file-readers/doc-extract.service.js';
+import { WorkerDocumentExtractor } from '../file-readers/extract-in-worker.js';
 import { OCTET_STREAM_FALLBACK_NOTE } from '../file-readers/content-mode.js';
 import type { IAccessControl } from '../../access/access-control.interface.js';
 import { AccessControlService } from '../../access/access-control.service.js';
@@ -143,8 +145,13 @@ let raceHook: (() => Promise<void>) | null = null;
  * The grep matcher `start()` wires, built from `matcherOptions` (the
  * defaults, unless a test narrows the pool) and closed after each test.
  */
-let patternMatcher: WorkerPatternMatcher | undefined;
-let matcherOptions: WorkerPatternMatcherOptions = {};
+let patternMatcher: RipgrepPatternMatcher | undefined;
+/**
+ * The document extractor `start()` wires, closed after each test: its workers
+ * are made lazily, and `unref()` lets the process exit but keeps them alive.
+ */
+let docExtractor: WorkerDocumentExtractor | undefined;
+let matcherOptions: RipgrepPatternMatcherOptions = {};
 
 async function start(
   scope: 'read' | 'write' = 'write',
@@ -154,7 +161,10 @@ async function start(
 ): Promise<string> {
   raceHook = null;
   await patternMatcher?.close();
-  patternMatcher = new WorkerPatternMatcher(matcherOptions);
+  patternMatcher = new RipgrepPatternMatcher(matcherOptions);
+  await docExtractor?.close();
+  const extractor = new WorkerDocumentExtractor();
+  docExtractor = extractor;
   hooks = new WorkflowHooks();
   notes = new ToolDescriptionNotes();
   callerSource = 'internal';
@@ -309,7 +319,7 @@ async function start(
   const app = express();
   app.use(express.json());
   const router = express.Router();
-  registerWorkspaceTools(registry, router, fakeAuth, toolHandler, new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(docCacheDir), access, testKbContext({ kbDirName: KB_DIR }), {
+  registerWorkspaceTools(registry, router, fakeAuth, toolHandler, new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(docCacheDir, extractor), access, testKbContext({ kbDirName: KB_DIR }), {
     // The hooks and the notes a test registers against; with neither
     // registered (the default, and every Hexis-only deployment) the gate
     // refuses nothing and the descriptions say nothing extra.
@@ -354,6 +364,8 @@ afterEach(async () => {
   httpServer = undefined;
   await patternMatcher?.close();
   patternMatcher = undefined;
+  await docExtractor?.close();
+  docExtractor = undefined;
   if (tempDir) {
     await rm(tempDir, { recursive: true, force: true });
     tempDir = '';
@@ -1004,6 +1016,10 @@ describe("the agent guide at the guide's name", () => {
     expect((await grep(base, 'needle', 'AGENTS.md')).matches).toEqual([{ path: GUIDE, line: 3, text: 'Name every needle you plant.' }]);
     // A search under a folder does not reach a file that is not under it.
     expect((await grep(base, 'needle', `${KB_DIR}/Handbook`)).matches.map((m) => m.path)).toEqual([`${KB_DIR}/Handbook/a.md`]);
+    // A folder named with a trailing slash answers the same paths, spelled
+    // once, and the guide once.
+    expect((await grep(base, 'needle', `${KB_DIR}/Handbook/`)).matches.map((m) => m.path)).toEqual([`${KB_DIR}/Handbook/a.md`]);
+    expect((await grep(base, 'needle', `${KB_DIR}/`)).matches).toEqual(fromRoot.matches);
 
     // With the knowledge base's own AGENTS.md, the search covers the composed
     // text — the own file first, then the guide — and once: the walk's own
@@ -1018,17 +1034,18 @@ describe("the agent guide at the guide's name", () => {
     ]);
     expect((await grep(base, 'needle', GUIDE)).matches).toEqual(composed);
 
-    // The own file does not use up `max_results` twice: the walk leaves it to
-    // the composed search, so a file later in the tree is still reached when
-    // the own file alone has more matches than the cap. Without that, the
-    // walk filled the cap from AGENTS.md, those matches were dropped as
-    // duplicates, and Handbook/ was never searched.
+    // The own file does not use up `max_results` twice: the tree search
+    // leaves it to the composed search, so its lines are counted once and the
+    // file after it in path order is still reached. Counted twice, its five
+    // lines and their duplicates would fill a cap of seven, and Handbook/
+    // would be cut.
     await fs.writeFile(`${KB_DIR}/AGENTS.md`, `# Acme\n\n${'needle\n'.repeat(5)}`);
-    const capped = (await (await post(`${base}/api/agent/tools/grep`, { pattern: 'needle', max_results: 3 })).json()) as {
-      matches: { path: string }[];
+    const capped = (await (await post(`${base}/api/agent/tools/grep`, { pattern: 'needle', max_results: 7 })).json()) as {
+      matches: { path: string; line: number }[];
     };
-    expect(capped.matches.map((m) => m.path)).toContain(`${KB_DIR}/Handbook/a.md`);
-    expect(capped.matches).toHaveLength(3);
+    // Five lines of the own file and the guide's one, then the file after it.
+    expect(capped.matches.map((m) => m.path)).toEqual([...Array<string>(6).fill(GUIDE), `${KB_DIR}/Handbook/a.md`]);
+    expect(new Set(capped.matches.map((m) => `${m.path}:${m.line}`)).size).toBe(7);
     await fs.writeFile(`${KB_DIR}/AGENTS.md`, '# Acme\n\nOur needle rule.\n');
 
     // A caller who may not read the own file searches the guide alone, from
@@ -1181,6 +1198,22 @@ describe('grep with a path that names a file', () => {
     }
   });
 
+  it('a named text file answers the lines a folder search answers from it, valid UTF-8 lines of a file holding others included', async () => {
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/mixed/latin.txt`, Buffer.concat([Buffer.from('needle \xff\n', 'latin1'), Buffer.from('needle ok\n')]));
+    const named = await grep(base, { pattern: 'needle', path: `${KB_DIR}/mixed/latin.txt` });
+    const folder = await grep(base, { pattern: 'needle', path: `${KB_DIR}/mixed` });
+    expect(named.matches).toEqual([{ path: `${KB_DIR}/mixed/latin.txt`, line: 2, text: 'needle ok' }]);
+    expect(named.matches).toEqual(folder.matches);
+    expect(named.note).toBeUndefined();
+    // A line that is not UTF-8 counts toward ripgrep's cap, yet the line after it is still answered.
+    for (const path of [`${KB_DIR}/mixed/latin.txt`, `${KB_DIR}/mixed`]) {
+      const one = await grep(base, { pattern: 'needle', path, max_results: 1 });
+      expect(one.matches).toEqual([{ path: `${KB_DIR}/mixed/latin.txt`, line: 2, text: 'needle ok' }]);
+      expect(one.truncated).toBe(true);
+    }
+  });
+
   it('a path with nothing at it fails with an error naming the path — never an empty success', async () => {
     const base = await start();
     const res = await post(`${base}/api/agent/tools/grep`, { pattern: 'needle', path: `${KB_DIR}/notes/ghost.md` });
@@ -1267,119 +1300,356 @@ describe('grep with a path that names a file', () => {
 });
 
 /**
- * A caller's pattern runs off the event loop under a budget (WorkerPatternMatcher):
- * one that backtracks for ever is stopped with a worded 400, and every other
- * request keeps being served meanwhile — here, a `read_file` on the same
- * server, which shares the one loop the pattern used to hold.
+ * grep runs ripgrep (RipgrepPatternMatcher), as a child process: the pattern
+ * is ripgrep's default engine's to judge, a search is stopped at its budget
+ * while every other request keeps being served, and the walk covers every
+ * text file the caller may read — hidden, git-ignored and untracked alike —
+ * without following a link or entering `.git`.
  */
-describe('a pattern too expensive to match', () => {
-  const LONG_LINE = `${'a'.repeat(39)}!`;
+describe('grep runs ripgrep', () => {
+  type GrepBody = { matches: { path: string; line: number; text: string }[]; truncated: boolean; note?: string; error?: string; code?: string };
+  const grep = async (base: string, body: Record<string, unknown>): Promise<{ status: number; body: GrepBody }> => {
+    const res = await post(`${base}/api/agent/tools/grep`, body);
+    return { status: res.status, body: (await res.json()) as GrepBody };
+  };
+  const git = (...args: string[]) => promisify(execFile)('git', args, { cwd: join(tempDir, KB_DIR) });
+
+  it('stops a search at its 5 s budget with the 400 "too expensive", leaves no ripgrep running, and serves other calls meanwhile', async () => {
+    const slow = await slowRipgrep();
+    try {
+      matcherOptions = {};
+      const base = await start();
+      await fs.writeFile(`${KB_DIR}/hit.md`, 'needle\n');
+      const started = Date.now();
+      const searching = grep(base, { pattern: 'needle' }).then((r) => ({ ...r, took: Date.now() - started }));
+      // Probe while the search runs: each read must answer well within a second.
+      const probes: number[] = [];
+      let done = false;
+      void searching.finally(() => (done = true));
+      while (!done) {
+        const t = Date.now();
+        const res = await post(`${base}/api/agent/tools/read_file`, { path: `${KB_DIR}/a.md` });
+        expect(res.status).toBe(200);
+        probes.push(Date.now() - t);
+        await new Promise((r) => setTimeout(r, 100));
+      }
+      const { status, body, took } = await searching;
+      expect(status).toBe(400);
+      expect(body.code).toBe('pattern_too_expensive');
+      expect(body.error).toMatch(/^Pattern too expensive: /);
+      expect(body.error).toContain("this search's 5 s budget");
+      expect(took).toBeGreaterThanOrEqual(5_000);
+      expect(took).toBeLessThan(5_000 + 2_000);
+      expect(probes.length).toBeGreaterThan(10);
+      expect(Math.max(...probes)).toBeLessThan(1_000);
+      expect(await slow.alive()).toEqual([]);
+    } finally {
+      await slow.remove();
+    }
+  }, 15_000);
+
+  it('answers 503 when every search slot is taken and the queue is full, rather than an incomplete result', async () => {
+    const slow = await slowRipgrep();
+    try {
+      matcherOptions = { budgetMs: 1_000, maxRunning: 1, maxWaiting: 1 };
+      const base = await start();
+      await fs.writeFile(`${KB_DIR}/hit.md`, 'needle\n');
+      const holding = grep(base, { pattern: 'needle' }).then((r) => r.status);
+      while ((await slow.alive()).length === 0) await new Promise((r) => setTimeout(r, 20));
+      const queued = grep(base, { pattern: 'needle' }).then((r) => r.status);
+      await new Promise((r) => setTimeout(r, 100));
+      const refused = await grep(base, { pattern: 'needle' });
+      expect(refused.status).toBe(503);
+      expect(refused.body.code).toBe('search_busy');
+      expect(await Promise.all([holding, queued])).toEqual([400, 400]);
+      expect(await slow.alive()).toEqual([]);
+      // A slot is free again: with the real ripgrep back, a search answers.
+      await slow.remove();
+      const after = await grep(base, { pattern: 'needle', path: `${KB_DIR}/hit.md` });
+      expect(after.status).toBe(200);
+      expect(after.body.matches).toEqual([{ path: `${KB_DIR}/hit.md`, line: 1, text: 'needle' }]);
+    } finally {
+      await slow.remove();
+    }
+  });
+
+  it.each([
+    ['look-ahead', 'foo(?=bar)', /look-around/],
+    ['look-behind', '(?<=foo)bar', /look-around/],
+    ['a backreference', '(a)\\1', /backreferences are not supported/],
+    ['invalid syntax', 'a(', /unclosed group/],
+  ])('refuses %s with a 400 quoting ripgrep and naming the dialect, whatever path it names', async (_what, pattern, reason) => {
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/img.png`, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0]));
+    for (const path of [undefined, `${KB_DIR}/a.md`, `${KB_DIR}/img.png`]) {
+      const res = await grep(base, path === undefined ? { pattern } : { pattern, path });
+      expect(res.status).toBe(400);
+      expect(res.body.code).toBe('invalid_pattern');
+      expect(res.body.error).toMatch(/^Invalid regex: /);
+      expect(res.body.error).toMatch(reason);
+      expect(res.body.error).toContain('RE2-style patterns: no look-around and no backreferences');
+      // ripgrep's advice to pass a flag the caller cannot pass is not repeated.
+      expect(res.body.error).not.toContain('--pcre2');
+    }
+  });
+
+  it('the description and the pattern input say which patterns the engine takes', async () => {
+    await start();
+    const def = (await toolRegistry.listInternal()).find((t) => t.name === 'grep');
+    expect(def?.description).toContain("grep runs ripgrep's default regex engine, which takes RE2-style patterns: no look-around and no backreferences.");
+    expect(JSON.stringify(def?.inputSchema ?? def)).toContain('RE2-style (ripgrep): no look-around, no backreferences.');
+  });
 
-  it('answers 400 "too expensive" fast over a 40-character line, while other calls stay under a second', async () => {
+  it('a pattern that backtracks for ever in JavaScript is answered at once: the engine does not backtrack', async () => {
     const base = await start();
-    await fs.writeFile(`${KB_DIR}/redos.md`, `intro\n${LONG_LINE}\n`);
+    const longLine = `${'a'.repeat(5_000)}!`;
+    await fs.writeFile(`${KB_DIR}/redos.md`, `intro\n${longLine}\n`);
     const started = Date.now();
-    const searching = post(`${base}/api/agent/tools/grep`, { pattern: '^(a+)+$', path: `${KB_DIR}/redos.md` }).then(
-      async (res) => ({ status: res.status, body: (await res.json()) as { error: string; code?: string }, took: Date.now() - started }),
-    );
-    // Probe while the search runs: each read must answer within a second.
-    const probes: number[] = [];
-    let done = false;
-    void searching.finally(() => (done = true));
-    while (!done) {
-      const t = Date.now();
-      const res = await post(`${base}/api/agent/tools/read_file`, { path: `${KB_DIR}/a.md` });
+    const none = await grep(base, { pattern: '^(a+)+$' });
+    expect(none.status).toBe(200);
+    expect(none.body.matches).toEqual([]);
+    const some = await grep(base, { pattern: '^a+!$', path: `${KB_DIR}/redos.md` });
+    expect(some.body.matches).toEqual([{ path: `${KB_DIR}/redos.md`, line: 2, text: longLine.slice(0, 300) }]);
+    expect(Date.now() - started).toBeLessThan(2_000);
+  });
+
+  it('searches hidden, git-ignored and untracked files, never .git, and never through a link', async () => {
+    const base = await start();
+    const outside = await mkdtemp(join(tmpdir(), 'grep-outside-'));
+    try {
+      await writeFile(join(outside, 'secret.md'), 'needle from outside\n');
+      await git('init', '-q');
+      await fs.writeFile(`${KB_DIR}/.gitignore`, 'build/\nignored.md\n');
+      await fs.writeFile(`${KB_DIR}/tracked.md`, 'needle tracked\n');
+      await git('add', '.gitignore', 'tracked.md');
+      await git('-c', 'user.email=t@x', '-c', 'user.name=t', 'commit', '-qm', 'init');
+      await fs.writeFile(`${KB_DIR}/.notes.md`, 'needle hidden\n');
+      await fs.writeFile(`${KB_DIR}/.hidden/deep.md`, 'needle in a hidden folder\n');
+      await fs.writeFile(`${KB_DIR}/build/out.txt`, 'needle ignored folder\n');
+      await fs.writeFile(`${KB_DIR}/ignored.md`, 'needle ignored file\n');
+      await fs.writeFile(`${KB_DIR}/untracked.md`, 'needle untracked\n');
+      await writeFile(join(tempDir, KB_DIR, '.git', 'needle-in-git'), 'needle git internals\n');
+      await mkdir(join(tempDir, KB_DIR, 'sub', '.GIT'), { recursive: true });
+      await writeFile(join(tempDir, KB_DIR, 'sub', '.GIT', 'x'), 'needle git internals\n');
+      await symlink(join(outside, 'secret.md'), join(tempDir, KB_DIR, 'link.md'));
+      await symlink(outside, join(tempDir, KB_DIR, 'linked-dir'));
+      await fs.writeFile(`${KB_DIR}/binary.dat`, Buffer.concat([Buffer.from('needle in binary\n'), Buffer.from([0, 1, 2])]));
+      const res = await grep(base, { pattern: 'needle' });
       expect(res.status).toBe(200);
-      probes.push(Date.now() - t);
-      await new Promise((r) => setTimeout(r, 50));
+      expect(res.body.matches.map((m) => m.path)).toEqual([
+        `${KB_DIR}/.hidden/deep.md`,
+        `${KB_DIR}/.notes.md`,
+        `${KB_DIR}/build/out.txt`,
+        `${KB_DIR}/ignored.md`,
+        `${KB_DIR}/tracked.md`,
+        `${KB_DIR}/untracked.md`,
+      ]);
+      expect(JSON.stringify(res.body)).not.toContain('outside');
+      expect(JSON.stringify(res.body)).not.toContain('git internals');
+    } finally {
+      await rm(outside, { recursive: true, force: true });
     }
-    const { status, body, took } = await searching;
-    expect(status).toBe(400);
-    expect(body.error).toMatch(/^Pattern too expensive: /);
-    expect(body.code).toBe('pattern_too_expensive');
-    expect(body.error).toContain("this search's 5 s budget");
-    expect(took).toBeLessThan(5_000 + 2_000);
-    expect(probes.length).toBeGreaterThan(0);
-    expect(Math.max(...probes)).toBeLessThan(1_000);
   });
 
-  it('stops it in a directory walk too, rather than skipping the file as unreadable', async () => {
-    const base = await start();
-    await fs.writeFile(`${KB_DIR}/redos.md`, `${LONG_LINE}\n`);
-    const res = await post(`${base}/api/agent/tools/grep`, { pattern: '^(a+)+$' });
-    expect(res.status).toBe(400);
-    expect(((await res.json()) as { error: string }).error).toMatch(/^Pattern too expensive: /);
+  it('never answers a line from a file the caller may not read, nor counts it against max_results', async () => {
+    const base = await start('read', denyReads(new Set(['a-secret.md', 'Locked/b.md'])));
+    await fs.writeFile(`${KB_DIR}/a-secret.md`, 'needle 1\nneedle 2\nneedle 3\n');
+    await fs.writeFile(`${KB_DIR}/Locked/b.md`, 'needle locked\n');
+    await fs.writeFile(`${KB_DIR}/open.md`, 'needle open\n');
+    const res = await grep(base, { pattern: 'needle', max_results: 1 });
+    expect(res.body.matches).toEqual([{ path: `${KB_DIR}/open.md`, line: 1, text: 'needle open' }]);
+    expect(res.body.truncated).toBe(true);
   });
 
-  it('answers a directory walk 503 when no worker is free, rather than skipping every file as unreadable', async () => {
-    // One worker and room for one search to wait: two expensive greps fill both.
-    matcherOptions = { budgetMs: 1_000, poolMax: 1, maxWaiting: 1 };
+  // Four million matching lines: a file ripgrep searched would spend far more
+  // than the 1.5 s budget answering them.
+  const huge = 'needle\n'.repeat(4_000_000);
+
+  it('never searches a file the caller may not read, so what it holds cannot turn the answer into a 400', async () => {
+    matcherOptions = { budgetMs: 1_500 };
+    const base = await start('read', denyReads(new Set(['Locked/huge.md'])));
+    await fs.writeFile(`${KB_DIR}/Locked/huge.md`, huge);
+    await fs.writeFile(`${KB_DIR}/open.md`, 'needle open\n');
+    const res = await grep(base, { pattern: 'needle' });
+    expect(res.status).toBe(200);
+    expect(res.body.matches).toEqual([{ path: `${KB_DIR}/open.md`, line: 1, text: 'needle open' }]);
+  }, 20_000);
+
+  it('spends the budget on judging who may read what too: a verdict that outlasts it answers the 400 at the budget', async () => {
+    matcherOptions = { budgetMs: 1_000 };
+    // An access check that takes ten seconds, as a cold one over a large tree could.
+    const slowAccess = {
+      ...allowAll,
+      canReadBatch: (_w: string, _u: string, paths: string[]) =>
+        new Promise<Map<string, boolean>>((resolve) => setTimeout(() => resolve(new Map(paths.map((p) => [p, true]))), 10_000)),
+    } as unknown as IAccessControl;
+    const base = await start('read', slowAccess);
+    await fs.writeFile(`${KB_DIR}/open.md`, 'needle open\n');
+    const started = Date.now();
+    const res = await grep(base, { pattern: 'needle' });
+    expect(res.status).toBe(400);
+    expect(res.body.code).toBe('pattern_too_expensive');
+    expect(Date.now() - started).toBeLessThan(3_000);
+  }, 20_000);
+
+  it('stops the search at the file that fills max_results, so what comes after it costs nothing', async () => {
+    matcherOptions = { budgetMs: 1_500 };
     const base = await start();
-    await fs.writeFile(`${KB_DIR}/redos.md`, `${LONG_LINE}\n`);
-    await fs.writeFile(`${KB_DIR}/hit.md`, 'needle\n');
-    const expensive = () =>
-      post(`${base}/api/agent/tools/grep`, { pattern: '^(a+)+$', path: `${KB_DIR}/redos.md` }).then((r) => r.status);
-    const holding = expensive();
-    await new Promise((r) => setTimeout(r, 200));
-    const queued = expensive();
-    await new Promise((r) => setTimeout(r, 100));
-    const res = await post(`${base}/api/agent/tools/grep`, { pattern: 'needle' });
-    expect(res.status).toBe(503);
-    expect(((await res.json()) as { code?: string }).code).toBe('search_busy');
-    // The two that held the pool answered as they always would, and it serves again after.
-    expect(await Promise.all([holding, queued])).toEqual([400, 400]);
-    const after = await post(`${base}/api/agent/tools/grep`, { pattern: 'needle' });
-    expect(after.status).toBe(200);
-  });
-
-  it('spends one budget across every file a search reads: each alone fits, all of them together do not', async () => {
-    // How long the pattern takes over one line IN A WORKER, which backtracks at
-    // its own pace, not this thread's: once on a fresh worker (`first`, which
-    // can be several times slower) and then again on the same one (`again`).
-    // Timed through a probe matcher of the same kind, on a line long enough
-    // that `again` is at least 50 ms.
-    let line = '';
-    let first = 0;
-    let again = 0;
-    for (let n = 16; again < 50; n++) {
-      line = `${'a'.repeat(n)}!`;
-      const probe = new WorkerPatternMatcher();
-      try {
-        await probe.search('x', '').matchingLines(['x'], 1); // the worker is up before anything is timed
-        let t = performance.now();
-        await probe.search('^(a+)+$', '').matchingLines([line], 1);
-        first = performance.now() - t;
-        t = performance.now();
-        await probe.search('^(a+)+$', '').matchingLines([line], 1);
-        again = performance.now() - t;
-      } finally {
-        await probe.close();
-      }
+    await fs.writeFile(`${KB_DIR}/a.md`, 'needle first\n');
+    await fs.writeFile(`${KB_DIR}/z/huge.md`, huge);
+    const res = await grep(base, { pattern: 'needle', max_results: 1 });
+    expect(res.status).toBe(200);
+    expect(res.body.matches).toEqual([{ path: `${KB_DIR}/a.md`, line: 1, text: 'needle first' }]);
+    expect(res.body.truncated).toBe(true);
+  }, 20_000);
+
+  it('reads a file of millions of matching lines only as far as max_results, in a tree and named alone', async () => {
+    matcherOptions = { budgetMs: 1_500 };
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/a/huge.md`, huge);
+    await fs.writeFile(`${KB_DIR}/b.md`, 'needle b\n');
+    const expected = Array.from({ length: 200 }, (_, i) => ({ path: `${KB_DIR}/a/huge.md`, line: i + 1, text: 'needle' }));
+    for (const path of [KB_DIR, `${KB_DIR}/a/huge.md`]) {
+      const res = await grep(base, { pattern: 'needle', path });
+      expect(res.status).toBe(200);
+      expect(res.body.matches).toEqual(expected);
+      expect(res.body.truncated).toBe(true);
     }
-    // A budget well above what any one file costs, and enough files that
-    // together they cost about twice the budget.
-    const budgetMs = Math.ceil(Math.max(first, again) * 1.5) + 200;
-    const files = Math.ceil((2 * budgetMs) / again);
-    matcherOptions = { budgetMs };
+  }, 20_000);
+
+  it('a named file whose only NUL lies past the first 64 KB has no searchable text; the same length of text has no matches and no note', async () => {
+    matcherOptions = {};
     const base = await start();
-    for (let i = 1; i <= files; i++) await fs.writeFile(`${KB_DIR}/slow-${String(i).padStart(3, '0')}.md`, `${line}\n`);
-    const all = await post(`${base}/api/agent/tools/grep`, { pattern: '^(a+)+$' });
-    expect(all.status).toBe(400);
-    const body = (await all.json()) as { error: string; code?: string };
-    expect(body.code).toBe('pattern_too_expensive');
-    expect(body.error).toMatch(/^Pattern too expensive: /);
-    // Any one of the same files, searched alone, fits the same budget.
-    const one = await post(`${base}/api/agent/tools/grep`, { pattern: '^(a+)+$', path: `${KB_DIR}/slow-001.md` });
-    expect(one.status).toBe(200);
-    expect(((await one.json()) as { matches: unknown[] }).matches).toEqual([]);
-  });
-
-  it('a control pattern over the same line still matches', async () => {
+    // ripgrep stops reading a file at the lines the answer can take, so a NUL
+    // far past the pattern's last chance is found by the read that follows,
+    // chunk by chunk — here in the fourth chunk.
+    const filler = Buffer.alloc(200 * 1024, 'x');
+    await fs.writeFile(`${KB_DIR}/late.md`, Buffer.concat([filler, Buffer.from([0])]));
+    await fs.writeFile(`${KB_DIR}/plain.md`, filler);
+    const binary = await grep(base, { pattern: 'needle', path: `${KB_DIR}/late.md` });
+    expect(binary.status).toBe(200);
+    expect(binary.body.matches).toEqual([]);
+    expect(binary.body.note).toContain('no searchable text');
+    const text = await grep(base, { pattern: 'needle', path: `${KB_DIR}/plain.md` });
+    expect(text.status).toBe(200);
+    expect(text.body.matches).toEqual([]);
+    expect(text.body.note).toBeUndefined();
+  });
+
+  it('stops the search when its caller goes away, leaving no ripgrep running', async () => {
+    const slow = await slowRipgrep();
+    try {
+      matcherOptions = {};
+      const base = await start();
+      const gone = new AbortController();
+      const call = fetch(`${base}/api/agent/tools/grep`, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json', authorization: 'Bearer x' },
+        body: JSON.stringify({ branch: 'main', pattern: 'needle' }),
+        signal: gone.signal,
+      }).catch(() => undefined);
+      while ((await slow.alive()).length === 0) await new Promise((r) => setTimeout(r, 20));
+      const left = Date.now();
+      gone.abort();
+      await call;
+      // Well inside the 5 s budget the search would otherwise have run to.
+      while ((await slow.alive()).length > 0 && Date.now() - left < 4_000) await new Promise((r) => setTimeout(r, 20));
+      expect(await slow.alive()).toEqual([]);
+      expect(Date.now() - left).toBeLessThan(1_000);
+    } finally {
+      await slow.remove();
+    }
+  }, 15_000);
+
+  it('searches node_modules and folders of any depth, and a link inside the workspace only where it points', async () => {
     const base = await start();
-    await fs.writeFile(`${KB_DIR}/redos.md`, `intro\n${LONG_LINE}\n`);
-    const res = await post(`${base}/api/agent/tools/grep`, { pattern: '^a+!$', path: `${KB_DIR}/redos.md` });
+    const deep = Array.from({ length: 15 }, (_, i) => `d${i}`).join('/');
+    await fs.writeFile(`${KB_DIR}/node_modules/pkg/readme.md`, 'needle in node_modules\n');
+    await fs.writeFile(`${KB_DIR}/${deep}/deep.md`, 'needle fifteen levels down\n');
+    await fs.writeFile(`${KB_DIR}/real/target.md`, 'needle at the target\n');
+    await symlink(join(tempDir, KB_DIR, 'real', 'target.md'), join(tempDir, KB_DIR, 'file-link.md'));
+    await symlink(join(tempDir, KB_DIR, 'real'), join(tempDir, KB_DIR, 'folder-link'));
+    const res = await grep(base, { pattern: 'needle' });
     expect(res.status).toBe(200);
-    expect(((await res.json()) as { matches: unknown[] }).matches).toEqual([{ path: `${KB_DIR}/redos.md`, line: 2, text: LONG_LINE }]);
+    expect(res.body.matches.map((m) => m.path)).toEqual([
+      `${KB_DIR}/${deep}/deep.md`,
+      `${KB_DIR}/node_modules/pkg/readme.md`,
+      `${KB_DIR}/real/target.md`,
+    ]);
+  });
+
+  it('leaves out a file whose name holds a line break, which no tool can read, and answers the rest', async () => {
+    const base = await start();
+    await writeFile(join(tempDir, KB_DIR, 'two\nlines.md'), 'needle in an odd name\n');
+    await fs.writeFile(`${KB_DIR}/plain.md`, 'needle plain\n');
+    expect((await post(`${base}/api/agent/tools/read_file`, { path: `${KB_DIR}/two\nlines.md` })).status).toBe(400);
+    const res = await grep(base, { pattern: 'needle' });
+    expect(res.status).toBe(200);
+    expect(res.body.matches).toEqual([{ path: `${KB_DIR}/plain.md`, line: 1, text: 'needle plain' }]);
+  });
+
+  it('answers the first max_results by path then line, whatever reads them, and marks the answer truncated', async () => {
+    const docx = (text: string): Buffer => {
+      const zip = new AdmZip();
+      zip.addFile('[Content_Types].xml', Buffer.from('<?xml version="1.0"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"/>'));
+      zip.addFile(
+        'word/document.xml',
+        Buffer.from(`<?xml version="1.0"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>${text}</w:t></w:r></w:p></w:body></w:document>`),
+      );
+      return zip.toBuffer();
+    };
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/b.md`, 'x\nneedle b2\nneedle b3\n');
+    await fs.writeFile(`${KB_DIR}/a/z.md`, 'needle a/z\n');
+    await fs.writeFile(`${KB_DIR}/a.md`, 'needle a.md\n');
+    await fs.writeFile(`${KB_DIR}/a/m.docx`, docx('needle in a document'));
+    await fs.writeFile(`${KB_DIR}/c.md`, `needle ${'c'.repeat(400)}\n`);
+    const all = await grep(base, { pattern: 'needle' });
+    expect(all.body.matches.map((m) => [m.path, m.line])).toEqual([
+      [`${KB_DIR}/a/m.docx`, 2],
+      [`${KB_DIR}/a/z.md`, 1],
+      [`${KB_DIR}/a.md`, 1],
+      [`${KB_DIR}/b.md`, 2],
+      [`${KB_DIR}/b.md`, 3],
+      [`${KB_DIR}/c.md`, 1],
+    ]);
+    expect(all.body.truncated).toBe(false);
+    expect(all.body.matches[5]!.text).toHaveLength(300);
+    const capped = await grep(base, { pattern: 'needle', max_results: 4 });
+    expect(capped.body.matches).toEqual(all.body.matches.slice(0, 4));
+    expect(capped.body.truncated).toBe(true);
+    // The default cap is 200.
+    await fs.writeFile(`${KB_DIR}/many.md`, 'needle\n'.repeat(250));
+    const many = await grep(base, { pattern: 'needle', path: `${KB_DIR}/many.md` });
+    expect(many.body.matches).toHaveLength(200);
+    expect(many.body.truncated).toBe(true);
+  });
+
+  it('honours ignore_case', async () => {
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/case.md`, 'Needle\nneedle\nNEEDLE\n');
+    expect((await grep(base, { pattern: 'NEEDLE', path: `${KB_DIR}/case.md` })).body.matches.map((m) => m.line)).toEqual([3]);
+    expect((await grep(base, { pattern: 'NEEDLE', path: `${KB_DIR}/case.md`, ignore_case: true })).body.matches.map((m) => m.line)).toEqual([1, 2, 3]);
+    expect((await grep(base, { pattern: 'NEEDLE', ignore_case: true })).body.matches.map((m) => m.line)).toEqual([1, 2, 3]);
+  });
+
+  it('searches a pattern and a path beginning with `-` as text, never as an option', async () => {
+    const base = await start();
+    await fs.writeFile(`${KB_DIR}/-dir/-n.md`, 'run it with -n\nnothing here\n');
+    for (const path of [undefined, `${KB_DIR}/-dir`, `${KB_DIR}/-dir/-n.md`, '-dir']) {
+      const res = await grep(base, path === undefined ? { pattern: '-n' } : { pattern: '-n', path });
+      expect(res.status).toBe(200);
+      expect(res.body.matches).toEqual([{ path: path === '-dir' ? `${KB_DIR}/-dir/-n.md` : `${KB_DIR}/-dir/-n.md`, line: 1, text: 'run it with -n' }]);
+    }
+    // `--files` as a pattern is a pattern too: it matches nothing here rather than listing files.
+    expect((await grep(base, { pattern: '--files' })).body.matches).toEqual([]);
+  });
+
+  it('a pattern carrying a NUL is refused with the escape to use instead', async () => {
+    const base = await start();
+    const res = await grep(base, { pattern: 'a\u0000b' });
+    expect(res.status).toBe(400);
+    expect(res.body.error).toContain('\\x00');
   });
 });
 
@@ -2137,6 +2407,7 @@ describe('images', () => {
 describe('start_session', () => {
   // Minimal harness: own app + a fake ISessionSink that records createSession.
   let server: HttpServer | undefined;
+  let sessionExtractor: WorkerDocumentExtractor | undefined;
   let created: Array<{ userId: string; startedAt: Date }> = [];
 
   async function startSessionApp(
@@ -2189,9 +2460,12 @@ describe('start_session', () => {
     const app = express();
     app.use(express.json());
     const router = express.Router();
+    await sessionExtractor?.close();
+    const extractor = new WorkerDocumentExtractor();
+    sessionExtractor = extractor;
     registerWorkspaceTools(
       registry, router, auth, toolHandler,
-      new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract')), access, testKbContext({ kbDirName: KB_DIR }),
+      new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract'), extractor), access, testKbContext({ kbDirName: KB_DIR }),
       { recoveryBotEmail: RECOVERY_BOT, hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },
       new RoutineWritePolicyService(),
       sink ?? fakeSessionSink,
@@ -2213,6 +2487,8 @@ describe('start_session', () => {
   afterEach(async () => {
     if (server) await new Promise<void>((r) => server!.close(() => r()));
     server = undefined;
+    await sessionExtractor?.close();
+    sessionExtractor = undefined;
   });
 
   it('returns the sink-minted id as sessionId', async () => {
@@ -2321,9 +2597,10 @@ describe('start_session', () => {
     const registry = new ToolRegistry();
     const router = express.Router();
     const noopAuth: express.RequestHandler = (_req, _res, next) => next();
+    const extractor = new WorkerDocumentExtractor();
     registerWorkspaceTools(
       registry, router, noopAuth, (() => () => {}) as never,
-      new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract')), allowAll, testKbContext({ kbDirName: KB_DIR }),
+      new SpillStore(join(tmpdir(), 'bevel-test-spills')), new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract'), extractor), allowAll, testKbContext({ kbDirName: KB_DIR }),
       { recoveryBotEmail: RECOVERY_BOT, hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },
       new RoutineWritePolicyService(),
       {} as never,
@@ -2341,6 +2618,7 @@ describe('start_session', () => {
     // And that the answer may carry a note to act on, so an agent reading
     // only the catalog knows the field is not noise.
     expect(description).toContain('`firstRun`');
+    await extractor.close();
   });
 
   /**
@@ -2484,9 +2762,16 @@ describe('start_session', () => {
  * schema layer, complementing the handler-level guard exercised above.
  */
 describe('branch is a required parameter in the tool contract', () => {
+  /** Every extractor a registry was built with, closed after each test. */
+  const extractors: WorkerDocumentExtractor[] = [];
+  afterEach(async () => {
+    await Promise.all(extractors.splice(0).map((e) => e.close()));
+  });
   /** Register the workspace tools into a fresh registry (no server needed). */
   function buildRegistry(): ToolRegistry {
     const registry = new ToolRegistry();
+    const extractor = new WorkerDocumentExtractor();
+    extractors.push(extractor);
     const router = express.Router();
     const noopAuth: express.RequestHandler = (_req, _res, next) => next();
     // The handler factory is only invoked to build routes; its output is never
@@ -2498,7 +2783,7 @@ describe('branch is a required parameter in the tool contract', () => {
       noopAuth,
       toolHandler,
       new SpillStore(join(tmpdir(), 'bevel-test-spills')),
-      new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract')),
+      new DocExtractService(join(tmpdir(), 'bevel-test-doc-extract'), extractor),
       allowAll,
       testKbContext({ kbDirName: KB_DIR }),
       { recoveryBotEmail: RECOVERY_BOT, hooks: new WorkflowHooks(), notes: new ToolDescriptionNotes() },
@@ -2513,7 +2798,7 @@ describe('branch is a required parameter in the tool contract', () => {
       undefined,
       undefined,
       // So `grep` is mounted; it spawns no worker, as nothing here searches.
-      new WorkerPatternMatcher(),
+      new RipgrepPatternMatcher(),
     );
     return registry;
   }
@@ -4542,17 +4827,38 @@ describe('agent read/write hooks', () => {
     expect(writes).toEqual([]);
   });
 
-  it('grep tells the read hook about every file the walk opens, not only its root', async () => {
+  it('grep tells the read hook about every file it opens — a file with no matching line included — not only its root', async () => {
     const base = await start();
     record();
     await fs.mkdir(`${KB_DIR}/KnowledgeBase/Product/Knowledge`, { recursive: true });
     await fs.mkdir(`${KB_DIR}/KnowledgeBase/Legal/Knowledge`, { recursive: true });
     await fs.writeFile(`${KB_DIR}/KnowledgeBase/Product/Knowledge/P.md`, 'needle');
     await fs.writeFile(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/L.md`, 'needle');
+    // Searched and read by ripgrep like the others; it answers no line.
+    await fs.writeFile(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`, 'nothing of the sort');
     await post(`${base}/api/agent/tools/grep`, { pattern: 'needle', path: KB_DIR, sessionId: 's1' });
     const paths = reads.map((r) => r.wsPath);
     expect(paths).toContain(`${KB_DIR}/KnowledgeBase/Product/Knowledge/P.md`);
     expect(paths).toContain(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/L.md`);
+    expect(paths).toContain(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`);
+    expect(paths.filter((p) => p === `${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`)).toHaveLength(1);
+  });
+
+  it('a read hook that refuses a file the search would open ends the grep as it ends a read_file, even one that matches nothing', async () => {
+    const base = await start();
+    record();
+    hooks.onAgentRead(async (op) => {
+      if (op.wsPath === `${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`) throw new ToolError('Quiet.md is off limits to this run', 403);
+    });
+    await fs.mkdir(`${KB_DIR}/KnowledgeBase/Product/Knowledge`, { recursive: true });
+    await fs.mkdir(`${KB_DIR}/KnowledgeBase/Legal/Knowledge`, { recursive: true });
+    await fs.writeFile(`${KB_DIR}/KnowledgeBase/Product/Knowledge/P.md`, 'needle');
+    await fs.writeFile(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`, 'nothing of the sort');
+    const res = await post(`${base}/api/agent/tools/grep`, { pattern: 'needle', path: KB_DIR, sessionId: 's1' });
+    expect(res.status).toBe(403);
+    expect(await res.json()).toMatchObject({ error: 'Quiet.md is off limits to this run' });
+    // The refusal came at admission: the hook heard of the file, and the search answered nothing from any file.
+    expect(reads.map((r) => r.wsPath)).toContain(`${KB_DIR}/KnowledgeBase/Legal/Knowledge/Quiet.md`);
   });
 
   it('the write hook covers write_file, edit_file and mkdir, once per path', async () => {