@bevel-software/platform-core-backend 0.33.0 → 0.35.0

package/src/modules/workspace/agent-access.gate.ts

@@ -32,7 +32,7 @@ import type { WorkflowHooks } from '../workflow/index.js';
  * takes, so a run's reads and its questions share one conversation.
  */
 export const SESSION_ID_DESCRIPTION =
-  "This conversation's id, so the server can tell one run's calls apart. External agents (direct MCP calls and `call_tool_chain` alike): call the `start_session` tool ONCE at the very start of your work to obtain it, then pass that id here — the same id `ask` takes. The in-process agent runtime supplies this automatically (do not set it yourself).";
+  "This conversation's id: call `start_session` once, at the start of your work, and pass its id here, inside `call_tool_chain` too (and to `ask`, where the deployment offers it). The in-process agent runtime sets it itself.";
 
 /**
  * The `sessionId` input every gated tool declares. The description a MOUNTED

package/src/modules/workspace/file-readers/__tests__/doc-extract.service.test.ts

@@ -152,6 +152,16 @@ describe('DocExtractService — how long a failure is remembered', () => {
     expect(told).toHaveLength(1);
   });
 
+  it('two reads of the same bytes at different paths share one extraction, and each marker names its own path', async () => {
+    const extractor = scriptedExtractor(() => ({ ok: true, summary: '1 page', text: '[page 1]\nshared' }));
+    const service = new DocExtractService(cacheRoot, extractor, { now, onUnreadable });
+    const bytes = Buffer.from('same bytes, two names');
+    const [a, b] = await Promise.all([service.extract('a/deck.pdf', bytes, ORIGIN), service.extract('b/copy.pdf', bytes, ORIGIN)]);
+    expect(extractor.calls).toBe(1);
+    expect(a).toEqual({ ok: true, marker: '[extracted text of a/deck.pdf — 1 page]', text: '[page 1]\nshared' });
+    expect(b).toEqual({ ok: true, marker: '[extracted text of b/copy.pdf — 1 page]', text: '[page 1]\nshared' });
+  });
+
   it('still answers the read when whoever is told throws', async () => {
     const extractor = scriptedExtractor(() => TIMEOUT);
     const service = new DocExtractService(cacheRoot, extractor, {

package/src/modules/workspace/file-readers/doc-extract.service.ts

@@ -1,5 +1,5 @@
-import type { IDocumentExtractor, IsolatedExtraction } from './document-extractor.contract.js';
-import { extractionMarker, fileExtension } from './doc-extract.types.js';
+import type { IDocumentExtractor } from './document-extractor.contract.js';
+import { extractionMarker, fileExtension, type ExtractedDoc } from './doc-extract.types.js';
 import { DocExtractionCache, gitBlobSha } from './extraction-cache.js';
 import type { ReadOrigin } from './file-reader.js';
 
@@ -15,6 +15,18 @@ export type DocExtractOutcome =
   | ({ ok: true } & DocExtraction)
   | { ok: false; message: string };
 
+/**
+ * What one read of some bytes answers before any path names it: the cache's
+ * own shape (an extraction, or a failure's message). `named` turns it into
+ * what `extract` returns for the path it was asked about.
+ */
+type RawOutcome = ({ ok: true } & ExtractedDoc) | { ok: false; message: string };
+
+/** `raw` as the answer for `path`: its marker names that path. */
+function named(path: string, raw: RawOutcome): DocExtractOutcome {
+  return raw.ok ? { ok: true, marker: extractionMarker(path, raw.summary), text: raw.text } : raw;
+}
+
 /** A document whose extraction just failed, as the people looking at it are told. */
 export interface UnreadableDocument {
   workspaceId: string;
@@ -94,12 +106,23 @@ export class DocExtractService {
   private readonly now: () => number;
   private readonly onUnreadable: ((doc: UnreadableDocument) => void) | undefined;
   /**
-   * Cold extractions currently running, by cache key. Two reads of the same
-   * document arriving together would otherwise BOTH parse it — the expensive
-   * half of this service, run twice for one answer — because neither had
-   * written the cache entry yet when the other looked.
+   * Reads currently running, by cache key — the cache lookup included, from
+   * the moment `extract` is called. Two reads of the same document arriving
+   * together would otherwise BOTH parse it — the expensive half of this
+   * service, run twice for one answer — because neither had written the
+   * cache entry yet when the other looked. The entry is set before anything
+   * is awaited: were the lookup awaited first, a second read could miss the
+   * cache (its read of the file started before the first read's write) AND
+   * the map (the first read finished and left it in that window), and parse
+   * the document again, telling the people looking at it twice.
+   *
+   * What is shared is exactly what the key identifies — the bytes' extraction
+   * under a format, which is what the cache holds — and nothing of the read
+   * that started it: the key carries no path, so two reads of the same bytes
+   * at different paths share one promise, and each names the answer with its
+   * own path (`extract`).
    */
-  private readonly inFlight = new Map<string, Promise<IsolatedExtraction>>();
+  private readonly inFlight = new Map<string, Promise<RawOutcome>>();
 
   constructor(
     cacheRoot: string,
@@ -119,11 +142,8 @@ export class DocExtractService {
    * only to cold ones; `file_stat` uses it to report a remembered failure.
    */
   async getCached(path: string, bytes: Buffer): Promise<DocExtractOutcome | undefined> {
-    const hit = await this.cache.get(this.cacheKey(path, bytes), this.now());
-    if (!hit) return undefined;
-    return hit.failed
-      ? { ok: false, message: hit.message }
-      : { ok: true, marker: extractionMarker(path, hit.summary), text: hit.text };
+    const hit = await this.remembered(this.cacheKey(path, bytes));
+    return hit && named(path, hit);
   }
 
   /**
@@ -132,33 +152,46 @@ export class DocExtractService {
    * `origin`, when given, is where a fresh failure is reported.
    */
   async extract(path: string, bytes: Buffer, origin?: ReadOrigin): Promise<DocExtractOutcome> {
+    // The bytes are hashed BEFORE the extractor takes its copy of them.
     const key = this.cacheKey(path, bytes);
-    const remembered = await this.getCached(path, bytes);
-    if (remembered) return remembered;
-    const running = this.inFlight.get(key);
-    const result = await (running ??
-      (() => {
-        // The cache write stays INSIDE the shared promise: were the entry
-        // dropped as soon as parsing settled, a read arriving during the
-        // write would miss both the cache and the in-flight map — and parse
-        // the same document again. The bytes are hashed (above) BEFORE the
-        // extractor takes its copy of them.
-        const started = (async (): Promise<IsolatedExtraction> => {
-          const res = await this.extractor.extract(fileExtension(path), bytes);
-          if (res.ok) {
-            await this.cache.put(key, { summary: res.summary, text: res.text });
-          } else if (res.failure !== 'busy') {
-            const until = res.failure === 'parser' ? undefined : this.now() + TIMEOUT_MEMORY_MS;
-            await this.cache.put(key, { failed: true, message: res.message, ...(until === undefined ? {} : { until }) });
-            if (origin) this.notifyUnreadable({ workspaceId: origin.workspaceId, path, reason: reasonOf(res.message) });
-          }
-          return res;
-        })().finally(() => this.inFlight.delete(key));
-        this.inFlight.set(key, started);
-        return started;
-      })());
-    if (!result.ok) return { ok: false, message: result.message };
-    return { ok: true, marker: extractionMarker(path, result.summary), text: result.text };
+    return named(path, await (this.inFlight.get(key) ?? this.read(key, path, bytes, origin)));
+  }
+
+  /** What the cache holds for `key`, or undefined on a miss. */
+  private async remembered(key: string): Promise<RawOutcome | undefined> {
+    const hit = await this.cache.get(key, this.now());
+    if (!hit) return undefined;
+    return hit.failed ? { ok: false, message: hit.message } : { ok: true, summary: hit.summary, text: hit.text };
+  }
+
+  /**
+   * One read of `bytes`, shared by every `extract` of the same key that
+   * arrives before it settles (see `inFlight`): the cache lookup, then the
+   * extraction and its cache write. The write stays INSIDE the shared
+   * promise: were the entry dropped as soon as parsing settled, a read
+   * arriving during the write would miss both the cache and the map — and
+   * parse the same document again. A failure is reported to the FIRST
+   * read's origin and path, once; `path` otherwise gives only the format,
+   * which every read of the key shares.
+   */
+  private read(key: string, path: string, bytes: Buffer, origin: ReadOrigin | undefined): Promise<RawOutcome> {
+    const started = (async (): Promise<RawOutcome> => {
+      const hit = await this.remembered(key);
+      if (hit) return hit;
+      const res = await this.extractor.extract(fileExtension(path), bytes);
+      if (res.ok) {
+        await this.cache.put(key, { summary: res.summary, text: res.text });
+        return { ok: true, summary: res.summary, text: res.text };
+      }
+      if (res.failure !== 'busy') {
+        const until = res.failure === 'parser' ? undefined : this.now() + TIMEOUT_MEMORY_MS;
+        await this.cache.put(key, { failed: true, message: res.message, ...(until === undefined ? {} : { until }) });
+        if (origin) this.notifyUnreadable({ workspaceId: origin.workspaceId, path, reason: reasonOf(res.message) });
+      }
+      return { ok: false, message: res.message };
+    })().finally(() => this.inFlight.delete(key));
+    this.inFlight.set(key, started);
+    return started;
   }
 
   /** Best-effort, like every event: a listener that throws never fails the read. */

package/src/modules/workspace/index.ts

@@ -45,6 +45,26 @@ export { SpillStore } from './spill-store.js';
 export { DocExtractService } from './file-readers/doc-extract.service.js';
 export type { IDocumentExtractor } from './file-readers/document-extractor.contract.js';
 export { WorkerDocumentExtractor } from './file-readers/extract-in-worker.js';
+/**
+ * What `grep` runs a caller's pattern with: the matcher contract, its one
+ * implementation (ripgrep), and the boot check that ripgrep is there. The
+ * composition root wires them through this surface, as a suite does.
+ */
+export {
+  compareSearchPaths,
+  PATTERN_DIALECT,
+  PatternRefusedError,
+  PatternTooExpensiveError,
+  SearchAbandonedError,
+  SearchBusyError,
+  type FileMatches,
+  type IPatternMatcher,
+  type IPatternSearch,
+  type TreeScope,
+  type TreeSearchOptions,
+} from './pattern-matcher.contract.js';
+export { RipgrepPatternMatcher, type RipgrepPatternMatcherOptions } from './ripgrep-search.js';
+export { assertRipgrep } from './ripgrep-version.js';
 // Reached by other modules; exported here so they stop importing the files.
 export { isTextBytes } from './file-readers/text-reader.js';
 export { isUntouchedStarterPage, knowledgeFolderIsNew } from './first-run.js';

package/src/modules/workspace/nul-scan.ts

@@ -0,0 +1,50 @@
+import { constants as fsConstants, type Stats } from 'node:fs';
+import nodeFs from 'node:fs/promises';
+import type { IPatternSearch } from './pattern-matcher.contract.js';
+
+/** How much of a file {@link holdsNul} holds in memory at once. */
+export const NUL_SCAN_CHUNK = 64 * 1024;
+
+/**
+ * Whether the file at `absolute` holds a NUL, read a chunk at a time and
+ * stopped at the first, under the search's budget: a check that outlives it
+ * is answered as the search is, and reads no further. The file is opened
+ * without following a link, and only the file that was judged (`judged`, its
+ * device and inode) is read: a link or another file swapped in since, here or
+ * in a folder above it, is not, and holds nothing.
+ *
+ * The check a named text file gets from `grep` when ripgrep answers no line
+ * from it (see `workspace.tools.ts`, `grepTextFile`): ripgrep stops reading a
+ * file at the lines the answer can take, so a NUL past them goes unseen by it,
+ * and "no searchable text" must be told from "no match" by this read.
+ */
+export async function holdsNul(absolute: string, judged: Stats, search: IPatternSearch): Promise<boolean> {
+  let stopped = false;
+  try {
+    return await search.charged(async () => {
+      let handle: nodeFs.FileHandle;
+      try {
+        handle = await nodeFs.open(absolute, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
+      } catch (err) {
+        // ELOOP: the name is a link now. Anything else is the caller's to answer.
+        if ((err as { code?: unknown } | null)?.code === 'ELOOP') return false;
+        throw err;
+      }
+      try {
+        const opened = await handle.stat();
+        if (!opened.isFile() || opened.dev !== judged.dev || opened.ino !== judged.ino) return false;
+        const chunk = Buffer.alloc(NUL_SCAN_CHUNK);
+        while (!stopped) {
+          const { bytesRead } = await handle.read(chunk, 0, chunk.length, null);
+          if (bytesRead === 0) return false;
+          if (chunk.subarray(0, bytesRead).includes(0)) return true;
+        }
+        return false;
+      } finally {
+        await handle.close();
+      }
+    });
+  } finally {
+    stopped = true;
+  }
+}

package/src/modules/workspace/pattern-matcher.contract.ts

@@ -3,47 +3,123 @@ import { ToolError } from '../tool-helpers/index.js';
 /**
  * The contract for running a CALLER'S regular expression: what `grep` (REST,
  * MCP and `call_tool_chain` alike, one handler) depends on, so it never holds
- * a concrete engine. The one implementation is `WorkerPatternMatcher`
- * (`bounded-matcher.ts`), owned by the composition root and closed with the
+ * a concrete engine. The one implementation is `RipgrepPatternMatcher`
+ * (`ripgrep-search.ts`), owned by the composition root and closed with the
  * graph.
  *
- * JavaScript's engine backtracks, so a pattern like `^(a+)+$` over a line of
- * forty `a`s and a `!` never finishes. An implementation must therefore never
- * run the pattern on the request thread, must stop a search whose matching
- * outlives its budget with {@link PatternTooExpensiveError}, and must refuse
- * work it cannot take on with {@link SearchBusyError} rather than queue it
- * without bound. Within budget, a pattern matches exactly the lines
- * `RegExp#test` does.
+ * The engine is ripgrep's default one: linear-time, so no pattern can stall
+ * it, and RE2-style, so it refuses look-around and backreferences. It runs in
+ * child processes, never on the request thread. An implementation must stop a
+ * search that outlives its budget with {@link PatternTooExpensiveError},
+ * refuse a pattern its engine refuses with {@link PatternRefusedError}, and
+ * refuse work it cannot take on with {@link SearchBusyError} rather than queue
+ * it without bound.
  */
 export interface IPatternMatcher {
   /**
-   * One search's matcher for `source`/`flags`: its budget is spent across
-   * every call it makes. Throws the 400 `Invalid regex` the caller's syntax
-   * earns, or the 400 `pattern_too_long` an over-long pattern earns.
+   * One search's matcher for `pattern`: its budget is spent across every call
+   * it makes. Rejects with the 400 {@link PatternRefusedError} when the engine
+   * refuses the pattern, or the 400 `pattern_too_long` an over-long pattern
+   * earns, before any file is searched. When `signal` aborts (the caller went
+   * away), every run the search has going is killed, and the call it was
+   * serving rejects with {@link SearchAbandonedError}.
    */
-  search(source: string, flags: string): IPatternSearch;
-  /** Ends the workers; a search after this is refused. */
+  search(pattern: string, opts: { ignoreCase: boolean; signal?: AbortSignal }): Promise<IPatternSearch>;
+  /** Ends every running search; a search after this is refused. */
   close(): Promise<void>;
 }
 
-/** One search, under one budget. */
+/** Where a tree search runs: a directory, never a file. */
+export interface TreeScope {
+  /** Absolute, canonical directory the search runs in. */
+  cwd: string;
+  /** The directory searched, relative to `cwd` (`.` for `cwd` itself). */
+  root: string;
+}
+
+/** Which files a tree search opens, and how much of each it keeps. */
+export interface TreeSearchOptions {
+  /**
+   * The files searched, relative to the scope's `root` as
+   * {@link IPatternSearch.list} answers them; no other file is opened. A name
+   * holding a line break cannot be named to ripgrep exactly and is refused.
+   */
+  only: readonly string[];
+  /**
+   * The most lines one file answers: matching stops at it, so a file of
+   * millions of matching lines costs its first few and one look for a NUL
+   * in the rest, not a record for every line. A matching line that is not
+   * UTF-8 is not answered and does not use up the cap.
+   */
+  maxLinesPerFile: number;
+}
+
+/** One file's matching lines, as a tree search finds them. */
+export interface FileMatches {
+  /** The file, relative to the scope's `root`, `/`-separated. */
+  path: string;
+  /** Its matching lines, in order: 1-based numbers, the whole line without its `\n`. */
+  lines: { line: number; text: string }[];
+}
+
+/**
+ * One search, under one budget.
+ *
+ * A tree search covers every file under the scope's root: hidden files and
+ * folders, files a `.gitignore` or `.ignore` names, untracked files. It never
+ * enters a folder named `.git` (in any case), never follows a symbolic link,
+ * and reads no configuration of its own. Who may read what is not its
+ * business: the caller filters what it answers.
+ */
 export interface IPatternSearch {
   /**
-   * The indices of the lines the pattern matches, in order, at most `limit` of
-   * them (the first `limit`, exactly as a loop that stops at the cap finds).
-   * Throws {@link PatternTooExpensiveError} once the search's budget is spent.
+   * The lines the pattern matches in the `only` files under the scope, one
+   * file at a time in {@link compareSearchPaths} order, found by the same walk
+   * as {@link list}: a named path that has since become a link is not
+   * followed. A file whose content is binary contributes nothing, and neither
+   * does a line that is not UTF-8. Each file is answered as soon as ripgrep
+   * has finished it (whether a file is binary is known only then), so
+   * stopping the iteration stops the search at the file the caller had
+   * enough with.
+   */
+  files(scope: TreeScope, opts: TreeSearchOptions): AsyncIterable<FileMatches>;
+  /**
+   * Every file under the scope, relative to the root, in
+   * {@link compareSearchPaths} order: the walk {@link files} makes, without
+   * opening a file — what a caller judges before choosing what is searched.
+   */
+  list(scope: TreeScope): Promise<string[]>;
+  /**
+   * For each of `texts`, the 0-based indices of its lines (split on `\n`) that
+   * the pattern matches, in order.
+   */
+  matchingLines(texts: readonly string[]): Promise<number[][]>;
+  /**
+   * Run `work` the search waits on between its runs (judging which files the
+   * caller may read) under the search's one budget: the time it takes is
+   * spent from it, and work still running when the budget is gone is
+   * answered with {@link PatternTooExpensiveError} — or
+   * {@link SearchAbandonedError} when the caller has gone — at once, whatever
+   * `work` itself does after.
    */
-  matchingLines(lines: string[], limit: number): Promise<number[]>;
+  charged<T>(work: () => Promise<T>): Promise<T>;
 }
 
-/** A pattern stopped for taking too long — a 400, worded for the caller. */
+/**
+ * The order every search answers in: ripgrep's, path segment by segment, each
+ * compared by its UTF-8 bytes. Defined beside the platform's own walk order
+ * in `shared/path-order.ts`, which says how the two differ and why the search
+ * has one of its own; re-exported here because it is part of this contract.
+ */
+export { compareSearchPaths } from '../../shared/path-order.js';
+
+/** A search stopped for taking too long — a 400, worded for the caller. */
 export class PatternTooExpensiveError extends ToolError {
   constructor(pattern: string, budgetMs: number) {
     super(
-      `Pattern too expensive: /${pattern.length > 80 ? `${pattern.slice(0, 80)}…` : pattern}/ did not finish matching within this search's ${budgetMs / 1000} s budget, ` +
-        'so it was stopped and nothing was returned. Patterns with nested or overlapping repetition such as ' +
-        '`(a+)+` or `(a|aa)*` can take exponential time on some lines. Use a simpler pattern — a literal, ' +
-        'an anchor, a character class, a single repetition — or narrow `path`.',
+      `Pattern too expensive: /${shown(pattern)}/ did not finish within this search's ${budgetMs / 1000} s budget, ` +
+        'so it was stopped and nothing was returned. Narrow `path` to the folder you mean, or use a more ' +
+        'specific pattern.',
       400,
       { code: 'pattern_too_expensive' },
     );
@@ -51,7 +127,19 @@ export class PatternTooExpensiveError extends ToolError {
   }
 }
 
-/** Every worker busy and the queue for one full — a 503, so the caller retries rather than reads a partial answer. */
+/** What the engine accepts, as every refusal and the tool's description say it. */
+export const PATTERN_DIALECT =
+  "grep runs ripgrep's default regex engine, which takes RE2-style patterns: no look-around and no backreferences.";
+
+/** A pattern the engine refuses — a 400 quoting the engine's own reason. */
+export class PatternRefusedError extends ToolError {
+  constructor(reason: string) {
+    super(`Invalid regex: ${reason}\n${PATTERN_DIALECT}`, 400, { code: 'invalid_pattern' });
+    this.name = 'PatternRefusedError';
+  }
+}
+
+/** Every search slot busy and the queue for one full — a 503, so the caller retries rather than reads a partial answer. */
 export class SearchBusyError extends ToolError {
   constructor() {
     super('Too many searches are running at once; try again in a few seconds.', 503, { code: 'search_busy' });
@@ -59,11 +147,14 @@ export class SearchBusyError extends ToolError {
   }
 }
 
-/**
- * Whether `err` ends a whole search rather than one file's part of it: a
- * directory walk that skipped a file for either would answer an incomplete
- * result as if it were the whole one.
- */
-export function endsTheSearch(err: unknown): boolean {
-  return err instanceof PatternTooExpensiveError || err instanceof SearchBusyError;
+/** A search whose caller went away: its runs were killed, and nobody is waiting for the answer. */
+export class SearchAbandonedError extends ToolError {
+  constructor() {
+    super('The search was stopped because its caller went away.', 499, { code: 'search_abandoned' });
+    this.name = 'SearchAbandonedError';
+  }
+}
+
+function shown(pattern: string): string {
+  return pattern.length > 80 ? `${pattern.slice(0, 80)}…` : pattern;
 }