@bevel-software/platform-core-backend 0.33.0 → 0.34.0

package/dist/modules/workspace/workspace.tools.js

@@ -1,5 +1,5 @@
 import nodeFs from 'node:fs/promises';
-import { join } from 'node:path';
+import { isAbsolute, join, relative, sep } from 'node:path';
 import AdmZip from 'adm-zip';
 import { hasHttpStatus, ToolError, BRANCH_INPUT, toolDef, CAN_WRITE_CLAUSE, defaultBranchWriteVerdicts } from '../tool-helpers/index.js';
 import { notifyAgentRead, assertAgentWriteAllowed, SESSION_ID_INPUT, } from './agent-access.gate.js';
@@ -8,7 +8,7 @@ import { workspaceIdForBranch } from '../../shared/workspace-id.js';
 import { GitInternalsError, WorkflowValidationError } from '../../shared/domain-errors.js';
 // Leaf-level shared primitive (same exception `workspace.service.ts` already
 // relies on) — not a workflow service, so this stays inside the module boundary.
-import { assertInsideRepo, assertRepoRootNameFree, assertRepoRootNameFreeArgs, isInsideRepo, normalizePathArgs, normalizeWorkspacePath, } from '../kb-fs/index.js';
+import { assertInsideRepo, assertRepoRootNameFree, assertRepoRootNameFreeArgs, hasControlCharacter, isInsideRepo, isTmpRef, normalizePathArgs, normalizeWorkspacePath, } from '../kb-fs/index.js';
 import { GitGuardedFilesystem, MoveLockedError, MoveRacedError } from '../kb-fs/index.js';
 import { assertNoGitInternalsSegment, assertNotGitInternals, hasGitInternalsSegment } from '../../shared/git-internals.js';
 import { isRolesYamlPath, toKbRelative, resolveReadableMap, AccessDeniedError } from '../access-model/index.js';
@@ -18,11 +18,13 @@ import { displayPath } from './file-readers/file-reader.js';
 import { fileTypeOf, needsContent } from './file-readers/content-mode.js';
 import { createFileReaderRegistry } from './file-readers/file-reader.registry.js';
 import { DocumentReader } from './file-readers/document-reader.js';
+import { TextReader } from './file-readers/text-reader.js';
 import { mcpImageResult } from '@bevel-software/platform-mcp-core';
 import { folderPlaceholderPath, isFolderPlaceholder, isPlatformFile, isPlatformFolder, platformFileCreationRefusal, platformFileRefusal, platformFileUploadRefusal, platformFolderRefusal, isRepositoryOwnFile, repositoryOwnFileDeleteRefusal, entryExistsMessage, } from '@bevel-software/platform-shared';
 import { AGENT_GUIDE_FILE, isAgentGuidePath, isManagedGuide, withPlatformGuideAppended, } from '../agent-guide/index.js';
 import { removeEmptyDirs } from './empty-dirs.js';
-import { endsTheSearch } from './pattern-matcher.contract.js';
+import { compareSearchPaths, PATTERN_DIALECT, } from './pattern-matcher.contract.js';
+import { holdsNul } from './nul-scan.js';
 import { FIRST_RUN_SECTION_ID, firstRunNote, knowledgeFolderIsNew } from './first-run.js';
 import { planMoveLinks } from './move-links.js';
 import { rethrowAsWriteDenial } from './write-denial.js';
@@ -36,6 +38,12 @@ import { isSymlinkZipEntry, isZipNoiseEntry, readZipEntry, zipEntryName, zipEntr
 const log = logger('workspace-tools');
 /** Types that run scripts wherever they are opened, sent by `request_file_download` as plain bytes. */
 const ACTIVE_CONTENT_TYPES = new Set(['image/svg+xml', 'text/html', 'application/xhtml+xml']);
+/**
+ * The caller's verdicts on a file you staged in the workspace's tmp folder: it
+ * may be read (and copied out), nothing else — the folder is read-only
+ * through the ref, and `request_file_download` does not take one.
+ */
+const TMP_ACCESS = { read: true, write: false, download: false, owner: false };
 /** How many files `file_stat` counts under a folder before it stops and says so. */
 const DESCENDANTS_CAP = 10_000;
 /** How many of a folder's files a `delete_folder` answer names. */
@@ -67,13 +75,19 @@ async function filterReadableEntries(gate, dir, entries) {
     }
     if (fileWsPaths.length === 0)
         return entries;
-    const verdict = await resolveReadableMap((wid, email, rels) => gate.accessControl.canReadBatch(wid, email, rels), gate.workspaceId, gate.userEmail, gate.kbDirName, fileWsPaths);
-    return entries.filter((e) => {
-        if (e.type === 'directory')
-            return true;
-        const wsPath = dir ? `${dir}/${e.name}` : e.name;
-        return verdict.get(wsPath) === true;
-    });
+    const readable = await readableFiles(gate, fileWsPaths);
+    return entries.filter((e) => e.type === 'directory' || readable.has(dir ? `${dir}/${e.name}` : e.name));
+}
+/**
+ * The FILES among `wsPaths` the caller may read, judged in one batched ACL
+ * load by the FULL `canReadBatch` (per-node frontmatter honoured), via the
+ * shared `resolveReadableMap`. A path the verdict does not name is unreadable.
+ */
+async function readableFiles(gate, wsPaths) {
+    if (wsPaths.length === 0)
+        return new Set();
+    const verdict = await resolveReadableMap((wid, email, rels) => gate.accessControl.canReadBatch(wid, email, rels), gate.workspaceId, gate.userEmail, gate.kbDirName, wsPaths);
+    return new Set(wsPaths.filter((p) => verdict.get(p) === true));
 }
 /**
  * Drop the empty-folder placeholder from a directory listing: it keeps a
@@ -180,6 +194,30 @@ const RESERVED_ROOT_NAME_TARGETS = {
     // inside the handler — an archive chooses its entry names, not the caller.
     apply_file_upload: ['destination'],
 };
+/**
+ * The inputs that take a tmp ref (`__tmp__/<path>`, see `workspace-tmp.ts`)
+ * beside a repository path — the four reads of the one file it names. Every
+ * other tool, and every other input of these four (`copy_file`'s `dest`
+ * among them), refuses a ref before anything else runs — in
+ * `normalizePathArgs`, through the one admission gate (`kb-fs/tmp-ref.ts`):
+ * the folder is read-only through the ref, and `copy_file` to a repository
+ * path is how a file is kept.
+ */
+const TMP_REF_INPUTS = {
+    read_file: ['path'],
+    file_stat: ['path'],
+    grep: ['path'],
+    copy_file: ['src'],
+};
+/**
+ * A path input that also takes a tmp ref (see `TMP_REF_INPUTS`): said on the
+ * input, because the descriptions of the tools that take one have no room
+ * left, and what the ref is is in the shared rules.
+ */
+const orTmpRef = (schema) => ({
+    ...schema,
+    description: `${schema.description ?? ''} Or a \`__tmp__/…\` ref to one file you staged in the workspace's tmp folder.`,
+});
 function asText(content) {
     return typeof content === 'string' ? content : content.toString('utf8');
 }
@@ -467,35 +505,49 @@ function modeOf(a) {
         return a.mode;
     throw new ToolError(`"${String(a.mode)}" is not a write mode: use ${WRITE_MODES.map((m) => `\`${m}\``).join(', ')} (default \`create\`).`, 400, { code: 'bad_mode' });
 }
+/** The longest stretch of a matching line a grep answer carries. */
+const GREP_LINE_CHARS = 300;
+/** {@link searchableText}'s `bytesOf` for a repository path: the workspace filesystem's own read. */
+function fileBytes(fs) {
+    return async (path) => asBytes(await fs.readFile(path));
+}
 /**
- * Search ONE file and append its matches to `out`. Shared by the directory
- * walk and by a grep whose `path` names a file, so both produce the same match
- * shape (that path, 1-based line numbers, 300-char text) under the same cap.
- *
- * What is searched is the file's reader's business: text content for the text
- * reader (null on NUL bytes / invalid UTF-8 — binary is not searchable), the
- * EXTRACTION for a document reader (marker line included, so the
- * `[slide N]`/`[sheet: …]`/`[page N]` lines are themselves searchable and line
- * numbers match what read_file returns), nothing for images.
+ * Files the text pass judges, then searches, at a time: first this many, then
+ * twice as many each time, up to {@link GREP_MAX_BATCH}. A verdict reads the
+ * file's own access rules, so a batch costs one read of each of its files.
+ * Its names also go to ripgrep on the command line, so a batch also ends
+ * once they reach {@link GREP_BATCH_CHARS}, well inside the system's limit.
+ */
+const GREP_FIRST_BATCH = 256;
+const GREP_MAX_BATCH = 2_048;
+const GREP_BATCH_CHARS = 256 * 1024;
+/**
+ * The text ONE file holds for `grep`, read through its reader: the text
+ * content for a text reader (none on NUL bytes / invalid UTF-8 — binary is not
+ * searchable), the EXTRACTION for a document reader (marker line included, so
+ * the `[slide N]`/`[sheet: …]`/`[page N]` lines are themselves searchable and
+ * line numbers match what read_file returns), nothing for images.
  *
  * Throws whatever reading the file throws — the caller decides whether that is
- * a file to skip (the walk) or an error to surface (a single-file grep).
+ * a file to skip (a tree search) or an error to surface (a single-file grep).
+ * `bytesOf` is where the file's bytes come from: the workspace filesystem for
+ * a repository path, the tmp resolver for a tmp ref.
  */
-async function grepOneFile(fs, path, matcher, out, max, docs) {
+async function searchableText(bytesOf, path, docs) {
     const reader = docs.readers.readerFor(path);
-    const bytes = asBytes(await fs.readFile(path));
+    const bytes = await bytesOf(path);
     let content;
     if (reader instanceof DocumentReader) {
         // What the cache remembers is free — an extraction is searched, a failure
         // (a damaged file, or one that overran its budget within the hour) is
         // skipped without extracting it again. A cold document draws on the
-        // per-walk budget (see UNCACHED_DOCS_PER_GREP) — beyond it the search
+        // per-call budget (see UNCACHED_DOCS_PER_GREP) — beyond it the search
         // skips the document and counts it.
         let res = await reader.cachedRead(bytes, path);
         if (res === undefined) {
             if (docs.uncachedBudget <= 0) {
                 docs.skippedUncached++;
-                return 'budget-skipped';
+                return { kind: 'budget-skipped' };
             }
             docs.uncachedBudget--;
             res = await reader.read(bytes, path, docs.origin);
@@ -506,15 +558,41 @@ async function grepOneFile(fs, path, matcher, out, max, docs) {
     else {
         content = reader.greppableText ? await reader.greppableText(bytes, path) : null;
     }
-    if (content === null)
-        return 'no-text';
-    const lines = content.split('\n');
-    // matchingLines posts these to the worker in bounded batches, so no one
-    // copy of a large file runs on the request thread.
-    for (const i of await matcher.matchingLines(lines, max - out.length)) {
-        out.push({ path, line: i + 1, text: lines[i].slice(0, 300) });
+    return content === null ? { kind: 'no-text' } : { kind: 'text', text: content };
+}
+/**
+ * grep over the one text file `path`, by ripgrep from the folder holding it,
+ * its lines pushed onto `out`, when it is a regular file; `null`, with
+ * nothing searched, for anything else (a link, one that loops, a file gone
+ * since), which is read as read_file reads it instead. A file that answers no
+ * line has no text when ripgrep finds it binary: it holds a NUL, which
+ * {@link holdsNul} looks for in the very file that was judged.
+ */
+async function grepTextFile(fs, path, search, max, out) {
+    const slash = path.lastIndexOf('/');
+    const scope = await treeScope(fs, slash < 0 ? '' : path.slice(0, slash));
+    const name = path.slice(slash + 1);
+    const absolute = join(scope.cwd, scope.root, name);
+    const judged = await nodeFs.lstat(absolute).catch(() => null);
+    if (judged === null || !judged.isFile())
+        return null;
+    const from = out.length;
+    for await (const file of search.files(scope, { only: [name], maxLinesPerFile: max })) {
+        for (const l of file.lines)
+            out.push({ path, line: l.line, text: l.text.slice(0, GREP_LINE_CHARS) });
     }
-    return 'searched';
+    if (out.length > from)
+        return { kind: 'searched' };
+    const binary = await orNotFound(path, () => holdsNul(absolute, judged, search), 'Nothing to search');
+    return binary ? { kind: 'no-text' } : { kind: 'searched' };
+}
+/** Append the lines `hits` names (0-based indices into `text`'s lines) to `out`, as matches in `path`. */
+function pushMatches(out, path, text, hits) {
+    if (hits.length === 0)
+        return;
+    const lines = text.split('\n');
+    for (const i of hits)
+        out.push({ path, line: i + 1, text: lines[i].slice(0, GREP_LINE_CHARS) });
 }
 /**
  * What a grep's `path` actually names. The walk starts with `readdir`, which
@@ -548,55 +626,145 @@ async function searchRootKind(fs, path) {
         return 'file';
     }
 }
-/** JS grep over the workspace tree (read methods only) — bounded by match + depth caps. */
-async function grepWalk(fs, dir, matcher, out, max, depth, gate, notifyRead, docs, 
-/** A file the walk leaves to its caller: never opened, never counted against `max`. */
-skip = () => false) {
-    if (out.length >= max || depth > 12)
-        return;
-    let entries;
-    try {
-        entries = (await fs.readdir(dir || '.'));
-    }
-    catch {
-        return;
+/**
+ * Where ripgrep searches the directory `root`: from the workspace's canonical
+ * directory, with the root's canonical path relative to it. The stat that
+ * called `root` a directory already refused one outside the workspace; this
+ * refuses it again where the search is started, so ripgrep, which follows the
+ * root it is given, is never started on anything outside.
+ */
+async function treeScope(fs, root) {
+    const cwd = await nodeFs.realpath(fs.basePath);
+    const absolute = fs.resolveAbsolutePath(root);
+    const rel = absolute === undefined ? '..' : relative(cwd, await nodeFs.realpath(absolute));
+    if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) {
+        throw new ToolError(`"${displayPath(root)}" is outside the workspace.`, 403);
     }
-    // Filter unreadable KB nodes out of the walk so grep never opens — or leaks
-    // a line from — a file the caller may not read.
-    entries = await filterReadableEntries(gate, dir, entries);
-    for (const e of entries) {
-        if (out.length >= max)
-            return;
-        if (hasGitInternalsSegment(e.name) || e.name === 'node_modules')
-            continue;
-        if (e.type !== 'directory' && isFolderPlaceholder(e.name))
-            continue;
-        const p = dir ? `${dir}/${e.name}` : e.name;
-        if (e.type === 'directory') {
-            await grepWalk(fs, p, matcher, out, max, depth + 1, gate, notifyRead, docs, skip);
+    return { cwd, root: rel === '' ? '.' : rel.split(sep).join('/') };
+}
+/**
+ * grep over the directory `root`: the tree is listed, then searched in two
+ * passes, and no file is opened before it is admitted.
+ *
+ * The listing opens no file. Admission is the ONE step a file passes before
+ * either pass opens it: it is judged readable by the gate, and then — being a
+ * file the search will read, as `read_file` reads one — it is told to the
+ * read hook, which may refuse it, and with it the search. A file the caller
+ * may not read is never opened and never told of (there is no read to tell):
+ * what it holds can change neither the answer nor how long the search takes,
+ * and so not whether it ends within its budget. Judging and telling are both
+ * spent from that budget. A file that was not in the listing (it appeared
+ * since) is never admitted, so never searched.
+ *
+ * The text pass is ripgrep's own: every file whose reader greps its bytes as
+ * text (a TextReader) is searched by it, in path order, and read nowhere else.
+ * It admits a batch at a time, before ripgrep opens any of it, so the hook
+ * hears of each file of a batch whether or not a line of it is answered —
+ * and, past the line that fills the cap, of the rest of that batch, which the
+ * pass then never opens. Each file is answered as ripgrep finishes it, and
+ * the pass stops at the `max`-th line.
+ *
+ * The document pass reads the files every other reader owns through that
+ * reader, as `read_file` does — extractions under the per-call budget — and
+ * has their text matched by the same engine. A document that sorts after the
+ * text pass's last line cannot be among the first `max`, so it is not read.
+ *
+ * Answers what both found, unsorted: the caller merges, sorts and caps.
+ */
+async function grepTree(fs, root, search, max, gate, notifyRead, docs, 
+/** A file the search leaves to its caller: never answered, never counted against `max`. */
+skip = () => false) {
+    const scope = await treeScope(fs, root);
+    // A folder named with a trailing slash answers the paths it is spelled
+    // with everywhere else (`a/b.md`, never `a//b.md`), so the guide's own
+    // path and every verdict meet the one spelling.
+    const base = root.replace(/\/+$/, '');
+    const at = (p) => (base === '' ? p : `${base}/${p}`);
+    // ripgrep already stays out of `.git`; the folder's every other spelling,
+    // and the folder placeholder, are left out as everywhere else.
+    const shown = (p) => !hasGitInternalsSegment(p) && !isFolderPlaceholder(p) && !skip(p);
+    // Which reader owns an extension is the registry's answer, asked of a name
+    // carrying it; a TextReader greps the bytes themselves, which is ripgrep's
+    // pass. Extensions any other reader owns are left out of it.
+    const readerOf = (ext) => docs.readers.readerFor(`file${ext}`);
+    const notRaw = docs.readers.ownedExtensions().filter((ext) => !(readerOf(ext) instanceof TextReader));
+    const ownedBy = (p, exts) => {
+        const lower = p.toLowerCase();
+        return exts.some((ext) => lower.endsWith(ext));
+    };
+    const readerOwned = notRaw.filter((ext) => {
+        const reader = readerOf(ext);
+        return reader instanceof DocumentReader || reader.greppableText !== undefined;
+    });
+    const out = [];
+    // Admission (see above): the gate's verdict on `paths`, then the hook told
+    // of each readable one in path order — a refusal ends the search here, as
+    // it ends a `read_file`, before anything is opened. Spent from the budget.
+    const admit = (paths) => search.charged(async () => {
+        const readable = await readableFiles(gate, paths);
+        for (const p of paths) {
+            if (readable.has(p))
+                await notifyRead(p);
         }
-        else if (skip(p)) {
-            continue;
+        return readable;
+    });
+    // Every file the walk covers, in path order. Only what is shown is ever
+    // admitted, so what is admitted is what may be answered. A name holding a
+    // control character is no file name to the platform — no tool reads or
+    // writes one — so it is not searched either.
+    const all = (await search.list(scope)).filter((p) => shown(at(p)) && !hasControlCharacter(p));
+    const raw = all.filter((p) => !ownedBy(p, notRaw));
+    // The text pass: a batch of files is admitted, then its admitted files, and
+    // only they, are searched. A batch is admitted before ripgrep opens any of
+    // it, and the pass stops at the `max`-th line, so the verdicts asked for are
+    // those of the files the answer reached — few for a common word — and the
+    // batches double so a rare one still walks the tree in a handful of runs.
+    let size = GREP_FIRST_BATCH;
+    for (let from = 0; from < raw.length && out.length < max; size = Math.min(size * 2, GREP_MAX_BATCH)) {
+        const batch = [];
+        for (let chars = 0; from < raw.length && batch.length < size && (batch.length === 0 || chars < GREP_BATCH_CHARS); from++) {
+            batch.push(raw[from]);
+            chars += raw[from].length;
         }
-        else {
-            // Opening a file is a read of it, even when the walk started at a root
-            // the read hook was already told about — so every file the walk opens
-            // reaches the hook by name (closes the read-leak).
-            await notifyRead(p);
-            // A file the walk cannot read is silently skipped: one unreadable entry
-            // must not fail a search over the whole tree.
-            // A pattern too expensive to finish, or no worker free to run it, is not
-            // an unreadable file: either ends the whole search.
-            try {
-                await grepOneFile(fs, p, matcher, out, max, docs);
-            }
-            catch (err) {
-                if (endsTheSearch(err))
-                    throw err;
+        const readable = await admit(batch.map(at));
+        const only = batch.filter((p) => readable.has(at(p)));
+        for await (const file of search.files(scope, { only, maxLinesPerFile: max - out.length })) {
+            const p = at(file.path);
+            // Only what was admitted answers, whatever ripgrep printed.
+            if (!readable.has(p))
                 continue;
+            for (const l of file.lines.slice(0, max - out.length)) {
+                out.push({ path: p, line: l.line, text: l.text.slice(0, GREP_LINE_CHARS) });
             }
+            // Enough: leaving the loop kills ripgrep at the file that filled the cap.
+            if (out.length >= max)
+                break;
+        }
+    }
+    const last = out.length >= max ? out[out.length - 1].path : undefined;
+    const candidates = all
+        .filter((p) => ownedBy(p, readerOwned))
+        .map(at)
+        .filter((p) => last === undefined || compareSearchPaths(p, last) <= 0);
+    const readable = await admit(candidates);
+    const texts = [];
+    for (const p of candidates) {
+        if (!readable.has(p))
+            continue;
+        let found;
+        try {
+            found = await searchableText(fileBytes(fs), p, docs);
+        }
+        catch {
+            // One document that cannot be read must not fail a search over the tree.
+            continue;
         }
+        if (found.kind === 'text')
+            texts.push({ path: p, text: found.text });
     }
+    const hits = await search.matchingLines(texts.map((t) => t.text));
+    texts.forEach((t, i) => pushMatches(out, t.path, t.text, hits[i]));
+    return out;
 }
 const SAVE_WARNINGS_OUTPUT = {
     type: 'array',
@@ -842,7 +1010,9 @@ patternMatcher) {
         const paths = toolPathArgs(args);
         for (const p of paths)
             assertNoGitInternalsSegment(p);
-        const onDisk = paths.filter((p) => !spillStore.isSpillRef(p));
+        // A spill ref belongs to no workspace, and a tmp ref to the folder beside
+        // the checkout: neither is judged against the checkout's git folder.
+        const onDisk = paths.filter((p) => !spillStore.isSpillRef(p) && !isTmpRef(p));
         if (onDisk.length === 0)
             return;
         const root = await gitCheckRootFor(args, ctx);
@@ -896,9 +1066,14 @@ patternMatcher) {
      * caller WILL have, never whether they may do it. The write verdicts that
      * gate the move are `writeBlocked` and the lock gate, both of which read
      * the tree as it is.
+     *
+     * `sourceContent` is the text of a source that is no repository path (a
+     * tmp ref): it carries rules in exactly as a repository file would, so it
+     * is previewed through the same relocation reading, never as "nothing
+     * travels".
      */
     const accessAfter = async (branch, ctx, src, dest, opts) => {
-        const from = toKbRelative(src, kbDirName);
+        const from = opts?.sourceContent !== undefined ? src : toKbRelative(src, kbDirName);
         const to = toKbRelative(dest, kbDirName);
         // Outside the repository there are no rules to carry, and none to land
         // among — the same answer `accessAt` gives for such a path.
@@ -906,6 +1081,16 @@ patternMatcher) {
             return accessAt(branch, ctx, dest);
         return accessControl.previewAccessAfterRelocation(workspaceIdForBranch(branch), ctx.user.email, from, to, opts);
     };
+    /**
+     * The text a tmp source carries into the repository, for the preview only:
+     * read as a repository source is read there — a file that cannot be read
+     * carries no rules — so the preview never answers before the write
+     * verdict. Whether the file is there at all is `statTmpFile`'s to say.
+     */
+    const tmpSourceText = (branch, ctx, ref) => ctx.workspaceService
+        .readTmpFile(workspaceIdForBranch(branch), ctx.user.id, ref)
+        .then((bytes) => bytes.toString('utf-8'))
+        .catch(() => '');
     /**
      * What `copy_file`'s dry run answers: the same impact shape `move_file`
      * previews, over a copy's own rules.
@@ -933,10 +1118,16 @@ patternMatcher) {
      * them nothing they could not ask `file_stat` for.
      */
     const copyImpact = async (branch, ctx, src, dest) => {
-        const [before, after, blocked] = await Promise.all([
-            accessAt(branch, ctx, src),
-            accessAfter(branch, ctx, src, dest, { sourceRemains: true }),
-            writeBlocked(branch, ctx, [dest]),
+        const fromTmp = isTmpRef(src);
+        // The write verdict FIRST, and alone: a staged file's bytes are read only
+        // for a caller who may write the destination. A denied preview carries
+        // no rules from the source ('' — what an unreadable source carries), so
+        // its `access.after` is the destination's own rules for the caller.
+        const blocked = await writeBlocked(branch, ctx, [dest]);
+        const sourceContent = fromTmp ? (blocked.length > 0 ? '' : await tmpSourceText(branch, ctx, src)) : undefined;
+        const [before, after] = await Promise.all([
+            fromTmp ? TMP_ACCESS : accessAt(branch, ctx, src),
+            accessAfter(branch, ctx, src, dest, { sourceRemains: true, sourceContent }),
         ]);
         const access = { before, after };
         const accessChanges = verbsDiffer(before, after);
@@ -953,7 +1144,11 @@ patternMatcher) {
             };
         }
         const fs = await ctx.getFilesystem(branch);
-        const kind = await kindOf(fs, src);
+        // A tmp ref names one file by rule: resolving it is the whole look, and
+        // a folder or nothing there is refused by the resolver itself.
+        const kind = fromTmp
+            ? (await orNotFound(src, () => ctx.workspaceService.statTmpFile(workspaceIdForBranch(branch), ctx.user.id, src), 'Nothing to copy'), 'file')
+            : await kindOf(fs, src);
         if (kind === null)
             throw notFound(src, 'Nothing to copy');
         const srcFiles = kind === 'folder' ? (await filesUnder(fs, src)).files : [src];
@@ -978,6 +1173,39 @@ patternMatcher) {
             copied: false,
         };
     };
+    /**
+     * `copy_file` out of the workspace's tmp folder: the bytes the ref names
+     * (through the one tmp resolver), landed at `dest` by the ordinary write —
+     * the lock, the pre-disk validation, the creator's read grant for a new
+     * folder and the commit every write to the repository takes — and
+     * exclusively, so a name taken since the handler's look is refused with
+     * the same sentence rather than replaced. From then on the copy is a
+     * repository file like any other; the staged file is left where it was.
+     *
+     * A missing source is the ordinary 404 naming the ref, asked only after
+     * the write verdict (as for any copy); an absence met while landing is the
+     * destination's.
+     */
+    const copyFromTmp = async (fs, branch, ctx, src, dest) => {
+        const bytes = await orNotFound(src, () => ctx.workspaceService.readTmpFile(workspaceIdForBranch(branch), ctx.user.id, src), 'Nothing to copy');
+        try {
+            // Exclusive, under the destination's lock: a name taken since the look
+            // is refused there as `DestinationTakenError` — releasing the lock
+            // untouched, so the winner's queued file stays — and answered as a
+            // repository copy's lost race is (`asEntryExists`).
+            await asEntryExists(() => fs.writeFile(dest, bytes, { overwrite: false }));
+        }
+        catch (err) {
+            if (err instanceof ToolError)
+                throw err;
+            const occupiedBy = await existingAt(await workspaceRoot(branch, ctx), dest);
+            if (occupiedBy !== null)
+                throw new ToolError(entryExistsMessage(occupiedBy, dest), 409);
+            if (isAbsence(err))
+                throw notFound(dest, 'Nowhere to copy to');
+            throw err;
+        }
+    };
     /**
      * The paths among `paths` the caller may NOT write, judged exactly as the
      * lock gate judges them (`WorkflowService.acquireLock`): on a protected
@@ -1342,11 +1570,21 @@ patternMatcher) {
         // spill ref; for any other, `__tool_chain_spill__/x` is an ordinary
         // path, and exempting it there would be a workspace-relative path that
         // never reached the repository — the whole bug, spelled with a prefix.
+        //
+        // The other is a tmp ref (`__tmp__/…`), left as written in exactly the
+        // inputs `TMP_REF_INPUTS` names and refused in every other one first,
+        // so no tool sees one it does not resolve itself.
         toolHandler(async (args, ctx) => {
             // BEFORE the normaliser: see `assertToolPathsNotGitInternals`.
             if (spec.fileTool !== false)
                 await assertToolPathsNotGitInternals(args, ctx);
-            const normalized = normalizePathArgs(args, kbDirName, spec.name === 'read_file' ? (v) => spillStore.isSpillRef(v) : undefined);
+            // The normaliser admits a tmp ref in exactly the inputs
+            // `TMP_REF_INPUTS` names and refuses it in every other one first.
+            const normalized = normalizePathArgs(args, kbDirName, {
+                tool: spec.name,
+                tmpRefInputs: TMP_REF_INPUTS[spec.name],
+                skip: spec.name === 'read_file' ? (v) => spillStore.isSpillRef(v) : undefined,
+            });
             // The checkout's own name is reserved at the repository root on this
             // surface too. The routes meet that rule inside `WorkspaceService`;
             // these tools write through the locking filesystem, which never
@@ -1497,6 +1735,61 @@ patternMatcher) {
      * copy drifts on the first change to any of them.
      */
     const readForTool = async (branch, p, ctx) => (await readServed(branch, p, ctx)).result;
+    /**
+     * What a tmp ref (see `workspace-tmp.ts`) reads as: the bytes of the one
+     * file it names in the CALLER's own folder of the branch workspace's `tmp/`
+     * — through `readTmpFile`, the resolver a distribution's tool reads it with
+     * — put through the SAME reader a repository file of that name gets. No
+     * access gate and no read hook: the folder is outside the repository, so
+     * no rule governs it and no knowledge-base path is read; what keeps one
+     * person's staged file from another is that the ref resolves inside the
+     * caller's folder alone. A missing file — somebody else's included — is the
+     * ordinary 404, naming the ref.
+     */
+    const readTmp = async (branch, ref, ctx) => {
+        const workspaceId = workspaceIdForBranch(branch);
+        const bytes = await orNotFound(ref, () => ctx.workspaceService.readTmpFile(workspaceId, ctx.user.id, ref));
+        return readers.readerFor(ref).read(bytes, ref, { workspaceId });
+    };
+    /**
+     * `file_stat` on a tmp ref: the shape a repository file's stat has, its
+     * content fields decided by the same reader as {@link readTmp} reads with —
+     * and nothing to DO with it but read and copy out. Not written, moved,
+     * deleted or downloaded through these tools (`managed` is false: the
+     * platform's own files are a repository notion), and so not text-editable
+     * either, whatever its content.
+     */
+    const statTmp = async (branch, ref, ctx) => {
+        const workspaceId = workspaceIdForBranch(branch);
+        const stat = await orNotFound(ref, () => ctx.workspaceService.statTmpFile(workspaceId, ctx.user.id, ref));
+        const reader = readers.readerFor(ref);
+        // One read of the file at most: the type decision and the remembered
+        // extraction below both want the bytes, and a staged document can be large.
+        let bytes;
+        const bytesOf = async () => (bytes ??= await orNotFound(ref, () => ctx.workspaceService.readTmpFile(workspaceId, ctx.user.id, ref)));
+        const type = fileTypeOf(reader, ref, needsContent(reader) ? await bytesOf() : undefined);
+        const out = {
+            ...stat,
+            path: ref,
+            type: 'file',
+            managed: false,
+            movable: false,
+            deletable: false,
+            access: TMP_ACCESS,
+            canWrite: false,
+            ...type,
+            textEditable: false,
+        };
+        // A document whose extraction failed answers that failure's notice, as a
+        // repository document does: from what the cache remembers, never by
+        // extracting.
+        if (reader instanceof DocumentReader) {
+            const remembered = await reader.cachedRead(await bytesOf(), ref);
+            if (remembered?.kind === 'refusal')
+                return { ...out, unreadable: remembered.message };
+        }
+        return out;
+    };
     /**
      * {@link readForTool}, also saying, at the guide's name, whether a file of
      * the knowledge base's own answered first on THIS branch (`guide`: `own`)
@@ -1558,7 +1851,7 @@ patternMatcher) {
             type: 'object',
             properties: {
                 branch: BRANCH_INPUT,
-                path: str(`Path to read, under \`${kbDirName}/\` (e.g. \`${kbDirName}/KnowledgeBase/Foo.md\`), with or without a leading slash — a path without that prefix is placed under \`${kbDirName}/\` — or a \`__tool_chain_spill__/…\` ref from a truncated \`call_tool_chain\`.`),
+                path: str(`Path to read, under \`${kbDirName}/\` (e.g. \`${kbDirName}/KnowledgeBase/Foo.md\`), with or without a leading slash — a path without that prefix is placed under \`${kbDirName}/\` — or a \`__tool_chain_spill__/…\` ref from a truncated \`call_tool_chain\`, or a \`__tmp__/…\` ref to a file you staged in the workspace's tmp folder.`),
                 offset: int('Start character index (default 0).'),
                 limit: int('Max characters to return from `offset`.'),
                 sessionId: SESSION_ID_INPUT,
@@ -1594,7 +1887,11 @@ patternMatcher) {
                     : content;
             };
             const branch = a.branch;
-            const { result, guide } = await readServed(branch, p, ctx);
+            // A tmp ref is no repository path, so the `canWrite` below is false for
+            // it by the same rule as for any path outside the repository.
+            const { result, guide } = isTmpRef(p)
+                ? { result: await readTmp(branch, p, ctx), guide: null }
+                : await readServed(branch, p, ctx);
             // Judged after the read, so a path the caller may not read is refused
             // before anything is said about writing it. At the guide's name this
             // answers for the knowledge base's own file there, as `file_stat` does:
@@ -1798,7 +2095,7 @@ patternMatcher) {
             type: 'object',
             properties: {
                 branch: BRANCH_INPUT,
-                path: wsPath(kbDirName, 'Path'),
+                path: orTmpRef(wsPath(kbDirName, 'Path')),
                 explainAccess: {
                     type: 'boolean',
                     description: 'Also explain `access` (default false): `access.why` says what decided each of your verdicts — the folder rules or file frontmatter and whether that is inherited; `access.roster` lists who holds each verb and where each grant is written, given only when you can manage this path\'s access (otherwise null, with `access.rosterReason`).',
@@ -1871,6 +2168,8 @@ patternMatcher) {
         handler: async (a, ctx) => {
             const p = a.path;
             const branch = a.branch;
+            if (isTmpRef(p))
+                return statTmp(branch, p, ctx);
             // The guide's name with no file of the knowledge base's own under it —
             // or one the caller may not read, or a copy of the guide an earlier
             // release wrote, none of which `read_file` serves: what a read answers
@@ -2034,13 +2333,14 @@ patternMatcher) {
         mount({
             name: 'grep',
             gated: true,
-            description: 'Regex search across the workspace. Returns `{ matches: [{ path, line, text }] }` (capped). Use to find where something is defined/referenced. `path` may name a DIRECTORY (searches the subtree) or a single FILE (searches just that file); a path with nothing at it is an error, never an empty result. Searches INSIDE Office and OpenDocument files (.docx/.pptx/.xlsx, .odt/.odp/.ods), PDFs and email files (.eml/.msg) via their extracted text — matches there carry the extraction\'s line numbers, and the `[slide N]`/`[sheet: Name]`/`[page N]`/`[from]`/`[subject]` marker lines locate them; a bounded number of not-yet-extracted documents is extracted per call, and the result notes how many were skipped (re-run to cover them).',
+            description: `Regex search across the workspace. ${PATTERN_DIALECT} Covers hidden and git-ignored files. ` +
+                'Returns `{ matches: [{ path, line, text }] }`, capped, in path then line order. `path` may name a DIRECTORY (its subtree) or a single FILE; a path with nothing at it is an error. Searches INSIDE Office/OpenDocument, PDF and email files (.eml/.msg) via their extracted text — matches carry the extraction\'s line numbers, and the `[slide N]`/`[sheet: Name]`/`[page N]`/`[from]`/`[subject]` marker lines locate them; a bounded number of not-yet-extracted documents is extracted per call; the result notes the rest (re-run to cover them).',
             inputs: {
                 type: 'object',
                 properties: {
                     branch: BRANCH_INPUT,
-                    pattern: str('JavaScript regular expression.'),
-                    path: str(`Subtree to search, or a single file to search on its own, with or without a leading slash — a path without the \`${kbDirName}/\` prefix is placed under \`${kbDirName}/\` (default: the whole repository, \`${kbDirName}/\`).`),
+                    pattern: str('Regular expression, RE2-style (ripgrep): no look-around, no backreferences.'),
+                    path: str(`Subtree to search, or a single file to search on its own, with or without a leading slash — a path without the \`${kbDirName}/\` prefix is placed under \`${kbDirName}/\` (default: the whole repository, \`${kbDirName}/\`) — or a \`__tmp__/…\` ref to one staged file.`),
                     ignore_case: { type: 'boolean', description: 'Case-insensitive match.' },
                     max_results: { type: 'integer', minimum: 1, maximum: 1000, description: 'Cap on matches (default 200).' },
                     sessionId: SESSION_ID_INPUT,
@@ -2067,9 +2367,14 @@ patternMatcher) {
             },
             write: false,
             handler: async (a, ctx) => {
-                // The caller's pattern never runs on the request thread: a backtracking
-                // one would hold the event loop for everyone (see IPatternMatcher).
-                const matcher = patterns.search(a.pattern, a.ignore_case ? 'i' : '');
+                // The caller's pattern runs in ripgrep, never on the request thread
+                // (see IPatternMatcher); one the engine refuses is answered here,
+                // before anything is searched. A caller that goes away takes the
+                // search's runs with it.
+                const search = await patterns.search(a.pattern, {
+                    ignoreCase: Boolean(a.ignore_case),
+                    signal: ctx.abortSignal,
+                });
                 // No path means the whole REPOSITORY, not the workspace directory above
                 // it. The workspace root also holds whatever an older build left beside
                 // the checkout, and the read gate has no rules for a path outside the
@@ -2095,11 +2400,17 @@ patternMatcher) {
                 // worth a stat.
                 // A placeholder named on its own is searched as what it is to every
                 // other tool: nothing.
-                const kind = searchRoot === ''
-                    ? 'directory'
-                    : isFolderPlaceholder(searchRoot)
-                        ? 'missing'
-                        : await searchRootKind(fs, searchRoot);
+                // A tmp ref names one file by rule (see `workspace-tmp.ts`), so it is
+                // searched as a single file — never walked, and outside the repository
+                // there is no read gate or read hook for it, as for `read_file`.
+                const tmp = isTmpRef(searchRoot);
+                const kind = tmp
+                    ? 'file'
+                    : searchRoot === ''
+                        ? 'directory'
+                        : isFolderPlaceholder(searchRoot)
+                            ? 'missing'
+                            : await searchRootKind(fs, searchRoot);
                 /** Why a single-file search found nothing, when "no matches" would be a lie. */
                 let fileNote;
                 // The guide is searched where it is read: a search of the repository
@@ -2120,22 +2431,26 @@ patternMatcher) {
                 // The guide's own path is not told here, because `guideAt` tells the
                 // hook of the organisation's file there exactly as `read_file` does
                 // (after the gate), and the platform's guide alone is nobody's file to
-                // note. Each file the walk opens goes to the hook per file below, so a
-                // hook sees every path a grep reached rather than only its root.
-                if (!(coversGuide && kind !== 'directory')) {
+                // note. Each file the search opens goes to the hook per file below,
+                // before it is opened and whether or not a line of it is answered
+                // (`grepTree`'s admission), so a hook sees every file a grep reads
+                // rather than only its root, and may refuse one as it refuses a
+                // read_file. A tmp ref is no repository content, so there is nothing
+                // to tell.
+                if (!tmp && !(coversGuide && kind !== 'directory')) {
                     await notifyAgentRead(agentAccessGate, ctx, a.branch, searchRoot);
                 }
+                const notifyRead = (p) => notifyAgentRead(agentAccessGate, ctx, a.branch, p);
                 if (coversGuide) {
                     if (kind === 'directory') {
-                        await grepWalk(fs, searchRoot, matcher, out, max, 0, gate, (p) => notifyAgentRead(agentAccessGate, ctx, a.branch, p), docs, (p) => p === guidePath);
-                    }
-                    const lines = (await guideAt(a.branch, ctx, guidePath)).split('\n');
-                    for (const i of await matcher.matchingLines(lines, max - out.length)) {
-                        out.push({ path: guidePath, line: i + 1, text: lines[i].slice(0, 300) });
+                        out.push(...(await grepTree(fs, searchRoot, search, max, gate, notifyRead, docs, (p) => p === guidePath)));
                     }
+                    const guide = await guideAt(a.branch, ctx, guidePath);
+                    const [hits] = await search.matchingLines([guide]);
+                    pushMatches(out, guidePath, guide, hits);
                 }
                 else if (kind === 'directory') {
-                    await grepWalk(fs, searchRoot, matcher, out, max, 0, gate, (p) => notifyAgentRead(agentAccessGate, ctx, a.branch, p), docs);
+                    out.push(...(await grepTree(fs, searchRoot, search, max, gate, notifyRead, docs)));
                 }
                 else {
                     // Not a directory: the permission verdict comes BEFORE every other
@@ -2147,11 +2462,28 @@ patternMatcher) {
                     await assertCanRead(gate, searchRoot);
                     if (kind === 'missing')
                         throw notFound(searchRoot, 'Nothing to search');
-                    // A named FILE is searched directly: routing it through the walk would
-                    // fail its readdir and answer an empty match list, which the caller
-                    // cannot tell from "the pattern is not in this file".
-                    const outcome = await orNotFound(searchRoot, () => grepOneFile(fs, searchRoot, matcher, out, max, docs), 'Nothing to search');
-                    if (outcome === 'no-text') {
+                    // A named text FILE is searched as a tree search searches it: by
+                    // ripgrep, from its folder, so it answers the same lines either way
+                    // (the valid UTF-8 lines of a file holding some that are not, and
+                    // nothing from one ripgrep finds binary). Any other file, and any
+                    // path that is not a regular file, is read through its reader,
+                    // exactly as read_file reads it, its text searched by the same
+                    // engine: a tree search would find nothing to walk and answer an
+                    // empty match list, which the caller cannot tell from "the pattern
+                    // is not in this file". A tmp ref is not in the checkout ripgrep
+                    // searches, so it is always read through its reader, its bytes from
+                    // the tmp resolver.
+                    const bytesOf = tmp
+                        ? (ref) => ctx.workspaceService.readTmpFile(docs.origin.workspaceId, ctx.user.id, ref)
+                        : fileBytes(fs);
+                    const found = (!tmp && readers.readerFor(searchRoot) instanceof TextReader
+                        ? await grepTextFile(fs, searchRoot, search, max, out)
+                        : null) ?? (await orNotFound(searchRoot, () => searchableText(bytesOf, searchRoot, docs), 'Nothing to search'));
+                    if (found.kind === 'text') {
+                        const [hits] = await search.matchingLines([found.text]);
+                        pushMatches(out, searchRoot, found.text, hits);
+                    }
+                    else if (found.kind === 'no-text') {
                         fileNote =
                             `"${displayPath(searchRoot)}" has no searchable text — it is an image, binary content, or a document ` +
                                 'whose text could not be extracted. There were no matches because there was nothing to search, not ' +
@@ -2164,8 +2496,11 @@ patternMatcher) {
                             `extracted and this call's extraction budget (${UNCACHED_DOCS_PER_GREP}) ran out. Re-run the ` +
                             'same grep to extract and search the next batch.'
                         : undefined);
+                // One order whatever found the lines, then the cap: the first
+                // `max_results` by path, then line.
+                out.sort((x, y) => compareSearchPaths(x.path, y.path) || x.line - y.line);
                 return {
-                    matches: out,
+                    matches: out.slice(0, max),
                     truncated: out.length >= max,
                     ...(note !== undefined ? { note } : {}),
                 };
@@ -2984,14 +3319,14 @@ patternMatcher) {
     mount({
         name: 'copy_file',
         gated: true,
-        description: 'Copy a workspace FILE to a new path. The destination must not exist — like a move, a copy never overwrites a file or a folder; to change what is in a file that already exists, write it. Committed + pushed as you. ' +
+        description: 'Copy a workspace FILE to a new path. The destination must not exist — like a move, a copy never overwrites a file or a folder; to change what is in a file that already exists, write it. Committed + pushed as you. `src` may be a `__tmp__/<path>` ref to a file you staged in the workspace\'s tmp folder: the copy is how such a file is kept, as a normal repository file. ' +
             'Preflight first: `dryRun: true` changes nothing and answers `{ src, dest, kind, descendants, access: { before, after }, accessChanges, allowed, reason? }` — `access` is your own `{ read, write, download, owner }` at the source and at the destination AS IT WILL BE once the copy has landed, with every `access.md` inside a copied folder counted at its new place. ' +
             'One exception to that shape: when the destination is one you may not write, the answer is the refusal alone — `allowed: false` with `reason`, and no `kind` and no `descendants`, because nothing about the source is read before that verdict.',
         inputs: {
             type: 'object',
             properties: {
                 branch: BRANCH_INPUT,
-                src: wsPath(kbDirName, 'Source path'),
+                src: orTmpRef(wsPath(kbDirName, 'Source path')),
                 dest: wsPath(kbDirName, 'Destination path — must not exist yet'),
                 dryRun: { type: 'boolean', description: 'Answer with the impact and change nothing.' },
                 sessionId: SESSION_ID_INPUT,
@@ -3019,11 +3354,16 @@ patternMatcher) {
         proposable: true,
         handler: async (a, ctx) => {
             // A copy CARRIES the source content into the destination, so BOTH ends
-            // go to the write hook. Ask about both before touching disk.
-            writePolicy.assertPathWritable(ctx.sessionId, a.src);
+            // go to the write hook. Ask about both before touching disk. A tmp ref
+            // is not a knowledge-base path, so only the destination is one to ask
+            // about when the copy is out of the workspace's tmp folder.
+            const fromTmp = isTmpRef(a.src);
+            if (!fromTmp)
+                writePolicy.assertPathWritable(ctx.sessionId, a.src);
             writePolicy.assertPathWritable(ctx.sessionId, a.dest);
             const branch = a.branch;
-            await assertAgentWriteAllowed(agentAccessGate, ctx, branch, a.src);
+            if (!fromTmp)
+                await assertAgentWriteAllowed(agentAccessGate, ctx, branch, a.src);
             await assertAgentWriteAllowed(agentAccessGate, ctx, branch, a.dest);
             const src = a.src;
             const dest = a.dest;
@@ -3073,6 +3413,10 @@ patternMatcher) {
             // the server's own absolute path. Anything that is not absence travels
             // on as it always did.
             const fs = await ctx.getFilesystem(branch);
+            if (fromTmp) {
+                await copyFromTmp(fs, branch, ctx, src, dest);
+                return { src, dest, copied: true };
+            }
             try {
                 await asEntryExists(() => fs.copyFile(src, dest));
             }