@giovannijecha/jecode 0.8.4 → 0.8.5

package/dist/tools/fs.js

@@ -1,7 +1,8 @@
 // Filesystem tools: read, list, write, edit.
-import { constants } from "node:fs";
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
+import { withStableFile } from "../bounded-file.js";
+import { readStableDirectory } from "../stable-directory.js";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { assertDirectWritableInRoot, displayPath, resolveDirectWritableInRoot, resolveExistingInRoot, } from "./paths.js";
 import { assertEditableText, assertReplacementFits, leadingText, MAX_EDITABLE_CHARS, MAX_EDITABLE_LINES, readEditableText, } from "./text-boundary.js";
@@ -10,6 +11,7 @@ const MAX_READ_CHARS = 60_000;
 const MAX_LIST_CHARS = 60_000;
 const MAX_LIST_ENTRIES = 2_000;
 const READ_CHUNK_BYTES = 64 * 1024;
+const MAX_READ_SCAN_BYTES = 16 * 1024 * 1024;
 const DEFAULT_MUTATION_DEPENDENCIES = { atomicWrite };
 export const readFile = {
     name: "read_file",
@@ -27,22 +29,32 @@ export const readFile = {
         required: ["path"],
     },
     async run(args, ctx) {
-        const root = await resolveExistingInRoot(ctx.root, ".");
-        const target = await resolveExistingInRoot(root, requireString(args, "path"));
-        const offset = optionalInt(args, "offset");
-        const limit = optionalInt(args, "limit");
-        const { text, truncated } = await readRange(target, offset, limit, ctx.signal);
-        if (truncated) {
-            return {
-                output: `${text}\n\n[truncated at ${MAX_READ_CHARS} characters — read a narrower range]`,
-                summary: `truncated at ${MAX_READ_CHARS} characters`,
-            };
-        }
-        if (text === "")
-            return { output: "[file is empty]", summary: "empty" };
-        return { output: text, summary: `${count(text, "line")}` };
+        return runReadFile(args, ctx);
     },
 };
+export async function runReadFile(args, ctx, dependencies = {}) {
+    const root = await resolveExistingInRoot(ctx.root, ".");
+    const target = await resolveExistingInRoot(root, requireString(args, "path"));
+    const offset = optionalInt(args, "offset");
+    const limit = optionalInt(args, "limit");
+    const { text, truncated, scanCapped } = await readRange(root, target, offset, limit, ctx.signal, dependencies);
+    if (truncated) {
+        return {
+            output: `${text}\n\n[truncated at ${MAX_READ_CHARS} characters — read a narrower range]`,
+            summary: `truncated at ${MAX_READ_CHARS} characters`,
+        };
+    }
+    if (scanCapped) {
+        const notice = `[read stopped after scanning ${MAX_READ_SCAN_BYTES} bytes — use a smaller offset or search_text]`;
+        return {
+            output: text === "" ? notice : `${text}\n\n${notice}`,
+            summary: `scan capped at ${MAX_READ_SCAN_BYTES} bytes`,
+        };
+    }
+    if (text === "")
+        return { output: "[file is empty]", summary: "empty" };
+    return { output: text, summary: `${count(text, "line")}` };
+}
 export const listDir = {
     name: "list_dir",
     description: "List the entries of a directory inside the workspace. Directories end with a slash.",
@@ -56,39 +68,44 @@ export const listDir = {
         required: [],
     },
     async run(args, ctx) {
-        const root = await resolveExistingInRoot(ctx.root, ".");
-        const requested = optionalString(args, "path");
-        const relative = requested === undefined || requested.trim() === "" ? "." : requested;
-        const target = await resolveExistingInRoot(root, relative);
-        const entries = [];
-        let chars = 0;
-        let truncated = false;
-        const directory = await fs.opendir(target);
-        for await (const entry of directory) {
-            const label = entry.isDirectory() ? `${entry.name}/` : entry.name;
-            const separator = entries.length === 0 ? 0 : 1;
-            if (entries.length >= MAX_LIST_ENTRIES ||
-                chars + separator + label.length > MAX_LIST_CHARS) {
-                truncated = true;
-                break;
-            }
-            entries.push(label);
-            chars += separator + label.length;
-        }
-        if (entries.length === 0)
-            return { output: "[empty directory]", summary: "empty" };
-        const listing = entries
-            .sort((a, b) => a.localeCompare(b))
-            .join("\n");
-        if (truncated) {
-            return {
-                output: `${listing}\n\n[truncated after ${entries.length} entries]`,
-                summary: `${entries.length}+ entries`,
-            };
-        }
-        return { output: listing, summary: plural(entries.length, "entry", "entries") };
+        return runListDir(args, ctx);
     },
 };
+export async function runListDir(args, ctx, dependencies = {}) {
+    const root = await resolveExistingInRoot(ctx.root, ".");
+    const requested = optionalString(args, "path");
+    const relative = requested === undefined || requested.trim() === "" ? "." : requested;
+    const target = await resolveExistingInRoot(root, relative);
+    const inspected = await readStableDirectory(root, target, {
+        maxEntries: MAX_LIST_ENTRIES + 1,
+        signal: ctx.signal,
+        beforeOpen: dependencies.beforeOpen,
+    });
+    const entries = [];
+    let chars = 0;
+    let truncated = inspected.capped;
+    for (const entry of inspected.entries) {
+        const label = entry.kind === "directory" ? `${entry.name}/` : entry.name;
+        const separator = entries.length === 0 ? 0 : 1;
+        if (entries.length >= MAX_LIST_ENTRIES ||
+            chars + separator + label.length > MAX_LIST_CHARS) {
+            truncated = true;
+            break;
+        }
+        entries.push(label);
+        chars += separator + label.length;
+    }
+    if (entries.length === 0)
+        return { output: "[empty directory]", summary: "empty" };
+    const listing = entries.join("\n");
+    if (truncated) {
+        return {
+            output: `${listing}\n\n[truncated after ${entries.length} entries]`,
+            summary: `${entries.length}+ entries`,
+        };
+    }
+    return { output: listing, summary: plural(entries.length, "entry", "entries") };
+}
 export const writeFile = {
     name: "write_file",
     description: "Create a file, or replace its entire contents. Parent directories are " +
@@ -204,81 +221,86 @@ export async function runEditFile(args, ctx, dependencies = DEFAULT_MUTATION_DEP
         summary: plural(made, "replacement", "replacements"),
     };
 }
-async function readRange(target, offset, limit, signal) {
+async function readRange(root, target, offset, limit, signal, dependencies) {
     throwIfAborted(signal);
     const firstLine = Math.max(1, offset ?? 1);
     const lineCount = limit === undefined ? undefined : Math.max(0, limit);
     const endLine = lineCount === undefined ? Number.POSITIVE_INFINITY : firstLine + lineCount;
     if (lineCount === 0)
-        return { text: "", truncated: false };
-    if (!(await fs.lstat(target)).isFile())
-        throw new Error("path must be a regular file");
-    // O_NONBLOCK prevents a path swapped to a FIFO between lstat and open from
-    // waiting forever for a writer. The handle stat closes that race before any
-    // content is accepted.
-    const flags = process.platform === "win32"
-        ? "r"
-        : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
-    const handle = await fs.open(target, flags);
-    const decoder = new TextDecoder();
-    let text = "";
-    let line = 1;
-    let stopped = false;
-    let truncated = false;
-    const selected = (at) => at >= firstLine && at < endLine;
-    const append = (fragment) => {
-        if (fragment === "")
-            return;
-        const room = MAX_READ_CHARS - text.length;
-        if (fragment.length > room) {
-            text += leadingText(fragment, room);
-            truncated = true;
-            stopped = true;
-            return;
-        }
-        text += fragment;
-    };
-    const consume = (chunk) => {
-        let start = 0;
-        while (!stopped && start < chunk.length) {
-            const newline = chunk.indexOf("\n", start);
-            const end = newline === -1 ? chunk.length : newline;
-            if (selected(line))
-                append(chunk.slice(start, end));
-            if (stopped || newline === -1)
+        return { text: "", truncated: false, scanCapped: false };
+    const result = await withStableFile(target, {
+        label: "read file",
+        signal,
+        beforeOpen: dependencies.beforeOpen,
+    }, async (handle, opened) => {
+        const decoder = new TextDecoder();
+        let text = "";
+        let line = 1;
+        let stopped = false;
+        let truncated = false;
+        let reachedEnd = false;
+        const selected = (at) => at >= firstLine && at < endLine;
+        const append = (fragment) => {
+            if (fragment === "")
                 return;
-            const nextLine = line + 1;
-            if (selected(line) && selected(nextLine))
-                append("\n");
-            line = nextLine;
-            if (line >= endLine) {
+            const room = MAX_READ_CHARS - text.length;
+            if (fragment.length > room) {
+                text += leadingText(fragment, room);
+                truncated = true;
                 stopped = true;
                 return;
             }
-            start = newline + 1;
-        }
-    };
-    try {
-        if (!(await handle.stat()).isFile())
-            throw new Error("path must be a regular file");
+            text += fragment;
+        };
+        const consume = (chunk) => {
+            let start = 0;
+            while (!stopped && start < chunk.length) {
+                const newline = chunk.indexOf("\n", start);
+                const end = newline === -1 ? chunk.length : newline;
+                if (selected(line))
+                    append(chunk.slice(start, end));
+                if (stopped || newline === -1)
+                    return;
+                const nextLine = line + 1;
+                if (selected(line) && selected(nextLine))
+                    append("\n");
+                line = nextLine;
+                if (line >= endLine) {
+                    stopped = true;
+                    return;
+                }
+                start = newline + 1;
+            }
+        };
         let position = 0;
         const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES);
-        while (!stopped) {
+        while (!stopped && position < MAX_READ_SCAN_BYTES) {
             throwIfAborted(signal);
-            const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+            const length = Math.min(buffer.length, MAX_READ_SCAN_BYTES - position);
+            const { bytesRead } = await handle.read(buffer, 0, length, position);
             throwIfAborted(signal);
-            if (bytesRead === 0)
+            if (bytesRead === 0) {
+                reachedEnd = true;
                 break;
+            }
             position += bytesRead;
             consume(decoder.decode(buffer.subarray(0, bytesRead), { stream: true }));
         }
-        if (!stopped)
+        if (!stopped && BigInt(position) >= opened.size)
+            reachedEnd = true;
+        if (!stopped && reachedEnd)
             consume(decoder.decode());
+        return {
+            text,
+            truncated,
+            scanCapped: !stopped && !reachedEnd && position >= MAX_READ_SCAN_BYTES,
+        };
+    });
+    const confirmed = await resolveExistingInRoot(root, target);
+    if (path.relative(target, confirmed) !== "") {
+        throw new Error("read file changed while it was being read");
     }
-    finally {
-        await handle.close();
-    }
-    return { text, truncated };
+    return result;
 }
 function throwIfAborted(signal) {
     if (signal?.aborted !== true)

package/dist/tools/search.js

@@ -1,9 +1,10 @@
 // Bounded, read-only workspace discovery without borrowing a shell.
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
+import { BoundedFileError, readBoundedFile } from "../bounded-file.js";
+import { readStableDirectory, StableDirectoryError, } from "../stable-directory.js";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { displayPath, resolveExistingInRoot } from "./paths.js";
-import { trySearchWithRipgrep } from "./ripgrep.js";
 import { leadingText } from "./text-boundary.js";
 const DEFAULT_RESULTS = 100;
 const MAX_RESULTS = 500;
@@ -12,10 +13,6 @@ const MAX_FILE_BYTES = 1_000_000;
 const MAX_MATCH_LINE = 500;
 const MAX_GLOB_CHARS = 512;
 const PORTABLE_SEARCH_CONCURRENCY = 8;
-const RG_PREFIX_BYTES = 2_000_000;
-const RG_PREFIX_FILES = 500;
-const MIN_RG_TAIL_BYTES = 2_000_000;
-const MIN_RG_TAIL_FILES = 500;
 const SKIP = new Set([".git", ".hg", ".svn", "node_modules"]);
 export const findFiles = {
     name: "find_files",
@@ -84,80 +81,30 @@ export const searchText = {
         const match = pattern === undefined || pattern === "" ? () => true : glob(pattern);
         const limit = resultLimit(args);
         const found = [];
-        const candidates = [];
-        const prefix = [];
-        const tail = [];
+        const pending = [];
         let skipped = 0;
-        let prefixBytes = 0;
-        let prefixFiles = 0;
-        let tailBytes = 0;
-        const flushPrefix = async () => {
-            if (prefix.length === 0)
+        const flushPending = async () => {
+            if (pending.length === 0)
                 return false;
-            const searched = await portableSearch(prefix.splice(0), scoped, needle, sensitive, limit - found.length);
+            const searched = await portableSearch(pending.splice(0), scoped, needle, sensitive, limit - found.length);
             found.push(...searched.matches);
             skipped += searched.skipped;
             return found.length >= limit;
         };
-        const flushCandidates = async () => {
-            if (candidates.length === 0)
-                return false;
-            const inspected = await Promise.all(candidates.splice(0).map((lexical) => inspectCandidate(scoped.root, lexical)));
-            checkAbort(ctx.signal);
-            for (const result of inspected) {
-                if ("error" in result) {
-                    if (!skippable(result.error))
-                        throw result.error;
-                    skipped++;
-                    continue;
-                }
-                const candidate = result.file;
-                if (candidate.bytes > MAX_FILE_BYTES) {
-                    skipped++;
-                    continue;
-                }
-                if (prefixFiles + 1 > RG_PREFIX_FILES ||
-                    prefixBytes + candidate.bytes > RG_PREFIX_BYTES) {
-                    tail.push(candidate);
-                    tailBytes += candidate.bytes;
-                    continue;
-                }
-                prefixFiles++;
-                prefixBytes += candidate.bytes;
-                prefix.push(candidate);
-                if (prefix.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPrefix())
-                    return true;
-            }
-            return false;
-        };
-        const walked = await walk(start, scoped, async (lexical) => {
+        const walked = await walk(start, scoped, async (lexical, expected) => {
             const relative = displayPath(scoped.root, lexical);
             if (!match(relative))
                 return false;
-            candidates.push(lexical);
-            return candidates.length >= PORTABLE_SEARCH_CONCURRENCY
-                ? await flushCandidates()
-                : false;
+            const bytes = Number(expected.size);
+            if (bytes > MAX_FILE_BYTES) {
+                skipped++;
+                return false;
+            }
+            pending.push({ path: lexical, bytes, expected });
+            return pending.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPending();
         });
         if (found.length < limit)
-            await flushCandidates();
-        if (found.length < limit)
-            await flushPrefix();
-        const accelerated = found.length < limit && preferRipgrep(tail, tailBytes)
-            ? await trySearchWithRipgrep({
-                root: scoped.root,
-                files: tail,
-                query,
-                caseSensitive: sensitive,
-                limit: limit - found.length,
-                signal: ctx.signal,
-            })
-            : undefined;
-        const portable = found.length < limit && accelerated === undefined && tail.length > 0
-            ? await portableSearch(tail, scoped, needle, sensitive, limit - found.length)
-            : undefined;
-        found.push(...(portable?.matches ?? accelerated?.matches.map((match) => (`${displayPath(scoped.root, match.path)}:${match.line}:${clip(match.text)}`)) ?? []));
-        skipped += portable?.skipped ?? accelerated?.binaryPaths.length ?? 0;
+            await flushPending();
         const extra = skipped === 0 ? "" : ` · skipped ${skipped} binary/large/unreadable`;
         return {
             output: found.length === 0 ? "[no matches]" : found.join("\n"),
@@ -165,7 +112,7 @@ export const searchText = {
         };
     },
 };
-export async function portableSearch(files, ctx, needle, sensitive, limit, read = readSearchFile) {
+export async function portableSearch(files, ctx, needle, sensitive, limit, read) {
     const found = [];
     let skipped = 0;
     for (let start = 0; start < files.length; start += PORTABLE_SEARCH_CONCURRENCY) {
@@ -189,19 +136,12 @@ export async function portableSearch(files, ctx, needle, sensitive, limit, read
     }
     return { matches: found, skipped };
 }
-async function inspectCandidate(root, lexical) {
-    try {
-        const file = await resolveExistingInRoot(root, lexical);
-        return { file: { path: file, bytes: (await fs.stat(file)).size } };
-    }
-    catch (error) {
-        return { error };
-    }
-}
 async function searchFile(file, ctx, needle, sensitive, limit, read) {
     let data;
     try {
-        data = await read(file.path, ctx.signal);
+        data = read === undefined
+            ? await readVerifiedSearchFile(file, ctx)
+            : await read(file.path, ctx.signal);
     }
     catch (error) {
         checkAbort(ctx.signal);
@@ -225,55 +165,79 @@ async function searchFile(file, ctx, needle, sensitive, limit, read) {
     }
     return found;
 }
-function readSearchFile(file, signal) {
-    return signal === undefined ? fs.readFile(file) : fs.readFile(file, { signal });
-}
-function preferRipgrep(files, bytes) {
-    return files.length >= MIN_RG_TAIL_FILES || bytes >= MIN_RG_TAIL_BYTES;
+async function readVerifiedSearchFile(file, ctx) {
+    if (file.expected === undefined) {
+        throw new BoundedFileError("changed", "search file has no verified identity");
+    }
+    return readBoundedFile(file.path, MAX_FILE_BYTES, {
+        label: "search file",
+        signal: ctx.signal,
+        expected: file.expected,
+        validate: async () => {
+            const confirmed = await resolveExistingInRoot(ctx.root, file.path);
+            if (confirmed !== file.path) {
+                throw new BoundedFileError("changed", "search file left the workspace");
+            }
+        },
+    });
 }
 async function walk(start, ctx, visit) {
     const pending = [];
     let seen = 0;
+    let buffered = 0;
+    let capped = false;
     const enter = async (lexical) => {
         checkAbort(ctx.signal);
         const directory = await resolveExistingInRoot(ctx.root, lexical);
-        let entries;
+        const room = MAX_VISITED - seen - buffered;
+        if (room <= 0) {
+            capped = true;
+            return false;
+        }
         try {
-            entries = await fs.readdir(directory, { withFileTypes: true, encoding: "utf8" });
+            const inspected = await readStableDirectory(ctx.root, directory, {
+                maxEntries: room,
+                signal: ctx.signal,
+            });
+            if (inspected.capped)
+                capped = true;
+            const entries = [...inspected.entries];
+            buffered += entries.length;
+            pending.push({ directory, entries, next: 0 });
         }
         catch (error) {
             if (skippable(error))
-                return;
+                return true;
             throw error;
         }
-        entries.sort((a, b) => a.name.localeCompare(b.name));
-        pending.push({ directory, entries, next: 0 });
+        return true;
     };
-    await enter(start);
+    if (!await enter(start))
+        return { capped: true };
     while (pending.length > 0) {
         checkAbort(ctx.signal);
         const frame = pending[pending.length - 1];
         if (frame === undefined)
             break;
-        const entry = frame.entries[frame.next++];
+        const index = frame.next++;
+        const entry = frame.entries[index];
         if (entry === undefined) {
             pending.pop();
             continue;
         }
-        if (++seen > MAX_VISITED)
-            return { capped: true };
-        if (entry.isSymbolicLink())
-            continue;
+        frame.entries[index] = undefined;
+        buffered--;
+        seen++;
         const target = path.join(frame.directory, entry.name);
-        if (entry.isDirectory()) {
+        if (entry.kind === "directory") {
             if (!SKIP.has(entry.name))
                 await enter(target);
         }
-        else if (entry.isFile() && (await visit(target))) {
-            return { capped: false };
+        else if (await visit(target, entry.expected)) {
+            return { capped };
         }
     }
-    return { capped: false };
+    return { capped };
 }
 async function startAt(args, ctx) {
     const candidate = optionalString(args, "path") ?? ".";
@@ -414,5 +378,6 @@ function checkAbort(signal) {
     throw signal.reason instanceof Error ? signal.reason : new Error("aborted");
 }
 function skippable(error) {
-    return ["EACCES", "EPERM", "ENOENT"].includes(error.code ?? "");
+    return error instanceof BoundedFileError || error instanceof StableDirectoryError ||
+        ["EACCES", "EPERM", "ENOENT"].includes(error.code ?? "");
 }

package/dist/tools/text-boundary.js

@@ -1,48 +1,22 @@
 // Shared size and truncation boundaries for text handled by workspace tools.
-import { constants } from "node:fs";
-import { lstat, open } from "node:fs/promises";
+import { BoundedFileError, readBoundedText } from "../bounded-file.js";
 export { leadingText, trailingText } from "../text-boundary.js";
 export const MAX_EDITABLE_BYTES = 4_000_000;
 export const MAX_EDITABLE_CHARS = 1_000_000;
 export const MAX_EDITABLE_LINES = 20_000;
-const READ_CHUNK_BYTES = 64 * 1024;
 /** Read a regular UTF-8 file without allowing an unbounded allocation. */
 export async function readEditableText(file, options = {}) {
     const label = options.label ?? "file";
-    const details = await lstat(file);
-    if (!details.isFile())
-        throw new Error(`${label} must be a regular file`);
-    const flags = process.platform === "win32"
-        ? "r"
-        : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
-    const handle = await open(file, flags);
     try {
-        const stat = await handle.stat();
-        if (!stat.isFile())
-            throw new Error(`${label} must be a regular file`);
-        if (stat.size > MAX_EDITABLE_BYTES) {
-            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
-        }
-        const chunks = [];
-        let total = 0;
-        while (total <= MAX_EDITABLE_BYTES) {
-            const room = MAX_EDITABLE_BYTES + 1 - total;
-            const buffer = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, room));
-            const { bytesRead } = await handle.read(buffer, 0, buffer.length, total);
-            if (bytesRead === 0)
-                break;
-            chunks.push(buffer.subarray(0, bytesRead));
-            total += bytesRead;
-        }
-        if (total > MAX_EDITABLE_BYTES) {
-            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
-        }
-        const text = Buffer.concat(chunks, total).toString("utf8");
+        const text = await readBoundedText(file, MAX_EDITABLE_BYTES, { label });
         assertEditableText(text, label);
         return text;
     }
-    finally {
-        await handle.close();
+    catch (error) {
+        if (error instanceof BoundedFileError && error.kind === "too-large") {
+            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
+        }
+        throw error;
     }
 }
 /** Reject whole-file writes that exceed Jecode's mutation budget. */