@giovannijecha/jecode 0.8.4 → 0.8.6

package/dist/tools/search.js

@@ -1,21 +1,18 @@
 // Bounded, read-only workspace discovery without borrowing a shell.
 import * as fs from "node:fs/promises";
 import * as path from "node:path";
+import { BoundedFileError, readBoundedFile } from "../bounded-file.js";
+import { readStableDirectory, StableDirectoryError, } from "../stable-directory.js";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { displayPath, resolveExistingInRoot } from "./paths.js";
-import { trySearchWithRipgrep } from "./ripgrep.js";
 import { leadingText } from "./text-boundary.js";
+import { glob } from "./glob.js";
 const DEFAULT_RESULTS = 100;
 const MAX_RESULTS = 500;
 const MAX_VISITED = 20_000;
 const MAX_FILE_BYTES = 1_000_000;
 const MAX_MATCH_LINE = 500;
-const MAX_GLOB_CHARS = 512;
 const PORTABLE_SEARCH_CONCURRENCY = 8;
-const RG_PREFIX_BYTES = 2_000_000;
-const RG_PREFIX_FILES = 500;
-const MIN_RG_TAIL_BYTES = 2_000_000;
-const MIN_RG_TAIL_FILES = 500;
 const SKIP = new Set([".git", ".hg", ".svn", "node_modules"]);
 export const findFiles = {
     name: "find_files",
@@ -84,80 +81,30 @@ export const searchText = {
         const match = pattern === undefined || pattern === "" ? () => true : glob(pattern);
         const limit = resultLimit(args);
         const found = [];
-        const candidates = [];
-        const prefix = [];
-        const tail = [];
+        const pending = [];
         let skipped = 0;
-        let prefixBytes = 0;
-        let prefixFiles = 0;
-        let tailBytes = 0;
-        const flushPrefix = async () => {
-            if (prefix.length === 0)
+        const flushPending = async () => {
+            if (pending.length === 0)
                 return false;
-            const searched = await portableSearch(prefix.splice(0), scoped, needle, sensitive, limit - found.length);
+            const searched = await portableSearch(pending.splice(0), scoped, needle, sensitive, limit - found.length);
             found.push(...searched.matches);
             skipped += searched.skipped;
             return found.length >= limit;
         };
-        const flushCandidates = async () => {
-            if (candidates.length === 0)
-                return false;
-            const inspected = await Promise.all(candidates.splice(0).map((lexical) => inspectCandidate(scoped.root, lexical)));
-            checkAbort(ctx.signal);
-            for (const result of inspected) {
-                if ("error" in result) {
-                    if (!skippable(result.error))
-                        throw result.error;
-                    skipped++;
-                    continue;
-                }
-                const candidate = result.file;
-                if (candidate.bytes > MAX_FILE_BYTES) {
-                    skipped++;
-                    continue;
-                }
-                if (prefixFiles + 1 > RG_PREFIX_FILES ||
-                    prefixBytes + candidate.bytes > RG_PREFIX_BYTES) {
-                    tail.push(candidate);
-                    tailBytes += candidate.bytes;
-                    continue;
-                }
-                prefixFiles++;
-                prefixBytes += candidate.bytes;
-                prefix.push(candidate);
-                if (prefix.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPrefix())
-                    return true;
-            }
-            return false;
-        };
-        const walked = await walk(start, scoped, async (lexical) => {
+        const walked = await walk(start, scoped, async (lexical, expected) => {
             const relative = displayPath(scoped.root, lexical);
             if (!match(relative))
                 return false;
-            candidates.push(lexical);
-            return candidates.length >= PORTABLE_SEARCH_CONCURRENCY
-                ? await flushCandidates()
-                : false;
+            const bytes = Number(expected.size);
+            if (bytes > MAX_FILE_BYTES) {
+                skipped++;
+                return false;
+            }
+            pending.push({ path: lexical, bytes, expected });
+            return pending.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPending();
         });
         if (found.length < limit)
-            await flushCandidates();
-        if (found.length < limit)
-            await flushPrefix();
-        const accelerated = found.length < limit && preferRipgrep(tail, tailBytes)
-            ? await trySearchWithRipgrep({
-                root: scoped.root,
-                files: tail,
-                query,
-                caseSensitive: sensitive,
-                limit: limit - found.length,
-                signal: ctx.signal,
-            })
-            : undefined;
-        const portable = found.length < limit && accelerated === undefined && tail.length > 0
-            ? await portableSearch(tail, scoped, needle, sensitive, limit - found.length)
-            : undefined;
-        found.push(...(portable?.matches ?? accelerated?.matches.map((match) => (`${displayPath(scoped.root, match.path)}:${match.line}:${clip(match.text)}`)) ?? []));
-        skipped += portable?.skipped ?? accelerated?.binaryPaths.length ?? 0;
+            await flushPending();
         const extra = skipped === 0 ? "" : ` · skipped ${skipped} binary/large/unreadable`;
         return {
             output: found.length === 0 ? "[no matches]" : found.join("\n"),
@@ -165,7 +112,7 @@ export const searchText = {
         };
     },
 };
-export async function portableSearch(files, ctx, needle, sensitive, limit, read = readSearchFile) {
+export async function portableSearch(files, ctx, needle, sensitive, limit, read) {
     const found = [];
     let skipped = 0;
     for (let start = 0; start < files.length; start += PORTABLE_SEARCH_CONCURRENCY) {
@@ -189,19 +136,12 @@ export async function portableSearch(files, ctx, needle, sensitive, limit, read
     }
     return { matches: found, skipped };
 }
-async function inspectCandidate(root, lexical) {
-    try {
-        const file = await resolveExistingInRoot(root, lexical);
-        return { file: { path: file, bytes: (await fs.stat(file)).size } };
-    }
-    catch (error) {
-        return { error };
-    }
-}
 async function searchFile(file, ctx, needle, sensitive, limit, read) {
     let data;
     try {
-        data = await read(file.path, ctx.signal);
+        data = read === undefined
+            ? await readVerifiedSearchFile(file, ctx)
+            : await read(file.path, ctx.signal);
     }
     catch (error) {
         checkAbort(ctx.signal);
@@ -225,55 +165,79 @@ async function searchFile(file, ctx, needle, sensitive, limit, read) {
     }
     return found;
 }
-function readSearchFile(file, signal) {
-    return signal === undefined ? fs.readFile(file) : fs.readFile(file, { signal });
-}
-function preferRipgrep(files, bytes) {
-    return files.length >= MIN_RG_TAIL_FILES || bytes >= MIN_RG_TAIL_BYTES;
+async function readVerifiedSearchFile(file, ctx) {
+    if (file.expected === undefined) {
+        throw new BoundedFileError("changed", "search file has no verified identity");
+    }
+    return readBoundedFile(file.path, MAX_FILE_BYTES, {
+        label: "search file",
+        signal: ctx.signal,
+        expected: file.expected,
+        validate: async () => {
+            const confirmed = await resolveExistingInRoot(ctx.root, file.path);
+            if (confirmed !== file.path) {
+                throw new BoundedFileError("changed", "search file left the workspace");
+            }
+        },
+    });
 }
 async function walk(start, ctx, visit) {
     const pending = [];
     let seen = 0;
+    let buffered = 0;
+    let capped = false;
     const enter = async (lexical) => {
         checkAbort(ctx.signal);
         const directory = await resolveExistingInRoot(ctx.root, lexical);
-        let entries;
+        const room = MAX_VISITED - seen - buffered;
+        if (room <= 0) {
+            capped = true;
+            return false;
+        }
         try {
-            entries = await fs.readdir(directory, { withFileTypes: true, encoding: "utf8" });
+            const inspected = await readStableDirectory(ctx.root, directory, {
+                maxEntries: room,
+                signal: ctx.signal,
+            });
+            if (inspected.capped)
+                capped = true;
+            const entries = [...inspected.entries];
+            buffered += entries.length;
+            pending.push({ directory, entries, next: 0 });
         }
         catch (error) {
             if (skippable(error))
-                return;
+                return true;
             throw error;
         }
-        entries.sort((a, b) => a.name.localeCompare(b.name));
-        pending.push({ directory, entries, next: 0 });
+        return true;
     };
-    await enter(start);
+    if (!await enter(start))
+        return { capped: true };
     while (pending.length > 0) {
         checkAbort(ctx.signal);
         const frame = pending[pending.length - 1];
         if (frame === undefined)
             break;
-        const entry = frame.entries[frame.next++];
+        const index = frame.next++;
+        const entry = frame.entries[index];
         if (entry === undefined) {
             pending.pop();
             continue;
         }
-        if (++seen > MAX_VISITED)
-            return { capped: true };
-        if (entry.isSymbolicLink())
-            continue;
+        frame.entries[index] = undefined;
+        buffered--;
+        seen++;
         const target = path.join(frame.directory, entry.name);
-        if (entry.isDirectory()) {
+        if (entry.kind === "directory") {
             if (!SKIP.has(entry.name))
                 await enter(target);
         }
-        else if (entry.isFile() && (await visit(target))) {
-            return { capped: false };
+        else if (await visit(target, entry.expected)) {
+            return { capped };
         }
     }
-    return { capped: false };
+    return { capped };
 }
 async function startAt(args, ctx) {
     const candidate = optionalString(args, "path") ?? ".";
@@ -291,110 +255,6 @@ function resultLimit(args) {
         throw new Error('"max_results" must be a positive integer');
     return Math.min(requested, MAX_RESULTS);
 }
-function glob(pattern) {
-    const normalized = pattern.replace(/\\/g, "/");
-    if (normalized.length > MAX_GLOB_CHARS) {
-        throw new Error(`"pattern" must be at most ${MAX_GLOB_CHARS} characters`);
-    }
-    const tokens = tokenizeGlob(normalized.toLowerCase());
-    const basenameOnly = !normalized.includes("/");
-    return (relative) => {
-        const candidate = relative.replace(/\\/g, "/");
-        const target = (basenameOnly ? path.posix.basename(candidate) : candidate).toLowerCase();
-        return matchGlob(tokens, Array.from(target));
-    };
-}
-function tokenizeGlob(pattern) {
-    const chars = Array.from(pattern);
-    const tokens = [];
-    let index = 0;
-    while (index < chars.length) {
-        const char = chars[index];
-        if (char === "*") {
-            let end = index + 1;
-            while (chars[end] === "*")
-                end++;
-            if (end - index >= 2) {
-                if (chars[end] === "/") {
-                    tokens.push({ kind: "globdir-start" }, { kind: "globdir-body" });
-                    index = end + 1;
-                }
-                else {
-                    tokens.push({ kind: "globstar" });
-                    index = end;
-                }
-            }
-            else {
-                tokens.push({ kind: "star" });
-                index = end;
-            }
-            continue;
-        }
-        tokens.push(char === "?" ? { kind: "one" } : { kind: "literal", value: char });
-        index++;
-    }
-    return tokens;
-}
-/** Thompson-style wildcard matching: O(pattern × path), with no regex backtracking. */
-function matchGlob(tokens, text) {
-    let states = epsilonClosure(new Set([0]), tokens);
-    for (const char of text) {
-        const next = new Set();
-        for (const state of states) {
-            const token = tokens[state];
-            if (token === undefined)
-                continue;
-            switch (token.kind) {
-                case "literal":
-                    if (token.value === char)
-                        next.add(state + 1);
-                    break;
-                case "one":
-                    if (char !== "/")
-                        next.add(state + 1);
-                    break;
-                case "star":
-                    if (char !== "/")
-                        next.add(state);
-                    break;
-                case "globstar":
-                    next.add(state);
-                    break;
-                case "globdir-body":
-                    next.add(state);
-                    if (char === "/")
-                        next.add(state + 1);
-                    break;
-                case "globdir-start":
-                    break;
-            }
-        }
-        states = epsilonClosure(next, tokens);
-        if (states.size === 0)
-            return false;
-    }
-    return epsilonClosure(states, tokens).has(tokens.length);
-}
-function epsilonClosure(seed, tokens) {
-    const states = new Set(seed);
-    const pending = [...seed];
-    while (pending.length > 0) {
-        const state = pending.pop();
-        const token = tokens[state];
-        const targets = token?.kind === "globdir-start"
-            ? [state + 1, state + 2]
-            : token?.kind === "star" || token?.kind === "globstar"
-                ? [state + 1]
-                : [];
-        for (const target of targets) {
-            if (states.has(target))
-                continue;
-            states.add(target);
-            pending.push(target);
-        }
-    }
-    return states;
-}
 function summary(count, limit, capped, one, many) {
     const noun = count === 1 ? one : many;
     if (count >= limit)
@@ -414,5 +274,6 @@ function checkAbort(signal) {
     throw signal.reason instanceof Error ? signal.reason : new Error("aborted");
 }
 function skippable(error) {
-    return ["EACCES", "EPERM", "ENOENT"].includes(error.code ?? "");
+    return error instanceof BoundedFileError || error instanceof StableDirectoryError ||
+        ["EACCES", "EPERM", "ENOENT"].includes(error.code ?? "");
 }

package/dist/tools/text-boundary.js

@@ -1,48 +1,22 @@
 // Shared size and truncation boundaries for text handled by workspace tools.
-import { constants } from "node:fs";
-import { lstat, open } from "node:fs/promises";
+import { BoundedFileError, readBoundedText } from "../bounded-file.js";
 export { leadingText, trailingText } from "../text-boundary.js";
 export const MAX_EDITABLE_BYTES = 4_000_000;
 export const MAX_EDITABLE_CHARS = 1_000_000;
 export const MAX_EDITABLE_LINES = 20_000;
-const READ_CHUNK_BYTES = 64 * 1024;
 /** Read a regular UTF-8 file without allowing an unbounded allocation. */
 export async function readEditableText(file, options = {}) {
     const label = options.label ?? "file";
-    const details = await lstat(file);
-    if (!details.isFile())
-        throw new Error(`${label} must be a regular file`);
-    const flags = process.platform === "win32"
-        ? "r"
-        : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
-    const handle = await open(file, flags);
     try {
-        const stat = await handle.stat();
-        if (!stat.isFile())
-            throw new Error(`${label} must be a regular file`);
-        if (stat.size > MAX_EDITABLE_BYTES) {
-            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
-        }
-        const chunks = [];
-        let total = 0;
-        while (total <= MAX_EDITABLE_BYTES) {
-            const room = MAX_EDITABLE_BYTES + 1 - total;
-            const buffer = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, room));
-            const { bytesRead } = await handle.read(buffer, 0, buffer.length, total);
-            if (bytesRead === 0)
-                break;
-            chunks.push(buffer.subarray(0, bytesRead));
-            total += bytesRead;
-        }
-        if (total > MAX_EDITABLE_BYTES) {
-            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
-        }
-        const text = Buffer.concat(chunks, total).toString("utf8");
+        const text = await readBoundedText(file, MAX_EDITABLE_BYTES, { label });
         assertEditableText(text, label);
         return text;
     }
-    finally {
-        await handle.close();
+    catch (error) {
+        if (error instanceof BoundedFileError && error.kind === "too-large") {
+            throw limitError(label, `${MAX_EDITABLE_BYTES} UTF-8 bytes`);
+        }
+        throw error;
     }
 }
 /** Reject whole-file writes that exceed Jecode's mutation budget. */