@giovannijecha/jecode 0.7.2 → 0.7.4

package/dist/tools/fs.js

@@ -4,12 +4,13 @@ import * as fs from "node:fs/promises";
 import * as path from "node:path";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { assertDirectWritableInRoot, displayPath, resolveDirectWritableInRoot, resolveExistingInRoot, } from "./paths.js";
-import { assertEditableText, assertReplacementFits, MAX_EDITABLE_CHARS, MAX_EDITABLE_LINES, readEditableText, } from "./text-boundary.js";
+import { assertEditableText, assertReplacementFits, leadingText, MAX_EDITABLE_CHARS, MAX_EDITABLE_LINES, readEditableText, } from "./text-boundary.js";
 import { atomicWrite } from "../atomic.js";
 const MAX_READ_CHARS = 60_000;
 const MAX_LIST_CHARS = 60_000;
 const MAX_LIST_ENTRIES = 2_000;
 const READ_CHUNK_BYTES = 64 * 1024;
+const DEFAULT_MUTATION_DEPENDENCIES = { atomicWrite };
 export const readFile = {
     name: "read_file",
     description: "Read a regular UTF-8 text file inside the workspace. Optionally start at a line " +
@@ -111,24 +112,38 @@ export const writeFile = {
         assertEditableText(content);
         // A write against a file that is already there is a replacement, and the
         // user is owed the difference rather than a wall of green.
-        return { before: await current(target), after: content };
+        const before = await current(target);
+        return { before: before.text, after: content, beforeExists: before.exists };
     },
     async run(args, ctx) {
-        const root = await resolveExistingInRoot(ctx.root, ".");
-        const target = await resolveDirectWritableInRoot(root, requireString(args, "path"));
-        const content = requireString(args, "content", true);
-        assertEditableText(content);
-        await fs.mkdir(path.dirname(target), { recursive: true });
-        const validate = () => assertDirectWritableInRoot(root, target);
-        await validate();
-        await unchangedSinceApproval(target, ctx.preview?.before);
-        await atomicWrite(target, content, { validate });
-        return {
-            output: `wrote ${displayPath(root, target)} (${content.length} characters)`,
-            summary: count(content, "line"),
-        };
+        return runWriteFile(args, ctx);
     },
 };
+export async function runWriteFile(args, ctx, dependencies = DEFAULT_MUTATION_DEPENDENCIES) {
+    throwIfAborted(ctx.signal);
+    const root = await resolveExistingInRoot(ctx.root, ".");
+    const target = await resolveDirectWritableInRoot(root, requireString(args, "path"));
+    const content = requireString(args, "content", true);
+    assertEditableText(content);
+    await fs.mkdir(path.dirname(target), { recursive: true });
+    throwIfAborted(ctx.signal);
+    await assertDirectWritableInRoot(root, target);
+    const before = await current(target);
+    assertApproved(before, ctx.preview, "write");
+    await dependencies.atomicWrite(target, content, {
+        signal: ctx.signal,
+        async validate(phase) {
+            await assertDirectWritableInRoot(root, target);
+            if (phase === "before-rename") {
+                await assertUnchanged(target, before, "write", ctx.preview !== undefined);
+            }
+        },
+    });
+    return {
+        output: `wrote ${displayPath(root, target)} (${content.length} characters)`,
+        summary: count(content, "line"),
+    };
+}
 export const editFile = {
     name: "edit_file",
     description: "Replace an exact string in a file. The old text must appear exactly once " +
@@ -150,32 +165,45 @@ export const editFile = {
     async preview(args, ctx) {
         const root = await resolveExistingInRoot(ctx.root, ".");
         const target = await resolveDirectWritableInRoot(root, requireString(args, "path"), true);
-        const before = await current(target);
+        const before = await current(target, true);
         // An edit that will not apply gets no preview: the run is about to say so
         // properly, and a diff of a match that does not exist would be a lie.
         try {
-            return { before, after: applied(before, args).after };
+            return {
+                before: before.text,
+                after: applied(before.text, args).after,
+                beforeExists: true,
+            };
         }
         catch {
             return undefined;
         }
     },
     async run(args, ctx) {
-        const root = await resolveExistingInRoot(ctx.root, ".");
-        const target = await resolveDirectWritableInRoot(root, requireString(args, "path"), true);
-        const before = await readEditableText(target);
-        if (ctx.preview !== undefined && before !== ctx.preview.before) {
-            throw new Error("file changed after the preview — inspect it and retry the edit");
-        }
-        const { after, made } = applied(before, args);
-        const validate = () => assertDirectWritableInRoot(root, target, true);
-        await atomicWrite(target, after, { validate });
-        return {
-            output: `edited ${displayPath(root, target)} (${made} replacement${made === 1 ? "" : "s"})`,
-            summary: plural(made, "replacement", "replacements"),
-        };
+        return runEditFile(args, ctx);
     },
 };
+export async function runEditFile(args, ctx, dependencies = DEFAULT_MUTATION_DEPENDENCIES) {
+    throwIfAborted(ctx.signal);
+    const root = await resolveExistingInRoot(ctx.root, ".");
+    const target = await resolveDirectWritableInRoot(root, requireString(args, "path"), true);
+    const before = await current(target, true);
+    assertApproved(before, ctx.preview, "edit");
+    const { after, made } = applied(before.text, args);
+    await dependencies.atomicWrite(target, after, {
+        signal: ctx.signal,
+        async validate(phase) {
+            await assertDirectWritableInRoot(root, target, true);
+            if (phase === "before-rename") {
+                await assertUnchanged(target, before, "edit", ctx.preview !== undefined);
+            }
+        },
+    });
+    return {
+        output: `edited ${displayPath(root, target)} (${made} replacement${made === 1 ? "" : "s"})`,
+        summary: plural(made, "replacement", "replacements"),
+    };
+}
 async function readRange(target, offset, limit, signal) {
     throwIfAborted(signal);
     const firstLine = Math.max(1, offset ?? 1);
@@ -203,8 +231,7 @@ async function readRange(target, offset, limit, signal) {
             return;
         const room = MAX_READ_CHARS - text.length;
         if (fragment.length > room) {
-            if (room > 0)
-                text += fragment.slice(0, room);
+            text += leadingText(fragment, room);
             truncated = true;
             stopped = true;
             return;
@@ -284,9 +311,17 @@ function applied(before, args) {
     assertEditableText(after, "edited content");
     return { after, made };
 }
-/** What is on disk now, or nothing at all — a file that is not there yet. */
-async function current(target) {
-    return readEditableText(target, { missingAsEmpty: true });
+/** What is on disk now, preserving the difference between absent and empty. */
+async function current(target, mustExist = false) {
+    try {
+        return { exists: true, text: await readEditableText(target) };
+    }
+    catch (error) {
+        if (!mustExist && error.code === "ENOENT") {
+            return { exists: false, text: "" };
+        }
+        throw error;
+    }
 }
 function countOccurrences(haystack, needle) {
     let occurrences = 0;
@@ -295,12 +330,24 @@ function countOccurrences(haystack, needle) {
     }
     return occurrences;
 }
-async function unchangedSinceApproval(target, approved) {
+function assertApproved(current, preview, operation) {
+    if (preview === undefined)
+        return;
+    const existenceChanged = preview.beforeExists !== undefined && current.exists !== preview.beforeExists;
+    if (existenceChanged || current.text !== preview.before) {
+        throw changedFile(operation, true);
+    }
+}
+async function assertUnchanged(target, expected, operation, previewed) {
     const onDisk = await current(target);
-    if (approved !== undefined && onDisk !== approved) {
-        throw new Error("file changed after the preview — inspect it and retry the write");
+    if (onDisk.exists !== expected.exists || onDisk.text !== expected.text) {
+        throw changedFile(operation, previewed);
     }
 }
+function changedFile(operation, previewed) {
+    const when = previewed ? "after the preview" : "while preparing the change";
+    return new Error(`file changed ${when} — inspect it and retry the ${operation}`);
+}
 function count(text, noun) {
     if (text === "")
         return "empty";

package/dist/tools/search.js

@@ -4,12 +4,14 @@ import * as path from "node:path";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { displayPath, resolveExistingInRoot } from "./paths.js";
 import { trySearchWithRipgrep } from "./ripgrep.js";
+import { leadingText } from "./text-boundary.js";
 const DEFAULT_RESULTS = 100;
 const MAX_RESULTS = 500;
 const MAX_VISITED = 20_000;
 const MAX_FILE_BYTES = 1_000_000;
 const MAX_MATCH_LINE = 500;
 const MAX_GLOB_CHARS = 512;
+const PORTABLE_SEARCH_CONCURRENCY = 8;
 const RG_PREFIX_BYTES = 2_000_000;
 const RG_PREFIX_FILES = 500;
 const MIN_RG_TAIL_BYTES = 2_000_000;
@@ -82,36 +84,66 @@ export const searchText = {
         const match = pattern === undefined || pattern === "" ? () => true : glob(pattern);
         const limit = resultLimit(args);
         const found = [];
+        const candidates = [];
+        const prefix = [];
         const tail = [];
         let skipped = 0;
         let prefixBytes = 0;
         let prefixFiles = 0;
         let tailBytes = 0;
-        const walked = await walk(start, scoped, async (lexical) => {
-            const relative = displayPath(scoped.root, lexical);
-            if (!match(relative))
+        const flushPrefix = async () => {
+            if (prefix.length === 0)
                 return false;
-            const file = await resolveExistingInRoot(scoped.root, lexical);
-            const info = await fs.stat(file);
-            if (info.size > MAX_FILE_BYTES) {
-                skipped++;
-                return false;
-            }
-            const candidate = { path: file, bytes: info.size };
-            if (prefixFiles + 1 > RG_PREFIX_FILES ||
-                prefixBytes + info.size > RG_PREFIX_BYTES) {
-                tail.push(candidate);
-                tailBytes += info.size;
-                return false;
-            }
-            prefixFiles++;
-            prefixBytes += info.size;
-            const searched = await portableSearch([candidate], scoped, needle, sensitive, limit - found.length);
+            const searched = await portableSearch(prefix.splice(0), scoped, needle, sensitive, limit - found.length);
             found.push(...searched.matches);
             skipped += searched.skipped;
             return found.length >= limit;
+        };
+        const flushCandidates = async () => {
+            if (candidates.length === 0)
+                return false;
+            const inspected = await Promise.all(candidates.splice(0).map((lexical) => inspectCandidate(scoped.root, lexical)));
+            checkAbort(ctx.signal);
+            for (const result of inspected) {
+                if ("error" in result) {
+                    if (!skippable(result.error))
+                        throw result.error;
+                    skipped++;
+                    continue;
+                }
+                const candidate = result.file;
+                if (candidate.bytes > MAX_FILE_BYTES) {
+                    skipped++;
+                    continue;
+                }
+                if (prefixFiles + 1 > RG_PREFIX_FILES ||
+                    prefixBytes + candidate.bytes > RG_PREFIX_BYTES) {
+                    tail.push(candidate);
+                    tailBytes += candidate.bytes;
+                    continue;
+                }
+                prefixFiles++;
+                prefixBytes += candidate.bytes;
+                prefix.push(candidate);
+                if (prefix.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPrefix())
+                    return true;
+            }
+            return false;
+        };
+        const walked = await walk(start, scoped, async (lexical) => {
+            const relative = displayPath(scoped.root, lexical);
+            if (!match(relative))
+                return false;
+            candidates.push(lexical);
+            return candidates.length >= PORTABLE_SEARCH_CONCURRENCY
+                ? await flushCandidates()
+                : false;
         });
-        const accelerated = preferRipgrep(tail, tailBytes)
+        if (found.length < limit)
+            await flushCandidates();
+        if (found.length < limit)
+            await flushPrefix();
+        const accelerated = found.length < limit && preferRipgrep(tail, tailBytes)
             ? await trySearchWithRipgrep({
                 root: scoped.root,
                 files: tail,
@@ -121,7 +153,7 @@ export const searchText = {
                 signal: ctx.signal,
             })
             : undefined;
-        const portable = accelerated === undefined && tail.length > 0
+        const portable = found.length < limit && accelerated === undefined && tail.length > 0
             ? await portableSearch(tail, scoped, needle, sensitive, limit - found.length)
             : undefined;
         found.push(...(portable?.matches ?? accelerated?.matches.map((match) => (`${displayPath(scoped.root, match.path)}:${match.line}:${clip(match.text)}`)) ?? []));
@@ -133,39 +165,69 @@ export const searchText = {
         };
     },
 };
-async function portableSearch(files, ctx, needle, sensitive, limit) {
+export async function portableSearch(files, ctx, needle, sensitive, limit, read = readSearchFile) {
     const found = [];
     let skipped = 0;
-    for (const file of files) {
+    for (let start = 0; start < files.length; start += PORTABLE_SEARCH_CONCURRENCY) {
         checkAbort(ctx.signal);
-        let text;
-        try {
-            const data = await fs.readFile(file.path);
-            if (data.includes(0)) {
+        const remaining = limit - found.length;
+        if (remaining <= 0)
+            break;
+        const batch = await Promise.all(files.slice(start, start + PORTABLE_SEARCH_CONCURRENCY)
+            .map((file) => searchFile(file, ctx, needle, sensitive, remaining, read)));
+        for (const searched of batch) {
+            if (searched === undefined) {
                 skipped++;
                 continue;
             }
-            text = data.toString("utf8");
-        }
-        catch (error) {
-            if (skippable(error)) {
-                skipped++;
-                continue;
+            for (const match of searched) {
+                found.push(match);
+                if (found.length >= limit)
+                    return { matches: found, skipped };
             }
-            throw error;
-        }
-        for (const [index, line] of text.replace(/\r\n?/g, "\n").split("\n").entries()) {
-            checkAbort(ctx.signal);
-            const haystack = sensitive ? line : line.toLocaleLowerCase();
-            if (!haystack.includes(needle))
-                continue;
-            found.push(`${displayPath(ctx.root, file.path)}:${index + 1}:${clip(line)}`);
-            if (found.length >= limit)
-                return { matches: found, skipped };
         }
     }
     return { matches: found, skipped };
 }
+async function inspectCandidate(root, lexical) {
+    try {
+        const file = await resolveExistingInRoot(root, lexical);
+        return { file: { path: file, bytes: (await fs.stat(file)).size } };
+    }
+    catch (error) {
+        return { error };
+    }
+}
+async function searchFile(file, ctx, needle, sensitive, limit, read) {
+    let data;
+    try {
+        data = await read(file.path, ctx.signal);
+    }
+    catch (error) {
+        checkAbort(ctx.signal);
+        if (skippable(error))
+            return undefined;
+        throw error;
+    }
+    checkAbort(ctx.signal);
+    if (data.byteLength > MAX_FILE_BYTES || data.includes(0))
+        return undefined;
+    const found = [];
+    const text = data.toString("utf8");
+    for (const [index, line] of text.replace(/\r\n?/g, "\n").split("\n").entries()) {
+        checkAbort(ctx.signal);
+        const haystack = sensitive ? line : line.toLocaleLowerCase();
+        if (!haystack.includes(needle))
+            continue;
+        found.push(`${displayPath(ctx.root, file.path)}:${index + 1}:${clip(line)}`);
+        if (found.length >= limit)
+            break;
+    }
+    return found;
+}
+function readSearchFile(file, signal) {
+    return signal === undefined ? fs.readFile(file) : fs.readFile(file, { signal });
+}
 function preferRipgrep(files, bytes) {
     return files.length >= MIN_RG_TAIL_FILES || bytes >= MIN_RG_TAIL_BYTES;
 }
@@ -335,7 +397,7 @@ function summary(count, limit, capped, one, many) {
 function clip(line) {
     if (line.length <= MAX_MATCH_LINE)
         return line;
-    return `${line.slice(0, MAX_MATCH_LINE - 1)}…`;
+    return `${leadingText(line, MAX_MATCH_LINE - 1)}…`;
 }
 function checkAbort(signal) {
     if (signal?.aborted !== true)

package/dist/tools/shell.js

@@ -5,6 +5,7 @@ import * as path from "node:path";
 import { optionalInt, requireString } from "./args.js";
 import { credentialRedactor, redactCredentials, shellEnvironment } from "../credential-safety.js";
 import { resolveExecutable } from "../executable.js";
+import { leadingText, trailingText } from "./text-boundary.js";
 const DEFAULT_TIMEOUT_MS = 120_000;
 const MAX_TIMEOUT_MS = 2_147_483_647;
 const MAX_OUTPUT_CHARS = 30_000;
@@ -128,18 +129,25 @@ function capture(onOutput) {
     let head = "";
     let tail = "";
     let total = 0;
+    let headClosed = false;
     const appendSafe = (chunk) => {
         total += chunk.length;
-        const room = Math.max(0, half - head.length);
-        head += chunk.slice(0, room);
-        const rest = chunk.slice(room);
-        if (rest !== "")
-            tail = `${tail}${rest}`.slice(-half);
+        let rest = chunk;
+        if (!headClosed) {
+            const candidate = `${head}${chunk}`;
+            head = leadingText(candidate, half);
+            rest = candidate.slice(head.length);
+            if (candidate.length > half)
+                headClosed = true;
+        }
+        if (rest !== "") {
+            tail = trailingText(`${tail}${rest}`, MAX_OUTPUT_CHARS - head.length);
+        }
         if (chunk !== "")
             onOutput?.(formatted().trimEnd());
     };
     const formatted = () => {
-        if (total <= MAX_OUTPUT_CHARS)
+        if (total === head.length + tail.length)
             return `${head}${tail}`;
         const cut = total - head.length - tail.length;
         return `${head}\n\n[... ${cut} characters cut ...]\n\n${tail}`;

package/dist/tools/text-boundary.js

@@ -1,6 +1,7 @@
-// Shared limits for tools that need to hold an entire text file in memory.
+// Shared size and truncation boundaries for text handled by workspace tools.
 import { constants } from "node:fs";
 import { lstat, open } from "node:fs/promises";
+export { leadingText, trailingText } from "../text-boundary.js";
 export const MAX_EDITABLE_BYTES = 4_000_000;
 export const MAX_EDITABLE_CHARS = 1_000_000;
 export const MAX_EDITABLE_LINES = 20_000;
@@ -8,29 +9,13 @@ const READ_CHUNK_BYTES = 64 * 1024;
 /** Read a regular UTF-8 file without allowing an unbounded allocation. */
 export async function readEditableText(file, options = {}) {
     const label = options.label ?? "file";
-    let details;
-    try {
-        details = await lstat(file);
-    }
-    catch (error) {
-        if (options.missingAsEmpty === true && isMissing(error))
-            return "";
-        throw error;
-    }
+    const details = await lstat(file);
     if (!details.isFile())
         throw new Error(`${label} must be a regular file`);
-    let handle;
-    try {
-        const flags = process.platform === "win32"
-            ? "r"
-            : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
-        handle = await open(file, flags);
-    }
-    catch (error) {
-        if (options.missingAsEmpty === true && isMissing(error))
-            return "";
-        throw error;
-    }
+    const flags = process.platform === "win32"
+        ? "r"
+        : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
+    const handle = await open(file, flags);
     try {
         const stat = await handle.stat();
         if (!stat.isFile())
@@ -97,6 +82,3 @@ function newlineCount(text) {
 function limitError(label, limit) {
     return new Error(`${label} exceeds the whole-file mutation limit of ${limit}`);
 }
-function isMissing(error) {
-    return error.code === "ENOENT";
-}

package/dist/tui/app-state.js

@@ -11,7 +11,6 @@ export function appState() {
         past: [],
         recall: -1,
         draft: "",
-        spin: 0,
         closeWhenIdle: false,
         committedNodeId: 0,
     };

package/dist/tui/app-workflows.js

@@ -4,10 +4,10 @@ import { runTurn } from "../controller.js";
 import { resolveContextPolicy } from "../context/capacity.js";
 import { compactContext } from "../context/compactor.js";
 import { compactSession } from "../context/manual.js";
-import { isContextOverflow, shouldResolveContextPolicy } from "../context/policy.js";
+import { isContextOverflow } from "../context/policy.js";
 import { updateSettings } from "../settings.js";
 import { saveTranscript } from "../transcript-export.js";
-import { recordAuxiliaryUsage, recordUsage } from "../usage.js";
+import { recordAuxiliaryUsage, recordRequestInput, recordUsage } from "../usage.js";
 import { selectTimeline } from "../timeline.js";
 import { answerAt } from "./approve.js";
 import * as edit from "./editor.js";
@@ -117,7 +117,31 @@ export function appWorkflows(options) {
         const prospectiveNodeId = session.conversation.nodes.length + 1;
         let nodeId;
         let context;
-        let contextPolicy;
+        let firstPolicy = true;
+        const policy = () => {
+            let visible = firstPolicy;
+            firstPolicy = false;
+            if (visible) {
+                state.status = "Checking context";
+                options.render();
+            }
+            return resolveContextPolicy({
+                provider: session.provider,
+                model: session.model,
+                compactionPercent: session.config.compactionPercent,
+                signal: activity.control.signal,
+                onStatus: (status) => {
+                    visible = true;
+                    state.status = status;
+                    options.render();
+                },
+            }).finally(() => {
+                if (visible) {
+                    state.status = WAITING;
+                    options.render();
+                }
+            });
+        };
         options.emit({ kind: "user", text });
         const user = { role: "user", content: [{ kind: "text", text }] };
         history.push(user);
@@ -136,6 +160,7 @@ export function appWorkflows(options) {
                 state.status = text;
             },
             usage: (usage) => recordUsage(session.usage, usage),
+            requestInput: (inputTokens) => recordRequestInput(session.usage, inputTokens),
         });
         const persist = async (checkpoint, settlement, failure) => {
             const next = session.conversation.commit({
@@ -157,31 +182,12 @@ export function appWorkflows(options) {
             nodeId = next.activeNodeId;
             state.committedNodeId = next.activeNodeId;
         };
-        const compact = async (checkpoint, projected, reason, error) => {
-            if (reason === "overflow" && (error === undefined || !isContextOverflow(error))) {
-                return undefined;
-            }
-            const force = reason === "overflow";
-            if (!shouldResolveContextPolicy(projected, session.usage.lastInputTokens, force)) {
+        const compact = async (checkpoint, projected, request) => {
+            if (request.reason === "overflow" &&
+                (request.error === undefined || !isContextOverflow(request.error))) {
                 return undefined;
             }
-            if (contextPolicy === undefined) {
-                state.status = "Checking context";
-                options.render();
-                contextPolicy = resolveContextPolicy({
-                    provider: session.provider,
-                    model: session.model,
-                    compactionPercent: session.config.compactionPercent,
-                    signal: activity.control.signal,
-                    onStatus: (status) => {
-                        state.status = status;
-                        options.render();
-                    },
-                }).finally(() => {
-                    state.status = WAITING;
-                    options.render();
-                });
-            }
+            const force = request.reason === "overflow";
             const result = await compactContext({
                 provider: session.provider,
                 model: session.model,
@@ -190,10 +196,10 @@ export function appWorkflows(options) {
                 turn: checkpoint.slice(historyStart),
                 nodeId: nodeId ?? prospectiveNodeId,
                 coveredMessages: context?.messageCount ?? 0,
-                lastInputTokens: session.usage.lastInputTokens,
+                lastInputTokens: Math.max(session.usage.lastInputTokens, request.inputTokens),
                 signal: activity.control.signal,
                 force,
-                policy: await contextPolicy,
+                policy: request.policy,
                 onBegin: () => {
                     state.status = "Compacting";
                     options.render();
@@ -213,7 +219,11 @@ export function appWorkflows(options) {
         events.onContext = compact;
         events.onCheckpoint = async (checkpoint, settlement, projected) => {
             await persist(checkpoint, settlement);
-            const compacted = await compact(checkpoint, projected, "budget");
+            const compacted = await compact(checkpoint, projected, {
+                reason: "budget",
+                policy: await policy(),
+                inputTokens: session.usage.lastInputTokens,
+            });
             if (compacted !== undefined)
                 await persist(checkpoint, settlement);
             return compacted;
@@ -221,7 +231,7 @@ export function appWorkflows(options) {
         let finishReason;
         let failed;
         try {
-            await runTurn(history, controllerOptions(session, permissions.availableTools()), events, activity.control.signal, modelHistory);
+            await runTurn(history, controllerOptions(session, policy, permissions.availableTools()), events, activity.control.signal, modelHistory);
         }
         catch (error) {
             const interrupted = activity.control.signal.aborted;