@giovannijecha/jecode 0.7.1 → 0.7.3

package/dist/tools/search.js

@@ -4,12 +4,14 @@ import * as path from "node:path";
 import { optionalBool, optionalInt, optionalString, requireString } from "./args.js";
 import { displayPath, resolveExistingInRoot } from "./paths.js";
 import { trySearchWithRipgrep } from "./ripgrep.js";
+import { leadingText } from "./text-boundary.js";
 const DEFAULT_RESULTS = 100;
 const MAX_RESULTS = 500;
 const MAX_VISITED = 20_000;
 const MAX_FILE_BYTES = 1_000_000;
 const MAX_MATCH_LINE = 500;
 const MAX_GLOB_CHARS = 512;
+const PORTABLE_SEARCH_CONCURRENCY = 8;
 const RG_PREFIX_BYTES = 2_000_000;
 const RG_PREFIX_FILES = 500;
 const MIN_RG_TAIL_BYTES = 2_000_000;
@@ -82,36 +84,66 @@ export const searchText = {
         const match = pattern === undefined || pattern === "" ? () => true : glob(pattern);
         const limit = resultLimit(args);
         const found = [];
+        const candidates = [];
+        const prefix = [];
         const tail = [];
         let skipped = 0;
         let prefixBytes = 0;
         let prefixFiles = 0;
         let tailBytes = 0;
-        const walked = await walk(start, scoped, async (lexical) => {
-            const relative = displayPath(scoped.root, lexical);
-            if (!match(relative))
+        const flushPrefix = async () => {
+            if (prefix.length === 0)
                 return false;
-            const file = await resolveExistingInRoot(scoped.root, lexical);
-            const info = await fs.stat(file);
-            if (info.size > MAX_FILE_BYTES) {
-                skipped++;
-                return false;
-            }
-            const candidate = { path: file, bytes: info.size };
-            if (prefixFiles + 1 > RG_PREFIX_FILES ||
-                prefixBytes + info.size > RG_PREFIX_BYTES) {
-                tail.push(candidate);
-                tailBytes += info.size;
-                return false;
-            }
-            prefixFiles++;
-            prefixBytes += info.size;
-            const searched = await portableSearch([candidate], scoped, needle, sensitive, limit - found.length);
+            const searched = await portableSearch(prefix.splice(0), scoped, needle, sensitive, limit - found.length);
             found.push(...searched.matches);
             skipped += searched.skipped;
             return found.length >= limit;
+        };
+        const flushCandidates = async () => {
+            if (candidates.length === 0)
+                return false;
+            const inspected = await Promise.all(candidates.splice(0).map((lexical) => inspectCandidate(scoped.root, lexical)));
+            checkAbort(ctx.signal);
+            for (const result of inspected) {
+                if ("error" in result) {
+                    if (!skippable(result.error))
+                        throw result.error;
+                    skipped++;
+                    continue;
+                }
+                const candidate = result.file;
+                if (candidate.bytes > MAX_FILE_BYTES) {
+                    skipped++;
+                    continue;
+                }
+                if (prefixFiles + 1 > RG_PREFIX_FILES ||
+                    prefixBytes + candidate.bytes > RG_PREFIX_BYTES) {
+                    tail.push(candidate);
+                    tailBytes += candidate.bytes;
+                    continue;
+                }
+                prefixFiles++;
+                prefixBytes += candidate.bytes;
+                prefix.push(candidate);
+                if (prefix.length >= PORTABLE_SEARCH_CONCURRENCY && await flushPrefix())
+                    return true;
+            }
+            return false;
+        };
+        const walked = await walk(start, scoped, async (lexical) => {
+            const relative = displayPath(scoped.root, lexical);
+            if (!match(relative))
+                return false;
+            candidates.push(lexical);
+            return candidates.length >= PORTABLE_SEARCH_CONCURRENCY
+                ? await flushCandidates()
+                : false;
         });
-        const accelerated = preferRipgrep(tail, tailBytes)
+        if (found.length < limit)
+            await flushCandidates();
+        if (found.length < limit)
+            await flushPrefix();
+        const accelerated = found.length < limit && preferRipgrep(tail, tailBytes)
             ? await trySearchWithRipgrep({
                 root: scoped.root,
                 files: tail,
@@ -121,7 +153,7 @@ export const searchText = {
                 signal: ctx.signal,
             })
             : undefined;
-        const portable = accelerated === undefined && tail.length > 0
+        const portable = found.length < limit && accelerated === undefined && tail.length > 0
             ? await portableSearch(tail, scoped, needle, sensitive, limit - found.length)
             : undefined;
         found.push(...(portable?.matches ?? accelerated?.matches.map((match) => (`${displayPath(scoped.root, match.path)}:${match.line}:${clip(match.text)}`)) ?? []));
@@ -133,39 +165,69 @@ export const searchText = {
         };
     },
 };
-async function portableSearch(files, ctx, needle, sensitive, limit) {
+export async function portableSearch(files, ctx, needle, sensitive, limit, read = readSearchFile) {
     const found = [];
     let skipped = 0;
-    for (const file of files) {
+    for (let start = 0; start < files.length; start += PORTABLE_SEARCH_CONCURRENCY) {
         checkAbort(ctx.signal);
-        let text;
-        try {
-            const data = await fs.readFile(file.path);
-            if (data.includes(0)) {
+        const remaining = limit - found.length;
+        if (remaining <= 0)
+            break;
+        const batch = await Promise.all(files.slice(start, start + PORTABLE_SEARCH_CONCURRENCY)
+            .map((file) => searchFile(file, ctx, needle, sensitive, remaining, read)));
+        for (const searched of batch) {
+            if (searched === undefined) {
                 skipped++;
                 continue;
             }
-            text = data.toString("utf8");
-        }
-        catch (error) {
-            if (skippable(error)) {
-                skipped++;
-                continue;
+            for (const match of searched) {
+                found.push(match);
+                if (found.length >= limit)
+                    return { matches: found, skipped };
             }
-            throw error;
-        }
-        for (const [index, line] of text.replace(/\r\n?/g, "\n").split("\n").entries()) {
-            checkAbort(ctx.signal);
-            const haystack = sensitive ? line : line.toLocaleLowerCase();
-            if (!haystack.includes(needle))
-                continue;
-            found.push(`${displayPath(ctx.root, file.path)}:${index + 1}:${clip(line)}`);
-            if (found.length >= limit)
-                return { matches: found, skipped };
         }
     }
     return { matches: found, skipped };
 }
+async function inspectCandidate(root, lexical) {
+    try {
+        const file = await resolveExistingInRoot(root, lexical);
+        return { file: { path: file, bytes: (await fs.stat(file)).size } };
+    }
+    catch (error) {
+        return { error };
+    }
+}
+async function searchFile(file, ctx, needle, sensitive, limit, read) {
+    let data;
+    try {
+        data = await read(file.path, ctx.signal);
+    }
+    catch (error) {
+        checkAbort(ctx.signal);
+        if (skippable(error))
+            return undefined;
+        throw error;
+    }
+    checkAbort(ctx.signal);
+    if (data.byteLength > MAX_FILE_BYTES || data.includes(0))
+        return undefined;
+    const found = [];
+    const text = data.toString("utf8");
+    for (const [index, line] of text.replace(/\r\n?/g, "\n").split("\n").entries()) {
+        checkAbort(ctx.signal);
+        const haystack = sensitive ? line : line.toLocaleLowerCase();
+        if (!haystack.includes(needle))
+            continue;
+        found.push(`${displayPath(ctx.root, file.path)}:${index + 1}:${clip(line)}`);
+        if (found.length >= limit)
+            break;
+    }
+    return found;
+}
+function readSearchFile(file, signal) {
+    return signal === undefined ? fs.readFile(file) : fs.readFile(file, { signal });
+}
 function preferRipgrep(files, bytes) {
     return files.length >= MIN_RG_TAIL_FILES || bytes >= MIN_RG_TAIL_BYTES;
 }
@@ -335,7 +397,7 @@ function summary(count, limit, capped, one, many) {
 function clip(line) {
     if (line.length <= MAX_MATCH_LINE)
         return line;
-    return `${line.slice(0, MAX_MATCH_LINE - 1)}…`;
+    return `${leadingText(line, MAX_MATCH_LINE - 1)}…`;
 }
 function checkAbort(signal) {
     if (signal?.aborted !== true)

package/dist/tools/shell.js

@@ -5,6 +5,7 @@ import * as path from "node:path";
 import { optionalInt, requireString } from "./args.js";
 import { credentialRedactor, redactCredentials, shellEnvironment } from "../credential-safety.js";
 import { resolveExecutable } from "../executable.js";
+import { leadingText, trailingText } from "./text-boundary.js";
 const DEFAULT_TIMEOUT_MS = 120_000;
 const MAX_TIMEOUT_MS = 2_147_483_647;
 const MAX_OUTPUT_CHARS = 30_000;
@@ -128,18 +129,25 @@ function capture(onOutput) {
     let head = "";
     let tail = "";
     let total = 0;
+    let headClosed = false;
     const appendSafe = (chunk) => {
         total += chunk.length;
-        const room = Math.max(0, half - head.length);
-        head += chunk.slice(0, room);
-        const rest = chunk.slice(room);
-        if (rest !== "")
-            tail = `${tail}${rest}`.slice(-half);
+        let rest = chunk;
+        if (!headClosed) {
+            const candidate = `${head}${chunk}`;
+            head = leadingText(candidate, half);
+            rest = candidate.slice(head.length);
+            if (candidate.length > half)
+                headClosed = true;
+        }
+        if (rest !== "") {
+            tail = trailingText(`${tail}${rest}`, MAX_OUTPUT_CHARS - head.length);
+        }
         if (chunk !== "")
             onOutput?.(formatted().trimEnd());
     };
     const formatted = () => {
-        if (total <= MAX_OUTPUT_CHARS)
+        if (total === head.length + tail.length)
             return `${head}${tail}`;
         const cut = total - head.length - tail.length;
         return `${head}\n\n[... ${cut} characters cut ...]\n\n${tail}`;

package/dist/tools/text-boundary.js

@@ -1,36 +1,51 @@
-// Shared limits for tools that need to hold an entire text file in memory.
+// Shared size and truncation boundaries for text handled by workspace tools.
 import { constants } from "node:fs";
 import { lstat, open } from "node:fs/promises";
 export const MAX_EDITABLE_BYTES = 4_000_000;
 export const MAX_EDITABLE_CHARS = 1_000_000;
 export const MAX_EDITABLE_LINES = 20_000;
 const READ_CHUNK_BYTES = 64 * 1024;
+const GRAPHEME_SEGMENTER = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+/** Keep a bounded prefix without returning part of a user-perceived character. */
+export function leadingText(text, maxCodeUnits) {
+    if (maxCodeUnits <= 0)
+        return "";
+    if (text.length <= maxCodeUnits)
+        return text;
+    let end = 0;
+    for (const { index, segment } of GRAPHEME_SEGMENTER.segment(text)) {
+        const next = index + segment.length;
+        if (next > maxCodeUnits)
+            break;
+        end = next;
+    }
+    return text.slice(0, end);
+}
+/** Keep a bounded suffix without returning part of a user-perceived character. */
+export function trailingText(text, maxCodeUnits) {
+    if (maxCodeUnits <= 0)
+        return "";
+    if (text.length <= maxCodeUnits)
+        return text;
+    let start = text.length;
+    for (const { index } of GRAPHEME_SEGMENTER.segment(text)) {
+        if (text.length - index <= maxCodeUnits) {
+            start = index;
+            break;
+        }
+    }
+    return text.slice(start);
+}
 /** Read a regular UTF-8 file without allowing an unbounded allocation. */
 export async function readEditableText(file, options = {}) {
     const label = options.label ?? "file";
-    let details;
-    try {
-        details = await lstat(file);
-    }
-    catch (error) {
-        if (options.missingAsEmpty === true && isMissing(error))
-            return "";
-        throw error;
-    }
+    const details = await lstat(file);
     if (!details.isFile())
         throw new Error(`${label} must be a regular file`);
-    let handle;
-    try {
-        const flags = process.platform === "win32"
-            ? "r"
-            : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
-        handle = await open(file, flags);
-    }
-    catch (error) {
-        if (options.missingAsEmpty === true && isMissing(error))
-            return "";
-        throw error;
-    }
+    const flags = process.platform === "win32"
+        ? "r"
+        : constants.O_RDONLY | (constants.O_NONBLOCK ?? 0);
+    const handle = await open(file, flags);
     try {
         const stat = await handle.stat();
         if (!stat.isFile())
@@ -97,6 +112,3 @@ function newlineCount(text) {
 function limitError(label, limit) {
     return new Error(`${label} exceeds the whole-file mutation limit of ${limit}`);
 }
-function isMissing(error) {
-    return error.code === "ENOENT";
-}

package/dist/tui/app-state.js

@@ -11,7 +11,6 @@ export function appState() {
         past: [],
         recall: -1,
         draft: "",
-        spin: 0,
         closeWhenIdle: false,
         committedNodeId: 0,
     };

package/dist/tui/app-workflows.js

@@ -4,12 +4,13 @@ import { runTurn } from "../controller.js";
 import { resolveContextPolicy } from "../context/capacity.js";
 import { compactContext } from "../context/compactor.js";
 import { compactSession } from "../context/manual.js";
-import { isContextOverflow, shouldResolveContextPolicy } from "../context/policy.js";
+import { isContextOverflow } from "../context/policy.js";
 import { updateSettings } from "../settings.js";
 import { saveTranscript } from "../transcript-export.js";
 import { recordAuxiliaryUsage, recordUsage } from "../usage.js";
 import { selectTimeline } from "../timeline.js";
 import { answerAt } from "./approve.js";
+import * as edit from "./editor.js";
 import { cancel as cancelOpen } from "./overlay.js";
 import { controllerOptions, turnFailure } from "./session-view.js";
 import { transcribe } from "./turn.js";
@@ -116,7 +117,31 @@ export function appWorkflows(options) {
         const prospectiveNodeId = session.conversation.nodes.length + 1;
         let nodeId;
         let context;
-        let contextPolicy;
+        let firstPolicy = true;
+        const policy = () => {
+            let visible = firstPolicy;
+            firstPolicy = false;
+            if (visible) {
+                state.status = "Checking context";
+                options.render();
+            }
+            return resolveContextPolicy({
+                provider: session.provider,
+                model: session.model,
+                compactionPercent: session.config.compactionPercent,
+                signal: activity.control.signal,
+                onStatus: (status) => {
+                    visible = true;
+                    state.status = status;
+                    options.render();
+                },
+            }).finally(() => {
+                if (visible) {
+                    state.status = WAITING;
+                    options.render();
+                }
+            });
+        };
         options.emit({ kind: "user", text });
         const user = { role: "user", content: [{ kind: "text", text }] };
         history.push(user);
@@ -136,7 +161,7 @@ export function appWorkflows(options) {
             },
             usage: (usage) => recordUsage(session.usage, usage),
         });
-        const persist = async (checkpoint, settlement) => {
+        const persist = async (checkpoint, settlement, failure) => {
             const next = session.conversation.commit({
                 ...(nodeId === undefined ? {} : { nodeId }),
                 parentId,
@@ -149,37 +174,19 @@ export function appWorkflows(options) {
                 messages: checkpoint.slice(historyStart),
                 blocks: state.blocks.slice(blockStart),
                 ...(context === undefined ? {} : { context }),
+                ...(failure === undefined ? {} : { failure }),
             }, settlement);
             await session.persistence?.checkpoint(next);
             session.conversation = next;
             nodeId = next.activeNodeId;
             state.committedNodeId = next.activeNodeId;
         };
-        const compact = async (checkpoint, projected, reason, error) => {
-            if (reason === "overflow" && (error === undefined || !isContextOverflow(error))) {
+        const compact = async (checkpoint, projected, request) => {
+            if (request.reason === "overflow" &&
+                (request.error === undefined || !isContextOverflow(request.error))) {
                 return undefined;
             }
-            const force = reason === "overflow";
-            if (!shouldResolveContextPolicy(projected, session.usage.lastInputTokens, force)) {
-                return undefined;
-            }
-            if (contextPolicy === undefined) {
-                state.status = "Checking context";
-                options.render();
-                contextPolicy = resolveContextPolicy({
-                    provider: session.provider,
-                    model: session.model,
-                    compactionPercent: session.config.compactionPercent,
-                    signal: activity.control.signal,
-                    onStatus: (status) => {
-                        state.status = status;
-                        options.render();
-                    },
-                }).finally(() => {
-                    state.status = WAITING;
-                    options.render();
-                });
-            }
+            const force = request.reason === "overflow";
             const result = await compactContext({
                 provider: session.provider,
                 model: session.model,
@@ -188,10 +195,10 @@ export function appWorkflows(options) {
                 turn: checkpoint.slice(historyStart),
                 nodeId: nodeId ?? prospectiveNodeId,
                 coveredMessages: context?.messageCount ?? 0,
-                lastInputTokens: session.usage.lastInputTokens,
+                lastInputTokens: Math.max(session.usage.lastInputTokens, request.inputTokens),
                 signal: activity.control.signal,
                 force,
-                policy: await contextPolicy,
+                policy: request.policy,
                 onBegin: () => {
                     state.status = "Compacting";
                     options.render();
@@ -211,24 +218,75 @@ export function appWorkflows(options) {
         events.onContext = compact;
         events.onCheckpoint = async (checkpoint, settlement, projected) => {
             await persist(checkpoint, settlement);
-            const compacted = await compact(checkpoint, projected, "budget");
+            const compacted = await compact(checkpoint, projected, {
+                reason: "budget",
+                policy: await policy(),
+                inputTokens: session.usage.lastInputTokens,
+            });
             if (compacted !== undefined)
                 await persist(checkpoint, settlement);
             return compacted;
         };
         let finishReason;
+        let failed;
         try {
-            await runTurn(history, controllerOptions(session, permissions.availableTools()), events, activity.control.signal, modelHistory);
+            await runTurn(history, controllerOptions(session, policy, permissions.availableTools()), events, activity.control.signal, modelHistory);
         }
         catch (error) {
             const interrupted = activity.control.signal.aborted;
-            finishReason = interrupted ? "interrupted" : "failed";
-            options.emit(turnFailure(session, error, interrupted));
+            const completed = nodeId !== undefined && session.conversation.activeNodeId === nodeId &&
+                session.conversation.activeNode?.settlement === "completed";
+            if (completed) {
+                const notice = turnFailure(session, error, interrupted);
+                feedback.show({ text: notice.text, tone: notice.tone, timeoutMs: 6_000 });
+            }
+            else {
+                finishReason = interrupted ? "interrupted" : "failed";
+                failed = { error: error, interrupted };
+            }
         }
         finally {
-            events.finish(finishReason);
-            options.finishActivity(activity);
+            try {
+                events.finish(finishReason);
+                if (failed !== undefined) {
+                    const notice = turnFailure(session, failed.error, failed.interrupted);
+                    const settlement = failed.interrupted ? "interrupted" : "failed";
+                    const failure = {
+                        text: notice.text,
+                        tone: failed.interrupted ? "warn" : "error",
+                    };
+                    options.emit(notice);
+                    try {
+                        await persist(closeFailedTurn(history, settlement), settlement, failure);
+                    }
+                    catch (error) {
+                        // A failed persistence boundary cannot remain visible as if it had
+                        // been saved. Revert to the last durable path and return the input
+                        // to the composer so the user can retry without losing it.
+                        options.replaceTranscript();
+                        state.editor = edit.of(text);
+                        feedback.show({
+                            text: error.message,
+                            tone: "error",
+                            timeoutMs: 6_000,
+                        });
+                    }
+                }
+            }
+            finally {
+                options.finishActivity(activity);
+            }
         }
     }
     return { command, turn };
 }
+function closeFailedTurn(history, settlement) {
+    const closed = [...history];
+    if (closed.at(-1)?.role === "assistant")
+        return closed;
+    const text = settlement === "interrupted"
+        ? "The previous attempt was interrupted by the user before completion."
+        : "The previous attempt failed before completion.";
+    closed.push({ role: "assistant", content: [{ kind: "text", text }] });
+    return closed;
+}