jev-lens 0.5.0

package/dist/treesitter.js

@@ -0,0 +1,263 @@
+/**
+ * Tree-sitter backed structure for code views: exact top-level blocks (functions, classes,
+ * methods, top-level assignments) with signature lines, for the languages that ship in
+ * tree-sitter-wasms. Falls back to undefined when the language is unknown or parsing fails,
+ * in which case views.ts uses its regex heuristics.
+ */
+import { existsSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, extname, join } from "node:path";
+const require = createRequire(import.meta.url);
+/** Grammars shipped by @vscode/tree-sitter-wasm (ABI-compatible with web-tree-sitter 0.27). */
+const LANG_BY_EXT = {
+    ".js": "javascript", ".mjs": "javascript", ".cjs": "javascript", ".jsx": "javascript",
+    ".ts": "typescript", ".mts": "typescript", ".cts": "typescript", ".tsx": "tsx",
+    ".py": "python", ".pyx": "python", ".go": "go", ".rs": "rust", ".java": "java", ".rb": "ruby",
+    ".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", ".cs": "c-sharp", ".php": "php",
+    ".sh": "bash", ".bash": "bash", ".css": "css",
+    ".kt": "kotlin", ".kts": "kotlin",
+};
+/** Extra grammar packages: language → wasm path resolver (the VS Code bundle has no Kotlin). */
+const EXTRA_WASM = {
+    kotlin: () => {
+        try {
+            const dir = dirname(require.resolve("@binclusive/tree-sitter-kotlin-wasm/package.json"));
+            const { readdirSync } = require("node:fs");
+            const walk = (d) => { for (const f of readdirSync(d, { withFileTypes: true })) {
+                const p = join(d, f.name);
+                if (f.isDirectory() && f.name !== "node_modules") {
+                    const r = walk(p);
+                    if (r)
+                        return r;
+                }
+                else if (f.name.endsWith(".wasm"))
+                    return p;
+            } return undefined; };
+            return walk(dir);
+        }
+        catch {
+            return undefined;
+        }
+    },
+};
+/** Node types that count as top-level blocks, per language family. */
+const BLOCK_TYPES = new Set([
+    "function_declaration", "function_definition", "generator_function_declaration", "class_declaration", "class_definition",
+    "method_definition", "method_declaration", "abstract_class_declaration", "interface_declaration", "type_alias_declaration",
+    "enum_declaration", "module", "internal_module", "lexical_declaration", "variable_declaration", "export_statement",
+    "decorated_definition", "function_item", "impl_item", "struct_item", "enum_item", "trait_item", "mod_item", "const_item", "static_item",
+    "type_item", "func_literal", "method_declaration", "type_declaration", "var_declaration", "const_declaration",
+    "class_specifier", "struct_specifier", "namespace_definition", "template_declaration", "preproc_function_def",
+    "function_signature", "singleton_method", "module", "class", "method", "object_declaration", "property_declaration",
+    "companion_object", "constructor_declaration", "record_declaration", "annotation_type_declaration", "macro_definition", "extern_crate_declaration",
+]);
+const HEADER_TYPES = new Set(["import_statement", "import_declaration", "import_from_statement", "package_clause", "package_declaration", "use_declaration", "preproc_include", "require_call", "using_directive", "comment", "expression_statement", "attribute_item", "mod_item"]);
+let ts;
+let inited;
+const languages = new Map();
+function wasmDir() {
+    try {
+        return join(dirname(require.resolve("@vscode/tree-sitter-wasm/package.json")), "wasm");
+    }
+    catch {
+        return undefined;
+    }
+}
+async function init() {
+    if (ts)
+        return ts;
+    if (!inited) {
+        inited = (async () => {
+            try {
+                const mod = (await import("web-tree-sitter"));
+                await mod.Parser.init();
+                ts = mod;
+            }
+            catch {
+                ts = undefined;
+            }
+        })();
+    }
+    await inited;
+    return ts;
+}
+async function language(name) {
+    if (!languages.has(name)) {
+        languages.set(name, (async () => {
+            const mod = await init();
+            const dir = wasmDir();
+            if (!mod || !dir)
+                return undefined;
+            const file = EXTRA_WASM[name]?.() ?? join(dir, `tree-sitter-${name}.wasm`);
+            if (!file || !existsSync(file))
+                return undefined;
+            try {
+                return await mod.Language.load(file);
+            }
+            catch {
+                return undefined;
+            }
+        })());
+    }
+    return languages.get(name);
+}
+export function languageForPath(path) {
+    return LANG_BY_EXT[extname(path).toLowerCase()];
+}
+/**
+ * Top-level blocks from the syntax tree: each named child of the root that is a declaration
+ * becomes a block spanning its full line range (including a directly preceding comment).
+ * Leading imports and other non-block statements are folded into a header block.
+ */
+export async function treeSitterBlocks(path, text, maxBlocks = 48) {
+    const lang = languageForPath(path);
+    if (!lang)
+        return undefined;
+    const mod = await init();
+    const L = await language(lang);
+    if (!mod || !L)
+        return undefined;
+    const parser = new mod.Parser();
+    parser.setLanguage(L);
+    const tree = parser.parse(text);
+    if (!tree)
+        return undefined;
+    const lines = text.split("\n");
+    const root = tree.rootNode;
+    // Python and Ruby put everything under "module"/"program"; unwrap one level when the root has a single block child.
+    let children = root.namedChildren;
+    if (children.length === 1 && (children[0].type === "module" || children[0].type === "program"))
+        children = children[0].namedChildren;
+    const blocks = [];
+    let pendingComment;
+    let headerEnd = 0;
+    for (const c of children) {
+        if (!c)
+            continue;
+        const startLine = c.startPosition.row;
+        const endLine = c.endPosition.row;
+        if (c.type === "comment") {
+            if (pendingComment === undefined)
+                pendingComment = startLine;
+            continue;
+        }
+        let node = c;
+        // export const x = ...; export default class ...; decorated defs
+        if ((c.type === "export_statement" || c.type === "decorated_definition") && c.namedChildren.length) {
+            const inner = c.namedChildren.find((n) => n && BLOCK_TYPES.has(n.type));
+            if (inner)
+                node = inner;
+        }
+        // Multi-line top-level assignments (config dicts, tables, constants) are blocks too.
+        const isBigAssignment = (node.type === "expression_statement" || node.type === "assignment") && endLine - startLine >= 3;
+        // one-line declarations (type aliases, Kotlin data classes, Rust consts) are blocks too: they belong in the outline
+        const isBlock = BLOCK_TYPES.has(node.type) || isBigAssignment;
+        if (!isBlock) {
+            pendingComment = undefined;
+            if (blocks.length === 0)
+                headerEnd = endLine;
+            continue;
+        }
+        const from = (pendingComment ?? startLine) + 1;
+        pendingComment = undefined;
+        const sig = lines[startLine].trim().slice(0, 120);
+        // Large classes: expose their methods as blocks so the second step can pick individual bodies.
+        const body = node.namedChildren.find((n) => n && (n.type === "class_body" || n.type === "block" || n.type === "declaration_list" || n.type === "field_declaration_list"));
+        const methods = body ? body.namedChildren.filter((n) => n && (n.type === "method_definition" || n.type === "function_definition" || n.type === "method_declaration" || n.type === "constructor_declaration" || n.type === "decorated_definition" || n.type === "function_item" || n.type === "function_declaration" || n.type === "companion_object" || n.type === "property_declaration")) : [];
+        if (endLine - startLine > 40 && methods.length >= 2) {
+            blocks.push({ name: sig, from, to: methods[0].startPosition.row });
+            for (let k = 0; k < methods.length; k++) {
+                const mm = methods[k];
+                const mEnd = k + 1 < methods.length ? methods[k + 1].startPosition.row : endLine + 1;
+                blocks.push({ name: `${sig.replace(/[{:]\s*$/, "")} › ${lines[mm.startPosition.row].trim().slice(0, 80)}`, from: mm.startPosition.row + 1, to: mEnd });
+            }
+            continue;
+        }
+        blocks.push({ name: sig, from, to: endLine + 1 });
+    }
+    tree.delete();
+    parser.delete();
+    if (blocks.length < 2)
+        return blocks.length === 0 ? [] : undefined;
+    // fill gaps so the block list partitions the file
+    const out = [];
+    if (blocks[0].from > 1)
+        out.push({ name: "(header: imports, constants)", from: 1, to: blocks[0].from - 1 });
+    for (let i = 0; i < blocks.length; i++) {
+        const b = { ...blocks[i] };
+        const next = blocks[i + 1];
+        if (next && next.from > b.to + 1)
+            b.to = next.from - 1;
+        if (!next && b.to < lines.length)
+            b.to = lines.length;
+        out.push(b);
+    }
+    void headerEnd;
+    return out.slice(0, maxBlocks);
+}
+const SIGNATURE_TYPES = new Set(["function_declaration", "function_definition", "method_definition", "method_declaration", "constructor_declaration", "function_item", "decorated_definition", "class_declaration", "class_definition", "interface_declaration", "struct_item", "enum_item", "trait_item", "impl_item", "object_declaration", "companion_object", "type_alias_declaration", "type_item", "record_declaration", "enum_declaration", "abstract_class_declaration", "singleton_method", "method", "class", "module", "func_literal", "generator_function_declaration", "lexical_declaration", "property_declaration"]);
+/** 0-based rows of every declaration signature in the tree, at any nesting depth (methods in small classes, nested functions). */
+async function signatureRows(path, text) {
+    const lang = languageForPath(path);
+    if (!lang)
+        return undefined;
+    const mod = await init();
+    const L = await language(lang);
+    if (!mod || !L)
+        return undefined;
+    const parser = new mod.Parser();
+    parser.setLanguage(L);
+    const tree = parser.parse(text);
+    if (!tree)
+        return undefined;
+    const rows = new Set();
+    const walk = (n, depth) => {
+        if (depth > 6)
+            return;
+        for (const c of n.namedChildren) {
+            if (!c)
+                continue;
+            if (SIGNATURE_TYPES.has(c.type)) {
+                // property/lexical declarations only when they hold a function (arrow functions, lambdas) or are top-level
+                if ((c.type === "lexical_declaration" || c.type === "property_declaration") && depth > 0 && !/=>|lambda|fun\b|function\b/.test(text.split("\n")[c.startPosition.row]))
+                    continue;
+                rows.add(c.startPosition.row);
+            }
+            walk(c, depth + 1);
+        }
+    };
+    walk(tree.rootNode, 0);
+    tree.delete();
+    parser.delete();
+    return [...rows].sort((a, b) => a - b);
+}
+/** Signature lines (block starts) as an outline index list, 0-based. */
+export async function treeSitterOutline(path, text) {
+    const sigs = await signatureRows(path, text);
+    if (!sigs)
+        return undefined;
+    const blocks = (await treeSitterBlocks(path, text)) ?? [];
+    const idx = [...sigs];
+    const lines = text.split("\n");
+    if (blocks.length === 0)
+        for (let i = 0; i < Math.min(lines.length, 60); i++)
+            if (/^\s*(import|from|require|use|package|#include|using)\b/.test(lines[i]))
+                idx.push(i);
+    for (const b of blocks) {
+        if (b.name.startsWith("(header")) {
+            // imports plus any one-line declarations (Kotlin data classes, type aliases, constants) that were too short to be blocks
+            for (let i = b.from - 1; i < b.to; i++)
+                if (/^\s*(import|from|require|use|package|#include|using)\b/.test(lines[i]) || /^(export\s+)?(const|let|var|val|type|typealias|data class|sealed class|enum class|class|interface|object|fun|def|pub|static|final)\b/.test(lines[i]))
+                    idx.push(i);
+            continue;
+        }
+        // the signature line is the first non-comment line of the block
+        for (let i = b.from - 1; i < b.to; i++) {
+            if (!/^\s*(\/\/|\/\*|\*|#|"""|''')/.test(lines[i]) && lines[i].trim()) {
+                idx.push(i);
+                break;
+            }
+        }
+    }
+    return idx;
+}

package/dist/types.d.ts

@@ -0,0 +1,33 @@
+export type Bucket = "keep" | "trim" | "forget";
+export interface Probabilities {
+    needed: number;
+    outcomeOnly: number;
+}
+export interface Decision {
+    /** toolCallId of the tool result this decision is about. */
+    id: string;
+    toolName: string;
+    bucket: Bucket;
+    p: Probabilities;
+    /** One-line description used in the stub, fixed at decision time so the stub never changes. */
+    summary: string;
+    tokensBefore: number;
+    decidedAt: number;
+    /** "pending" until first applied in a context call; then frozen forever. */
+    status: "pending" | "applied";
+    appliedAtCall?: number;
+    /** Why the decision was applied (rolling, cold-cache, compaction, forced). */
+    appliedReason?: string;
+}
+export interface CallStats {
+    call: number;
+    at: number;
+    messages: number;
+    tokensOriginal: number;
+    tokensSent: number;
+    tokensPruned: number;
+    appliedNow: number;
+    frozen: number;
+    pendingHeld: number;
+    coldCache: boolean;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/views.d.ts

@@ -0,0 +1,129 @@
+/**
+ * Candidate views of a tool result. Every view is a deterministic subset of the original
+ * text (never generated), with line numbers so the agent can ask for exact ranges later.
+ */
+export type ViewKind = "full" | "outline" | "relevant" | "focus" | "signals" | "testlog" | "tree" | "matches" | "log" | "sample" | "head_tail" | "sections";
+export interface View {
+    kind: ViewKind;
+    text: string;
+    lines: number;
+    chars: number;
+    /** 1-based line numbers of the original that are included. */
+    included: number[];
+}
+export type ContentKind = "code" | "data" | "prose" | "command" | "listing";
+export { displayedFiles } from "./shell-display.ts";
+/** Mixed or unknown file types retain the ordinary command policy. */
+export declare function kindOfFiles(files: string[]): ContentKind | undefined;
+/** Guess what kind of content this is from the tool, its arguments and the text itself. */
+export declare function detectKind(toolName: string, args: unknown, text: string): ContentKind;
+/** Mostly lines that are file paths → a directory listing or find output. */
+export declare function looksLikePathList(text: string): boolean;
+/** Many lines with the same delimiter count → tabular or log-like data. */
+export declare function looksRepetitive(text: string): boolean;
+/** Collapse decorative runs (=====, -----, ......) and very long lines so views stay small. */
+export declare function tidyLine(l: string): string;
+export declare function fullView(text: string): View;
+export declare function headTailView(text: string, head?: number, tail?: number, tidy?: boolean): View;
+/** Code and prose structure: signatures, exports, imports, doc comments, headings. */
+export declare function outlineView(text: string, kind: ContentKind): View;
+/** Lines mentioning any of the given terms, with context. */
+export declare function focusView(text: string, terms: string[], ctx?: number, tidy?: boolean): View | undefined;
+/** Command output: error/warning/summary lines with context, plus the tail. */
+export declare function signalsView(text: string, ctx?: number, tail?: number, tidy?: boolean): View;
+/** Is this command output a test run? */
+export declare function looksLikeTestLog(text: string): boolean;
+/**
+ * Test run output reduced to what the agent acts on: the session header, every failing test's
+ * section (test id, assertion, the last frames of its traceback), the short summary, and the
+ * final counts. Passing tests, dots and decorative bars are dropped.
+ */
+export declare function testlogView(text: string, ctx?: number, maxFailLines?: number, maxIds?: number): View | undefined;
+/**
+ * Directory listings and find output: group paths by directory, keep the first entries of each
+ * directory and say how many more there are. Directories with many files (tests, fixtures) collapse.
+ */
+export declare function treeView(text: string, perDir?: number, terms?: string[], tidy?: boolean): View | undefined;
+/**
+ * grep / rg / git grep output (path:line:content): keep the first matches of every file and say how
+ * many more each file has. Files are what the agent navigates by; the tail of a long match list rarely matters.
+ */
+export declare function matchesView(text: string, perFile?: number, tidy?: boolean): View | undefined;
+/**
+ * Log-like output (scripts, servers, repeated progress lines): keep the first two and the last
+ * occurrence of every line template, so repeated lines collapse while the story stays readable.
+ */
+export declare function logView(text: string, tidy?: boolean): View | undefined;
+/** Data files: header plus a sample of rows and the count. */
+export declare function sampleView(text: string, rows?: number, tidy?: boolean): View;
+/** Pull identifier-like terms out of task text and tool arguments to drive the focus view. */
+export declare function extractTerms(...texts: string[]): string[];
+export interface ViewParams {
+    headLines: number;
+    tailLines: number;
+    focusCtx: number;
+    sampleRows: number;
+    signalsCtx: number;
+    signalsTail: number;
+    /** How many passing test ids the testlog view keeps as an index (0 = none). */
+    testIds: number;
+    /** Max lines kept per failure section in testlog. */
+    testFailLines: number;
+    /** grep-style output: matches kept per file in the "matches" view. */
+    matchesPerFile: number;
+    /** Log-like output: offer the "log" view that collapses repeated line templates. */
+    logView: boolean;
+    /** Command output: offer the "sections" view (first line of every section; bodies expanded in a second step). */
+    sectionsView: boolean;
+    /** Sections shorter than this are merged into their predecessor. */
+    sectionMinLines: number;
+    /** At most this many sections (adjacent ones are merged beyond it). */
+    sectionMaxBlocks: number;
+    /** Output without any section structure is cut into chunks of this many lines (0 = no sections then). */
+    sectionChunkLines: number;
+    minShrink: number;
+}
+export declare const DEFAULT_VIEW_PARAMS: ViewParams;
+export interface Candidates {
+    kind: ContentKind;
+    views: View[];
+}
+/** Build the candidate views for a tool result. Full is always first. Views that do not shrink the text enough are dropped. */
+export declare function buildCandidates(toolName: string, args: unknown, text: string, terms: string[], params?: Partial<ViewParams>): Candidates;
+export interface FooterOptions {
+    /** How the host names the recall tool in the note (default: pi's `recall(id: "...")` phrasing). */
+    recall?: (toolCallId: string) => string;
+    /** An extra sentence, for hosts that add their own line numbers in front of the view's. */
+    note?: string;
+}
+export declare function footer(view: View, toolCallId: string, total: number, opts?: FooterOptions): string;
+export interface Block {
+    /** The signature line (trimmed). */
+    name: string;
+    /** 1-based inclusive line range. */
+    from: number;
+    to: number;
+}
+/**
+ * Split code into top-level blocks: each block starts at a signature line with no indentation
+ * (or the least indentation seen) and runs until the next one. Leading imports form one block.
+ */
+export declare function splitBlocks(text: string, maxBlocks?: number): Block[];
+/**
+ * Split command output into sections: grep context groups (separated by `--` or a change of file), the
+ * top-level keys or items of a JSON document, markdown headings, marker lines (COMMAND:, URL:, ALL-CAPS labels,
+ * ===== bars, tracebacks) and paragraphs separated by blank lines. Small sections are merged into their
+ * predecessor; at most maxBlocks sections are kept. Output with no structure at all falls back to fixed chunks.
+ */
+export declare function splitSections(text: string, minLines?: number, maxBlocks?: number, chunkLines?: number): Block[];
+/** The first non-empty line of every section, line-numbered, with omission markers between. */
+export declare function sectionsView(text: string, blocks: Block[], tidy?: boolean): View | undefined;
+/** Outline plus the full bodies of the chosen blocks. */
+export declare function relevantView(text: string, kind: ContentKind, blocks: Block[], expand: Set<number>, outlineIncluded?: number[]): View;
+/**
+ * Async variant of buildCandidates that uses tree-sitter for the outline of code files when the
+ * grammar is available, and returns the blocks so the second step can reuse them.
+ */
+export declare function buildCandidatesAsync(toolName: string, args: unknown, text: string, terms: string[], params?: Partial<ViewParams>): Promise<Candidates & {
+    blocks?: Block[];
+}>;