codesearch-mcp 0.2.2-alpha → 0.3.0-alpha

package/dist/src/chunking/DeclarationChunkStrategy.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DeclarationChunkStrategy = void 0;
+const FragmentMerger_1 = require("./FragmentMerger");
+const RunPacker_1 = require("./RunPacker");
+class DeclarationChunkStrategy {
+    policy;
+    packer;
+    constructor(policy, packer) {
+        this.policy = policy;
+        this.packer = packer;
+    }
+    async chunk(ctx) {
+        const roots = ctx.declarations.roots;
+        const removed = DeclarationChunkStrategy.gaps({ start: 1, end: ctx.lines.count }, roots)
+            .flatMap(gap => ctx.lines.importRegions(gap.start, gap.end));
+        if (removed.length > 0 && DeclarationChunkStrategy.isShortPrologue(removed[0].start - 1, ctx.lines)) {
+            removed[0] = { start: 1, end: removed[0].end };
+        }
+        ctx.removed.push(...removed);
+        return this.splitLevel({ start: 1, end: ctx.lines.count }, roots, null, removed, ctx);
+    }
+    async splitNode(node, ctx) {
+        if (ctx.lines.length(node.start, node.end) <= ctx.chunkSize)
+            return RunPacker_1.RunPacker.piece(node, ctx.lines);
+        let children = node.children;
+        while (children.length === 1 && this.coversParent(children[0], node, ctx))
+            children = children[0].children;
+        return this.splitLevel(node, children, node, [], ctx);
+    }
+    coversParent(child, parent, ctx) {
+        if (child.end !== parent.end || child.start < parent.start)
+            return false;
+        for (let n = parent.start; n < child.start; n++) {
+            if (!ctx.lines.isBlank(n) && !ctx.lines.isCommentOnly(n))
+                return false;
+        }
+        return true;
+    }
+    async splitLevel(range, children, boundary, removed, ctx) {
+        const splitPoints = children.filter(c => this.policy.isSplitPoint(c, children, boundary, ctx.lines));
+        const packed = children.filter(c => !splitPoints.includes(c));
+        ctx.splitPoints.push(...splitPoints);
+        const pieces = [];
+        let cursor = range.start;
+        for (const sp of splitPoints) {
+            if (cursor < sp.start)
+                pieces.push(...await this.packRuns({ start: cursor, end: sp.start - 1 }, packed, boundary, removed, ctx));
+            pieces.push(...await this.splitNode(sp, ctx));
+            cursor = sp.end + 1;
+        }
+        if (cursor <= range.end)
+            pieces.push(...await this.packRuns({ start: cursor, end: range.end }, packed, boundary, removed, ctx));
+        return pieces;
+    }
+    async packRuns(span, children, owner, removed, ctx) {
+        const pieces = [];
+        for (const run of DeclarationChunkStrategy.gaps(span, removed)) {
+            pieces.push(...await this.packer.pack(run, children, ctx.lines, ctx.chunkSize, ctx.characterSplit, owner));
+        }
+        return pieces;
+    }
+    static isShortPrologue(end, lines) {
+        if (end < 1)
+            return false;
+        for (let n = lines.line(1).startsWith('#!') ? 2 : 1; n <= end; n++) {
+            if (!lines.isBlank(n) && !lines.isCommentOnly(n) && !lines.isDirective(n))
+                return false;
+        }
+        return lines.slice(1, end).replace(/\s+/g, '').length < FragmentMerger_1.MIN_FRAGMENT_CHARS;
+    }
+    static gaps(span, taken) {
+        const out = [];
+        let cursor = span.start;
+        for (const t of [...taken].sort((a, b) => a.start - b.start)) {
+            if (t.end < cursor || t.start > span.end)
+                continue;
+            if (cursor < t.start)
+                out.push({ start: cursor, end: t.start - 1 });
+            cursor = Math.max(cursor, t.end + 1);
+        }
+        if (cursor <= span.end)
+            out.push({ start: cursor, end: span.end });
+        return out;
+    }
+}
+exports.DeclarationChunkStrategy = DeclarationChunkStrategy;

package/dist/src/chunking/FragmentMerger.d.ts

@@ -0,0 +1,11 @@
+import type { ChunkPiece } from '../interfaces/IChunker';
+import type { ChunkContext, IChunkPostProcessor } from './interfaces';
+export declare const MIN_FRAGMENT_CHARS = 60;
+export declare class FragmentMerger implements IChunkPostProcessor {
+    process(pieces: ChunkPiece[], ctx: ChunkContext): ChunkPiece[];
+    private isFragment;
+    private canMerge;
+    private static scope;
+    private static within;
+    private static merge;
+}

package/dist/src/chunking/FragmentMerger.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FragmentMerger = exports.MIN_FRAGMENT_CHARS = void 0;
+const owner_path_1 = require("../owner-path");
+exports.MIN_FRAGMENT_CHARS = 60;
+class FragmentMerger {
+    process(pieces, ctx) {
+        const queue = [...pieces].sort((a, b) => a.startLine - b.startLine);
+        const out = [];
+        for (let i = 0; i < queue.length; i++) {
+            const piece = queue[i];
+            if (!this.isFragment(piece, ctx)) {
+                out.push(piece);
+                continue;
+            }
+            const scope = FragmentMerger.scope(piece, ctx);
+            const prev = out[out.length - 1];
+            if (prev && FragmentMerger.within(prev, scope) && this.canMerge(prev, piece, ctx)) {
+                out[out.length - 1] = FragmentMerger.merge(prev, piece, ctx);
+                continue;
+            }
+            const next = queue[i + 1];
+            if (next && FragmentMerger.within(next, scope) && this.canMerge(piece, next, ctx)) {
+                queue[i + 1] = FragmentMerger.merge(piece, next, ctx);
+                continue;
+            }
+            out.push(piece);
+        }
+        return out;
+    }
+    isFragment(piece, ctx) {
+        return ctx.lines.significantLength(piece.startLine, piece.endLine) < exports.MIN_FRAGMENT_CHARS
+            && (0, owner_path_1.ownedLines)(piece, ctx.declarations, ctx.lines).length === 0;
+    }
+    canMerge(first, second, ctx) {
+        const start = Math.min(first.startLine, second.startLine);
+        const end = Math.max(first.endLine, second.endLine);
+        if (ctx.removed.some(r => r.start > first.endLine && r.end < second.startLine))
+            return false;
+        return ctx.lines.length(start, end) <= ctx.chunkSize;
+    }
+    static scope(piece, ctx) {
+        let best = null;
+        for (const sp of [...ctx.splitPoints, ...ctx.characterSplit]) {
+            if (sp.start > piece.startLine || sp.end < piece.endLine)
+                continue;
+            if (!best || sp.end - sp.start < best.end - best.start)
+                best = sp;
+        }
+        return best;
+    }
+    static within(piece, scope) {
+        return !scope || (scope.start <= piece.startLine && piece.endLine <= scope.end);
+    }
+    static merge(first, second, ctx) {
+        const startLine = Math.min(first.startLine, second.startLine);
+        const endLine = Math.max(first.endLine, second.endLine);
+        return { content: ctx.lines.slice(startLine, endLine), startLine, endLine };
+    }
+}
+exports.FragmentMerger = FragmentMerger;

package/dist/src/chunking/ImportOnlyFilter.d.ts

@@ -0,0 +1,5 @@
+import type { ChunkPiece } from '../interfaces/IChunker';
+import type { ChunkContext, IChunkPostProcessor } from './interfaces';
+export declare class ImportOnlyFilter implements IChunkPostProcessor {
+    process(pieces: ChunkPiece[], ctx: ChunkContext): ChunkPiece[];
+}

package/dist/src/chunking/ImportOnlyFilter.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImportOnlyFilter = void 0;
+class ImportOnlyFilter {
+    process(pieces, ctx) {
+        return pieces.filter((p) => {
+            if (!ctx.lines.isImportOnly(p.startLine, p.endLine))
+                return true;
+            ctx.removed.push({ start: p.startLine, end: p.endLine });
+            return false;
+        });
+    }
+}
+exports.ImportOnlyFilter = ImportOnlyFilter;

package/dist/src/chunking/LangchainTextSplitter.d.ts

@@ -0,0 +1,21 @@
+import type { Config } from '../config';
+import type { ChunkPiece } from '../interfaces/IChunker';
+import type { ITextSplitter, KeepWindow } from './interfaces';
+export interface SplitterOptions {
+    chunkSize: number;
+    chunkOverlap: number;
+    separators: string[];
+}
+export declare class LangchainTextSplitter implements ITextSplitter {
+    private readonly opts;
+    constructor(opts: SplitterOptions);
+    static forJs(config: Config): LangchainTextSplitter;
+    static forSchema(config: Config): LangchainTextSplitter;
+    split(text: string, firstLine: number, keep?: KeepWindow): Promise<ChunkPiece[]>;
+    private splitSpan;
+    private static splitOn;
+    private static joinKept;
+    private merge;
+    private static trim;
+    private static newlines;
+}

package/dist/src/chunking/LangchainTextSplitter.js

@@ -0,0 +1,133 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LangchainTextSplitter = void 0;
+const textsplitters_1 = require("@langchain/textsplitters");
+class LangchainTextSplitter {
+    opts;
+    constructor(opts) {
+        this.opts = opts;
+        if (opts.chunkOverlap >= opts.chunkSize)
+            throw new Error('Cannot have chunkOverlap >= chunkSize');
+    }
+    static forJs(config) {
+        return new LangchainTextSplitter({
+            chunkSize: config.indexer.chunkSize,
+            chunkOverlap: config.indexer.chunkOverlap,
+            separators: textsplitters_1.RecursiveCharacterTextSplitter.getSeparatorsForLanguage('js'),
+        });
+    }
+    static forSchema(config) {
+        return new LangchainTextSplitter({
+            chunkSize: config.indexer.chunkSize,
+            chunkOverlap: config.indexer.chunkOverlap,
+            separators: ['\n\n', '\n', ' '],
+        });
+    }
+    async split(text, firstLine, keep = { from: 0, to: 0 }) {
+        const pieces = [];
+        let offset = 0;
+        let line = firstLine;
+        for (const span of this.splitSpan(text, { start: 0, end: text.length }, this.opts.separators, keep)) {
+            line += LangchainTextSplitter.newlines(text, offset, span.start);
+            offset = span.start;
+            const content = text.slice(span.start, span.end);
+            pieces.push({ content, startLine: line, endLine: line + LangchainTextSplitter.newlines(content, 0, content.length) });
+        }
+        return pieces;
+    }
+    splitSpan(text, span, separators, keep) {
+        const sub = text.slice(span.start, span.end);
+        let separator = separators[separators.length - 1];
+        let next;
+        for (let i = 0; i < separators.length; i++) {
+            if (separators[i] === '') {
+                separator = '';
+                break;
+            }
+            if (sub.includes(separators[i])) {
+                separator = separators[i];
+                next = separators.slice(i + 1);
+                break;
+            }
+        }
+        const out = [];
+        let good = [];
+        const splits = LangchainTextSplitter.splitOn(sub, separator, span.start);
+        for (const s of separator ? LangchainTextSplitter.joinKept(splits, keep) : splits) {
+            if (s.end - s.start < this.opts.chunkSize) {
+                good.push(s);
+                continue;
+            }
+            if (good.length) {
+                out.push(...this.merge(text, good));
+                good = [];
+            }
+            out.push(...(next ? this.splitSpan(text, s, next, keep) : [s]));
+        }
+        if (good.length)
+            out.push(...this.merge(text, good));
+        return out;
+    }
+    static splitOn(sub, separator, base) {
+        const parts = separator
+            ? sub.split(new RegExp(`(?=${separator.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')})`))
+            : sub.split('');
+        const spans = [];
+        let at = base;
+        for (const p of parts) {
+            if (p !== '')
+                spans.push({ start: at, end: at + p.length });
+            at += p.length;
+        }
+        return spans;
+    }
+    static joinKept(splits, keep) {
+        const out = [];
+        for (const s of splits) {
+            if (out.length && s.start > keep.from && s.start <= keep.to)
+                out[out.length - 1] = { start: out[out.length - 1].start, end: s.end };
+            else
+                out.push(s);
+        }
+        return out;
+    }
+    merge(text, splits) {
+        const { chunkSize, chunkOverlap } = this.opts;
+        const docs = [];
+        const current = [];
+        let total = 0;
+        const flush = () => {
+            const doc = LangchainTextSplitter.trim(text, { start: current[0].start, end: current[current.length - 1].end });
+            if (doc)
+                docs.push(doc);
+        };
+        for (const d of splits) {
+            const len = d.end - d.start;
+            if (total + len > chunkSize && current.length > 0) {
+                flush();
+                while (total > chunkOverlap || (total + len > chunkSize && total > 0)) {
+                    total -= current[0].end - current[0].start;
+                    current.shift();
+                }
+            }
+            current.push(d);
+            total += len;
+        }
+        if (current.length > 0)
+            flush();
+        return docs;
+    }
+    static trim(text, span) {
+        let { start, end } = span;
+        while (start < end && /\s/.test(text[start]))
+            start++;
+        while (end > start && /\s/.test(text[end - 1]))
+            end--;
+        return start < end ? { start, end } : null;
+    }
+    static newlines(text, from, to) {
+        const count = (a, b) => (text.slice(a, b).match(/\n/g) ?? []).length;
+        return to >= from ? count(from, to) : -count(to, from);
+    }
+}
+exports.LangchainTextSplitter = LangchainTextSplitter;

package/dist/src/chunking/MinSizeSplitPolicy.d.ts

@@ -0,0 +1,9 @@
+import type { DeclarationNode } from '../declarations/DeclarationTree';
+import type { SourceLines } from '../declarations/SourceLines';
+import type { ISplitPolicy } from './interfaces';
+export declare const MIN_MEMBER_CHARS = 200;
+export declare class MinSizeSplitPolicy implements ISplitPolicy {
+    private readonly minChars;
+    constructor(minChars?: number);
+    isSplitPoint(child: DeclarationNode, siblings: DeclarationNode[], parent: DeclarationNode | null, lines: SourceLines): boolean;
+}

package/dist/src/chunking/MinSizeSplitPolicy.js

@@ -0,0 +1,18 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MinSizeSplitPolicy = exports.MIN_MEMBER_CHARS = void 0;
+exports.MIN_MEMBER_CHARS = 200;
+class MinSizeSplitPolicy {
+    minChars;
+    constructor(minChars = exports.MIN_MEMBER_CHARS) {
+        this.minChars = minChars;
+    }
+    isSplitPoint(child, siblings, parent, lines) {
+        if (lines.length(child.start, child.end) < this.minChars)
+            return false;
+        if (parent && (child.start <= parent.nameLine || child.end >= parent.end))
+            return false;
+        return !siblings.some(s => s !== child && s.start <= child.end && child.start <= s.end);
+    }
+}
+exports.MinSizeSplitPolicy = MinSizeSplitPolicy;

package/dist/src/chunking/RunPacker.d.ts

@@ -0,0 +1,12 @@
+import type { ChunkPiece } from '../interfaces/IChunker';
+import type { DeclarationNode } from '../declarations/DeclarationTree';
+import type { LineRange, SourceLines } from '../declarations/SourceLines';
+import type { ITextSplitter } from './interfaces';
+export declare class RunPacker {
+    private readonly splitter;
+    constructor(splitter: ITextSplitter);
+    pack(run: LineRange, children: DeclarationNode[], lines: SourceLines, chunkSize: number, characterSplit?: LineRange[], owner?: DeclarationNode | null): Promise<ChunkPiece[]>;
+    private static leadWindow;
+    static units(run: LineRange, children: DeclarationNode[]): LineRange[];
+    static piece(range: LineRange, lines: SourceLines): ChunkPiece[];
+}

package/dist/src/chunking/RunPacker.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunPacker = void 0;
+class RunPacker {
+    splitter;
+    constructor(splitter) {
+        this.splitter = splitter;
+    }
+    async pack(run, children, lines, chunkSize, characterSplit = [], owner = null) {
+        const pieces = [];
+        let current = null;
+        const flush = () => {
+            if (current)
+                pieces.push(...RunPacker.piece(current, lines));
+            current = null;
+        };
+        for (const unit of RunPacker.units(run, children)) {
+            if (lines.length(unit.start, unit.end) > chunkSize) {
+                flush();
+                characterSplit.push(unit);
+                const keep = RunPacker.leadWindow(unit, [owner, ...children], lines, chunkSize);
+                pieces.push(...await this.splitter.split(lines.slice(unit.start, unit.end), unit.start, keep));
+                continue;
+            }
+            if (current && lines.length(current.start, unit.end) <= chunkSize) {
+                current.end = unit.end;
+                continue;
+            }
+            flush();
+            current = { ...unit };
+        }
+        flush();
+        return pieces;
+    }
+    static leadWindow(unit, nodes, lines, chunkSize) {
+        const lead = nodes.find(n => n !== null && n.start === unit.start && n.nameLine > unit.start);
+        if (!lead)
+            return undefined;
+        const to = lines.length(unit.start, lead.nameLine - 1);
+        if (lines.length(unit.start, lead.nameLine) < chunkSize)
+            return { from: 0, to };
+        let first = unit.start;
+        while (first < lead.nameLine && (lines.isBlank(first) || lines.isCommentOnly(first)))
+            first++;
+        if (first === lead.nameLine || 1 + lines.length(first, lead.nameLine) >= chunkSize)
+            return undefined;
+        return { from: lines.length(unit.start, first - 1), to };
+    }
+    static units(run, children) {
+        const inside = children
+            .filter(c => c.end >= run.start && c.start <= run.end)
+            .sort((a, b) => a.start - b.start);
+        const units = [];
+        let cursor = run.start;
+        for (let i = 0; i < inside.length;) {
+            const start = Math.max(inside[i].start, run.start);
+            let end = Math.min(inside[i].end, run.end);
+            let j = i + 1;
+            while (j < inside.length && inside[j].start <= end) {
+                end = Math.min(Math.max(end, inside[j].end), run.end);
+                j++;
+            }
+            if (cursor < start)
+                units.push({ start: cursor, end: start - 1 });
+            units.push({ start, end });
+            cursor = end + 1;
+            i = j;
+        }
+        if (cursor <= run.end)
+            units.push({ start: cursor, end: run.end });
+        return units;
+    }
+    static piece(range, lines) {
+        let { start, end } = range;
+        while (start <= end && lines.isBlank(start))
+            start++;
+        while (end >= start && lines.isBlank(end))
+            end--;
+        if (start > end)
+            return [];
+        return [{ content: lines.slice(start, end), startLine: start, endLine: end }];
+    }
+}
+exports.RunPacker = RunPacker;

package/dist/src/chunking/create-chunker.d.ts

@@ -0,0 +1,3 @@
+import type { Config } from '../config';
+import { AstChunker } from './AstChunker';
+export declare function createChunker(config: Config): AstChunker;

package/dist/src/chunking/create-chunker.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createChunker = createChunker;
+const AstChunker_1 = require("./AstChunker");
+const CharacterChunkStrategy_1 = require("./CharacterChunkStrategy");
+const DeclarationChunkStrategy_1 = require("./DeclarationChunkStrategy");
+const FragmentMerger_1 = require("./FragmentMerger");
+const ImportOnlyFilter_1 = require("./ImportOnlyFilter");
+const LangchainTextSplitter_1 = require("./LangchainTextSplitter");
+const MinSizeSplitPolicy_1 = require("./MinSizeSplitPolicy");
+const RunPacker_1 = require("./RunPacker");
+function createChunker(config) {
+    const jsSplitter = LangchainTextSplitter_1.LangchainTextSplitter.forJs(config);
+    return new AstChunker_1.AstChunker(new DeclarationChunkStrategy_1.DeclarationChunkStrategy(new MinSizeSplitPolicy_1.MinSizeSplitPolicy(), new RunPacker_1.RunPacker(jsSplitter)), new CharacterChunkStrategy_1.CharacterChunkStrategy(jsSplitter, LangchainTextSplitter_1.LangchainTextSplitter.forSchema(config)), [new ImportOnlyFilter_1.ImportOnlyFilter()], [new FragmentMerger_1.FragmentMerger()], config);
+}

package/dist/src/chunking/interfaces.d.ts

@@ -0,0 +1,27 @@
+import type { ChunkOpts, ChunkPiece } from '../interfaces/IChunker';
+import type { DeclarationNode, DeclarationTree } from '../declarations/DeclarationTree';
+import type { LineRange, SourceLines } from '../declarations/SourceLines';
+export interface ChunkContext {
+    lines: SourceLines;
+    declarations: DeclarationTree;
+    chunkSize: number;
+    removed: LineRange[];
+    splitPoints: LineRange[];
+    characterSplit: LineRange[];
+}
+export interface IChunkStrategy {
+    chunk(ctx: ChunkContext, opts: ChunkOpts): Promise<ChunkPiece[]>;
+}
+export interface ISplitPolicy {
+    isSplitPoint(child: DeclarationNode, siblings: DeclarationNode[], parent: DeclarationNode | null, lines: SourceLines): boolean;
+}
+export interface KeepWindow {
+    from: number;
+    to: number;
+}
+export interface ITextSplitter {
+    split(text: string, firstLine: number, keep?: KeepWindow): Promise<ChunkPiece[]>;
+}
+export interface IChunkPostProcessor {
+    process(pieces: ChunkPiece[], ctx: ChunkContext): ChunkPiece[];
+}

package/dist/src/chunking/interfaces.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/src/commands/ConfigCommand.js

@@ -72,7 +72,6 @@ class ConfigCommand {
             console.log(`No config found at ${config_store_1.CONFIG_FILE}. Run rag-setup.`);
             return;
         }
-        // redact password for display
         const display = JSON.parse(JSON.stringify(cfg));
         if (display.db?.password !== undefined)
             display.db.password = '***';
@@ -131,24 +130,18 @@ class ConfigCommand {
         }
     }
     edit() {
-        // Ensure config + schema exist so the editor has something to open and
-        // resolve the schema against.
         try {
             if (!fs.existsSync(config_store_1.CONFIG_FILE)) {
                 (0, config_store_1.writeConfigJson)({});
             }
             else {
                 (0, config_store_1.writeSchemaFile)();
-                // Migrate existing configs: re-add the "$schema" reference if absent
-                // (rewrite through writeConfigJson) so editors bind to the schema.
-                // Skip silently if the file is currently invalid — the user will fix
-                // it in the editor and validation runs on save.
                 try {
                     const cfg = (0, config_store_1.readConfigJson)();
                     if (cfg)
                         (0, config_store_1.writeConfigJson)(cfg);
                 }
-                catch { /* invalid file — leave as-is for the user to edit */ }
+                catch { }
             }
         }
         catch (e) {
@@ -164,7 +157,6 @@ class ConfigCommand {
             console.error('Set $EDITOR or edit the file directly: ' + config_store_1.CONFIG_FILE);
             process.exit(1);
         }
-        // Validate what the user saved; readConfigJson throws with a helpful message.
         try {
             (0, config_store_1.readConfigJson)();
             console.log('Config OK');

package/dist/src/commands/EvalCommand.d.ts

@@ -0,0 +1,11 @@
+import type { Container } from '../container';
+import type { ICommand } from './ICommand';
+import type { ICorpusInspector } from '../interfaces/ICorpusInspector';
+export declare class EvalCommand implements ICommand {
+    private container;
+    private argv1;
+    private corpusInspector;
+    private resultsDir;
+    constructor(container: Container, argv1: string, corpusInspector?: ICorpusInspector, resultsDir?: string);
+    execute(): Promise<void>;
+}