arceus-s 1.6.4

package/dist/core/search/bm25-index.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Full-Text Search via LadybugDB FTS
+ *
+ * Uses LadybugDB's built-in full-text search indexes for keyword-based search.
+ * Always reads from the database (no cached state to drift).
+ */
+export interface BM25SearchResult {
+    filePath: string;
+    score: number;
+    rank: number;
+    nodeIds?: string[];
+}
+export interface FTSSearchResponse {
+    results: BM25SearchResult[];
+    /** True when at least one FTS index query succeeded (index exists). */
+    ftsAvailable: boolean;
+}
+/**
+ * Search using LadybugDB's built-in FTS (always fresh, reads from disk)
+ *
+ * Queries multiple node tables (File, Function, Class, Method) in parallel
+ * and merges results by filePath, summing scores for the same file.
+ *
+ * @param query - Search query string
+ * @param limit - Maximum results
+ * @param repoId - If provided, queries will be routed via the MCP connection pool
+ * @returns Ranked search results from FTS indexes
+ */
+export declare const searchFTSFromLbug: (query: string, limit?: number, repoId?: string) => Promise<FTSSearchResponse>;

package/dist/core/search/bm25-index.js

@@ -0,0 +1,118 @@
+/**
+ * Full-Text Search via LadybugDB FTS
+ *
+ * Uses LadybugDB's built-in full-text search indexes for keyword-based search.
+ * Always reads from the database (no cached state to drift).
+ */
+import { queryFTS } from '../lbug/lbug-adapter.js';
+import { FTS_INDEXES } from './fts-schema.js';
+/**
+ * Execute a single FTS query via a custom executor (for MCP connection pool).
+ * Returns `null` when the query fails (e.g. FTS index does not exist) so the
+ * caller can distinguish "zero matches" from "index missing".
+ */
+async function queryFTSViaExecutor(executor, tableName, indexName, query, limit) {
+    // Escape single quotes and backslashes to prevent Cypher injection
+    const escapedQuery = query.replace(/\\/g, '\\\\').replace(/'/g, "''");
+    const cypher = `
+    CALL QUERY_FTS_INDEX('${tableName}', '${indexName}', '${escapedQuery}', conjunctive := false)
+    RETURN node, score
+    ORDER BY score DESC
+    LIMIT ${limit}
+  `;
+    try {
+        const rows = await executor(cypher);
+        return rows.map((row) => {
+            const node = row.node || row[0] || {};
+            const score = row.score ?? row[1] ?? 0;
+            return {
+                filePath: node.filePath || '',
+                score: typeof score === 'number' ? score : parseFloat(score) || 0,
+                nodeId: node.nodeId || node.id || '',
+            };
+        });
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Search using LadybugDB's built-in FTS (always fresh, reads from disk)
+ *
+ * Queries multiple node tables (File, Function, Class, Method) in parallel
+ * and merges results by filePath, summing scores for the same file.
+ *
+ * @param query - Search query string
+ * @param limit - Maximum results
+ * @param repoId - If provided, queries will be routed via the MCP connection pool
+ * @returns Ranked search results from FTS indexes
+ */
+export const searchFTSFromLbug = async (query, limit = 20, repoId) => {
+    const resultsByIndex = [];
+    let queriesSucceeded = 0;
+    if (repoId) {
+        // Use MCP connection pool via dynamic import
+        // IMPORTANT: FTS queries run sequentially to avoid connection contention.
+        // The MCP pool supports multiple connections, but FTS is best run serially.
+        const poolMod = await import('../lbug/pool-adapter.js');
+        const { executeQuery } = poolMod;
+        const executor = (cypher) => executeQuery(repoId, cypher);
+        for (const { table, indexName } of FTS_INDEXES) {
+            const result = await queryFTSViaExecutor(executor, table, indexName, query, limit);
+            if (result !== null) {
+                queriesSucceeded++;
+                resultsByIndex.push(result);
+            }
+        }
+    }
+    else {
+        // Use core lbug adapter (CLI / pipeline context) — also sequential for safety.
+        for (const { table, indexName } of FTS_INDEXES) {
+            try {
+                const result = await queryFTS(table, indexName, query, limit, false);
+                queriesSucceeded++;
+                resultsByIndex.push(result);
+            }
+            catch {
+                // FTS index may not exist — count as failed
+            }
+        }
+    }
+    const ftsAvailable = queriesSucceeded > 0;
+    // Collect all node scores per filePath to track which nodes actually matched
+    const fileNodeScores = new Map();
+    const addResults = (results) => {
+        for (const r of results) {
+            if (!fileNodeScores.has(r.filePath))
+                fileNodeScores.set(r.filePath, []);
+            fileNodeScores.get(r.filePath).push({ score: r.score, nodeId: r.nodeId });
+        }
+    };
+    for (const results of resultsByIndex)
+        addResults(results);
+    // Sum the top-3 highest-scoring nodes per file and collect their nodeIds.
+    // Summing all nodes naively inflates scores for files with many mediocre
+    // matches (e.g. test files) over files with a single highly-relevant symbol.
+    const merged = new Map();
+    for (const [filePath, entries] of fileNodeScores) {
+        const top3 = [...entries].sort((a, b) => b.score - a.score).slice(0, 3);
+        merged.set(filePath, {
+            filePath,
+            score: top3.reduce((acc, e) => acc + e.score, 0),
+            nodeIds: top3.map((e) => e.nodeId).filter((id) => id),
+        });
+    }
+    // Sort by score descending and add rank
+    const sorted = Array.from(merged.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    return {
+        results: sorted.map((r, index) => ({
+            filePath: r.filePath,
+            score: r.score,
+            rank: index + 1,
+            nodeIds: r.nodeIds,
+        })),
+        ftsAvailable,
+    };
+};

package/dist/core/search/fts-indexes.d.ts

@@ -0,0 +1 @@
+export declare function createSearchFTSIndexes(): Promise<void>;

package/dist/core/search/fts-indexes.js

@@ -0,0 +1,7 @@
+import { createFTSIndex } from '../lbug/lbug-adapter.js';
+import { FTS_INDEXES } from './fts-schema.js';
+export async function createSearchFTSIndexes() {
+    for (const { table, indexName, properties } of FTS_INDEXES) {
+        await createFTSIndex(table, indexName, [...properties]);
+    }
+}

package/dist/core/search/fts-schema.d.ts

@@ -0,0 +1,6 @@
+export interface FTSIndexDefinition {
+    readonly table: string;
+    readonly indexName: string;
+    readonly properties: readonly string[];
+}
+export declare const FTS_INDEXES: readonly FTSIndexDefinition[];

package/dist/core/search/fts-schema.js

@@ -0,0 +1,7 @@
+export const FTS_INDEXES = [
+    { table: 'File', indexName: 'file_fts', properties: ['name', 'content'] },
+    { table: 'Function', indexName: 'function_fts', properties: ['name', 'content'] },
+    { table: 'Class', indexName: 'class_fts', properties: ['name', 'content'] },
+    { table: 'Method', indexName: 'method_fts', properties: ['name', 'content'] },
+    { table: 'Interface', indexName: 'interface_fts', properties: ['name', 'content'] },
+];

package/dist/core/search/hybrid-search.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Hybrid Search with Reciprocal Rank Fusion (RRF)
+ *
+ * Combines BM25 (keyword) and semantic (embedding) search results.
+ * Uses RRF to merge rankings without needing score normalization.
+ *
+ * This is the same approach used by Elasticsearch, Pinecone, and other
+ * production search systems.
+ */
+import { type BM25SearchResult } from './bm25-index.js';
+import type { SemanticSearchResult } from '../embeddings/types.js';
+export interface HybridSearchResult {
+    filePath: string;
+    score: number;
+    rank: number;
+    sources: ('bm25' | 'semantic')[];
+    nodeId?: string;
+    name?: string;
+    label?: string;
+    startLine?: number;
+    endLine?: number;
+    bm25Score?: number;
+    semanticScore?: number;
+}
+/**
+ * Perform hybrid search combining BM25 and semantic results
+ *
+ * @param bm25Results - Results from BM25 keyword search
+ * @param semanticResults - Results from semantic/embedding search
+ * @param limit - Maximum results to return
+ * @returns Merged and re-ranked results
+ */
+export declare const mergeWithRRF: (bm25Results: BM25SearchResult[], semanticResults: SemanticSearchResult[], limit?: number) => HybridSearchResult[];
+/**
+ * Check if hybrid search is available.
+ * FTS indexes may be missing on read-only MCP connections (see #1403);
+ * callers should inspect `ftsAvailable` from searchFTSFromLbug for
+ * per-query availability. This helper is a coarse gate only.
+ */
+export declare const isHybridSearchReady: () => boolean;
+/**
+ * Format hybrid results for LLM consumption
+ */
+export declare const formatHybridResults: (results: HybridSearchResult[]) => string;
+/**
+ * Execute BM25 + semantic search and merge with RRF.
+ * Uses LadybugDB FTS for always-fresh BM25 results (no cached data).
+ * The semanticSearch function is injected to keep this module environment-agnostic.
+ *
+ * When FTS is unavailable (e.g. read-only MCP connection, missing indexes),
+ * falls back to semantic-only results instead of crashing (#1489).
+ */
+export declare const hybridSearch: (query: string, limit: number, executeQuery: (cypher: string) => Promise<any[]>, semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]>) => Promise<HybridSearchResult[]>;

package/dist/core/search/hybrid-search.js

@@ -0,0 +1,136 @@
+/**
+ * Hybrid Search with Reciprocal Rank Fusion (RRF)
+ *
+ * Combines BM25 (keyword) and semantic (embedding) search results.
+ * Uses RRF to merge rankings without needing score normalization.
+ *
+ * This is the same approach used by Elasticsearch, Pinecone, and other
+ * production search systems.
+ */
+import { searchFTSFromLbug } from './bm25-index.js';
+/**
+ * RRF constant - standard value used in the literature
+ * Higher values give more weight to lower-ranked results
+ */
+const RRF_K = 60;
+/**
+ * Perform hybrid search combining BM25 and semantic results
+ *
+ * @param bm25Results - Results from BM25 keyword search
+ * @param semanticResults - Results from semantic/embedding search
+ * @param limit - Maximum results to return
+ * @returns Merged and re-ranked results
+ */
+export const mergeWithRRF = (bm25Results, semanticResults, limit = 10) => {
+    const merged = new Map();
+    // Guard against undefined/null inputs (#1489) — when FTS is unavailable
+    // in the MCP process, bm25Results can arrive as undefined and the
+    // for-loop would throw "bm25Results is not iterable".
+    const safeBm25 = bm25Results ?? [];
+    const safeSemantic = semanticResults ?? [];
+    // Process BM25 results
+    for (let i = 0; i < safeBm25.length; i++) {
+        const r = safeBm25[i];
+        const rrfScore = 1 / (RRF_K + i + 1); // i+1 because rank starts at 1
+        merged.set(r.filePath, {
+            filePath: r.filePath,
+            score: rrfScore,
+            rank: 0, // Will be set after sorting
+            sources: ['bm25'],
+            bm25Score: r.score,
+        });
+    }
+    // Process semantic results and merge
+    for (let i = 0; i < safeSemantic.length; i++) {
+        const r = safeSemantic[i];
+        const rrfScore = 1 / (RRF_K + i + 1);
+        const existing = merged.get(r.filePath);
+        if (existing) {
+            // Found by both methods - add scores
+            existing.score += rrfScore;
+            existing.sources.push('semantic');
+            existing.semanticScore = 1 - r.distance;
+            // Add semantic metadata
+            existing.nodeId = r.nodeId;
+            existing.name = r.name;
+            existing.label = r.label;
+            existing.startLine = r.startLine;
+            existing.endLine = r.endLine;
+        }
+        else {
+            // Only found by semantic
+            merged.set(r.filePath, {
+                filePath: r.filePath,
+                score: rrfScore,
+                rank: 0,
+                sources: ['semantic'],
+                semanticScore: 1 - r.distance,
+                nodeId: r.nodeId,
+                name: r.name,
+                label: r.label,
+                startLine: r.startLine,
+                endLine: r.endLine,
+            });
+        }
+    }
+    // Sort by RRF score descending
+    const sorted = Array.from(merged.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    // Assign final ranks
+    sorted.forEach((r, i) => {
+        r.rank = i + 1;
+    });
+    return sorted;
+};
+/**
+ * Check if hybrid search is available.
+ * FTS indexes may be missing on read-only MCP connections (see #1403);
+ * callers should inspect `ftsAvailable` from searchFTSFromLbug for
+ * per-query availability. This helper is a coarse gate only.
+ */
+export const isHybridSearchReady = () => {
+    return true; // FTS is attempted on every query; ftsAvailable signals actual availability
+};
+/**
+ * Format hybrid results for LLM consumption
+ */
+export const formatHybridResults = (results) => {
+    if (results.length === 0) {
+        return 'No results found.';
+    }
+    const formatted = results.map((r, i) => {
+        const sources = r.sources.join(' + ');
+        const location = r.startLine ? ` (lines ${r.startLine}-${r.endLine})` : '';
+        const label = r.label ? `${r.label}: ` : 'File: ';
+        const name = r.name || r.filePath.split('/').pop() || r.filePath;
+        return `[${i + 1}] ${label}${name}
+    File: ${r.filePath}${location}
+    Found by: ${sources}
+    Relevance: ${r.score.toFixed(4)}`;
+    });
+    return `Found ${results.length} results:\n\n${formatted.join('\n\n')}`;
+};
+/**
+ * Execute BM25 + semantic search and merge with RRF.
+ * Uses LadybugDB FTS for always-fresh BM25 results (no cached data).
+ * The semanticSearch function is injected to keep this module environment-agnostic.
+ *
+ * When FTS is unavailable (e.g. read-only MCP connection, missing indexes),
+ * falls back to semantic-only results instead of crashing (#1489).
+ */
+export const hybridSearch = async (query, limit, executeQuery, semanticSearch) => {
+    // Use LadybugDB FTS for always-fresh BM25 results.
+    // If FTS fails (e.g. extension not loaded in MCP process), fall back to
+    // semantic-only search instead of crashing with "bm25Results is not iterable".
+    let bm25Results = [];
+    try {
+        const ftsResponse = await searchFTSFromLbug(query, limit);
+        bm25Results = ftsResponse?.results ?? [];
+    }
+    catch {
+        // FTS unavailable — continue with semantic-only search
+    }
+    const semanticResults = await semanticSearch(executeQuery, query, limit);
+    return mergeWithRRF(bm25Results, semanticResults, limit);
+};

package/dist/core/search/phase-timer.d.ts

@@ -0,0 +1,72 @@
+/**
+ * Per-phase wall-clock timing for the search pipeline and similar
+ * multi-stage flows. Designed to be called from query() with minimal
+ * ceremony and negligible overhead (< 0.1 ms per phase recorded).
+ *
+ * ### Sequential usage
+ *
+ * ```ts
+ * const t = new PhaseTimer();
+ * t.start('bm25'); await bm25Search(...); t.stop();
+ * t.start('merge'); doMerge(); t.stop();
+ * const phases = t.summary(); // { bm25: 42, merge: 3 }
+ * ```
+ *
+ * ### Concurrent usage (Promise.all)
+ *
+ * `start`/`stop` assume a single active phase at a time, which is wrong
+ * for concurrent work inside `Promise.all` — the second `start` would
+ * auto-stop the first and only one of the two would get timed. Use
+ * {@link PhaseTimer.time} to wrap each concurrent promise instead:
+ *
+ * ```ts
+ * const [a, b] = await Promise.all([
+ *   t.time('bm25', bm25Search(...)),
+ *   t.time('vector', semanticSearch(...)),
+ * ]);
+ * ```
+ *
+ * ### Pre-measured durations
+ *
+ * ```ts
+ * t.mark('inherited', 12.5);
+ * ```
+ */
+export declare class PhaseTimer {
+    private phases;
+    private current;
+    private t0;
+    /** Start a new phase. Implicitly stops the previous one, if any. */
+    start(phase: string): void;
+    /** Stop the current phase. No-op if no phase is active. */
+    stop(): void;
+    /**
+     * Record a pre-measured duration without touching the active phase.
+     * Use for concurrent operations inside `Promise.all` where
+     * `start`/`stop` would step on each other, or for durations imported
+     * from sub-systems. Additive across repeated calls with the same
+     * phase name. Ignores negative / non-finite inputs.
+     */
+    mark(phase: string, durationMs: number): void;
+    /**
+     * Wrap a promise with automatic timing. Records wall time via
+     * {@link PhaseTimer.mark} regardless of which other phases are
+     * active — safe to use inside `Promise.all`.
+     */
+    time<T>(phase: string, promise: Promise<T>): Promise<T>;
+    /**
+     * Snapshot of accumulated durations rounded to 0.1 ms. Stops the
+     * current phase if one is still running.
+     */
+    summary(): Record<string, number>;
+    /**
+     * Sum of every recorded phase duration.
+     *
+     * Note: for phases recorded via {@link PhaseTimer.time} or
+     * {@link PhaseTimer.mark} this is the *sum*, not the wall time —
+     * concurrent work overlaps and the sum can exceed the end-to-end
+     * wall time. Record wall time separately with `mark('wall', …)` if
+     * that distinction matters.
+     */
+    totalMs(): number;
+}

package/dist/core/search/phase-timer.js

@@ -0,0 +1,106 @@
+/**
+ * Per-phase wall-clock timing for the search pipeline and similar
+ * multi-stage flows. Designed to be called from query() with minimal
+ * ceremony and negligible overhead (< 0.1 ms per phase recorded).
+ *
+ * ### Sequential usage
+ *
+ * ```ts
+ * const t = new PhaseTimer();
+ * t.start('bm25'); await bm25Search(...); t.stop();
+ * t.start('merge'); doMerge(); t.stop();
+ * const phases = t.summary(); // { bm25: 42, merge: 3 }
+ * ```
+ *
+ * ### Concurrent usage (Promise.all)
+ *
+ * `start`/`stop` assume a single active phase at a time, which is wrong
+ * for concurrent work inside `Promise.all` — the second `start` would
+ * auto-stop the first and only one of the two would get timed. Use
+ * {@link PhaseTimer.time} to wrap each concurrent promise instead:
+ *
+ * ```ts
+ * const [a, b] = await Promise.all([
+ *   t.time('bm25', bm25Search(...)),
+ *   t.time('vector', semanticSearch(...)),
+ * ]);
+ * ```
+ *
+ * ### Pre-measured durations
+ *
+ * ```ts
+ * t.mark('inherited', 12.5);
+ * ```
+ */
+export class PhaseTimer {
+    phases = new Map();
+    current = null;
+    t0 = 0;
+    /** Start a new phase. Implicitly stops the previous one, if any. */
+    start(phase) {
+        this.stop();
+        this.current = phase;
+        this.t0 = performance.now();
+    }
+    /** Stop the current phase. No-op if no phase is active. */
+    stop() {
+        if (this.current !== null) {
+            const elapsed = performance.now() - this.t0;
+            this.phases.set(this.current, (this.phases.get(this.current) ?? 0) + elapsed);
+            this.current = null;
+        }
+    }
+    /**
+     * Record a pre-measured duration without touching the active phase.
+     * Use for concurrent operations inside `Promise.all` where
+     * `start`/`stop` would step on each other, or for durations imported
+     * from sub-systems. Additive across repeated calls with the same
+     * phase name. Ignores negative / non-finite inputs.
+     */
+    mark(phase, durationMs) {
+        if (!Number.isFinite(durationMs) || durationMs < 0)
+            return;
+        this.phases.set(phase, (this.phases.get(phase) ?? 0) + durationMs);
+    }
+    /**
+     * Wrap a promise with automatic timing. Records wall time via
+     * {@link PhaseTimer.mark} regardless of which other phases are
+     * active — safe to use inside `Promise.all`.
+     */
+    async time(phase, promise) {
+        const t0 = performance.now();
+        try {
+            return await promise;
+        }
+        finally {
+            this.mark(phase, performance.now() - t0);
+        }
+    }
+    /**
+     * Snapshot of accumulated durations rounded to 0.1 ms. Stops the
+     * current phase if one is still running.
+     */
+    summary() {
+        this.stop();
+        const out = {};
+        for (const [k, v] of this.phases)
+            out[k] = Math.round(v * 10) / 10;
+        return out;
+    }
+    /**
+     * Sum of every recorded phase duration.
+     *
+     * Note: for phases recorded via {@link PhaseTimer.time} or
+     * {@link PhaseTimer.mark} this is the *sum*, not the wall time —
+     * concurrent work overlaps and the sum can exceed the end-to-end
+     * wall time. Record wall time separately with `mark('wall', …)` if
+     * that distinction matters.
+     */
+    totalMs() {
+        this.stop();
+        let t = 0;
+        for (const v of this.phases.values())
+            t += v;
+        return Math.round(t * 10) / 10;
+    }
+}

package/dist/core/tree-sitter/parser-loader.d.ts

@@ -0,0 +1,8 @@
+import Parser from 'tree-sitter';
+import { SupportedLanguages } from '../../_shared/index.js';
+export declare const resolveLanguageKey: (language: SupportedLanguages, filePath?: string) => string;
+export declare const isLanguageAvailable: (language: SupportedLanguages, filePath?: string) => boolean;
+export declare const getLanguageGrammar: (language: SupportedLanguages, filePath?: string) => unknown;
+export declare const loadParser: () => Promise<Parser>;
+export declare const loadLanguage: (language: SupportedLanguages, filePath?: string) => Promise<void>;
+export declare const createParserForLanguage: (language: SupportedLanguages, filePath?: string) => Promise<Parser>;