@duytransipher/gitnexus 1.4.6-sipher.0

package/dist/core/search/hybrid-search.js

@@ -0,0 +1,118 @@
+/**
+ * Hybrid Search with Reciprocal Rank Fusion (RRF)
+ *
+ * Combines BM25 (keyword) and semantic (embedding) search results.
+ * Uses RRF to merge rankings without needing score normalization.
+ *
+ * This is the same approach used by Elasticsearch, Pinecone, and other
+ * production search systems.
+ */
+import { searchFTSFromLbug } from './bm25-index.js';
+/**
+ * RRF constant - standard value used in the literature
+ * Higher values give more weight to lower-ranked results
+ */
+const RRF_K = 60;
+/**
+ * Perform hybrid search combining BM25 and semantic results
+ *
+ * @param bm25Results - Results from BM25 keyword search
+ * @param semanticResults - Results from semantic/embedding search
+ * @param limit - Maximum results to return
+ * @returns Merged and re-ranked results
+ */
+export const mergeWithRRF = (bm25Results, semanticResults, limit = 10) => {
+    const merged = new Map();
+    // Process BM25 results
+    for (let i = 0; i < bm25Results.length; i++) {
+        const r = bm25Results[i];
+        const rrfScore = 1 / (RRF_K + i + 1); // i+1 because rank starts at 1
+        merged.set(r.filePath, {
+            filePath: r.filePath,
+            score: rrfScore,
+            rank: 0, // Will be set after sorting
+            sources: ['bm25'],
+            bm25Score: r.score,
+        });
+    }
+    // Process semantic results and merge
+    for (let i = 0; i < semanticResults.length; i++) {
+        const r = semanticResults[i];
+        const rrfScore = 1 / (RRF_K + i + 1);
+        const existing = merged.get(r.filePath);
+        if (existing) {
+            // Found by both methods - add scores
+            existing.score += rrfScore;
+            existing.sources.push('semantic');
+            existing.semanticScore = 1 - r.distance;
+            // Add semantic metadata
+            existing.nodeId = r.nodeId;
+            existing.name = r.name;
+            existing.label = r.label;
+            existing.startLine = r.startLine;
+            existing.endLine = r.endLine;
+        }
+        else {
+            // Only found by semantic
+            merged.set(r.filePath, {
+                filePath: r.filePath,
+                score: rrfScore,
+                rank: 0,
+                sources: ['semantic'],
+                semanticScore: 1 - r.distance,
+                nodeId: r.nodeId,
+                name: r.name,
+                label: r.label,
+                startLine: r.startLine,
+                endLine: r.endLine,
+            });
+        }
+    }
+    // Sort by RRF score descending
+    const sorted = Array.from(merged.values())
+        .sort((a, b) => b.score - a.score)
+        .slice(0, limit);
+    // Assign final ranks
+    sorted.forEach((r, i) => {
+        r.rank = i + 1;
+    });
+    return sorted;
+};
+/**
+ * Check if hybrid search is available
+ * LadybugDB FTS is always available once the database is initialized.
+ * Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
+ */
+export const isHybridSearchReady = () => {
+    return true; // FTS is always available via LadybugDB when DB is open
+};
+/**
+ * Format hybrid results for LLM consumption
+ */
+export const formatHybridResults = (results) => {
+    if (results.length === 0) {
+        return 'No results found.';
+    }
+    const formatted = results.map((r, i) => {
+        const sources = r.sources.join(' + ');
+        const location = r.startLine ? ` (lines ${r.startLine}-${r.endLine})` : '';
+        const label = r.label ? `${r.label}: ` : 'File: ';
+        const name = r.name || r.filePath.split('/').pop() || r.filePath;
+        return `[${i + 1}] ${label}${name}
+    File: ${r.filePath}${location}
+    Found by: ${sources}
+    Relevance: ${r.score.toFixed(4)}`;
+    });
+    return `Found ${results.length} results:\n\n${formatted.join('\n\n')}`;
+};
+/**
+ * Execute BM25 + semantic search and merge with RRF.
+ * Uses LadybugDB FTS for always-fresh BM25 results (no cached data).
+ * The semanticSearch function is injected to keep this module environment-agnostic.
+ */
+export const hybridSearch = async (query, limit, executeQuery, semanticSearch) => {
+    // Use LadybugDB FTS for always-fresh BM25 results
+    const bm25Results = await searchFTSFromLbug(query, limit);
+    const semanticResults = await semanticSearch(executeQuery, query, limit);
+    return mergeWithRRF(bm25Results, semanticResults, limit);
+};

package/dist/core/tree-sitter/parser-loader.d.ts

@@ -0,0 +1,5 @@
+import Parser from 'tree-sitter';
+import { SupportedLanguages } from '../../config/supported-languages.js';
+export declare const isLanguageAvailable: (language: SupportedLanguages) => boolean;
+export declare const loadParser: () => Promise<Parser>;
+export declare const loadLanguage: (language: SupportedLanguages, filePath?: string) => Promise<void>;

package/dist/core/tree-sitter/parser-loader.js

@@ -0,0 +1,63 @@
+import Parser from 'tree-sitter';
+import JavaScript from 'tree-sitter-javascript';
+import TypeScript from 'tree-sitter-typescript';
+import Python from 'tree-sitter-python';
+import Java from 'tree-sitter-java';
+import C from 'tree-sitter-c';
+import CPP from 'tree-sitter-cpp';
+import CSharp from 'tree-sitter-c-sharp';
+import Go from 'tree-sitter-go';
+import Rust from 'tree-sitter-rust';
+import PHP from 'tree-sitter-php';
+import Ruby from 'tree-sitter-ruby';
+import { createRequire } from 'node:module';
+import { SupportedLanguages } from '../../config/supported-languages.js';
+// tree-sitter-swift is an optionalDependency — may not be installed
+const _require = createRequire(import.meta.url);
+let Swift = null;
+try {
+    Swift = _require('tree-sitter-swift');
+}
+catch { }
+// tree-sitter-kotlin is an optionalDependency — may not be installed
+let Kotlin = null;
+try {
+    Kotlin = _require('tree-sitter-kotlin');
+}
+catch { }
+let parser = null;
+const languageMap = {
+    [SupportedLanguages.JavaScript]: JavaScript,
+    [SupportedLanguages.TypeScript]: TypeScript.typescript,
+    [`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx,
+    [SupportedLanguages.Python]: Python,
+    [SupportedLanguages.Java]: Java,
+    [SupportedLanguages.C]: C,
+    [SupportedLanguages.CPlusPlus]: CPP,
+    [SupportedLanguages.CSharp]: CSharp,
+    [SupportedLanguages.Go]: Go,
+    [SupportedLanguages.Rust]: Rust,
+    ...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
+    [SupportedLanguages.PHP]: PHP.php_only,
+    [SupportedLanguages.Ruby]: Ruby,
+    ...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
+};
+export const isLanguageAvailable = (language) => language in languageMap;
+export const loadParser = async () => {
+    if (parser)
+        return parser;
+    parser = new Parser();
+    return parser;
+};
+export const loadLanguage = async (language, filePath) => {
+    if (!parser)
+        await loadParser();
+    const key = language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
+        ? `${language}:tsx`
+        : language;
+    const lang = languageMap[key];
+    if (!lang) {
+        throw new Error(`Unsupported language: ${language}`);
+    }
+    parser.setLanguage(lang);
+};

package/dist/core/wiki/generator.d.ts

@@ -0,0 +1,120 @@
+/**
+ * Wiki Generator
+ *
+ * Orchestrates the full wiki generation pipeline:
+ *   Phase 0: Validate prerequisites + gather graph structure
+ *   Phase 1: Build module tree (one LLM call)
+ *   Phase 2: Generate module pages (one LLM call per module, bottom-up)
+ *   Phase 3: Generate overview page
+ *
+ * Supports incremental updates via git diff + module-file mapping.
+ */
+import { type FileWithExports } from './graph-queries.js';
+import { type LLMConfig } from './llm-client.js';
+export interface WikiOptions {
+    force?: boolean;
+    model?: string;
+    baseUrl?: string;
+    apiKey?: string;
+    maxTokensPerModule?: number;
+    concurrency?: number;
+}
+export interface WikiMeta {
+    fromCommit: string;
+    generatedAt: string;
+    model: string;
+    moduleFiles: Record<string, string[]>;
+    moduleTree: ModuleTreeNode[];
+}
+export interface ModuleTreeNode {
+    name: string;
+    slug: string;
+    files: string[];
+    children?: ModuleTreeNode[];
+}
+export type ProgressCallback = (phase: string, percent: number, detail?: string) => void;
+export declare const MAX_WIKI_PROMPT_TOKENS = 120000;
+export declare function isWikiPromptExcludedPath(filePath: string): boolean;
+export declare function isWikiCodeFilePath(filePath: string): boolean;
+export declare function filterWikiPromptPaths(filePaths: string[]): string[];
+export declare function estimateGroupingPromptTokens(files: FileWithExports[]): number;
+export declare function shouldUseDeterministicGrouping(fileCount: number, promptTokens: number): boolean;
+export declare class WikiGenerator {
+    private repoPath;
+    private storagePath;
+    private wikiDir;
+    private lbugPath;
+    private llmConfig;
+    private maxTokensPerModule;
+    private concurrency;
+    private options;
+    private onProgress;
+    private failedModules;
+    constructor(repoPath: string, storagePath: string, lbugPath: string, llmConfig: LLMConfig, options?: WikiOptions, onProgress?: ProgressCallback);
+    private lastPercent;
+    /**
+     * Create streaming options that report LLM progress to the progress bar.
+     * Uses the last known percent so streaming doesn't reset the bar backwards.
+     */
+    private streamOpts;
+    /**
+     * Main entry point. Runs the full pipeline or incremental update.
+     */
+    run(): Promise<{
+        pagesGenerated: number;
+        mode: 'full' | 'incremental' | 'up-to-date';
+        failedModules: string[];
+    }>;
+    private ensureHTMLViewer;
+    private fullGeneration;
+    private buildModuleTree;
+    /**
+     * Parse LLM grouping response. Validates all files are assigned.
+     */
+    private parseGroupingResponse;
+    /**
+     * Fallback grouping by top-level directory when LLM parsing fails.
+     */
+    private fallbackGrouping;
+    /**
+     * Split a large module into sub-modules by subdirectory.
+     */
+    private splitBySubdirectory;
+    /**
+     * Generate a leaf module page from source code + graph data.
+     */
+    private generateLeafPage;
+    /**
+     * Generate a parent module page from children's documentation.
+     */
+    private generateParentPage;
+    private generateOverview;
+    private incrementalUpdate;
+    private getCurrentCommit;
+    private getChangedFiles;
+    private readSourceFiles;
+    private truncateSource;
+    private estimateModuleTokens;
+    private readProjectInfo;
+    private assertPromptWithinBudget;
+    private buildDeterministicModuleTree;
+    private chunkFiles;
+    private extractModuleFiles;
+    private countModules;
+    /**
+     * Flatten the module tree into leaf nodes and parent nodes.
+     * Leaves can be processed in parallel; parents must wait for children.
+     */
+    private flattenModuleTree;
+    /**
+     * Run async tasks in parallel with a concurrency limit and adaptive rate limiting.
+     * If a 429 rate limit is hit, concurrency is temporarily reduced.
+     */
+    private runParallel;
+    private findNodeBySlug;
+    private slugify;
+    private fileExists;
+    private loadWikiMeta;
+    private saveWikiMeta;
+    private saveModuleTree;
+}