gitmem-mcp 1.4.4 → 1.6.0

package/dist/services/doc-chunker.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Document Chunker — Split markdown files into searchable chunks
+ *
+ * Strategy:
+ * 1. Split on H2 headers first (natural semantic boundaries)
+ * 2. If a section exceeds target size, split on paragraph boundaries
+ * 3. Each chunk carries metadata: file path, title, category, chunk index
+ *
+ * Target chunk size: 500-800 tokens (~2000-3200 chars)
+ */
+export interface DocChunk {
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    file_hash: string;
+}
+export interface DocFile {
+    absolute_path: string;
+    relative_path: string;
+    content: string;
+    hash: string;
+}
+/**
+ * Chunk a single markdown file into searchable segments
+ */
+export declare function chunkDocument(doc: DocFile): DocChunk[];
+/**
+ * Scan a directory for markdown files
+ */
+export declare function scanDirectory(dirPath: string, options?: {
+    exclude?: string[];
+}): DocFile[];
+/**
+ * Chunk all markdown files in a directory
+ */
+export declare function chunkDirectory(dirPath: string, options?: {
+    exclude?: string[];
+}): {
+    files: DocFile[];
+    chunks: DocChunk[];
+};
+//# sourceMappingURL=doc-chunker.d.ts.map

package/dist/services/doc-chunker.js

@@ -0,0 +1,208 @@
+/**
+ * Document Chunker — Split markdown files into searchable chunks
+ *
+ * Strategy:
+ * 1. Split on H2 headers first (natural semantic boundaries)
+ * 2. If a section exceeds target size, split on paragraph boundaries
+ * 3. Each chunk carries metadata: file path, title, category, chunk index
+ *
+ * Target chunk size: 500-800 tokens (~2000-3200 chars)
+ */
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+const TARGET_CHUNK_CHARS = 2400; // ~600 tokens
+const MAX_CHUNK_CHARS = 3600; // ~900 tokens hard limit
+const MIN_CHUNK_CHARS = 200; // Don't create tiny chunks
+/**
+ * Extract title from markdown content (first H1, or filename)
+ */
+function extractTitle(content, filePath) {
+    const h1Match = content.match(/^#\s+(.+)$/m);
+    if (h1Match)
+        return h1Match[1].trim();
+    // Fall back to filename without extension
+    return path.basename(filePath, ".md").replace(/[-_]/g, " ");
+}
+/**
+ * Extract category from directory structure
+ */
+function extractCategory(relativePath) {
+    const parts = relativePath.split(path.sep);
+    if (parts.length > 1)
+        return parts[0];
+    return "root";
+}
+/**
+ * Split markdown into sections by H2 headers
+ */
+function splitByH2(content) {
+    const sections = [];
+    const lines = content.split("\n");
+    let currentTitle = "";
+    let currentLines = [];
+    for (const line of lines) {
+        const h2Match = line.match(/^##\s+(.+)$/);
+        if (h2Match) {
+            // Save previous section if it has content
+            if (currentLines.length > 0) {
+                const text = currentLines.join("\n").trim();
+                if (text.length > 0) {
+                    sections.push({ title: currentTitle, content: text });
+                }
+            }
+            currentTitle = h2Match[1].trim();
+            currentLines = [];
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    // Don't forget the last section
+    if (currentLines.length > 0) {
+        const text = currentLines.join("\n").trim();
+        if (text.length > 0) {
+            sections.push({ title: currentTitle, content: text });
+        }
+    }
+    return sections;
+}
+/**
+ * Split a text blob on paragraph boundaries to fit within target size
+ */
+function splitByParagraphs(text, maxChars) {
+    if (text.length <= maxChars)
+        return [text];
+    const chunks = [];
+    const paragraphs = text.split(/\n\n+/);
+    let current = "";
+    for (const para of paragraphs) {
+        if (current.length + para.length + 2 > maxChars && current.length > 0) {
+            chunks.push(current.trim());
+            current = para;
+        }
+        else {
+            current = current ? current + "\n\n" + para : para;
+        }
+    }
+    if (current.trim().length > 0) {
+        chunks.push(current.trim());
+    }
+    return chunks;
+}
+/**
+ * Compute SHA-256 hash of content
+ */
+function hashContent(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+/**
+ * Chunk a single markdown file into searchable segments
+ */
+export function chunkDocument(doc) {
+    const title = extractTitle(doc.content, doc.relative_path);
+    const category = extractCategory(doc.relative_path);
+    const chunks = [];
+    let chunkIndex = 0;
+    // Split by H2 headers
+    const sections = splitByH2(doc.content);
+    for (const section of sections) {
+        // If section fits in one chunk, use it directly
+        if (section.content.length <= MAX_CHUNK_CHARS) {
+            if (section.content.length >= MIN_CHUNK_CHARS) {
+                chunks.push({
+                    file_path: doc.relative_path,
+                    chunk_index: chunkIndex++,
+                    title,
+                    section_title: section.title,
+                    category,
+                    content: section.content,
+                    file_hash: doc.hash,
+                });
+            }
+        }
+        else {
+            // Section too large — split by paragraphs
+            const subChunks = splitByParagraphs(section.content, TARGET_CHUNK_CHARS);
+            for (const sub of subChunks) {
+                if (sub.length >= MIN_CHUNK_CHARS) {
+                    chunks.push({
+                        file_path: doc.relative_path,
+                        chunk_index: chunkIndex++,
+                        title,
+                        section_title: section.title,
+                        category,
+                        content: sub,
+                        file_hash: doc.hash,
+                    });
+                }
+            }
+        }
+    }
+    // Edge case: file with no H2 headers and short content — one chunk
+    if (chunks.length === 0 && doc.content.trim().length >= MIN_CHUNK_CHARS) {
+        chunks.push({
+            file_path: doc.relative_path,
+            chunk_index: 0,
+            title,
+            section_title: "",
+            category,
+            content: doc.content.trim().slice(0, MAX_CHUNK_CHARS),
+            file_hash: doc.hash,
+        });
+    }
+    return chunks;
+}
+/**
+ * Scan a directory for markdown files
+ */
+export function scanDirectory(dirPath, options = {}) {
+    const exclude = options.exclude || ["_archive", "node_modules", ".git"];
+    const files = [];
+    function walk(currentPath) {
+        let entries;
+        try {
+            entries = fs.readdirSync(currentPath, { withFileTypes: true });
+        }
+        catch {
+            return; // Permission denied or inaccessible
+        }
+        for (const entry of entries) {
+            const fullPath = path.join(currentPath, entry.name);
+            if (entry.isDirectory()) {
+                if (!exclude.includes(entry.name)) {
+                    walk(fullPath);
+                }
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md")) {
+                try {
+                    const content = fs.readFileSync(fullPath, "utf-8");
+                    const relativePath = path.relative(dirPath, fullPath);
+                    files.push({
+                        absolute_path: fullPath,
+                        relative_path: relativePath,
+                        content,
+                        hash: hashContent(content),
+                    });
+                }
+                catch {
+                    // Skip unreadable files
+                }
+            }
+        }
+    }
+    walk(dirPath);
+    return files;
+}
+/**
+ * Chunk all markdown files in a directory
+ */
+export function chunkDirectory(dirPath, options = {}) {
+    const files = scanDirectory(dirPath, options);
+    const chunks = [];
+    for (const file of files) {
+        chunks.push(...chunkDocument(file));
+    }
+    return { files, chunks };
+}
+//# sourceMappingURL=doc-chunker.js.map

package/dist/services/doc-index.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Document Index — Storage and search for indexed doc chunks
+ *
+ * Supports two backends:
+ * - Free tier: Local JSON file with BM25 keyword search
+ * - Pro/dev tier: In-memory vector index with embeddings
+ *
+ * Follows the same patterns as local-vector-search.ts and local-file-storage.ts
+ */
+import type { DocChunk } from "./doc-chunker.js";
+export interface IndexedDocChunk {
+    id: string;
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    file_hash: string;
+    project: string;
+    embedding?: number[];
+    indexed_at: string;
+}
+export interface DocSearchResult {
+    id: string;
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    similarity: number;
+    project: string;
+}
+export interface IndexStats {
+    total_chunks: number;
+    total_files: number;
+    files_indexed: string[];
+    categories: Record<string, number>;
+    project: string;
+    has_embeddings: boolean;
+}
+/**
+ * Index doc chunks into storage.
+ *
+ * - Removes old chunks for the same project + file_path
+ * - Generates embeddings if available (pro/dev tier)
+ * - Stores to local JSON file
+ * - Loads into in-memory vector index if embeddings present
+ *
+ * Returns count of chunks indexed.
+ */
+export declare function indexChunks(chunks: DocChunk[], project: string, options?: {
+    batchSize?: number;
+}): Promise<{
+    indexed: number;
+    embedded: number;
+    errors: number;
+}>;
+/**
+ * Search indexed docs using semantic similarity (pro/dev) or BM25 (free)
+ */
+export declare function searchDocs(query: string, options?: {
+    project?: string;
+    category?: string;
+    match_count?: number;
+}): Promise<DocSearchResult[]>;
+/**
+ * Get index statistics
+ */
+export declare function getIndexStats(project?: string): IndexStats;
+/**
+ * Check which files have changed since last index (by hash)
+ */
+export declare function getChangedFiles(fileHashes: Map<string, string>, project: string): {
+    changed: string[];
+    unchanged: string[];
+    new_files: string[];
+};
+/**
+ * Initialize vector index from local storage on startup
+ */
+export declare function initDocVectorIndex(): void;
+/**
+ * Clear the doc index for a project (or all)
+ */
+export declare function clearDocIndex(project?: string): number;
+//# sourceMappingURL=doc-index.d.ts.map

package/dist/services/doc-index.js

@@ -0,0 +1,328 @@
+/**
+ * Document Index — Storage and search for indexed doc chunks
+ *
+ * Supports two backends:
+ * - Free tier: Local JSON file with BM25 keyword search
+ * - Pro/dev tier: In-memory vector index with embeddings
+ *
+ * Follows the same patterns as local-vector-search.ts and local-file-storage.ts
+ */
+import * as fs from "fs";
+import * as path from "path";
+import { v4 as uuidv4 } from "uuid";
+import { getGitmemDir } from "./gitmem-dir.js";
+import { bm25Search } from "./bm25.js";
+import { embed as generateEmbedding, isEmbeddingAvailable } from "./embedding.js";
+// --- Local File Index ---
+const INDEX_FILE = "docs-index.json";
+const MAX_INDEX_SIZE = 20 * 1024 * 1024; // 20MB
+/**
+ * Get the path to the local docs index file
+ */
+function getIndexPath() {
+    return path.join(getGitmemDir(), INDEX_FILE);
+}
+/**
+ * Read the local index from disk
+ */
+function readLocalIndex() {
+    const indexPath = getIndexPath();
+    if (!fs.existsSync(indexPath))
+        return [];
+    try {
+        const raw = fs.readFileSync(indexPath, "utf-8");
+        return JSON.parse(raw);
+    }
+    catch {
+        console.error("[doc-index] Failed to read docs-index.json, starting fresh");
+        return [];
+    }
+}
+/**
+ * Write the local index to disk
+ */
+function writeLocalIndex(chunks) {
+    const indexPath = getIndexPath();
+    const dir = path.dirname(indexPath);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+    // Strip embeddings from local file to save space
+    const stripped = chunks.map(({ embedding: _e, ...rest }) => rest);
+    const json = JSON.stringify(stripped, null, 2);
+    if (Buffer.byteLength(json, "utf-8") > MAX_INDEX_SIZE) {
+        console.error("[doc-index] Warning: docs-index.json exceeds 20MB");
+    }
+    fs.writeFileSync(indexPath, json, "utf-8");
+}
+let vectorIndex = [];
+/**
+ * Compute cosine similarity between two normalized vectors
+ */
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+    }
+    return dot;
+}
+// --- Public API ---
+/**
+ * Index doc chunks into storage.
+ *
+ * - Removes old chunks for the same project + file_path
+ * - Generates embeddings if available (pro/dev tier)
+ * - Stores to local JSON file
+ * - Loads into in-memory vector index if embeddings present
+ *
+ * Returns count of chunks indexed.
+ */
+export async function indexChunks(chunks, project, options = {}) {
+    const batchSize = options.batchSize || 10;
+    const now = new Date().toISOString();
+    let embedded = 0;
+    let errors = 0;
+    // Read existing index
+    const existing = readLocalIndex();
+    // Build set of file paths being re-indexed
+    const reindexedPaths = new Set(chunks.map((c) => `${project}:${c.file_path}`));
+    // Remove old chunks for files being re-indexed
+    const kept = existing.filter((c) => !reindexedPaths.has(`${c.project}:${c.file_path}`));
+    // Create new indexed chunks
+    const newChunks = [];
+    // Process in batches for embedding
+    for (let i = 0; i < chunks.length; i += batchSize) {
+        const batch = chunks.slice(i, i + batchSize);
+        for (const chunk of batch) {
+            const indexed = {
+                id: uuidv4(),
+                file_path: chunk.file_path,
+                chunk_index: chunk.chunk_index,
+                title: chunk.title,
+                section_title: chunk.section_title,
+                category: chunk.category,
+                content: chunk.content,
+                file_hash: chunk.file_hash,
+                project,
+                indexed_at: now,
+            };
+            // Generate embedding if available
+            if (isEmbeddingAvailable()) {
+                try {
+                    // Embed title + section + content for richer representation
+                    const textToEmbed = [
+                        indexed.title,
+                        indexed.section_title,
+                        indexed.content,
+                    ]
+                        .filter(Boolean)
+                        .join(" | ");
+                    const embedding = await generateEmbedding(textToEmbed);
+                    if (embedding) {
+                        indexed.embedding = embedding;
+                        embedded++;
+                    }
+                }
+                catch (err) {
+                    console.error(`[doc-index] Embedding failed for ${chunk.file_path}:${chunk.chunk_index}:`, err instanceof Error ? err.message : err);
+                    errors++;
+                }
+            }
+            newChunks.push(indexed);
+        }
+        // Progress logging for large batches (every ~100 chunks)
+        if (chunks.length > 50 && i + batchSize < chunks.length && (i + batchSize) % 100 < batchSize) {
+            console.error(`[doc-index] Progress: ${Math.min(i + batchSize, chunks.length)}/${chunks.length} chunks`);
+        }
+    }
+    // Merge and write
+    const merged = [...kept, ...newChunks];
+    writeLocalIndex(merged);
+    // Update in-memory vector index
+    rebuildVectorIndex(merged);
+    return { indexed: newChunks.length, embedded, errors };
+}
+/**
+ * Rebuild the in-memory vector index from stored chunks
+ */
+function rebuildVectorIndex(chunks) {
+    vectorIndex = chunks
+        .filter((c) => c.embedding && Array.isArray(c.embedding) && c.embedding.length > 0)
+        .map((c) => ({
+        chunk: c,
+        embedding: c.embedding,
+    }));
+    console.error(`[doc-index] Vector index rebuilt: ${vectorIndex.length} entries with embeddings`);
+}
+/**
+ * Search indexed docs using semantic similarity (pro/dev) or BM25 (free)
+ */
+export async function searchDocs(query, options = {}) {
+    const matchCount = options.match_count || 5;
+    // Try vector search first (pro/dev tier with embeddings)
+    if (isEmbeddingAvailable() && vectorIndex.length > 0) {
+        return vectorSearchDocs(query, options);
+    }
+    // Fall back to BM25 keyword search
+    return bm25SearchDocs(query, options);
+}
+/**
+ * Vector-based semantic search over doc chunks
+ */
+async function vectorSearchDocs(query, options) {
+    const matchCount = options.match_count || 5;
+    // Generate query embedding
+    const queryEmbedding = await generateEmbedding(query);
+    if (!queryEmbedding) {
+        console.error("[doc-index] Query embedding failed, falling back to BM25");
+        return bm25SearchDocs(query, options);
+    }
+    // Filter candidates
+    let candidates = vectorIndex;
+    if (options.project) {
+        candidates = candidates.filter((e) => e.chunk.project === options.project);
+    }
+    if (options.category) {
+        candidates = candidates.filter((e) => e.chunk.category === options.category);
+    }
+    // Score by cosine similarity
+    const scored = candidates.map((entry) => ({
+        chunk: entry.chunk,
+        similarity: cosineSimilarity(queryEmbedding, entry.embedding),
+    }));
+    // Sort and take top k
+    scored.sort((a, b) => b.similarity - a.similarity);
+    const topK = scored.slice(0, matchCount);
+    return topK.map(({ chunk, similarity }) => ({
+        id: chunk.id,
+        file_path: chunk.file_path,
+        chunk_index: chunk.chunk_index,
+        title: chunk.title,
+        section_title: chunk.section_title,
+        category: chunk.category,
+        content: chunk.content,
+        similarity: Math.round(similarity * 1000) / 1000,
+        project: chunk.project,
+    }));
+}
+/**
+ * BM25 keyword search over doc chunks (free tier)
+ */
+function bm25SearchDocs(query, options) {
+    const matchCount = options.match_count || 5;
+    const chunks = readLocalIndex();
+    // Filter by project and category
+    let filtered = chunks;
+    if (options.project) {
+        filtered = filtered.filter((c) => c.project === options.project);
+    }
+    if (options.category) {
+        filtered = filtered.filter((c) => c.category === options.category);
+    }
+    if (filtered.length === 0)
+        return [];
+    // Build BM25 documents with field boosting
+    const docs = filtered.map((c) => ({
+        id: c.id,
+        fields: [
+            { text: c.title, boost: 3 },
+            { text: c.section_title || "", boost: 2 },
+            { text: c.category, boost: 1.5 },
+            { text: c.content, boost: 1 },
+        ],
+    }));
+    const results = bm25Search(query, docs, matchCount);
+    // Map back to DocSearchResult
+    const byId = new Map(filtered.map((c) => [c.id, c]));
+    return results
+        .map((r) => {
+        const c = byId.get(r.id);
+        if (!c)
+            return null;
+        return {
+            id: c.id,
+            file_path: c.file_path,
+            chunk_index: c.chunk_index,
+            title: c.title,
+            section_title: c.section_title,
+            category: c.category,
+            content: c.content,
+            similarity: r.similarity,
+            project: c.project,
+        };
+    })
+        .filter((r) => r !== null);
+}
+/**
+ * Get index statistics
+ */
+export function getIndexStats(project) {
+    const chunks = readLocalIndex();
+    const filtered = project
+        ? chunks.filter((c) => c.project === project)
+        : chunks;
+    const files = new Set(filtered.map((c) => c.file_path));
+    const categories = {};
+    for (const c of filtered) {
+        categories[c.category] = (categories[c.category] || 0) + 1;
+    }
+    return {
+        total_chunks: filtered.length,
+        total_files: files.size,
+        files_indexed: Array.from(files).sort(),
+        categories,
+        project: project || "all",
+        has_embeddings: vectorIndex.length > 0,
+    };
+}
+/**
+ * Check which files have changed since last index (by hash)
+ */
+export function getChangedFiles(fileHashes, project) {
+    const existing = readLocalIndex().filter((c) => c.project === project);
+    const existingHashes = new Map();
+    for (const c of existing) {
+        existingHashes.set(c.file_path, c.file_hash);
+    }
+    const changed = [];
+    const unchanged = [];
+    const newFiles = [];
+    for (const [filePath, hash] of fileHashes) {
+        const existingHash = existingHashes.get(filePath);
+        if (!existingHash) {
+            newFiles.push(filePath);
+        }
+        else if (existingHash !== hash) {
+            changed.push(filePath);
+        }
+        else {
+            unchanged.push(filePath);
+        }
+    }
+    return { changed, unchanged, new_files: newFiles };
+}
+/**
+ * Initialize vector index from local storage on startup
+ */
+export function initDocVectorIndex() {
+    const chunks = readLocalIndex();
+    rebuildVectorIndex(chunks);
+}
+/**
+ * Clear the doc index for a project (or all)
+ */
+export function clearDocIndex(project) {
+    const existing = readLocalIndex();
+    if (!project) {
+        writeLocalIndex([]);
+        vectorIndex = [];
+        return existing.length;
+    }
+    const kept = existing.filter((c) => c.project !== project);
+    writeLocalIndex(kept);
+    rebuildVectorIndex(kept);
+    return existing.length - kept.length;
+}
+//# sourceMappingURL=doc-index.js.map