gitmem-mcp 1.4.4 → 1.5.1

package/CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.5.1] - 2026-05-13
+
+### Fixed
+- **CI smoke test tool count**: Updated `EXPECTED_TOOL_COUNTS` to reflect `index_docs` and `search_docs` additions (+2 per tier). The 1.5.0 release failed to publish because the smoke test expected 23 free-tier tools but found 25.
+
+## [1.5.0] - 2026-05-11
+
+### Added
+- **`index_docs` tool**: Scan a directory of markdown files, chunk them, and store in a local doc index for semantic search. Supports incremental indexing (only re-processes changed files), force re-index, and project-scoped indexes. Aliases: `gitmem-idx`.
+- **`search_docs` tool**: Search indexed repository documentation using semantic similarity (pro tier) or BM25 keyword search (free tier). Returns relevant chunks with file paths for targeted reading. Aliases: `gitmem-sd`.
+- **Citation protocol**: `recall`, `search`, and `prepare_context` now include a citation rule instructing agents to cite record IDs when referencing facts from institutional memory.
+- **Low confidence tagging**: Recall and search results with similarity below 0.55 are tagged `[low confidence]` — these matches have a 66% N/A rate historically.
+- **Session duration on resume**: `session_start` now shows elapsed session time and loaded scar count when resuming or refreshing an existing session.
+
+### Changed
+- **Quick close hard gate**: `session_close` with `close_type: "quick"` now rejects sessions over 30 minutes, requiring standard close instead.
+- **Standard close recall gate**: `session_close` with `close_type: "standard"` now requires at least one `recall()` call during the session (exemptions: quick close, autonomous agents, sessions with inline reflection).
+
 ## [1.4.4] - 2026-03-31
 
 ### Fixed

package/dist/hooks/format-utils.js

@@ -57,6 +57,10 @@ export function formatCompact(scars, plan, maxTokens) {
         lines.push(line);
         included++;
     }
+    // Citation reminder for sub-agent context (compact — one line)
+    if (included > 0) {
+        lines.push("Cite record IDs for any factual claims from these scars.");
+    }
     return { payload: lines.join("\n"), included };
 }
 /**

package/dist/server.js

@@ -36,6 +36,8 @@ import { dismissSuggestion } from "./tools/dismiss-suggestion.js";
 import { cleanupThreads } from "./tools/cleanup-threads.js";
 import { archiveLearning } from "./tools/archive-learning.js";
 import { contributeFeedback } from "./tools/contribute-feedback.js";
+import { indexDocs } from "./tools/index-docs.js";
+import { searchDocsHandler } from "./tools/search-docs.js";
 import { getCacheStatus, checkCacheHealth, flushCache, startBackgroundInit, } from "./services/startup.js";
 import { getEffectTracker } from "./services/effect-tracker.js";
 import { RIPPLE, ANSI } from "./services/display-protocol.js";
@@ -246,6 +248,8 @@ export function createServer() {
                         { alias: "gitmem-al", full: "archive_learning", description: "Archive a scar/win/pattern (is_active=false)" },
                         { alias: "gitmem-graph", full: "graph_traverse", description: "Traverse knowledge graph over institutional memory" },
                         { alias: "gitmem-fb", full: "contribute_feedback", description: "Submit feedback about gitmem (10/session limit)" },
+                        { alias: "gitmem-idx", full: "index_docs", description: "Index markdown docs for semantic search" },
+                        { alias: "gitmem-sd", full: "search_docs", description: "Search indexed repository docs" },
                     ];
                     if (hasBatchOperations()) {
                         commands.push({ alias: "gitmem-rsb", full: "record_scar_usage_batch", description: "Track multiple scars (batch)" });
@@ -315,6 +319,15 @@ export function createServer() {
                 case "gm-cache-f":
                     result = await flushCache(toolArgs.project || getProject() || "default");
                     break;
+                // Doc indexing and search
+                case "index_docs":
+                case "gitmem-idx":
+                    result = await indexDocs(toolArgs);
+                    break;
+                case "search_docs":
+                case "gitmem-sd":
+                    result = await searchDocsHandler(toolArgs);
+                    break;
                 default:
                     throw new Error(`Unknown tool: ${name}`);
             }

package/dist/services/doc-chunker.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Document Chunker — Split markdown files into searchable chunks
+ *
+ * Strategy:
+ * 1. Split on H2 headers first (natural semantic boundaries)
+ * 2. If a section exceeds target size, split on paragraph boundaries
+ * 3. Each chunk carries metadata: file path, title, category, chunk index
+ *
+ * Target chunk size: 500-800 tokens (~2000-3200 chars)
+ */
+export interface DocChunk {
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    file_hash: string;
+}
+export interface DocFile {
+    absolute_path: string;
+    relative_path: string;
+    content: string;
+    hash: string;
+}
+/**
+ * Chunk a single markdown file into searchable segments
+ */
+export declare function chunkDocument(doc: DocFile): DocChunk[];
+/**
+ * Scan a directory for markdown files
+ */
+export declare function scanDirectory(dirPath: string, options?: {
+    exclude?: string[];
+}): DocFile[];
+/**
+ * Chunk all markdown files in a directory
+ */
+export declare function chunkDirectory(dirPath: string, options?: {
+    exclude?: string[];
+}): {
+    files: DocFile[];
+    chunks: DocChunk[];
+};
+//# sourceMappingURL=doc-chunker.d.ts.map

package/dist/services/doc-chunker.js

@@ -0,0 +1,208 @@
+/**
+ * Document Chunker — Split markdown files into searchable chunks
+ *
+ * Strategy:
+ * 1. Split on H2 headers first (natural semantic boundaries)
+ * 2. If a section exceeds target size, split on paragraph boundaries
+ * 3. Each chunk carries metadata: file path, title, category, chunk index
+ *
+ * Target chunk size: 500-800 tokens (~2000-3200 chars)
+ */
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+const TARGET_CHUNK_CHARS = 2400; // ~600 tokens
+const MAX_CHUNK_CHARS = 3600; // ~900 tokens hard limit
+const MIN_CHUNK_CHARS = 200; // Don't create tiny chunks
+/**
+ * Extract title from markdown content (first H1, or filename)
+ */
+function extractTitle(content, filePath) {
+    const h1Match = content.match(/^#\s+(.+)$/m);
+    if (h1Match)
+        return h1Match[1].trim();
+    // Fall back to filename without extension
+    return path.basename(filePath, ".md").replace(/[-_]/g, " ");
+}
+/**
+ * Extract category from directory structure
+ */
+function extractCategory(relativePath) {
+    const parts = relativePath.split(path.sep);
+    if (parts.length > 1)
+        return parts[0];
+    return "root";
+}
+/**
+ * Split markdown into sections by H2 headers
+ */
+function splitByH2(content) {
+    const sections = [];
+    const lines = content.split("\n");
+    let currentTitle = "";
+    let currentLines = [];
+    for (const line of lines) {
+        const h2Match = line.match(/^##\s+(.+)$/);
+        if (h2Match) {
+            // Save previous section if it has content
+            if (currentLines.length > 0) {
+                const text = currentLines.join("\n").trim();
+                if (text.length > 0) {
+                    sections.push({ title: currentTitle, content: text });
+                }
+            }
+            currentTitle = h2Match[1].trim();
+            currentLines = [];
+        }
+        else {
+            currentLines.push(line);
+        }
+    }
+    // Don't forget the last section
+    if (currentLines.length > 0) {
+        const text = currentLines.join("\n").trim();
+        if (text.length > 0) {
+            sections.push({ title: currentTitle, content: text });
+        }
+    }
+    return sections;
+}
+/**
+ * Split a text blob on paragraph boundaries to fit within target size
+ */
+function splitByParagraphs(text, maxChars) {
+    if (text.length <= maxChars)
+        return [text];
+    const chunks = [];
+    const paragraphs = text.split(/\n\n+/);
+    let current = "";
+    for (const para of paragraphs) {
+        if (current.length + para.length + 2 > maxChars && current.length > 0) {
+            chunks.push(current.trim());
+            current = para;
+        }
+        else {
+            current = current ? current + "\n\n" + para : para;
+        }
+    }
+    if (current.trim().length > 0) {
+        chunks.push(current.trim());
+    }
+    return chunks;
+}
+/**
+ * Compute SHA-256 hash of content
+ */
+function hashContent(content) {
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+/**
+ * Chunk a single markdown file into searchable segments
+ */
+export function chunkDocument(doc) {
+    const title = extractTitle(doc.content, doc.relative_path);
+    const category = extractCategory(doc.relative_path);
+    const chunks = [];
+    let chunkIndex = 0;
+    // Split by H2 headers
+    const sections = splitByH2(doc.content);
+    for (const section of sections) {
+        // If section fits in one chunk, use it directly
+        if (section.content.length <= MAX_CHUNK_CHARS) {
+            if (section.content.length >= MIN_CHUNK_CHARS) {
+                chunks.push({
+                    file_path: doc.relative_path,
+                    chunk_index: chunkIndex++,
+                    title,
+                    section_title: section.title,
+                    category,
+                    content: section.content,
+                    file_hash: doc.hash,
+                });
+            }
+        }
+        else {
+            // Section too large — split by paragraphs
+            const subChunks = splitByParagraphs(section.content, TARGET_CHUNK_CHARS);
+            for (const sub of subChunks) {
+                if (sub.length >= MIN_CHUNK_CHARS) {
+                    chunks.push({
+                        file_path: doc.relative_path,
+                        chunk_index: chunkIndex++,
+                        title,
+                        section_title: section.title,
+                        category,
+                        content: sub,
+                        file_hash: doc.hash,
+                    });
+                }
+            }
+        }
+    }
+    // Edge case: file with no H2 headers and short content — one chunk
+    if (chunks.length === 0 && doc.content.trim().length >= MIN_CHUNK_CHARS) {
+        chunks.push({
+            file_path: doc.relative_path,
+            chunk_index: 0,
+            title,
+            section_title: "",
+            category,
+            content: doc.content.trim().slice(0, MAX_CHUNK_CHARS),
+            file_hash: doc.hash,
+        });
+    }
+    return chunks;
+}
+/**
+ * Scan a directory for markdown files
+ */
+export function scanDirectory(dirPath, options = {}) {
+    const exclude = options.exclude || ["_archive", "node_modules", ".git"];
+    const files = [];
+    function walk(currentPath) {
+        let entries;
+        try {
+            entries = fs.readdirSync(currentPath, { withFileTypes: true });
+        }
+        catch {
+            return; // Permission denied or inaccessible
+        }
+        for (const entry of entries) {
+            const fullPath = path.join(currentPath, entry.name);
+            if (entry.isDirectory()) {
+                if (!exclude.includes(entry.name)) {
+                    walk(fullPath);
+                }
+            }
+            else if (entry.isFile() && entry.name.endsWith(".md")) {
+                try {
+                    const content = fs.readFileSync(fullPath, "utf-8");
+                    const relativePath = path.relative(dirPath, fullPath);
+                    files.push({
+                        absolute_path: fullPath,
+                        relative_path: relativePath,
+                        content,
+                        hash: hashContent(content),
+                    });
+                }
+                catch {
+                    // Skip unreadable files
+                }
+            }
+        }
+    }
+    walk(dirPath);
+    return files;
+}
+/**
+ * Chunk all markdown files in a directory
+ */
+export function chunkDirectory(dirPath, options = {}) {
+    const files = scanDirectory(dirPath, options);
+    const chunks = [];
+    for (const file of files) {
+        chunks.push(...chunkDocument(file));
+    }
+    return { files, chunks };
+}
+//# sourceMappingURL=doc-chunker.js.map

package/dist/services/doc-index.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Document Index — Storage and search for indexed doc chunks
+ *
+ * Supports two backends:
+ * - Free tier: Local JSON file with BM25 keyword search
+ * - Pro/dev tier: In-memory vector index with embeddings
+ *
+ * Follows the same patterns as local-vector-search.ts and local-file-storage.ts
+ */
+import type { DocChunk } from "./doc-chunker.js";
+export interface IndexedDocChunk {
+    id: string;
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    file_hash: string;
+    project: string;
+    embedding?: number[];
+    indexed_at: string;
+}
+export interface DocSearchResult {
+    id: string;
+    file_path: string;
+    chunk_index: number;
+    title: string;
+    section_title: string;
+    category: string;
+    content: string;
+    similarity: number;
+    project: string;
+}
+export interface IndexStats {
+    total_chunks: number;
+    total_files: number;
+    files_indexed: string[];
+    categories: Record<string, number>;
+    project: string;
+    has_embeddings: boolean;
+}
+/**
+ * Index doc chunks into storage.
+ *
+ * - Removes old chunks for the same project + file_path
+ * - Generates embeddings if available (pro/dev tier)
+ * - Stores to local JSON file
+ * - Loads into in-memory vector index if embeddings present
+ *
+ * Returns count of chunks indexed.
+ */
+export declare function indexChunks(chunks: DocChunk[], project: string, options?: {
+    batchSize?: number;
+}): Promise<{
+    indexed: number;
+    embedded: number;
+    errors: number;
+}>;
+/**
+ * Search indexed docs using semantic similarity (pro/dev) or BM25 (free)
+ */
+export declare function searchDocs(query: string, options?: {
+    project?: string;
+    category?: string;
+    match_count?: number;
+}): Promise<DocSearchResult[]>;
+/**
+ * Get index statistics
+ */
+export declare function getIndexStats(project?: string): IndexStats;
+/**
+ * Check which files have changed since last index (by hash)
+ */
+export declare function getChangedFiles(fileHashes: Map<string, string>, project: string): {
+    changed: string[];
+    unchanged: string[];
+    new_files: string[];
+};
+/**
+ * Initialize vector index from local storage on startup
+ */
+export declare function initDocVectorIndex(): void;
+/**
+ * Clear the doc index for a project (or all)
+ */
+export declare function clearDocIndex(project?: string): number;
+//# sourceMappingURL=doc-index.d.ts.map