@telvok/librarian-mcp 1.1.0 → 1.2.0

package/dist/library/embeddings.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Get embedding for a text string.
+ * Returns a 384-dimensional normalized vector.
+ */
+export declare function getEmbedding(text: string): Promise<number[]>;
+/**
+ * Check if embeddings are available (model can load).
+ */
+export declare function isEmbeddingAvailable(): Promise<boolean>;
+/**
+ * Calculate cosine similarity between two vectors.
+ * Since vectors are normalized, this is just the dot product.
+ */
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+/**
+ * Split text into chunks at sentence boundaries.
+ * Aims for ~500 chars per chunk to preserve semantic meaning.
+ */
+export declare function chunkText(text: string, maxChars?: number): string[];
+export declare const EMBEDDING_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
+export declare const EMBEDDING_DIMENSION = 384;

package/dist/library/embeddings.js

@@ -0,0 +1,86 @@
+import { pipeline, env } from '@huggingface/transformers';
+import * as path from 'path';
+import { getLibraryPath } from './storage.js';
+// ============================================================================
+// Configuration
+// ============================================================================
+// Cache model in .librarian/models
+env.allowRemoteModels = true;
+const MODEL_ID = 'Xenova/all-MiniLM-L6-v2';
+// ============================================================================
+// Embedding Generation
+// ============================================================================
+let embedder = null;
+/**
+ * Get embedding for a text string.
+ * Returns a 384-dimensional normalized vector.
+ */
+export async function getEmbedding(text) {
+    if (!embedder) {
+        // Set local model path on first call
+        const libraryPath = getLibraryPath();
+        env.localModelPath = path.join(libraryPath, 'models');
+        embedder = await pipeline('feature-extraction', MODEL_ID);
+    }
+    const result = await embedder(text, { pooling: 'mean', normalize: true });
+    return Array.from(result.data);
+}
+/**
+ * Check if embeddings are available (model can load).
+ */
+export async function isEmbeddingAvailable() {
+    try {
+        await getEmbedding('test');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// ============================================================================
+// Similarity Calculation
+// ============================================================================
+/**
+ * Calculate cosine similarity between two vectors.
+ * Since vectors are normalized, this is just the dot product.
+ */
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+        throw new Error('Vectors must have same dimension');
+    }
+    return a.reduce((sum, val, i) => sum + val * b[i], 0);
+}
+// ============================================================================
+// Text Chunking
+// ============================================================================
+/**
+ * Split text into chunks at sentence boundaries.
+ * Aims for ~500 chars per chunk to preserve semantic meaning.
+ */
+export function chunkText(text, maxChars = 500) {
+    // Split at sentence boundaries (. ! ? followed by whitespace)
+    const sentences = text.split(/(?<=[.!?])\s+/);
+    const chunks = [];
+    let current = '';
+    for (const sentence of sentences) {
+        // If adding this sentence exceeds limit and we have content, start new chunk
+        if ((current + ' ' + sentence).length > maxChars && current.trim()) {
+            chunks.push(current.trim());
+            current = sentence;
+        }
+        else {
+            current = current ? current + ' ' + sentence : sentence;
+        }
+    }
+    // Don't forget the last chunk
+    if (current.trim()) {
+        chunks.push(current.trim());
+    }
+    // If no chunks created (e.g., no sentence boundaries), return original text
+    return chunks.length > 0 ? chunks : [text];
+}
+// ============================================================================
+// Constants
+// ============================================================================
+export const EMBEDDING_MODEL_ID = MODEL_ID;
+export const EMBEDDING_DIMENSION = 384;

package/dist/library/vector-index.d.ts

@@ -0,0 +1,55 @@
+export interface IndexEntry {
+    path: string;
+    title: string;
+    embedding: number[];
+    chunk: number;
+    preview: string;
+}
+export interface VectorIndex {
+    version: number;
+    rebuilt: string;
+    modelId: string;
+    entries: IndexEntry[];
+}
+export interface SemanticMatch {
+    path: string;
+    title: string;
+    similarity: number;
+    preview: string;
+}
+/**
+ * Load the vector index from disk.
+ * Returns empty index if file doesn't exist or is invalid.
+ */
+export declare function loadIndex(): Promise<VectorIndex>;
+/**
+ * Save the vector index to disk.
+ */
+export declare function saveIndex(index: VectorIndex): Promise<void>;
+/**
+ * Add or update an entry in the index.
+ * Chunks the content and generates embeddings for each chunk.
+ */
+export declare function addToIndex(index: VectorIndex, entryPath: string, title: string, content: string): Promise<void>;
+/**
+ * Remove an entry from the index.
+ */
+export declare function removeFromIndex(index: VectorIndex, entryPath: string): void;
+/**
+ * Search the index for entries semantically similar to the query.
+ * Returns paths ranked by similarity, deduped to best chunk per entry.
+ */
+export declare function semanticSearch(index: VectorIndex, query: string, limit?: number): Promise<SemanticMatch[]>;
+/**
+ * Check if the index might be stale (model changed).
+ */
+export declare function isIndexStale(index: VectorIndex): boolean;
+/**
+ * Get index statistics.
+ */
+export declare function getIndexStats(index: VectorIndex): {
+    entryCount: number;
+    chunkCount: number;
+    modelId: string;
+    rebuilt: string;
+};

package/dist/library/vector-index.js

@@ -0,0 +1,160 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { getLibraryPath } from './storage.js';
+import { getEmbedding, chunkText, cosineSimilarity, EMBEDDING_MODEL_ID } from './embeddings.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const INDEX_FILENAME = 'index.json';
+const CURRENT_VERSION = 1;
+// ============================================================================
+// Index File Operations
+// ============================================================================
+/**
+ * Get path to the index file.
+ */
+function getIndexPath() {
+    return path.join(getLibraryPath(), INDEX_FILENAME);
+}
+/**
+ * Load the vector index from disk.
+ * Returns empty index if file doesn't exist or is invalid.
+ */
+export async function loadIndex() {
+    const indexPath = getIndexPath();
+    try {
+        const data = await fs.readFile(indexPath, 'utf-8');
+        const index = JSON.parse(data);
+        // Validate structure
+        if (!index.version || !Array.isArray(index.entries)) {
+            return createEmptyIndex();
+        }
+        return index;
+    }
+    catch {
+        // File doesn't exist or is invalid
+        return createEmptyIndex();
+    }
+}
+/**
+ * Save the vector index to disk.
+ */
+export async function saveIndex(index) {
+    const indexPath = getIndexPath();
+    // Update metadata
+    index.rebuilt = new Date().toISOString();
+    index.modelId = EMBEDDING_MODEL_ID;
+    // Ensure directory exists
+    await fs.mkdir(path.dirname(indexPath), { recursive: true });
+    // Write atomically by writing to temp file first
+    const tempPath = indexPath + '.tmp';
+    await fs.writeFile(tempPath, JSON.stringify(index, null, 2), 'utf-8');
+    await fs.rename(tempPath, indexPath);
+}
+/**
+ * Create an empty index.
+ */
+function createEmptyIndex() {
+    return {
+        version: CURRENT_VERSION,
+        rebuilt: '',
+        modelId: EMBEDDING_MODEL_ID,
+        entries: [],
+    };
+}
+// ============================================================================
+// Index Operations
+// ============================================================================
+/**
+ * Add or update an entry in the index.
+ * Chunks the content and generates embeddings for each chunk.
+ */
+export async function addToIndex(index, entryPath, title, content) {
+    // Remove any existing entries for this path
+    index.entries = index.entries.filter(e => e.path !== entryPath);
+    // Chunk the content
+    const chunks = chunkText(content);
+    // Generate embeddings for each chunk
+    for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        try {
+            const embedding = await getEmbedding(chunk);
+            index.entries.push({
+                path: entryPath,
+                title,
+                embedding,
+                chunk: i,
+                preview: chunk.slice(0, 100) + (chunk.length > 100 ? '...' : ''),
+            });
+        }
+        catch (error) {
+            // Log but don't fail - entry will still be searchable via keywords
+            console.error(`Failed to embed chunk ${i} for ${entryPath}:`, error);
+        }
+    }
+}
+/**
+ * Remove an entry from the index.
+ */
+export function removeFromIndex(index, entryPath) {
+    index.entries = index.entries.filter(e => e.path !== entryPath);
+}
+// ============================================================================
+// Semantic Search
+// ============================================================================
+/**
+ * Search the index for entries semantically similar to the query.
+ * Returns paths ranked by similarity, deduped to best chunk per entry.
+ */
+export async function semanticSearch(index, query, limit = 5) {
+    if (index.entries.length === 0) {
+        return [];
+    }
+    // Get query embedding
+    const queryEmbedding = await getEmbedding(query);
+    // Score all entries
+    const scored = index.entries.map(entry => ({
+        ...entry,
+        similarity: cosineSimilarity(queryEmbedding, entry.embedding),
+    }));
+    // Dedupe by path - keep the chunk with highest similarity
+    const byPath = new Map();
+    for (const entry of scored) {
+        const existing = byPath.get(entry.path);
+        if (!existing || entry.similarity > existing.similarity) {
+            byPath.set(entry.path, entry);
+        }
+    }
+    // Sort by similarity descending and apply limit
+    const results = [...byPath.values()]
+        .sort((a, b) => b.similarity - a.similarity)
+        .slice(0, limit)
+        .map(entry => ({
+        path: entry.path,
+        title: entry.title,
+        similarity: entry.similarity,
+        preview: entry.preview,
+    }));
+    return results;
+}
+// ============================================================================
+// Index Health
+// ============================================================================
+/**
+ * Check if the index might be stale (model changed).
+ */
+export function isIndexStale(index) {
+    return index.modelId !== EMBEDDING_MODEL_ID;
+}
+/**
+ * Get index statistics.
+ */
+export function getIndexStats(index) {
+    const uniquePaths = new Set(index.entries.map(e => e.path));
+    return {
+        entryCount: uniquePaths.size,
+        chunkCount: index.entries.length,
+        modelId: index.modelId,
+        rebuilt: index.rebuilt,
+    };
+}

package/dist/tools/brief.js

@@ -3,6 +3,7 @@ import * as path from 'path';
 import matter from 'gray-matter';
 import { glob } from 'glob';
 import { getLibraryPath, getLocalPath, getImportedPath } from '../library/storage.js';
+import { loadIndex, semanticSearch, isIndexStale } from '../library/vector-index.js';
 // ============================================================================
 // Tool Definition
 // ============================================================================
@@ -39,6 +40,49 @@ Examples:
         const localPath = getLocalPath(libraryPath);
         const importedPath = getImportedPath(libraryPath);
         let allEntries = [];
+        let useSemanticSearch = false;
+        let semanticMatches = [];
+        // Try semantic search if query is provided
+        if (query) {
+            try {
+                const index = await loadIndex();
+                // Only use semantic search if index has entries and isn't stale
+                if (index.entries.length > 0 && !isIndexStale(index)) {
+                    semanticMatches = await semanticSearch(index, query, limit);
+                    useSemanticSearch = semanticMatches.length > 0;
+                }
+            }
+            catch {
+                // Semantic search unavailable, fall back to keyword search
+                useSemanticSearch = false;
+            }
+        }
+        if (useSemanticSearch && semanticMatches.length > 0) {
+            // Load only the entries that matched semantically
+            const matchedPaths = new Set(semanticMatches.map(m => m.path));
+            for (const match of semanticMatches) {
+                const fullPath = path.join(libraryPath, match.path);
+                const entry = await readEntry(fullPath, libraryPath);
+                if (entry) {
+                    allEntries.push(entry);
+                }
+            }
+            // Sort by semantic similarity (order preserved from semanticSearch)
+            // Re-order allEntries to match semanticMatches order
+            const pathToEntry = new Map(allEntries.map(e => [e.path, e]));
+            allEntries = semanticMatches
+                .map(m => pathToEntry.get(m.path))
+                .filter((e) => e !== undefined);
+            const total = allEntries.length;
+            const entries = allEntries.slice(0, limit);
+            return {
+                entries,
+                total,
+                message: `Found ${total} ${total === 1 ? 'entry' : 'entries'} for "${query}" (semantic search).`,
+                libraryPath: localPath,
+            };
+        }
+        // Fall back to keyword search
         // Read local entries
         try {
             const localFiles = await glob(path.join(localPath, '**/*.md'), { nodir: true });

package/dist/tools/record.js

@@ -1,6 +1,7 @@
 import * as fs from 'fs/promises';
 import * as path from 'path';
 import { getLibraryPath, getLocalPath } from '../library/storage.js';
+import { loadIndex, saveIndex, addToIndex } from '../library/vector-index.js';
 // ============================================================================
 // Tool Definition
 // ============================================================================
@@ -140,6 +141,26 @@ Rich:
         const fileContent = frontmatterLines.join('\n') + '\n\n' + bodyLines.join('\n') + '\n';
         await fs.writeFile(filePath, fileContent, 'utf-8');
         const relativePath = path.relative(libraryPath, filePath);
+        // Add to vector index for semantic search
+        try {
+            const index = await loadIndex();
+            // Combine all text for embedding
+            const fullContent = [
+                title,
+                intent || '',
+                insight,
+                reasoning || '',
+                example || '',
+                context || '',
+            ].filter(Boolean).join('\n\n');
+            await addToIndex(index, relativePath, title, fullContent);
+            await saveIndex(index);
+        }
+        catch (embeddingError) {
+            // Don't fail the record operation if embedding fails
+            // Entry is still saved and searchable via keywords
+            console.error('Failed to add embedding:', embeddingError);
+        }
         return {
             success: true,
             path: relativePath,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@telvok/librarian-mcp",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Knowledge capture MCP server - remember what you learn with AI",
   "type": "module",
   "main": "dist/server.js",
@@ -26,11 +26,12 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/telvokdev/librarian.git"
+    "url": "git+https://github.com/telvokdev/librarian.git"
   },
   "author": "Telvok",
   "license": "MIT",
   "dependencies": {
+    "@huggingface/transformers": "^3.0.0",
     "@modelcontextprotocol/sdk": "^1.0.0",
     "glob": "^11.0.0",
     "gray-matter": "^4.0.3",