@grec0/memory-bank-mcp 0.0.2

package/dist/common/embeddingService.js

@@ -0,0 +1,302 @@
+/**
+ * @fileoverview Embedding service for Memory Bank using OpenAI
+ * Generates vector embeddings for code chunks
+ */
+import OpenAI from "openai";
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as path from "path";
+/**
+ * Embedding service with caching and batch processing
+ */
+export class EmbeddingService {
+    client;
+    options;
+    cache;
+    constructor(apiKey, options) {
+        if (!apiKey) {
+            throw new Error("OpenAI API key is required");
+        }
+        this.client = new OpenAI({ apiKey });
+        this.options = {
+            model: options?.model || "text-embedding-3-small",
+            dimensions: options?.dimensions || 1536,
+            batchSize: options?.batchSize || 100,
+            enableCache: options?.enableCache !== undefined ? options.enableCache : true,
+            cachePath: options?.cachePath || ".memorybank/embedding-cache.json",
+        };
+        this.cache = new Map();
+        if (this.options.enableCache) {
+            this.loadCache();
+        }
+    }
+    /**
+     * Loads embedding cache from disk
+     */
+    loadCache() {
+        try {
+            if (fs.existsSync(this.options.cachePath)) {
+                const data = fs.readFileSync(this.options.cachePath, "utf-8");
+                const entries = JSON.parse(data);
+                for (const entry of entries) {
+                    this.cache.set(entry.chunkId, entry);
+                }
+                console.error(`Loaded ${entries.length} cached embeddings`);
+            }
+        }
+        catch (error) {
+            console.error(`Warning: Could not load embedding cache: ${error}`);
+        }
+    }
+    /**
+     * Saves embedding cache to disk
+     */
+    saveCache() {
+        try {
+            const dir = path.dirname(this.options.cachePath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            const entries = Array.from(this.cache.values());
+            fs.writeFileSync(this.options.cachePath, JSON.stringify(entries, null, 2));
+            console.error(`Saved ${entries.length} embeddings to cache`);
+        }
+        catch (error) {
+            console.error(`Warning: Could not save embedding cache: ${error}`);
+        }
+    }
+    /**
+     * Generates hash of content for cache lookup
+     */
+    hashContent(content) {
+        return crypto.createHash("sha256").update(content).digest("hex");
+    }
+    /**
+     * Checks if embedding is cached
+     */
+    getCachedEmbedding(chunkId, content) {
+        if (!this.options.enableCache) {
+            return null;
+        }
+        const cached = this.cache.get(chunkId);
+        if (!cached) {
+            return null;
+        }
+        // Verify content hasn't changed
+        const contentHash = this.hashContent(content);
+        if (cached.contentHash !== contentHash) {
+            // Content changed, invalidate cache
+            this.cache.delete(chunkId);
+            return null;
+        }
+        // Verify model matches
+        if (cached.model !== this.options.model) {
+            return null;
+        }
+        return cached.vector;
+    }
+    /**
+     * Caches an embedding
+     */
+    cacheEmbedding(chunkId, content, vector) {
+        if (!this.options.enableCache) {
+            return;
+        }
+        const contentHash = this.hashContent(content);
+        this.cache.set(chunkId, {
+            chunkId,
+            contentHash,
+            vector,
+            model: this.options.model,
+            timestamp: Date.now(),
+        });
+    }
+    /**
+     * Sleeps for a specified duration (for retry backoff)
+     */
+    sleep(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    /**
+     * Generates embeddings for a batch of texts with retry logic
+     */
+    async generateBatchWithRetry(texts, maxRetries = 3) {
+        let lastError = null;
+        for (let attempt = 0; attempt < maxRetries; attempt++) {
+            try {
+                const response = await this.client.embeddings.create({
+                    model: this.options.model,
+                    input: texts,
+                    dimensions: this.options.dimensions,
+                });
+                return response.data.map((item) => item.embedding);
+            }
+            catch (error) {
+                lastError = error;
+                // Check if it's a rate limit error
+                if (error?.status === 429 || error?.code === "rate_limit_exceeded") {
+                    const backoffMs = Math.pow(2, attempt) * 1000; // Exponential backoff
+                    console.error(`Rate limit hit, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
+                    await this.sleep(backoffMs);
+                    continue;
+                }
+                // Check if it's a temporary error
+                if (error?.status >= 500 && error?.status < 600) {
+                    const backoffMs = Math.pow(2, attempt) * 1000;
+                    console.error(`Server error ${error.status}, retrying in ${backoffMs}ms (attempt ${attempt + 1}/${maxRetries})`);
+                    await this.sleep(backoffMs);
+                    continue;
+                }
+                // For other errors, don't retry
+                throw error;
+            }
+        }
+        // All retries failed
+        throw lastError || new Error("Failed to generate embeddings after retries");
+    }
+    /**
+     * Generates embedding for a single chunk
+     */
+    async generateEmbedding(chunkId, content) {
+        // Check cache first
+        const cached = this.getCachedEmbedding(chunkId, content);
+        if (cached) {
+            return {
+                chunkId,
+                vector: cached,
+                model: this.options.model,
+                tokens: 0, // Not tracked for cached
+            };
+        }
+        // Generate new embedding
+        const vectors = await this.generateBatchWithRetry([content]);
+        const vector = vectors[0];
+        // Cache the result
+        this.cacheEmbedding(chunkId, content, vector);
+        // Estimate tokens (rough approximation: ~4 chars per token)
+        const tokens = Math.ceil(content.length / 4);
+        return {
+            chunkId,
+            vector,
+            model: this.options.model,
+            tokens,
+        };
+    }
+    /**
+     * Generates embeddings for multiple chunks in batches
+     */
+    async generateBatchEmbeddings(chunks) {
+        const results = [];
+        const toGenerate = [];
+        // Check cache and collect chunks that need generation
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            const cached = this.getCachedEmbedding(chunk.id, chunk.content);
+            if (cached) {
+                results[i] = {
+                    chunkId: chunk.id,
+                    vector: cached,
+                    model: this.options.model,
+                    tokens: 0,
+                };
+            }
+            else {
+                toGenerate.push({ ...chunk, index: i });
+            }
+        }
+        console.error(`Generating embeddings: ${toGenerate.length} new, ${chunks.length - toGenerate.length} cached`);
+        // Process in batches
+        for (let i = 0; i < toGenerate.length; i += this.options.batchSize) {
+            const batch = toGenerate.slice(i, i + this.options.batchSize);
+            const batchTexts = batch.map((item) => item.content);
+            console.error(`Processing batch ${Math.floor(i / this.options.batchSize) + 1}/${Math.ceil(toGenerate.length / this.options.batchSize)}`);
+            try {
+                const vectors = await this.generateBatchWithRetry(batchTexts);
+                // Store results and cache
+                for (let j = 0; j < batch.length; j++) {
+                    const item = batch[j];
+                    const vector = vectors[j];
+                    // Cache the result
+                    this.cacheEmbedding(item.id, item.content, vector);
+                    // Estimate tokens
+                    const tokens = Math.ceil(item.content.length / 4);
+                    results[item.index] = {
+                        chunkId: item.id,
+                        vector,
+                        model: this.options.model,
+                        tokens,
+                    };
+                }
+            }
+            catch (error) {
+                console.error(`Error generating batch embeddings: ${error}`);
+                throw error;
+            }
+            // Small delay between batches to avoid rate limits
+            if (i + this.options.batchSize < toGenerate.length) {
+                await this.sleep(100);
+            }
+        }
+        // Save cache after batch processing
+        if (this.options.enableCache && toGenerate.length > 0) {
+            this.saveCache();
+        }
+        return results;
+    }
+    /**
+     * Generates embedding for a query (for search)
+     */
+    async generateQueryEmbedding(query) {
+        try {
+            const response = await this.client.embeddings.create({
+                model: this.options.model,
+                input: query,
+                dimensions: this.options.dimensions,
+            });
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            console.error(`Error generating query embedding: ${error}`);
+            throw error;
+        }
+    }
+    /**
+     * Clears the embedding cache
+     */
+    clearCache() {
+        this.cache.clear();
+        if (fs.existsSync(this.options.cachePath)) {
+            fs.unlinkSync(this.options.cachePath);
+        }
+        console.error("Embedding cache cleared");
+    }
+    /**
+     * Gets cache statistics
+     */
+    getCacheStats() {
+        const models = {};
+        for (const entry of this.cache.values()) {
+            models[entry.model] = (models[entry.model] || 0) + 1;
+        }
+        return {
+            size: this.cache.size,
+            models,
+        };
+    }
+}
+/**
+ * Creates an embedding service from environment variables
+ */
+export function createEmbeddingService() {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+        throw new Error("OPENAI_API_KEY environment variable is required. Get your API key from https://platform.openai.com/api-keys");
+    }
+    const options = {
+        model: process.env.MEMORYBANK_EMBEDDING_MODEL || "text-embedding-3-small",
+        dimensions: parseInt(process.env.MEMORYBANK_EMBEDDING_DIMENSIONS || "1536"),
+        enableCache: true,
+        cachePath: path.join(process.env.MEMORYBANK_STORAGE_PATH || ".memorybank", "embedding-cache.json"),
+    };
+    return new EmbeddingService(apiKey, options);
+}

package/dist/common/errors.js

@@ -0,0 +1,71 @@
+export class PlankaError extends Error {
+    status;
+    response;
+    constructor(message, status, response) {
+        super(message);
+        this.status = status;
+        this.response = response;
+        this.name = "PlankaError";
+    }
+}
+export class PlankaValidationError extends PlankaError {
+    constructor(message, status, response) {
+        super(message, status, response);
+        this.name = "PlankaValidationError";
+    }
+}
+export class PlankaResourceNotFoundError extends PlankaError {
+    constructor(resource) {
+        super(`Resource not found: ${resource}`, 404, {
+            message: `${resource} not found`,
+        });
+        this.name = "PlankaResourceNotFoundError";
+    }
+}
+export class PlankaAuthenticationError extends PlankaError {
+    constructor(message = "Authentication failed") {
+        super(message, 401, { message });
+        this.name = "PlankaAuthenticationError";
+    }
+}
+export class PlankaPermissionError extends PlankaError {
+    constructor(message = "Insufficient permissions") {
+        super(message, 403, { message });
+        this.name = "PlankaPermissionError";
+    }
+}
+export class PlankaRateLimitError extends PlankaError {
+    resetAt;
+    constructor(message = "Rate limit exceeded", resetAt) {
+        super(message, 429, { message, reset_at: resetAt.toISOString() });
+        this.resetAt = resetAt;
+        this.name = "PlankaRateLimitError";
+    }
+}
+export class PlankaConflictError extends PlankaError {
+    constructor(message) {
+        super(message, 409, { message });
+        this.name = "PlankaConflictError";
+    }
+}
+export function isPlankaError(error) {
+    return error instanceof PlankaError;
+}
+export function createPlankaError(status, response) {
+    switch (status) {
+        case 401:
+            return new PlankaAuthenticationError(response?.message);
+        case 403:
+            return new PlankaPermissionError(response?.message);
+        case 404:
+            return new PlankaResourceNotFoundError(response?.message || "Resource");
+        case 409:
+            return new PlankaConflictError(response?.message || "Conflict occurred");
+        case 422:
+            return new PlankaValidationError(response?.message || "Validation failed", status, response);
+        case 429:
+            return new PlankaRateLimitError(response?.message, new Date(response?.reset_at || Date.now() + 60000));
+        default:
+            return new PlankaError(response?.message || "Planka API error", status, response);
+    }
+}

package/dist/common/fileScanner.js

@@ -0,0 +1,261 @@
+/**
+ * @fileoverview File scanner for Memory Bank
+ * Scans workspace files respecting .gitignore and .memoryignore patterns
+ */
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+import ignoreLib from "ignore";
+// Handle ignore library export
+const ignore = typeof ignoreLib === 'function' ? ignoreLib : ignoreLib.default;
+// Language detection by file extension
+const LANGUAGE_MAP = {
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".py": "python",
+    ".java": "java",
+    ".c": "c",
+    ".cpp": "cpp",
+    ".cc": "cpp",
+    ".cxx": "cpp",
+    ".h": "c",
+    ".hpp": "cpp",
+    ".cs": "csharp",
+    ".go": "go",
+    ".rs": "rust",
+    ".rb": "ruby",
+    ".php": "php",
+    ".swift": "swift",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+    ".scala": "scala",
+    ".r": "r",
+    ".R": "r",
+    ".sql": "sql",
+    ".sh": "shell",
+    ".bash": "shell",
+    ".zsh": "shell",
+    ".fish": "shell",
+    ".md": "markdown",
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".xml": "xml",
+    ".html": "html",
+    ".htm": "html",
+    ".css": "css",
+    ".scss": "scss",
+    ".sass": "sass",
+    ".vue": "vue",
+    ".svelte": "svelte",
+};
+// Binary file extensions to skip
+const BINARY_EXTENSIONS = new Set([
+    ".exe", ".dll", ".so", ".dylib", ".bin",
+    ".pdf", ".jpg", ".jpeg", ".png", ".gif", ".svg", ".ico",
+    ".mp3", ".mp4", ".avi", ".mov", ".wav",
+    ".zip", ".tar", ".gz", ".rar", ".7z",
+    ".db", ".sqlite", ".sqlite3",
+    ".woff", ".woff2", ".ttf", ".eot",
+    ".pyc", ".class", ".o", ".a",
+]);
+/**
+ * Loads ignore patterns from .gitignore and .memoryignore files
+ */
+export function loadIgnorePatterns(rootPath) {
+    const ig = ignore();
+    // Always ignore .git directory and .memorybank storage
+    ig.add([".git", ".memorybank", "node_modules", "dist", "build", "out"]);
+    // Load .gitignore if exists
+    const gitignorePath = path.join(rootPath, ".gitignore");
+    if (fs.existsSync(gitignorePath)) {
+        try {
+            const gitignoreContent = fs.readFileSync(gitignorePath, "utf-8");
+            ig.add(gitignoreContent);
+            console.error(`Loaded .gitignore patterns from ${gitignorePath}`);
+        }
+        catch (error) {
+            console.error(`Warning: Could not read .gitignore: ${error}`);
+        }
+    }
+    // Load .memoryignore if exists
+    const memoryignorePath = path.join(rootPath, ".memoryignore");
+    if (fs.existsSync(memoryignorePath)) {
+        try {
+            const memoryignoreContent = fs.readFileSync(memoryignorePath, "utf-8");
+            ig.add(memoryignoreContent);
+            console.error(`Loaded .memoryignore patterns from ${memoryignorePath}`);
+        }
+        catch (error) {
+            console.error(`Warning: Could not read .memoryignore: ${error}`);
+        }
+    }
+    return ig;
+}
+/**
+ * Calculates SHA-256 hash of file content
+ */
+export function calculateFileHash(filePath) {
+    const content = fs.readFileSync(filePath);
+    return crypto.createHash("sha256").update(content).digest("hex");
+}
+/**
+ * Detects language from file extension
+ */
+export function detectLanguage(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return LANGUAGE_MAP[ext] || "unknown";
+}
+/**
+ * Checks if file is binary based on extension
+ */
+export function isBinaryFile(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    return BINARY_EXTENSIONS.has(ext);
+}
+/**
+ * Checks if file is a code file that should be indexed
+ */
+export function isCodeFile(filePath) {
+    if (isBinaryFile(filePath)) {
+        return false;
+    }
+    const ext = path.extname(filePath).toLowerCase();
+    // Check if it's a known code file
+    if (LANGUAGE_MAP[ext]) {
+        return true;
+    }
+    // Additional checks for files without extension or special cases
+    const basename = path.basename(filePath);
+    const codeFileNames = new Set([
+        "Makefile", "Dockerfile", "Jenkinsfile", "Vagrantfile",
+        "Rakefile", "Gemfile", "Podfile", "Fastfile",
+    ]);
+    return codeFileNames.has(basename);
+}
+/**
+ * Recursively scans directory for code files
+ */
+function scanDirectoryRecursive(dirPath, rootPath, ig, options, results) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dirPath, { withFileTypes: true });
+    }
+    catch (error) {
+        console.error(`Warning: Could not read directory ${dirPath}: ${error}`);
+        return;
+    }
+    for (const entry of entries) {
+        const fullPath = path.join(dirPath, entry.name);
+        const relativePath = path.relative(rootPath, fullPath);
+        // Skip hidden files if not included
+        if (!options.includeHidden && entry.name.startsWith(".")) {
+            continue;
+        }
+        // Check ignore patterns (use forward slashes for cross-platform compatibility)
+        const relativePathForward = relativePath.split(path.sep).join("/");
+        if (ig.ignores(relativePathForward)) {
+            continue;
+        }
+        if (entry.isDirectory()) {
+            if (options.recursive) {
+                scanDirectoryRecursive(fullPath, rootPath, ig, options, results);
+            }
+        }
+        else if (entry.isFile()) {
+            try {
+                // Check if it's a code file
+                if (!isCodeFile(fullPath)) {
+                    continue;
+                }
+                const stats = fs.statSync(fullPath);
+                // Check file size limit
+                if (stats.size > options.maxFileSize) {
+                    console.error(`Skipping large file (${stats.size} bytes): ${relativePath}`);
+                    continue;
+                }
+                // Calculate hash and collect metadata
+                const hash = calculateFileHash(fullPath);
+                const language = detectLanguage(fullPath);
+                const extension = path.extname(fullPath);
+                results.push({
+                    path: relativePath,
+                    absolutePath: fullPath,
+                    hash,
+                    size: stats.size,
+                    mtime: stats.mtime,
+                    language,
+                    extension,
+                });
+            }
+            catch (error) {
+                console.error(`Warning: Could not process file ${fullPath}: ${error}`);
+            }
+        }
+    }
+}
+/**
+ * Scans workspace for code files
+ */
+export function scanFiles(options) {
+    const fullOptions = {
+        rootPath: options.rootPath,
+        recursive: options.recursive !== undefined ? options.recursive : true,
+        includeHidden: options.includeHidden !== undefined ? options.includeHidden : false,
+        maxFileSize: options.maxFileSize || 10 * 1024 * 1024, // 10MB default
+    };
+    // Validate root path
+    if (!fs.existsSync(fullOptions.rootPath)) {
+        throw new Error(`Root path does not exist: ${fullOptions.rootPath}`);
+    }
+    const stats = fs.statSync(fullOptions.rootPath);
+    if (!stats.isDirectory()) {
+        throw new Error(`Root path is not a directory: ${fullOptions.rootPath}`);
+    }
+    console.error(`Scanning files in: ${fullOptions.rootPath}`);
+    // Load ignore patterns
+    const ig = loadIgnorePatterns(fullOptions.rootPath);
+    // Scan files
+    const results = [];
+    scanDirectoryRecursive(fullOptions.rootPath, fullOptions.rootPath, ig, fullOptions, results);
+    console.error(`Found ${results.length} code files to index`);
+    return results;
+}
+/**
+ * Scans a single file and returns its metadata
+ */
+export function scanSingleFile(filePath, rootPath) {
+    try {
+        if (!fs.existsSync(filePath)) {
+            throw new Error(`File does not exist: ${filePath}`);
+        }
+        const stats = fs.statSync(filePath);
+        if (!stats.isFile()) {
+            throw new Error(`Path is not a file: ${filePath}`);
+        }
+        if (!isCodeFile(filePath)) {
+            return null;
+        }
+        const hash = calculateFileHash(filePath);
+        const language = detectLanguage(filePath);
+        const extension = path.extname(filePath);
+        const relativePath = path.relative(rootPath, filePath);
+        return {
+            path: relativePath,
+            absolutePath: filePath,
+            hash,
+            size: stats.size,
+            mtime: stats.mtime,
+            language,
+            extension,
+        };
+    }
+    catch (error) {
+        console.error(`Error scanning file ${filePath}: ${error}`);
+        return null;
+    }
+}