@ruvector/rvf-mcp-server 0.1.5 → 0.2.0

package/dist/server.js

@@ -1,18 +1,35 @@
 /**
  * RVF MCP Server — core server implementation.
  *
- * Registers all RVF tools, resources, and prompts with the MCP SDK.
+ * Registers all RVF tools, resources, and prompts with the MCP SDK. Stores
+ * are real on-disk `.rvf` files (via @ruvector/rvf) confined to a data dir,
+ * so queries are index-backed and data survives server restarts.
  */
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { RvfDatabase } from '@ruvector/rvf';
+import * as fs from 'node:fs';
 import { z } from 'zod';
+import { META_SUFFIX, MetadataLog, defaultDataDir, matchesFilter, resolveStorePath, } from './stores.js';
 /** Upper bounds on model-supplied sizes (tool arguments are untrusted). */
 export const LIMITS = {
     maxDimensions: 65_536,
     maxK: 10_000,
     maxBatch: 10_000,
 };
+/** Auto-compaction after ingest: file larger than `minBytes` AND `ratio` x live vector bytes. */
+export const AUTO_COMPACT = {
+    minBytes: 64 * 1024 * 1024,
+    ratio: 8,
+};
 const vectorSchema = z.array(z.number()).max(LIMITS.maxDimensions);
 const metadataSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
+const reply = (body) => ({
+    content: [{ type: 'text', text: JSON.stringify(body, null, 2) }],
+});
+const fail = (message) => ({
+    content: [{ type: 'text', text: `Error: ${message}` }],
+});
+const errMessage = (err) => (err instanceof Error ? err.message : String(err));
 // ─── Server ─────────────────────────────────────────────────────────────────
 export class RvfMcpServer {
     mcp;
@@ -25,6 +42,8 @@ export class RvfMcpServer {
             version: options?.version ?? '0.1.0',
             defaultDimensions: options?.defaultDimensions ?? 128,
             maxStores: options?.maxStores ?? 64,
+            dataDir: options?.dataDir ?? defaultDataDir(),
+            autoCompact: options?.autoCompact ?? AUTO_COMPACT,
         };
         this.mcp = new McpServer({ name: this.opts.name, version: this.opts.version }, {
             capabilities: {
@@ -37,88 +56,111 @@ export class RvfMcpServer {
         this.registerResources();
         this.registerPrompts();
     }
+    /** Directory that holds this server's stores. */
+    get dataDir() {
+        return this.opts.dataDir;
+    }
+    withStore(storeId, write, fn) {
+        const handle = this.stores.get(storeId);
+        if (!handle)
+            return Promise.resolve(fail(`store ${storeId} not found`));
+        if (write && handle.readOnly)
+            return Promise.resolve(fail('store is read-only'));
+        return fn(handle).catch((err) => fail(errMessage(err)));
+    }
+    register(handle) {
+        const full = { ...handle, id: `store_${this.nextId++}`, createdAt: Date.now() };
+        this.stores.set(full.id, full);
+        return full;
+    }
+    /** k-NN with exact-match metadata filtering applied to an over-fetched result. */
+    async search(h, vector, k, filter) {
+        const q = new Float32Array(vector);
+        if (!filter) {
+            const results = await h.db.query(q, k);
+            return { results, scanned: results.length };
+        }
+        const candidates = [...h.meta.entries.values()].filter((m) => matchesFilter(m, filter)).length;
+        if (candidates === 0)
+            return { results: [], scanned: 0 };
+        const total = (await h.db.status()).totalVectors;
+        let fetch = Math.min(total, Math.max(k * 4, 64));
+        for (;;) {
+            const raw = await h.db.query(q, fetch);
+            const hits = raw.filter((r) => matchesFilter(h.meta.entries.get(r.id), filter));
+            if (hits.length >= Math.min(k, candidates) || fetch >= total) {
+                return { results: hits.slice(0, k), scanned: raw.length };
+            }
+            fetch = Math.min(total, fetch * 4);
+        }
+    }
     // ─── Tool Registration ──────────────────────────────────────────────────
     registerTools() {
         // ── rvf_create_store ──────────────────────────────────────────────────
-        this.mcp.tool('rvf_create_store', 'Create a new RVF vector store at the given path', {
-            path: z.string().describe('File path for the new .rvf store'),
+        this.mcp.tool('rvf_create_store', 'Create a new RVF vector store at the given path (relative to the server data dir)', {
+            path: z.string().describe('File path for the new .rvf store, relative to the data dir'),
             dimensions: z.number().int().positive().max(LIMITS.maxDimensions).describe('Vector dimensionality'),
             metric: z.enum(['l2', 'cosine', 'dotproduct']).default('l2').describe('Distance metric'),
         }, async ({ path, dimensions, metric }) => {
             if (this.stores.size >= this.opts.maxStores) {
-                return { content: [{ type: 'text', text: `Error: max stores (${this.opts.maxStores}) reached` }] };
+                return fail(`max stores (${this.opts.maxStores}) reached`);
+            }
+            try {
+                const file = resolveStorePath(this.opts.dataDir, path, true);
+                const meta = new MetadataLog(file + META_SUFFIX, false);
+                if (fs.existsSync(meta.file))
+                    throw new Error(`${path} already has metadata; use rvf_open_store`);
+                const db = await RvfDatabase.create(file, { dimensions, metric: metric }, 'node');
+                meta.header = { dimensions, metric };
+                meta.rewrite();
+                const h = this.register({ path, file, dimensions, metric, readOnly: false, db, meta });
+                return reply({ storeId: h.id, path, dimensions, metric, status: 'created' });
+            }
+            catch (err) {
+                return fail(errMessage(err));
             }
-            const id = `store_${this.nextId++}`;
-            const handle = {
-                id,
-                path,
-                dimensions,
-                metric,
-                readOnly: false,
-                vectors: new Map(),
-                createdAt: Date.now(),
-            };
-            this.stores.set(id, handle);
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            storeId: id,
-                            path,
-                            dimensions,
-                            metric,
-                            status: 'created',
-                        }, null, 2),
-                    }],
-            };
         });
         // ── rvf_open_store ────────────────────────────────────────────────────
-        this.mcp.tool('rvf_open_store', 'Open an existing RVF store for reading and writing', {
-            path: z.string().describe('Path to existing .rvf file'),
+        this.mcp.tool('rvf_open_store', 'Open an existing RVF store (relative to the server data dir) for reading and writing', {
+            path: z.string().describe('Path to existing .rvf file, relative to the data dir'),
             readOnly: z.boolean().default(false).describe('Open in read-only mode'),
         }, async ({ path, readOnly }) => {
             if (this.stores.size >= this.opts.maxStores) {
-                return { content: [{ type: 'text', text: `Error: max stores (${this.opts.maxStores}) reached` }] };
+                return fail(`max stores (${this.opts.maxStores}) reached`);
+            }
+            try {
+                const file = resolveStorePath(this.opts.dataDir, path, false);
+                if (!fs.existsSync(file))
+                    throw new Error(`store ${path} does not exist`);
+                const db = readOnly
+                    ? await RvfDatabase.openReadonly(file, 'node')
+                    : await RvfDatabase.open(file, 'node');
+                const meta = new MetadataLog(file + META_SUFFIX, readOnly);
+                try {
+                    meta.load();
+                }
+                catch (err) {
+                    await db.close();
+                    throw err;
+                }
+                const dimensions = await db.dimension();
+                const metric = meta.header?.metric ?? 'l2';
+                meta.header ??= { dimensions, metric };
+                const h = this.register({ path, file, dimensions, metric, readOnly, db, meta });
+                return reply({ storeId: h.id, path, readOnly, dimensions, metric, status: 'opened' });
+            }
+            catch (err) {
+                return fail(errMessage(err));
             }
-            const id = `store_${this.nextId++}`;
-            const handle = {
-                id,
-                path,
-                dimensions: this.opts.defaultDimensions,
-                metric: 'l2',
-                readOnly,
-                vectors: new Map(),
-                createdAt: Date.now(),
-            };
-            this.stores.set(id, handle);
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            storeId: id,
-                            path,
-                            readOnly,
-                            status: 'opened',
-                        }, null, 2),
-                    }],
-            };
         });
         // ── rvf_close_store ───────────────────────────────────────────────────
         this.mcp.tool('rvf_close_store', 'Close an open RVF store, releasing the writer lock', {
             storeId: z.string().describe('Store ID returned by create/open'),
-        }, async ({ storeId }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
+        }, async ({ storeId }) => this.withStore(storeId, false, async (h) => {
             this.stores.delete(storeId);
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({ storeId, status: 'closed', path: handle.path }, null, 2),
-                    }],
-            };
-        });
+            await this.closeHandle(h);
+            return reply({ storeId, status: 'closed', path: h.path });
+        }));
         // ── rvf_ingest ────────────────────────────────────────────────────────
         this.mcp.tool('rvf_ingest', 'Insert vectors into an RVF store', {
             storeId: z.string().describe('Target store ID'),
@@ -128,38 +170,46 @@ export class RvfMcpServer {
                 metadata: metadataSchema.optional()
                     .describe('Optional metadata key-value pairs'),
             })).max(LIMITS.maxBatch).describe('Vectors to insert'),
-        }, async ({ storeId, entries }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
-            if (handle.readOnly) {
-                return { content: [{ type: 'text', text: 'Error: store is read-only' }] };
-            }
-            let accepted = 0;
+        }, async ({ storeId, entries }) => this.withStore(storeId, true, async (h) => {
+            // Last write wins for duplicate ids within a batch (upsert semantics).
+            const batch = new Map();
             let rejected = 0;
-            for (const entry of entries) {
-                if (entry.vector.length !== handle.dimensions) {
+            for (const e of entries) {
+                if (e.vector.length !== h.dimensions)
                     rejected++;
-                    continue;
+                else
+                    batch.set(e.id, { vector: e.vector, metadata: e.metadata });
+            }
+            if (batch.size > 0) {
+                await h.db.ingestBatch([...batch].map(([id, e]) => ({ id, vector: new Float32Array(e.vector) })));
+                const sets = [];
+                const cleared = [];
+                for (const [id, e] of batch) {
+                    if (e.metadata && Object.keys(e.metadata).length > 0)
+                        sets.push([id, e.metadata]);
+                    else
+                        cleared.push(id);
                 }
-                handle.vectors.set(entry.id, {
-                    vector: entry.vector,
-                    metadata: entry.metadata,
-                });
-                accepted++;
+                h.meta.apply(sets, cleared);
             }
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            accepted,
-                            rejected,
-                            totalVectors: handle.vectors.size,
-                        }, null, 2),
-                    }],
-            };
-        });
+            let status = await h.db.status();
+            // rvf-runtime appends a manifest listing every segment on each ingest,
+            // so many small ingest calls grow the file quadratically (2,500 single
+            // ingests -> 231 MB for 1.3 MB of vectors). Bound it by compacting
+            // once the file is both large and mostly overhead. Compaction drops
+            // the in-memory HNSW, which is rebuilt lazily on the next query.
+            const liveBytes = status.totalVectors * h.dimensions * 4;
+            let autoCompacted = false;
+            // Stat the file directly: status().fileSizeBytes reads 0 in @ruvector/rvf <= 0.4.2.
+            const fileBytes = fs.statSync(h.file).size;
+            const { minBytes, ratio } = this.opts.autoCompact;
+            if (fileBytes > minBytes && fileBytes > ratio * Math.max(liveBytes, 1)) {
+                await h.db.compact();
+                status = await h.db.status();
+                autoCompacted = true;
+            }
+            return reply({ accepted: batch.size, rejected, totalVectors: status.totalVectors, autoCompacted });
+        }));
         // ── rvf_query ─────────────────────────────────────────────────────────
         this.mcp.tool('rvf_query', 'k-NN vector similarity search', {
             storeId: z.string().describe('Store ID to query'),
@@ -167,182 +217,122 @@ export class RvfMcpServer {
             k: z.number().int().positive().max(LIMITS.maxK).default(10).describe('Number of nearest neighbors'),
             filter: metadataSchema.optional()
                 .describe('Metadata filter (exact match on fields)'),
-        }, async ({ storeId, vector, k, filter }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
+        }, async ({ storeId, vector, k, filter }) => this.withStore(storeId, false, async (h) => {
+            if (vector.length !== h.dimensions) {
+                return fail(`dimension mismatch (query=${vector.length}, store=${h.dimensions})`);
             }
-            if (vector.length !== handle.dimensions) {
-                return {
-                    content: [{
-                            type: 'text',
-                            text: `Error: dimension mismatch (query=${vector.length}, store=${handle.dimensions})`,
-                        }],
-                };
-            }
-            // Compute distances and sort
-            const results = [];
-            for (const [id, entry] of handle.vectors) {
-                // Apply metadata filter if provided
-                if (filter && entry.metadata) {
-                    let match = true;
-                    for (const [key, val] of Object.entries(filter)) {
-                        if (entry.metadata[key] !== val) {
-                            match = false;
-                            break;
-                        }
-                    }
-                    if (!match)
-                        continue;
-                }
-                else if (filter && !entry.metadata) {
-                    continue;
-                }
-                const dist = computeDistance(vector, entry.vector, handle.metric);
-                results.push({ id, distance: dist });
-            }
-            results.sort((a, b) => a.distance - b.distance);
-            const topK = results.slice(0, k);
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            results: topK,
-                            totalScanned: handle.vectors.size,
-                            metric: handle.metric,
-                        }, null, 2),
-                    }],
-            };
-        });
+            const { results, scanned } = await this.search(h, vector, k, filter);
+            return reply({ results, totalScanned: scanned, metric: h.metric });
+        }));
         // ── rvf_delete ────────────────────────────────────────────────────────
         this.mcp.tool('rvf_delete', 'Delete vectors by their IDs', {
             storeId: z.string().describe('Store ID'),
             ids: z.array(z.string()).max(LIMITS.maxBatch).describe('Vector IDs to delete'),
-        }, async ({ storeId, ids }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
-            if (handle.readOnly) {
-                return { content: [{ type: 'text', text: 'Error: store is read-only' }] };
-            }
-            let deleted = 0;
-            for (const id of ids) {
-                if (handle.vectors.delete(id))
-                    deleted++;
-            }
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({ deleted, remaining: handle.vectors.size }, null, 2),
-                    }],
-            };
-        });
+        }, async ({ storeId, ids }) => this.withStore(storeId, true, async (h) => {
+            const { deleted } = await h.db.delete(ids);
+            h.meta.apply([], ids);
+            const status = await h.db.status();
+            return reply({ deleted, remaining: status.totalVectors });
+        }));
         // ── rvf_delete_filter ─────────────────────────────────────────────────
         this.mcp.tool('rvf_delete_filter', 'Delete vectors matching a metadata filter', {
             storeId: z.string().describe('Store ID'),
             filter: metadataSchema
                 .describe('Metadata filter — all matching vectors will be deleted'),
-        }, async ({ storeId, filter }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
-            if (handle.readOnly) {
-                return { content: [{ type: 'text', text: 'Error: store is read-only' }] };
-            }
-            let deleted = 0;
-            for (const [id, entry] of handle.vectors) {
-                if (!entry.metadata)
-                    continue;
-                let match = true;
-                for (const [key, val] of Object.entries(filter)) {
-                    if (entry.metadata[key] !== val) {
-                        match = false;
-                        break;
-                    }
-                }
-                if (match) {
-                    handle.vectors.delete(id);
-                    deleted++;
-                }
-            }
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({ deleted, remaining: handle.vectors.size }, null, 2),
-                    }],
-            };
-        });
+        }, async ({ storeId, filter }) => this.withStore(storeId, true, async (h) => {
+            const ids = [...h.meta.entries].filter(([, m]) => matchesFilter(m, filter)).map(([id]) => id);
+            const deleted = ids.length > 0 ? (await h.db.delete(ids)).deleted : 0;
+            h.meta.apply([], ids);
+            const status = await h.db.status();
+            return reply({ deleted, remaining: status.totalVectors });
+        }));
         // ── rvf_compact ───────────────────────────────────────────────────────
         this.mcp.tool('rvf_compact', 'Compact store to reclaim dead space from deleted vectors', {
             storeId: z.string().describe('Store ID'),
-        }, async ({ storeId }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            storeId,
-                            compacted: true,
-                            totalVectors: handle.vectors.size,
-                        }, null, 2),
-                    }],
-            };
-        });
+        }, async ({ storeId }) => this.withStore(storeId, true, async (h) => {
+            const result = await h.db.compact();
+            h.meta.rewrite();
+            const status = await h.db.status();
+            return reply({
+                storeId,
+                compacted: true,
+                segmentsCompacted: result.segmentsCompacted,
+                bytesReclaimed: result.bytesReclaimed,
+                totalVectors: status.totalVectors,
+            });
+        }));
         // ── rvf_status ────────────────────────────────────────────────────────
         this.mcp.tool('rvf_status', 'Get the current status of an RVF store', {
             storeId: z.string().describe('Store ID'),
-        }, async ({ storeId }) => {
-            const handle = this.stores.get(storeId);
-            if (!handle) {
-                return { content: [{ type: 'text', text: `Error: store ${storeId} not found` }] };
-            }
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({
-                            storeId: handle.id,
-                            path: handle.path,
-                            dimensions: handle.dimensions,
-                            metric: handle.metric,
-                            readOnly: handle.readOnly,
-                            totalVectors: handle.vectors.size,
-                            createdAt: new Date(handle.createdAt).toISOString(),
-                        }, null, 2),
-                    }],
-            };
-        });
+        }, async ({ storeId }) => this.withStore(storeId, false, async (h) => {
+            const status = await h.db.status();
+            return reply({
+                storeId: h.id,
+                path: h.path,
+                dimensions: h.dimensions,
+                metric: h.metric,
+                readOnly: h.readOnly,
+                totalVectors: status.totalVectors,
+                totalSegments: status.totalSegments,
+                fileSizeBytes: status.fileSizeBytes,
+                epoch: status.epoch,
+                createdAt: new Date(h.createdAt).toISOString(),
+            });
+        }));
         // ── rvf_list_stores ───────────────────────────────────────────────────
         this.mcp.tool('rvf_list_stores', 'List all open RVF stores', {}, async () => {
-            const list = Array.from(this.stores.values()).map((h) => ({
+            const list = await this.listStores();
+            return reply({
+                stores: list.map((h) => ({
+                    storeId: h.storeId,
+                    path: h.path,
+                    dimensions: h.dimensions,
+                    metric: h.metric,
+                    totalVectors: h.totalVectors,
+                    readOnly: h.readOnly,
+                })),
+                count: list.length,
+            });
+        });
+    }
+    async listStores() {
+        const out = [];
+        for (const h of this.stores.values()) {
+            let totalVectors = 0;
+            try {
+                totalVectors = (await h.db.status()).totalVectors;
+            }
+            catch {
+                // report the store even if its status call fails
+            }
+            out.push({
                 storeId: h.id,
                 path: h.path,
                 dimensions: h.dimensions,
                 metric: h.metric,
-                totalVectors: h.vectors.size,
                 readOnly: h.readOnly,
-            }));
-            return {
-                content: [{
-                        type: 'text',
-                        text: JSON.stringify({ stores: list, count: list.length }, null, 2),
-                    }],
-            };
-        });
+                totalVectors,
+            });
+        }
+        return out;
+    }
+    async closeHandle(h) {
+        try {
+            h.meta.rewrite();
+        }
+        finally {
+            await h.db.close();
+        }
     }
     // ─── Resource Registration ──────────────────────────────────────────────
     registerResources() {
         // List of open stores
         this.mcp.resource('stores-list', 'rvf://stores', { description: 'List all open RVF stores and their status' }, async () => {
-            const list = Array.from(this.stores.values()).map((h) => ({
-                storeId: h.id,
+            const list = (await this.listStores()).map((h) => ({
+                storeId: h.storeId,
                 path: h.path,
                 dimensions: h.dimensions,
-                totalVectors: h.vectors.size,
+                totalVectors: h.totalVectors,
             }));
             return {
                 contents: [{
@@ -388,52 +378,25 @@ export class RvfMcpServer {
         await this.mcp.connect(transport);
     }
     async close() {
-        // Close all stores
+        // Close every store so native writer locks are released and metadata
+        // logs are compacted before the transport goes away.
+        const handles = [...this.stores.values()];
         this.stores.clear();
+        let failure;
+        for (const h of handles) {
+            try {
+                await this.closeHandle(h);
+            }
+            catch (err) {
+                failure ??= err;
+            }
+        }
         await this.mcp.close();
+        if (failure)
+            throw failure;
     }
     get storeCount() {
         return this.stores.size;
     }
 }
-// ─── Distance Functions ─────────────────────────────────────────────────────
-function computeDistance(a, b, metric) {
-    switch (metric) {
-        case 'cosine':
-            return cosineDistance(a, b);
-        case 'dotproduct':
-            return -dotProduct(a, b);
-        default: // l2
-            return l2Distance(a, b);
-    }
-}
-function l2Distance(a, b) {
-    let sum = 0;
-    for (let i = 0; i < a.length; i++) {
-        const d = a[i] - b[i];
-        sum += d * d;
-    }
-    return sum;
-}
-function dotProduct(a, b) {
-    let sum = 0;
-    for (let i = 0; i < a.length; i++) {
-        sum += a[i] * b[i];
-    }
-    return sum;
-}
-function cosineDistance(a, b) {
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dot += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    if (denom === 0)
-        return 1;
-    return 1 - dot / denom;
-}
 //# sourceMappingURL=server.js.map