@aeriondyseti/vector-memory-mcp 2.2.6 → 2.3.0-dev.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.2.6",
+  "version": "2.3.0-dev.1",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "server/index.ts",
@@ -9,7 +9,6 @@
   },
   "files": [
     "server",
-    "scripts",
     "README.md",
     "LICENSE"
   ],
@@ -47,18 +46,18 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@huggingface/transformers": "^3.8.0",
+    "@huggingface/tokenizers": "^0.1.3",
     "@lancedb/lancedb": "^0.26.2",
     "@modelcontextprotocol/sdk": "^1.0.0",
     "arg": "^5.0.2",
-    "hono": "^4.11.3"
+    "hono": "^4.11.3",
+    "onnxruntime-node": "^1.21.0"
   },
   "devDependencies": {
     "@types/bun": "latest",
     "typescript": "^5.0.0"
   },
   "trustedDependencies": [
-    "protobufjs",
-    "sharp"
+    "protobufjs"
   ]
 }

package/server/core/embeddings.service.ts

@@ -1,9 +1,17 @@
-import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
+import * as ort from "onnxruntime-node";
+import { Tokenizer } from "@huggingface/tokenizers";
+import { join, dirname } from "path";
+import { mkdir } from "fs/promises";
+import { existsSync } from "fs";
+
+const HF_CDN = "https://huggingface.co";
+const MAX_SEQ_LENGTH = 512;
 
 export class EmbeddingsService {
   private modelName: string;
-  private extractor: FeatureExtractionPipeline | null = null;
-  private initPromise: Promise<FeatureExtractionPipeline> | null = null;
+  private session: ort.InferenceSession | null = null;
+  private tokenizer: Tokenizer | null = null;
+  private initPromise: Promise<void> | null = null;
   private _dimension: number;
 
   constructor(modelName: string, dimension: number) {
@@ -15,27 +23,71 @@ export class EmbeddingsService {
     return this._dimension;
   }
 
-  private async getExtractor(): Promise<FeatureExtractionPipeline> {
-    if (this.extractor) {
-      return this.extractor;
-    }
-
+  private async initialize(): Promise<void> {
+    if (this.session) return;
     if (!this.initPromise) {
-      this.initPromise = pipeline(
-        "feature-extraction",
-        this.modelName,
-        { dtype: "fp32" } as any
-      ) as Promise<FeatureExtractionPipeline>;
+      this.initPromise = this._init();
     }
+    await this.initPromise;
+  }
+
+  private get cacheDir(): string {
+    const packageRoot = join(dirname(Bun.main), "..");
+    return join(packageRoot, ".cache", "models", this.modelName);
+  }
 
-    this.extractor = await this.initPromise;
-    return this.extractor;
+  private async downloadIfMissing(fileName: string): Promise<string> {
+    const filePath = join(this.cacheDir, fileName);
+    if (existsSync(filePath)) return filePath;
+
+    const url = `${HF_CDN}/${this.modelName}/resolve/main/${fileName}`;
+    await mkdir(dirname(filePath), { recursive: true });
+    const response = await fetch(url);
+    if (!response.ok) throw new Error(`Failed to download ${url}: ${response.status}`);
+    const buffer = await response.arrayBuffer();
+    await Bun.write(filePath, buffer);
+    return filePath;
+  }
+
+  private async _init(): Promise<void> {
+    const modelPath = await this.downloadIfMissing("onnx/model.onnx");
+    const tokenizerJsonPath = await this.downloadIfMissing("tokenizer.json");
+    const tokenizerConfigPath = await this.downloadIfMissing("tokenizer_config.json");
+
+    this.session = await ort.InferenceSession.create(modelPath, {
+      executionProviders: ["cpu"],
+    });
+
+    const tokenizerJson = await Bun.file(tokenizerJsonPath).json();
+    const tokenizerConfig = await Bun.file(tokenizerConfigPath).json();
+    this.tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
   }
 
   async embed(text: string): Promise<number[]> {
-    const extractor = await this.getExtractor();
-    const output = await extractor(text, { pooling: "mean", normalize: true });
-    return Array.from(output.data as Float32Array);
+    await this.initialize();
+
+    const encoded = this.tokenizer!.encode(text);
+
+    // Truncate to model's max sequence length
+    const seqLen = Math.min(encoded.ids.length, MAX_SEQ_LENGTH);
+    const ids = encoded.ids.slice(0, seqLen);
+    const mask = encoded.attention_mask.slice(0, seqLen);
+
+    const inputIds = BigInt64Array.from(ids.map(BigInt));
+    const attentionMask = BigInt64Array.from(mask.map(BigInt));
+    const tokenTypeIds = new BigInt64Array(seqLen); // zeros for single-sequence input
+
+    const feeds: Record<string, ort.Tensor> = {
+      input_ids: new ort.Tensor("int64", inputIds, [1, seqLen]),
+      attention_mask: new ort.Tensor("int64", attentionMask, [1, seqLen]),
+      token_type_ids: new ort.Tensor("int64", tokenTypeIds, [1, seqLen]),
+    };
+
+    const output = await this.session!.run(feeds);
+    const lastHidden = output["last_hidden_state"];
+
+    const pooled = this.meanPool(lastHidden.data as Float32Array, mask, seqLen);
+    return this.normalize(pooled);
   }
 
   async embedBatch(texts: string[]): Promise<number[][]> {
@@ -45,4 +97,29 @@ export class EmbeddingsService {
     }
     return results;
   }
+
+  private meanPool(data: Float32Array, mask: number[], seqLen: number): number[] {
+    const dim = this._dimension;
+    const pooled = new Array(dim).fill(0);
+    let maskSum = 0;
+    for (let t = 0; t < seqLen; t++) {
+      if (mask[t]) {
+        maskSum += 1;
+        for (let d = 0; d < dim; d++) {
+          pooled[d] += data[t * dim + d];
+        }
+      }
+    }
+    for (let d = 0; d < dim; d++) {
+      pooled[d] /= maskSum;
+    }
+    return pooled;
+  }
+
+  private normalize(vec: number[]): number[] {
+    let norm = 0;
+    for (const v of vec) norm += v * v;
+    norm = Math.sqrt(norm);
+    return vec.map(v => v / norm);
+  }
 }

package/server/core/migrations.ts

@@ -1,4 +1,6 @@
 import type { Database } from "bun:sqlite";
+import type { EmbeddingsService } from "./embeddings.service.js";
+import { serializeVector } from "./sqlite-utils.js";
 
 /**
  * Pre-migration step: remove vec0 virtual table entries from sqlite_master
@@ -113,3 +115,149 @@ export function runMigrations(db: Database): void {
   db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_role ON conversation_history(role)`);
   db.exec(`CREATE INDEX IF NOT EXISTS idx_conversation_created_at ON conversation_history(created_at)`);
 }
+
+/**
+ * Backfill missing vectors in memories_vec and conversation_history_vec.
+ *
+ * After the vec0-to-BLOB migration, existing rows may lack vector embeddings.
+ * This re-embeds their content and inserts into the _vec tables.
+ * Idempotent: skips rows that already have vectors. Fast no-op when fully backfilled.
+ */
+export async function backfillVectors(
+  db: Database,
+  embeddings: EmbeddingsService,
+): Promise<void> {
+  // Fast sentinel check: skip the LEFT JOIN queries entirely when backfill is done
+  const sentinel = db
+    .prepare("SELECT 1 FROM memories_vec LIMIT 1")
+    .get();
+  const memoriesExist = db.prepare("SELECT 1 FROM memories LIMIT 1").get();
+  const convosExist = db.prepare("SELECT 1 FROM conversation_history LIMIT 1").get();
+
+  // If vec tables have data and source tables have data, backfill is likely complete.
+  // Only run the expensive LEFT JOIN when there's reason to suspect gaps.
+  const convoSentinel = db
+    .prepare("SELECT 1 FROM conversation_history_vec LIMIT 1")
+    .get();
+  const mayNeedMemoryBackfill = memoriesExist && !sentinel;
+  const mayNeedConvoBackfill = convosExist && !convoSentinel;
+
+  // If both vec tables are populated, do a quick count check to confirm
+  if (!mayNeedMemoryBackfill && !mayNeedConvoBackfill) {
+    if (memoriesExist) {
+      const gap = db.prepare(
+        `SELECT 1 FROM memories m LEFT JOIN memories_vec v ON m.id = v.id
+         WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
+      ).get();
+      if (!gap && convosExist) {
+        const convoGap = db.prepare(
+          `SELECT 1 FROM conversation_history c LEFT JOIN conversation_history_vec v ON c.id = v.id
+           WHERE v.id IS NULL OR length(v.vector) = 0 LIMIT 1`,
+        ).get();
+        if (!convoGap) return;
+      } else if (!gap && !convosExist) {
+        return;
+      }
+    } else {
+      return; // No data at all
+    }
+  }
+
+  // ── Memories ──────────────────────────────────────────────────────
+  const missingMemories = db
+    .prepare(
+      `SELECT m.id, m.content, json_extract(m.metadata, '$.type') AS type
+       FROM memories m
+       LEFT JOIN memories_vec v ON m.id = v.id
+       WHERE v.id IS NULL OR length(v.vector) = 0`,
+    )
+    .all() as Array<{ id: string; content: string; type: string | null }>;
+
+  if (missingMemories.length > 0) {
+    console.error(
+      `[vector-memory-mcp] Backfilling vectors for ${missingMemories.length} memories...`,
+    );
+
+    const insertVec = db.prepare(
+      "INSERT OR REPLACE INTO memories_vec (id, vector) VALUES (?, ?)",
+    );
+
+    const zeroVector = serializeVector(
+      new Array(embeddings.dimension).fill(0),
+    );
+
+    // Separate waypoints from content that needs embedding
+    const toEmbed = missingMemories.filter((r) => r.type !== "waypoint");
+    const waypoints = missingMemories.filter((r) => r.type === "waypoint");
+
+    // Batch embed all non-waypoint content
+    const vectors = toEmbed.length > 0
+      ? await embeddings.embedBatch(toEmbed.map((r) => r.content))
+      : [];
+
+    db.exec("BEGIN");
+    try {
+      for (const row of waypoints) {
+        insertVec.run(row.id, zeroVector);
+      }
+      for (let i = 0; i < toEmbed.length; i++) {
+        insertVec.run(toEmbed[i].id, serializeVector(vectors[i]));
+      }
+      db.exec("COMMIT");
+    } catch (e) {
+      db.exec("ROLLBACK");
+      throw e;
+    }
+
+    console.error(
+      `[vector-memory-mcp] Backfilled ${missingMemories.length} memory vectors`,
+    );
+  }
+
+  // ── Conversation history ──────────────────────────────────────────
+  const missingConvos = db
+    .prepare(
+      `SELECT c.id, c.content
+       FROM conversation_history c
+       LEFT JOIN conversation_history_vec v ON c.id = v.id
+       WHERE v.id IS NULL OR length(v.vector) = 0`,
+    )
+    .all() as Array<{ id: string; content: string }>;
+
+  if (missingConvos.length > 0) {
+    console.error(
+      `[vector-memory-mcp] Backfilling vectors for ${missingConvos.length} conversation chunks...`,
+    );
+
+    const insertConvoVec = db.prepare(
+      "INSERT OR REPLACE INTO conversation_history_vec (id, vector) VALUES (?, ?)",
+    );
+
+    // Batch embed in chunks of 32
+    const BATCH_SIZE = 32;
+    db.exec("BEGIN");
+    try {
+      for (let i = 0; i < missingConvos.length; i += BATCH_SIZE) {
+        const batch = missingConvos.slice(i, i + BATCH_SIZE);
+        const vecs = await embeddings.embedBatch(batch.map((r) => r.content));
+        for (let j = 0; j < batch.length; j++) {
+          insertConvoVec.run(batch[j].id, serializeVector(vecs[j]));
+        }
+
+        if ((i + BATCH_SIZE) % 100 < BATCH_SIZE) {
+          console.error(
+            `[vector-memory-mcp]   ...${Math.min(i + BATCH_SIZE, missingConvos.length)}/${missingConvos.length} conversation chunks`,
+          );
+        }
+      }
+      db.exec("COMMIT");
+    } catch (e) {
+      db.exec("ROLLBACK");
+      throw e;
+    }
+
+    console.error(
+      `[vector-memory-mcp] Backfilled ${missingConvos.length} conversation vectors`,
+    );
+  }
+}

package/server/index.ts

@@ -2,6 +2,7 @@
 
 import { loadConfig, parseCliArgs } from "./config/index.js";
 import { connectToDatabase } from "./core/connection.js";
+import { backfillVectors } from "./core/migrations.js";
 import { MemoryRepository } from "./core/memory.repository.js";
 import { ConversationRepository } from "./core/conversation.repository.js";
 import { EmbeddingsService } from "./core/embeddings.service.js";
@@ -9,26 +10,6 @@ import { MemoryService } from "./core/memory.service.js";
 import { ConversationHistoryService } from "./core/conversation.service.js";
 import { startServer } from "./transports/mcp/server.js";
 import { startHttpServer } from "./transports/http/server.js";
-import { isLanceDbDirectory, migrate, formatMigrationSummary } from "./migration.js";
-
-async function runMigrate(args: string[]): Promise<void> {
-  const overrides = parseCliArgs(args.slice(1)); // skip "migrate"
-  const config = loadConfig(overrides);
-
-  const source = config.dbPath;
-  const target = source.endsWith(".sqlite") ? source.replace(/\.sqlite$/, "-migrated.sqlite") : source + ".sqlite";
-
-  if (!isLanceDbDirectory(source)) {
-    console.error(
-      `[vector-memory-mcp] No LanceDB data found at ${source}\n` +
-      `  Nothing to migrate. The server will create a fresh SQLite database on startup.`
-    );
-    return;
-  }
-
-  const result = await migrate({ source, target });
-  console.error(formatMigrationSummary(source, target, result));
-}
 
 async function main(): Promise<void> {
   const args = process.argv.slice(2);
@@ -40,33 +21,17 @@ async function main(): Promise<void> {
     return;
   }
 
-  // Check for migrate command
-  if (args[0] === "migrate") {
-    await runMigrate(args);
-    return;
-  }
-
   // Parse CLI args and load config
   const overrides = parseCliArgs(args);
   const config = loadConfig(overrides);
 
-  // Detect legacy LanceDB data and warn
-  if (isLanceDbDirectory(config.dbPath)) {
-    console.error(
-      `[vector-memory-mcp] ⚠️  Legacy LanceDB data detected at ${config.dbPath}\n` +
-      `  Your data must be migrated to the new SQLite format.\n` +
-      `  Run: vector-memory-mcp migrate\n` +
-      `  Or:  bun run server/index.ts migrate\n`
-    );
-    process.exit(1);
-  }
-
-  // Initialize database
+  // Initialize database and backfill any missing vectors before services start
   const db = connectToDatabase(config.dbPath);
+  const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
+  await backfillVectors(db, embeddings);
 
   // Initialize layers
   const repository = new MemoryRepository(db);
-  const embeddings = new EmbeddingsService(config.embeddingModel, config.embeddingDimension);
   const memoryService = new MemoryService(repository, embeddings);
 
   if (config.pluginMode) {

package/server/transports/http/server.ts

@@ -8,7 +8,7 @@ import type { Config } from "../../config/index.js";
 import { isDeleted } from "../../core/memory.js";
 import { createMcpRoutes } from "./mcp-transport.js";
 import type { Memory, SearchIntent } from "../../core/memory.js";
-import { MigrationService } from "../../core/migration.service.js";
+
 
 /**
  * Check if a port is available by attempting to bind to it
@@ -245,34 +245,6 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
     }
   });
 
-  // Migrate from external memory database
-  app.post("/migrate", async (c) => {
-    try {
-      const body = await c.req.json().catch(() => null);
-      if (!body || typeof body !== "object") {
-        return c.json({ error: "Invalid or missing JSON body" }, 400);
-      }
-      const source = body.source;
-
-      if (!source || typeof source !== "string") {
-        return c.json({ error: "Missing or invalid 'source' field" }, 400);
-      }
-
-      const repository = memoryService.getRepository();
-      const migrationService = new MigrationService(
-        repository,
-        memoryService.getEmbeddings(),
-        repository.getDb(),
-      );
-
-      const result = await migrationService.migrate(source);
-      return c.json(result);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : "Unknown error";
-      return c.json({ error: message }, 500);
-    }
-  });
-
   // Get single memory
   app.get("/memories/:id", async (c) => {
     try {

package/server/transports/mcp/resources.ts

@@ -1,152 +1,11 @@
-const MIGRATE_GUIDE = `# Migrating External Memory Databases
-
-The vector-memory-mcp server exposes a \`POST /migrate\` HTTP endpoint that imports
-memories from other database formats into the running instance. All imported
-content is re-embedded with the server's current embedding model to guarantee
-consistency.
-
-## Endpoint
-
-\`\`\`
-POST http://<host>:<port>/migrate
-Content-Type: application/json
-
-{ "source": "/absolute/path/to/source/database" }
-\`\`\`
-
-## Discovering the Server Port
-
-The HTTP server writes a lockfile at \`.vector-memory/server.lock\` in the
-project's working directory. Read it to discover the current port:
-
-\`\`\`json
-{ "port": 3271, "pid": 12345 }
-\`\`\`
-
-## Supported Source Formats
-
-The endpoint auto-detects the source format from the path provided.
-
-### 1. LanceDB Directory
-Provide the path to a LanceDB data directory (contains \`.lance\` files or
-\`_versions\`/\`_indices\` subdirectories). Both memories and conversation
-history are imported.
-
-\`\`\`json
-{ "source": "/path/to/project/.vector-memory" }
-\`\`\`
-
-### 2. Own SQLite (Current or Older Schema)
-Provide the path to a \`.db\` file that was created by any version of
-vector-memory-mcp. The migrator handles missing columns (e.g. \`usefulness\`,
-\`access_count\`) by using sensible defaults. Both memories and conversation
-history are imported.
-
-\`\`\`json
-{ "source": "/path/to/old-project/.vector-memory/memories.db" }
-\`\`\`
-
-### 3. CCCMemory SQLite
-Provide the path to a CCCMemory database. The migrator extracts from the
-\`decisions\`, \`mistakes\`, \`methodologies\`, \`research_findings\`,
-\`solution_patterns\`, and \`working_memory\` tables. Each record is tagged
-with \`source_type: "cccmemory"\` and the appropriate \`memory_type\` in
-metadata.
-
-\`\`\`json
-{ "source": "/path/to/cccmemory.db" }
-\`\`\`
-
-### 4. MCP Memory Service SQLite
-Provide the path to an mcp-memory-service database. Memories with
-\`deleted_at IS NULL\` are imported. Tags and memory type are preserved in
-metadata.
-
-\`\`\`json
-{ "source": "/path/to/mcp-memory-service.db" }
-\`\`\`
-
-### 5. MIF JSON (Shodh Memory Interchange Format)
-Provide the path to a \`.json\` file exported from Shodh Memory. The file must
-contain a top-level \`memories\` array. Memory type, tags, entities, and source
-metadata are preserved.
-
-\`\`\`json
-{ "source": "/path/to/export.mif.json" }
-\`\`\`
-
-## Response
-
-The endpoint returns a JSON summary upon completion:
-
-\`\`\`json
-{
-  "source": "/path/to/source",
-  "format": "own-sqlite",
-  "memoriesImported": 142,
-  "memoriesSkipped": 3,
-  "conversationsImported": 0,
-  "conversationsSkipped": 0,
-  "errors": [],
-  "durationMs": 8320
-}
-\`\`\`
-
-- **memoriesImported**: Number of new memories written to the database.
-- **memoriesSkipped**: Records skipped because a memory with the same ID
-  already exists (safe for idempotent re-runs).
-- **conversationsImported / conversationsSkipped**: Same, for conversation
-  history chunks (LanceDB and own-sqlite formats only).
-- **errors**: Per-record errors that did not abort the migration.
-- **durationMs**: Wall-clock time for the entire operation.
-
-## Important Notes
-
-- **Re-embedding**: All content is re-embedded regardless of the source format.
-  This ensures vector consistency with the server's current model but means the
-  operation can take time for large databases (~50ms per record).
-- **Idempotent**: Running the same migration twice is safe. Duplicate IDs are
-  skipped.
-- **Non-destructive**: The source database is opened read-only and is never
-  modified.
-- **Batched writes**: Records are inserted in batches of 100 within
-  transactions. If the process is interrupted, already-committed batches are
-  durable.
-- **Error isolation**: A single bad record does not abort the migration. Check
-  the \`errors\` array in the response for any per-record failures.
-
-## Workflow Example
-
-1. Locate the source database file or directory.
-2. Read \`.vector-memory/server.lock\` to get the port.
-3. Send the migrate request:
-   \`\`\`bash
-   curl -X POST http://127.0.0.1:3271/migrate \\
-     -H "Content-Type: application/json" \\
-     -d '{"source": "/path/to/old/memories.db"}'
-   \`\`\`
-4. Inspect the response summary.
-5. Verify imported memories with a search:
-   \`\`\`bash
-   curl -X POST http://127.0.0.1:3271/search \\
-     -H "Content-Type: application/json" \\
-     -d '{"query": "test query", "limit": 5}'
-   \`\`\`
-`;
-
-export const resources = [
-  {
-    uri: "vector-memory://guides/migrate",
-    name: "Migration Guide",
-    description:
-      "How to use the POST /migrate HTTP endpoint to import memories from external database formats (LanceDB, older SQLite, CCCMemory, MCP Memory Service, MIF JSON) into the running vector-memory instance.",
-    mimeType: "text/markdown",
-  },
-];
-
-const RESOURCE_CONTENT: Record<string, string> = {
-  "vector-memory://guides/migrate": MIGRATE_GUIDE,
-};
+export const resources: Array<{
+  uri: string;
+  name: string;
+  description: string;
+  mimeType: string;
+}> = [];
+
+const RESOURCE_CONTENT: Record<string, string> = {};
 
 export function readResource(uri: string): {
   contents: Array<{ uri: string; mimeType: string; text: string }>;