@aeriondyseti/vector-memory-mcp 2.1.1 → 2.2.0-dev.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.1.1",
+  "version": "2.2.0-dev.1",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "src/index.ts",
@@ -51,8 +51,7 @@
     "@modelcontextprotocol/sdk": "^1.0.0",
     "apache-arrow": "^21.1.0",
     "arg": "^5.0.2",
-    "hono": "^4.11.3",
-    "sqlite-vec": "^0.1.6"
+    "hono": "^4.11.3"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/scripts/lancedb-extract.ts

@@ -0,0 +1,106 @@
+#!/usr/bin/env bun
+/**
+ * Standalone LanceDB data extractor — runs in a child process so that
+ * @lancedb/lancedb native bindings never coexist with bun:sqlite's
+ * extension loading in the same process.
+ *
+ * Usage: bun scripts/lancedb-extract.ts <lance-db-path>
+ * Output: JSON on stdout — { memories: Row[], conversations: Row[] }
+ */
+
+const source = process.argv[2];
+if (!source) {
+  console.error("Usage: bun scripts/lancedb-extract.ts <lance-db-path>");
+  process.exit(1);
+}
+
+function toEpochMs(value: unknown): number {
+  if (typeof value === "number") return value;
+  if (value instanceof Date) return value.getTime();
+  if (typeof value === "bigint") return Number(value);
+  return Date.now();
+}
+
+function toFloatArray(vec: unknown): number[] {
+  if (Array.isArray(vec)) return vec;
+  if (vec instanceof Float32Array) return Array.from(vec);
+  if (vec && typeof (vec as any).toArray === "function") {
+    return Array.from((vec as any).toArray());
+  }
+  if (ArrayBuffer.isView(vec)) {
+    const view = vec as DataView;
+    return Array.from(new Float32Array(view.buffer, view.byteOffset, view.byteLength / 4));
+  }
+  return [];
+}
+
+const lancedb = await import("@lancedb/lancedb");
+const db = await lancedb.connect(source);
+const tableNames = await db.tableNames();
+console.error(`Found tables: ${tableNames.join(", ")}`);
+
+const result: { memories: any[]; conversations: any[] } = {
+  memories: [],
+  conversations: [],
+};
+
+const BATCH_SIZE = 500;
+
+if (tableNames.includes("memories")) {
+  const table = await db.openTable("memories");
+  const total = await table.countRows();
+  console.error(`Reading ${total} memories...`);
+
+  let offset = 0;
+  while (true) {
+    const rows = await table.query().limit(BATCH_SIZE).offset(offset).toArray();
+    if (rows.length === 0) break;
+    for (const row of rows) {
+      result.memories.push({
+        id: row.id,
+        content: row.content,
+        metadata: row.metadata ?? "{}",
+        vector: toFloatArray(row.vector),
+        created_at: toEpochMs(row.created_at),
+        updated_at: toEpochMs(row.updated_at),
+        last_accessed: row.last_accessed != null ? toEpochMs(row.last_accessed) : null,
+        superseded_by: row.superseded_by ?? null,
+        usefulness: row.usefulness ?? 0,
+        access_count: row.access_count ?? 0,
+      });
+    }
+    offset += BATCH_SIZE;
+  }
+  console.error(`  ${result.memories.length} memories read`);
+}
+
+if (tableNames.includes("conversation_history")) {
+  const table = await db.openTable("conversation_history");
+  const total = await table.countRows();
+  console.error(`Reading ${total} conversation chunks...`);
+
+  let offset = 0;
+  while (true) {
+    const rows = await table.query().limit(BATCH_SIZE).offset(offset).toArray();
+    if (rows.length === 0) break;
+    for (const row of rows) {
+      result.conversations.push({
+        id: row.id,
+        content: row.content,
+        metadata: row.metadata ?? "{}",
+        vector: toFloatArray(row.vector),
+        created_at: toEpochMs(row.created_at),
+        session_id: row.session_id,
+        role: row.role,
+        message_index_start: row.message_index_start ?? 0,
+        message_index_end: row.message_index_end ?? 0,
+        project: row.project ?? "",
+      });
+    }
+    offset += BATCH_SIZE;
+  }
+  console.error(`  ${result.conversations.length} conversation chunks read`);
+}
+
+await db.close?.();
+process.stdout.write(JSON.stringify(result));

package/src/db/connection.ts

@@ -1,12 +1,11 @@
 import { Database } from "bun:sqlite";
-import * as sqliteVec from "sqlite-vec";
 import { mkdirSync } from "fs";
 import { dirname } from "path";
 import { runMigrations } from "./migrations.js";
 
 /**
- * Open (or create) a SQLite database at the given path,
- * load the sqlite-vec extension, and run schema migrations.
+ * Open (or create) a SQLite database at the given path
+ * and run schema migrations.
  */
 export function connectToDatabase(dbPath: string): Database {
   mkdirSync(dirname(dbPath), { recursive: true });
@@ -15,9 +14,6 @@ export function connectToDatabase(dbPath: string): Database {
   // WAL mode for concurrent read performance
   db.exec("PRAGMA journal_mode=WAL");
 
-  // Load sqlite-vec extension
-  sqliteVec.load(db);
-
   // Ensure schema is up to date
   runMigrations(db);
 

package/src/db/conversation.repository.ts

@@ -9,6 +9,7 @@ import {
   sanitizeFtsQuery,
   hybridRRF,
   topByRRF,
+  knnSearch,
 } from "./sqlite-utils.js";
 
 export class ConversationRepository {
@@ -112,14 +113,8 @@ export class ConversationRepository {
   ): Promise<ConversationHybridRow[]> {
     const candidateCount = limit * 3;
 
-    // Vector KNN search
-    const vecResults = this.db
-      .prepare(
-        `SELECT id FROM conversation_history_vec
-         WHERE vector MATCH ? AND k = ?
-         ORDER BY distance`
-      )
-      .all(serializeVector(embedding), candidateCount) as Array<{ id: string }>;
+    // Vector KNN search (brute-force cosine similarity in JS)
+    const vecResults = knnSearch(this.db, "conversation_history_vec", embedding, candidateCount);
 
     // FTS5 search
     const ftsQuery = sanitizeFtsQuery(query);

package/src/db/memory.repository.ts

@@ -6,6 +6,7 @@ import {
   sanitizeFtsQuery,
   hybridRRF,
   topByRRF,
+  knnSearch,
 } from "./sqlite-utils.js";
 import {
   type Memory,
@@ -46,7 +47,7 @@ export class MemoryRepository {
   }
 
   /**
-   * Fetch the embedding vector for a memory id from the vec0 table.
+   * Fetch the embedding vector for a memory id.
    */
   private getEmbedding(id: string): number[] {
     const row = this.db
@@ -110,7 +111,6 @@ export class MemoryRepository {
           memory.lastAccessed?.getTime() ?? null,
         );
 
-      // vec0 virtual tables don't support REPLACE — delete then insert
       this.db.prepare("DELETE FROM memories_vec WHERE id = ?").run(memory.id);
       this.db
         .prepare("INSERT INTO memories_vec (id, vector) VALUES (?, ?)")
@@ -170,14 +170,9 @@ export class MemoryRepository {
     limit: number,
   ): Promise<HybridRow[]> {
     const candidateLimit = limit * 3;
-    const vecBuf = serializeVector(embedding);
 
-    // Vector KNN search
-    const vectorResults = this.db
-      .prepare(
-        "SELECT id, distance FROM memories_vec WHERE vector MATCH ? AND k = ? ORDER BY distance",
-      )
-      .all(vecBuf, candidateLimit) as Array<{ id: string; distance: number }>;
+    // Vector KNN search (brute-force cosine similarity in JS)
+    const vectorResults = knnSearch(this.db, "memories_vec", embedding, candidateLimit);
 
     // Full-text search
     const ftsQuery = sanitizeFtsQuery(query);

package/src/db/migrations.ts

@@ -1,5 +1,33 @@
 import type { Database } from "bun:sqlite";
 
+/**
+ * Check if a table exists and is a vec0 virtual table (from the old sqlite-vec schema).
+ */
+function isVec0Table(db: Database, tableName: string): boolean {
+  const row = db
+    .prepare(
+      `SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?`,
+    )
+    .get(tableName) as { sql: string } | null;
+  return row?.sql?.toLowerCase().includes("vec0") ?? false;
+}
+
+/**
+ * Migrate a vec0 virtual table to a plain BLOB table.
+ * Copies id + vector data, drops the vec0 table and its shadow tables, then
+ * creates the new plain table with the copied data.
+ */
+function migrateVec0ToBlob(db: Database, tableName: string): void {
+  const tmpTable = `${tableName}_migration_tmp`;
+
+  db.exec(`CREATE TABLE IF NOT EXISTS ${tmpTable} (id TEXT PRIMARY KEY, vector BLOB NOT NULL)`);
+  db.exec(`INSERT OR IGNORE INTO ${tmpTable} (id, vector) SELECT id, vector FROM ${tableName}`);
+  db.exec(`DROP TABLE ${tableName}`);
+  db.exec(`CREATE TABLE ${tableName} (id TEXT PRIMARY KEY, vector BLOB NOT NULL)`);
+  db.exec(`INSERT INTO ${tableName} (id, vector) SELECT id, vector FROM ${tmpTable}`);
+  db.exec(`DROP TABLE ${tmpTable}`);
+}
+
 /**
  * Run all schema migrations. Safe to call on every startup (uses IF NOT EXISTS).
  */
@@ -19,12 +47,17 @@ export function runMigrations(db: Database): void {
     )
   `);
 
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS memories_vec USING vec0(
-      id TEXT PRIMARY KEY,
-      vector float[384]
-    )
-  `);
+  // Migrate vec0 -> plain blob table if upgrading from sqlite-vec schema
+  if (isVec0Table(db, "memories_vec")) {
+    migrateVec0ToBlob(db, "memories_vec");
+  } else {
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS memories_vec (
+        id     TEXT PRIMARY KEY,
+        vector BLOB NOT NULL
+      )
+    `);
+  }
 
   db.exec(`
     CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
@@ -48,12 +81,17 @@ export function runMigrations(db: Database): void {
     )
   `);
 
-  db.exec(`
-    CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_vec USING vec0(
-      id TEXT PRIMARY KEY,
-      vector float[384]
-    )
-  `);
+  // Migrate vec0 -> plain blob table if upgrading from sqlite-vec schema
+  if (isVec0Table(db, "conversation_history_vec")) {
+    migrateVec0ToBlob(db, "conversation_history_vec");
+  } else {
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS conversation_history_vec (
+        id     TEXT PRIMARY KEY,
+        vector BLOB NOT NULL
+      )
+    `);
+  }
 
   db.exec(`
     CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_fts USING fts5(

package/src/db/sqlite-utils.ts

@@ -4,19 +4,60 @@ import type { Database } from "bun:sqlite";
 export const RRF_K = 60;
 
 /**
- * Serialize a number[] embedding to the raw float32 bytes sqlite-vec expects.
+ * Serialize a number[] embedding to raw float32 bytes for BLOB storage.
  */
 export function serializeVector(vec: number[]): Buffer {
   return Buffer.from(new Float32Array(vec).buffer);
 }
 
 /**
- * Deserialize raw float32 bytes from sqlite-vec back to number[].
+ * Deserialize raw float32 bytes back to number[].
  */
 export function deserializeVector(buf: Buffer): number[] {
   return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
 }
 
+/**
+ * Cosine similarity between two pre-normalized Float32Arrays.
+ * Returns dot product (equivalent to cosine sim when vectors are unit-length).
+ */
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot;
+}
+
+/**
+ * Brute-force KNN search over a vector blob table.
+ * Loads all vectors, computes cosine similarity, returns top-K results
+ * sorted by descending similarity (ascending distance).
+ */
+export function knnSearch(
+  db: Database,
+  table: string,
+  queryVec: number[],
+  k: number,
+): Array<{ id: string; distance: number }> {
+  const rows = db
+    .prepare(`SELECT id, vector FROM ${table}`)
+    .all() as Array<{ id: string; vector: Buffer }>;
+
+  const qv = new Float32Array(queryVec);
+  const scored = rows.map((r) => {
+    const vec = new Float32Array(
+      r.vector.buffer,
+      r.vector.byteOffset,
+      r.vector.byteLength / 4,
+    );
+    const sim = cosineSimilarity(qv, vec);
+    // Convert similarity to distance (1 - sim) for consistency with previous API
+    return { id: r.id, distance: 1 - sim };
+  });
+
+  scored.sort((a, b) => a.distance - b.distance);
+  return scored.slice(0, k);
+}
+
 /**
  * Sanitize a user query for FTS5 by quoting each token as a literal.
  * Prevents FTS5 syntax errors from special characters like AND, OR, *, etc.

package/src/http/server.ts

@@ -189,7 +189,8 @@ export function createHttpApp(memoryService: MemoryService, config: Config): Hon
   // Get latest waypoint
   app.get("/waypoint", async (c) => {
     try {
-      const waypoint = await memoryService.getLatestWaypoint();
+      const project = c.req.query("project");
+      const waypoint = await memoryService.getLatestWaypoint(project);
 
       if (!waypoint) {
         return c.json({ error: "No waypoint found" }, 404);

package/src/mcp/handlers.ts

@@ -38,6 +38,10 @@ function errorText(e: unknown): string {
   return e instanceof Error ? e.message : String(e);
 }
 
+function errorResult(text: string): CallToolResult {
+  return { isError: true, content: [{ type: "text", text }] };
+}
+
 function parseDate(value: unknown, fieldName: string): Date | undefined {
   if (value === undefined) return undefined;
   const date = new Date(value as string);
@@ -59,7 +63,7 @@ export async function handleStoreMemories(
   try {
     memories = asArray(args?.memories, "memories");
   } catch (e) {
-    return { isError: true, content: [{ type: "text", text: errorText(e) }] };
+    return errorResult(errorText(e));
   }
 
   const ids: string[] = [];
@@ -93,7 +97,7 @@ export async function handleDeleteMemories(
   try {
     ids = asArray(args?.ids, "ids");
   } catch (e) {
-    return { isError: true, content: [{ type: "text", text: errorText(e) }] };
+    return errorResult(errorText(e));
   }
   const results: string[] = [];
 
@@ -128,7 +132,7 @@ export async function handleUpdateMemories(
   try {
     updates = asArray(args?.updates, "updates");
   } catch (e) {
-    return { isError: true, content: [{ type: "text", text: errorText(e) }] };
+    return errorResult(errorText(e));
   }
 
   const results: string[] = [];
@@ -168,10 +172,11 @@ export async function handleSearchMemories(
 ): Promise<CallToolResult> {
   const query = args?.query;
   if (typeof query !== "string" || query.trim() === "") {
-    return { isError: true, content: [{ type: "text", text: "query is required and must be a non-empty string" }] };
+    return errorResult("query is required and must be a non-empty string");
   }
   const intent = (args?.intent as SearchIntent) ?? "fact_check";
   const limit = (args?.limit as number) ?? 10;
+  const offset = (args?.offset as number) ?? 0;
   const includeDeleted = (args?.include_deleted as boolean) ?? false;
   const historyOnly = (args?.history_only as boolean) ?? false;
   // history_only implies include_history
@@ -181,13 +186,14 @@ export async function handleSearchMemories(
   try {
     historyFilters = parseHistoryFilters(args);
   } catch (e) {
-    return { isError: true, content: [{ type: "text", text: errorText(e) }] };
+    return errorResult(errorText(e));
   }
 
   const results = await service.search(query, intent, limit, includeDeleted, {
     includeHistory,
     historyOnly,
     historyFilters,
+    offset,
   });
 
   if (results.length === 0) {
@@ -243,7 +249,7 @@ export async function handleGetMemories(
   try {
     ids = asArray(args?.ids, "ids");
   } catch (e) {
-    return { isError: true, content: [{ type: "text", text: errorText(e) }] };
+    return errorResult(errorText(e));
   }
 
   const memories = await service.getMultiple(ids);
@@ -267,10 +273,7 @@ export async function handleReportMemoryUsefulness(
   const memory = await service.vote(memoryId, useful ? 1 : -1);
 
   if (!memory) {
-    return {
-      content: [{ type: "text", text: `Memory ${memoryId} not found` }],
-      isError: true,
-    };
+    return errorResult(`Memory ${memoryId} not found`);
   }
 
   return {
@@ -305,10 +308,11 @@ export async function handleSetWaypoint(
 }
 
 export async function handleGetWaypoint(
-  _args: Record<string, unknown> | undefined,
+  args: Record<string, unknown> | undefined,
   service: MemoryService
 ): Promise<CallToolResult> {
-  const waypoint = await service.getLatestWaypoint();
+  const project = args?.project as string | undefined;
+  const waypoint = await service.getLatestWaypoint(project);
 
   if (!waypoint) {
     return {
@@ -350,15 +354,9 @@ function requireConversationService(
   const conversationService = service.getConversationService();
   if (!conversationService) {
     return {
-      error: {
-        content: [
-          {
-            type: "text",
-            text: "Conversation history indexing is not enabled. Enable it with --enable-history.",
-          },
-        ],
-        isError: true,
-      },
+      error: errorResult(
+        "Conversation history indexing is not enabled. Enable it with --enable-history."
+      ),
     };
   }
   return { service: conversationService };
@@ -376,7 +374,7 @@ export async function handleIndexConversations(
   const sinceStr = args?.since as string | undefined;
   const since = sinceStr ? new Date(sinceStr) : undefined;
   if (since && isNaN(since.getTime())) {
-    return { isError: true, content: [{ type: "text", text: "Invalid 'since' date format" }] };
+    return errorResult("Invalid 'since' date format");
   }
 
   const result = await conversationService.indexConversations(path, since);
@@ -444,18 +442,12 @@ export async function handleReindexSession(
 
   const sessionId = args?.session_id as string | undefined;
   if (!sessionId) {
-    return {
-      content: [{ type: "text", text: "session_id is required" }],
-      isError: true,
-    };
+    return errorResult("session_id is required");
   }
   const result = await conversationService.reindexSession(sessionId);
 
   if (!result.success) {
-    return {
-      content: [{ type: "text", text: `Reindex failed: ${result.error}` }],
-      isError: true,
-    };
+    return errorResult(`Reindex failed: ${result.error}`);
   }
 
   return {
@@ -497,9 +489,6 @@ export async function handleToolCall(
     case "reindex_session":
       return handleReindexSession(args, service);
     default:
-      return {
-        content: [{ type: "text", text: `Unknown tool: ${name}` }],
-        isError: true,
-      };
+      return errorResult(`Unknown tool: ${name}`);
   }
 }

package/src/mcp/tools.ts

@@ -162,6 +162,11 @@ When in doubt, search. Missing context is costlier than an extra query.`,
         description: "Maximum results to return (default: 10).",
         default: 10,
       },
+      offset: {
+        type: "integer",
+        description: "Number of results to skip for pagination (default: 0).",
+        default: 0,
+      },
       include_deleted: {
         type: "boolean",
         description: "Include soft-deleted memories in results (default: false). Useful for recovering prior information.",
@@ -297,7 +302,13 @@ export const getWaypointTool: Tool = {
     "Load the current project waypoint snapshot. Call at conversation start or when resuming a project.",
   inputSchema: {
     type: "object",
-    properties: {},
+    properties: {
+      project: {
+        type: "string",
+        description:
+          "Project name to retrieve waypoint for. If omitted, retrieves the default (legacy) waypoint.",
+      },
+    },
   },
 };
 

package/src/migration.ts

@@ -1,57 +1,32 @@
 /**
- * LanceDB → SQLite (sqlite-vec) migration logic.
+ * LanceDB -> SQLite migration logic.
  *
- * This module is the shared core used by both the `migrate` subcommand
- * and the standalone `scripts/migrate-from-lancedb.ts` script.
+ * Reads LanceDB data in a child process (scripts/lancedb-extract.ts) to avoid
+ * a native symbol collision between @lancedb/lancedb and bun:sqlite.
+ * The extracted JSON is then written to SQLite in-process.
  *
  * @deprecated Will be removed in the next major version once LanceDB
  *   support is dropped.
  */
 
 import { existsSync, statSync } from "fs";
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
 import { connectToDatabase } from "./db/connection.js";
 import { serializeVector } from "./db/sqlite-utils.js";
 
-// ── Helpers ─────────────────────────────────────────────────────────
-
-function toEpochMs(value: unknown): number {
-  if (typeof value === "number") return value;
-  if (value instanceof Date) return value.getTime();
-  if (typeof value === "bigint") return Number(value);
-  console.warn(`⚠️ Unexpected timestamp type: ${typeof value} (value: ${value}), using current time`);
-  return Date.now();
-}
-
-function toFloatArray(vec: unknown): number[] {
-  if (Array.isArray(vec)) return vec;
-  if (vec instanceof Float32Array) return Array.from(vec);
-  // Arrow Vector objects have a .toArray() method that returns Float32Array
-  if (vec && typeof (vec as any).toArray === "function") {
-    return Array.from((vec as any).toArray());
-  }
-  if (ArrayBuffer.isView(vec)) {
-    const view = vec as DataView;
-    return Array.from(new Float32Array(view.buffer, view.byteOffset, view.byteLength / 4));
-  }
-  return [];
-}
+const __dirname = dirname(fileURLToPath(import.meta.url));
 
 // ── Detection ───────────────────────────────────────────────────────
 
-/**
- * Check if a path is a LanceDB directory (i.e. needs migration).
- * Returns true if the path exists and is a directory.
- */
 export function isLanceDbDirectory(dbPath: string): boolean {
   return existsSync(dbPath) && statSync(dbPath).isDirectory();
 }
 
-// ── Migration ───────────────────────────────────────────────────────
+// ── Types ───────────────────────────────────────────────────────────
 
 export interface MigrateOptions {
-  /** Path to the LanceDB directory (source). */
   source: string;
-  /** Path to the new SQLite file (target). */
   target: string;
 }
 
@@ -61,24 +36,44 @@ export interface MigrateResult {
   outputSizeMB: string;
 }
 
-/**
- * Run the full LanceDB → SQLite migration.
- *
- * Dynamically imports @lancedb/lancedb so the cost is only paid
- * when the migration is actually invoked.
- */
+interface ExtractedData {
+  memories: Array<{
+    id: string;
+    content: string;
+    metadata: string;
+    vector: number[];
+    created_at: number;
+    updated_at: number;
+    last_accessed: number | null;
+    superseded_by: string | null;
+    usefulness: number;
+    access_count: number;
+  }>;
+  conversations: Array<{
+    id: string;
+    content: string;
+    metadata: string;
+    vector: number[];
+    created_at: number;
+    session_id: string;
+    role: string;
+    message_index_start: number;
+    message_index_end: number;
+    project: string;
+  }>;
+}
+
+// ── Migration ───────────────────────────────────────────────────────
+
 export async function migrate(opts: MigrateOptions): Promise<MigrateResult> {
   const { source, target } = opts;
 
-  // Validate source
   if (!existsSync(source)) {
     throw new Error(`Source not found: ${source}`);
   }
   if (!statSync(source).isDirectory()) {
     throw new Error(`Source is not a directory (expected LanceDB): ${source}`);
   }
-
-  // Prevent overwriting
   if (existsSync(target)) {
     throw new Error(
       `Target already exists: ${target}\n   Delete it first or choose a different target path.`
@@ -89,148 +84,100 @@ export async function migrate(opts: MigrateOptions): Promise<MigrateResult> {
   console.error(`📄 Target (SQLite):  ${target}`);
   console.error();
 
-  // Dynamic import — only loads LanceDB when migration is actually run
-  const lancedb = await import("@lancedb/lancedb");
+  // Phase 1: Extract data from LanceDB in a subprocess.
+  // This avoids a native symbol collision between @lancedb/lancedb and bun:sqlite.
+  const extractScript = resolve(__dirname, "..", "scripts", "lancedb-extract.ts");
+  const proc = Bun.spawn(["bun", extractScript, source], {
+    stdout: "pipe",
+    stderr: "inherit",
+  });
+
+  const output = await new Response(proc.stdout).text();
+  const exitCode = await proc.exited;
+
+  if (exitCode !== 0) {
+    throw new Error(`LanceDB extraction failed (exit code ${exitCode})`);
+  }
 
-  // Open LanceDB
-  const lanceDb = await lancedb.connect(source);
-  const tableNames = await lanceDb.tableNames();
-  console.error(`Found tables: ${tableNames.join(", ")}`);
+  const data: ExtractedData = JSON.parse(output);
 
-  // Open SQLite (reuses shared connection setup: WAL, sqlite-vec, migrations)
+  // Phase 2: Write to SQLite (no LanceDB in this process).
   const sqliteDb = connectToDatabase(target);
 
   let memoriesMigrated = 0;
   let conversationChunksMigrated = 0;
 
-  // ── Migrate memories ────────────────────────────────────────────
-  if (tableNames.includes("memories")) {
-    const memoriesTable = await lanceDb.openTable("memories");
-    const totalMemories = await memoriesTable.countRows();
-    console.error(`\n🧠 Migrating ${totalMemories} memories...`);
+  if (data.memories.length > 0) {
+    console.error(`\n🧠 Writing ${data.memories.length} memories to SQLite...`);
 
     const insertMain = sqliteDb.prepare(
       `INSERT OR REPLACE INTO memories
         (id, content, metadata, created_at, updated_at, superseded_by, usefulness, access_count, last_accessed)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
     );
+    const deleteVec = sqliteDb.prepare(`DELETE FROM memories_vec WHERE id = ?`);
     const insertVec = sqliteDb.prepare(
       `INSERT INTO memories_vec (id, vector) VALUES (?, ?)`
     );
     const insertFts = sqliteDb.prepare(
-      `INSERT INTO memories_fts (id, content) VALUES (?, ?)`
+      `INSERT OR REPLACE INTO memories_fts (id, content) VALUES (?, ?)`
     );
 
-    const BATCH_SIZE = 500;
-    let offset = 0;
-
-    while (true) {
-      const rows = await memoriesTable.query().limit(BATCH_SIZE).offset(offset).toArray();
-      if (rows.length === 0) break;
-
-      const tx = sqliteDb.transaction(() => {
-        for (const row of rows) {
-          const vec = toFloatArray(row.vector);
-          const createdAt = toEpochMs(row.created_at);
-          const updatedAt = toEpochMs(row.updated_at);
-          const lastAccessed = row.last_accessed != null ? toEpochMs(row.last_accessed) : null;
-
-          insertMain.run(
-            row.id,
-            row.content,
-            row.metadata ?? "{}",
-            createdAt,
-            updatedAt,
-            row.superseded_by ?? null,
-            row.usefulness ?? 0,
-            row.access_count ?? 0,
-            lastAccessed,
-          );
-
-          if (vec.length > 0) {
-            insertVec.run(row.id, serializeVector(vec));
-          }
-
-          insertFts.run(row.id, row.content);
+    const tx = sqliteDb.transaction(() => {
+      for (const row of data.memories) {
+        insertMain.run(
+          row.id, row.content, row.metadata,
+          row.created_at, row.updated_at,
+          row.superseded_by, row.usefulness,
+          row.access_count, row.last_accessed,
+        );
+        if (row.vector.length > 0) {
+          deleteVec.run(row.id);
+          insertVec.run(row.id, serializeVector(row.vector));
         }
-      });
-
-      tx();
-      memoriesMigrated += rows.length;
-      offset += BATCH_SIZE;
-
-      if (totalMemories > BATCH_SIZE) {
-        process.stderr.write(`   ${memoriesMigrated}/${totalMemories}\r`);
+        insertFts.run(row.id, row.content);
       }
-    }
-
+    });
+    tx();
+    memoriesMigrated = data.memories.length;
     console.error(`   ✅ ${memoriesMigrated} memories migrated`);
   }
 
-  // ── Migrate conversation history ────────────────────────────────
-  if (tableNames.includes("conversation_history")) {
-    const convTable = await lanceDb.openTable("conversation_history");
-    const totalConv = await convTable.countRows();
-    console.error(`\n💬 Migrating ${totalConv} conversation chunks...`);
+  if (data.conversations.length > 0) {
+    console.error(`\n💬 Writing ${data.conversations.length} conversation chunks to SQLite...`);
 
     const insertMain = sqliteDb.prepare(
       `INSERT OR REPLACE INTO conversation_history
         (id, content, metadata, created_at, session_id, role, message_index_start, message_index_end, project)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
     );
+    const deleteVec = sqliteDb.prepare(`DELETE FROM conversation_history_vec WHERE id = ?`);
     const insertVec = sqliteDb.prepare(
       `INSERT INTO conversation_history_vec (id, vector) VALUES (?, ?)`
     );
     const insertFts = sqliteDb.prepare(
-      `INSERT INTO conversation_history_fts (id, content) VALUES (?, ?)`
+      `INSERT OR REPLACE INTO conversation_history_fts (id, content) VALUES (?, ?)`
     );
 
-    const BATCH_SIZE = 500;
-    let offset = 0;
-
-    while (true) {
-      const rows = await convTable.query().limit(BATCH_SIZE).offset(offset).toArray();
-      if (rows.length === 0) break;
-
-      const tx = sqliteDb.transaction(() => {
-        for (const row of rows) {
-          const vec = toFloatArray(row.vector);
-          const createdAt = toEpochMs(row.created_at);
-
-          insertMain.run(
-            row.id,
-            row.content,
-            row.metadata ?? "{}",
-            createdAt,
-            row.session_id,
-            row.role,
-            row.message_index_start ?? 0,
-            row.message_index_end ?? 0,
-            row.project ?? "",
-          );
-
-          if (vec.length > 0) {
-            insertVec.run(row.id, serializeVector(vec));
-          }
-
-          insertFts.run(row.id, row.content);
+    const tx = sqliteDb.transaction(() => {
+      for (const row of data.conversations) {
+        insertMain.run(
+          row.id, row.content, row.metadata,
+          row.created_at, row.session_id, row.role,
+          row.message_index_start, row.message_index_end, row.project,
+        );
+        if (row.vector.length > 0) {
+          deleteVec.run(row.id);
+          insertVec.run(row.id, serializeVector(row.vector));
         }
-      });
-
-      tx();
-      conversationChunksMigrated += rows.length;
-      offset += BATCH_SIZE;
-
-      if (totalConv > BATCH_SIZE) {
-        process.stderr.write(`   ${conversationChunksMigrated}/${totalConv}\r`);
+        insertFts.run(row.id, row.content);
       }
-    }
-
+    });
+    tx();
+    conversationChunksMigrated = data.conversations.length;
     console.error(`   ✅ ${conversationChunksMigrated} conversation chunks migrated`);
   }
 
-  // ── Finalize ────────────────────────────────────────────────────
-  await lanceDb.close?.();
   sqliteDb.close();
 
   const { size } = statSync(target);
@@ -239,9 +186,6 @@ export async function migrate(opts: MigrateOptions): Promise<MigrateResult> {
   return { memoriesMigrated, conversationChunksMigrated, outputSizeMB };
 }
 
-/**
- * Format a human-readable summary after migration completes.
- */
 export function formatMigrationSummary(
   source: string,
   target: string,

package/src/services/memory.service.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from "crypto";
+import { randomUUID, createHash } from "crypto";
 import type { Memory, SearchIntent, IntentProfile, HybridRow } from "../types/memory.js";
 import { isDeleted } from "../types/memory.js";
 import type { SearchResult, SearchOptions } from "../types/conversation.js";
@@ -185,6 +185,7 @@ export class MemoryService {
     const queryEmbedding = await this.embeddings.embed(query);
     const profile = INTENT_PROFILES[intent];
     const now = new Date();
+    const offset = Math.min(options?.offset ?? 0, 500);
 
     const hasConversationService = this.conversationService !== null;
     const historyOnly = (options?.historyOnly ?? false) && hasConversationService;
@@ -195,11 +196,14 @@ export class MemoryService {
       this.conversationService?.config.historyWeight ??
       0.75;
 
+    // Widen the candidate pool to account for offset
+    const effectiveLimit = offset + limit;
+
     // Run memory + history queries in parallel
     const memoryPromise =
       !historyOnly
         ? this.repository
-            .findHybrid(queryEmbedding, query, limit * 5)
+            .findHybrid(queryEmbedding, query, effectiveLimit * 5)
             .then((candidates) =>
               candidates
                 .filter((m) => includeDeleted || !isDeleted(m))
@@ -225,7 +229,7 @@ export class MemoryService {
             .searchHistory(
               query,
               queryEmbedding,
-              historyOnly ? limit * 5 : limit * 3,
+              historyOnly ? effectiveLimit * 5 : effectiveLimit * 3,
               options?.historyFilters
             )
             .then((historyRows) =>
@@ -255,7 +259,7 @@ export class MemoryService {
     const merged = [...memoryResults, ...historyResults];
     merged.sort((a, b) => b.score - a.score);
 
-    return merged.slice(0, limit);
+    return merged.slice(offset, offset + limit);
   }
 
   async trackAccess(ids: string[]): Promise<void> {
@@ -278,6 +282,19 @@ export class MemoryService {
   private static readonly UUID_ZERO =
     "00000000-0000-0000-0000-000000000000";
 
+  private static waypointId(project?: string): string {
+    if (!project?.length) return MemoryService.UUID_ZERO;
+    const hex = createHash("sha256").update(`waypoint:${project}`).digest("hex");
+    // Format as UUID: 8-4-4-4-12
+    return [
+      hex.slice(0, 8),
+      hex.slice(8, 12),
+      hex.slice(12, 16),
+      hex.slice(16, 20),
+      hex.slice(20, 32),
+    ].join("-");
+  }
+
   async setWaypoint(args: {
     project: string;
     branch?: string;
@@ -336,7 +353,7 @@ ${list(args.memory_ids)}`;
     };
 
     const memory: Memory = {
-      id: MemoryService.UUID_ZERO,
+      id: MemoryService.waypointId(args.project),
       content,
       embedding: new Array(this.embeddings.dimension).fill(0),
       metadata,
@@ -352,7 +369,7 @@ ${list(args.memory_ids)}`;
     return memory;
   }
 
-  async getLatestWaypoint(): Promise<Memory | null> {
-    return await this.get(MemoryService.UUID_ZERO);
+  async getLatestWaypoint(project?: string): Promise<Memory | null> {
+    return await this.get(MemoryService.waypointId(project));
   }
 }

package/src/types/conversation.ts

@@ -118,4 +118,5 @@ export interface SearchOptions {
   historyOnly?: boolean;
   historyWeight?: number;
   historyFilters?: HistoryFilters;
+  offset?: number;
 }