@aeriondyseti/vector-memory-mcp 2.2.2 → 2.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeriondyseti/vector-memory-mcp",
-  "version": "2.2.2",
+  "version": "2.2.3",
   "description": "A zero-configuration RAG memory server for MCP clients",
   "type": "module",
   "main": "src/index.ts",

package/scripts/lancedb-extract.ts

@@ -14,24 +14,59 @@ if (!source) {
   process.exit(1);
 }
 
-/**
- * Read a value from an Arrow column at a given row index.
- * Arrow timestamp columns return BigInt — we convert to epoch-ms here
- * without going through Arrow's bigIntToNumber safety check.
- */
+// Arrow TimeUnit enum → divisor to convert to milliseconds.
+// 0=SECOND, 1=MILLISECOND, 2=MICROSECOND, 3=NANOSECOND
+// Negative divisor = multiply (seconds → ms needs ×1000).
+const TIME_UNIT_TO_MS_DIVISOR: Record<number, bigint> = {
+  0: -1000n,   // seconds → ms (multiply by 1000)
+  1: 1n,       // ms → no conversion
+  2: 1000n,    // μs → ms
+  3: 1000000n, // ns → ms
+};
+
+function buildTimestampDivisors(schema: any): Map<string, bigint> {
+  const map = new Map<string, bigint>();
+  for (const field of schema.fields) {
+    if (field.type.typeId === 10) {
+      map.set(field.name, TIME_UNIT_TO_MS_DIVISOR[field.type.unit] ?? 1n);
+    }
+  }
+  return map;
+}
+
 function columnValue(batch: any, colName: string, rowIdx: number): unknown {
   const col = batch.getChild(colName);
   if (!col) return undefined;
-  return col.get(rowIdx);
+  try {
+    return col.get(rowIdx);
+  } catch {
+    // Arrow's getter can throw on BigInt timestamps exceeding MAX_SAFE_INTEGER;
+    // fall back to the raw typed array.
+    let offset = rowIdx;
+    for (const data of col.data) {
+      if (offset < data.length) {
+        return (data.values instanceof BigInt64Array || data.values instanceof BigUint64Array)
+          ? data.values[offset]
+          : null;
+      }
+      offset -= data.length;
+    }
+    return null;
+  }
 }
 
-function toEpochMs(value: unknown): number {
-  if (typeof value === "number") return value;
+function toEpochMs(value: unknown, divisor: bigint = 1n): number {
+  if (value == null) return Date.now();
   if (value instanceof Date) return value.getTime();
   if (typeof value === "bigint") {
-    // Arrow timestamps are microseconds; convert to milliseconds.
-    const ms = value / 1000n;
-    return Number(ms);
+    if (divisor < 0n) return Number(value * -divisor);  // seconds → ms
+    if (divisor === 1n) return Number(value);
+    return Number(value / divisor);
+  }
+  if (typeof value === "number") {
+    if (divisor < 0n) return value * Number(-divisor);
+    if (divisor === 1n) return value;
+    return Math.floor(value / Number(divisor));
   }
   return Date.now();
 }
@@ -49,6 +84,7 @@ function toFloatArray(vec: unknown): number[] {
   return [];
 }
 
+const BATCH_SIZE = 100;
 const lancedb = await import("@lancedb/lancedb");
 const db = await lancedb.connect(source);
 const tableNames = await db.tableNames();
@@ -64,27 +100,45 @@ if (tableNames.includes("memories")) {
   const total = await table.countRows();
   console.error(`Reading ${total} memories...`);
 
-  // Use toArrow() to get raw Arrow RecordBatches, bypassing StructRow
-  // property accessors that throw on BigInt timestamps.
-  const arrowTable = await table.query().toArrow();
-  for (const batch of arrowTable.batches) {
-    for (let i = 0; i < batch.numRows; i++) {
-      const lastAccessed = columnValue(batch, "last_accessed", i);
-      result.memories.push({
-        id: columnValue(batch, "id", i),
-        content: columnValue(batch, "content", i),
-        metadata: columnValue(batch, "metadata", i) ?? "{}",
-        vector: toFloatArray(columnValue(batch, "vector", i)),
-        created_at: toEpochMs(columnValue(batch, "created_at", i)),
-        updated_at: toEpochMs(columnValue(batch, "updated_at", i)),
-        last_accessed: lastAccessed != null ? toEpochMs(lastAccessed) : null,
-        superseded_by: columnValue(batch, "superseded_by", i) ?? null,
-        usefulness: columnValue(batch, "usefulness", i) ?? 0,
-        access_count: columnValue(batch, "access_count", i) ?? 0,
-      });
+  // Paginated scan — query().toArrow() without offset/limit returns
+  // non-deterministic results that can duplicate some rows and skip others.
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const lastAccessed = columnValue(batch, "last_accessed", i);
+        const accessedMs = lastAccessed != null ? toEpochMs(lastAccessed, tsDivisors.get("last_accessed")) : null;
+        // Deduplicate by ID: prefer most recently accessed, then longest content.
+        const existing = seen.get(id);
+        if (existing) {
+          const existingAccess = existing.last_accessed ?? 0;
+          const newAccess = accessedMs ?? 0;
+          if (newAccess < existingAccess) continue;
+          if (newAccess === existingAccess && content.length <= existing.content.length) continue;
+        }
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          updated_at: toEpochMs(columnValue(batch, "updated_at", i), tsDivisors.get("updated_at")),
+          last_accessed: accessedMs,
+          superseded_by: columnValue(batch, "superseded_by", i) ?? null,
+          usefulness: columnValue(batch, "usefulness", i) ?? 0,
+          access_count: columnValue(batch, "access_count", i) ?? 0,
+        });
+      }
     }
   }
-  console.error(`  ${result.memories.length} memories read`);
+  result.memories = [...seen.values()];
+  console.error(`  ${result.memories.length} unique memories read (${total} rows scanned)`);
 }
 
 if (tableNames.includes("conversation_history")) {
@@ -92,24 +146,35 @@ if (tableNames.includes("conversation_history")) {
   const total = await table.countRows();
   console.error(`Reading ${total} conversation chunks...`);
 
-  const arrowTable = await table.query().toArrow();
-  for (const batch of arrowTable.batches) {
-    for (let i = 0; i < batch.numRows; i++) {
-      result.conversations.push({
-        id: columnValue(batch, "id", i),
-        content: columnValue(batch, "content", i),
-        metadata: columnValue(batch, "metadata", i) ?? "{}",
-        vector: toFloatArray(columnValue(batch, "vector", i)),
-        created_at: toEpochMs(columnValue(batch, "created_at", i)),
-        session_id: columnValue(batch, "session_id", i),
-        role: columnValue(batch, "role", i),
-        message_index_start: columnValue(batch, "message_index_start", i) ?? 0,
-        message_index_end: columnValue(batch, "message_index_end", i) ?? 0,
-        project: columnValue(batch, "project", i) ?? "",
-      });
+  const schemaSample = await table.query().limit(1).toArrow();
+  const tsDivisors = buildTimestampDivisors(schemaSample.schema);
+  const seen = new Map<string, any>();
+
+  for (let offset = 0; offset < total; offset += BATCH_SIZE) {
+    const arrowTable = await table.query().offset(offset).limit(BATCH_SIZE).toArrow();
+    for (const batch of arrowTable.batches) {
+      for (let i = 0; i < batch.numRows; i++) {
+        const id = columnValue(batch, "id", i) as string;
+        const content = columnValue(batch, "content", i) as string;
+        const existing = seen.get(id);
+        if (existing && existing.content.length >= content.length) continue;
+        seen.set(id, {
+          id,
+          content,
+          metadata: columnValue(batch, "metadata", i) ?? "{}",
+          vector: toFloatArray(columnValue(batch, "vector", i)),
+          created_at: toEpochMs(columnValue(batch, "created_at", i), tsDivisors.get("created_at")),
+          session_id: columnValue(batch, "session_id", i),
+          role: columnValue(batch, "role", i),
+          message_index_start: columnValue(batch, "message_index_start", i) ?? 0,
+          message_index_end: columnValue(batch, "message_index_end", i) ?? 0,
+          project: columnValue(batch, "project", i) ?? "",
+        });
+      }
     }
   }
-  console.error(`  ${result.conversations.length} conversation chunks read`);
+  result.conversations = [...seen.values()];
+  console.error(`  ${result.conversations.length} unique conversation chunks read (${total} rows scanned)`);
 }
 
 await db.close?.();