@mnemoai/core 1.1.0

package/src/adapters/lancedb.ts

@@ -0,0 +1,237 @@
+// SPDX-License-Identifier: MIT
+/**
+ * LanceDB Storage Adapter — Default backend for Mnemo.
+ *
+ * Implements StorageAdapter using @lancedb/lancedb.
+ * This is the reference implementation; other backends should
+ * produce equivalent behavior.
+ */
+
+import type {
+  StorageAdapter,
+  MemoryRecord,
+  SearchResult,
+  QueryOptions,
+} from "../storage-adapter.js";
+import { registerAdapter } from "../storage-adapter.js";
+
+/** Strict allowlist sanitizer — prevents SQL injection in LanceDB filters */
+function sanitize(value: string): string {
+  if (typeof value !== "string") return "";
+  return value.replace(/[^a-zA-Z0-9\-_.:@ \u4e00-\u9fff\u3400-\u4dbf]/g, "");
+}
+
+// Dynamic import to avoid hard dependency at module level
+let _lancedb: typeof import("@lancedb/lancedb") | null = null;
+
+async function loadLanceDB() {
+  if (!_lancedb) {
+    _lancedb = await import("@lancedb/lancedb");
+  }
+  return _lancedb;
+}
+
+const TABLE_NAME = "memories";
+
+export class LanceDBAdapter implements StorageAdapter {
+  readonly name = "lancedb";
+
+  private db: any = null;
+  private table: any = null;
+  private ftsReady = false;
+  private vectorDim = 0;
+
+  async connect(dbPath: string): Promise<void> {
+    const lancedb = await loadLanceDB();
+    this.db = await lancedb.connect(dbPath);
+  }
+
+  async ensureTable(vectorDimensions: number): Promise<void> {
+    this.vectorDim = vectorDimensions;
+
+    try {
+      this.table = await this.db.openTable(TABLE_NAME);
+    } catch {
+      // Table doesn't exist — create with schema
+      const lancedb = await loadLanceDB();
+      const schemaEntry: MemoryRecord = {
+        id: "__schema__",
+        text: "",
+        vector: new Array(vectorDimensions).fill(0),
+        timestamp: 0,
+        scope: "global",
+        importance: 0,
+        category: "other",
+        metadata: "{}",
+      };
+
+      try {
+        this.table = await this.db.createTable(TABLE_NAME, [schemaEntry]);
+        await this.table.delete('id = "__schema__"');
+      } catch (err) {
+        if (String(err).includes("already exists")) {
+          this.table = await this.db.openTable(TABLE_NAME);
+        } else {
+          throw err;
+        }
+      }
+    }
+
+    // Validate dimensions
+    const sample = await this.table.query().limit(1).toArray();
+    if (sample.length > 0 && sample[0]?.vector?.length) {
+      const existing = sample[0].vector.length;
+      if (existing !== vectorDimensions) {
+        throw new Error(
+          `Vector dimension mismatch: table=${existing}, config=${vectorDimensions}`
+        );
+      }
+    }
+
+    // Create FTS index
+    await this.ensureFullTextIndex();
+  }
+
+  async add(records: MemoryRecord[]): Promise<void> {
+    if (!this.table) throw new Error("Table not initialized");
+    await this.table.add(records);
+  }
+
+  async update(id: string, record: MemoryRecord): Promise<void> {
+    if (!this.table) throw new Error("Table not initialized");
+    await this.table.delete(`id = '${sanitize(id)}'`);
+    await this.table.add([record]);
+  }
+
+  async delete(filter: string): Promise<void> {
+    if (!this.table) throw new Error("Table not initialized");
+    await this.table.delete(filter);
+  }
+
+  async vectorSearch(
+    vector: number[],
+    limit: number,
+    minScore = 0,
+    scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    if (!this.table) throw new Error("Table not initialized");
+
+    let query = this.table.vectorSearch(vector).distanceType("cosine").limit(limit * 3);
+
+    if (scopeFilter?.length) {
+      const scopeExpr = scopeFilter.map((s) => `'${sanitize(s)}'`).join(", ");
+      query = query.where(`scope IN (${scopeExpr})`);
+    }
+
+    const raw = await query.toArray();
+
+    return raw
+      .map((row: any) => {
+        const distance = row._distance ?? row.distance ?? 1;
+        const score = 1 / (1 + distance);
+        return { record: this.toRecord(row), score };
+      })
+      .filter((r: SearchResult) => r.score >= minScore)
+      .slice(0, limit);
+  }
+
+  async fullTextSearch(
+    queryText: string,
+    limit: number,
+    scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    if (!this.table || !this.ftsReady) return [];
+
+    let query = this.table.search(queryText, "fts").limit(limit * 2);
+
+    if (scopeFilter?.length) {
+      const scopeExpr = scopeFilter.map((s) => `'${sanitize(s)}'`).join(", ");
+      query = query.where(`scope IN (${scopeExpr})`);
+    }
+
+    const raw = await query.toArray();
+
+    return raw
+      .map((row: any) => {
+        const score = row._relevance_score ?? row.score ?? 0.5;
+        return { record: this.toRecord(row), score };
+      })
+      .slice(0, limit);
+  }
+
+  async query(options: QueryOptions): Promise<MemoryRecord[]> {
+    if (!this.table) throw new Error("Table not initialized");
+
+    let q = this.table.query();
+
+    if (options.select?.length) {
+      q = q.select(options.select);
+    }
+    if (options.where) {
+      q = q.where(options.where);
+    }
+    if (options.limit) {
+      q = q.limit(options.limit);
+    }
+
+    const raw = await q.toArray();
+    return raw.map((row: any) => this.toRecord(row));
+  }
+
+  async count(filter?: string): Promise<number> {
+    if (!this.table) throw new Error("Table not initialized");
+    let q = this.table.query();
+    if (filter) q = q.where(filter);
+    const rows = await q.toArray();
+    return rows.length;
+  }
+
+  async ensureFullTextIndex(): Promise<void> {
+    if (!this.table) return;
+
+    try {
+      const indices = await this.table.listIndices();
+      const hasFts = indices?.some(
+        (idx: any) => idx.indexType === "FTS" || idx.columns?.includes("text"),
+      );
+
+      if (!hasFts) {
+        const lancedb = await loadLanceDB();
+        await this.table.createIndex("text", {
+          config: (lancedb as any).Index.fts(),
+        });
+      }
+      this.ftsReady = true;
+    } catch {
+      this.ftsReady = false;
+    }
+  }
+
+  hasFullTextSearch(): boolean {
+    return this.ftsReady;
+  }
+
+  async close(): Promise<void> {
+    this.table = null;
+    this.db = null;
+  }
+
+  // ── Helpers ──
+
+  private toRecord(row: any): MemoryRecord {
+    return {
+      id: row.id,
+      text: row.text,
+      vector: row.vector ? Array.from(row.vector) : [],
+      timestamp: row.timestamp ?? 0,
+      scope: row.scope ?? "global",
+      importance: row.importance ?? 0.5,
+      category: row.category ?? "other",
+      metadata: row.metadata ?? "{}",
+      ...row, // preserve extra fields
+    };
+  }
+}
+
+// ── Auto-register ──
+registerAdapter("lancedb", () => new LanceDBAdapter());

package/src/adapters/pgvector.ts

@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: MIT
+/**
+ * PGVector Storage Adapter for Mnemo
+ *
+ * Requirements:
+ *   npm install pg pgvector
+ *   PostgreSQL with pgvector extension enabled
+ *
+ * Config:
+ *   storage: "pgvector"
+ *   storageConfig: { connectionString: "postgres://user:pass@localhost:5432/mnemo" }
+ */
+
+import type {
+  StorageAdapter,
+  MemoryRecord,
+  SearchResult,
+  QueryOptions,
+} from "../storage-adapter.js";
+import { registerAdapter } from "../storage-adapter.js";
+
+const TABLE = "mnemo_memories";
+
+export class PGVectorAdapter implements StorageAdapter {
+  readonly name = "pgvector";
+
+  private pool: any = null;
+  private vectorDim = 0;
+  private connectionString: string;
+
+  constructor(config?: Record<string, unknown>) {
+    this.connectionString = (config?.connectionString as string) ||
+      "postgres://localhost:5432/mnemo";
+  }
+
+  async connect(dbPath: string): Promise<void> {
+    const { Pool } = await import("pg");
+    this.pool = new Pool({
+      connectionString: dbPath || this.connectionString,
+    });
+
+    // Enable pgvector extension
+    await this.pool.query("CREATE EXTENSION IF NOT EXISTS vector");
+  }
+
+  async ensureTable(vectorDimensions: number): Promise<void> {
+    this.vectorDim = vectorDimensions;
+
+    await this.pool.query(`
+      CREATE TABLE IF NOT EXISTS ${TABLE} (
+        id TEXT PRIMARY KEY,
+        text TEXT NOT NULL,
+        vector vector(${vectorDimensions}),
+        timestamp BIGINT DEFAULT 0,
+        scope TEXT DEFAULT 'global',
+        importance REAL DEFAULT 0.5,
+        category TEXT DEFAULT 'other',
+        metadata JSONB DEFAULT '{}'::jsonb
+      )
+    `);
+
+    // Create HNSW index for vector search
+    await this.pool.query(`
+      CREATE INDEX IF NOT EXISTS ${TABLE}_vector_idx
+      ON ${TABLE} USING hnsw (vector vector_cosine_ops)
+    `).catch(() => {}); // ignore if already exists
+
+    // Create GIN index for full-text search
+    await this.pool.query(`
+      CREATE INDEX IF NOT EXISTS ${TABLE}_text_idx
+      ON ${TABLE} USING gin (to_tsvector('simple', text))
+    `).catch(() => {});
+
+    // Create index on scope for filtering
+    await this.pool.query(`
+      CREATE INDEX IF NOT EXISTS ${TABLE}_scope_idx ON ${TABLE} (scope)
+    `).catch(() => {});
+  }
+
+  async add(records: MemoryRecord[]): Promise<void> {
+    for (const r of records) {
+      await this.pool.query(
+        `INSERT INTO ${TABLE} (id, text, vector, timestamp, scope, importance, category, metadata)
+         VALUES ($1, $2, $3::vector, $4, $5, $6, $7, $8)
+         ON CONFLICT (id) DO UPDATE SET
+           text = EXCLUDED.text,
+           vector = EXCLUDED.vector,
+           timestamp = EXCLUDED.timestamp,
+           scope = EXCLUDED.scope,
+           importance = EXCLUDED.importance,
+           category = EXCLUDED.category,
+           metadata = EXCLUDED.metadata`,
+        [r.id, r.text, `[${r.vector.join(",")}]`, r.timestamp, r.scope, r.importance, r.category, r.metadata],
+      );
+    }
+  }
+
+  async update(id: string, record: MemoryRecord): Promise<void> {
+    await this.add([record]);
+  }
+
+  async delete(filter: string): Promise<void> {
+    const idMatch = filter.match(/id\s*=\s*'([^']+)'/);
+    if (idMatch) {
+      await this.pool.query(`DELETE FROM ${TABLE} WHERE id = $1`, [idMatch[1]]);
+    } else {
+      // Pass filter as-is for simple SQL WHERE clauses
+      await this.pool.query(`DELETE FROM ${TABLE} WHERE ${filter}`);
+    }
+  }
+
+  async vectorSearch(
+    vector: number[],
+    limit: number,
+    minScore = 0,
+    scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    const vectorStr = `[${vector.join(",")}]`;
+    let query = `
+      SELECT *, 1 - (vector <=> $1::vector) AS score
+      FROM ${TABLE}
+      WHERE 1 - (vector <=> $1::vector) >= $2
+    `;
+    const params: any[] = [vectorStr, minScore];
+
+    if (scopeFilter?.length) {
+      query += ` AND scope = ANY($3)`;
+      params.push(scopeFilter);
+    }
+
+    query += ` ORDER BY vector <=> $1::vector LIMIT $${params.length + 1}`;
+    params.push(limit);
+
+    const result = await this.pool.query(query, params);
+
+    return result.rows.map((row: any) => ({
+      record: this.toRecord(row),
+      score: parseFloat(row.score),
+    }));
+  }
+
+  async fullTextSearch(
+    queryText: string,
+    limit: number,
+    scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    // Use PostgreSQL full-text search with ts_rank
+    let query = `
+      SELECT *, ts_rank(to_tsvector('simple', text), plainto_tsquery('simple', $1)) AS score
+      FROM ${TABLE}
+      WHERE to_tsvector('simple', text) @@ plainto_tsquery('simple', $1)
+    `;
+    const params: any[] = [queryText];
+
+    if (scopeFilter?.length) {
+      query += ` AND scope = ANY($2)`;
+      params.push(scopeFilter);
+    }
+
+    query += ` ORDER BY score DESC LIMIT $${params.length + 1}`;
+    params.push(limit);
+
+    const result = await this.pool.query(query, params);
+
+    return result.rows.map((row: any) => ({
+      record: this.toRecord(row),
+      score: parseFloat(row.score),
+    }));
+  }
+
+  async query(options: QueryOptions): Promise<MemoryRecord[]> {
+    let query = `SELECT * FROM ${TABLE}`;
+    if (options.where) query += ` WHERE ${options.where}`;
+    query += ` LIMIT ${options.limit || 100}`;
+
+    const result = await this.pool.query(query);
+    return result.rows.map((row: any) => this.toRecord(row));
+  }
+
+  async count(filter?: string): Promise<number> {
+    let query = `SELECT COUNT(*) FROM ${TABLE}`;
+    if (filter) query += ` WHERE ${filter}`;
+    const result = await this.pool.query(query);
+    return parseInt(result.rows[0].count);
+  }
+
+  async ensureFullTextIndex(): Promise<void> {
+    // Created in ensureTable
+  }
+
+  hasFullTextSearch(): boolean {
+    return true; // PostgreSQL has native full-text search
+  }
+
+  async close(): Promise<void> {
+    if (this.pool) {
+      await this.pool.end();
+      this.pool = null;
+    }
+  }
+
+  // ── Helpers ──
+
+  private toRecord(row: any): MemoryRecord {
+    return {
+      id: row.id,
+      text: row.text,
+      vector: row.vector ? (typeof row.vector === "string" ? JSON.parse(row.vector) : Array.from(row.vector)) : [],
+      timestamp: parseInt(row.timestamp) || 0,
+      scope: row.scope ?? "global",
+      importance: parseFloat(row.importance) || 0.5,
+      category: row.category ?? "other",
+      metadata: typeof row.metadata === "object" ? JSON.stringify(row.metadata) : (row.metadata ?? "{}"),
+    };
+  }
+}
+
+registerAdapter("pgvector", (config) => new PGVectorAdapter(config));

package/src/adapters/qdrant.ts

@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: MIT
+/**
+ * Qdrant Storage Adapter for Mnemo
+ *
+ * Requirements:
+ *   npm install @qdrant/js-client-rest
+ *
+ * Config:
+ *   storage: "qdrant"
+ *   storageConfig: { url: "http://localhost:6333", apiKey?: "..." }
+ */
+
+import type {
+  StorageAdapter,
+  MemoryRecord,
+  SearchResult,
+  QueryOptions,
+} from "../storage-adapter.js";
+import { registerAdapter } from "../storage-adapter.js";
+
+const COLLECTION = "mnemo_memories";
+
+export class QdrantAdapter implements StorageAdapter {
+  readonly name = "qdrant";
+
+  private client: any = null;
+  private url: string = "http://localhost:6333";
+  private apiKey?: string;
+  private vectorDim = 0;
+
+  constructor(config?: Record<string, unknown>) {
+    if (config?.url) this.url = config.url as string;
+    if (config?.apiKey) this.apiKey = config.apiKey as string;
+  }
+
+  async connect(): Promise<void> {
+    const { QdrantClient } = await import("@qdrant/js-client-rest");
+    this.client = new QdrantClient({
+      url: this.url,
+      ...(this.apiKey ? { apiKey: this.apiKey } : {}),
+    });
+  }
+
+  async ensureTable(vectorDimensions: number): Promise<void> {
+    this.vectorDim = vectorDimensions;
+    const collections = await this.client.getCollections();
+    const exists = collections.collections.some((c: any) => c.name === COLLECTION);
+
+    if (!exists) {
+      await this.client.createCollection(COLLECTION, {
+        vectors: { size: vectorDimensions, distance: "Cosine" },
+      });
+      // Create payload indices for filtering
+      await this.client.createPayloadIndex(COLLECTION, {
+        field_name: "scope",
+        field_schema: "keyword",
+      });
+      await this.client.createPayloadIndex(COLLECTION, {
+        field_name: "category",
+        field_schema: "keyword",
+      });
+    }
+  }
+
+  async add(records: MemoryRecord[]): Promise<void> {
+    const points = records.map((r) => ({
+      id: r.id,
+      vector: r.vector,
+      payload: {
+        text: r.text,
+        timestamp: r.timestamp,
+        scope: r.scope,
+        importance: r.importance,
+        category: r.category,
+        metadata: r.metadata,
+      },
+    }));
+    await this.client.upsert(COLLECTION, { points });
+  }
+
+  async update(id: string, record: MemoryRecord): Promise<void> {
+    await this.add([record]);
+  }
+
+  async delete(filter: string): Promise<void> {
+    // Parse simple "id = 'xxx'" filter
+    const idMatch = filter.match(/id\s*=\s*'([^']+)'/);
+    if (idMatch) {
+      await this.client.delete(COLLECTION, {
+        points: [idMatch[1]],
+      });
+    }
+  }
+
+  async vectorSearch(
+    vector: number[],
+    limit: number,
+    minScore = 0,
+    scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    const filter = scopeFilter?.length
+      ? { must: [{ key: "scope", match: { any: scopeFilter } }] }
+      : undefined;
+
+    const results = await this.client.search(COLLECTION, {
+      vector,
+      limit,
+      with_payload: true,
+      score_threshold: minScore,
+      ...(filter ? { filter } : {}),
+    });
+
+    return results.map((r: any) => ({
+      record: this.toRecord(r.id, r.payload),
+      score: r.score,
+    }));
+  }
+
+  async fullTextSearch(
+    _query: string,
+    _limit: number,
+    _scopeFilter?: string[],
+  ): Promise<SearchResult[]> {
+    // Qdrant doesn't have native BM25 — fall back to empty
+    // Users should pair with a separate FTS engine or use vector search
+    return [];
+  }
+
+  async query(options: QueryOptions): Promise<MemoryRecord[]> {
+    const filter = options.where
+      ? this.parseFilter(options.where)
+      : undefined;
+
+    const result = await this.client.scroll(COLLECTION, {
+      limit: options.limit || 100,
+      with_payload: true,
+      with_vectors: true,
+      ...(filter ? { filter } : {}),
+    });
+
+    return result.points.map((p: any) => this.toRecord(p.id, p.payload, p.vector));
+  }
+
+  async count(filter?: string): Promise<number> {
+    const result = await this.client.count(COLLECTION, {
+      ...(filter ? { filter: this.parseFilter(filter) } : {}),
+      exact: true,
+    });
+    return result.count;
+  }
+
+  async ensureFullTextIndex(): Promise<void> {
+    // Qdrant uses payload indices, not FTS indices
+    // Text search via Qdrant requires external FTS or payload keyword match
+  }
+
+  hasFullTextSearch(): boolean {
+    return false; // Qdrant doesn't have native BM25
+  }
+
+  async close(): Promise<void> {
+    this.client = null;
+  }
+
+  // ── Helpers ──
+
+  private toRecord(id: string, payload: any, vector?: number[]): MemoryRecord {
+    return {
+      id,
+      text: payload.text ?? "",
+      vector: vector ? Array.from(vector) : [],
+      timestamp: payload.timestamp ?? 0,
+      scope: payload.scope ?? "global",
+      importance: payload.importance ?? 0.5,
+      category: payload.category ?? "other",
+      metadata: payload.metadata ?? "{}",
+    };
+  }
+
+  private parseFilter(where: string): any {
+    // Simple parser for common filters
+    const scopeMatch = where.match(/scope\s+IN\s*\(([^)]+)\)/i);
+    if (scopeMatch) {
+      const scopes = scopeMatch[1].split(",").map((s) => s.trim().replace(/'/g, ""));
+      return { must: [{ key: "scope", match: { any: scopes } }] };
+    }
+    return undefined;
+  }
+}
+
+registerAdapter("qdrant", (config) => new QdrantAdapter(config));