memory-lancedb-pro 1.0.0

package/src/embedder.ts

@@ -0,0 +1,354 @@
+/**
+ * Embedding Abstraction Layer
+ * OpenAI-compatible API for various embedding providers.
+ *
+ * Note: Some providers (e.g. Jina) support extra parameters like `task` and
+ * `normalized` on the embeddings endpoint. The OpenAI SDK types do not include
+ * these fields, so we pass them via a narrow `any` cast.
+ */
+
+import OpenAI from "openai";
+import { createHash } from "node:crypto";
+
+// ============================================================================
+// Embedding Cache (LRU with TTL)
+// ============================================================================
+
+interface CacheEntry {
+  vector: number[];
+  createdAt: number;
+}
+
+class EmbeddingCache {
+  private cache = new Map<string, CacheEntry>();
+  private readonly maxSize: number;
+  private readonly ttlMs: number;
+  public hits = 0;
+  public misses = 0;
+
+  constructor(maxSize = 256, ttlMinutes = 30) {
+    this.maxSize = maxSize;
+    this.ttlMs = ttlMinutes * 60_000;
+  }
+
+  private key(text: string, task?: string): string {
+    const hash = createHash("sha256").update(`${task || ""}:${text}`).digest("hex").slice(0, 24);
+    return hash;
+  }
+
+  get(text: string, task?: string): number[] | undefined {
+    const k = this.key(text, task);
+    const entry = this.cache.get(k);
+    if (!entry) {
+      this.misses++;
+      return undefined;
+    }
+    if (Date.now() - entry.createdAt > this.ttlMs) {
+      this.cache.delete(k);
+      this.misses++;
+      return undefined;
+    }
+    // Move to end (most recently used)
+    this.cache.delete(k);
+    this.cache.set(k, entry);
+    this.hits++;
+    return entry.vector;
+  }
+
+  set(text: string, task: string | undefined, vector: number[]): void {
+    const k = this.key(text, task);
+    // Evict oldest if full
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey !== undefined) this.cache.delete(firstKey);
+    }
+    this.cache.set(k, { vector, createdAt: Date.now() });
+  }
+
+  get size(): number { return this.cache.size; }
+  get stats(): { size: number; hits: number; misses: number; hitRate: string } {
+    const total = this.hits + this.misses;
+    return {
+      size: this.cache.size,
+      hits: this.hits,
+      misses: this.misses,
+      hitRate: total > 0 ? `${((this.hits / total) * 100).toFixed(1)}%` : "N/A",
+    };
+  }
+}
+
+// ============================================================================
+// Types & Configuration
+// ============================================================================
+
+export interface EmbeddingConfig {
+  provider: "openai-compatible";
+  apiKey: string;
+  model: string;
+  baseURL?: string;
+  dimensions?: number;
+
+  /** Optional task type for query embeddings (e.g. "retrieval.query") */
+  taskQuery?: string;
+  /** Optional task type for passage/document embeddings (e.g. "retrieval.passage") */
+  taskPassage?: string;
+  /** Optional flag to request normalized embeddings (provider-dependent, e.g. Jina v5) */
+  normalized?: boolean;
+}
+
+// Known embedding model dimensions
+const EMBEDDING_DIMENSIONS: Record<string, number> = {
+  "text-embedding-3-small": 1536,
+  "text-embedding-3-large": 3072,
+  "text-embedding-004": 768,
+  "gemini-embedding-001": 3072,
+  "nomic-embed-text": 768,
+  "mxbai-embed-large": 1024,
+  "BAAI/bge-m3": 1024,
+  "all-MiniLM-L6-v2": 384,
+  "all-mpnet-base-v2": 768,
+
+  // Jina v5
+  "jina-embeddings-v5-text-small": 1024,
+  "jina-embeddings-v5-text-nano": 768,
+};
+
+// ============================================================================
+// Utility Functions
+// ============================================================================
+
+function resolveEnvVars(value: string): string {
+  return value.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
+    const envValue = process.env[envVar];
+    if (!envValue) {
+      throw new Error(`Environment variable ${envVar} is not set`);
+    }
+    return envValue;
+  });
+}
+
+export function getVectorDimensions(model: string, overrideDims?: number): number {
+  if (overrideDims && overrideDims > 0) {
+    return overrideDims;
+  }
+
+  const dims = EMBEDDING_DIMENSIONS[model];
+  if (!dims) {
+    throw new Error(
+      `Unsupported embedding model: ${model}. Either add it to EMBEDDING_DIMENSIONS or set embedding.dimensions in config.`
+    );
+  }
+
+  return dims;
+}
+
+// ============================================================================
+// Embedder Class
+// ============================================================================
+
+export class Embedder {
+  private client: OpenAI;
+  public readonly dimensions: number;
+  private readonly _cache: EmbeddingCache;
+
+  private readonly _model: string;
+  private readonly _taskQuery?: string;
+  private readonly _taskPassage?: string;
+  private readonly _normalized?: boolean;
+
+  constructor(config: EmbeddingConfig) {
+    // Resolve environment variables in API key
+    const resolvedApiKey = resolveEnvVars(config.apiKey);
+
+    this._model = config.model;
+    this._taskQuery = config.taskQuery;
+    this._taskPassage = config.taskPassage;
+    this._normalized = config.normalized;
+
+    this.client = new OpenAI({
+      apiKey: resolvedApiKey,
+      ...(config.baseURL ? { baseURL: config.baseURL } : {}),
+    });
+
+    this.dimensions = getVectorDimensions(config.model, config.dimensions);
+    this._cache = new EmbeddingCache(256, 30); // 256 entries, 30 min TTL
+  }
+
+  // --------------------------------------------------------------------------
+  // Backward-compatible API
+  // --------------------------------------------------------------------------
+
+  /**
+   * Backward-compatible embedding API.
+   *
+   * Historically the plugin used a single `embed()` method for both query and
+   * passage embeddings. With task-aware providers we treat this as passage.
+   */
+  async embed(text: string): Promise<number[]> {
+    return this.embedPassage(text);
+  }
+
+  /** Backward-compatible batch embedding API (treated as passage). */
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    return this.embedBatchPassage(texts);
+  }
+
+  // --------------------------------------------------------------------------
+  // Task-aware API
+  // --------------------------------------------------------------------------
+
+  async embedQuery(text: string): Promise<number[]> {
+    return this.embedSingle(text, this._taskQuery);
+  }
+
+  async embedPassage(text: string): Promise<number[]> {
+    return this.embedSingle(text, this._taskPassage);
+  }
+
+  async embedBatchQuery(texts: string[]): Promise<number[][]> {
+    return this.embedMany(texts, this._taskQuery);
+  }
+
+  async embedBatchPassage(texts: string[]): Promise<number[][]> {
+    return this.embedMany(texts, this._taskPassage);
+  }
+
+  // --------------------------------------------------------------------------
+  // Internals
+  // --------------------------------------------------------------------------
+
+  private validateEmbedding(embedding: number[]): void {
+    if (!Array.isArray(embedding)) {
+      throw new Error(`Embedding is not an array (got ${typeof embedding})`);
+    }
+    if (embedding.length !== this.dimensions) {
+      throw new Error(
+        `Embedding dimension mismatch: expected ${this.dimensions}, got ${embedding.length}`
+      );
+    }
+  }
+
+  private buildPayload(input: string | string[], task?: string): any {
+    const payload: any = {
+      model: this.model,
+      input,
+    };
+
+    if (task) payload.task = task;
+    if (this._normalized !== undefined) payload.normalized = this._normalized;
+
+    return payload;
+  }
+
+  private async embedSingle(text: string, task?: string): Promise<number[]> {
+    if (!text || text.trim().length === 0) {
+      throw new Error("Cannot embed empty text");
+    }
+
+    // Check cache first
+    const cached = this._cache.get(text, task);
+    if (cached) return cached;
+
+    try {
+      const response = await this.client.embeddings.create(this.buildPayload(text, task) as any);
+      const embedding = response.data[0]?.embedding as number[] | undefined;
+      if (!embedding) {
+        throw new Error("No embedding returned from provider");
+      }
+
+      this.validateEmbedding(embedding);
+      this._cache.set(text, task, embedding);
+      return embedding;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new Error(`Failed to generate embedding: ${error.message}`, { cause: error });
+      }
+      throw new Error(`Failed to generate embedding: ${String(error)}`);
+    }
+  }
+
+  private async embedMany(texts: string[], task?: string): Promise<number[][]> {
+    if (!texts || texts.length === 0) {
+      return [];
+    }
+
+    // Filter out empty texts and track indices
+    const validTexts: string[] = [];
+    const validIndices: number[] = [];
+
+    texts.forEach((text, index) => {
+      if (text && text.trim().length > 0) {
+        validTexts.push(text);
+        validIndices.push(index);
+      }
+    });
+
+    if (validTexts.length === 0) {
+      return texts.map(() => []);
+    }
+
+    try {
+      const response = await this.client.embeddings.create(
+        this.buildPayload(validTexts, task) as any
+      );
+
+      // Create result array with proper length
+      const results: number[][] = new Array(texts.length);
+
+      // Fill in embeddings for valid texts
+      response.data.forEach((item, idx) => {
+        const originalIndex = validIndices[idx];
+        const embedding = item.embedding as number[];
+
+        this.validateEmbedding(embedding);
+        results[originalIndex] = embedding;
+      });
+
+      // Fill empty arrays for invalid texts
+      for (let i = 0; i < texts.length; i++) {
+        if (!results[i]) {
+          results[i] = [];
+        }
+      }
+
+      return results;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new Error(`Failed to generate batch embeddings: ${error.message}`, { cause: error });
+      }
+      throw new Error(`Failed to generate batch embeddings: ${String(error)}`);
+    }
+  }
+
+  get model(): string {
+    return this._model;
+  }
+
+  // Test connection and validate configuration
+  async test(): Promise<{ success: boolean; error?: string; dimensions?: number }> {
+    try {
+      const testEmbedding = await this.embedPassage("test");
+      return {
+        success: true,
+        dimensions: testEmbedding.length,
+      };
+    } catch (error) {
+      return {
+        success: false,
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+
+  get cacheStats() {
+    return this._cache.stats;
+  }
+}
+
+// ============================================================================
+// Factory Function
+// ============================================================================
+
+export function createEmbedder(config: EmbeddingConfig): Embedder {
+  return new Embedder(config);
+}

package/src/migrate.ts

@@ -0,0 +1,356 @@
+/**
+ * Migration Utilities
+ * Migrates data from old memory-lancedb plugin to memory-lancedb-pro
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+import fs from "node:fs/promises";
+import type { MemoryStore, MemoryEntry } from "./store.js";
+import { loadLanceDB } from "./store.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+interface LegacyMemoryEntry {
+  id: string;
+  text: string;
+  vector: number[];
+  importance: number;
+  category: "preference" | "fact" | "decision" | "entity" | "other";
+  createdAt: number;
+  scope?: string;
+}
+
+interface MigrationResult {
+  success: boolean;
+  migratedCount: number;
+  skippedCount: number;
+  errors: string[];
+  summary: string;
+}
+
+interface MigrationOptions {
+  sourceDbPath?: string;
+  dryRun?: boolean;
+  defaultScope?: string;
+  skipExisting?: boolean;
+}
+
+// ============================================================================
+// Default Paths
+// ============================================================================
+
+function getDefaultLegacyPaths(): string[] {
+  const home = homedir();
+  return [
+    join(home, ".openclaw", "memory", "lancedb"),
+    join(home, ".claude", "memory", "lancedb"),
+    // Add more legacy paths as needed
+  ];
+}
+
+// ============================================================================
+// Migration Functions
+// ============================================================================
+
+export class MemoryMigrator {
+  constructor(private targetStore: MemoryStore) {}
+
+  async migrate(options: MigrationOptions = {}): Promise<MigrationResult> {
+    const result: MigrationResult = {
+      success: false,
+      migratedCount: 0,
+      skippedCount: 0,
+      errors: [],
+      summary: "",
+    };
+
+    try {
+      // Find source database
+      const sourceDbPath = await this.findSourceDatabase(options.sourceDbPath);
+      if (!sourceDbPath) {
+        result.errors.push("No legacy database found to migrate from");
+        result.summary = "Migration failed: No source database found";
+        return result;
+      }
+
+      console.log(`Migrating from: ${sourceDbPath}`);
+
+      // Load legacy data
+      const legacyEntries = await this.loadLegacyData(sourceDbPath);
+      if (legacyEntries.length === 0) {
+        result.summary = "Migration completed: No data to migrate";
+        result.success = true;
+        return result;
+      }
+
+      console.log(`Found ${legacyEntries.length} entries to migrate`);
+
+      // Migrate entries
+      if (!options.dryRun) {
+        const migrationStats = await this.migrateEntries(legacyEntries, options);
+        result.migratedCount = migrationStats.migrated;
+        result.skippedCount = migrationStats.skipped;
+        result.errors.push(...migrationStats.errors);
+      } else {
+        result.summary = `Dry run: Would migrate ${legacyEntries.length} entries`;
+        result.success = true;
+        return result;
+      }
+
+      result.success = result.errors.length === 0;
+      result.summary = `Migration ${result.success ? 'completed' : 'completed with errors'}: ` +
+        `${result.migratedCount} migrated, ${result.skippedCount} skipped`;
+
+    } catch (error) {
+      result.errors.push(`Migration failed: ${error instanceof Error ? error.message : String(error)}`);
+      result.summary = "Migration failed due to unexpected error";
+    }
+
+    return result;
+  }
+
+  private async findSourceDatabase(explicitPath?: string): Promise<string | null> {
+    if (explicitPath) {
+      try {
+        await fs.access(explicitPath);
+        return explicitPath;
+      } catch {
+        return null;
+      }
+    }
+
+    // Check default legacy paths
+    for (const path of getDefaultLegacyPaths()) {
+      try {
+        await fs.access(path);
+        const files = await fs.readdir(path);
+        // Check for LanceDB files
+        if (files.some(f => f.endsWith('.lance') || f === 'memories.lance')) {
+          return path;
+        }
+      } catch {
+        continue;
+      }
+    }
+
+    return null;
+  }
+
+  private async loadLegacyData(sourceDbPath: string, limit?: number): Promise<LegacyMemoryEntry[]> {
+    const lancedb = await loadLanceDB();
+    const db = await lancedb.connect(sourceDbPath);
+
+    try {
+      const table = await db.openTable("memories");
+      let query = table.query();
+      if (limit) query = query.limit(limit);
+      const entries = await query.toArray();
+
+      return entries.map((row): LegacyMemoryEntry => ({
+        id: row.id as string,
+        text: row.text as string,
+        vector: row.vector as number[],
+        importance: row.importance as number,
+        category: (row.category as LegacyMemoryEntry["category"]) || "other",
+        createdAt: row.createdAt as number,
+        scope: row.scope as string | undefined,
+      }));
+    } catch (error) {
+      console.warn(`Failed to load legacy data: ${error}`);
+      return [];
+    }
+  }
+
+  private async migrateEntries(
+    legacyEntries: LegacyMemoryEntry[],
+    options: MigrationOptions
+  ): Promise<{ migrated: number; skipped: number; errors: string[] }> {
+    let migrated = 0;
+    let skipped = 0;
+    const errors: string[] = [];
+
+    const defaultScope = options.defaultScope || "global";
+
+    for (const legacy of legacyEntries) {
+      try {
+        // Check if entry already exists (if skipExisting is enabled)
+        if (options.skipExisting) {
+          const existing = await this.targetStore.vectorSearch(
+            legacy.vector, 1, 0.9, [legacy.scope || defaultScope]
+          );
+          if (existing.length > 0 && existing[0].score > 0.95) {
+            skipped++;
+            continue;
+          }
+        }
+
+        // Convert legacy entry to new format
+        const newEntry: Omit<MemoryEntry, "id" | "timestamp"> = {
+          text: legacy.text,
+          vector: legacy.vector,
+          category: legacy.category,
+          scope: legacy.scope || defaultScope, // Use legacy scope or default
+          importance: legacy.importance,
+          metadata: JSON.stringify({
+            migratedFrom: "memory-lancedb",
+            originalId: legacy.id,
+            originalCreatedAt: legacy.createdAt,
+          }),
+        };
+
+        await this.targetStore.store(newEntry);
+        migrated++;
+
+        if (migrated % 100 === 0) {
+          console.log(`Migrated ${migrated}/${legacyEntries.length} entries...`);
+        }
+
+      } catch (error) {
+        errors.push(`Failed to migrate entry ${legacy.id}: ${error}`);
+        skipped++;
+      }
+    }
+
+    return { migrated, skipped, errors };
+  }
+
+  // Check if migration is needed
+  async checkMigrationNeeded(sourceDbPath?: string): Promise<{
+    needed: boolean;
+    sourceFound: boolean;
+    sourceDbPath?: string;
+    entryCount?: number;
+  }> {
+    const sourcePath = await this.findSourceDatabase(sourceDbPath);
+
+    if (!sourcePath) {
+      return {
+        needed: false,
+        sourceFound: false,
+      };
+    }
+
+    try {
+      const entries = await this.loadLegacyData(sourcePath, 1);
+      return {
+        needed: entries.length > 0,
+        sourceFound: true,
+        sourceDbPath: sourcePath,
+        entryCount: entries.length > 0 ? undefined : 0, // Avoid full scan; count unknown
+      };
+    } catch (error) {
+      return {
+        needed: false,
+        sourceFound: true,
+        sourceDbPath: sourcePath,
+      };
+    }
+  }
+
+  // Verify migration results
+  async verifyMigration(sourceDbPath?: string): Promise<{
+    valid: boolean;
+    sourceCount: number;
+    targetCount: number;
+    issues: string[];
+  }> {
+    const issues: string[] = [];
+
+    try {
+      const sourcePath = await this.findSourceDatabase(sourceDbPath);
+      if (!sourcePath) {
+        return {
+          valid: false,
+          sourceCount: 0,
+          targetCount: 0,
+          issues: ["Source database not found"],
+        };
+      }
+
+      const sourceEntries = await this.loadLegacyData(sourcePath);
+      const targetStats = await this.targetStore.stats();
+
+      const sourceCount = sourceEntries.length;
+      const targetCount = targetStats.totalCount;
+
+      // Basic validation - target should have at least as many entries as source
+      if (targetCount < sourceCount) {
+        issues.push(`Target has fewer entries (${targetCount}) than source (${sourceCount})`);
+      }
+
+      return {
+        valid: issues.length === 0,
+        sourceCount,
+        targetCount,
+        issues,
+      };
+
+    } catch (error) {
+      return {
+        valid: false,
+        sourceCount: 0,
+        targetCount: 0,
+        issues: [`Verification failed: ${error}`],
+      };
+    }
+  }
+}
+
+// ============================================================================
+// Factory Function
+// ============================================================================
+
+export function createMigrator(targetStore: MemoryStore): MemoryMigrator {
+  return new MemoryMigrator(targetStore);
+}
+
+// ============================================================================
+// Standalone Migration Function
+// ============================================================================
+
+export async function migrateFromLegacy(
+  targetStore: MemoryStore,
+  options: MigrationOptions = {}
+): Promise<MigrationResult> {
+  const migrator = createMigrator(targetStore);
+  return migrator.migrate(options);
+}
+
+// ============================================================================
+// CLI Helper Functions
+// ============================================================================
+
+export async function checkForLegacyData(): Promise<{
+  found: boolean;
+  paths: string[];
+  totalEntries: number;
+}> {
+  const paths: string[] = [];
+  let totalEntries = 0;
+
+  for (const path of getDefaultLegacyPaths()) {
+    try {
+      const lancedb = await loadLanceDB();
+      const db = await lancedb.connect(path);
+      const table = await db.openTable("memories");
+      const entries = await table.query().select(["id"]).toArray();
+
+      if (entries.length > 0) {
+        paths.push(path);
+        totalEntries += entries.length;
+      }
+    } catch {
+      // Path doesn't exist or isn't a valid LanceDB
+      continue;
+    }
+  }
+
+  return {
+    found: paths.length > 0,
+    paths,
+    totalEntries,
+  };
+}