@mnemoai/core 1.1.0

package/src/migrate.ts

@@ -0,0 +1,364 @@
+// SPDX-License-Identifier: MIT
+/**
+ * Migration Utilities
+ * Migrates data from old memory-lancedb plugin to memory-lancedb-pro
+ */
+
+import { homedir } from "node:os";
+import { join } from "node:path";
+import fs from "node:fs/promises";
+import type { MemoryStore, MemoryEntry } from "./store.js";
+import { loadLanceDB } from "./store.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+interface LegacyMemoryEntry {
+  id: string;
+  text: string;
+  vector: number[];
+  importance: number;
+  category: "preference" | "fact" | "decision" | "entity" | "other";
+  createdAt: number;
+  scope?: string;
+}
+
+interface MigrationResult {
+  success: boolean;
+  migratedCount: number;
+  skippedCount: number;
+  errors: string[];
+  summary: string;
+}
+
+interface MigrationOptions {
+  sourceDbPath?: string;
+  dryRun?: boolean;
+  defaultScope?: string;
+  skipExisting?: boolean;
+}
+
+function normalizeLegacyVector(value: unknown): number[] {
+  if (Array.isArray(value)) {
+    return value.map((n) => Number(n));
+  }
+
+  if (
+    value &&
+    typeof value === "object" &&
+    Symbol.iterator in (value as Record<PropertyKey, unknown>)
+  ) {
+    return Array.from(value as Iterable<unknown>, (n) => Number(n));
+  }
+
+  return [];
+}
+
+// ============================================================================
+// Default Paths
+// ============================================================================
+
+function getDefaultLegacyPaths(): string[] {
+  const home = homedir();
+  return [
+    join(home, ".openclaw", "memory", "lancedb"),
+    join(home, ".claude", "memory", "lancedb"),
+    // Add more legacy paths as needed
+  ];
+}
+
+// ============================================================================
+// Migration Functions
+// ============================================================================
+
+export class MemoryMigrator {
+  constructor(private targetStore: MemoryStore) {}
+
+  async migrate(options: MigrationOptions = {}): Promise<MigrationResult> {
+    const result: MigrationResult = {
+      success: false,
+      migratedCount: 0,
+      skippedCount: 0,
+      errors: [],
+      summary: "",
+    };
+
+    try {
+      // Find source database
+      const sourceDbPath = await this.findSourceDatabase(options.sourceDbPath);
+      if (!sourceDbPath) {
+        result.errors.push("No legacy database found to migrate from");
+        result.summary = "Migration failed: No source database found";
+        return result;
+      }
+
+      console.log(`Migrating from: ${sourceDbPath}`);
+
+      // Load legacy data
+      const legacyEntries = await this.loadLegacyData(sourceDbPath);
+      if (legacyEntries.length === 0) {
+        result.summary = "Migration completed: No data to migrate";
+        result.success = true;
+        return result;
+      }
+
+      console.log(`Found ${legacyEntries.length} entries to migrate`);
+
+      // Migrate entries
+      if (!options.dryRun) {
+        const migrationStats = await this.migrateEntries(legacyEntries, options);
+        result.migratedCount = migrationStats.migrated;
+        result.skippedCount = migrationStats.skipped;
+        result.errors.push(...migrationStats.errors);
+      } else {
+        result.summary = `Dry run: Would migrate ${legacyEntries.length} entries`;
+        result.success = true;
+        return result;
+      }
+
+      result.success = result.errors.length === 0;
+      result.summary = `Migration ${result.success ? 'completed' : 'completed with errors'}: ` +
+        `${result.migratedCount} migrated, ${result.skippedCount} skipped`;
+
+    } catch (error) {
+      result.errors.push(`Migration failed: ${error instanceof Error ? error.message : String(error)}`);
+      result.summary = "Migration failed due to unexpected error";
+    }
+
+    return result;
+  }
+
+  private async findSourceDatabase(explicitPath?: string): Promise<string | null> {
+    if (explicitPath) {
+      try {
+        await fs.access(explicitPath);
+        return explicitPath;
+      } catch {
+        return null;
+      }
+    }
+
+    // Check default legacy paths
+    for (const path of getDefaultLegacyPaths()) {
+      try {
+        await fs.access(path);
+        const files = await fs.readdir(path);
+        // Check for LanceDB files
+        if (files.some(f => f.endsWith('.lance') || f === 'memories.lance')) {
+          return path;
+        }
+      } catch {
+        continue;
+      }
+    }
+
+    return null;
+  }
+
+  private async loadLegacyData(sourceDbPath: string, limit?: number): Promise<LegacyMemoryEntry[]> {
+    const lancedb = await loadLanceDB();
+    const db = await lancedb.connect(sourceDbPath);
+
+    try {
+      const table = await db.openTable("memories");
+      let query = table.query();
+      if (limit) query = query.limit(limit);
+      const entries = await query.toArray();
+
+      return entries.map((row): LegacyMemoryEntry => ({
+        id: row.id as string,
+        text: row.text as string,
+        vector: normalizeLegacyVector(row.vector),
+        importance: Number(row.importance),
+        category: (row.category as LegacyMemoryEntry["category"]) || "other",
+        createdAt: Number(row.createdAt),
+        scope: row.scope as string | undefined,
+      }));
+    } catch (error) {
+      console.warn(`Failed to load legacy data: ${error}`);
+      return [];
+    }
+  }
+
+  private async migrateEntries(
+    legacyEntries: LegacyMemoryEntry[],
+    options: MigrationOptions
+  ): Promise<{ migrated: number; skipped: number; errors: string[] }> {
+    let migrated = 0;
+    let skipped = 0;
+    const errors: string[] = [];
+
+    const defaultScope = options.defaultScope || "global";
+
+    for (const legacy of legacyEntries) {
+      try {
+        // Check if entry already exists (if skipExisting is enabled)
+        if (options.skipExisting) {
+          if (legacy.id && (await this.targetStore.hasId(legacy.id))) {
+            skipped++;
+            continue;
+          }
+
+          const existing = await this.targetStore.vectorSearch(
+            legacy.vector, 1, 0.9, [legacy.scope || defaultScope]
+          );
+          if (existing.length > 0 && existing[0].score > 0.95) {
+            skipped++;
+            continue;
+          }
+        }
+
+        // Convert legacy entry to new format while preserving legacy identity.
+        const newEntry: MemoryEntry = {
+          id: legacy.id,
+          text: legacy.text,
+          vector: legacy.vector,
+          category: legacy.category,
+          scope: legacy.scope || defaultScope,
+          importance: legacy.importance,
+          timestamp: Number.isFinite(legacy.createdAt) ? legacy.createdAt : Date.now(),
+          metadata: JSON.stringify({
+            migratedFrom: "memory-lancedb",
+            originalId: legacy.id,
+            originalCreatedAt: legacy.createdAt,
+          }),
+        };
+
+        await this.targetStore.importEntry(newEntry);
+        migrated++;
+
+        if (migrated % 100 === 0) {
+          console.log(`Migrated ${migrated}/${legacyEntries.length} entries...`);
+        }
+
+      } catch (error) {
+        errors.push(`Failed to migrate entry ${legacy.id}: ${error}`);
+        skipped++;
+      }
+    }
+
+    return { migrated, skipped, errors };
+  }
+
+  async checkMigrationNeeded(sourceDbPath?: string): Promise<{
+    needed: boolean;
+    sourceFound: boolean;
+    sourceDbPath?: string;
+    entryCount?: number;
+  }> {
+    const sourcePath = await this.findSourceDatabase(sourceDbPath);
+
+    if (!sourcePath) {
+      return {
+        needed: false,
+        sourceFound: false,
+      };
+    }
+
+    try {
+      const entries = await this.loadLegacyData(sourcePath, 1);
+      return {
+        needed: entries.length > 0,
+        sourceFound: true,
+        sourceDbPath: sourcePath,
+        entryCount: entries.length > 0 ? undefined : 0,
+      };
+    } catch (error) {
+      return {
+        needed: false,
+        sourceFound: true,
+        sourceDbPath: sourcePath,
+      };
+    }
+  }
+
+  async verifyMigration(sourceDbPath?: string): Promise<{
+    valid: boolean;
+    sourceCount: number;
+    targetCount: number;
+    issues: string[];
+  }> {
+    const issues: string[] = [];
+
+    try {
+      const sourcePath = await this.findSourceDatabase(sourceDbPath);
+      if (!sourcePath) {
+        return {
+          valid: false,
+          sourceCount: 0,
+          targetCount: 0,
+          issues: ["Source database not found"],
+        };
+      }
+
+      const sourceEntries = await this.loadLegacyData(sourcePath);
+      const targetStats = await this.targetStore.stats();
+
+      const sourceCount = sourceEntries.length;
+      const targetCount = targetStats.totalCount;
+
+      if (targetCount < sourceCount) {
+        issues.push(`Target has fewer entries (${targetCount}) than source (${sourceCount})`);
+      }
+
+      return {
+        valid: issues.length === 0,
+        sourceCount,
+        targetCount,
+        issues,
+      };
+
+    } catch (error) {
+      return {
+        valid: false,
+        sourceCount: 0,
+        targetCount: 0,
+        issues: [`Verification failed: ${error}`],
+      };
+    }
+  }
+}
+
+export function createMigrator(targetStore: MemoryStore): MemoryMigrator {
+  return new MemoryMigrator(targetStore);
+}
+
+export async function migrateFromLegacy(
+  targetStore: MemoryStore,
+  options: MigrationOptions = {}
+): Promise<MigrationResult> {
+  const migrator = createMigrator(targetStore);
+  return migrator.migrate(options);
+}
+
+export async function checkForLegacyData(): Promise<{
+  found: boolean;
+  paths: string[];
+  totalEntries: number;
+}> {
+  const paths: string[] = [];
+  let totalEntries = 0;
+
+  for (const path of getDefaultLegacyPaths()) {
+    try {
+      const lancedb = await loadLanceDB();
+      const db = await lancedb.connect(path);
+      const table = await db.openTable("memories");
+      const entries = await table.query().select(["id"]).toArray();
+
+      if (entries.length > 0) {
+        paths.push(path);
+        totalEntries += entries.length;
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return {
+    found: paths.length > 0,
+    paths,
+    totalEntries,
+  };
+}

package/src/mnemo.ts

@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: MIT
+/**
+ * Mnemo Core — simplified entry point
+ * Usage: const mnemo = await createMnemo(config)
+ */
+
+import { MemoryStore } from "./store.js";
+import { createRetriever, DEFAULT_RETRIEVAL_CONFIG } from "./retriever.js";
+import { Embedder } from "./embedder.js";
+import { createDecayEngine, DEFAULT_DECAY_CONFIG } from "./decay-engine.js";
+import { SmartExtractor } from "./smart-extractor.js";
+import { createLlmClient } from "./llm-client.js";
+
+export interface MnemoConfig {
+  embedding: {
+    provider: "openai-compatible";
+    apiKey: string;
+    baseURL?: string;
+    model?: string;
+    dimensions?: number;
+    taskQuery?: string;
+    taskPassage?: string;
+  };
+  dbPath: string;
+  decay?: {
+    recencyHalfLifeDays?: number;
+    recencyWeight?: number;
+    frequencyWeight?: number;
+    intrinsicWeight?: number;
+  };
+  tier?: {
+    coreAccessThreshold?: number;
+    coreImportanceThreshold?: number;
+    peripheralAgeDays?: number;
+  };
+  llm?: {
+    model?: string;
+    baseURL?: string;
+    apiKey?: string;
+  };
+  retrieval?: {
+    candidatePoolSize?: number;
+    rerank?: "cross-encoder" | "lightweight" | "none";
+    rerankApiKey?: string;
+    rerankModel?: string;
+    rerankEndpoint?: string;
+    rerankProvider?: string;
+  };
+}
+
+export interface MnemoInstance {
+  store(entry: {
+    text: string;
+    category?: string;
+    importance?: number;
+    scope?: string;
+  }): Promise<{ id: string }>;
+
+  recall(query: string, options?: {
+    limit?: number;
+    scopeFilter?: string[];
+    category?: string;
+  }): Promise<Array<{
+    text: string;
+    score: number;
+    category: string;
+    importance: number;
+    timestamp: number;
+  }>>;
+
+  stats(): Promise<{ totalEntries: number }>;
+
+  close(): Promise<void>;
+}
+
+export async function createMnemo(config: MnemoConfig): Promise<MnemoInstance> {
+  const embedder = new Embedder({
+    apiKey: config.embedding.apiKey,
+    baseURL: config.embedding.baseURL,
+    model: config.embedding.model || "voyage-3-large",
+    dimensions: config.embedding.dimensions || 1024,
+    taskQuery: config.embedding.taskQuery,
+    taskPassage: config.embedding.taskPassage,
+  });
+
+  const store = new MemoryStore({
+    dbPath: config.dbPath,
+    embedder,
+  });
+
+  await store.initialize();
+
+  const decayEngine = createDecayEngine({
+    ...DEFAULT_DECAY_CONFIG,
+    ...(config.decay || {}),
+  });
+
+  const retriever = createRetriever(store, embedder, {
+    ...DEFAULT_RETRIEVAL_CONFIG,
+    ...(config.retrieval || {}),
+  }, { decayEngine });
+
+  return {
+    async store(entry) {
+      const vector = await embedder.embed(entry.text);
+      const result = await store.store({
+        text: entry.text,
+        vector,
+        category: entry.category || "fact",
+        importance: entry.importance ?? 0.7,
+        scope: entry.scope || "global",
+      });
+      return { id: result.id };
+    },
+
+    async recall(query, options = {}) {
+      const results = await retriever.retrieve({
+        query,
+        limit: options.limit ?? 5,
+        scopeFilter: options.scopeFilter,
+        category: options.category,
+        source: "manual",
+      });
+      return results.map(r => ({
+        text: r.entry.text,
+        score: r.score,
+        category: r.entry.category || "fact",
+        importance: r.entry.importance ?? 0.7,
+        timestamp: r.entry.timestamp ?? Date.now(),
+      }));
+    },
+
+    async stats() {
+      const count = await store.count();
+      return { totalEntries: count };
+    },
+
+    async close() {
+      // LanceDB handles cleanup automatically
+    },
+  };
+}

package/src/noise-filter.ts

@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: MIT
+/**
+ * Noise Filter
+ * Filters out low-quality memories (meta-questions, agent denials, session boilerplate)
+ * Inspired by openclaw-plugin-continuity's noise filtering approach.
+ */
+
+// Agent-side denial patterns
+const DENIAL_PATTERNS = [
+  /i don'?t have (any )?(information|data|memory|record)/i,
+  /i'?m not sure about/i,
+  /i don'?t recall/i,
+  /i don'?t remember/i,
+  /it looks like i don'?t/i,
+  /i wasn'?t able to find/i,
+  /no (relevant )?memories found/i,
+  /i don'?t have access to/i,
+];
+
+// User-side meta-question patterns (about memory itself, not content)
+const META_QUESTION_PATTERNS = [
+  /\bdo you (remember|recall|know about)\b/i,
+  /\bcan you (remember|recall)\b/i,
+  /\bdid i (tell|mention|say|share)\b/i,
+  /\bhave i (told|mentioned|said)\b/i,
+  /\bwhat did i (tell|say|mention)\b/i,
+  /如果你知道.+只回复/i,
+  /如果不知道.+只回复\s*none/i,
+  /只回复精确代号/i,
+  /只回复\s*none/i,
+  // Chinese recall / meta-question patterns
+  /你还?记得/,
+  /记不记得/,
+  /还记得.*吗/,
+  /你[知晓]道.+吗/,
+  /我(?:之前|上次|以前)(?:说|提|讲).*(?:吗|呢|？|\?)/,
+];
+
+// Session boilerplate
+const BOILERPLATE_PATTERNS = [
+  /^(hi|hello|hey|good morning|good evening|greetings)/i,
+  /^fresh session/i,
+  /^new session/i,
+  /^HEARTBEAT/i,
+];
+
+// Extractor artifacts from validation prompts / synthetic summaries
+const DIAGNOSTIC_ARTIFACT_PATTERNS = [
+  /\bquery\s*->\s*(none|no explicit solution|unknown|not found)\b/i,
+  /\buser asked for\b.*\b(none|no explicit solution|unknown|not found)\b/i,
+  /\bno explicit solution\b/i,
+];
+
+export interface NoiseFilterOptions {
+  /** Filter agent denial responses (default: true) */
+  filterDenials?: boolean;
+  /** Filter meta-questions about memory (default: true) */
+  filterMetaQuestions?: boolean;
+  /** Filter session boilerplate (default: true) */
+  filterBoilerplate?: boolean;
+}
+
+const DEFAULT_OPTIONS: Required<NoiseFilterOptions> = {
+  filterDenials: true,
+  filterMetaQuestions: true,
+  filterBoilerplate: true,
+};
+
+/**
+ * Check if a memory text is noise that should be filtered out.
+ * Returns true if the text is noise.
+ */
+export function isNoise(text: string, options: NoiseFilterOptions = {}): boolean {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  const trimmed = text.trim();
+
+  if (trimmed.length < 5) return true;
+
+  if (opts.filterDenials && DENIAL_PATTERNS.some(p => p.test(trimmed))) return true;
+  if (opts.filterMetaQuestions && META_QUESTION_PATTERNS.some(p => p.test(trimmed))) return true;
+  if (opts.filterBoilerplate && BOILERPLATE_PATTERNS.some(p => p.test(trimmed))) return true;
+  if (DIAGNOSTIC_ARTIFACT_PATTERNS.some(p => p.test(trimmed))) return true;
+
+  return false;
+}
+
+/**
+ * Filter an array of items, removing noise entries.
+ */
+export function filterNoise<T>(
+  items: T[],
+  getText: (item: T) => string,
+  options?: NoiseFilterOptions
+): T[] {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  return items.filter(item => !isNoise(getText(item), opts));
+}