@yamo/memory-mesh 2.3.2 → 3.0.0

package/lib/memory/schema.ts

@@ -0,0 +1,267 @@
+/**
+ * LanceDB Schema Definitions for MemoryManager
+ * Uses Apache Arrow Schema format for LanceDB JavaScript SDK
+ *
+ * Supports dynamic vector dimensions for different embedding models:
+ * - all-MiniLM-L6-v2: 384 dimensions
+ * - all-mpnet-base-v2: 768 dimensions
+ * - text-embedding-3-small: 1536 dimensions
+ */
+
+import * as arrow from "apache-arrow";
+import * as lancedb from "@lancedb/lancedb";
+
+/**
+ * Default vector dimension (all-MiniLM-L6-v2)
+ */
+export const DEFAULT_VECTOR_DIMENSION = 384;
+
+/**
+ * Common embedding model dimensions
+ */
+export const EMBEDDING_DIMENSIONS: Record<string, number> = {
+  "Xenova/all-MiniLM-L6-v2": 384,
+  "Xenova/all-mpnet-base-v2": 768,
+  "Xenova/distiluse-base-multilingual-cased-v1": 512,
+  "sentence-transformers/all-MiniLM-L6-v2": 384,
+  "sentence-transformers/all-mpnet-base-v2": 768,
+  "openai/text-embedding-3-small": 1536,
+  "openai/text-embedding-3-large": 3072,
+  "cohere/embed-english-light-v3.0": 1024,
+  "cohere/embed-english-v3.0": 1024,
+};
+
+/**
+ * Get dimension for a given embedding model
+ * @param {string} modelName - Embedding model name or path
+ * @returns {number} Vector dimension
+ */
+export function getEmbeddingDimension(modelName?: string): number {
+  if (!modelName) {
+    return DEFAULT_VECTOR_DIMENSION;
+  }
+
+  // Check exact match
+  if (EMBEDDING_DIMENSIONS[modelName]) {
+    return EMBEDDING_DIMENSIONS[modelName];
+  }
+
+  // Check for partial matches
+  for (const [key, dimension] of Object.entries(EMBEDDING_DIMENSIONS)) {
+    if (modelName.toLowerCase().includes(key.toLowerCase())) {
+      return dimension;
+    }
+  }
+
+  // Fallback to default
+  return DEFAULT_VECTOR_DIMENSION;
+}
+
+/**
+ * Create a memory schema with a specific vector dimension
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {arrow.Schema} Arrow schema with specified dimension
+ */
+export function createMemorySchema(
+  vectorDim: number = DEFAULT_VECTOR_DIMENSION,
+): arrow.Schema {
+  return new arrow.Schema([
+    new arrow.Field("id", new arrow.Utf8(), false),
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        vectorDim,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+      false,
+    ),
+    new arrow.Field("content", new arrow.Utf8(), false),
+    new arrow.Field("metadata", new arrow.Utf8(), true), // Stored as JSON string
+    new arrow.Field(
+      "created_at",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      false,
+    ),
+    new arrow.Field(
+      "updated_at",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      true,
+    ),
+  ]);
+}
+
+/**
+ * Create V2 memory schema with automatic recall fields
+ * All new fields are nullable for backward compatibility
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {arrow.Schema} Arrow schema with V2 fields
+ */
+export function createMemorySchemaV2(
+  vectorDim: number = DEFAULT_VECTOR_DIMENSION,
+): arrow.Schema {
+  return new arrow.Schema([
+    // ========== V1 Fields (Backward Compatible) ==========
+    new arrow.Field("id", new arrow.Utf8(), false),
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        vectorDim,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+      false,
+    ),
+    new arrow.Field("content", new arrow.Utf8(), false),
+    new arrow.Field("metadata", new arrow.Utf8(), true),
+    new arrow.Field(
+      "created_at",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      false,
+    ),
+    new arrow.Field(
+      "updated_at",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      true,
+    ),
+
+    // ========== V2 Fields (All Nullable) ==========
+    new arrow.Field("session_id", new arrow.Utf8(), true), // Session association
+    new arrow.Field("agent_id", new arrow.Utf8(), true), // Agent/skill that created memory
+    new arrow.Field("memory_type", new arrow.Utf8(), true), // 'global', 'session', 'agent'
+    new arrow.Field("importance_score", new arrow.Float32(), true), // 0.0-1.0 importance
+    new arrow.Field("access_count", new arrow.Int32(), true), // Popularity tracking
+    new arrow.Field(
+      "last_accessed",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      true,
+    ),
+  ]);
+}
+
+/**
+ * Create schema for synthesized skills (Recursive Skill Synthesis)
+ * @param {number} vectorDim - Vector dimension for intent embedding
+ * @returns {arrow.Schema} Arrow schema
+ */
+export function createSynthesizedSkillSchema(
+  vectorDim: number = DEFAULT_VECTOR_DIMENSION,
+): arrow.Schema {
+  return new arrow.Schema([
+    new arrow.Field("id", new arrow.Utf8(), false),
+    new arrow.Field("name", new arrow.Utf8(), false),
+    new arrow.Field("intent", new arrow.Utf8(), false),
+    new arrow.Field("yamo_text", new arrow.Utf8(), false),
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        vectorDim,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+      false,
+    ),
+    new arrow.Field("metadata", new arrow.Utf8(), true), // Stored as JSON: {reliability, use_count, created_at}
+    new arrow.Field(
+      "created_at",
+      new arrow.Timestamp(arrow.TimeUnit.MILLISECOND),
+      false,
+    ),
+  ]);
+}
+
+/**
+ * Check if a table is using V2 schema
+ * @param {arrow.Schema} schema - Table schema to check
+ * @returns {boolean} True if V2 schema detected
+ */
+export function isSchemaV2(schema: arrow.Schema): boolean {
+  return schema.fields.some((f) => f.name === "session_id");
+}
+
+/**
+ * Memory table schema using Apache Arrow format (default 384 dimensions)
+ * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
+ */
+export const MEMORY_SCHEMA = createMemorySchema(DEFAULT_VECTOR_DIMENSION);
+
+/**
+ * Index configuration for memory table
+ * Indices should be created after data is inserted
+ */
+export const INDEX_CONFIG = {
+  vector: {
+    index_type: "ivf_pq",
+    metric: "cosine",
+    num_partitions: 256,
+    num_sub_vectors: 8,
+  },
+  full_text: {
+    fields: ["content"],
+  },
+};
+
+/**
+ * Creates a memory table in LanceDB with the predefined schema (384 dimensions)
+ * @param {lancedb.Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create (default: 'memory_entries')
+ * @returns {Promise<lancedb.Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ * @deprecated Use createMemoryTableWithDimension() for dynamic dimensions
+ */
+export async function createMemoryTable(
+  db: lancedb.Connection,
+  tableName: string = "memory_entries",
+): Promise<lancedb.Table> {
+  return createMemoryTableWithDimension(
+    db,
+    tableName,
+    DEFAULT_VECTOR_DIMENSION,
+  );
+}
+
+/**
+ * Creates a memory table in LanceDB with a specific vector dimension
+ * @param {lancedb.Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create
+ * @param {number} vectorDim - Vector dimension (384, 768, 1536, etc.)
+ * @returns {Promise<lancedb.Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ */
+export async function createMemoryTableWithDimension(
+  db: lancedb.Connection,
+  tableName: string,
+  vectorDim: number,
+): Promise<lancedb.Table> {
+  try {
+    // Check if table already exists
+    const existingTables = await db.tableNames();
+
+    if (existingTables.includes(tableName)) {
+      return await db.openTable(tableName);
+    }
+
+    // Create schema with specified dimension
+    const schema = createMemorySchema(vectorDim);
+
+    // Create table with schema
+    // LanceDB v0.23.0+ accepts empty array as initial data with schema option
+    const table = await db.createTable(tableName, [], { schema } as any); // Cast to any because lancedb types might be strict about options
+    return table;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(
+      `Failed to create memory table with dimension ${vectorDim}: ${message}`,
+    );
+  }
+}
+
+export default {
+  MEMORY_SCHEMA,
+  INDEX_CONFIG,
+  createMemoryTable,
+  createMemoryTableWithDimension,
+  createMemorySchema,
+  createMemorySchemaV2,
+  isSchemaV2,
+  getEmbeddingDimension,
+  DEFAULT_VECTOR_DIMENSION,
+  EMBEDDING_DIMENSIONS,
+};

package/lib/memory/scorer.d.ts

@@ -0,0 +1,26 @@
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+import { MemoryMesh } from "./memory-mesh.js";
+export declare class MemoryScorer {
+    #private;
+    /**
+     * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+     */
+    constructor(mesh: MemoryMesh);
+    /**
+     * Calculate importance score for content
+     * @param {string} content - Content to score
+     * @param {Object} metadata - Associated metadata
+     * @returns {Promise<number>} Importance score (0-1)
+     */
+    calculateImportance(content: string, metadata?: any): number;
+    /**
+     * Check if content is duplicate of existing memory
+     * @param {string} content - Content to check
+     * @param {number} threshold - Similarity threshold (default 0.9)
+     * @returns {Promise<boolean>} True if duplicate exists
+     */
+    isDuplicate(content: string, threshold?: number): Promise<boolean>;
+}
+export default MemoryScorer;

package/lib/memory/scorer.js

@@ -0,0 +1,77 @@
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+export class MemoryScorer {
+    #mesh;
+    /**
+     * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+     */
+    constructor(mesh) {
+        this.#mesh = mesh;
+    }
+    /**
+     * Calculate importance score for content
+     * @param {string} content - Content to score
+     * @param {Object} metadata - Associated metadata
+     * @returns {Promise<number>} Importance score (0-1)
+     */
+    calculateImportance(content, metadata = {}) {
+        let score = 0;
+        // Content length (longer = more important, up to a point)
+        const length = content.length;
+        score += Math.min(length / 1000, 0.2);
+        // Has structured data (JSON, code blocks)
+        if (content.includes("```") || content.includes("{")) {
+            score += 0.1;
+        }
+        // Interaction type bonuses
+        if (metadata.interaction_type === "tool_execution") {
+            score += 0.15;
+        }
+        if (metadata.interaction_type === "file_operation") {
+            score += 0.1;
+        }
+        // Tool usage indicates importance
+        if (metadata.tools_used?.length > 0) {
+            score += Math.min(metadata.tools_used.length * 0.05, 0.15);
+        }
+        // File involvement
+        if (metadata.files_involved?.length > 0) {
+            score += Math.min(metadata.files_involved.length * 0.05, 0.15);
+        }
+        // Keywords that indicate importance
+        const importantKeywords = [
+            "error",
+            "bug",
+            "fix",
+            "important",
+            "critical",
+            "note",
+            "remember",
+        ];
+        const lowerContent = content.toLowerCase();
+        const keywordMatches = importantKeywords.filter((k) => lowerContent.includes(k)).length;
+        score += Math.min(keywordMatches * 0.05, 0.15);
+        return Math.min(score, 1.0);
+    }
+    /**
+     * Check if content is duplicate of existing memory
+     * @param {string} content - Content to check
+     * @param {number} threshold - Similarity threshold (default 0.9)
+     * @returns {Promise<boolean>} True if duplicate exists
+     */
+    async isDuplicate(content, threshold = 0.9) {
+        try {
+            const results = await this.#mesh.search(content, {
+                limit: 1,
+                useCache: false,
+            });
+            return results.length > 0 && results[0].score >= threshold;
+        }
+        catch (_error) {
+            // On error, assume not duplicate to allow storage
+            return false;
+        }
+    }
+}
+export default MemoryScorer;

package/lib/memory/scorer.ts

@@ -0,0 +1,95 @@
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+
+import { MemoryMesh } from "./memory-mesh.js";
+
+export class MemoryScorer {
+  #mesh: MemoryMesh;
+
+  /**
+   * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+   */
+  constructor(mesh: MemoryMesh) {
+    this.#mesh = mesh;
+  }
+
+  /**
+   * Calculate importance score for content
+   * @param {string} content - Content to score
+   * @param {Object} metadata - Associated metadata
+   * @returns {Promise<number>} Importance score (0-1)
+   */
+  calculateImportance(content: string, metadata: any = {}): number {
+    let score = 0;
+
+    // Content length (longer = more important, up to a point)
+    const length = content.length;
+    score += Math.min(length / 1000, 0.2);
+
+    // Has structured data (JSON, code blocks)
+    if (content.includes("```") || content.includes("{")) {
+      score += 0.1;
+    }
+
+    // Interaction type bonuses
+    if (metadata.interaction_type === "tool_execution") {
+      score += 0.15;
+    }
+    if (metadata.interaction_type === "file_operation") {
+      score += 0.1;
+    }
+
+    // Tool usage indicates importance
+    if (metadata.tools_used?.length > 0) {
+      score += Math.min(metadata.tools_used.length * 0.05, 0.15);
+    }
+
+    // File involvement
+    if (metadata.files_involved?.length > 0) {
+      score += Math.min(metadata.files_involved.length * 0.05, 0.15);
+    }
+
+    // Keywords that indicate importance
+    const importantKeywords = [
+      "error",
+      "bug",
+      "fix",
+      "important",
+      "critical",
+      "note",
+      "remember",
+    ];
+    const lowerContent = content.toLowerCase();
+    const keywordMatches = importantKeywords.filter((k) =>
+      lowerContent.includes(k),
+    ).length;
+    score += Math.min(keywordMatches * 0.05, 0.15);
+
+    return Math.min(score, 1.0);
+  }
+
+  /**
+   * Check if content is duplicate of existing memory
+   * @param {string} content - Content to check
+   * @param {number} threshold - Similarity threshold (default 0.9)
+   * @returns {Promise<boolean>} True if duplicate exists
+   */
+  async isDuplicate(
+    content: string,
+    threshold: number = 0.9,
+  ): Promise<boolean> {
+    try {
+      const results = await this.#mesh.search(content, {
+        limit: 1,
+        useCache: false,
+      });
+      return results.length > 0 && results[0].score >= threshold;
+    } catch (_error) {
+      // On error, assume not duplicate to allow storage
+      return false;
+    }
+  }
+}
+
+export default MemoryScorer;

package/lib/memory/search/index.d.ts

@@ -0,0 +1 @@
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/index.js

@@ -0,0 +1 @@
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/index.ts

@@ -0,0 +1 @@
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/keyword-search.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+export interface KeywordDoc {
+    content: string;
+    metadata?: any;
+}
+export interface KeywordSearchResult extends KeywordDoc {
+    id: string;
+    score: number;
+    matches: string[];
+}
+export interface SearchOptions {
+    limit?: number;
+}
+export declare class KeywordSearch {
+    index: Map<string, Map<string, number>>;
+    docLengths: Map<string, number>;
+    idf: Map<string, number>;
+    docs: Map<string, KeywordDoc>;
+    isDirty: boolean;
+    constructor();
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text: string): string[];
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id: string, content: string, metadata?: any): void;
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id: string): void;
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats(): void;
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query: string, options?: SearchOptions): KeywordSearchResult[];
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records: {
+        id: string;
+        content: string;
+        metadata?: any;
+    }[]): void;
+}

package/lib/memory/search/keyword-search.js

@@ -0,0 +1,135 @@
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+export class KeywordSearch {
+    index; // token -> Map<docId, tf>
+    docLengths; // docId -> length
+    idf; // token -> idf value
+    docs; // docId -> content (optional, for snippet)
+    isDirty;
+    constructor() {
+        this.index = new Map();
+        this.docLengths = new Map();
+        this.idf = new Map();
+        this.docs = new Map();
+        this.isDirty = false;
+    }
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text) {
+        if (!text) {
+            return [];
+        }
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, "") // Remove punctuation
+            .split(/\s+/)
+            .filter((t) => t.length > 2) // Filter stopwords/short
+            .map((t) => t.substring(0, 20)); // Truncate
+    }
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id, content, metadata = {}) {
+        const tokens = this.tokenize(content);
+        const termFreqs = new Map();
+        tokens.forEach((t) => {
+            termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+        });
+        this.docLengths.set(id, tokens.length);
+        this.docs.set(id, { content, metadata });
+        // Update index
+        for (const [token, freq] of termFreqs.entries()) {
+            if (!this.index.has(token)) {
+                this.index.set(token, new Map());
+            }
+            this.index.get(token).set(id, freq);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id) {
+        this.docLengths.delete(id);
+        this.docs.delete(id);
+        // This is expensive O(Vocab), but okay for small scale
+        for (const docMap of this.index.values()) {
+            docMap.delete(id);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats() {
+        if (!this.isDirty) {
+            return;
+        }
+        const N = this.docLengths.size;
+        this.idf.clear();
+        for (const [token, docMap] of this.index.entries()) {
+            const df = docMap.size;
+            // Standard IDF: log(N / (df + 1)) + 1
+            const idf = Math.log(N / (df + 1)) + 1;
+            this.idf.set(token, idf);
+        }
+        this.isDirty = false;
+    }
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query, options = {}) {
+        this._computeStats();
+        const tokens = this.tokenize(query);
+        const scores = new Map(); // docId -> score
+        const matches = new Map(); // docId -> matched tokens
+        const limit = options.limit || 10;
+        for (const token of tokens) {
+            const docMap = this.index.get(token);
+            if (!docMap) {
+                continue;
+            }
+            const idf = this.idf.get(token) || 0;
+            for (const [docId, tf] of docMap.entries()) {
+                // TF-IDF Score
+                // Score = tf * idf * (normalization?)
+                // Simple variant:
+                const score = tf * idf;
+                scores.set(docId, (scores.get(docId) || 0) + score);
+                if (!matches.has(docId)) {
+                    matches.set(docId, []);
+                }
+                matches.get(docId).push(token);
+            }
+        }
+        // Convert to array and sort
+        return Array.from(scores.entries())
+            .map(([id, score]) => ({
+            id,
+            score,
+            matches: matches.get(id) || [],
+            ...this.docs.get(id),
+        }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records) {
+        records.forEach((r) => this.add(r.id, r.content, r.metadata));
+    }
+}