@yamo/memory-mesh 2.3.2 → 3.0.1

package/lib/memory/schema.ts

@@ -0,0 +1,184 @@
+// @ts-nocheck
+/**
+ * LanceDB Schema Definitions for MemoryManager
+ * Uses Apache Arrow Schema format for LanceDB JavaScript SDK
+ *
+ * Supports dynamic vector dimensions for different embedding models:
+ * - all-MiniLM-L6-v2: 384 dimensions
+ * - all-mpnet-base-v2: 768 dimensions
+ * - text-embedding-3-small: 1536 dimensions
+ */
+import * as arrow from "apache-arrow";
+/**
+ * Default vector dimension (all-MiniLM-L6-v2)
+ */
+export const DEFAULT_VECTOR_DIMENSION = 384;
+/**
+ * Common embedding model dimensions
+ */
+export const EMBEDDING_DIMENSIONS = {
+    "Xenova/all-MiniLM-L6-v2": 384,
+    "Xenova/all-mpnet-base-v2": 768,
+    "Xenova/distiluse-base-multilingual-cased-v1": 512,
+    "sentence-transformers/all-MiniLM-L6-v2": 384,
+    "sentence-transformers/all-mpnet-base-v2": 768,
+    "openai/text-embedding-3-small": 1536,
+    "openai/text-embedding-3-large": 3072,
+    "cohere/embed-english-light-v3.0": 1024,
+    "cohere/embed-english-v3.0": 1024,
+};
+/**
+ * Get dimension for a given embedding model
+ * @param {string} modelName - Embedding model name or path
+ * @returns {number} Vector dimension
+ */
+export function getEmbeddingDimension(modelName) {
+    if (!modelName) {
+        return DEFAULT_VECTOR_DIMENSION;
+    }
+    // Check exact match
+    if (EMBEDDING_DIMENSIONS[modelName]) {
+        return EMBEDDING_DIMENSIONS[modelName];
+    }
+    // Check for partial matches
+    for (const [key, dimension] of Object.entries(EMBEDDING_DIMENSIONS)) {
+        if (modelName.toLowerCase().includes(key.toLowerCase())) {
+            return dimension;
+        }
+    }
+    // Fallback to default
+    return DEFAULT_VECTOR_DIMENSION;
+}
+/**
+ * Create a memory schema with a specific vector dimension
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {arrow.Schema} Arrow schema with specified dimension
+ */
+export function createMemorySchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
+    return new arrow.Schema([
+        new arrow.Field("id", new arrow.Utf8(), false),
+        new arrow.Field("vector", new arrow.FixedSizeList(vectorDim, new arrow.Field("item", new arrow.Float32(), true)), false),
+        new arrow.Field("content", new arrow.Utf8(), false),
+        new arrow.Field("metadata", new arrow.Utf8(), true), // Stored as JSON string
+        new arrow.Field("created_at", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+        new arrow.Field("updated_at", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
+    ]);
+}
+/**
+ * Create V2 memory schema with automatic recall fields
+ * All new fields are nullable for backward compatibility
+ * @param {number} vectorDim - Vector dimension (e.g., 384, 768, 1536)
+ * @returns {arrow.Schema} Arrow schema with V2 fields
+ */
+export function createMemorySchemaV2(vectorDim = DEFAULT_VECTOR_DIMENSION) {
+    return new arrow.Schema([
+        // ========== V1 Fields (Backward Compatible) ==========
+        new arrow.Field("id", new arrow.Utf8(), false),
+        new arrow.Field("vector", new arrow.FixedSizeList(vectorDim, new arrow.Field("item", new arrow.Float32(), true)), false),
+        new arrow.Field("content", new arrow.Utf8(), false),
+        new arrow.Field("metadata", new arrow.Utf8(), true),
+        new arrow.Field("created_at", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+        new arrow.Field("updated_at", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
+        // ========== V2 Fields (All Nullable) ==========
+        new arrow.Field("session_id", new arrow.Utf8(), true), // Session association
+        new arrow.Field("agent_id", new arrow.Utf8(), true), // Agent/skill that created memory
+        new arrow.Field("memory_type", new arrow.Utf8(), true), // 'global', 'session', 'agent'
+        new arrow.Field("importance_score", new arrow.Float32(), true), // 0.0-1.0 importance
+        new arrow.Field("access_count", new arrow.Int32(), true), // Popularity tracking
+        new arrow.Field("last_accessed", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), true),
+    ]);
+}
+/**
+ * Create schema for synthesized skills (Recursive Skill Synthesis)
+ * @param {number} vectorDim - Vector dimension for intent embedding
+ * @returns {arrow.Schema} Arrow schema
+ */
+export function createSynthesizedSkillSchema(vectorDim = DEFAULT_VECTOR_DIMENSION) {
+    return new arrow.Schema([
+        new arrow.Field("id", new arrow.Utf8(), false),
+        new arrow.Field("name", new arrow.Utf8(), false),
+        new arrow.Field("intent", new arrow.Utf8(), false),
+        new arrow.Field("yamo_text", new arrow.Utf8(), false),
+        new arrow.Field("vector", new arrow.FixedSizeList(vectorDim, new arrow.Field("item", new arrow.Float32(), true)), false),
+        new arrow.Field("metadata", new arrow.Utf8(), true), // Stored as JSON: {reliability, use_count, created_at}
+        new arrow.Field("created_at", new arrow.Timestamp(arrow.TimeUnit.MILLISECOND), false),
+    ]);
+}
+/**
+ * Check if a table is using V2 schema
+ * @param {arrow.Schema} schema - Table schema to check
+ * @returns {boolean} True if V2 schema detected
+ */
+export function isSchemaV2(schema) {
+    return schema.fields.some((f) => f.name === "session_id");
+}
+/**
+ * Memory table schema using Apache Arrow format (default 384 dimensions)
+ * @deprecated Use createMemorySchema(vectorDim) for dynamic dimensions
+ */
+export const MEMORY_SCHEMA = createMemorySchema(DEFAULT_VECTOR_DIMENSION);
+/**
+ * Index configuration for memory table
+ * Indices should be created after data is inserted
+ */
+export const INDEX_CONFIG = {
+    vector: {
+        index_type: "ivf_pq",
+        metric: "cosine",
+        num_partitions: 256,
+        num_sub_vectors: 8,
+    },
+    full_text: {
+        fields: ["content"],
+    },
+};
+/**
+ * Creates a memory table in LanceDB with the predefined schema (384 dimensions)
+ * @param {lancedb.Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create (default: 'memory_entries')
+ * @returns {Promise<lancedb.Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ * @deprecated Use createMemoryTableWithDimension() for dynamic dimensions
+ */
+export async function createMemoryTable(db, tableName = "memory_entries") {
+    return createMemoryTableWithDimension(db, tableName, DEFAULT_VECTOR_DIMENSION);
+}
+/**
+ * Creates a memory table in LanceDB with a specific vector dimension
+ * @param {lancedb.Connection} db - LanceDB connection
+ * @param {string} tableName - Name of the table to create
+ * @param {number} vectorDim - Vector dimension (384, 768, 1536, etc.)
+ * @returns {Promise<lancedb.Table>} The created or opened table
+ * @throws {Error} If table creation fails
+ */
+export async function createMemoryTableWithDimension(db, tableName, vectorDim) {
+    try {
+        // Check if table already exists
+        const existingTables = await db.tableNames();
+        if (existingTables.includes(tableName)) {
+            return await db.openTable(tableName);
+        }
+        // Create schema with specified dimension
+        const schema = createMemorySchema(vectorDim);
+        // Create table with schema
+        // LanceDB v0.23.0+ accepts empty array as initial data with schema option
+        const table = await db.createTable(tableName, [], { schema }); // Cast to any because lancedb types might be strict about options
+        return table;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to create memory table with dimension ${vectorDim}: ${message}`);
+    }
+}
+export default {
+    MEMORY_SCHEMA,
+    INDEX_CONFIG,
+    createMemoryTable,
+    createMemoryTableWithDimension,
+    createMemorySchema,
+    createMemorySchemaV2,
+    isSchemaV2,
+    getEmbeddingDimension,
+    DEFAULT_VECTOR_DIMENSION,
+    EMBEDDING_DIMENSIONS,
+};

package/lib/memory/scorer.d.ts

@@ -0,0 +1,25 @@
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+export declare class MemoryScorer {
+    #private;
+    /**
+     * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+     */
+    constructor(mesh: any);
+    /**
+     * Calculate importance score for content
+     * @param {string} content - Content to score
+     * @param {Object} metadata - Associated metadata
+     * @returns {Promise<number>} Importance score (0-1)
+     */
+    calculateImportance(content: any, metadata?: {}): number;
+    /**
+     * Check if content is duplicate of existing memory
+     * @param {string} content - Content to check
+     * @param {number} threshold - Similarity threshold (default 0.9)
+     * @returns {Promise<boolean>} True if duplicate exists
+     */
+    isDuplicate(content: any, threshold?: number): Promise<boolean>;
+}
+export default MemoryScorer;

package/lib/memory/scorer.js

@@ -0,0 +1,78 @@
+// @ts-nocheck
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+export class MemoryScorer {
+    #mesh;
+    /**
+     * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+     */
+    constructor(mesh) {
+        this.#mesh = mesh;
+    }
+    /**
+     * Calculate importance score for content
+     * @param {string} content - Content to score
+     * @param {Object} metadata - Associated metadata
+     * @returns {Promise<number>} Importance score (0-1)
+     */
+    calculateImportance(content, metadata = {}) {
+        let score = 0;
+        // Content length (longer = more important, up to a point)
+        const length = content.length;
+        score += Math.min(length / 1000, 0.2);
+        // Has structured data (JSON, code blocks)
+        if (content.includes("```") || content.includes("{")) {
+            score += 0.1;
+        }
+        // Interaction type bonuses
+        if (metadata.interaction_type === "tool_execution") {
+            score += 0.15;
+        }
+        if (metadata.interaction_type === "file_operation") {
+            score += 0.1;
+        }
+        // Tool usage indicates importance
+        if (metadata.tools_used?.length > 0) {
+            score += Math.min(metadata.tools_used.length * 0.05, 0.15);
+        }
+        // File involvement
+        if (metadata.files_involved?.length > 0) {
+            score += Math.min(metadata.files_involved.length * 0.05, 0.15);
+        }
+        // Keywords that indicate importance
+        const importantKeywords = [
+            "error",
+            "bug",
+            "fix",
+            "important",
+            "critical",
+            "note",
+            "remember",
+        ];
+        const lowerContent = content.toLowerCase();
+        const keywordMatches = importantKeywords.filter((k) => lowerContent.includes(k)).length;
+        score += Math.min(keywordMatches * 0.05, 0.15);
+        return Math.min(score, 1.0);
+    }
+    /**
+     * Check if content is duplicate of existing memory
+     * @param {string} content - Content to check
+     * @param {number} threshold - Similarity threshold (default 0.9)
+     * @returns {Promise<boolean>} True if duplicate exists
+     */
+    async isDuplicate(content, threshold = 0.9) {
+        try {
+            const results = await this.#mesh.search(content, {
+                limit: 1,
+                useCache: false,
+            });
+            return results.length > 0 && results[0].score >= threshold;
+        }
+        catch (_error) {
+            // On error, assume not duplicate to allow storage
+            return false;
+        }
+    }
+}
+export default MemoryScorer;

package/lib/memory/scorer.ts

@@ -0,0 +1,78 @@
+// @ts-nocheck
+/**
+ * MemoryScorer - Calculate memory importance and detect duplicates
+ */
+export class MemoryScorer {
+    #mesh;
+    /**
+     * @param {MemoryMesh} mesh - MemoryMesh instance for duplicate checking
+     */
+    constructor(mesh) {
+        this.#mesh = mesh;
+    }
+    /**
+     * Calculate importance score for content
+     * @param {string} content - Content to score
+     * @param {Object} metadata - Associated metadata
+     * @returns {Promise<number>} Importance score (0-1)
+     */
+    calculateImportance(content, metadata = {}) {
+        let score = 0;
+        // Content length (longer = more important, up to a point)
+        const length = content.length;
+        score += Math.min(length / 1000, 0.2);
+        // Has structured data (JSON, code blocks)
+        if (content.includes("```") || content.includes("{")) {
+            score += 0.1;
+        }
+        // Interaction type bonuses
+        if (metadata.interaction_type === "tool_execution") {
+            score += 0.15;
+        }
+        if (metadata.interaction_type === "file_operation") {
+            score += 0.1;
+        }
+        // Tool usage indicates importance
+        if (metadata.tools_used?.length > 0) {
+            score += Math.min(metadata.tools_used.length * 0.05, 0.15);
+        }
+        // File involvement
+        if (metadata.files_involved?.length > 0) {
+            score += Math.min(metadata.files_involved.length * 0.05, 0.15);
+        }
+        // Keywords that indicate importance
+        const importantKeywords = [
+            "error",
+            "bug",
+            "fix",
+            "important",
+            "critical",
+            "note",
+            "remember",
+        ];
+        const lowerContent = content.toLowerCase();
+        const keywordMatches = importantKeywords.filter((k) => lowerContent.includes(k)).length;
+        score += Math.min(keywordMatches * 0.05, 0.15);
+        return Math.min(score, 1.0);
+    }
+    /**
+     * Check if content is duplicate of existing memory
+     * @param {string} content - Content to check
+     * @param {number} threshold - Similarity threshold (default 0.9)
+     * @returns {Promise<boolean>} True if duplicate exists
+     */
+    async isDuplicate(content, threshold = 0.9) {
+        try {
+            const results = await this.#mesh.search(content, {
+                limit: 1,
+                useCache: false,
+            });
+            return results.length > 0 && results[0].score >= threshold;
+        }
+        catch (_error) {
+            // On error, assume not duplicate to allow storage
+            return false;
+        }
+    }
+}
+export default MemoryScorer;

package/lib/memory/search/index.d.ts

@@ -0,0 +1 @@
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/index.js

@@ -0,0 +1,2 @@
+// @ts-nocheck
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/index.ts

@@ -0,0 +1,2 @@
+// @ts-nocheck
+export { KeywordSearch } from "./keyword-search.js";

package/lib/memory/search/keyword-search.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+export declare class KeywordSearch {
+    index: any;
+    docLengths: any;
+    idf: any;
+    docs: any;
+    isDirty: any;
+    constructor();
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text: any): any;
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id: any, content: any, metadata?: {}): void;
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id: any): void;
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats(): void;
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query: any, options?: {}): any[];
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records: any): void;
+}

package/lib/memory/search/keyword-search.js

@@ -0,0 +1,136 @@
+// @ts-nocheck
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+export class KeywordSearch {
+    index; // token -> Map<docId, tf>
+    docLengths; // docId -> length
+    idf; // token -> idf value
+    docs; // docId -> content (optional, for snippet)
+    isDirty;
+    constructor() {
+        this.index = new Map();
+        this.docLengths = new Map();
+        this.idf = new Map();
+        this.docs = new Map();
+        this.isDirty = false;
+    }
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text) {
+        if (!text) {
+            return [];
+        }
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, "") // Remove punctuation
+            .split(/\s+/)
+            .filter((t) => t.length > 2) // Filter stopwords/short
+            .map((t) => t.substring(0, 20)); // Truncate
+    }
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id, content, metadata = {}) {
+        const tokens = this.tokenize(content);
+        const termFreqs = new Map();
+        tokens.forEach((t) => {
+            termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+        });
+        this.docLengths.set(id, tokens.length);
+        this.docs.set(id, { content, metadata });
+        // Update index
+        for (const [token, freq] of termFreqs.entries()) {
+            if (!this.index.has(token)) {
+                this.index.set(token, new Map());
+            }
+            this.index.get(token).set(id, freq);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id) {
+        this.docLengths.delete(id);
+        this.docs.delete(id);
+        // This is expensive O(Vocab), but okay for small scale
+        for (const docMap of this.index.values()) {
+            docMap.delete(id);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats() {
+        if (!this.isDirty) {
+            return;
+        }
+        const N = this.docLengths.size;
+        this.idf.clear();
+        for (const [token, docMap] of this.index.entries()) {
+            const df = docMap.size;
+            // Standard IDF: log(N / (df + 1)) + 1
+            const idf = Math.log(N / (df + 1)) + 1;
+            this.idf.set(token, idf);
+        }
+        this.isDirty = false;
+    }
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query, options = {}) {
+        this._computeStats();
+        const tokens = this.tokenize(query);
+        const scores = new Map(); // docId -> score
+        const matches = new Map(); // docId -> matched tokens
+        const limit = options.limit || 10;
+        for (const token of tokens) {
+            const docMap = this.index.get(token);
+            if (!docMap) {
+                continue;
+            }
+            const idf = this.idf.get(token) || 0;
+            for (const [docId, tf] of docMap.entries()) {
+                // TF-IDF Score
+                // Score = tf * idf * (normalization?)
+                // Simple variant:
+                const score = tf * idf;
+                scores.set(docId, (scores.get(docId) || 0) + score);
+                if (!matches.has(docId)) {
+                    matches.set(docId, []);
+                }
+                matches.get(docId).push(token);
+            }
+        }
+        // Convert to array and sort
+        return Array.from(scores.entries())
+            .map(([id, score]) => ({
+            id,
+            score,
+            matches: matches.get(id) || [],
+            ...this.docs.get(id),
+        }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records) {
+        records.forEach((r) => this.add(r.id, r.content, r.metadata));
+    }
+}

package/lib/memory/search/keyword-search.ts

@@ -0,0 +1,136 @@
+// @ts-nocheck
+/**
+ * Simple Keyword Search Engine (In-Memory)
+ * Provides basic TF-IDF style retrieval to complement vector search
+ */
+export class KeywordSearch {
+    index; // token -> Map<docId, tf>
+    docLengths; // docId -> length
+    idf; // token -> idf value
+    docs; // docId -> content (optional, for snippet)
+    isDirty;
+    constructor() {
+        this.index = new Map();
+        this.docLengths = new Map();
+        this.idf = new Map();
+        this.docs = new Map();
+        this.isDirty = false;
+    }
+    /**
+     * Tokenize text into normalized terms
+     * @param {string} text
+     * @returns {string[]} tokens
+     */
+    tokenize(text) {
+        if (!text) {
+            return [];
+        }
+        return text
+            .toLowerCase()
+            .replace(/[^\w\s]/g, "") // Remove punctuation
+            .split(/\s+/)
+            .filter((t) => t.length > 2) // Filter stopwords/short
+            .map((t) => t.substring(0, 20)); // Truncate
+    }
+    /**
+     * Add a document to the index
+     * @param {string} id
+     * @param {string} content
+     * @param {Object} [metadata]
+     */
+    add(id, content, metadata = {}) {
+        const tokens = this.tokenize(content);
+        const termFreqs = new Map();
+        tokens.forEach((t) => {
+            termFreqs.set(t, (termFreqs.get(t) || 0) + 1);
+        });
+        this.docLengths.set(id, tokens.length);
+        this.docs.set(id, { content, metadata });
+        // Update index
+        for (const [token, freq] of termFreqs.entries()) {
+            if (!this.index.has(token)) {
+                this.index.set(token, new Map());
+            }
+            this.index.get(token).set(id, freq);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Remove a document
+     * @param {string} id
+     */
+    remove(id) {
+        this.docLengths.delete(id);
+        this.docs.delete(id);
+        // This is expensive O(Vocab), but okay for small scale
+        for (const docMap of this.index.values()) {
+            docMap.delete(id);
+        }
+        this.isDirty = true;
+    }
+    /**
+     * Recalculate IDF scores
+     */
+    _computeStats() {
+        if (!this.isDirty) {
+            return;
+        }
+        const N = this.docLengths.size;
+        this.idf.clear();
+        for (const [token, docMap] of this.index.entries()) {
+            const df = docMap.size;
+            // Standard IDF: log(N / (df + 1)) + 1
+            const idf = Math.log(N / (df + 1)) + 1;
+            this.idf.set(token, idf);
+        }
+        this.isDirty = false;
+    }
+    /**
+     * Search for query terms
+     * @param {string} query
+     * @param {Object} options
+     * @returns {Array<{id: string, score: number, matches: string[], content: string, metadata: Object}>}
+     */
+    search(query, options = {}) {
+        this._computeStats();
+        const tokens = this.tokenize(query);
+        const scores = new Map(); // docId -> score
+        const matches = new Map(); // docId -> matched tokens
+        const limit = options.limit || 10;
+        for (const token of tokens) {
+            const docMap = this.index.get(token);
+            if (!docMap) {
+                continue;
+            }
+            const idf = this.idf.get(token) || 0;
+            for (const [docId, tf] of docMap.entries()) {
+                // TF-IDF Score
+                // Score = tf * idf * (normalization?)
+                // Simple variant:
+                const score = tf * idf;
+                scores.set(docId, (scores.get(docId) || 0) + score);
+                if (!matches.has(docId)) {
+                    matches.set(docId, []);
+                }
+                matches.get(docId).push(token);
+            }
+        }
+        // Convert to array and sort
+        return Array.from(scores.entries())
+            .map(([id, score]) => ({
+            id,
+            score,
+            matches: matches.get(id) || [],
+            ...this.docs.get(id),
+        }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    /**
+     * Bulk load records
+     * @param {Array} records
+     */
+    load(records) {
+        records.forEach((r) => this.add(r.id, r.content, r.metadata));
+    }
+}