memories-lite 0.9.0

package/dist/vectorstores/lite.js

@@ -0,0 +1,319 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LiteVectorStore = void 0;
+// vector persistence
+const path_1 = __importDefault(require("path"));
+const sqlite3_1 = __importDefault(require("sqlite3"));
+const crypto_1 = require("crypto");
+/**
+ * LiteVectorStore provides a simple vector storage implementation.
+ *
+ * This class manages user-sensitive vector data, with each instance tied to a specific user.
+ * Data is stored in a local SQLite database with appropriate isolation between users.
+ * The user's identity is hashed for privacy and security purposes.
+ * ⚠️ User memories are typically small (usually <1000 vectors), therefore database optimization is not a priority.
+ *
+ * @implements {VectorStore}
+ */
+class LiteVectorStore {
+    constructor(config, currentUserId) {
+        if (!config.scoring) {
+            throw new Error("Scoring configuration is missing in VectorStoreConfig");
+        }
+        this.dimension = config.dimension || 1536;
+        this.currentUserId = currentUserId;
+        this.isSecure = config.secure || false;
+        this.scoringConfig = config.scoring;
+        this.cleanupThreshold = config.recencyCleanupThreshold; // Store threshold
+        config.rootPath = config.rootPath || process.cwd();
+        const filename = this.isSecure ? `memories-lite-${currentUserId}.db` : `memories-lite-global.db`;
+        const dbPath = (config.rootPath == ':memory:') ? ':memory:' : path_1.default.join(config.rootPath, filename);
+        // Add error handling callback for the database connection
+        this.db = new sqlite3_1.default.Database(dbPath);
+    }
+    async init() {
+        await this.run(`
+      CREATE TABLE IF NOT EXISTS vectors (
+        id TEXT PRIMARY KEY,
+        vector BLOB NOT NULL,
+        payload TEXT NOT NULL
+      )
+    `);
+        await this.run(`
+      CREATE TABLE IF NOT EXISTS memory_migrations (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        user_id TEXT NOT NULL UNIQUE
+      )
+    `);
+    }
+    async run(sql, params = []) {
+        return new Promise((resolve, reject) => {
+            this.db.run(sql, params, (err) => {
+                if (err)
+                    reject(err);
+                else
+                    resolve();
+            });
+        });
+    }
+    async all(sql, params = []) {
+        return new Promise((resolve, reject) => {
+            this.db.all(sql, params, (err, rows) => {
+                if (err)
+                    reject(err);
+                else
+                    resolve(rows);
+            });
+        });
+    }
+    async getOne(sql, params = []) {
+        return new Promise((resolve, reject) => {
+            this.db.get(sql, params, (err, row) => {
+                if (err)
+                    reject(err);
+                else
+                    resolve(row);
+            });
+        });
+    }
+    //
+    // TODO: Memory Decay & Hybrid Scoring (α·cosSim + β·when + γ·1 [+ δ·tag])
+    // scoreTotal = α·cos + β·recency + γ·1 + δ·tag
+    // https://chatgpt.com/g/g-p-677932764d20819181f8805995ddab8b-the-coder/c/68121341-7fdc-800b-8734-af49719eb5e1
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error("Vectors must have the same dimension for cosine similarity.");
+        }
+        if (a.length === 0) {
+            return 0;
+        }
+        let dotProduct = 0;
+        let normA = 0;
+        let normB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            normA += a[i] * a[i];
+            normB += b[i] * b[i];
+        }
+        const magnitudeA = Math.sqrt(normA);
+        const magnitudeB = Math.sqrt(normB);
+        if (magnitudeA === 0 || magnitudeB === 0) {
+            return 0;
+        }
+        return dotProduct / (magnitudeA * magnitudeB);
+    }
+    filterVector(vector, filters) {
+        if (!filters)
+            return true;
+        return Object.entries(filters).every(([key, value]) => vector.payload && vector.payload[key] === value);
+    }
+    // by default, the userId is the SHA256 derivation 
+    getInstanceId() {
+        return this.currentUserId;
+    }
+    static async from(userId, config) {
+        if (!userId) {
+            throw new Error("userId is required");
+        }
+        if (!config.scoring) {
+            throw new Error("Scoring configuration is missing in VectorStoreConfig");
+        }
+        const hashedUserId = (0, crypto_1.createHash)('sha256').update(userId).digest('hex');
+        if (!LiteVectorStore.cache) {
+            LiteVectorStore.cache = new Map();
+        }
+        const cachedStore = LiteVectorStore.cache.get(hashedUserId);
+        if (cachedStore) {
+            Object.setPrototypeOf(cachedStore, LiteVectorStore.prototype);
+            cachedStore.currentUserId = hashedUserId;
+            // Ensure scoring config and threshold are updated if config object changed
+            cachedStore.scoringConfig = config.scoring;
+            cachedStore.cleanupThreshold = config.recencyCleanupThreshold;
+            return cachedStore;
+        }
+        // Pass the full config (including scoring) to the constructor
+        const newStore = new LiteVectorStore(config, hashedUserId);
+        await newStore.init();
+        LiteVectorStore.cache.set(hashedUserId, newStore);
+        return newStore;
+    }
+    async insert(vectors, ids, payloads) {
+        for (let i = 0; i < vectors.length; i++) {
+            if (vectors[i].length !== this.dimension) {
+                throw new Error(`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`);
+            }
+            //
+            // remove the userId from the payload as sensitive data
+            this.isSecure && delete payloads[i].userId;
+            const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
+            await this.run(`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`, [ids[i], vectorBuffer, JSON.stringify(payloads[i])]);
+        }
+    }
+    async search(query, limit = 10, filters) {
+        if (query.length !== this.dimension) {
+            throw new Error(`Query dimension mismatch. Expected ${this.dimension}, got ${query.length}`);
+        }
+        const results = [];
+        const rows = await this.all(`SELECT * FROM vectors`);
+        for (const row of rows) {
+            const vector = Array.from(new Float32Array(row.vector.buffer)); // Convert buffer to number array
+            const payload = JSON.parse(row.payload);
+            const memoryVector = {
+                id: row.id,
+                vector: vector,
+                payload,
+            };
+            if (this.filterVector(memoryVector, filters)) {
+                const cosineScore = this.cosineSimilarity(query, vector);
+                const hybridScore = this.calculateHybridScore(cosineScore, payload);
+                results.push({
+                    id: memoryVector.id,
+                    payload: memoryVector.payload,
+                    score: hybridScore,
+                });
+            }
+        }
+        results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
+        const finalResults = results.slice(0, limit);
+        // Trigger cleanup after search, if threshold is set
+        if (this.cleanupThreshold !== undefined && this.cleanupThreshold >= 0) {
+            // Run cleanup asynchronously in the background (fire and forget)
+            // to avoid delaying the search response.
+            // Consider adding mutex or debouncing in high-frequency scenarios.
+            this._cleanupByRecency(this.cleanupThreshold).catch(err => {
+                console.error("Error during background recency cleanup:", err);
+            });
+        }
+        return finalResults;
+    }
+    async get(vectorId) {
+        const row = await this.getOne(`SELECT * FROM vectors WHERE id = ?`, [
+            vectorId,
+        ]);
+        if (!row)
+            return null;
+        const payload = JSON.parse(row.payload);
+        return {
+            id: row.id,
+            payload,
+        };
+    }
+    async update(vectorId, vector, payload) {
+        if (vector.length !== this.dimension) {
+            throw new Error(`Vector dimension mismatch. Expected ${this.dimension}, got ${vector.length}`);
+        }
+        //
+        // remove the userId from the payload as sensitive data
+        this.isSecure && delete payload.userId;
+        const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
+        await this.run(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`, [
+            vectorBuffer,
+            JSON.stringify(payload),
+            vectorId,
+        ]);
+    }
+    async delete(vectorId) {
+        await this.run(`DELETE FROM vectors WHERE id = ?`, [vectorId]);
+    }
+    async deleteCol() {
+        await this.run(`DROP TABLE IF EXISTS vectors`);
+        await this.init();
+    }
+    async list(filters, limit = 100) {
+        const rows = await this.all(`SELECT * FROM vectors`);
+        const results = [];
+        for (const row of rows) {
+            const memoryVector = {
+                id: row.id,
+                vector: Array.from(new Float32Array(row.vector.buffer)),
+                payload: {},
+            };
+            if (this.filterVector(memoryVector, filters)) {
+                // load payload at the end
+                results.push({
+                    id: memoryVector.id,
+                    payload: JSON.parse(row.payload),
+                });
+            }
+        }
+        return [results.slice(0, limit), results.length];
+    }
+    // Calculate recency score using exponential decay
+    calculateRecencyScore(createdAt, halfLifeDays) {
+        if (halfLifeDays === Infinity || !createdAt) {
+            return 1.0; // No decay or no date
+        }
+        if (halfLifeDays <= 0) {
+            return 0.0; // Instant decay
+        }
+        const createdAtDate = new Date(createdAt);
+        if (isNaN(createdAtDate.getTime())) {
+            console.warn('Invalid createdAt date format:', createdAt);
+            return 0.5; // Neutral score if date is invalid
+        }
+        const now = new Date();
+        const ageInMilliseconds = now.getTime() - createdAtDate.getTime();
+        const ageInDays = ageInMilliseconds / (1000 * 60 * 60 * 24);
+        if (ageInDays < 0) {
+            console.warn('createdAt date is in the future:', createdAt);
+            return 1.0; // Treat future dates as brand new
+        }
+        const lambda = Math.log(2) / halfLifeDays;
+        return Math.exp(-lambda * ageInDays);
+    }
+    // Hybrid Scoring: Combines Cosine Similarity, Recency, and Base Importance
+    calculateHybridScore(cosineScore, payload) {
+        const memoryType = payload.type;
+        const createdAt = payload.createdAt; // Expecting ISO string
+        // Get scoring parameters for the memory type or use default
+        const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
+        const { alpha, beta, gamma, halfLifeDays } = typeConfig || this.scoringConfig.default;
+        const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
+        // Calculate final score
+        // scoreTotal = α * cosSim + β * recency + γ * baseImportance (using 1 as base)
+        const hybridScore = alpha * cosineScore + beta * recencyScore + gamma;
+        // Ensure score is within a reasonable range (e.g., 0 to alpha+beta+gamma)
+        return Math.max(0, hybridScore);
+    }
+    // Internal method to clean up vectors based on recency score threshold
+    async _cleanupByRecency(threshold) {
+        const rows = await this.all(`SELECT id, payload FROM vectors`);
+        let deletedCount = 0;
+        const idsToDelete = [];
+        for (const row of rows) {
+            try {
+                const payload = JSON.parse(row.payload);
+                const memoryType = payload.type;
+                const createdAt = payload.createdAt;
+                // Get halfLifeDays for the type or use default
+                const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
+                const { halfLifeDays } = typeConfig || this.scoringConfig.default;
+                // Calculate current recency score
+                const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
+                // Mark for deletion if score is below threshold
+                // Add extra check for halfLifeDays > 0 to avoid deleting permanent items instantly if threshold is 0
+                if (recencyScore < threshold && halfLifeDays > 0) {
+                    // Optional: Add type-specific exceptions (e.g., never delete 'factual')
+                    // if (payload.type !== 'factual') {
+                    idsToDelete.push(row.id);
+                    // }
+                }
+            }
+            catch (error) {
+                console.error(`Error processing vector ${row.id} for cleanup:`, error);
+                // Optionally delete vectors with bad payload/date based on policy
+            }
+        }
+        if (idsToDelete.length > 0) {
+            const placeholders = idsToDelete.map(() => '?').join(',');
+            await this.run(`DELETE FROM vectors WHERE id IN (${placeholders})`, idsToDelete);
+            deletedCount = idsToDelete.length;
+            console.log(`Recency Cleanup: Removed ${deletedCount} vectors with score < ${threshold}.`);
+        }
+        return deletedCount;
+    }
+}
+exports.LiteVectorStore = LiteVectorStore;

package/dist/vectorstores/llm.d.ts

@@ -0,0 +1,31 @@
+import { VectorStore } from './base';
+import { SearchFilters, VectorStoreConfig, VectorStoreResult } from '../types';
+export interface FakeVectorStore extends VectorStore {
+    lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]>;
+    dump(filePath?: string, userId?: string): Promise<void>;
+    load(filePath?: string, userId?: string): Promise<void>;
+}
+interface MemoryVector {
+    id: string;
+    vector: number[];
+    payload: Record<string, any>;
+}
+export declare class LLMVectorStore implements FakeVectorStore {
+    constructor(config?: VectorStoreConfig);
+    dump(filePath?: string, userId?: string): Promise<void>;
+    load(filePath?: string, userId?: string): Promise<void>;
+    add(record: MemoryVector, userId?: string): Promise<string>;
+    search(query: number[], limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]>;
+    lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]>;
+    get(id: string, userId?: string): Promise<MemoryVector | null>;
+    update(vectorId: string, vector: number[], payload: Record<string, any>): Promise<void>;
+    delete(id: string, userId?: string): Promise<void>;
+    reset(userId?: string): Promise<void>;
+    insert(vectors: number[][], ids: string[], payloads: Record<string, any>[]): Promise<void>;
+    deleteCol(userId?: string): Promise<void>;
+    list(filters?: SearchFilters, limit?: number): Promise<[VectorStoreResult[], number]>;
+    getInstanceId(): string;
+    setUserId(userId: string): Promise<void>;
+    initialize(): Promise<void>;
+}
+export {};

package/dist/vectorstores/llm.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LLMVectorStore = void 0;
+// Removed fs and path imports as they are not needed for empty methods
+class LLMVectorStore {
+    constructor(config) {
+        console.log('LLMVectorStore initialized', config);
+        // Initialization logic based on config, if any
+    }
+    async dump(filePath, userId) {
+        // TODO: Implement LLMVectorStore dump logic
+        console.log('LLMVectorStore dump called for:', userId || 'all users', 'to path:', filePath);
+        // Placeholder implementation
+    }
+    async load(filePath, userId) {
+        // TODO: Implement LLMVectorStore load logic
+        console.log('LLMVectorStore load called for:', userId || 'all users', 'from path:', filePath);
+        // Placeholder implementation
+    }
+    async add(record, userId) {
+        // TODO: Implement LLMVectorStore add logic
+        console.log('LLMVectorStore add called with:', record, userId);
+        return 'placeholder-id'; // Placeholder return
+    }
+    async search(query, limit, filters) {
+        return [];
+    }
+    async lookup(query, limit, filters) {
+        // TODO: Implement LLMVectorStore search logic using query vector
+        // Note: userId parameter is removed as it's not in the base interface signature.
+        // If userId is needed, it might have to be passed via filters or managed differently.
+        console.log('LLMVectorStore search called with vector query:', query, 'limit:', limit, 'filters:', filters);
+        // Linter error on VectorStoreResult likely means its definition in ../types.ts needs checking/creation.
+        return []; // Placeholder return
+    }
+    async get(id, userId) {
+        // TODO: Implement LLMVectorStore get logic
+        console.log('LLMVectorStore get called with:', id, userId);
+        return null; // Placeholder return
+    }
+    async update(vectorId, vector, payload) {
+        // TODO: Implement LLMVectorStore update logic matching the base interface
+        console.log('LLMVectorStore update called with:', vectorId, vector, payload);
+        // Placeholder implementation
+    }
+    async delete(id, userId) {
+        // TODO: Implement LLMVectorStore delete logic
+        console.log('LLMVectorStore delete called with:', id, userId);
+        // Placeholder implementation
+    }
+    async reset(userId) {
+        // TODO: Implement LLMVectorStore reset logic
+        console.log('LLMVectorStore reset called for:', userId || 'all users');
+        // Placeholder implementation
+    }
+    // TODO: Implement other missing methods from base VectorStore interface if needed
+    // Placeholder for missing method from VectorStore interface
+    async insert(vectors, ids, payloads) {
+        // TODO: Implement LLMVectorStore insert logic
+    }
+    // Placeholder for missing method from VectorStore interface
+    async deleteCol(userId) {
+        // TODO: Implement LLMVectorStore deleteCol logic
+        console.log('LLMVectorStore deleteCol called for:', userId || 'default collection');
+    }
+    // Placeholder for missing method from VectorStore interface
+    async list(filters, limit) {
+        // TODO: Implement LLMVectorStore list logic
+        return [[], 0];
+    }
+    // Placeholder for missing method from VectorStore interface
+    getInstanceId() {
+        // TODO: Implement LLMVectorStore getUserId logic (if applicable to this store)
+        console.log('LLMVectorStore getUserId called');
+        return 'placeholder-user-id'; // Or null if not managed per-user this way
+    }
+    // Placeholder for missing method from VectorStore interface
+    async setUserId(userId) {
+        // TODO: Implement LLMVectorStore setUserId logic (if applicable)
+        console.log('LLMVectorStore setUserId called with:', userId);
+    }
+    // Placeholder for missing method from VectorStore interface
+    async initialize() {
+        // TODO: Implement LLMVectorStore initialize logic (if needed)
+        console.log('LLMVectorStore initialize called');
+    }
+}
+exports.LLMVectorStore = LLMVectorStore;

package/jest.config.js

@@ -0,0 +1,22 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} **/
+module.exports = {
+  verbose: true,
+  testEnvironment: "node",
+  transform: {
+    "^.+.ts?$": ["ts-jest",{}],
+  },
+  // Désactiver l'exécution parallèle des tests
+  maxWorkers: 1,
+  // Exécuter un seul fichier de test à la fois
+  maxConcurrency: 1,
+  testTimeout: 100000,
+  collectCoverage: false,
+  coverageDirectory: 'coverage',
+  collectCoverageFrom: [
+    'src/**/*.ts',
+    '!src/**/*.d.ts',
+    '!src/**/*.test.ts',
+    '!src/__tests__/**/*.ts'
+  ],
+
+};

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "memories-lite",
+  "version": "0.9.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "build": "tsc -b",
+    "test": "jest"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.18.0",
+    "@google/genai": "^0.7.0",
+    "@keyv/sqlite": "^4.0.3",
+    "@types/node": "^20.11.19",
+    "@types/uuid": "^9.0.8",
+    "cache-manager": "^6.4.2",
+    "dotenv": "^16.5.0",
+    "keyv": "^5.3.3",
+    "openai": "^4.96.0",
+    "sqlite3": "^5.1.7",
+    "uuid": "^9.0.1",
+    "zod": "^3.24.3"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.12",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.4.0"
+  },
+  "keywords": [
+    "memory",
+    "openai",
+    "embeddings",
+    "vector-store",
+    "typescript"
+  ]
+}

package/src/config/defaults.ts

@@ -0,0 +1,61 @@
+import { MemoryConfig, MemoryScoringConfig } from "../types";
+
+const DEFAULT_SCORING_CONFIG: MemoryScoringConfig = {
+  // Values from memories-lite rule & user request
+  procedural: { alpha: 0.30, beta: 0.40, gamma: 0.05, halfLifeDays: 1 / 24 }, // ~1 hour
+  episodic: { alpha: 0.40, beta: 0.50, gamma: 0.10, halfLifeDays: 2 }, // ~2 days (user request 'temporary')
+  factual: { alpha: 0.70, beta: 0.20, gamma: 0.10, halfLifeDays: 365 },
+  semantic: { alpha: 0.50, beta: 0.25, gamma: 0.25, halfLifeDays: 120 },
+  assistant_preference: { alpha: 0.60, beta: 0.05, gamma: 0.35, halfLifeDays: Infinity },
+  default: { alpha: 0.5, beta: 0.3, gamma: 0.1, halfLifeDays: 30 } // Fallback default
+};
+
+export const DEFAULT_MEMORY_CONFIG: MemoryConfig = {
+  disableHistory: true,
+  enableGraph: false,
+  version: "v1.1",
+  embedder: {
+    provider: "openai",
+    config: {
+      dimension: 768,
+      apiKey: process.env.OPENAI_API_KEY || "",
+      model: "text-embedding-3-small",
+    },
+  },
+  vectorStore: {
+    provider: "lite",
+    config: {
+      collectionName: "memories",
+      dimension: 768,
+      scoring: DEFAULT_SCORING_CONFIG,
+    },
+  },
+  llm: {
+    provider: "openai",
+    config: {
+      apiKey: process.env.OPENAI_API_KEY || "",
+      model: "gpt-4o-mini",
+      modelProperties: undefined,
+    },
+  },
+  graphStore: {
+    provider: "neo4j",
+    config: {
+      url: process.env.NEO4J_URL || "neo4j://localhost:7687",
+      username: process.env.NEO4J_USERNAME || "neo4j",
+      password: process.env.NEO4J_PASSWORD || "password",
+    },
+    llm: {
+      provider: "openai",
+      config: {
+        model: "gpt-4o-mini",
+      },
+    },
+  },
+  historyStore: {
+    provider: "dummy",
+    config: {
+      historyDbPath: "memory.db",
+    },
+  },
+};