memories-lite 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/MEMORIES.md +39 -0
  2. package/README.md +221 -0
  3. package/TECHNICAL.md +135 -0
  4. package/dist/config/defaults.d.ts +2 -0
  5. package/dist/config/defaults.js +61 -0
  6. package/dist/config/manager.d.ts +4 -0
  7. package/dist/config/manager.js +121 -0
  8. package/dist/embeddings/base.d.ts +4 -0
  9. package/dist/embeddings/base.js +2 -0
  10. package/dist/embeddings/google.d.ts +10 -0
  11. package/dist/embeddings/google.js +28 -0
  12. package/dist/embeddings/openai.d.ts +10 -0
  13. package/dist/embeddings/openai.js +31 -0
  14. package/dist/graphs/configs.d.ts +14 -0
  15. package/dist/graphs/configs.js +19 -0
  16. package/dist/graphs/tools.d.ts +271 -0
  17. package/dist/graphs/tools.js +220 -0
  18. package/dist/graphs/utils.d.ts +9 -0
  19. package/dist/graphs/utils.js +105 -0
  20. package/dist/index.d.ts +14 -0
  21. package/dist/index.js +30 -0
  22. package/dist/llms/base.d.ts +16 -0
  23. package/dist/llms/base.js +2 -0
  24. package/dist/llms/google.d.ts +11 -0
  25. package/dist/llms/google.js +44 -0
  26. package/dist/llms/openai.d.ts +9 -0
  27. package/dist/llms/openai.js +73 -0
  28. package/dist/llms/openai_structured.d.ts +11 -0
  29. package/dist/llms/openai_structured.js +72 -0
  30. package/dist/memory/index.d.ts +42 -0
  31. package/dist/memory/index.js +499 -0
  32. package/dist/memory/memory.types.d.ts +23 -0
  33. package/dist/memory/memory.types.js +2 -0
  34. package/dist/prompts/index.d.ts +102 -0
  35. package/dist/prompts/index.js +233 -0
  36. package/dist/storage/DummyHistoryManager.d.ts +7 -0
  37. package/dist/storage/DummyHistoryManager.js +19 -0
  38. package/dist/storage/MemoryHistoryManager.d.ts +8 -0
  39. package/dist/storage/MemoryHistoryManager.js +36 -0
  40. package/dist/storage/base.d.ts +6 -0
  41. package/dist/storage/base.js +2 -0
  42. package/dist/storage/index.d.ts +3 -0
  43. package/dist/storage/index.js +19 -0
  44. package/dist/types/index.d.ts +1071 -0
  45. package/dist/types/index.js +100 -0
  46. package/dist/utils/bm25.d.ts +13 -0
  47. package/dist/utils/bm25.js +51 -0
  48. package/dist/utils/factory.d.ts +13 -0
  49. package/dist/utils/factory.js +49 -0
  50. package/dist/utils/logger.d.ts +7 -0
  51. package/dist/utils/logger.js +9 -0
  52. package/dist/utils/memory.d.ts +3 -0
  53. package/dist/utils/memory.js +44 -0
  54. package/dist/utils/telemetry.d.ts +11 -0
  55. package/dist/utils/telemetry.js +74 -0
  56. package/dist/utils/telemetry.types.d.ts +27 -0
  57. package/dist/utils/telemetry.types.js +2 -0
  58. package/dist/vectorstores/base.d.ts +11 -0
  59. package/dist/vectorstores/base.js +2 -0
  60. package/dist/vectorstores/lite.d.ts +40 -0
  61. package/dist/vectorstores/lite.js +319 -0
  62. package/dist/vectorstores/llm.d.ts +31 -0
  63. package/dist/vectorstores/llm.js +88 -0
  64. package/jest.config.js +22 -0
  65. package/memories-lite.db +0 -0
  66. package/package.json +38 -0
  67. package/src/config/defaults.ts +61 -0
  68. package/src/config/manager.ts +132 -0
  69. package/src/embeddings/base.ts +4 -0
  70. package/src/embeddings/google.ts +32 -0
  71. package/src/embeddings/openai.ts +33 -0
  72. package/src/graphs/configs.ts +30 -0
  73. package/src/graphs/tools.ts +267 -0
  74. package/src/graphs/utils.ts +114 -0
  75. package/src/index.ts +14 -0
  76. package/src/llms/base.ts +20 -0
  77. package/src/llms/google.ts +56 -0
  78. package/src/llms/openai.ts +85 -0
  79. package/src/llms/openai_structured.ts +82 -0
  80. package/src/memory/index.ts +723 -0
  81. package/src/memory/memory.types.ts +27 -0
  82. package/src/prompts/index.ts +268 -0
  83. package/src/storage/DummyHistoryManager.ts +27 -0
  84. package/src/storage/MemoryHistoryManager.ts +58 -0
  85. package/src/storage/base.ts +14 -0
  86. package/src/storage/index.ts +3 -0
  87. package/src/types/index.ts +243 -0
  88. package/src/utils/bm25.ts +64 -0
  89. package/src/utils/factory.ts +59 -0
  90. package/src/utils/logger.ts +13 -0
  91. package/src/utils/memory.ts +48 -0
  92. package/src/utils/telemetry.ts +98 -0
  93. package/src/utils/telemetry.types.ts +34 -0
  94. package/src/vectorstores/base.ts +27 -0
  95. package/src/vectorstores/lite.ts +402 -0
  96. package/src/vectorstores/llm.ts +126 -0
  97. package/tests/lite.spec.ts +158 -0
  98. package/tests/memory.facts.test.ts +211 -0
  99. package/tests/memory.test.ts +406 -0
  100. package/tsconfig.json +16 -0
  101. package/tsconfig.tsbuildinfo +1 -0
@@ -0,0 +1,402 @@
1
+ // vector persistence
2
+ import path from 'path';
3
+ import sqlite3 from 'sqlite3';
4
+
5
+ import { VectorStore } from "./base";
6
+ import { SearchFilters, VectorStoreConfig, VectorStoreResult, MemoryPayload, MemoryScoringConfig, MemoryType } from "../types";
7
+ import { createHash } from 'crypto';
8
+
9
+
10
+ // Define interface for database rows
11
+ interface MemoryVector {
12
+ id: string;
13
+ vector: number[];
14
+ payload: Record<string, any>;
15
+ }
16
+ /**
17
+ * LiteVectorStore provides a simple vector storage implementation.
18
+ *
19
+ * This class manages user-sensitive vector data, with each instance tied to a specific user.
20
+ * Data is stored in a local SQLite database with appropriate isolation between users.
21
+ * The user's identity is hashed for privacy and security purposes.
22
+ * ⚠️ User memories are typically small (usually <1000 vectors), therefore database optimization is not a priority.
23
+ *
24
+ * @implements {VectorStore}
25
+ */
26
+ export class LiteVectorStore implements VectorStore {
27
+ private db: sqlite3.Database;
28
+ private isSecure: boolean;
29
+ private dimension: number;
30
+ private currentUserId: string;
31
+ private scoringConfig: MemoryScoringConfig;
32
+ private cleanupThreshold?: number; // Store the threshold
33
+ private static cache: Map<string, LiteVectorStore>;
34
+ constructor(config: VectorStoreConfig, currentUserId: string) {
35
+ if (!config.scoring) {
36
+ throw new Error("Scoring configuration is missing in VectorStoreConfig");
37
+ }
38
+ this.dimension = config.dimension || 1536;
39
+ this.currentUserId = currentUserId;
40
+ this.isSecure = config.secure || false;
41
+ this.scoringConfig = config.scoring;
42
+ this.cleanupThreshold = config.recencyCleanupThreshold; // Store threshold
43
+ config.rootPath = config.rootPath || process.cwd();
44
+ const filename = this.isSecure ? `memories-lite-${currentUserId}.db` : `memories-lite-global.db`;
45
+ const dbPath = (config.rootPath == ':memory:') ? ':memory:' : path.join(config.rootPath, filename);
46
+
47
+ // Add error handling callback for the database connection
48
+ this.db = new sqlite3.Database(dbPath);
49
+ }
50
+
51
+
52
+ private async init() {
53
+ await this.run(`
54
+ CREATE TABLE IF NOT EXISTS vectors (
55
+ id TEXT PRIMARY KEY,
56
+ vector BLOB NOT NULL,
57
+ payload TEXT NOT NULL
58
+ )
59
+ `);
60
+
61
+ await this.run(`
62
+ CREATE TABLE IF NOT EXISTS memory_migrations (
63
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
64
+ user_id TEXT NOT NULL UNIQUE
65
+ )
66
+ `);
67
+ }
68
+
69
+ private async run(sql: string, params: any[] = []): Promise<void> {
70
+ return new Promise((resolve, reject) => {
71
+ this.db.run(sql, params, (err) => {
72
+ if (err) reject(err);
73
+ else resolve();
74
+ });
75
+ });
76
+ }
77
+
78
+ private async all(sql: string, params: any[] = []): Promise<any[]> {
79
+ return new Promise((resolve, reject) => {
80
+ this.db.all(sql, params, (err, rows) => {
81
+ if (err) reject(err);
82
+ else resolve(rows);
83
+ });
84
+ });
85
+ }
86
+
87
+ private async getOne(sql: string, params: any[] = []): Promise<any> {
88
+ return new Promise((resolve, reject) => {
89
+ this.db.get(sql, params, (err, row) => {
90
+ if (err) reject(err);
91
+ else resolve(row);
92
+ });
93
+ });
94
+ }
95
+
96
+
97
+ //
98
+ // TODO: Memory Decay & Hybrid Scoring (α·cosSim + β·when + γ·1 [+ δ·tag])
99
+ // scoreTotal = α·cos + β·recency + γ·1 + δ·tag
100
+ // https://chatgpt.com/g/g-p-677932764d20819181f8805995ddab8b-the-coder/c/68121341-7fdc-800b-8734-af49719eb5e1
101
+ private cosineSimilarity(a: number[], b: number[]): number {
102
+ if (a.length !== b.length) {
103
+ throw new Error("Vectors must have the same dimension for cosine similarity.");
104
+ }
105
+ if (a.length === 0) {
106
+ return 0;
107
+ }
108
+ let dotProduct = 0;
109
+ let normA = 0;
110
+ let normB = 0;
111
+ for (let i = 0; i < a.length; i++) {
112
+ dotProduct += a[i] * b[i];
113
+ normA += a[i] * a[i];
114
+ normB += b[i] * b[i];
115
+ }
116
+ const magnitudeA = Math.sqrt(normA);
117
+ const magnitudeB = Math.sqrt(normB);
118
+ if (magnitudeA === 0 || magnitudeB === 0) {
119
+ return 0;
120
+ }
121
+ return dotProduct / (magnitudeA * magnitudeB);
122
+ }
123
+
124
+ private filterVector(vector: MemoryVector, filters?: SearchFilters): boolean {
125
+ if (!filters) return true;
126
+ return Object.entries(filters).every(
127
+ ([key, value]) => vector.payload && vector.payload[key] === value,
128
+ );
129
+ }
130
+
131
+ // by default, the userId is the SHA256 derivation
132
+ getInstanceId(): string {
133
+ return this.currentUserId;
134
+ }
135
+
136
+ static async from(userId: string, config: VectorStoreConfig): Promise<LiteVectorStore> {
137
+ if (!userId) {
138
+ throw new Error("userId is required");
139
+ }
140
+ if (!config.scoring) {
141
+ throw new Error("Scoring configuration is missing in VectorStoreConfig");
142
+ }
143
+
144
+ const hashedUserId = createHash('sha256').update(userId).digest('hex');
145
+
146
+ if (!LiteVectorStore.cache) {
147
+ LiteVectorStore.cache = new Map<string, LiteVectorStore>();
148
+ }
149
+
150
+ const cachedStore = LiteVectorStore.cache.get(hashedUserId);
151
+ if (cachedStore) {
152
+ Object.setPrototypeOf(cachedStore, LiteVectorStore.prototype);
153
+ cachedStore.currentUserId = hashedUserId;
154
+ // Ensure scoring config and threshold are updated if config object changed
155
+ cachedStore.scoringConfig = config.scoring;
156
+ cachedStore.cleanupThreshold = config.recencyCleanupThreshold;
157
+ return cachedStore;
158
+ }
159
+
160
+ // Pass the full config (including scoring) to the constructor
161
+ const newStore = new LiteVectorStore(config, hashedUserId);
162
+
163
+ await newStore.init();
164
+ LiteVectorStore.cache.set(hashedUserId, newStore);
165
+ return newStore;
166
+ }
167
+
168
+ async insert(
169
+ vectors: number[][],
170
+ ids: string[],
171
+ payloads: Record<string, any>[],
172
+ ): Promise<void> {
173
+ for (let i = 0; i < vectors.length; i++) {
174
+ if (vectors[i].length !== this.dimension) {
175
+ throw new Error(
176
+ `Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`,
177
+ );
178
+ }
179
+ //
180
+ // remove the userId from the payload as sensitive data
181
+ this.isSecure && delete payloads[i].userId;
182
+ const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
183
+ await this.run(
184
+ `INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`,
185
+ [ids[i], vectorBuffer, JSON.stringify(payloads[i])],
186
+ );
187
+ }
188
+
189
+ }
190
+
191
+ async search(
192
+ query: number[],
193
+ limit: number = 10,
194
+ filters?: SearchFilters,
195
+ ): Promise<VectorStoreResult[]> {
196
+ if (query.length !== this.dimension) {
197
+ throw new Error(
198
+ `Query dimension mismatch. Expected ${this.dimension}, got ${query.length}`,
199
+ );
200
+ }
201
+
202
+ const results: VectorStoreResult[] = [];
203
+ const rows = await this.all(`SELECT * FROM vectors`);
204
+
205
+ for (const row of rows) {
206
+ const vector = Array.from(new Float32Array(row.vector.buffer)); // Convert buffer to number array
207
+ const payload: MemoryPayload = JSON.parse(row.payload);
208
+ const memoryVector: MemoryVector = {
209
+ id: row.id,
210
+ vector: vector,
211
+ payload,
212
+ };
213
+
214
+ if (this.filterVector(memoryVector, filters)) {
215
+ const cosineScore = this.cosineSimilarity(query, vector);
216
+ const hybridScore = this.calculateHybridScore(cosineScore, payload);
217
+
218
+ results.push({
219
+ id: memoryVector.id,
220
+ payload: memoryVector.payload,
221
+ score: hybridScore,
222
+ });
223
+ }
224
+ }
225
+
226
+ results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
227
+ const finalResults = results.slice(0, limit);
228
+
229
+ // Trigger cleanup after search, if threshold is set
230
+ if (this.cleanupThreshold !== undefined && this.cleanupThreshold >= 0) {
231
+ // Run cleanup asynchronously in the background (fire and forget)
232
+ // to avoid delaying the search response.
233
+ // Consider adding mutex or debouncing in high-frequency scenarios.
234
+ this._cleanupByRecency(this.cleanupThreshold).catch(err => {
235
+ console.error("Error during background recency cleanup:", err);
236
+ });
237
+ }
238
+
239
+ return finalResults;
240
+ }
241
+
242
+ async get(vectorId: string): Promise<VectorStoreResult | null> {
243
+ const row = await this.getOne(`SELECT * FROM vectors WHERE id = ?`, [
244
+ vectorId,
245
+ ]);
246
+ if (!row) return null;
247
+
248
+ const payload = JSON.parse(row.payload);
249
+ return {
250
+ id: row.id,
251
+ payload,
252
+ };
253
+ }
254
+
255
+ async update(
256
+ vectorId: string,
257
+ vector: number[],
258
+ payload: Record<string, any>,
259
+ ): Promise<void> {
260
+ if (vector.length !== this.dimension) {
261
+ throw new Error(
262
+ `Vector dimension mismatch. Expected ${this.dimension}, got ${vector.length}`,
263
+ );
264
+ }
265
+ //
266
+ // remove the userId from the payload as sensitive data
267
+ this.isSecure && delete payload.userId;
268
+ const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
269
+ await this.run(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`, [
270
+ vectorBuffer,
271
+ JSON.stringify(payload),
272
+ vectorId,
273
+ ]);
274
+ }
275
+
276
+ async delete(vectorId: string): Promise<void> {
277
+ await this.run(`DELETE FROM vectors WHERE id = ?`, [vectorId]);
278
+ }
279
+
280
+ async deleteCol(): Promise<void> {
281
+ await this.run(`DROP TABLE IF EXISTS vectors`);
282
+ await this.init();
283
+ }
284
+
285
+
286
+ async list(
287
+ filters?: SearchFilters,
288
+ limit: number = 100,
289
+ ): Promise<[VectorStoreResult[], number]> {
290
+ const rows = await this.all(`SELECT * FROM vectors`);
291
+ const results: VectorStoreResult[] = [];
292
+
293
+ for (const row of rows) {
294
+ const memoryVector: MemoryVector = {
295
+ id: row.id,
296
+ vector: Array.from(new Float32Array(row.vector.buffer)),
297
+ payload: {},
298
+ };
299
+
300
+ if (this.filterVector(memoryVector, filters)) {
301
+ // load payload at the end
302
+ results.push({
303
+ id: memoryVector.id,
304
+ payload: JSON.parse(row.payload),
305
+ });
306
+ }
307
+ }
308
+
309
+ return [results.slice(0, limit), results.length];
310
+ }
311
+
312
+ // Calculate recency score using exponential decay
313
+ private calculateRecencyScore(createdAt: string, halfLifeDays: number): number {
314
+ if (halfLifeDays === Infinity || !createdAt) {
315
+ return 1.0; // No decay or no date
316
+ }
317
+ if (halfLifeDays <= 0) {
318
+ return 0.0; // Instant decay
319
+ }
320
+
321
+ const createdAtDate = new Date(createdAt);
322
+ if (isNaN(createdAtDate.getTime())) {
323
+ console.warn('Invalid createdAt date format:', createdAt);
324
+ return 0.5; // Neutral score if date is invalid
325
+ }
326
+
327
+ const now = new Date();
328
+ const ageInMilliseconds = now.getTime() - createdAtDate.getTime();
329
+ const ageInDays = ageInMilliseconds / (1000 * 60 * 60 * 24);
330
+
331
+ if (ageInDays < 0) {
332
+ console.warn('createdAt date is in the future:', createdAt);
333
+ return 1.0; // Treat future dates as brand new
334
+ }
335
+
336
+ const lambda = Math.log(2) / halfLifeDays;
337
+ return Math.exp(-lambda * ageInDays);
338
+ }
339
+
340
+ // Hybrid Scoring: Combines Cosine Similarity, Recency, and Base Importance
341
+ private calculateHybridScore(cosineScore: number, payload: MemoryPayload): number {
342
+ const memoryType: MemoryType | undefined = payload.type;
343
+ const createdAt = payload.createdAt; // Expecting ISO string
344
+
345
+ // Get scoring parameters for the memory type or use default
346
+ const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
347
+ const { alpha, beta, gamma, halfLifeDays } = typeConfig || this.scoringConfig.default;
348
+
349
+ const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
350
+
351
+ // Calculate final score
352
+ // scoreTotal = α * cosSim + β * recency + γ * baseImportance (using 1 as base)
353
+ const hybridScore = alpha * cosineScore + beta * recencyScore + gamma;
354
+
355
+ // Ensure score is within a reasonable range (e.g., 0 to alpha+beta+gamma)
356
+ return Math.max(0, hybridScore);
357
+ }
358
+
359
+ // Internal method to clean up vectors based on recency score threshold
360
+ private async _cleanupByRecency(threshold: number): Promise<number> {
361
+ const rows = await this.all(`SELECT id, payload FROM vectors`);
362
+ let deletedCount = 0;
363
+ const idsToDelete: string[] = [];
364
+
365
+ for (const row of rows) {
366
+ try {
367
+ const payload: MemoryPayload = JSON.parse(row.payload);
368
+ const memoryType: MemoryType | undefined = payload.type;
369
+ const createdAt = payload.createdAt;
370
+
371
+ // Get halfLifeDays for the type or use default
372
+ const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
373
+ const { halfLifeDays } = typeConfig || this.scoringConfig.default;
374
+
375
+ // Calculate current recency score
376
+ const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
377
+
378
+ // Mark for deletion if score is below threshold
379
+ // Add extra check for halfLifeDays > 0 to avoid deleting permanent items instantly if threshold is 0
380
+ if (recencyScore < threshold && halfLifeDays > 0) {
381
+ // Optional: Add type-specific exceptions (e.g., never delete 'factual')
382
+ // if (payload.type !== 'factual') {
383
+ idsToDelete.push(row.id);
384
+ // }
385
+ }
386
+ } catch (error) {
387
+ console.error(`Error processing vector ${row.id} for cleanup:`, error);
388
+ // Optionally delete vectors with bad payload/date based on policy
389
+ }
390
+ }
391
+
392
+ if (idsToDelete.length > 0) {
393
+ const placeholders = idsToDelete.map(() => '?').join(',');
394
+ await this.run(`DELETE FROM vectors WHERE id IN (${placeholders})`, idsToDelete);
395
+ deletedCount = idsToDelete.length;
396
+ console.log(`Recency Cleanup: Removed ${deletedCount} vectors with score < ${threshold}.`);
397
+ }
398
+
399
+ return deletedCount;
400
+ }
401
+
402
+ }
@@ -0,0 +1,126 @@
1
+ // Linter errors exist because './base' or '../types' might be incorrect paths or lack expected exports.
2
+ // Ensure VectorStore is exported from './base.ts'
3
+ // Ensure MemoryVector, SearchResult, SearchFilters, VectorStoreConfig are exported from '../types.ts'
4
+ import { VectorStore } from './base';
5
+ import { SearchFilters, VectorStoreConfig, VectorStoreResult } from '../types';
6
+
7
+
8
+ export interface FakeVectorStore extends VectorStore {
9
+ lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]>;
10
+ dump(filePath?: string, userId?: string): Promise<void> ;
11
+ load(filePath?: string, userId?: string): Promise<void>;
12
+ }
13
+
14
+ interface MemoryVector {
15
+ id: string;
16
+ vector: number[];
17
+ payload: Record<string, any>;
18
+ }
19
+
20
+ // Removed fs and path imports as they are not needed for empty methods
21
+
22
+ export class LLMVectorStore implements FakeVectorStore {
23
+
24
+ constructor(config?: VectorStoreConfig) {
25
+ console.log('LLMVectorStore initialized', config);
26
+ // Initialization logic based on config, if any
27
+ }
28
+
29
+
30
+ async dump(filePath?: string, userId?: string): Promise<void> {
31
+ // TODO: Implement LLMVectorStore dump logic
32
+ console.log('LLMVectorStore dump called for:', userId || 'all users', 'to path:', filePath);
33
+ // Placeholder implementation
34
+ }
35
+
36
+ async load(filePath?: string, userId?: string): Promise<void> {
37
+ // TODO: Implement LLMVectorStore load logic
38
+ console.log('LLMVectorStore load called for:', userId || 'all users', 'from path:', filePath);
39
+ // Placeholder implementation
40
+ }
41
+
42
+
43
+ async add(record: MemoryVector, userId?: string): Promise<string> {
44
+ // TODO: Implement LLMVectorStore add logic
45
+ console.log('LLMVectorStore add called with:', record, userId);
46
+ return 'placeholder-id'; // Placeholder return
47
+ }
48
+
49
+ async search(query: number[], limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]> {
50
+ return [];
51
+ }
52
+
53
+
54
+ async lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]> {
55
+ // TODO: Implement LLMVectorStore search logic using query vector
56
+ // Note: userId parameter is removed as it's not in the base interface signature.
57
+ // If userId is needed, it might have to be passed via filters or managed differently.
58
+ console.log('LLMVectorStore search called with vector query:', query, 'limit:', limit, 'filters:', filters);
59
+ // Linter error on VectorStoreResult likely means its definition in ../types.ts needs checking/creation.
60
+ return []; // Placeholder return
61
+ }
62
+
63
+ async get(id: string, userId?: string): Promise<MemoryVector | null> {
64
+ // TODO: Implement LLMVectorStore get logic
65
+ console.log('LLMVectorStore get called with:', id, userId);
66
+ return null; // Placeholder return
67
+ }
68
+
69
+ async update(vectorId: string, vector: number[], payload: Record<string, any>): Promise<void> {
70
+ // TODO: Implement LLMVectorStore update logic matching the base interface
71
+ console.log('LLMVectorStore update called with:', vectorId, vector, payload);
72
+ // Placeholder implementation
73
+ }
74
+
75
+ async delete(id: string, userId?: string): Promise<void> {
76
+ // TODO: Implement LLMVectorStore delete logic
77
+ console.log('LLMVectorStore delete called with:', id, userId);
78
+ // Placeholder implementation
79
+ }
80
+
81
+ async reset(userId?: string): Promise<void> {
82
+ // TODO: Implement LLMVectorStore reset logic
83
+ console.log('LLMVectorStore reset called for:', userId || 'all users');
84
+ // Placeholder implementation
85
+ }
86
+
87
+ // TODO: Implement other missing methods from base VectorStore interface if needed
88
+
89
+ // Placeholder for missing method from VectorStore interface
90
+ async insert(vectors: number[][], ids: string[],payloads: Record<string, any>[],): Promise<void> {
91
+ // TODO: Implement LLMVectorStore insert logic
92
+ }
93
+
94
+ // Placeholder for missing method from VectorStore interface
95
+ async deleteCol(userId?: string): Promise<void> {
96
+ // TODO: Implement LLMVectorStore deleteCol logic
97
+ console.log('LLMVectorStore deleteCol called for:', userId || 'default collection');
98
+ }
99
+
100
+ // Placeholder for missing method from VectorStore interface
101
+ async list(filters?: SearchFilters,limit?: number): Promise<[VectorStoreResult[], number]>{
102
+ // TODO: Implement LLMVectorStore list logic
103
+ return [[],0];
104
+ }
105
+
106
+ // Placeholder for missing method from VectorStore interface
107
+ getInstanceId(): string {
108
+ // TODO: Implement LLMVectorStore getUserId logic (if applicable to this store)
109
+ console.log('LLMVectorStore getUserId called');
110
+ return 'placeholder-user-id'; // Or null if not managed per-user this way
111
+ }
112
+
113
+ // Placeholder for missing method from VectorStore interface
114
+ async setUserId(userId: string): Promise<void> {
115
+ // TODO: Implement LLMVectorStore setUserId logic (if applicable)
116
+ console.log('LLMVectorStore setUserId called with:', userId);
117
+ }
118
+
119
+ // Placeholder for missing method from VectorStore interface
120
+ async initialize(): Promise<void> {
121
+ // TODO: Implement LLMVectorStore initialize logic (if needed)
122
+ console.log('LLMVectorStore initialize called');
123
+ }
124
+
125
+ // Add any other methods required by the VectorStore interface or custom methods
126
+ }
@@ -0,0 +1,158 @@
1
+ import { describe, it, expect, beforeEach, jest } from '@jest/globals'; // Use Jest globals
2
+ import { LiteVectorStore } from '../src/vectorstores/lite';
3
+ import { VectorStoreConfig, MemoryPayload } from '../src/types';
4
+ import { DEFAULT_MEMORY_CONFIG } from '../src/config/defaults'; // To get default scoring
5
+
6
+ // Mock crypto using jest.mock
7
+ jest.mock('crypto', () => ({
8
+ createHash: jest.fn().mockReturnThis(),
9
+ update: jest.fn().mockReturnThis(),
10
+ digest: jest.fn().mockReturnValue('hashed-user-id'),
11
+ }));
12
+
13
+ // Mock sqlite3 using jest.mock
14
+ jest.mock('sqlite3', () => {
15
+ const mockDb = {
16
+ run: jest.fn((_sql: any, _params: any, callback: any) => callback(null)),
17
+ all: jest.fn((_sql: any, _params: any, callback: any) => callback(null, [])),
18
+ get: jest.fn((_sql: any, _params: any, callback: any) => callback(null, null)),
19
+ close: jest.fn((callback: any) => callback(null)),
20
+ };
21
+ return {
22
+ Database: jest.fn(() => mockDb),
23
+ verbose: jest.fn(() => ({ Database: jest.fn(() => mockDb) })), // Handle verbose()
24
+ };
25
+ });
26
+
27
+
28
+ describe('LiteVectorStore Private Methods', () => {
29
+ let store: LiteVectorStore;
30
+ const userId = 'test-user';
31
+ const mockConfig: VectorStoreConfig = {
32
+ ...DEFAULT_MEMORY_CONFIG.vectorStore.config, // Use defaults including scoring
33
+ rootPath: ':memory:', // Use in-memory for tests
34
+ dimension: 3, // Use smaller dimension for test vectors
35
+ };
36
+
37
+ beforeEach(() => {
38
+ // Reset mocks using Jest's API
39
+ jest.clearAllMocks();
40
+ // Create a new store instance before each test
41
+ store = new LiteVectorStore(mockConfig, userId);
42
+ // db interaction isn't needed for these calculation tests.
43
+ });
44
+
45
+ describe('calculateRecencyScore', () => {
46
+ const now = new Date();
47
+ const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000).toISOString();
48
+ const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000).toISOString();
49
+ const futureDate = new Date(now.getTime() + 1 * 24 * 60 * 60 * 1000).toISOString();
50
+
51
+ it('should return 1.0 for infinite half-life', () => {
52
+ const score = (store as any).calculateRecencyScore(sevenDaysAgo, Infinity);
53
+ expect(score).toBe(1.0);
54
+ });
55
+
56
+ it('should return 1.0 for missing createdAt date', () => {
57
+ const score = (store as any).calculateRecencyScore(undefined, 7);
58
+ expect(score).toBe(1.0);
59
+ });
60
+
61
+ it('should return 0.0 for zero half-life', () => {
62
+ const score = (store as any).calculateRecencyScore(sevenDaysAgo, 0);
63
+ expect(score).toBe(0.0);
64
+ });
65
+
66
+ it('should return ~0.5 for age equal to half-life (e.g., 7 days ago, HL=7)', () => {
67
+ const score = (store as any).calculateRecencyScore(sevenDaysAgo, 7);
68
+ expect(score).toBeCloseTo(0.5, 5); // Use toBeCloseTo for float comparisons
69
+ });
70
+
71
+ it('should return ~0.25 for age double the half-life (e.g., 4 days ago, HL=2)', () => {
72
+ const fourDaysAgo = new Date(now.getTime() - 4 * 24 * 60 * 60 * 1000).toISOString();
73
+ const score = (store as any).calculateRecencyScore(fourDaysAgo, 2);
74
+ expect(score).toBeCloseTo(0.25, 5);
75
+ });
76
+
77
+ it('should return 0.5 for invalid date format', () => {
78
+ const score = (store as any).calculateRecencyScore('invalid-date', 7);
79
+ expect(score).toBe(0.5);
80
+ });
81
+
82
+ it('should return 1.0 for future dates', () => {
83
+ const score = (store as any).calculateRecencyScore(futureDate, 7);
84
+ expect(score).toBe(1.0);
85
+ });
86
+
87
+ it('should handle negative ageInDays gracefully (treat as future)', () => {
88
+ // Mock Date.now() to simulate the scenario if needed, or test with future date
89
+ const score = (store as any).calculateRecencyScore(futureDate, 7);
90
+ expect(score).toBe(1.0);
91
+ });
92
+ });
93
+
94
+ describe('calculateHybridScore', () => {
95
+ const now = new Date();
96
+ const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000).toISOString();
97
+ const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000).toISOString();
98
+ const veryOldDate = new Date(now.getTime() - 400 * 24 * 60 * 60 * 1000).toISOString();
99
+ const scoring = DEFAULT_MEMORY_CONFIG.vectorStore.config.scoring!;
100
+
101
+ it('should prioritize cosine similarity for factual memory (high alpha)', () => {
102
+ const payload: MemoryPayload = { memoryId: 'mem-f1', userId: userId, type: 'factual', createdAt: twoDaysAgo };
103
+ const cosineScore = 0.9;
104
+ const expectedRecency = (store as any).calculateRecencyScore(twoDaysAgo, scoring.factual.halfLifeDays);
105
+ const expectedScore = scoring.factual.alpha * cosineScore + scoring.factual.beta * expectedRecency + scoring.factual.gamma;
106
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
107
+ expect(hybridScore).toBeCloseTo(expectedScore, 5);
108
+ expect(scoring.factual.alpha * cosineScore).toBeGreaterThan(scoring.factual.beta * expectedRecency);
109
+ });
110
+
111
+ it('should prioritize recency for episodic memory (high beta, short half-life)', () => {
112
+ const payload: MemoryPayload = { memoryId: 'mem-e1', userId: userId, type: 'episodic', createdAt: twoDaysAgo };
113
+ const cosineScore = 0.5;
114
+ const expectedRecency = (store as any).calculateRecencyScore(twoDaysAgo, scoring.episodic.halfLifeDays);
115
+ const expectedScore = scoring.episodic.alpha * cosineScore + scoring.episodic.beta * expectedRecency + scoring.episodic.gamma;
116
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
117
+ expect(hybridScore).toBeCloseTo(expectedScore, 5);
118
+ expect(scoring.episodic.beta * expectedRecency).toBeGreaterThan(0.2);
119
+ });
120
+
121
+ it('should have low score for old episodic memory', () => {
122
+ const payload: MemoryPayload = { memoryId: 'mem-e2', userId: userId, type: 'episodic', createdAt: veryOldDate };
123
+ const cosineScore = 0.9;
124
+ const expectedRecency = (store as any).calculateRecencyScore(veryOldDate, scoring.episodic.halfLifeDays);
125
+ const expectedScore = scoring.episodic.alpha * cosineScore + scoring.episodic.beta * expectedRecency + scoring.episodic.gamma;
126
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
127
+ expect(expectedRecency).toBeLessThan(0.01);
128
+ expect(hybridScore).toBeCloseTo(scoring.episodic.alpha * cosineScore + scoring.episodic.gamma, 5);
129
+ });
130
+
131
+ it('should handle assistant_preference with no decay (Infinity half-life)', () => {
132
+ const payload: MemoryPayload = { memoryId: 'mem-a1', userId: userId, type: 'assistant_preference', createdAt: veryOldDate };
133
+ const cosineScore = 0.7;
134
+ const expectedRecency = 1.0;
135
+ const expectedScore = scoring.assistant_preference.alpha * cosineScore + scoring.assistant_preference.beta * expectedRecency + scoring.assistant_preference.gamma;
136
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
137
+ expect(hybridScore).toBeCloseTo(expectedScore, 5);
138
+ });
139
+
140
+ it('should use default scoring if type is missing', () => {
141
+ const payload: MemoryPayload = { memoryId: 'mem-d1', userId: userId, createdAt: sevenDaysAgo }; // No type specified
142
+ const cosineScore = 0.8;
143
+ const expectedRecency = (store as any).calculateRecencyScore(sevenDaysAgo, scoring.default.halfLifeDays);
144
+ const expectedScore = scoring.default.alpha * cosineScore + scoring.default.beta * expectedRecency + scoring.default.gamma;
145
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
146
+ expect(hybridScore).toBeCloseTo(expectedScore, 5);
147
+ });
148
+
149
+ it('should return score >= 0 even with negative cosine similarity', () => {
150
+ const payload: MemoryPayload = { memoryId: 'mem-s1', userId: userId, type: 'semantic', createdAt: twoDaysAgo };
151
+ const cosineScore = -0.5;
152
+ const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
153
+ expect(hybridScore).toBeGreaterThanOrEqual(0);
154
+ });
155
+
156
+ });
157
+
158
+ });