memories-lite 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MEMORIES.md +39 -0
- package/README.md +221 -0
- package/TECHNICAL.md +135 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.js +61 -0
- package/dist/config/manager.d.ts +4 -0
- package/dist/config/manager.js +121 -0
- package/dist/embeddings/base.d.ts +4 -0
- package/dist/embeddings/base.js +2 -0
- package/dist/embeddings/google.d.ts +10 -0
- package/dist/embeddings/google.js +28 -0
- package/dist/embeddings/openai.d.ts +10 -0
- package/dist/embeddings/openai.js +31 -0
- package/dist/graphs/configs.d.ts +14 -0
- package/dist/graphs/configs.js +19 -0
- package/dist/graphs/tools.d.ts +271 -0
- package/dist/graphs/tools.js +220 -0
- package/dist/graphs/utils.d.ts +9 -0
- package/dist/graphs/utils.js +105 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +30 -0
- package/dist/llms/base.d.ts +16 -0
- package/dist/llms/base.js +2 -0
- package/dist/llms/google.d.ts +11 -0
- package/dist/llms/google.js +44 -0
- package/dist/llms/openai.d.ts +9 -0
- package/dist/llms/openai.js +73 -0
- package/dist/llms/openai_structured.d.ts +11 -0
- package/dist/llms/openai_structured.js +72 -0
- package/dist/memory/index.d.ts +42 -0
- package/dist/memory/index.js +499 -0
- package/dist/memory/memory.types.d.ts +23 -0
- package/dist/memory/memory.types.js +2 -0
- package/dist/prompts/index.d.ts +102 -0
- package/dist/prompts/index.js +233 -0
- package/dist/storage/DummyHistoryManager.d.ts +7 -0
- package/dist/storage/DummyHistoryManager.js +19 -0
- package/dist/storage/MemoryHistoryManager.d.ts +8 -0
- package/dist/storage/MemoryHistoryManager.js +36 -0
- package/dist/storage/base.d.ts +6 -0
- package/dist/storage/base.js +2 -0
- package/dist/storage/index.d.ts +3 -0
- package/dist/storage/index.js +19 -0
- package/dist/types/index.d.ts +1071 -0
- package/dist/types/index.js +100 -0
- package/dist/utils/bm25.d.ts +13 -0
- package/dist/utils/bm25.js +51 -0
- package/dist/utils/factory.d.ts +13 -0
- package/dist/utils/factory.js +49 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.js +9 -0
- package/dist/utils/memory.d.ts +3 -0
- package/dist/utils/memory.js +44 -0
- package/dist/utils/telemetry.d.ts +11 -0
- package/dist/utils/telemetry.js +74 -0
- package/dist/utils/telemetry.types.d.ts +27 -0
- package/dist/utils/telemetry.types.js +2 -0
- package/dist/vectorstores/base.d.ts +11 -0
- package/dist/vectorstores/base.js +2 -0
- package/dist/vectorstores/lite.d.ts +40 -0
- package/dist/vectorstores/lite.js +319 -0
- package/dist/vectorstores/llm.d.ts +31 -0
- package/dist/vectorstores/llm.js +88 -0
- package/jest.config.js +22 -0
- package/memories-lite.db +0 -0
- package/package.json +38 -0
- package/src/config/defaults.ts +61 -0
- package/src/config/manager.ts +132 -0
- package/src/embeddings/base.ts +4 -0
- package/src/embeddings/google.ts +32 -0
- package/src/embeddings/openai.ts +33 -0
- package/src/graphs/configs.ts +30 -0
- package/src/graphs/tools.ts +267 -0
- package/src/graphs/utils.ts +114 -0
- package/src/index.ts +14 -0
- package/src/llms/base.ts +20 -0
- package/src/llms/google.ts +56 -0
- package/src/llms/openai.ts +85 -0
- package/src/llms/openai_structured.ts +82 -0
- package/src/memory/index.ts +723 -0
- package/src/memory/memory.types.ts +27 -0
- package/src/prompts/index.ts +268 -0
- package/src/storage/DummyHistoryManager.ts +27 -0
- package/src/storage/MemoryHistoryManager.ts +58 -0
- package/src/storage/base.ts +14 -0
- package/src/storage/index.ts +3 -0
- package/src/types/index.ts +243 -0
- package/src/utils/bm25.ts +64 -0
- package/src/utils/factory.ts +59 -0
- package/src/utils/logger.ts +13 -0
- package/src/utils/memory.ts +48 -0
- package/src/utils/telemetry.ts +98 -0
- package/src/utils/telemetry.types.ts +34 -0
- package/src/vectorstores/base.ts +27 -0
- package/src/vectorstores/lite.ts +402 -0
- package/src/vectorstores/llm.ts +126 -0
- package/tests/lite.spec.ts +158 -0
- package/tests/memory.facts.test.ts +211 -0
- package/tests/memory.test.ts +406 -0
- package/tsconfig.json +16 -0
- package/tsconfig.tsbuildinfo +1 -0
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
// vector persistence
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import sqlite3 from 'sqlite3';
|
|
4
|
+
|
|
5
|
+
import { VectorStore } from "./base";
|
|
6
|
+
import { SearchFilters, VectorStoreConfig, VectorStoreResult, MemoryPayload, MemoryScoringConfig, MemoryType } from "../types";
|
|
7
|
+
import { createHash } from 'crypto';
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
// Define interface for database rows
|
|
11
|
+
interface MemoryVector {
|
|
12
|
+
id: string;
|
|
13
|
+
vector: number[];
|
|
14
|
+
payload: Record<string, any>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* LiteVectorStore provides a simple vector storage implementation.
|
|
18
|
+
*
|
|
19
|
+
* This class manages user-sensitive vector data, with each instance tied to a specific user.
|
|
20
|
+
* Data is stored in a local SQLite database with appropriate isolation between users.
|
|
21
|
+
* The user's identity is hashed for privacy and security purposes.
|
|
22
|
+
* ⚠️ User memories are typically small (usually <1000 vectors), therefore database optimization is not a priority.
|
|
23
|
+
*
|
|
24
|
+
* @implements {VectorStore}
|
|
25
|
+
*/
|
|
26
|
+
export class LiteVectorStore implements VectorStore {
|
|
27
|
+
private db: sqlite3.Database;
|
|
28
|
+
private isSecure: boolean;
|
|
29
|
+
private dimension: number;
|
|
30
|
+
private currentUserId: string;
|
|
31
|
+
private scoringConfig: MemoryScoringConfig;
|
|
32
|
+
private cleanupThreshold?: number; // Store the threshold
|
|
33
|
+
private static cache: Map<string, LiteVectorStore>;
|
|
34
|
+
constructor(config: VectorStoreConfig, currentUserId: string) {
|
|
35
|
+
if (!config.scoring) {
|
|
36
|
+
throw new Error("Scoring configuration is missing in VectorStoreConfig");
|
|
37
|
+
}
|
|
38
|
+
this.dimension = config.dimension || 1536;
|
|
39
|
+
this.currentUserId = currentUserId;
|
|
40
|
+
this.isSecure = config.secure || false;
|
|
41
|
+
this.scoringConfig = config.scoring;
|
|
42
|
+
this.cleanupThreshold = config.recencyCleanupThreshold; // Store threshold
|
|
43
|
+
config.rootPath = config.rootPath || process.cwd();
|
|
44
|
+
const filename = this.isSecure ? `memories-lite-${currentUserId}.db` : `memories-lite-global.db`;
|
|
45
|
+
const dbPath = (config.rootPath == ':memory:') ? ':memory:' : path.join(config.rootPath, filename);
|
|
46
|
+
|
|
47
|
+
// Add error handling callback for the database connection
|
|
48
|
+
this.db = new sqlite3.Database(dbPath);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
private async init() {
|
|
53
|
+
await this.run(`
|
|
54
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
55
|
+
id TEXT PRIMARY KEY,
|
|
56
|
+
vector BLOB NOT NULL,
|
|
57
|
+
payload TEXT NOT NULL
|
|
58
|
+
)
|
|
59
|
+
`);
|
|
60
|
+
|
|
61
|
+
await this.run(`
|
|
62
|
+
CREATE TABLE IF NOT EXISTS memory_migrations (
|
|
63
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
64
|
+
user_id TEXT NOT NULL UNIQUE
|
|
65
|
+
)
|
|
66
|
+
`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private async run(sql: string, params: any[] = []): Promise<void> {
|
|
70
|
+
return new Promise((resolve, reject) => {
|
|
71
|
+
this.db.run(sql, params, (err) => {
|
|
72
|
+
if (err) reject(err);
|
|
73
|
+
else resolve();
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private async all(sql: string, params: any[] = []): Promise<any[]> {
|
|
79
|
+
return new Promise((resolve, reject) => {
|
|
80
|
+
this.db.all(sql, params, (err, rows) => {
|
|
81
|
+
if (err) reject(err);
|
|
82
|
+
else resolve(rows);
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private async getOne(sql: string, params: any[] = []): Promise<any> {
|
|
88
|
+
return new Promise((resolve, reject) => {
|
|
89
|
+
this.db.get(sql, params, (err, row) => {
|
|
90
|
+
if (err) reject(err);
|
|
91
|
+
else resolve(row);
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
//
|
|
98
|
+
// TODO: Memory Decay & Hybrid Scoring (α·cosSim + β·when + γ·1 [+ δ·tag])
|
|
99
|
+
// scoreTotal = α·cos + β·recency + γ·1 + δ·tag
|
|
100
|
+
// https://chatgpt.com/g/g-p-677932764d20819181f8805995ddab8b-the-coder/c/68121341-7fdc-800b-8734-af49719eb5e1
|
|
101
|
+
private cosineSimilarity(a: number[], b: number[]): number {
|
|
102
|
+
if (a.length !== b.length) {
|
|
103
|
+
throw new Error("Vectors must have the same dimension for cosine similarity.");
|
|
104
|
+
}
|
|
105
|
+
if (a.length === 0) {
|
|
106
|
+
return 0;
|
|
107
|
+
}
|
|
108
|
+
let dotProduct = 0;
|
|
109
|
+
let normA = 0;
|
|
110
|
+
let normB = 0;
|
|
111
|
+
for (let i = 0; i < a.length; i++) {
|
|
112
|
+
dotProduct += a[i] * b[i];
|
|
113
|
+
normA += a[i] * a[i];
|
|
114
|
+
normB += b[i] * b[i];
|
|
115
|
+
}
|
|
116
|
+
const magnitudeA = Math.sqrt(normA);
|
|
117
|
+
const magnitudeB = Math.sqrt(normB);
|
|
118
|
+
if (magnitudeA === 0 || magnitudeB === 0) {
|
|
119
|
+
return 0;
|
|
120
|
+
}
|
|
121
|
+
return dotProduct / (magnitudeA * magnitudeB);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
private filterVector(vector: MemoryVector, filters?: SearchFilters): boolean {
|
|
125
|
+
if (!filters) return true;
|
|
126
|
+
return Object.entries(filters).every(
|
|
127
|
+
([key, value]) => vector.payload && vector.payload[key] === value,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// by default, the userId is the SHA256 derivation
|
|
132
|
+
getInstanceId(): string {
|
|
133
|
+
return this.currentUserId;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
static async from(userId: string, config: VectorStoreConfig): Promise<LiteVectorStore> {
|
|
137
|
+
if (!userId) {
|
|
138
|
+
throw new Error("userId is required");
|
|
139
|
+
}
|
|
140
|
+
if (!config.scoring) {
|
|
141
|
+
throw new Error("Scoring configuration is missing in VectorStoreConfig");
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const hashedUserId = createHash('sha256').update(userId).digest('hex');
|
|
145
|
+
|
|
146
|
+
if (!LiteVectorStore.cache) {
|
|
147
|
+
LiteVectorStore.cache = new Map<string, LiteVectorStore>();
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const cachedStore = LiteVectorStore.cache.get(hashedUserId);
|
|
151
|
+
if (cachedStore) {
|
|
152
|
+
Object.setPrototypeOf(cachedStore, LiteVectorStore.prototype);
|
|
153
|
+
cachedStore.currentUserId = hashedUserId;
|
|
154
|
+
// Ensure scoring config and threshold are updated if config object changed
|
|
155
|
+
cachedStore.scoringConfig = config.scoring;
|
|
156
|
+
cachedStore.cleanupThreshold = config.recencyCleanupThreshold;
|
|
157
|
+
return cachedStore;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Pass the full config (including scoring) to the constructor
|
|
161
|
+
const newStore = new LiteVectorStore(config, hashedUserId);
|
|
162
|
+
|
|
163
|
+
await newStore.init();
|
|
164
|
+
LiteVectorStore.cache.set(hashedUserId, newStore);
|
|
165
|
+
return newStore;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async insert(
|
|
169
|
+
vectors: number[][],
|
|
170
|
+
ids: string[],
|
|
171
|
+
payloads: Record<string, any>[],
|
|
172
|
+
): Promise<void> {
|
|
173
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
174
|
+
if (vectors[i].length !== this.dimension) {
|
|
175
|
+
throw new Error(
|
|
176
|
+
`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`,
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
//
|
|
180
|
+
// remove the userId from the payload as sensitive data
|
|
181
|
+
this.isSecure && delete payloads[i].userId;
|
|
182
|
+
const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
|
|
183
|
+
await this.run(
|
|
184
|
+
`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`,
|
|
185
|
+
[ids[i], vectorBuffer, JSON.stringify(payloads[i])],
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async search(
|
|
192
|
+
query: number[],
|
|
193
|
+
limit: number = 10,
|
|
194
|
+
filters?: SearchFilters,
|
|
195
|
+
): Promise<VectorStoreResult[]> {
|
|
196
|
+
if (query.length !== this.dimension) {
|
|
197
|
+
throw new Error(
|
|
198
|
+
`Query dimension mismatch. Expected ${this.dimension}, got ${query.length}`,
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const results: VectorStoreResult[] = [];
|
|
203
|
+
const rows = await this.all(`SELECT * FROM vectors`);
|
|
204
|
+
|
|
205
|
+
for (const row of rows) {
|
|
206
|
+
const vector = Array.from(new Float32Array(row.vector.buffer)); // Convert buffer to number array
|
|
207
|
+
const payload: MemoryPayload = JSON.parse(row.payload);
|
|
208
|
+
const memoryVector: MemoryVector = {
|
|
209
|
+
id: row.id,
|
|
210
|
+
vector: vector,
|
|
211
|
+
payload,
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
if (this.filterVector(memoryVector, filters)) {
|
|
215
|
+
const cosineScore = this.cosineSimilarity(query, vector);
|
|
216
|
+
const hybridScore = this.calculateHybridScore(cosineScore, payload);
|
|
217
|
+
|
|
218
|
+
results.push({
|
|
219
|
+
id: memoryVector.id,
|
|
220
|
+
payload: memoryVector.payload,
|
|
221
|
+
score: hybridScore,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
|
227
|
+
const finalResults = results.slice(0, limit);
|
|
228
|
+
|
|
229
|
+
// Trigger cleanup after search, if threshold is set
|
|
230
|
+
if (this.cleanupThreshold !== undefined && this.cleanupThreshold >= 0) {
|
|
231
|
+
// Run cleanup asynchronously in the background (fire and forget)
|
|
232
|
+
// to avoid delaying the search response.
|
|
233
|
+
// Consider adding mutex or debouncing in high-frequency scenarios.
|
|
234
|
+
this._cleanupByRecency(this.cleanupThreshold).catch(err => {
|
|
235
|
+
console.error("Error during background recency cleanup:", err);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return finalResults;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async get(vectorId: string): Promise<VectorStoreResult | null> {
|
|
243
|
+
const row = await this.getOne(`SELECT * FROM vectors WHERE id = ?`, [
|
|
244
|
+
vectorId,
|
|
245
|
+
]);
|
|
246
|
+
if (!row) return null;
|
|
247
|
+
|
|
248
|
+
const payload = JSON.parse(row.payload);
|
|
249
|
+
return {
|
|
250
|
+
id: row.id,
|
|
251
|
+
payload,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async update(
|
|
256
|
+
vectorId: string,
|
|
257
|
+
vector: number[],
|
|
258
|
+
payload: Record<string, any>,
|
|
259
|
+
): Promise<void> {
|
|
260
|
+
if (vector.length !== this.dimension) {
|
|
261
|
+
throw new Error(
|
|
262
|
+
`Vector dimension mismatch. Expected ${this.dimension}, got ${vector.length}`,
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
//
|
|
266
|
+
// remove the userId from the payload as sensitive data
|
|
267
|
+
this.isSecure && delete payload.userId;
|
|
268
|
+
const vectorBuffer = Buffer.from(new Float32Array(vector).buffer);
|
|
269
|
+
await this.run(`UPDATE vectors SET vector = ?, payload = ? WHERE id = ?`, [
|
|
270
|
+
vectorBuffer,
|
|
271
|
+
JSON.stringify(payload),
|
|
272
|
+
vectorId,
|
|
273
|
+
]);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async delete(vectorId: string): Promise<void> {
|
|
277
|
+
await this.run(`DELETE FROM vectors WHERE id = ?`, [vectorId]);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async deleteCol(): Promise<void> {
|
|
281
|
+
await this.run(`DROP TABLE IF EXISTS vectors`);
|
|
282
|
+
await this.init();
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
async list(
|
|
287
|
+
filters?: SearchFilters,
|
|
288
|
+
limit: number = 100,
|
|
289
|
+
): Promise<[VectorStoreResult[], number]> {
|
|
290
|
+
const rows = await this.all(`SELECT * FROM vectors`);
|
|
291
|
+
const results: VectorStoreResult[] = [];
|
|
292
|
+
|
|
293
|
+
for (const row of rows) {
|
|
294
|
+
const memoryVector: MemoryVector = {
|
|
295
|
+
id: row.id,
|
|
296
|
+
vector: Array.from(new Float32Array(row.vector.buffer)),
|
|
297
|
+
payload: {},
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
if (this.filterVector(memoryVector, filters)) {
|
|
301
|
+
// load payload at the end
|
|
302
|
+
results.push({
|
|
303
|
+
id: memoryVector.id,
|
|
304
|
+
payload: JSON.parse(row.payload),
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
return [results.slice(0, limit), results.length];
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Calculate recency score using exponential decay
|
|
313
|
+
private calculateRecencyScore(createdAt: string, halfLifeDays: number): number {
|
|
314
|
+
if (halfLifeDays === Infinity || !createdAt) {
|
|
315
|
+
return 1.0; // No decay or no date
|
|
316
|
+
}
|
|
317
|
+
if (halfLifeDays <= 0) {
|
|
318
|
+
return 0.0; // Instant decay
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const createdAtDate = new Date(createdAt);
|
|
322
|
+
if (isNaN(createdAtDate.getTime())) {
|
|
323
|
+
console.warn('Invalid createdAt date format:', createdAt);
|
|
324
|
+
return 0.5; // Neutral score if date is invalid
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const now = new Date();
|
|
328
|
+
const ageInMilliseconds = now.getTime() - createdAtDate.getTime();
|
|
329
|
+
const ageInDays = ageInMilliseconds / (1000 * 60 * 60 * 24);
|
|
330
|
+
|
|
331
|
+
if (ageInDays < 0) {
|
|
332
|
+
console.warn('createdAt date is in the future:', createdAt);
|
|
333
|
+
return 1.0; // Treat future dates as brand new
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const lambda = Math.log(2) / halfLifeDays;
|
|
337
|
+
return Math.exp(-lambda * ageInDays);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Hybrid Scoring: Combines Cosine Similarity, Recency, and Base Importance
|
|
341
|
+
private calculateHybridScore(cosineScore: number, payload: MemoryPayload): number {
|
|
342
|
+
const memoryType: MemoryType | undefined = payload.type;
|
|
343
|
+
const createdAt = payload.createdAt; // Expecting ISO string
|
|
344
|
+
|
|
345
|
+
// Get scoring parameters for the memory type or use default
|
|
346
|
+
const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
|
|
347
|
+
const { alpha, beta, gamma, halfLifeDays } = typeConfig || this.scoringConfig.default;
|
|
348
|
+
|
|
349
|
+
const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
|
|
350
|
+
|
|
351
|
+
// Calculate final score
|
|
352
|
+
// scoreTotal = α * cosSim + β * recency + γ * baseImportance (using 1 as base)
|
|
353
|
+
const hybridScore = alpha * cosineScore + beta * recencyScore + gamma;
|
|
354
|
+
|
|
355
|
+
// Ensure score is within a reasonable range (e.g., 0 to alpha+beta+gamma)
|
|
356
|
+
return Math.max(0, hybridScore);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// Internal method to clean up vectors based on recency score threshold
|
|
360
|
+
private async _cleanupByRecency(threshold: number): Promise<number> {
|
|
361
|
+
const rows = await this.all(`SELECT id, payload FROM vectors`);
|
|
362
|
+
let deletedCount = 0;
|
|
363
|
+
const idsToDelete: string[] = [];
|
|
364
|
+
|
|
365
|
+
for (const row of rows) {
|
|
366
|
+
try {
|
|
367
|
+
const payload: MemoryPayload = JSON.parse(row.payload);
|
|
368
|
+
const memoryType: MemoryType | undefined = payload.type;
|
|
369
|
+
const createdAt = payload.createdAt;
|
|
370
|
+
|
|
371
|
+
// Get halfLifeDays for the type or use default
|
|
372
|
+
const typeConfig = memoryType ? this.scoringConfig[memoryType] : this.scoringConfig.default;
|
|
373
|
+
const { halfLifeDays } = typeConfig || this.scoringConfig.default;
|
|
374
|
+
|
|
375
|
+
// Calculate current recency score
|
|
376
|
+
const recencyScore = this.calculateRecencyScore(createdAt, halfLifeDays);
|
|
377
|
+
|
|
378
|
+
// Mark for deletion if score is below threshold
|
|
379
|
+
// Add extra check for halfLifeDays > 0 to avoid deleting permanent items instantly if threshold is 0
|
|
380
|
+
if (recencyScore < threshold && halfLifeDays > 0) {
|
|
381
|
+
// Optional: Add type-specific exceptions (e.g., never delete 'factual')
|
|
382
|
+
// if (payload.type !== 'factual') {
|
|
383
|
+
idsToDelete.push(row.id);
|
|
384
|
+
// }
|
|
385
|
+
}
|
|
386
|
+
} catch (error) {
|
|
387
|
+
console.error(`Error processing vector ${row.id} for cleanup:`, error);
|
|
388
|
+
// Optionally delete vectors with bad payload/date based on policy
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
if (idsToDelete.length > 0) {
|
|
393
|
+
const placeholders = idsToDelete.map(() => '?').join(',');
|
|
394
|
+
await this.run(`DELETE FROM vectors WHERE id IN (${placeholders})`, idsToDelete);
|
|
395
|
+
deletedCount = idsToDelete.length;
|
|
396
|
+
console.log(`Recency Cleanup: Removed ${deletedCount} vectors with score < ${threshold}.`);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return deletedCount;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// Linter errors exist because './base' or '../types' might be incorrect paths or lack expected exports.
|
|
2
|
+
// Ensure VectorStore is exported from './base.ts'
|
|
3
|
+
// Ensure MemoryVector, SearchResult, SearchFilters, VectorStoreConfig are exported from '../types.ts'
|
|
4
|
+
import { VectorStore } from './base';
|
|
5
|
+
import { SearchFilters, VectorStoreConfig, VectorStoreResult } from '../types';
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
export interface FakeVectorStore extends VectorStore {
|
|
9
|
+
lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]>;
|
|
10
|
+
dump(filePath?: string, userId?: string): Promise<void> ;
|
|
11
|
+
load(filePath?: string, userId?: string): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
interface MemoryVector {
|
|
15
|
+
id: string;
|
|
16
|
+
vector: number[];
|
|
17
|
+
payload: Record<string, any>;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Removed fs and path imports as they are not needed for empty methods
|
|
21
|
+
|
|
22
|
+
export class LLMVectorStore implements FakeVectorStore {
|
|
23
|
+
|
|
24
|
+
constructor(config?: VectorStoreConfig) {
|
|
25
|
+
console.log('LLMVectorStore initialized', config);
|
|
26
|
+
// Initialization logic based on config, if any
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
async dump(filePath?: string, userId?: string): Promise<void> {
|
|
31
|
+
// TODO: Implement LLMVectorStore dump logic
|
|
32
|
+
console.log('LLMVectorStore dump called for:', userId || 'all users', 'to path:', filePath);
|
|
33
|
+
// Placeholder implementation
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async load(filePath?: string, userId?: string): Promise<void> {
|
|
37
|
+
// TODO: Implement LLMVectorStore load logic
|
|
38
|
+
console.log('LLMVectorStore load called for:', userId || 'all users', 'from path:', filePath);
|
|
39
|
+
// Placeholder implementation
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async add(record: MemoryVector, userId?: string): Promise<string> {
|
|
44
|
+
// TODO: Implement LLMVectorStore add logic
|
|
45
|
+
console.log('LLMVectorStore add called with:', record, userId);
|
|
46
|
+
return 'placeholder-id'; // Placeholder return
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async search(query: number[], limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]> {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
async lookup(query: string, limit?: number, filters?: SearchFilters): Promise<VectorStoreResult[]> {
|
|
55
|
+
// TODO: Implement LLMVectorStore search logic using query vector
|
|
56
|
+
// Note: userId parameter is removed as it's not in the base interface signature.
|
|
57
|
+
// If userId is needed, it might have to be passed via filters or managed differently.
|
|
58
|
+
console.log('LLMVectorStore search called with vector query:', query, 'limit:', limit, 'filters:', filters);
|
|
59
|
+
// Linter error on VectorStoreResult likely means its definition in ../types.ts needs checking/creation.
|
|
60
|
+
return []; // Placeholder return
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async get(id: string, userId?: string): Promise<MemoryVector | null> {
|
|
64
|
+
// TODO: Implement LLMVectorStore get logic
|
|
65
|
+
console.log('LLMVectorStore get called with:', id, userId);
|
|
66
|
+
return null; // Placeholder return
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async update(vectorId: string, vector: number[], payload: Record<string, any>): Promise<void> {
|
|
70
|
+
// TODO: Implement LLMVectorStore update logic matching the base interface
|
|
71
|
+
console.log('LLMVectorStore update called with:', vectorId, vector, payload);
|
|
72
|
+
// Placeholder implementation
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async delete(id: string, userId?: string): Promise<void> {
|
|
76
|
+
// TODO: Implement LLMVectorStore delete logic
|
|
77
|
+
console.log('LLMVectorStore delete called with:', id, userId);
|
|
78
|
+
// Placeholder implementation
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async reset(userId?: string): Promise<void> {
|
|
82
|
+
// TODO: Implement LLMVectorStore reset logic
|
|
83
|
+
console.log('LLMVectorStore reset called for:', userId || 'all users');
|
|
84
|
+
// Placeholder implementation
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// TODO: Implement other missing methods from base VectorStore interface if needed
|
|
88
|
+
|
|
89
|
+
// Placeholder for missing method from VectorStore interface
|
|
90
|
+
async insert(vectors: number[][], ids: string[],payloads: Record<string, any>[],): Promise<void> {
|
|
91
|
+
// TODO: Implement LLMVectorStore insert logic
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Placeholder for missing method from VectorStore interface
|
|
95
|
+
async deleteCol(userId?: string): Promise<void> {
|
|
96
|
+
// TODO: Implement LLMVectorStore deleteCol logic
|
|
97
|
+
console.log('LLMVectorStore deleteCol called for:', userId || 'default collection');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Placeholder for missing method from VectorStore interface
|
|
101
|
+
async list(filters?: SearchFilters,limit?: number): Promise<[VectorStoreResult[], number]>{
|
|
102
|
+
// TODO: Implement LLMVectorStore list logic
|
|
103
|
+
return [[],0];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Placeholder for missing method from VectorStore interface
|
|
107
|
+
getInstanceId(): string {
|
|
108
|
+
// TODO: Implement LLMVectorStore getUserId logic (if applicable to this store)
|
|
109
|
+
console.log('LLMVectorStore getUserId called');
|
|
110
|
+
return 'placeholder-user-id'; // Or null if not managed per-user this way
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Placeholder for missing method from VectorStore interface
|
|
114
|
+
async setUserId(userId: string): Promise<void> {
|
|
115
|
+
// TODO: Implement LLMVectorStore setUserId logic (if applicable)
|
|
116
|
+
console.log('LLMVectorStore setUserId called with:', userId);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Placeholder for missing method from VectorStore interface
|
|
120
|
+
async initialize(): Promise<void> {
|
|
121
|
+
// TODO: Implement LLMVectorStore initialize logic (if needed)
|
|
122
|
+
console.log('LLMVectorStore initialize called');
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Add any other methods required by the VectorStore interface or custom methods
|
|
126
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, jest } from '@jest/globals'; // Use Jest globals
|
|
2
|
+
import { LiteVectorStore } from '../src/vectorstores/lite';
|
|
3
|
+
import { VectorStoreConfig, MemoryPayload } from '../src/types';
|
|
4
|
+
import { DEFAULT_MEMORY_CONFIG } from '../src/config/defaults'; // To get default scoring
|
|
5
|
+
|
|
6
|
+
// Mock crypto using jest.mock
|
|
7
|
+
jest.mock('crypto', () => ({
|
|
8
|
+
createHash: jest.fn().mockReturnThis(),
|
|
9
|
+
update: jest.fn().mockReturnThis(),
|
|
10
|
+
digest: jest.fn().mockReturnValue('hashed-user-id'),
|
|
11
|
+
}));
|
|
12
|
+
|
|
13
|
+
// Mock sqlite3 using jest.mock
|
|
14
|
+
jest.mock('sqlite3', () => {
|
|
15
|
+
const mockDb = {
|
|
16
|
+
run: jest.fn((_sql: any, _params: any, callback: any) => callback(null)),
|
|
17
|
+
all: jest.fn((_sql: any, _params: any, callback: any) => callback(null, [])),
|
|
18
|
+
get: jest.fn((_sql: any, _params: any, callback: any) => callback(null, null)),
|
|
19
|
+
close: jest.fn((callback: any) => callback(null)),
|
|
20
|
+
};
|
|
21
|
+
return {
|
|
22
|
+
Database: jest.fn(() => mockDb),
|
|
23
|
+
verbose: jest.fn(() => ({ Database: jest.fn(() => mockDb) })), // Handle verbose()
|
|
24
|
+
};
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
describe('LiteVectorStore Private Methods', () => {
|
|
29
|
+
let store: LiteVectorStore;
|
|
30
|
+
const userId = 'test-user';
|
|
31
|
+
const mockConfig: VectorStoreConfig = {
|
|
32
|
+
...DEFAULT_MEMORY_CONFIG.vectorStore.config, // Use defaults including scoring
|
|
33
|
+
rootPath: ':memory:', // Use in-memory for tests
|
|
34
|
+
dimension: 3, // Use smaller dimension for test vectors
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
// Reset mocks using Jest's API
|
|
39
|
+
jest.clearAllMocks();
|
|
40
|
+
// Create a new store instance before each test
|
|
41
|
+
store = new LiteVectorStore(mockConfig, userId);
|
|
42
|
+
// db interaction isn't needed for these calculation tests.
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
describe('calculateRecencyScore', () => {
|
|
46
|
+
const now = new Date();
|
|
47
|
+
const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000).toISOString();
|
|
48
|
+
const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
|
49
|
+
const futureDate = new Date(now.getTime() + 1 * 24 * 60 * 60 * 1000).toISOString();
|
|
50
|
+
|
|
51
|
+
it('should return 1.0 for infinite half-life', () => {
|
|
52
|
+
const score = (store as any).calculateRecencyScore(sevenDaysAgo, Infinity);
|
|
53
|
+
expect(score).toBe(1.0);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('should return 1.0 for missing createdAt date', () => {
|
|
57
|
+
const score = (store as any).calculateRecencyScore(undefined, 7);
|
|
58
|
+
expect(score).toBe(1.0);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('should return 0.0 for zero half-life', () => {
|
|
62
|
+
const score = (store as any).calculateRecencyScore(sevenDaysAgo, 0);
|
|
63
|
+
expect(score).toBe(0.0);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('should return ~0.5 for age equal to half-life (e.g., 7 days ago, HL=7)', () => {
|
|
67
|
+
const score = (store as any).calculateRecencyScore(sevenDaysAgo, 7);
|
|
68
|
+
expect(score).toBeCloseTo(0.5, 5); // Use toBeCloseTo for float comparisons
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('should return ~0.25 for age double the half-life (e.g., 4 days ago, HL=2)', () => {
|
|
72
|
+
const fourDaysAgo = new Date(now.getTime() - 4 * 24 * 60 * 60 * 1000).toISOString();
|
|
73
|
+
const score = (store as any).calculateRecencyScore(fourDaysAgo, 2);
|
|
74
|
+
expect(score).toBeCloseTo(0.25, 5);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('should return 0.5 for invalid date format', () => {
|
|
78
|
+
const score = (store as any).calculateRecencyScore('invalid-date', 7);
|
|
79
|
+
expect(score).toBe(0.5);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('should return 1.0 for future dates', () => {
|
|
83
|
+
const score = (store as any).calculateRecencyScore(futureDate, 7);
|
|
84
|
+
expect(score).toBe(1.0);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('should handle negative ageInDays gracefully (treat as future)', () => {
|
|
88
|
+
// Mock Date.now() to simulate the scenario if needed, or test with future date
|
|
89
|
+
const score = (store as any).calculateRecencyScore(futureDate, 7);
|
|
90
|
+
expect(score).toBe(1.0);
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe('calculateHybridScore', () => {
|
|
95
|
+
const now = new Date();
|
|
96
|
+
const twoDaysAgo = new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000).toISOString();
|
|
97
|
+
const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000).toISOString();
|
|
98
|
+
const veryOldDate = new Date(now.getTime() - 400 * 24 * 60 * 60 * 1000).toISOString();
|
|
99
|
+
const scoring = DEFAULT_MEMORY_CONFIG.vectorStore.config.scoring!;
|
|
100
|
+
|
|
101
|
+
it('should prioritize cosine similarity for factual memory (high alpha)', () => {
|
|
102
|
+
const payload: MemoryPayload = { memoryId: 'mem-f1', userId: userId, type: 'factual', createdAt: twoDaysAgo };
|
|
103
|
+
const cosineScore = 0.9;
|
|
104
|
+
const expectedRecency = (store as any).calculateRecencyScore(twoDaysAgo, scoring.factual.halfLifeDays);
|
|
105
|
+
const expectedScore = scoring.factual.alpha * cosineScore + scoring.factual.beta * expectedRecency + scoring.factual.gamma;
|
|
106
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
107
|
+
expect(hybridScore).toBeCloseTo(expectedScore, 5);
|
|
108
|
+
expect(scoring.factual.alpha * cosineScore).toBeGreaterThan(scoring.factual.beta * expectedRecency);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('should prioritize recency for episodic memory (high beta, short half-life)', () => {
|
|
112
|
+
const payload: MemoryPayload = { memoryId: 'mem-e1', userId: userId, type: 'episodic', createdAt: twoDaysAgo };
|
|
113
|
+
const cosineScore = 0.5;
|
|
114
|
+
const expectedRecency = (store as any).calculateRecencyScore(twoDaysAgo, scoring.episodic.halfLifeDays);
|
|
115
|
+
const expectedScore = scoring.episodic.alpha * cosineScore + scoring.episodic.beta * expectedRecency + scoring.episodic.gamma;
|
|
116
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
117
|
+
expect(hybridScore).toBeCloseTo(expectedScore, 5);
|
|
118
|
+
expect(scoring.episodic.beta * expectedRecency).toBeGreaterThan(0.2);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('should have low score for old episodic memory', () => {
|
|
122
|
+
const payload: MemoryPayload = { memoryId: 'mem-e2', userId: userId, type: 'episodic', createdAt: veryOldDate };
|
|
123
|
+
const cosineScore = 0.9;
|
|
124
|
+
const expectedRecency = (store as any).calculateRecencyScore(veryOldDate, scoring.episodic.halfLifeDays);
|
|
125
|
+
const expectedScore = scoring.episodic.alpha * cosineScore + scoring.episodic.beta * expectedRecency + scoring.episodic.gamma;
|
|
126
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
127
|
+
expect(expectedRecency).toBeLessThan(0.01);
|
|
128
|
+
expect(hybridScore).toBeCloseTo(scoring.episodic.alpha * cosineScore + scoring.episodic.gamma, 5);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('should handle assistant_preference with no decay (Infinity half-life)', () => {
|
|
132
|
+
const payload: MemoryPayload = { memoryId: 'mem-a1', userId: userId, type: 'assistant_preference', createdAt: veryOldDate };
|
|
133
|
+
const cosineScore = 0.7;
|
|
134
|
+
const expectedRecency = 1.0;
|
|
135
|
+
const expectedScore = scoring.assistant_preference.alpha * cosineScore + scoring.assistant_preference.beta * expectedRecency + scoring.assistant_preference.gamma;
|
|
136
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
137
|
+
expect(hybridScore).toBeCloseTo(expectedScore, 5);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('should use default scoring if type is missing', () => {
|
|
141
|
+
const payload: MemoryPayload = { memoryId: 'mem-d1', userId: userId, createdAt: sevenDaysAgo }; // No type specified
|
|
142
|
+
const cosineScore = 0.8;
|
|
143
|
+
const expectedRecency = (store as any).calculateRecencyScore(sevenDaysAgo, scoring.default.halfLifeDays);
|
|
144
|
+
const expectedScore = scoring.default.alpha * cosineScore + scoring.default.beta * expectedRecency + scoring.default.gamma;
|
|
145
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
146
|
+
expect(hybridScore).toBeCloseTo(expectedScore, 5);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('should return score >= 0 even with negative cosine similarity', () => {
|
|
150
|
+
const payload: MemoryPayload = { memoryId: 'mem-s1', userId: userId, type: 'semantic', createdAt: twoDaysAgo };
|
|
151
|
+
const cosineScore = -0.5;
|
|
152
|
+
const hybridScore = (store as any).calculateHybridScore(cosineScore, payload);
|
|
153
|
+
expect(hybridScore).toBeGreaterThanOrEqual(0);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
});
|