memories-lite 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -152
- package/dist/memory/index.js +10 -3
- package/dist/prompts/index.d.ts +1 -1
- package/dist/prompts/index.js +14 -9
- package/dist/vectorstores/lite.d.ts +10 -0
- package/dist/vectorstores/lite.js +46 -15
- package/{memories-lite.db → memories-lite-a42ac5108869b599bcbac21069f63fb47f07452fcc4b87e89b3c06a945612d0b.db} +0 -0
- package/memories-lite-a9137698d8d3fdbf27efcdc8cd372084b52d484e8db866c5455bbb3f85299b54.db +0 -0
- package/package.json +1 -1
- package/src/memory/index.ts +10 -4
- package/src/prompts/index.ts +14 -9
- package/src/vectorstores/lite.ts +52 -14
- package/tests/init.mem.ts +40 -0
- package/tests/memory.facts.test.ts +40 -83
- package/tests/memory.test.ts +16 -74
- package/tests/memory.update.test.ts +150 -0
- package/tests/memory.users.test.ts +235 -0
|
@@ -8,6 +8,7 @@ exports.LiteVectorStore = void 0;
|
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
9
|
const sqlite3_1 = __importDefault(require("sqlite3"));
|
|
10
10
|
const crypto_1 = require("crypto");
|
|
11
|
+
const fs_1 = require("fs");
|
|
11
12
|
/**
|
|
12
13
|
* LiteVectorStore provides a simple vector storage implementation.
|
|
13
14
|
*
|
|
@@ -27,27 +28,32 @@ class LiteVectorStore {
|
|
|
27
28
|
this.currentUserId = currentUserId;
|
|
28
29
|
this.isSecure = config.secure || false;
|
|
29
30
|
this.scoringConfig = config.scoring;
|
|
30
|
-
this.cleanupThreshold = config.recencyCleanupThreshold; //
|
|
31
|
+
this.cleanupThreshold = config.recencyCleanupThreshold || 0.25; // (default 0.25 means 2 times the half-life )
|
|
31
32
|
config.rootPath = config.rootPath || process.cwd();
|
|
32
33
|
const filename = this.isSecure ? `memories-lite-${currentUserId}.db` : `memories-lite-global.db`;
|
|
33
|
-
|
|
34
|
+
this.dbPath = (config.rootPath == ':memory:') ? ':memory:' : path_1.default.join(config.rootPath, filename);
|
|
34
35
|
// Add error handling callback for the database connection
|
|
35
|
-
this.db = new sqlite3_1.default.Database(dbPath);
|
|
36
|
+
this.db = new sqlite3_1.default.Database(this.dbPath);
|
|
36
37
|
}
|
|
37
38
|
async init() {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
try {
|
|
40
|
+
await this.run(`
|
|
41
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
42
|
+
id TEXT PRIMARY KEY,
|
|
41
43
|
vector BLOB NOT NULL,
|
|
42
44
|
payload TEXT NOT NULL
|
|
43
45
|
)
|
|
44
46
|
`);
|
|
45
|
-
|
|
47
|
+
await this.run(`
|
|
46
48
|
CREATE TABLE IF NOT EXISTS memory_migrations (
|
|
47
49
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
48
50
|
user_id TEXT NOT NULL UNIQUE
|
|
49
51
|
)
|
|
50
52
|
`);
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
console.log("-- DBG init error:", err);
|
|
56
|
+
}
|
|
51
57
|
}
|
|
52
58
|
async run(sql, params = []) {
|
|
53
59
|
return new Promise((resolve, reject) => {
|
|
@@ -129,9 +135,11 @@ class LiteVectorStore {
|
|
|
129
135
|
if (cachedStore) {
|
|
130
136
|
Object.setPrototypeOf(cachedStore, LiteVectorStore.prototype);
|
|
131
137
|
cachedStore.currentUserId = hashedUserId;
|
|
132
|
-
//
|
|
133
|
-
|
|
134
|
-
cachedStore.
|
|
138
|
+
//
|
|
139
|
+
// if the database file does not exist, we need to reinitialize the store
|
|
140
|
+
if (cachedStore.dbPath !== ':memory:' && !(0, fs_1.existsSync)(cachedStore.dbPath)) {
|
|
141
|
+
return new LiteVectorStore(config, hashedUserId);
|
|
142
|
+
}
|
|
135
143
|
return cachedStore;
|
|
136
144
|
}
|
|
137
145
|
// Pass the full config (including scoring) to the constructor
|
|
@@ -145,17 +153,28 @@ class LiteVectorStore {
|
|
|
145
153
|
if (vectors[i].length !== this.dimension) {
|
|
146
154
|
throw new Error(`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`);
|
|
147
155
|
}
|
|
156
|
+
const payload = { ...payloads[i] };
|
|
157
|
+
//
|
|
158
|
+
// case of global store (insecure)
|
|
159
|
+
if (!payload.userId) {
|
|
160
|
+
throw new Error("userId is required in payload");
|
|
161
|
+
}
|
|
148
162
|
//
|
|
149
163
|
// remove the userId from the payload as sensitive data
|
|
150
|
-
this.isSecure && delete
|
|
164
|
+
this.isSecure && delete payload.userId;
|
|
151
165
|
const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
|
|
152
|
-
await this.run(`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`, [ids[i], vectorBuffer, JSON.stringify(
|
|
166
|
+
await this.run(`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`, [ids[i], vectorBuffer, JSON.stringify(payload)]);
|
|
153
167
|
}
|
|
154
168
|
}
|
|
155
169
|
async search(query, limit = 10, filters) {
|
|
156
170
|
if (query.length !== this.dimension) {
|
|
157
171
|
throw new Error(`Query dimension mismatch. Expected ${this.dimension}, got ${query.length}`);
|
|
158
172
|
}
|
|
173
|
+
if (!filters || !filters.userId) {
|
|
174
|
+
throw new Error("userId is mandatory in search");
|
|
175
|
+
}
|
|
176
|
+
filters = { ...filters };
|
|
177
|
+
this.isSecure && delete filters.userId;
|
|
159
178
|
const results = [];
|
|
160
179
|
const rows = await this.all(`SELECT * FROM vectors`);
|
|
161
180
|
for (const row of rows) {
|
|
@@ -225,17 +244,21 @@ class LiteVectorStore {
|
|
|
225
244
|
async list(filters, limit = 100) {
|
|
226
245
|
const rows = await this.all(`SELECT * FROM vectors`);
|
|
227
246
|
const results = [];
|
|
247
|
+
//
|
|
248
|
+
// remove the userId from the payload as sensitive data
|
|
249
|
+
filters = { ...filters };
|
|
250
|
+
this.isSecure && delete filters?.userId;
|
|
228
251
|
for (const row of rows) {
|
|
229
252
|
const memoryVector = {
|
|
230
253
|
id: row.id,
|
|
231
254
|
vector: Array.from(new Float32Array(row.vector.buffer)),
|
|
232
|
-
payload:
|
|
255
|
+
payload: JSON.parse(row.payload),
|
|
233
256
|
};
|
|
234
257
|
if (this.filterVector(memoryVector, filters)) {
|
|
235
258
|
// load payload at the end
|
|
236
259
|
results.push({
|
|
237
260
|
id: memoryVector.id,
|
|
238
|
-
payload:
|
|
261
|
+
payload: memoryVector.payload,
|
|
239
262
|
});
|
|
240
263
|
}
|
|
241
264
|
}
|
|
@@ -278,7 +301,15 @@ class LiteVectorStore {
|
|
|
278
301
|
// Ensure score is within a reasonable range (e.g., 0 to alpha+beta+gamma)
|
|
279
302
|
return Math.max(0, hybridScore);
|
|
280
303
|
}
|
|
281
|
-
|
|
304
|
+
/**
|
|
305
|
+
* Internal method to clean up vectors based on recency score threshold.
|
|
306
|
+
*
|
|
307
|
+
* @param threshold - The minimum recency score required for a memory to be retained.
|
|
308
|
+
* - Recency score is calculated using exponential decay: 1.0 means brand new, 0.5 means at half-life, 0.0 means fully decayed.
|
|
309
|
+
* - Memories with a recency score below this threshold will be deleted (unless their half-life is infinite or zero).
|
|
310
|
+
* - For example, a threshold of 0.25 will remove all memories whose recency score has decayed 2 times the half-life.
|
|
311
|
+
* - Use a lower threshold to keep more old memories, or a higher threshold to keep only fresher ones.
|
|
312
|
+
*/
|
|
282
313
|
async _cleanupByRecency(threshold) {
|
|
283
314
|
const rows = await this.all(`SELECT id, payload FROM vectors`);
|
|
284
315
|
let deletedCount = 0;
|
|
Binary file
|
|
Binary file
|
package/package.json
CHANGED
package/src/memory/index.ts
CHANGED
|
@@ -162,7 +162,7 @@ export class MemoriesLite {
|
|
|
162
162
|
|
|
163
163
|
const $t = this.$t;
|
|
164
164
|
const vectorStore = await this.getVectorStore(userId);
|
|
165
|
-
const parsedMessages = messages.filter((m) => typeof m.content === 'string').map((m) => `${m.role=='user' ? '**USER**: ' : '**ASSISTANT**: '}${$t(m.content as string)}\n`).join("\n");
|
|
165
|
+
const parsedMessages = messages.filter((m) => typeof m.content === 'string' && m.role=='user').map((m) => `${m.role=='user' ? '**USER**: ' : '**ASSISTANT**: '}${$t(m.content as string)}\n`).join("\n");
|
|
166
166
|
|
|
167
167
|
const [systemPrompt, userPrompt] = getFactRetrievalMessages(parsedMessages, customFacts||this.customPrompt);
|
|
168
168
|
|
|
@@ -202,6 +202,9 @@ export class MemoriesLite {
|
|
|
202
202
|
const newMessageEmbeddings: Record<string, number[]> = {};
|
|
203
203
|
const retrievedOldMemory: Array<{ id: string; text: string; type: string }> = [];
|
|
204
204
|
|
|
205
|
+
//
|
|
206
|
+
// add the userId to the filters
|
|
207
|
+
filters.userId = userId;
|
|
205
208
|
// Create embeddings and search for similar memories
|
|
206
209
|
for (const elem of facts) {
|
|
207
210
|
const fact = elem.fact;
|
|
@@ -250,7 +253,7 @@ export class MemoriesLite {
|
|
|
250
253
|
console.log(`-- ⛔ LLM Error: ${action.event}, ${action.type}, "${action.text}"`);
|
|
251
254
|
continue;
|
|
252
255
|
}
|
|
253
|
-
console.log(`-- DBG memory
|
|
256
|
+
console.log(`-- DBG memory "${userId}": ${action.event}, ${action.type}, "${action.text}", why: "${action.reason}"`);
|
|
254
257
|
try {
|
|
255
258
|
switch (action.event) {
|
|
256
259
|
case "ADD": {
|
|
@@ -276,7 +279,7 @@ export class MemoriesLite {
|
|
|
276
279
|
}
|
|
277
280
|
case "UPDATE": {
|
|
278
281
|
const realMemoryId = tempUuidMapping[action.id];
|
|
279
|
-
const type = uniqueOldMemories[action.id].type
|
|
282
|
+
const type = metadata.type = uniqueOldMemories[action.id].type || action.type;
|
|
280
283
|
await this.updateMemory(
|
|
281
284
|
realMemoryId,
|
|
282
285
|
action.text,
|
|
@@ -455,7 +458,7 @@ export class MemoriesLite {
|
|
|
455
458
|
}
|
|
456
459
|
|
|
457
460
|
const vectorStore = await this.getVectorStore(userId);
|
|
458
|
-
|
|
461
|
+
filters.userId = userId;
|
|
459
462
|
|
|
460
463
|
// Search vector store
|
|
461
464
|
const queryEmbedding = await this.embedder.embed(query);
|
|
@@ -597,6 +600,7 @@ export class MemoriesLite {
|
|
|
597
600
|
if (agentId) filters.agentId = agentId;
|
|
598
601
|
if (runId) filters.runId = runId;
|
|
599
602
|
if (type) filters.type = type;
|
|
603
|
+
filters.userId = userId;
|
|
600
604
|
const [memories] = await vectorStore.list(filters, limit);
|
|
601
605
|
|
|
602
606
|
const excludedKeys = new Set([
|
|
@@ -640,6 +644,7 @@ export class MemoriesLite {
|
|
|
640
644
|
...metadata,
|
|
641
645
|
data,
|
|
642
646
|
hash: createHash("md5").update(data).digest("hex"),
|
|
647
|
+
userId,
|
|
643
648
|
createdAt: new Date().toISOString(),
|
|
644
649
|
};
|
|
645
650
|
|
|
@@ -676,6 +681,7 @@ export class MemoriesLite {
|
|
|
676
681
|
...metadata,
|
|
677
682
|
data,
|
|
678
683
|
hash: createHash("md5").update(data).digest("hex"),
|
|
684
|
+
type: existingMemory.payload.type,
|
|
679
685
|
createdAt: existingMemory.payload.createdAt,
|
|
680
686
|
updatedAt: new Date().toISOString(),
|
|
681
687
|
...(existingMemory.payload.agentId && {
|
package/src/prompts/index.ts
CHANGED
|
@@ -20,7 +20,12 @@ export const FactRetrievalSchema_extended = z.object({
|
|
|
20
20
|
z.object({
|
|
21
21
|
fact: z.string().describe("The fact extracted from the conversation."),
|
|
22
22
|
existing: z.boolean().describe("Whether the fact is already present"),
|
|
23
|
-
type: z.enum(["assistant_preference","factual", "episodic", "procedural", "semantic"])
|
|
23
|
+
type: z.enum(["assistant_preference","factual", "episodic", "procedural", "semantic"])
|
|
24
|
+
.describe(`The type of the fact.
|
|
25
|
+
Use 'assistant_preference' for Assistant behavior preferences.
|
|
26
|
+
Use 'episodic' always for time-based events.
|
|
27
|
+
Use 'procedural' always when it concerns a business question.
|
|
28
|
+
Use 'semantic' for Understanding of concepts, relationships and general meanings.`),
|
|
24
29
|
})
|
|
25
30
|
)
|
|
26
31
|
});
|
|
@@ -71,7 +76,7 @@ export const MEMORY_STRING_SYSTEM = `# DIRECTIVES FOR MEMORIES
|
|
|
71
76
|
- You must adapt your answer based on the contents found within the <memories> section.
|
|
72
77
|
- If the memories are irrelevant to the user's query, you MUST ignore them.
|
|
73
78
|
- By default, do not reference this section or the memories in your response.
|
|
74
|
-
- Use
|
|
79
|
+
- Use memories only to guide your reasoning. Do not respond to the memories themselves.`;
|
|
75
80
|
|
|
76
81
|
export const MEMORY_STRING_PREFIX = "Use these contextual memories to guide your response. Prioritize the user's question. Ignore irrelevant memories."
|
|
77
82
|
|
|
@@ -97,17 +102,17 @@ Your mission is to analyze a input content line by line and produce:
|
|
|
97
102
|
|
|
98
103
|
Filter content before extracting triplets:
|
|
99
104
|
- Ignore content with no direct relevance to user (e.g., "today is sunny", "I'm working").
|
|
100
|
-
-
|
|
101
|
-
- Eliminate introductions, sub-facts, detailed repetitive elements, stylistic fillers, or vague statements. A general fact always takes precedence over multiple sub-facts (signal vs noise).
|
|
105
|
+
- Eliminate introductions, vague statements and detailed repetitive elements.
|
|
102
106
|
|
|
103
107
|
You must extract {Subject, Predicate, Object} triplets by following these rules:
|
|
104
108
|
1. Identify named entities, preferences, and meaningful user-related concepts:
|
|
105
|
-
- All extracted triplets describe the user query intention as: the user’s preferences, beliefs, actions, experiences, learning, identity, work, or relationships (e.g., "I
|
|
109
|
+
- All extracted triplets describe the user query intention as: the user’s preferences, beliefs, actions, experiences, learning, identity, work, or relationships (e.g., "I love working with precise Agents").
|
|
110
|
+
- Merge triplets from sub-facts or detailed objects. A general fact always takes precedence over multiple sub-facts (signal vs noise).
|
|
106
111
|
- If the user asks about third-party business information classify it as "procedural" type.
|
|
107
112
|
- The query intention can include specific preferences about how the Assistant should respond (e.g., "answer concisely", "explain in detail").
|
|
108
113
|
- Use inference to compress each fact (max 10 words).
|
|
109
114
|
- DO NOT infer personal facts from third-party informations.
|
|
110
|
-
- Treat "Assistant
|
|
115
|
+
- Treat "Assistant:" messages as external and transient responses, there is no fact to extract from them. These responses MUST be used to enrich your reasoning process.
|
|
111
116
|
2. Compress the facts:
|
|
112
117
|
- Keep only the most shortest version of the Triplet.
|
|
113
118
|
3. Rewrite comparatives, conditionals, or temporals into explicit predicates (e.g., "prefers", "available during", "used because of").
|
|
@@ -148,15 +153,15 @@ You must strictly extract {Subject, Predicate, Object} triplets by following the
|
|
|
148
153
|
- Extract triplets that describe facts *about the user* based on their statements, covering areas like preferences, beliefs, actions, experiences, learning, identity, work, or relationships (e.g., "I love working").
|
|
149
154
|
- Apply explicit, precise, and unambiguous predicates (e.g., "owns", "is located at", "is a", "has function", "causes", etc.).
|
|
150
155
|
- Determine the triplet type (e.g., "factual", "episodic", "procedural", "semantic") based on the content and meaning.
|
|
151
|
-
- "episodic"
|
|
156
|
+
- "episodic" If a fact depends on a temporal, situational, or immediate personal context, then that fact AND ALL OF ITS sub-facts MUST be classified as episodic.
|
|
152
157
|
- "procedural" for business processes (e.g., "Looking for customer John Doe address", "How to create a new contract").
|
|
153
158
|
- "factual" for stable user data (except procedural that prevails).
|
|
154
159
|
|
|
155
160
|
- Eliminate introductions, sub-facts, detailed repetitive elements, stylistic fillers, or vague statements. General facts always takes precedence over multiple sub-facts (signal vs noise).
|
|
156
161
|
- The query intention can include specific preferences about how the Assistant should respond (e.g., "answer concisely", "explain in detail").
|
|
157
|
-
- Compress each fact
|
|
162
|
+
- Compress each OUTPUT (fact and reason) with less than 10 words.
|
|
158
163
|
- DO NOT infer personal facts from third-party informations.
|
|
159
|
-
- Treat "
|
|
164
|
+
- Treat "**ASSISTANT**:" as responses to enrich context of your reasoning process about the USER query.
|
|
160
165
|
2. Use pronoun "I" instead of "The user" in the subject of the triplet.
|
|
161
166
|
3. Do not output any facts already present in section # PRE-EXISTING FACTS.
|
|
162
167
|
- If you find facts already present in section # PRE-EXISTING FACTS, use field "existing" to store them.
|
package/src/vectorstores/lite.ts
CHANGED
|
@@ -5,6 +5,7 @@ import sqlite3 from 'sqlite3';
|
|
|
5
5
|
import { VectorStore } from "./base";
|
|
6
6
|
import { SearchFilters, VectorStoreConfig, VectorStoreResult, MemoryPayload, MemoryScoringConfig, MemoryType } from "../types";
|
|
7
7
|
import { createHash } from 'crypto';
|
|
8
|
+
import { existsSync } from 'fs';
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
// Define interface for database rows
|
|
@@ -25,6 +26,7 @@ interface MemoryVector {
|
|
|
25
26
|
*/
|
|
26
27
|
export class LiteVectorStore implements VectorStore {
|
|
27
28
|
private db: sqlite3.Database;
|
|
29
|
+
private dbPath: string;
|
|
28
30
|
private isSecure: boolean;
|
|
29
31
|
private dimension: number;
|
|
30
32
|
private currentUserId: string;
|
|
@@ -39,20 +41,21 @@ export class LiteVectorStore implements VectorStore {
|
|
|
39
41
|
this.currentUserId = currentUserId;
|
|
40
42
|
this.isSecure = config.secure || false;
|
|
41
43
|
this.scoringConfig = config.scoring;
|
|
42
|
-
this.cleanupThreshold = config.recencyCleanupThreshold; //
|
|
44
|
+
this.cleanupThreshold = config.recencyCleanupThreshold || 0.25; // (default 0.25 means 2 times the half-life )
|
|
43
45
|
config.rootPath = config.rootPath || process.cwd();
|
|
44
46
|
const filename = this.isSecure ? `memories-lite-${currentUserId}.db` : `memories-lite-global.db`;
|
|
45
|
-
|
|
47
|
+
this.dbPath = (config.rootPath == ':memory:') ? ':memory:' : path.join(config.rootPath, filename);
|
|
46
48
|
|
|
47
49
|
// Add error handling callback for the database connection
|
|
48
|
-
this.db = new sqlite3.Database(dbPath);
|
|
50
|
+
this.db = new sqlite3.Database(this.dbPath);
|
|
49
51
|
}
|
|
50
52
|
|
|
51
53
|
|
|
52
54
|
private async init() {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
try{
|
|
56
|
+
await this.run(`
|
|
57
|
+
CREATE TABLE IF NOT EXISTS vectors (
|
|
58
|
+
id TEXT PRIMARY KEY,
|
|
56
59
|
vector BLOB NOT NULL,
|
|
57
60
|
payload TEXT NOT NULL
|
|
58
61
|
)
|
|
@@ -64,6 +67,9 @@ export class LiteVectorStore implements VectorStore {
|
|
|
64
67
|
user_id TEXT NOT NULL UNIQUE
|
|
65
68
|
)
|
|
66
69
|
`);
|
|
70
|
+
}catch(err){
|
|
71
|
+
console.log("-- DBG init error:",err);
|
|
72
|
+
}
|
|
67
73
|
}
|
|
68
74
|
|
|
69
75
|
private async run(sql: string, params: any[] = []): Promise<void> {
|
|
@@ -151,9 +157,12 @@ export class LiteVectorStore implements VectorStore {
|
|
|
151
157
|
if (cachedStore) {
|
|
152
158
|
Object.setPrototypeOf(cachedStore, LiteVectorStore.prototype);
|
|
153
159
|
cachedStore.currentUserId = hashedUserId;
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
160
|
+
|
|
161
|
+
//
|
|
162
|
+
// if the database file does not exist, we need to reinitialize the store
|
|
163
|
+
if (cachedStore.dbPath!==':memory:' && !existsSync(cachedStore.dbPath)) {
|
|
164
|
+
return new LiteVectorStore(config, hashedUserId);
|
|
165
|
+
}
|
|
157
166
|
return cachedStore;
|
|
158
167
|
}
|
|
159
168
|
|
|
@@ -176,13 +185,20 @@ export class LiteVectorStore implements VectorStore {
|
|
|
176
185
|
`Vector dimension mismatch. Expected ${this.dimension}, got ${vectors[i].length}`,
|
|
177
186
|
);
|
|
178
187
|
}
|
|
188
|
+
|
|
189
|
+
const payload = {...payloads[i]};
|
|
190
|
+
//
|
|
191
|
+
// case of global store (insecure)
|
|
192
|
+
if(!payload.userId){
|
|
193
|
+
throw new Error("userId is required in payload");
|
|
194
|
+
}
|
|
179
195
|
//
|
|
180
196
|
// remove the userId from the payload as sensitive data
|
|
181
|
-
this.isSecure && delete
|
|
197
|
+
this.isSecure && delete payload.userId;
|
|
182
198
|
const vectorBuffer = Buffer.from(new Float32Array(vectors[i]).buffer);
|
|
183
199
|
await this.run(
|
|
184
200
|
`INSERT OR REPLACE INTO vectors (id, vector, payload) VALUES (?, ?, ?)`,
|
|
185
|
-
[ids[i], vectorBuffer, JSON.stringify(
|
|
201
|
+
[ids[i], vectorBuffer, JSON.stringify(payload)],
|
|
186
202
|
);
|
|
187
203
|
}
|
|
188
204
|
|
|
@@ -199,6 +215,15 @@ export class LiteVectorStore implements VectorStore {
|
|
|
199
215
|
);
|
|
200
216
|
}
|
|
201
217
|
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
if(!filters || !filters.userId){
|
|
221
|
+
throw new Error("userId is mandatory in search");
|
|
222
|
+
}
|
|
223
|
+
filters = {...filters};
|
|
224
|
+
this.isSecure && delete filters.userId;
|
|
225
|
+
|
|
226
|
+
|
|
202
227
|
const results: VectorStoreResult[] = [];
|
|
203
228
|
const rows = await this.all(`SELECT * FROM vectors`);
|
|
204
229
|
|
|
@@ -290,18 +315,23 @@ export class LiteVectorStore implements VectorStore {
|
|
|
290
315
|
const rows = await this.all(`SELECT * FROM vectors`);
|
|
291
316
|
const results: VectorStoreResult[] = [];
|
|
292
317
|
|
|
318
|
+
//
|
|
319
|
+
// remove the userId from the payload as sensitive data
|
|
320
|
+
filters = {...filters};
|
|
321
|
+
this.isSecure && delete filters?.userId;
|
|
322
|
+
|
|
293
323
|
for (const row of rows) {
|
|
294
324
|
const memoryVector: MemoryVector = {
|
|
295
325
|
id: row.id,
|
|
296
326
|
vector: Array.from(new Float32Array(row.vector.buffer)),
|
|
297
|
-
payload:
|
|
327
|
+
payload: JSON.parse(row.payload),
|
|
298
328
|
};
|
|
299
329
|
|
|
300
330
|
if (this.filterVector(memoryVector, filters)) {
|
|
301
331
|
// load payload at the end
|
|
302
332
|
results.push({
|
|
303
333
|
id: memoryVector.id,
|
|
304
|
-
payload:
|
|
334
|
+
payload:memoryVector.payload,
|
|
305
335
|
});
|
|
306
336
|
}
|
|
307
337
|
}
|
|
@@ -356,7 +386,15 @@ export class LiteVectorStore implements VectorStore {
|
|
|
356
386
|
return Math.max(0, hybridScore);
|
|
357
387
|
}
|
|
358
388
|
|
|
359
|
-
|
|
389
|
+
/**
|
|
390
|
+
* Internal method to clean up vectors based on recency score threshold.
|
|
391
|
+
*
|
|
392
|
+
* @param threshold - The minimum recency score required for a memory to be retained.
|
|
393
|
+
* - Recency score is calculated using exponential decay: 1.0 means brand new, 0.5 means at half-life, 0.0 means fully decayed.
|
|
394
|
+
* - Memories with a recency score below this threshold will be deleted (unless their half-life is infinite or zero).
|
|
395
|
+
* - For example, a threshold of 0.25 will remove all memories whose recency score has decayed 2 times the half-life.
|
|
396
|
+
* - Use a lower threshold to keep more old memories, or a higher threshold to keep only fresher ones.
|
|
397
|
+
*/
|
|
360
398
|
private async _cleanupByRecency(threshold: number): Promise<number> {
|
|
361
399
|
const rows = await this.all(`SELECT id, payload FROM vectors`);
|
|
362
400
|
let deletedCount = 0;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/// <reference types="jest" />
|
|
2
|
+
import { MemoriesLite } from "../src";
|
|
3
|
+
import dotenv from "dotenv";
|
|
4
|
+
|
|
5
|
+
dotenv.config();
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Helper to initialize MemoriesLite instance and generate a random userId.
|
|
9
|
+
* @param customPrompt Optional prompt to inject into the memory config.
|
|
10
|
+
*/
|
|
11
|
+
export function createTestMemory({customPrompt, dimension, rootPath, secure}:any) {
|
|
12
|
+
dimension = dimension || 768;
|
|
13
|
+
const userId =
|
|
14
|
+
Math.random().toString(36).substring(2, 15) +
|
|
15
|
+
Math.random().toString(36).substring(2, 15);
|
|
16
|
+
|
|
17
|
+
const memory = new MemoriesLite({
|
|
18
|
+
version: "v1.1",
|
|
19
|
+
disableHistory: true,
|
|
20
|
+
...(customPrompt ? { customPrompt } : {}),
|
|
21
|
+
embedder: {
|
|
22
|
+
provider: "openai",
|
|
23
|
+
config: { dimension, apiKey: process.env.OPENAI_API_KEY!, model: "text-embedding-3-small" }
|
|
24
|
+
},
|
|
25
|
+
vectorStore: {
|
|
26
|
+
provider: "lite",
|
|
27
|
+
config: {
|
|
28
|
+
dimension,
|
|
29
|
+
rootPath: (rootPath || ":memory:"),
|
|
30
|
+
secure: secure || false }
|
|
31
|
+
},
|
|
32
|
+
llm: {
|
|
33
|
+
provider: "openai",
|
|
34
|
+
config: { apiKey: process.env.OPENAI_API_KEY || "", model: "gpt-4.1-mini" }
|
|
35
|
+
},
|
|
36
|
+
historyDbPath: ":memory:"
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
return { memory, userId };
|
|
40
|
+
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { MemoriesLite } from "../src";
|
|
3
3
|
import { MemoryItem, SearchResult } from "../src/types";
|
|
4
4
|
import dotenv from "dotenv";
|
|
5
|
+
import { createTestMemory } from "./init.mem";
|
|
5
6
|
|
|
6
7
|
dotenv.config();
|
|
7
8
|
|
|
@@ -9,42 +10,11 @@ jest.setTimeout(30000); // Increase timeout to 30 seconds
|
|
|
9
10
|
|
|
10
11
|
describe("Memory Class facts regression tests", () => {
|
|
11
12
|
let memory: MemoriesLite;
|
|
12
|
-
|
|
13
|
-
Math.random().toString(36).substring(2, 15) +
|
|
14
|
-
Math.random().toString(36).substring(2, 15);
|
|
15
|
-
|
|
16
|
-
const dimension = 768;
|
|
13
|
+
let userId: string;
|
|
17
14
|
|
|
18
15
|
beforeEach(async () => {
|
|
19
|
-
// Initialize
|
|
20
|
-
memory =
|
|
21
|
-
version: "v1.1",
|
|
22
|
-
disableHistory: true,
|
|
23
|
-
customPrompt: "L'utilisateur travail pour une régie immobilière!",
|
|
24
|
-
embedder: {
|
|
25
|
-
provider: "openai",
|
|
26
|
-
config: {
|
|
27
|
-
dimension,
|
|
28
|
-
apiKey: process.env.OPENAI_API_KEY || "",
|
|
29
|
-
model: "text-embedding-3-small",
|
|
30
|
-
},
|
|
31
|
-
},
|
|
32
|
-
vectorStore: {
|
|
33
|
-
provider: "lite",
|
|
34
|
-
config: {
|
|
35
|
-
dimension,
|
|
36
|
-
rootPath: ":memory:",
|
|
37
|
-
},
|
|
38
|
-
},
|
|
39
|
-
llm: {
|
|
40
|
-
provider: "openai",
|
|
41
|
-
config: {
|
|
42
|
-
apiKey: process.env.OPENAI_API_KEY || "",
|
|
43
|
-
model: "gpt-4.1-mini",
|
|
44
|
-
},
|
|
45
|
-
},
|
|
46
|
-
historyDbPath: ":memory:", // Use in-memory SQLite for tests
|
|
47
|
-
});
|
|
16
|
+
// Initialize memory via helper
|
|
17
|
+
({ memory, userId } = createTestMemory({customPrompt:"L'utilisateur travail pour une régie immobilière!"}));
|
|
48
18
|
// Reset all memories before each test
|
|
49
19
|
await memory.reset(userId);
|
|
50
20
|
});
|
|
@@ -56,69 +26,37 @@ describe("Memory Class facts regression tests", () => {
|
|
|
56
26
|
|
|
57
27
|
describe("Edge cases for Facts", () => {
|
|
58
28
|
|
|
59
|
-
it("should not extract personal information as facts from business queries", async () => {
|
|
60
|
-
// type?: "factual" | "episodic" | "semantic"|"procedural" | "assistant_preference";
|
|
61
|
-
// Capture a query that contains a name but is asking for contact information
|
|
62
|
-
const result = (await memory.capture(
|
|
63
|
-
"je cherche le téléphone de mon client Alphonse MAGLOIRE",
|
|
64
|
-
userId,
|
|
65
|
-
{},
|
|
66
|
-
)) as SearchResult;
|
|
67
|
-
|
|
68
|
-
// Verify no memory was created (business query)
|
|
69
|
-
expect(result).toBeDefined();
|
|
70
|
-
expect(result.results).toBeDefined();
|
|
71
|
-
expect(Array.isArray(result.results)).toBe(true);
|
|
72
|
-
expect(result.results.length).toBe(1);
|
|
73
|
-
const type = result.results[0]?.type;
|
|
74
|
-
expect(["procedural","episodic"].includes(type)).toBe(true);
|
|
75
|
-
// Now search for memories that might contain "Alphonse MAGLOIRE"
|
|
76
|
-
// const searchResult = (await memory.retrieve(
|
|
77
|
-
// "Qui est Alphonse MAGLOIRE?",
|
|
78
|
-
// userId,
|
|
79
|
-
// {},
|
|
80
|
-
// )) as SearchResult;
|
|
81
|
-
|
|
82
|
-
// // Verify no personal fact like "Je m'appelle Alphonse MAGLOIRE" was created
|
|
83
|
-
// expect(searchResult).toBeDefined();
|
|
84
|
-
// expect(searchResult.results).toBeDefined();
|
|
85
|
-
// expect(Array.isArray(searchResult.results)).toBe(true);
|
|
86
|
-
// expect(searchResult.results.length).toBe(0);
|
|
87
|
-
|
|
88
|
-
// // Ensure no memory contains the name as a personal fact
|
|
89
|
-
// const allMemories = await memory.getAll(userId, {});
|
|
90
|
-
// const personalFacts = allMemories.results.filter(mem =>
|
|
91
|
-
// mem.memory.toLowerCase().includes("Alphonse MAGLOIRE")
|
|
92
|
-
// );
|
|
93
|
-
// expect(personalFacts.length).toBe(0);
|
|
94
|
-
});
|
|
95
29
|
|
|
96
|
-
it("should add
|
|
97
|
-
const customFacts = "Je suis Olivier Poulain\nIT chez Immeuble SA";
|
|
30
|
+
it("should not add memory: Qui suis-je ?", async () => {
|
|
31
|
+
const customFacts = "Je suis Olivier Poulain\nIT et je travaille chez Immeuble SA";
|
|
98
32
|
const result = (await memory.capture([
|
|
99
33
|
{role:"user", content:"Qui suis-je ?"},
|
|
100
|
-
{role:"
|
|
34
|
+
{role:"assistant", content:"Vous êtes Olivier Poulain, Chef de Projets au département IT & Gestion de projet, dans l'équipe IT de Immeuble SA"}],
|
|
101
35
|
userId,
|
|
102
36
|
{customFacts},
|
|
103
37
|
)) as SearchResult;
|
|
104
38
|
expect(result).toBeDefined();
|
|
105
39
|
expect(result.results).toBeDefined();
|
|
106
|
-
expect(result.results.length).toBe(
|
|
107
|
-
expect(result.results[0]?.type).toBe("factual");
|
|
40
|
+
expect(result.results.length).toBe(0);
|
|
41
|
+
// expect(result.results[0]?.type).toBe("factual");
|
|
108
42
|
});
|
|
109
|
-
it("
|
|
43
|
+
it("episodic: Je veux manger des sushis pour ma pause de midi.", async () => {
|
|
44
|
+
const customFacts = "Je suis Olivier Poulain\nIT et je travaille chez Immeuble SA";
|
|
110
45
|
const result = (await memory.capture([
|
|
111
|
-
{role:"user", content:"je veux
|
|
46
|
+
{role:"user", content:"J'ai faim, je veux manger des sushis pour ma pause de midi."},
|
|
47
|
+
{role:"user", content:"Cherche un restaurant de sushis près de chez moi."}],
|
|
112
48
|
userId,
|
|
113
|
-
{},
|
|
49
|
+
{customFacts},
|
|
114
50
|
)) as SearchResult;
|
|
115
51
|
|
|
116
52
|
expect(result).toBeDefined();
|
|
117
53
|
expect(result.results).toBeDefined();
|
|
118
|
-
expect(result.results.length).toBeGreaterThan(
|
|
54
|
+
expect(result.results.length).toBeGreaterThan(1);
|
|
119
55
|
expect(result.results[0]?.type).toBe("episodic");
|
|
56
|
+
expect(result.results[1]?.type).toBe("episodic");
|
|
120
57
|
});
|
|
121
58
|
|
|
59
|
+
|
|
122
60
|
it("should add assistant_preference memory", async () => {
|
|
123
61
|
const result = (await memory.capture(
|
|
124
62
|
"tu dois répondre de manière concise et précise",
|
|
@@ -134,9 +72,28 @@ describe("Memory Class facts regression tests", () => {
|
|
|
134
72
|
expect(result.results[0]?.type).toBe("assistant_preference");
|
|
135
73
|
});
|
|
136
74
|
|
|
137
|
-
it("business
|
|
75
|
+
it("business:je cherche le téléphone de mon client Alphonse MAGLOIRE", async () => {
|
|
76
|
+
// type?: "factual" | "episodic" | "semantic"|"procedural" | "assistant_preference";
|
|
77
|
+
// Capture a query that contains a name but is asking for contact information
|
|
78
|
+
const result = (await memory.capture(
|
|
79
|
+
"je cherche le téléphone de mon client Alphonse MAGLOIRE",
|
|
80
|
+
userId,
|
|
81
|
+
{},
|
|
82
|
+
)) as SearchResult;
|
|
83
|
+
|
|
84
|
+
// Verify no memory was created (business query)
|
|
85
|
+
expect(result).toBeDefined();
|
|
86
|
+
expect(result.results).toBeDefined();
|
|
87
|
+
expect(Array.isArray(result.results)).toBe(true);
|
|
88
|
+
expect(result.results.length).toBe(1);
|
|
89
|
+
const type = result.results[0]?.type;
|
|
90
|
+
expect(["procedural","episodic"].includes(type)).toBe(true);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("business:Le logement de Alphonse MAGLOIRE au 5ème étage est de combien pièces", async () => {
|
|
138
94
|
const result = (await memory.capture([
|
|
139
|
-
{role:"user", content:"
|
|
95
|
+
{role:"user", content:"Le logement de Alphonse MAGLOIRE au 5ème étage est de combien pièces.",},
|
|
96
|
+
{role:"assitant", content:"Alphonse MAGLOIRE a un logement de 4 pièces au 5ème étage",}],
|
|
140
97
|
userId,
|
|
141
98
|
{customFacts:"Je suis Olivier Poulain, Je m'occupe de la gérance locataire chez Immeuble SA"},
|
|
142
99
|
)) as SearchResult;
|
|
@@ -171,9 +128,9 @@ describe("Memory Class facts regression tests", () => {
|
|
|
171
128
|
}
|
|
172
129
|
});
|
|
173
130
|
|
|
174
|
-
it("
|
|
131
|
+
it("business:Est-ce que Claude RIBUR est à jour avec son loyer ?", async () => {
|
|
175
132
|
const result = (await memory.capture([
|
|
176
|
-
{role:"user", content:"Est-ce que
|
|
133
|
+
{role:"user", content:"Est-ce que Claude RIBUR est à jour avec son loyer ?"}],
|
|
177
134
|
userId,
|
|
178
135
|
{customFacts:"Je suis Olivier Poulain, Je m'occupe de la gérance locataire chez Immeuble SA"},
|
|
179
136
|
)) as SearchResult;
|