@memrosetta/core 0.2.19 → 0.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +23 -3
- package/dist/index.js +252 -62
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import Database from 'better-sqlite3';
|
|
2
|
-
import { Memory, MemoryInput, RelationType, MemoryRelation, SearchResult, SearchFilters, SearchQuery, SearchResponse, IMemoryEngine, CompressResult, MaintenanceResult, MemoryTier, TierConfig } from '@memrosetta/types';
|
|
2
|
+
import { Memory, MemoryState, MemoryInput, RelationType, MemoryRelation, SearchResult, SearchFilters, SearchQuery, SearchResponse, IMemoryEngine, CompressResult, MaintenanceResult, MemoryTier, MemoryQuality, TierConfig } from '@memrosetta/types';
|
|
3
3
|
import { Embedder, ContradictionDetector } from '@memrosetta/embeddings';
|
|
4
4
|
|
|
5
5
|
interface SchemaOptions {
|
|
@@ -12,6 +12,13 @@ declare function generateMemoryId(): string;
|
|
|
12
12
|
declare function nowIso(): string;
|
|
13
13
|
declare function keywordsToString(keywords: readonly string[] | undefined): string | null;
|
|
14
14
|
declare function stringToKeywords(str: string | null): readonly string[];
|
|
15
|
+
/**
|
|
16
|
+
* Derive the logical state of a memory from its existing fields.
|
|
17
|
+
* - invalidated: invalidatedAt is set (takes precedence)
|
|
18
|
+
* - superseded: isLatest is false
|
|
19
|
+
* - current: isLatest is true and not invalidated
|
|
20
|
+
*/
|
|
21
|
+
declare function deriveMemoryState(memory: Memory): MemoryState;
|
|
15
22
|
|
|
16
23
|
interface MemoryRow {
|
|
17
24
|
readonly id: number;
|
|
@@ -37,6 +44,8 @@ interface MemoryRow {
|
|
|
37
44
|
readonly access_count: number | null;
|
|
38
45
|
readonly last_accessed_at: string | null;
|
|
39
46
|
readonly compressed_from: string | null;
|
|
47
|
+
readonly use_count: number | null;
|
|
48
|
+
readonly success_count: number | null;
|
|
40
49
|
}
|
|
41
50
|
declare function rowToMemory(row: MemoryRow): Memory;
|
|
42
51
|
declare function serializeEmbedding(embedding: readonly number[] | Float32Array): Buffer;
|
|
@@ -204,7 +213,7 @@ declare function updateAccessTracking(db: Database.Database, memoryIds: readonly
|
|
|
204
213
|
*
|
|
205
214
|
* Results are weighted by activation score and access tracking is updated.
|
|
206
215
|
*/
|
|
207
|
-
declare function searchMemories(db: Database.Database, query: SearchQuery, queryVec?: Float32Array, useVecTable?: boolean): SearchResponse;
|
|
216
|
+
declare function searchMemories(db: Database.Database, query: SearchQuery, queryVec?: Float32Array, useVecTable?: boolean, skipAccessTracking?: boolean): SearchResponse;
|
|
208
217
|
|
|
209
218
|
interface SqliteEngineOptions {
|
|
210
219
|
readonly dbPath: string;
|
|
@@ -235,6 +244,8 @@ declare class SqliteMemoryEngine implements IMemoryEngine {
|
|
|
235
244
|
compress(userId: string): Promise<CompressResult>;
|
|
236
245
|
maintain(userId: string): Promise<MaintenanceResult>;
|
|
237
246
|
setTier(memoryId: string, tier: MemoryTier): Promise<void>;
|
|
247
|
+
feedback(memoryId: string, helpful: boolean): Promise<void>;
|
|
248
|
+
quality(userId: string): Promise<MemoryQuality>;
|
|
238
249
|
/**
|
|
239
250
|
* Check the newly stored memory against existing similar memories
|
|
240
251
|
* for contradictions using the NLI model.
|
|
@@ -254,6 +265,15 @@ declare class SqliteMemoryEngine implements IMemoryEngine {
|
|
|
254
265
|
* Only runs for single store() calls, not storeBatch() (too slow for bulk).
|
|
255
266
|
*/
|
|
256
267
|
private checkDuplicates;
|
|
268
|
+
/**
|
|
269
|
+
* Auto-create 'extends' relations when a new memory shares keywords
|
|
270
|
+
* with existing memories. Builds graph density for future graph-based retrieval.
|
|
271
|
+
*
|
|
272
|
+
* Only runs for individual store() calls, not storeBatch().
|
|
273
|
+
* Requires at least 2 overlapping keywords to create a relation.
|
|
274
|
+
* Graceful: errors are silently swallowed.
|
|
275
|
+
*/
|
|
276
|
+
private autoRelate;
|
|
257
277
|
private ensureInitialized;
|
|
258
278
|
}
|
|
259
279
|
declare function createEngine(options: SqliteEngineOptions): SqliteMemoryEngine;
|
|
@@ -311,4 +331,4 @@ declare function determineTier(memory: {
|
|
|
311
331
|
*/
|
|
312
332
|
declare function estimateTokens(content: string): number;
|
|
313
333
|
|
|
314
|
-
export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, applyThreeFactorReranking, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, computeEbbinghaus, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, rrfMergeWeighted, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };
|
|
334
|
+
export { DEFAULT_TIER_CONFIG, type MemoryRow, type PreparedStatements, type RelationStatements, type SchemaOptions, type SearchSqlResult, type SqliteEngineOptions, SqliteMemoryEngine, type VectorSearchResult, applyKeywordBoost, applyThreeFactorReranking, bruteForceVectorSearch, buildFtsQuery, buildSearchSql, computeActivation, computeEbbinghaus, createEngine, createPreparedStatements, createRelation, createRelationStatements, deduplicateResults, deriveMemoryState, determineTier, ensureSchema, estimateTokens, extractQueryTokens, ftsSearch, generateMemoryId, getRelationsByMemory, keywordsToString, normalizeScores, nowIso, rowToMemory, rrfMerge, rrfMergeWeighted, searchMemories, serializeEmbedding, storeBatchAsync, storeBatchInTransaction, storeMemory, storeMemoryAsync, stringToKeywords, updateAccessTracking, vectorSearch };
|
package/dist/index.js
CHANGED
|
@@ -24,7 +24,9 @@ CREATE TABLE memories (
|
|
|
24
24
|
activation_score REAL DEFAULT 1.0,
|
|
25
25
|
access_count INTEGER DEFAULT 0,
|
|
26
26
|
last_accessed_at TEXT,
|
|
27
|
-
compressed_from TEXT
|
|
27
|
+
compressed_from TEXT,
|
|
28
|
+
use_count INTEGER DEFAULT 0,
|
|
29
|
+
success_count INTEGER DEFAULT 0
|
|
28
30
|
);
|
|
29
31
|
|
|
30
32
|
CREATE INDEX idx_memories_user_id ON memories(user_id);
|
|
@@ -72,6 +74,10 @@ CREATE TRIGGER memories_au AFTER UPDATE ON memories BEGIN
|
|
|
72
74
|
INSERT INTO memories_fts(rowid, content, keywords) VALUES (new.id, new.content, new.keywords);
|
|
73
75
|
END;
|
|
74
76
|
`;
|
|
77
|
+
var SCHEMA_V5 = `
|
|
78
|
+
ALTER TABLE memories ADD COLUMN use_count INTEGER DEFAULT 0;
|
|
79
|
+
ALTER TABLE memories ADD COLUMN success_count INTEGER DEFAULT 0;
|
|
80
|
+
`;
|
|
75
81
|
function schemaV2(dim) {
|
|
76
82
|
return `CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(embedding float[${dim}]);`;
|
|
77
83
|
}
|
|
@@ -106,7 +112,7 @@ function ensureSchema(db, options) {
|
|
|
106
112
|
db.exec(schemaV2(dim));
|
|
107
113
|
version = 2;
|
|
108
114
|
}
|
|
109
|
-
version =
|
|
115
|
+
version = 5;
|
|
110
116
|
db.prepare("INSERT INTO schema_version (version, embedding_dimension) VALUES (?, ?)").run(version, dim);
|
|
111
117
|
return;
|
|
112
118
|
}
|
|
@@ -132,6 +138,12 @@ function ensureSchema(db, options) {
|
|
|
132
138
|
}
|
|
133
139
|
db.prepare("UPDATE schema_version SET version = ?").run(4);
|
|
134
140
|
}
|
|
141
|
+
if (currentVersion < 5) {
|
|
142
|
+
if (currentVersion >= 1) {
|
|
143
|
+
db.exec(SCHEMA_V5);
|
|
144
|
+
}
|
|
145
|
+
db.prepare("UPDATE schema_version SET version = ?").run(5);
|
|
146
|
+
}
|
|
135
147
|
if (options?.vectorEnabled) {
|
|
136
148
|
const hasVecTable = db.prepare(
|
|
137
149
|
"SELECT name FROM sqlite_master WHERE type='table' AND name='vec_memories'"
|
|
@@ -183,6 +195,11 @@ function stringToKeywords(str) {
|
|
|
183
195
|
if (!str || str.trim() === "") return [];
|
|
184
196
|
return str.split(" ").filter((s) => s.length > 0);
|
|
185
197
|
}
|
|
198
|
+
function deriveMemoryState(memory) {
|
|
199
|
+
if (memory.invalidatedAt) return "invalidated";
|
|
200
|
+
if (!memory.isLatest) return "superseded";
|
|
201
|
+
return "current";
|
|
202
|
+
}
|
|
186
203
|
|
|
187
204
|
// src/mapper.ts
|
|
188
205
|
function rowToMemory(row) {
|
|
@@ -208,7 +225,9 @@ function rowToMemory(row) {
|
|
|
208
225
|
activationScore: row.activation_score ?? 1,
|
|
209
226
|
accessCount: row.access_count ?? 0,
|
|
210
227
|
...row.last_accessed_at != null ? { lastAccessedAt: row.last_accessed_at } : {},
|
|
211
|
-
...row.compressed_from != null ? { compressedFrom: row.compressed_from } : {}
|
|
228
|
+
...row.compressed_from != null ? { compressedFrom: row.compressed_from } : {},
|
|
229
|
+
useCount: row.use_count ?? 0,
|
|
230
|
+
successCount: row.success_count ?? 0
|
|
212
231
|
};
|
|
213
232
|
}
|
|
214
233
|
function deserializeEmbedding(buf) {
|
|
@@ -224,8 +243,8 @@ function serializeEmbedding(embedding) {
|
|
|
224
243
|
function createPreparedStatements(db) {
|
|
225
244
|
return {
|
|
226
245
|
insertMemory: db.prepare(`
|
|
227
|
-
INSERT INTO memories (memory_id, user_id, namespace, memory_type, content, raw_text, document_date, learned_at, source_id, confidence, salience, is_latest, embedding, keywords, event_date_start, event_date_end, invalidated_at, tier, activation_score, access_count, last_accessed_at, compressed_from)
|
|
228
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
246
|
+
INSERT INTO memories (memory_id, user_id, namespace, memory_type, content, raw_text, document_date, learned_at, source_id, confidence, salience, is_latest, embedding, keywords, event_date_start, event_date_end, invalidated_at, tier, activation_score, access_count, last_accessed_at, compressed_from, use_count, success_count)
|
|
247
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
229
248
|
`),
|
|
230
249
|
getById: db.prepare("SELECT * FROM memories WHERE id = ?"),
|
|
231
250
|
getByMemoryId: db.prepare("SELECT * FROM memories WHERE memory_id = ?"),
|
|
@@ -264,8 +283,12 @@ function storeMemory(db, stmts, input) {
|
|
|
264
283
|
// access_count
|
|
265
284
|
null,
|
|
266
285
|
// last_accessed_at
|
|
267
|
-
null
|
|
286
|
+
null,
|
|
268
287
|
// compressed_from
|
|
288
|
+
0,
|
|
289
|
+
// use_count
|
|
290
|
+
0
|
|
291
|
+
// success_count
|
|
269
292
|
);
|
|
270
293
|
const row = stmts.getByMemoryId.get(memoryId);
|
|
271
294
|
return rowToMemory(row);
|
|
@@ -307,8 +330,12 @@ async function storeMemoryAsync(db, stmts, input, embedder) {
|
|
|
307
330
|
// access_count
|
|
308
331
|
null,
|
|
309
332
|
// last_accessed_at
|
|
310
|
-
null
|
|
333
|
+
null,
|
|
311
334
|
// compressed_from
|
|
335
|
+
0,
|
|
336
|
+
// use_count
|
|
337
|
+
0
|
|
338
|
+
// success_count
|
|
312
339
|
);
|
|
313
340
|
if (embeddingVec && info.lastInsertRowid) {
|
|
314
341
|
const rowid = Number(info.lastInsertRowid);
|
|
@@ -374,8 +401,12 @@ async function storeBatchAsync(db, stmts, inputs, embedder) {
|
|
|
374
401
|
// access_count
|
|
375
402
|
null,
|
|
376
403
|
// last_accessed_at
|
|
377
|
-
null
|
|
404
|
+
null,
|
|
378
405
|
// compressed_from
|
|
406
|
+
0,
|
|
407
|
+
// use_count
|
|
408
|
+
0
|
|
409
|
+
// success_count
|
|
379
410
|
);
|
|
380
411
|
if (info.lastInsertRowid) {
|
|
381
412
|
db.prepare("INSERT INTO vec_memories(rowid, embedding) VALUES (CAST(? AS INTEGER), ?)").run(
|
|
@@ -608,9 +639,29 @@ function buildSearchSql(query) {
|
|
|
608
639
|
whereClauses.push("m.confidence >= ?");
|
|
609
640
|
params.push(filters.minConfidence);
|
|
610
641
|
}
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
642
|
+
if (filters?.states && filters.states.length > 0) {
|
|
643
|
+
const stateConditions = [];
|
|
644
|
+
if (filters.states.includes("current")) {
|
|
645
|
+
stateConditions.push("(m.is_latest = 1 AND m.invalidated_at IS NULL)");
|
|
646
|
+
}
|
|
647
|
+
if (filters.states.includes("superseded")) {
|
|
648
|
+
stateConditions.push("(m.is_latest = 0 AND m.invalidated_at IS NULL)");
|
|
649
|
+
}
|
|
650
|
+
if (filters.states.includes("invalidated")) {
|
|
651
|
+
stateConditions.push("(m.invalidated_at IS NOT NULL)");
|
|
652
|
+
}
|
|
653
|
+
if (stateConditions.length > 0) {
|
|
654
|
+
whereClauses.push(`(${stateConditions.join(" OR ")})`);
|
|
655
|
+
}
|
|
656
|
+
} else {
|
|
657
|
+
const onlyLatest = filters?.onlyLatest ?? true;
|
|
658
|
+
if (onlyLatest) {
|
|
659
|
+
whereClauses.push("m.is_latest = 1");
|
|
660
|
+
}
|
|
661
|
+
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
662
|
+
if (excludeInvalidated) {
|
|
663
|
+
whereClauses.push("m.invalidated_at IS NULL");
|
|
664
|
+
}
|
|
614
665
|
}
|
|
615
666
|
if (filters?.eventDateRange?.start) {
|
|
616
667
|
whereClauses.push("m.event_date_start >= ?");
|
|
@@ -620,10 +671,6 @@ function buildSearchSql(query) {
|
|
|
620
671
|
whereClauses.push("m.event_date_end <= ?");
|
|
621
672
|
params.push(filters.eventDateRange.end);
|
|
622
673
|
}
|
|
623
|
-
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
624
|
-
if (excludeInvalidated) {
|
|
625
|
-
whereClauses.push("m.invalidated_at IS NULL");
|
|
626
|
-
}
|
|
627
674
|
const limit = query.limit ?? DEFAULT_LIMIT;
|
|
628
675
|
params.push(limit);
|
|
629
676
|
const sql = [
|
|
@@ -676,9 +723,29 @@ function ftsSearch(db, query) {
|
|
|
676
723
|
function bruteForceVectorSearch(db, queryVec, userId, limit, filters) {
|
|
677
724
|
const whereClauses = ["user_id = ?", "embedding IS NOT NULL"];
|
|
678
725
|
const params = [userId];
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
726
|
+
if (filters?.states && filters.states.length > 0) {
|
|
727
|
+
const stateConditions = [];
|
|
728
|
+
if (filters.states.includes("current")) {
|
|
729
|
+
stateConditions.push("(is_latest = 1 AND invalidated_at IS NULL)");
|
|
730
|
+
}
|
|
731
|
+
if (filters.states.includes("superseded")) {
|
|
732
|
+
stateConditions.push("(is_latest = 0 AND invalidated_at IS NULL)");
|
|
733
|
+
}
|
|
734
|
+
if (filters.states.includes("invalidated")) {
|
|
735
|
+
stateConditions.push("(invalidated_at IS NOT NULL)");
|
|
736
|
+
}
|
|
737
|
+
if (stateConditions.length > 0) {
|
|
738
|
+
whereClauses.push(`(${stateConditions.join(" OR ")})`);
|
|
739
|
+
}
|
|
740
|
+
} else {
|
|
741
|
+
const onlyLatest = filters?.onlyLatest ?? true;
|
|
742
|
+
if (onlyLatest) {
|
|
743
|
+
whereClauses.push("is_latest = 1");
|
|
744
|
+
}
|
|
745
|
+
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
746
|
+
if (excludeInvalidated) {
|
|
747
|
+
whereClauses.push("invalidated_at IS NULL");
|
|
748
|
+
}
|
|
682
749
|
}
|
|
683
750
|
if (filters?.memoryTypes && filters.memoryTypes.length > 0) {
|
|
684
751
|
const mtPlaceholders = filters.memoryTypes.map(() => "?").join(",");
|
|
@@ -707,10 +774,6 @@ function bruteForceVectorSearch(db, queryVec, userId, limit, filters) {
|
|
|
707
774
|
whereClauses.push("event_date_end <= ?");
|
|
708
775
|
params.push(filters.eventDateRange.end);
|
|
709
776
|
}
|
|
710
|
-
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
711
|
-
if (excludeInvalidated) {
|
|
712
|
-
whereClauses.push("invalidated_at IS NULL");
|
|
713
|
-
}
|
|
714
777
|
const sql = `SELECT * FROM memories WHERE ${whereClauses.join(" AND ")}`;
|
|
715
778
|
const rows = db.prepare(sql).all(...params);
|
|
716
779
|
if (rows.length === 0) {
|
|
@@ -748,9 +811,29 @@ function vectorSearch(db, queryVec, userId, limit, filters, useVecTable = true)
|
|
|
748
811
|
const placeholders = rowids.map(() => "?").join(",");
|
|
749
812
|
let sql = `SELECT * FROM memories WHERE id IN (${placeholders}) AND user_id = ?`;
|
|
750
813
|
const params = [...rowids, userId];
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
814
|
+
if (filters?.states && filters.states.length > 0) {
|
|
815
|
+
const stateConditions = [];
|
|
816
|
+
if (filters.states.includes("current")) {
|
|
817
|
+
stateConditions.push("(is_latest = 1 AND invalidated_at IS NULL)");
|
|
818
|
+
}
|
|
819
|
+
if (filters.states.includes("superseded")) {
|
|
820
|
+
stateConditions.push("(is_latest = 0 AND invalidated_at IS NULL)");
|
|
821
|
+
}
|
|
822
|
+
if (filters.states.includes("invalidated")) {
|
|
823
|
+
stateConditions.push("(invalidated_at IS NOT NULL)");
|
|
824
|
+
}
|
|
825
|
+
if (stateConditions.length > 0) {
|
|
826
|
+
sql += ` AND (${stateConditions.join(" OR ")})`;
|
|
827
|
+
}
|
|
828
|
+
} else {
|
|
829
|
+
const onlyLatest = filters?.onlyLatest ?? true;
|
|
830
|
+
if (onlyLatest) {
|
|
831
|
+
sql += " AND is_latest = 1";
|
|
832
|
+
}
|
|
833
|
+
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
834
|
+
if (excludeInvalidated) {
|
|
835
|
+
sql += " AND invalidated_at IS NULL";
|
|
836
|
+
}
|
|
754
837
|
}
|
|
755
838
|
if (filters?.memoryTypes && filters.memoryTypes.length > 0) {
|
|
756
839
|
const mtPlaceholders = filters.memoryTypes.map(() => "?").join(",");
|
|
@@ -779,10 +862,6 @@ function vectorSearch(db, queryVec, userId, limit, filters, useVecTable = true)
|
|
|
779
862
|
sql += " AND event_date_end <= ?";
|
|
780
863
|
params.push(filters.eventDateRange.end);
|
|
781
864
|
}
|
|
782
|
-
const excludeInvalidated = filters?.excludeInvalidated ?? true;
|
|
783
|
-
if (excludeInvalidated) {
|
|
784
|
-
sql += " AND invalidated_at IS NULL";
|
|
785
|
-
}
|
|
786
865
|
const rows = db.prepare(sql).all(...params);
|
|
787
866
|
return rows.map((row) => ({
|
|
788
867
|
memory: rowToMemory(row),
|
|
@@ -950,7 +1029,7 @@ function updateAccessTracking(db, memoryIds) {
|
|
|
950
1029
|
});
|
|
951
1030
|
updateAll(memoryIds);
|
|
952
1031
|
}
|
|
953
|
-
function searchMemories(db, query, queryVec, useVecTable = true) {
|
|
1032
|
+
function searchMemories(db, query, queryVec, useVecTable = true, skipAccessTracking = false) {
|
|
954
1033
|
const startTime = performance.now();
|
|
955
1034
|
const queryTokens = extractQueryTokens(query.query);
|
|
956
1035
|
const ftsResults = ftsSearch(db, query);
|
|
@@ -959,7 +1038,9 @@ function searchMemories(db, query, queryVec, useVecTable = true) {
|
|
|
959
1038
|
const weighted = applyThreeFactorReranking(ftsResults);
|
|
960
1039
|
const boosted = applyKeywordBoost(weighted, queryTokens);
|
|
961
1040
|
finalResults = deduplicateResults(boosted);
|
|
962
|
-
|
|
1041
|
+
if (!skipAccessTracking) {
|
|
1042
|
+
updateAccessTracking(db, finalResults.map((r) => r.memory.memoryId));
|
|
1043
|
+
}
|
|
963
1044
|
return {
|
|
964
1045
|
results: finalResults,
|
|
965
1046
|
totalCount: finalResults.length,
|
|
@@ -1088,19 +1169,6 @@ function estimateTokens(content) {
|
|
|
1088
1169
|
}
|
|
1089
1170
|
|
|
1090
1171
|
// src/engine.ts
|
|
1091
|
-
function cosineSim(a, b) {
|
|
1092
|
-
let dot = 0;
|
|
1093
|
-
let normA = 0;
|
|
1094
|
-
let normB = 0;
|
|
1095
|
-
const len = Math.min(a.length, b.length);
|
|
1096
|
-
for (let i = 0; i < len; i++) {
|
|
1097
|
-
dot += a[i] * b[i];
|
|
1098
|
-
normA += a[i] * a[i];
|
|
1099
|
-
normB += b[i] * b[i];
|
|
1100
|
-
}
|
|
1101
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
1102
|
-
return denom === 0 ? 0 : dot / denom;
|
|
1103
|
-
}
|
|
1104
1172
|
var SqliteMemoryEngine = class {
|
|
1105
1173
|
db = null;
|
|
1106
1174
|
stmts = null;
|
|
@@ -1156,6 +1224,7 @@ var SqliteMemoryEngine = class {
|
|
|
1156
1224
|
await this.checkContradictions(memory);
|
|
1157
1225
|
}
|
|
1158
1226
|
await this.checkDuplicates(memory);
|
|
1227
|
+
await this.autoRelate(memory);
|
|
1159
1228
|
return memory;
|
|
1160
1229
|
}
|
|
1161
1230
|
async storeBatch(inputs) {
|
|
@@ -1166,10 +1235,22 @@ var SqliteMemoryEngine = class {
|
|
|
1166
1235
|
} else {
|
|
1167
1236
|
memories = storeBatchInTransaction(this.db, this.stmts, inputs);
|
|
1168
1237
|
}
|
|
1169
|
-
if (
|
|
1238
|
+
if (memories.length <= 50) {
|
|
1170
1239
|
for (const memory of memories) {
|
|
1240
|
+
if (this.options.embedder && this.options.contradictionDetector) {
|
|
1241
|
+
try {
|
|
1242
|
+
await this.checkContradictions(memory);
|
|
1243
|
+
} catch {
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
if (this.options.embedder) {
|
|
1247
|
+
try {
|
|
1248
|
+
await this.checkDuplicates(memory);
|
|
1249
|
+
} catch {
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1171
1252
|
try {
|
|
1172
|
-
await this.
|
|
1253
|
+
await this.autoRelate(memory);
|
|
1173
1254
|
} catch {
|
|
1174
1255
|
}
|
|
1175
1256
|
}
|
|
@@ -1295,6 +1376,7 @@ var SqliteMemoryEngine = class {
|
|
|
1295
1376
|
const coldMemories = db.prepare(`
|
|
1296
1377
|
SELECT * FROM memories
|
|
1297
1378
|
WHERE user_id = ? AND tier = 'cold' AND activation_score < 0.1 AND is_latest = 1
|
|
1379
|
+
AND invalidated_at IS NULL
|
|
1298
1380
|
ORDER BY namespace, learned_at
|
|
1299
1381
|
`).all(userId);
|
|
1300
1382
|
if (coldMemories.length === 0) return { compressed: 0, removed: 0 };
|
|
@@ -1355,8 +1437,12 @@ var SqliteMemoryEngine = class {
|
|
|
1355
1437
|
// access_count
|
|
1356
1438
|
null,
|
|
1357
1439
|
// last_accessed_at
|
|
1358
|
-
rows[0].memory_id
|
|
1440
|
+
rows[0].memory_id,
|
|
1359
1441
|
// compressed_from
|
|
1442
|
+
0,
|
|
1443
|
+
// use_count
|
|
1444
|
+
0
|
|
1445
|
+
// success_count
|
|
1360
1446
|
);
|
|
1361
1447
|
compressed++;
|
|
1362
1448
|
for (const row of rows) {
|
|
@@ -1424,6 +1510,64 @@ var SqliteMemoryEngine = class {
|
|
|
1424
1510
|
this.ensureInitialized();
|
|
1425
1511
|
this.db.prepare("UPDATE memories SET tier = ? WHERE memory_id = ?").run(tier, memoryId);
|
|
1426
1512
|
}
|
|
1513
|
+
async feedback(memoryId, helpful) {
|
|
1514
|
+
this.ensureInitialized();
|
|
1515
|
+
const db = this.db;
|
|
1516
|
+
db.transaction(() => {
|
|
1517
|
+
if (helpful) {
|
|
1518
|
+
db.prepare(
|
|
1519
|
+
"UPDATE memories SET use_count = use_count + 1, success_count = success_count + 1 WHERE memory_id = ?"
|
|
1520
|
+
).run(memoryId);
|
|
1521
|
+
} else {
|
|
1522
|
+
db.prepare(
|
|
1523
|
+
"UPDATE memories SET use_count = use_count + 1 WHERE memory_id = ?"
|
|
1524
|
+
).run(memoryId);
|
|
1525
|
+
}
|
|
1526
|
+
const row = db.prepare(
|
|
1527
|
+
"SELECT salience, use_count, success_count FROM memories WHERE memory_id = ?"
|
|
1528
|
+
).get(memoryId);
|
|
1529
|
+
if (row && row.use_count > 0) {
|
|
1530
|
+
const successRate = row.success_count / row.use_count;
|
|
1531
|
+
const newSalience = Math.min(1, Math.max(0.1, 0.5 + 0.5 * successRate));
|
|
1532
|
+
db.prepare("UPDATE memories SET salience = ? WHERE memory_id = ?").run(newSalience, memoryId);
|
|
1533
|
+
}
|
|
1534
|
+
})();
|
|
1535
|
+
}
|
|
1536
|
+
async quality(userId) {
|
|
1537
|
+
this.ensureInitialized();
|
|
1538
|
+
const db = this.db;
|
|
1539
|
+
const total = db.prepare("SELECT COUNT(*) as c FROM memories WHERE user_id = ?").get(userId).c;
|
|
1540
|
+
const fresh = db.prepare(
|
|
1541
|
+
"SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND is_latest = 1 AND invalidated_at IS NULL"
|
|
1542
|
+
).get(userId).c;
|
|
1543
|
+
const invalidated = db.prepare(
|
|
1544
|
+
"SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND invalidated_at IS NOT NULL"
|
|
1545
|
+
).get(userId).c;
|
|
1546
|
+
const superseded = db.prepare(
|
|
1547
|
+
"SELECT COUNT(*) as c FROM memories WHERE user_id = ? AND is_latest = 0"
|
|
1548
|
+
).get(userId).c;
|
|
1549
|
+
const withRelations = db.prepare(`
|
|
1550
|
+
SELECT COUNT(DISTINCT mid) as c FROM (
|
|
1551
|
+
SELECT src_memory_id as mid FROM memory_relations
|
|
1552
|
+
WHERE src_memory_id IN (SELECT memory_id FROM memories WHERE user_id = ?)
|
|
1553
|
+
UNION
|
|
1554
|
+
SELECT dst_memory_id as mid FROM memory_relations
|
|
1555
|
+
WHERE dst_memory_id IN (SELECT memory_id FROM memories WHERE user_id = ?)
|
|
1556
|
+
)
|
|
1557
|
+
`).get(userId, userId).c;
|
|
1558
|
+
const avgRow = db.prepare(
|
|
1559
|
+
"SELECT AVG(activation_score) as avg FROM memories WHERE user_id = ? AND is_latest = 1"
|
|
1560
|
+
).get(userId);
|
|
1561
|
+
const avgActivation = avgRow.avg ?? 0;
|
|
1562
|
+
return {
|
|
1563
|
+
total,
|
|
1564
|
+
fresh,
|
|
1565
|
+
invalidated,
|
|
1566
|
+
superseded,
|
|
1567
|
+
withRelations,
|
|
1568
|
+
avgActivation
|
|
1569
|
+
};
|
|
1570
|
+
}
|
|
1427
1571
|
/**
|
|
1428
1572
|
* Check the newly stored memory against existing similar memories
|
|
1429
1573
|
* for contradictions using the NLI model.
|
|
@@ -1449,7 +1593,9 @@ var SqliteMemoryEngine = class {
|
|
|
1449
1593
|
filters: { onlyLatest: true }
|
|
1450
1594
|
},
|
|
1451
1595
|
queryVec,
|
|
1452
|
-
this.vectorEnabled
|
|
1596
|
+
this.vectorEnabled,
|
|
1597
|
+
true
|
|
1598
|
+
// skipAccessTracking
|
|
1453
1599
|
);
|
|
1454
1600
|
for (const result of similar.results) {
|
|
1455
1601
|
if (result.memory.memoryId === newMemory.memoryId) continue;
|
|
@@ -1483,27 +1629,21 @@ var SqliteMemoryEngine = class {
|
|
|
1483
1629
|
if (!this.options.embedder) return;
|
|
1484
1630
|
try {
|
|
1485
1631
|
const queryVec = await this.options.embedder.embed(newMemory.content);
|
|
1486
|
-
const
|
|
1632
|
+
const candidates = bruteForceVectorSearch(
|
|
1487
1633
|
this.db,
|
|
1488
|
-
{
|
|
1489
|
-
userId: newMemory.userId,
|
|
1490
|
-
query: newMemory.content,
|
|
1491
|
-
limit: 5,
|
|
1492
|
-
filters: { onlyLatest: true }
|
|
1493
|
-
},
|
|
1494
1634
|
queryVec,
|
|
1495
|
-
|
|
1635
|
+
newMemory.userId,
|
|
1636
|
+
10,
|
|
1637
|
+
{ onlyLatest: true }
|
|
1496
1638
|
);
|
|
1497
|
-
for (const
|
|
1498
|
-
if (
|
|
1499
|
-
|
|
1500
|
-
const embB = new Float32Array(result.memory.embedding);
|
|
1501
|
-
const similarity = cosineSim(queryVec, embB);
|
|
1639
|
+
for (const candidate of candidates) {
|
|
1640
|
+
if (candidate.memory.memoryId === newMemory.memoryId) continue;
|
|
1641
|
+
const similarity = 1 - candidate.distance;
|
|
1502
1642
|
if (similarity > 0.95) {
|
|
1503
1643
|
try {
|
|
1504
1644
|
await this.relate(
|
|
1505
1645
|
newMemory.memoryId,
|
|
1506
|
-
|
|
1646
|
+
candidate.memory.memoryId,
|
|
1507
1647
|
"updates",
|
|
1508
1648
|
`Auto-detected duplicate: cosine similarity ${similarity.toFixed(3)}`
|
|
1509
1649
|
);
|
|
@@ -1514,6 +1654,55 @@ var SqliteMemoryEngine = class {
|
|
|
1514
1654
|
} catch {
|
|
1515
1655
|
}
|
|
1516
1656
|
}
|
|
1657
|
+
/**
|
|
1658
|
+
* Auto-create 'extends' relations when a new memory shares keywords
|
|
1659
|
+
* with existing memories. Builds graph density for future graph-based retrieval.
|
|
1660
|
+
*
|
|
1661
|
+
* Only runs for individual store() calls, not storeBatch().
|
|
1662
|
+
* Requires at least 2 overlapping keywords to create a relation.
|
|
1663
|
+
* Graceful: errors are silently swallowed.
|
|
1664
|
+
*/
|
|
1665
|
+
async autoRelate(newMemory) {
|
|
1666
|
+
if (!newMemory.keywords || newMemory.keywords.length === 0) return;
|
|
1667
|
+
try {
|
|
1668
|
+
const keywordList = newMemory.keywords;
|
|
1669
|
+
const existing = this.db.prepare(`
|
|
1670
|
+
SELECT memory_id, keywords FROM memories
|
|
1671
|
+
WHERE user_id = ? AND is_latest = 1 AND memory_id != ?
|
|
1672
|
+
AND invalidated_at IS NULL
|
|
1673
|
+
ORDER BY learned_at DESC LIMIT 10
|
|
1674
|
+
`).all(newMemory.userId, newMemory.memoryId);
|
|
1675
|
+
for (const row of existing) {
|
|
1676
|
+
if (!row.keywords) continue;
|
|
1677
|
+
const existingKeywords = row.keywords.split(" ").map((k) => k.trim().toLowerCase());
|
|
1678
|
+
const overlap = keywordList.filter((k) => existingKeywords.includes(k.toLowerCase()));
|
|
1679
|
+
if (overlap.length >= 3) {
|
|
1680
|
+
const existingRelation = this.db.prepare(
|
|
1681
|
+
`SELECT 1 FROM memory_relations
|
|
1682
|
+
WHERE (src_memory_id = ? AND dst_memory_id = ?)
|
|
1683
|
+
OR (src_memory_id = ? AND dst_memory_id = ?)
|
|
1684
|
+
LIMIT 1`
|
|
1685
|
+
).get(
|
|
1686
|
+
newMemory.memoryId,
|
|
1687
|
+
row.memory_id,
|
|
1688
|
+
row.memory_id,
|
|
1689
|
+
newMemory.memoryId
|
|
1690
|
+
);
|
|
1691
|
+
if (existingRelation) continue;
|
|
1692
|
+
try {
|
|
1693
|
+
await this.relate(
|
|
1694
|
+
newMemory.memoryId,
|
|
1695
|
+
row.memory_id,
|
|
1696
|
+
"extends",
|
|
1697
|
+
`Auto: ${overlap.length} shared keywords (${overlap.join(", ")})`
|
|
1698
|
+
);
|
|
1699
|
+
} catch {
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
} catch {
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1517
1706
|
ensureInitialized() {
|
|
1518
1707
|
if (!this.db || !this.stmts || !this.relStmts) {
|
|
1519
1708
|
throw new Error("Engine not initialized. Call initialize() first.");
|
|
@@ -1538,6 +1727,7 @@ export {
|
|
|
1538
1727
|
createRelation,
|
|
1539
1728
|
createRelationStatements,
|
|
1540
1729
|
deduplicateResults,
|
|
1730
|
+
deriveMemoryState,
|
|
1541
1731
|
determineTier,
|
|
1542
1732
|
ensureSchema,
|
|
1543
1733
|
estimateTokens,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memrosetta/core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.21",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -20,10 +20,10 @@
|
|
|
20
20
|
"better-sqlite3": "^11.0.0",
|
|
21
21
|
"nanoid": "^5.0.0",
|
|
22
22
|
"sqlite-vec": "^0.1.7",
|
|
23
|
-
"@memrosetta/types": "0.2.
|
|
23
|
+
"@memrosetta/types": "0.2.21"
|
|
24
24
|
},
|
|
25
25
|
"optionalDependencies": {
|
|
26
|
-
"@memrosetta/embeddings": "0.2.
|
|
26
|
+
"@memrosetta/embeddings": "0.2.21"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
29
|
"@types/better-sqlite3": "^7.6.0",
|