@monoes/memory 1.0.10 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -11
- package/dist/controller-registry.d.ts +7 -5
- package/dist/controller-registry.d.ts.map +1 -1
- package/dist/controller-registry.js +8 -6
- package/dist/controller-registry.js.map +1 -1
- package/dist/index.d.ts +38 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +39 -12
- package/dist/index.js.map +1 -1
- package/dist/migration.d.ts +3 -2
- package/dist/migration.d.ts.map +1 -1
- package/dist/migration.js +3 -2
- package/dist/migration.js.map +1 -1
- package/dist/sql-backend.d.ts +81 -0
- package/dist/sql-backend.d.ts.map +1 -0
- package/dist/sql-backend.js +553 -0
- package/dist/sql-backend.js.map +1 -0
- package/dist/sql-driver.d.ts +135 -0
- package/dist/sql-driver.d.ts.map +1 -0
- package/dist/sql-driver.js +187 -0
- package/dist/sql-driver.js.map +1 -0
- package/dist/sql-schema.d.ts +74 -0
- package/dist/sql-schema.d.ts.map +1 -0
- package/dist/sql-schema.js +178 -0
- package/dist/sql-schema.js.map +1 -0
- package/dist/sqlite-backend.d.ts +16 -106
- package/dist/sqlite-backend.d.ts.map +1 -1
- package/dist/sqlite-backend.js +32 -726
- package/dist/sqlite-backend.js.map +1 -1
- package/dist/sqljs-backend.d.ts +26 -98
- package/dist/sqljs-backend.d.ts.map +1 -1
- package/dist/sqljs-backend.js +75 -609
- package/dist/sqljs-backend.js.map +1 -1
- package/dist/types.d.ts +5 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +5 -2
- package/dist/types.js.map +1 -1
- package/package.json +5 -4
- package/dist/lancedb-backend.d.ts +0 -79
- package/dist/lancedb-backend.d.ts.map +0 -1
- package/dist/lancedb-backend.js +0 -567
- package/dist/lancedb-backend.js.map +0 -1
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified SQL memory backend.
|
|
3
|
+
*
|
|
4
|
+
* One IMemoryBackend implementation over a pluggable SqlDriver, replacing the
|
|
5
|
+
* two near-parallel implementations (better-sqlite3 and sql.js) that together
|
|
6
|
+
* ran to ~1,780 lines. They had already drifted in ways users could observe:
|
|
7
|
+
* different schemas, different tag-filter semantics (ANY vs ALL for the same
|
|
8
|
+
* call), tag validation on one and not the other, and a crash on the sql.js
|
|
9
|
+
* side when reading a missing row. Sharing the logic removes that class of bug
|
|
10
|
+
* rather than testing for it after the fact.
|
|
11
|
+
*
|
|
12
|
+
* Driver differences (cursors, transactions, persistence, pragmas) live in
|
|
13
|
+
* sql-driver.ts; the canonical schema and the legacy-data migration live in
|
|
14
|
+
* sql-schema.ts. This file contains only behaviour.
|
|
15
|
+
*
|
|
16
|
+
* @module v1/memory/sql-backend
|
|
17
|
+
*/
|
|
18
|
+
import { EventEmitter } from 'node:events';
|
|
19
|
+
import type { SqlDriver } from './sql-driver.js';
|
|
20
|
+
import { type MigrationReport } from './sql-schema.js';
|
|
21
|
+
import { IMemoryBackend, MemoryEntry, MemoryEntryUpdate, MemoryQuery, SearchOptions, SearchResult, BackendStats, HealthCheckResult, EmbeddingGenerator } from './types.js';
|
|
22
|
+
export interface SqlBackendConfig {
|
|
23
|
+
/** Default namespace applied when an entry omits one. */
|
|
24
|
+
defaultNamespace: string;
|
|
25
|
+
/** Embedding generator, for callers that store text and want vectors. */
|
|
26
|
+
embeddingGenerator?: EmbeddingGenerator;
|
|
27
|
+
/** Soft cap used by healthCheck to report utilization. */
|
|
28
|
+
maxEntries: number;
|
|
29
|
+
verbose: boolean;
|
|
30
|
+
}
|
|
31
|
+
export declare class SqlBackend extends EventEmitter implements IMemoryBackend {
|
|
32
|
+
protected config: SqlBackendConfig;
|
|
33
|
+
protected driver: SqlDriver | null;
|
|
34
|
+
protected initialized: boolean;
|
|
35
|
+
/** Populated during initialize(); surfaced for diagnostics. */
|
|
36
|
+
migrationReport: MigrationReport | null;
|
|
37
|
+
private stats;
|
|
38
|
+
/** Debounce counter: the agent_reads purge is expensive, so it is amortised. */
|
|
39
|
+
private _readCount;
|
|
40
|
+
constructor(config?: Partial<SqlBackendConfig>);
|
|
41
|
+
/** Subclasses open their driver here. */
|
|
42
|
+
protected openDriver(): Promise<SqlDriver>;
|
|
43
|
+
initialize(): Promise<void>;
|
|
44
|
+
shutdown(): Promise<void>;
|
|
45
|
+
store(entry: MemoryEntry): Promise<void>;
|
|
46
|
+
/** Synchronous store body, shared by store() and bulkInsert(). */
|
|
47
|
+
private storeSync;
|
|
48
|
+
bulkInsert(entries: MemoryEntry[]): Promise<void>;
|
|
49
|
+
update(id: string, update: MemoryEntryUpdate): Promise<MemoryEntry | null>;
|
|
50
|
+
delete(id: string): Promise<boolean>;
|
|
51
|
+
bulkDelete(ids: string[]): Promise<number>;
|
|
52
|
+
clearNamespace(namespace: string): Promise<number>;
|
|
53
|
+
get(id: string, agentId?: string): Promise<MemoryEntry | null>;
|
|
54
|
+
/**
|
|
55
|
+
* Promote an entry to 'team' once 3+ distinct agents have read it within 24h.
|
|
56
|
+
* https://arxiv.org/abs/2505.18279
|
|
57
|
+
*/
|
|
58
|
+
private checkAndPromoteEntry;
|
|
59
|
+
getByKey(namespace: string, key: string): Promise<MemoryEntry | null>;
|
|
60
|
+
query(query: MemoryQuery): Promise<MemoryEntry[]>;
|
|
61
|
+
/**
|
|
62
|
+
* Semantic search — brute-force cosine over stored embeddings, namespace- and
|
|
63
|
+
* TTL-filtered in SQL. At second-brain scale (10^3–10^5 vectors of 384 dims)
|
|
64
|
+
* this is a few tens of ms; swap in an ANN index if a profiled workload
|
|
65
|
+
* outgrows it.
|
|
66
|
+
*/
|
|
67
|
+
search(embedding: Float32Array, options: SearchOptions): Promise<SearchResult[]>;
|
|
68
|
+
count(namespace?: string): Promise<number>;
|
|
69
|
+
listNamespaces(): Promise<string[]>;
|
|
70
|
+
getStats(): Promise<BackendStats>;
|
|
71
|
+
healthCheck(): Promise<HealthCheckResult>;
|
|
72
|
+
/** Flush to durable storage. No-op on write-through drivers. */
|
|
73
|
+
persist(): Promise<void>;
|
|
74
|
+
private static readonly TAG_RE;
|
|
75
|
+
private static readonly SRC_TAG_RE;
|
|
76
|
+
private static readonly MAX_TAG_LEN;
|
|
77
|
+
protected validateTags(tags: string[]): void;
|
|
78
|
+
protected ensureInitialized(): void;
|
|
79
|
+
private rowToEntry;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=sql-backend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sql-backend.d.ts","sourceRoot":"","sources":["../src/sql-backend.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,KAAK,EAAE,SAAS,EAAY,MAAM,iBAAiB,CAAC;AAC3D,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACzE,OAAO,EACL,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,iBAAiB,EAGjB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,gBAAgB;IAC/B,yDAAyD;IACzD,gBAAgB,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;CAClB;AAWD,qBAAa,UAAW,SAAQ,YAAa,YAAW,cAAc;IACpE,SAAS,CAAC,MAAM,EAAE,gBAAgB,CAAC;IACnC,SAAS,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAAQ;IAC1C,SAAS,CAAC,WAAW,UAAS;IAC9B,+DAA+D;IAC/D,eAAe,EAAE,eAAe,GAAG,IAAI,CAAQ;IAE/C,OAAO,CAAC,KAAK,CAA0E;IACvF,gFAAgF;IAChF,OAAO,CAAC,UAAU,CAAK;IAEvB,YAAY,MAAM,GAAE,OAAO,CAAC,gBAAgB,CAAM,EAGjD;IAED,yCAAyC;IACzC,UAAgB,UAAU,IAAI,OAAO,CAAC,SAAS,CAAC,CAE/C;IAEK,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAsBhC;IAEK,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAY9B;IAIK,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAW7C;IAED,kEAAkE;IAClE,OAAO,CAAC,SAAS;IAuDX,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAOtD;IAEK,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAkB/E;IAEK,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAazC;IAEK,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAc/C;IAEK,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAevD;IAIK,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAwBnE;IAED;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAgBtB,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAW1E;IAEK,KAAK,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA+DtD;IAED;;;;;OAKG;IACG,MAAM,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CA0BrF;IAEK,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAM/C;IAEK,cAAc,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAKxC;IAIK,QAAQ,IAAI,OAAO,CAAC,YAAY,CAAC,CAiCtC;IAEK,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAkE9C;IAED,gEAAgE;IAC1D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAG7B;IAQD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAA4B;IAC1D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAC/D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAO;IAE1C,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAQ3C;IAED,SAAS,CAAC,iBAAiB,IAAI,IAAI,CAIlC;IAED,OAAO,CAAC,UAAU;CAiCnB"}
|
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified SQL memory backend.
|
|
3
|
+
*
|
|
4
|
+
* One IMemoryBackend implementation over a pluggable SqlDriver, replacing the
|
|
5
|
+
* two near-parallel implementations (better-sqlite3 and sql.js) that together
|
|
6
|
+
* ran to ~1,780 lines. They had already drifted in ways users could observe:
|
|
7
|
+
* different schemas, different tag-filter semantics (ANY vs ALL for the same
|
|
8
|
+
* call), tag validation on one and not the other, and a crash on the sql.js
|
|
9
|
+
* side when reading a missing row. Sharing the logic removes that class of bug
|
|
10
|
+
* rather than testing for it after the fact.
|
|
11
|
+
*
|
|
12
|
+
* Driver differences (cursors, transactions, persistence, pragmas) live in
|
|
13
|
+
* sql-driver.ts; the canonical schema and the legacy-data migration live in
|
|
14
|
+
* sql-schema.ts. This file contains only behaviour.
|
|
15
|
+
*
|
|
16
|
+
* @module v1/memory/sql-backend
|
|
17
|
+
*/
|
|
18
|
+
import { EventEmitter } from 'node:events';
|
|
19
|
+
import { cosineSimilarity } from './math-utils.js';
|
|
20
|
+
import { initializeSchema } from './sql-schema.js';
|
|
21
|
+
const DEFAULT_CONFIG = {
|
|
22
|
+
defaultNamespace: 'default',
|
|
23
|
+
maxEntries: 1_000_000,
|
|
24
|
+
verbose: false,
|
|
25
|
+
};
|
|
26
|
+
/** Cap on votes/rows pulled in one go, guarding against unbounded memory use. */
|
|
27
|
+
const MAX_QUERY_LIMIT = 10_000;
|
|
28
|
+
export class SqlBackend extends EventEmitter {
|
|
29
|
+
config;
|
|
30
|
+
driver = null;
|
|
31
|
+
initialized = false;
|
|
32
|
+
/** Populated during initialize(); surfaced for diagnostics. */
|
|
33
|
+
migrationReport = null;
|
|
34
|
+
stats = { queryCount: 0, totalQueryTime: 0, writeCount: 0, totalWriteTime: 0 };
|
|
35
|
+
/** Debounce counter: the agent_reads purge is expensive, so it is amortised. */
|
|
36
|
+
_readCount = 0;
|
|
37
|
+
constructor(config = {}) {
|
|
38
|
+
super();
|
|
39
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
40
|
+
}
|
|
41
|
+
/** Subclasses open their driver here. */
|
|
42
|
+
async openDriver() {
|
|
43
|
+
throw new Error('SqlBackend.openDriver() must be implemented by a subclass');
|
|
44
|
+
}
|
|
45
|
+
async initialize() {
|
|
46
|
+
if (this.initialized)
|
|
47
|
+
return;
|
|
48
|
+
this.driver = await this.openDriver();
|
|
49
|
+
// Enable FK enforcement — required for ON DELETE CASCADE to fire.
|
|
50
|
+
try {
|
|
51
|
+
this.driver.exec('PRAGMA foreign_keys = ON');
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
/* unsupported on this driver */
|
|
55
|
+
}
|
|
56
|
+
this.migrationReport = initializeSchema(this.driver);
|
|
57
|
+
if (this.config.verbose && this.migrationReport.legacyColumnFound) {
|
|
58
|
+
console.log(`[SqlBackend] migrated ${this.migrationReport.migrated} inline embedding(s); ` +
|
|
59
|
+
`${this.migrationReport.skipped} already present`);
|
|
60
|
+
}
|
|
61
|
+
this.initialized = true;
|
|
62
|
+
this.emit('initialized');
|
|
63
|
+
}
|
|
64
|
+
async shutdown() {
|
|
65
|
+
if (!this.initialized || !this.driver)
|
|
66
|
+
return;
|
|
67
|
+
await this.driver.persist();
|
|
68
|
+
try {
|
|
69
|
+
this.driver.pragma('optimize');
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
/* not supported everywhere */
|
|
73
|
+
}
|
|
74
|
+
this.driver.close();
|
|
75
|
+
this.driver = null;
|
|
76
|
+
this.initialized = false;
|
|
77
|
+
this.emit('shutdown');
|
|
78
|
+
}
|
|
79
|
+
// ===== Writes ============================================================
|
|
80
|
+
async store(entry) {
|
|
81
|
+
this.ensureInitialized();
|
|
82
|
+
const startTime = performance.now();
|
|
83
|
+
this.validateTags(entry.tags);
|
|
84
|
+
this.driver.transaction(() => this.storeSync(entry));
|
|
85
|
+
const duration = performance.now() - startTime;
|
|
86
|
+
this.stats.writeCount++;
|
|
87
|
+
this.stats.totalWriteTime += duration;
|
|
88
|
+
this.emit('entry:stored', { id: entry.id, duration });
|
|
89
|
+
}
|
|
90
|
+
/** Synchronous store body, shared by store() and bulkInsert(). */
|
|
91
|
+
storeSync(entry) {
|
|
92
|
+
const d = this.driver;
|
|
93
|
+
// Read any existing embedding BEFORE INSERT OR REPLACE fires the CASCADE
|
|
94
|
+
// that would delete it — an entry updated without a vector must keep the
|
|
95
|
+
// one it already had.
|
|
96
|
+
let embeddingToStore = entry.embedding;
|
|
97
|
+
if (!embeddingToStore) {
|
|
98
|
+
const existing = d.get('SELECT embedding FROM memory_embeddings WHERE entry_id = ?', [entry.id]);
|
|
99
|
+
const buf = existing?.embedding;
|
|
100
|
+
if (buf && buf.byteLength > 0) {
|
|
101
|
+
embeddingToStore = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
d.run(`INSERT OR REPLACE INTO memory_entries (
|
|
105
|
+
id, key, content, type, namespace, tags, metadata, owner_id, access_level,
|
|
106
|
+
created_at, updated_at, expires_at, event_at, version, "references",
|
|
107
|
+
access_count, last_accessed_at
|
|
108
|
+
) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, [
|
|
109
|
+
entry.id, entry.key, entry.content, entry.type, entry.namespace,
|
|
110
|
+
JSON.stringify(entry.tags), JSON.stringify(entry.metadata),
|
|
111
|
+
entry.ownerId || null, entry.accessLevel,
|
|
112
|
+
entry.createdAt, entry.updatedAt, entry.expiresAt || null, entry.eventAt ?? null,
|
|
113
|
+
entry.version, JSON.stringify(entry.references), entry.accessCount, entry.lastAccessedAt,
|
|
114
|
+
]);
|
|
115
|
+
d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [entry.id]);
|
|
116
|
+
for (const tag of entry.tags) {
|
|
117
|
+
d.run('INSERT OR IGNORE INTO memory_entry_tags (entry_id, tag) VALUES (?, ?)', [entry.id, tag]);
|
|
118
|
+
}
|
|
119
|
+
if (embeddingToStore) {
|
|
120
|
+
// Slice by byteOffset/byteLength: for copies under Node's 4KB pooling
|
|
121
|
+
// threshold, `.buffer` refers to the entire shared pool, so writing it
|
|
122
|
+
// whole silently stores unrelated memory.
|
|
123
|
+
const bytes = Buffer.from(embeddingToStore.buffer, embeddingToStore.byteOffset, embeddingToStore.byteLength);
|
|
124
|
+
d.run('INSERT OR REPLACE INTO memory_embeddings (entry_id, embedding) VALUES (?, ?)', [
|
|
125
|
+
entry.id,
|
|
126
|
+
bytes,
|
|
127
|
+
]);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
async bulkInsert(entries) {
|
|
131
|
+
this.ensureInitialized();
|
|
132
|
+
for (const e of entries)
|
|
133
|
+
this.validateTags(e.tags);
|
|
134
|
+
this.driver.transaction(() => {
|
|
135
|
+
for (const entry of entries)
|
|
136
|
+
this.storeSync(entry);
|
|
137
|
+
});
|
|
138
|
+
this.emit('bulk:inserted', { count: entries.length });
|
|
139
|
+
}
|
|
140
|
+
async update(id, update) {
|
|
141
|
+
this.ensureInitialized();
|
|
142
|
+
const entry = await this.get(id);
|
|
143
|
+
if (!entry)
|
|
144
|
+
return null;
|
|
145
|
+
if (update.content !== undefined)
|
|
146
|
+
entry.content = update.content;
|
|
147
|
+
if (update.tags !== undefined)
|
|
148
|
+
entry.tags = update.tags;
|
|
149
|
+
if (update.metadata !== undefined)
|
|
150
|
+
entry.metadata = { ...entry.metadata, ...update.metadata };
|
|
151
|
+
if (update.accessLevel !== undefined)
|
|
152
|
+
entry.accessLevel = update.accessLevel;
|
|
153
|
+
if (update.expiresAt !== undefined)
|
|
154
|
+
entry.expiresAt = update.expiresAt;
|
|
155
|
+
if (update.references !== undefined)
|
|
156
|
+
entry.references = update.references;
|
|
157
|
+
entry.updatedAt = Date.now();
|
|
158
|
+
entry.version++;
|
|
159
|
+
await this.store(entry);
|
|
160
|
+
this.emit('entry:updated', { id });
|
|
161
|
+
return entry;
|
|
162
|
+
}
|
|
163
|
+
async delete(id) {
|
|
164
|
+
this.ensureInitialized();
|
|
165
|
+
const d = this.driver;
|
|
166
|
+
// Explicit tag cleanup as well as the CASCADE — belt and braces, since FK
|
|
167
|
+
// enforcement is a pragma that not every driver honours.
|
|
168
|
+
d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [id]);
|
|
169
|
+
d.run('DELETE FROM memory_embeddings WHERE entry_id = ?', [id]);
|
|
170
|
+
const changes = d.run('DELETE FROM memory_entries WHERE id = ?', [id]);
|
|
171
|
+
if (changes > 0) {
|
|
172
|
+
this.emit('entry:deleted', { id });
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
async bulkDelete(ids) {
|
|
178
|
+
this.ensureInitialized();
|
|
179
|
+
const d = this.driver;
|
|
180
|
+
const count = d.transaction(() => {
|
|
181
|
+
let deleted = 0;
|
|
182
|
+
for (const id of ids) {
|
|
183
|
+
d.run('DELETE FROM memory_entry_tags WHERE entry_id = ?', [id]);
|
|
184
|
+
d.run('DELETE FROM memory_embeddings WHERE entry_id = ?', [id]);
|
|
185
|
+
if (d.run('DELETE FROM memory_entries WHERE id = ?', [id]) > 0)
|
|
186
|
+
deleted++;
|
|
187
|
+
}
|
|
188
|
+
return deleted;
|
|
189
|
+
});
|
|
190
|
+
this.emit('bulk:deleted', { count });
|
|
191
|
+
return count;
|
|
192
|
+
}
|
|
193
|
+
async clearNamespace(namespace) {
|
|
194
|
+
this.ensureInitialized();
|
|
195
|
+
const d = this.driver;
|
|
196
|
+
const count = d.transaction(() => {
|
|
197
|
+
d.run(`DELETE FROM memory_entry_tags
|
|
198
|
+
WHERE entry_id IN (SELECT id FROM memory_entries WHERE namespace = ?)`, [namespace]);
|
|
199
|
+
const changes = d.run('DELETE FROM memory_entries WHERE namespace = ?', [namespace]);
|
|
200
|
+
d.run('DELETE FROM memory_embeddings WHERE entry_id NOT IN (SELECT id FROM memory_entries)');
|
|
201
|
+
return changes;
|
|
202
|
+
});
|
|
203
|
+
this.emit('namespace:cleared', { namespace, count });
|
|
204
|
+
return count;
|
|
205
|
+
}
|
|
206
|
+
// ===== Reads =============================================================
|
|
207
|
+
async get(id, agentId) {
|
|
208
|
+
this.ensureInitialized();
|
|
209
|
+
const startTime = performance.now();
|
|
210
|
+
const row = this.driver.get('SELECT * FROM memory_entries WHERE id = ?', [id]);
|
|
211
|
+
if (!row)
|
|
212
|
+
return null;
|
|
213
|
+
// Collaborative memory promotion — https://arxiv.org/abs/2505.18279
|
|
214
|
+
const AGENT_ID_RE = /^[a-zA-Z0-9_-]{1,128}$/;
|
|
215
|
+
if (agentId && AGENT_ID_RE.test(agentId)) {
|
|
216
|
+
try {
|
|
217
|
+
this.driver.run('INSERT OR IGNORE INTO agent_reads (entry_id, agent_id, read_at) VALUES (?, ?, ?)', [id, agentId, Date.now()]);
|
|
218
|
+
this._readCount++;
|
|
219
|
+
if (this._readCount % 1000 === 0)
|
|
220
|
+
this.checkAndPromoteEntry(id);
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
/* non-critical */
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const entry = this.rowToEntry(row);
|
|
227
|
+
this.emit('entry:retrieved', { id, duration: performance.now() - startTime });
|
|
228
|
+
return entry;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Promote an entry to 'team' once 3+ distinct agents have read it within 24h.
|
|
232
|
+
* https://arxiv.org/abs/2505.18279
|
|
233
|
+
*/
|
|
234
|
+
checkAndPromoteEntry(entryId) {
|
|
235
|
+
const d = this.driver;
|
|
236
|
+
if (!d)
|
|
237
|
+
return;
|
|
238
|
+
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
|
239
|
+
d.run('DELETE FROM agent_reads WHERE read_at <= ?', [cutoff]);
|
|
240
|
+
const row = d.get('SELECT COUNT(DISTINCT agent_id) as cnt FROM agent_reads WHERE entry_id = ? AND read_at > ?', [entryId, cutoff]);
|
|
241
|
+
if (Number(row?.cnt ?? 0) >= 3) {
|
|
242
|
+
d.run("UPDATE memory_entries SET access_level = 'team' WHERE id = ? AND access_level = 'private'", [
|
|
243
|
+
entryId,
|
|
244
|
+
]);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
async getByKey(namespace, key) {
|
|
248
|
+
this.ensureInitialized();
|
|
249
|
+
const startTime = performance.now();
|
|
250
|
+
const row = this.driver.get('SELECT * FROM memory_entries WHERE namespace = ? AND key = ?', [
|
|
251
|
+
namespace,
|
|
252
|
+
key,
|
|
253
|
+
]);
|
|
254
|
+
if (!row)
|
|
255
|
+
return null;
|
|
256
|
+
const entry = this.rowToEntry(row);
|
|
257
|
+
this.emit('entry:retrieved', { namespace, key, duration: performance.now() - startTime });
|
|
258
|
+
return entry;
|
|
259
|
+
}
|
|
260
|
+
async query(query) {
|
|
261
|
+
this.ensureInitialized();
|
|
262
|
+
const startTime = performance.now();
|
|
263
|
+
let sql = 'SELECT * FROM memory_entries WHERE 1=1';
|
|
264
|
+
const params = [];
|
|
265
|
+
if (query.namespace) {
|
|
266
|
+
sql += ' AND namespace = ?';
|
|
267
|
+
params.push(query.namespace);
|
|
268
|
+
}
|
|
269
|
+
if (query.key) {
|
|
270
|
+
sql += ' AND key = ?';
|
|
271
|
+
params.push(query.key);
|
|
272
|
+
}
|
|
273
|
+
if (query.keyPrefix) {
|
|
274
|
+
sql += ' AND key LIKE ?';
|
|
275
|
+
params.push(`${query.keyPrefix}%`);
|
|
276
|
+
}
|
|
277
|
+
if (query.memoryType) {
|
|
278
|
+
sql += ' AND type = ?';
|
|
279
|
+
params.push(query.memoryType);
|
|
280
|
+
}
|
|
281
|
+
if (query.accessLevel) {
|
|
282
|
+
sql += ' AND access_level = ?';
|
|
283
|
+
params.push(query.accessLevel);
|
|
284
|
+
}
|
|
285
|
+
if (query.ownerId) {
|
|
286
|
+
sql += ' AND owner_id = ?';
|
|
287
|
+
params.push(query.ownerId);
|
|
288
|
+
}
|
|
289
|
+
if (query.createdAfter) {
|
|
290
|
+
sql += ' AND created_at >= ?';
|
|
291
|
+
params.push(query.createdAfter);
|
|
292
|
+
}
|
|
293
|
+
if (query.createdBefore) {
|
|
294
|
+
sql += ' AND created_at <= ?';
|
|
295
|
+
params.push(query.createdBefore);
|
|
296
|
+
}
|
|
297
|
+
if (query.updatedAfter) {
|
|
298
|
+
sql += ' AND updated_at >= ?';
|
|
299
|
+
params.push(query.updatedAfter);
|
|
300
|
+
}
|
|
301
|
+
if (query.updatedBefore) {
|
|
302
|
+
sql += ' AND updated_at <= ?';
|
|
303
|
+
params.push(query.updatedBefore);
|
|
304
|
+
}
|
|
305
|
+
// Bi-temporal event-time filters (arXiv:2501.13956 — Zep/Graphiti)
|
|
306
|
+
if (query.eventAfter) {
|
|
307
|
+
sql += ' AND event_at >= ?';
|
|
308
|
+
params.push(query.eventAfter);
|
|
309
|
+
}
|
|
310
|
+
if (query.eventBefore) {
|
|
311
|
+
sql += ' AND event_at <= ?';
|
|
312
|
+
params.push(query.eventBefore);
|
|
313
|
+
}
|
|
314
|
+
if (!query.includeExpired) {
|
|
315
|
+
sql += ' AND (expires_at IS NULL OR expires_at > ?)';
|
|
316
|
+
params.push(Date.now());
|
|
317
|
+
}
|
|
318
|
+
// MemoryQuery.tags is documented as "entries must have all specified tags"
|
|
319
|
+
// (types.ts). Counting distinct matches enforces that; an EXISTS(... IN ...)
|
|
320
|
+
// would be ANY-match, which is exactly the divergence that made the two old
|
|
321
|
+
// backends return different result sets for the same call.
|
|
322
|
+
if (query.tags && query.tags.length > 0) {
|
|
323
|
+
this.validateTags(query.tags);
|
|
324
|
+
const placeholders = query.tags.map(() => '?').join(', ');
|
|
325
|
+
sql += ` AND (
|
|
326
|
+
SELECT COUNT(DISTINCT t.tag) FROM memory_entry_tags t
|
|
327
|
+
WHERE t.entry_id = memory_entries.id AND t.tag IN (${placeholders})
|
|
328
|
+
) = ?`;
|
|
329
|
+
params.push(...query.tags, query.tags.length);
|
|
330
|
+
}
|
|
331
|
+
const colMap = {
|
|
332
|
+
createdAt: 'created_at', updatedAt: 'updated_at',
|
|
333
|
+
lastAccessedAt: 'last_accessed_at', accessCount: 'access_count', key: 'key',
|
|
334
|
+
};
|
|
335
|
+
const orderCol = query.sortField && query.sortField !== 'score' && colMap[query.sortField]
|
|
336
|
+
? colMap[query.sortField]
|
|
337
|
+
: 'created_at';
|
|
338
|
+
const orderDir = query.sortDirection === 'asc' ? 'ASC' : 'DESC';
|
|
339
|
+
sql += ` ORDER BY ${orderCol} ${orderDir} LIMIT ?`;
|
|
340
|
+
const effectiveLimit = Math.min(Math.max(1, query.limit ?? MAX_QUERY_LIMIT), MAX_QUERY_LIMIT);
|
|
341
|
+
params.push(effectiveLimit);
|
|
342
|
+
if (query.offset) {
|
|
343
|
+
sql += ' OFFSET ?';
|
|
344
|
+
params.push(query.offset);
|
|
345
|
+
}
|
|
346
|
+
const rows = this.driver.all(sql, params);
|
|
347
|
+
const results = rows.map((r) => this.rowToEntry(r));
|
|
348
|
+
const duration = performance.now() - startTime;
|
|
349
|
+
this.stats.queryCount++;
|
|
350
|
+
this.stats.totalQueryTime += duration;
|
|
351
|
+
this.emit('query:executed', { query, resultCount: results.length, duration });
|
|
352
|
+
return results;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Semantic search — brute-force cosine over stored embeddings, namespace- and
|
|
356
|
+
* TTL-filtered in SQL. At second-brain scale (10^3–10^5 vectors of 384 dims)
|
|
357
|
+
* this is a few tens of ms; swap in an ANN index if a profiled workload
|
|
358
|
+
* outgrows it.
|
|
359
|
+
*/
|
|
360
|
+
async search(embedding, options) {
|
|
361
|
+
this.ensureInitialized();
|
|
362
|
+
const ns = options.filters?.namespace;
|
|
363
|
+
const rows = this.driver.iterate(`SELECT e.*, emb.embedding AS _emb
|
|
364
|
+
FROM memory_entries e
|
|
365
|
+
JOIN memory_embeddings emb ON emb.entry_id = e.id
|
|
366
|
+
WHERE (e.expires_at IS NULL OR e.expires_at = 0 OR e.expires_at > ?)
|
|
367
|
+
${ns ? 'AND e.namespace = ?' : ''}`, ns ? [Date.now(), ns] : [Date.now()]);
|
|
368
|
+
const results = [];
|
|
369
|
+
for (const row of rows) {
|
|
370
|
+
const buf = row._emb;
|
|
371
|
+
if (!buf || buf.byteLength % 4 !== 0)
|
|
372
|
+
continue;
|
|
373
|
+
const vec = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
374
|
+
if (vec.length !== embedding.length)
|
|
375
|
+
continue;
|
|
376
|
+
const similarity = cosineSimilarity(embedding, vec);
|
|
377
|
+
if (options.threshold !== undefined && similarity < options.threshold)
|
|
378
|
+
continue;
|
|
379
|
+
const clean = { ...row };
|
|
380
|
+
delete clean._emb;
|
|
381
|
+
results.push({ entry: this.rowToEntry(clean), score: similarity, distance: 1 - similarity });
|
|
382
|
+
}
|
|
383
|
+
results.sort((a, b) => b.score - a.score);
|
|
384
|
+
return results.slice(0, options.k);
|
|
385
|
+
}
|
|
386
|
+
async count(namespace) {
|
|
387
|
+
this.ensureInitialized();
|
|
388
|
+
const row = namespace
|
|
389
|
+
? this.driver.get('SELECT COUNT(*) as count FROM memory_entries WHERE namespace = ?', [namespace])
|
|
390
|
+
: this.driver.get('SELECT COUNT(*) as count FROM memory_entries');
|
|
391
|
+
return Number(row?.count ?? 0);
|
|
392
|
+
}
|
|
393
|
+
async listNamespaces() {
|
|
394
|
+
this.ensureInitialized();
|
|
395
|
+
return this.driver.all('SELECT DISTINCT namespace FROM memory_entries').map((r) => String(r.namespace));
|
|
396
|
+
}
|
|
397
|
+
// ===== Introspection =====================================================
|
|
398
|
+
async getStats() {
|
|
399
|
+
this.ensureInitialized();
|
|
400
|
+
const d = this.driver;
|
|
401
|
+
const entriesByNamespace = {};
|
|
402
|
+
for (const row of d.all('SELECT namespace, COUNT(*) as count FROM memory_entries GROUP BY namespace')) {
|
|
403
|
+
entriesByNamespace[String(row.namespace)] = Number(row.count);
|
|
404
|
+
}
|
|
405
|
+
const entriesByType = {
|
|
406
|
+
episodic: 0, semantic: 0, working: 0, cache: 0,
|
|
407
|
+
};
|
|
408
|
+
for (const row of d.all('SELECT type, COUNT(*) as count FROM memory_entries GROUP BY type')) {
|
|
409
|
+
entriesByType[String(row.type)] = Number(row.count);
|
|
410
|
+
}
|
|
411
|
+
// page_count/page_size are native-only; sql.js reports 0 rather than a
|
|
412
|
+
// fabricated figure.
|
|
413
|
+
let memoryUsage = 0;
|
|
414
|
+
const pageCount = d.pragma('page_count');
|
|
415
|
+
const pageSize = d.pragma('page_size');
|
|
416
|
+
if (typeof pageCount === 'number' && typeof pageSize === 'number') {
|
|
417
|
+
memoryUsage = pageCount * pageSize;
|
|
418
|
+
}
|
|
419
|
+
return {
|
|
420
|
+
totalEntries: await this.count(),
|
|
421
|
+
entriesByNamespace,
|
|
422
|
+
entriesByType,
|
|
423
|
+
memoryUsage,
|
|
424
|
+
avgQueryTime: this.stats.queryCount > 0 ? this.stats.totalQueryTime / this.stats.queryCount : 0,
|
|
425
|
+
avgSearchTime: 0,
|
|
426
|
+
};
|
|
427
|
+
}
|
|
428
|
+
async healthCheck() {
|
|
429
|
+
const issues = [];
|
|
430
|
+
const recommendations = [];
|
|
431
|
+
if (!this.initialized || !this.driver) {
|
|
432
|
+
return {
|
|
433
|
+
status: 'unhealthy',
|
|
434
|
+
components: {
|
|
435
|
+
storage: { status: 'unhealthy', latency: 0, message: 'Not initialized' },
|
|
436
|
+
index: { status: 'healthy', latency: 0 },
|
|
437
|
+
cache: { status: 'healthy', latency: 0 },
|
|
438
|
+
},
|
|
439
|
+
timestamp: Date.now(),
|
|
440
|
+
issues: ['Backend not initialized'],
|
|
441
|
+
recommendations: ['Call initialize() before using'],
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
let storageHealth;
|
|
445
|
+
try {
|
|
446
|
+
const integrity = this.driver.pragma('integrity_check');
|
|
447
|
+
if (integrity === undefined || integrity === 'ok') {
|
|
448
|
+
// undefined means the driver cannot run the check, not that it failed.
|
|
449
|
+
storageHealth = { status: 'healthy', latency: 0 };
|
|
450
|
+
}
|
|
451
|
+
else {
|
|
452
|
+
issues.push('Database integrity check failed');
|
|
453
|
+
recommendations.push('Run VACUUM to repair database');
|
|
454
|
+
storageHealth = { status: 'unhealthy', latency: 0, message: 'Integrity check failed' };
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
issues.push('Failed to check database integrity');
|
|
459
|
+
storageHealth = { status: 'unhealthy', latency: 0, message: String(error) };
|
|
460
|
+
}
|
|
461
|
+
const totalEntries = await this.count();
|
|
462
|
+
const utilizationPercent = (totalEntries / this.config.maxEntries) * 100;
|
|
463
|
+
if (utilizationPercent > 95) {
|
|
464
|
+
issues.push('Storage utilization critical (>95%)');
|
|
465
|
+
recommendations.push('Cleanup old data or increase maxEntries');
|
|
466
|
+
storageHealth = { status: 'unhealthy', latency: 0, message: 'Near capacity' };
|
|
467
|
+
}
|
|
468
|
+
else if (utilizationPercent > 80) {
|
|
469
|
+
issues.push('Storage utilization high (>80%)');
|
|
470
|
+
recommendations.push('Consider cleanup');
|
|
471
|
+
if (storageHealth.status === 'healthy') {
|
|
472
|
+
storageHealth = { status: 'degraded', latency: 0, message: 'High utilization' };
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
const status = storageHealth.status === 'unhealthy'
|
|
476
|
+
? 'unhealthy'
|
|
477
|
+
: storageHealth.status === 'degraded'
|
|
478
|
+
? 'degraded'
|
|
479
|
+
: 'healthy';
|
|
480
|
+
return {
|
|
481
|
+
status,
|
|
482
|
+
components: {
|
|
483
|
+
storage: storageHealth,
|
|
484
|
+
index: { status: 'healthy', latency: 0 },
|
|
485
|
+
cache: { status: 'healthy', latency: 0 },
|
|
486
|
+
},
|
|
487
|
+
timestamp: Date.now(),
|
|
488
|
+
issues,
|
|
489
|
+
recommendations,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
/** Flush to durable storage. No-op on write-through drivers. */
|
|
493
|
+
async persist() {
|
|
494
|
+
if (!this.driver)
|
|
495
|
+
return;
|
|
496
|
+
await this.driver.persist();
|
|
497
|
+
}
|
|
498
|
+
// ===== Internals =========================================================
|
|
499
|
+
// Every tag write is parameterized, so this is shape sanity rather than
|
|
500
|
+
// injection defense. `src:<absolute path>` provenance tags are first-class in
|
|
501
|
+
// the knowledge pipeline and legitimately contain spaces, parentheses and
|
|
502
|
+
// unicode, so they only exclude control characters.
|
|
503
|
+
static TAG_RE = /^[a-zA-Z0-9_\-.:/~ ]+$/;
|
|
504
|
+
static SRC_TAG_RE = /^src:[^\x00-\x1f\x7f]+$/;
|
|
505
|
+
static MAX_TAG_LEN = 512;
|
|
506
|
+
validateTags(tags) {
|
|
507
|
+
for (const tag of tags) {
|
|
508
|
+
const ok = typeof tag === 'string' &&
|
|
509
|
+
tag.length <= SqlBackend.MAX_TAG_LEN &&
|
|
510
|
+
(tag.startsWith('src:') ? SqlBackend.SRC_TAG_RE.test(tag) : SqlBackend.TAG_RE.test(tag));
|
|
511
|
+
if (!ok)
|
|
512
|
+
throw new Error(`Invalid tag format: "${String(tag).slice(0, 80)}"`);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
ensureInitialized() {
|
|
516
|
+
if (!this.initialized || !this.driver) {
|
|
517
|
+
throw new Error('Backend not initialized. Call initialize() first.');
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
rowToEntry(row) {
|
|
521
|
+
let embedding;
|
|
522
|
+
const embRow = this.driver.get('SELECT embedding FROM memory_embeddings WHERE entry_id = ?', [
|
|
523
|
+
String(row.id),
|
|
524
|
+
]);
|
|
525
|
+
const buf = embRow?.embedding;
|
|
526
|
+
if (buf && buf.byteLength > 0) {
|
|
527
|
+
// Slice by byteOffset/byteLength — see storeSync for why `.buffer` alone
|
|
528
|
+
// can span unrelated pooled memory.
|
|
529
|
+
embedding = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
530
|
+
}
|
|
531
|
+
return {
|
|
532
|
+
id: String(row.id),
|
|
533
|
+
key: String(row.key),
|
|
534
|
+
content: String(row.content),
|
|
535
|
+
embedding,
|
|
536
|
+
type: row.type,
|
|
537
|
+
namespace: String(row.namespace),
|
|
538
|
+
tags: JSON.parse(String(row.tags)),
|
|
539
|
+
metadata: JSON.parse(String(row.metadata)),
|
|
540
|
+
ownerId: row.owner_id ?? undefined,
|
|
541
|
+
accessLevel: row.access_level,
|
|
542
|
+
createdAt: Number(row.created_at),
|
|
543
|
+
updatedAt: Number(row.updated_at),
|
|
544
|
+
expiresAt: row.expires_at ?? undefined,
|
|
545
|
+
eventAt: row.event_at ?? undefined,
|
|
546
|
+
version: Number(row.version),
|
|
547
|
+
references: JSON.parse(String(row.references)),
|
|
548
|
+
accessCount: Number(row.access_count),
|
|
549
|
+
lastAccessedAt: Number(row.last_accessed_at),
|
|
550
|
+
};
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
//# sourceMappingURL=sql-backend.js.map
|