@alineo-labs/sqlite-memory 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +43 -3
- package/dist/index.mjs +68 -12
- package/package.json +2 -2
package/dist/index.d.mts
CHANGED
|
@@ -1,22 +1,45 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { Database } from "bun:sqlite";
|
|
2
|
+
import { EmbeddingProvider, IBulkSemanticMemoryProvider, IPagedWorkingMemoryProvider, IPrunableSemanticMemoryProvider, IRecentSemanticMemoryProvider, MemoryFact, RememberedFact, ResourceRef } from "@alineo-labs/memory";
|
|
2
3
|
//#region src/working.d.ts
|
|
3
4
|
/**
|
|
4
5
|
* Persisted `IWorkingMemoryProvider` backed by `bun:sqlite` — a real, file-based backend with
|
|
5
6
|
* zero external services, same "just a file" story as `@alineo-labs/sqlite`'s ledger adapter.
|
|
6
7
|
* Survives process restarts, unlike `InMemoryWorkingMemoryProvider`.
|
|
7
8
|
*/
|
|
8
|
-
export declare class SQLiteWorkingMemoryProvider implements
|
|
9
|
+
export declare class SQLiteWorkingMemoryProvider implements IPagedWorkingMemoryProvider {
|
|
9
10
|
private readonly db;
|
|
10
11
|
constructor(path: string);
|
|
11
12
|
get(ref: ResourceRef, key: string): Promise<unknown | undefined>;
|
|
12
13
|
set(ref: ResourceRef, key: string, value: unknown): Promise<void>;
|
|
13
14
|
list(ref: ResourceRef): Promise<Record<string, unknown>>;
|
|
15
|
+
/**
|
|
16
|
+
* One page, walking the `(scope, key)` primary-key index — cost follows `limit`, not how many
|
|
17
|
+
* keys the resource has. Ascending UTF-8 byte order (SQLite's BINARY collation), which is the
|
|
18
|
+
* order `listPage`'s contract names.
|
|
19
|
+
*/
|
|
20
|
+
listPage(ref: ResourceRef, opts: {
|
|
21
|
+
after?: string;
|
|
22
|
+
limit: number;
|
|
23
|
+
}): Promise<{
|
|
24
|
+
entries: Array<[string, unknown]>;
|
|
25
|
+
more: boolean;
|
|
26
|
+
}>;
|
|
14
27
|
delete(ref: ResourceRef, key: string): Promise<void>;
|
|
15
28
|
/** Release the underlying SQLite connection. */
|
|
16
29
|
close(): void;
|
|
17
30
|
}
|
|
18
31
|
//#endregion
|
|
19
32
|
//#region src/semantic.d.ts
|
|
33
|
+
/**
|
|
34
|
+
* Load the `sqlite-vec` extension into `db`. `false` (not a throw) when it can't load — an
|
|
35
|
+
* unsupported platform, or a host that blocks native extensions — because the provider has a
|
|
36
|
+
* correct JS fallback for that case.
|
|
37
|
+
*
|
|
38
|
+
* Exported so anything else that has to open this provider's database (a backup tool, say: a
|
|
39
|
+
* `vec0` virtual table can't be read, or `VACUUM`ed, without its module) loads the extension the
|
|
40
|
+
* same way the provider does, rather than carrying a second copy of how.
|
|
41
|
+
*/
|
|
42
|
+
export declare function loadSqliteVec(db: Database): boolean;
|
|
20
43
|
/**
|
|
21
44
|
* Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
|
|
22
45
|
*
|
|
@@ -35,7 +58,7 @@ export declare class SQLiteWorkingMemoryProvider implements IWorkingMemoryProvid
|
|
|
35
58
|
* regardless, so nothing about the fallback is a degraded schema — it's a genuinely
|
|
36
59
|
* lower-performance code path over the same data.
|
|
37
60
|
*/
|
|
38
|
-
export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider {
|
|
61
|
+
export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider, IRecentSemanticMemoryProvider {
|
|
39
62
|
private readonly embeddings;
|
|
40
63
|
private readonly db;
|
|
41
64
|
private vecAvailable;
|
|
@@ -45,6 +68,17 @@ export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMe
|
|
|
45
68
|
* fallback scan (false, either because the extension didn't load or no fact has been
|
|
46
69
|
* remembered yet to size the index from). */
|
|
47
70
|
get hasVectorIndex(): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Make sure the `vec0` index exists and was built for vectors of `dimensions` — and throw,
|
|
73
|
+
* before anything is written, when it was not.
|
|
74
|
+
*
|
|
75
|
+
* `CREATE VIRTUAL TABLE IF NOT EXISTS` is a silent no-op against a table that already exists
|
|
76
|
+
* with a different width, and the dimension this instance tracks in memory starts out null on
|
|
77
|
+
* every process start. So after switching to an embedding model of another size, nothing here
|
|
78
|
+
* would object: the index would be reported as usable (`hasVectorIndex`) and the first insert
|
|
79
|
+
* would fail deep in the native extension, after the fact's metadata row was already written.
|
|
80
|
+
* Compare against the width actually on disk instead, and name the way out.
|
|
81
|
+
*/
|
|
48
82
|
private ensureVecTable;
|
|
49
83
|
remember(ref: ResourceRef, fact: MemoryFact): Promise<void>;
|
|
50
84
|
/** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
|
|
@@ -54,6 +88,12 @@ export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMe
|
|
|
54
88
|
topK?: number;
|
|
55
89
|
}): Promise<MemoryFact[]>;
|
|
56
90
|
listAll(ref: ResourceRef): Promise<RememberedFact[]>;
|
|
91
|
+
/**
|
|
92
|
+
* The `limit` most recently remembered facts, newest first — a bounded read for callers that
|
|
93
|
+
* page through a resource's facts, so a large store isn't deserialized just to show ten.
|
|
94
|
+
* `listAll()` is the unbounded form (and what compaction and `Memory.fork()` need).
|
|
95
|
+
*/
|
|
96
|
+
listRecent(ref: ResourceRef, limit: number): Promise<RememberedFact[]>;
|
|
57
97
|
forget(ref: ResourceRef, ids: string[]): Promise<number>;
|
|
58
98
|
/** Release the underlying SQLite connection. */
|
|
59
99
|
close(): void;
|
package/dist/index.mjs
CHANGED
|
@@ -33,6 +33,10 @@ CREATE TABLE IF NOT EXISTS alineo_semantic_memory (
|
|
|
33
33
|
);
|
|
34
34
|
|
|
35
35
|
CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope ON alineo_semantic_memory(scope);
|
|
36
|
+
-- Lets listRecent() walk one resource's facts newest-first and stop at its LIMIT, instead of
|
|
37
|
+
-- sorting everything the resource ever remembered.
|
|
38
|
+
CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope_time
|
|
39
|
+
ON alineo_semantic_memory(scope, remembered_at);
|
|
36
40
|
`;
|
|
37
41
|
//#endregion
|
|
38
42
|
//#region src/working.ts
|
|
@@ -65,6 +69,20 @@ var SQLiteWorkingMemoryProvider = class {
|
|
|
65
69
|
const rows = this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ?").all(scopeKey(ref));
|
|
66
70
|
return Object.fromEntries(rows.map((r) => [r.key, JSON.parse(r.value)]));
|
|
67
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* One page, walking the `(scope, key)` primary-key index — cost follows `limit`, not how many
|
|
74
|
+
* keys the resource has. Ascending UTF-8 byte order (SQLite's BINARY collation), which is the
|
|
75
|
+
* order `listPage`'s contract names.
|
|
76
|
+
*/
|
|
77
|
+
async listPage(ref, opts) {
|
|
78
|
+
const limit = Math.max(0, Math.floor(opts.limit));
|
|
79
|
+
const scope = scopeKey(ref);
|
|
80
|
+
const rows = opts.after === void 0 ? this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ? ORDER BY key LIMIT ?").all(scope, limit + 1) : this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ? AND key > ? ORDER BY key LIMIT ?").all(scope, opts.after, limit + 1);
|
|
81
|
+
return {
|
|
82
|
+
entries: rows.slice(0, limit).map((r) => [r.key, JSON.parse(r.value)]),
|
|
83
|
+
more: rows.length > limit
|
|
84
|
+
};
|
|
85
|
+
}
|
|
68
86
|
async delete(ref, key) {
|
|
69
87
|
this.db.prepare("DELETE FROM alineo_working_memory WHERE scope = ? AND key = ?").run(scopeKey(ref), key);
|
|
70
88
|
}
|
|
@@ -76,6 +94,26 @@ var SQLiteWorkingMemoryProvider = class {
|
|
|
76
94
|
//#endregion
|
|
77
95
|
//#region src/semantic.ts
|
|
78
96
|
const VEC_TABLE = "alineo_semantic_vec";
|
|
97
|
+
function dimensionMismatch(existing, got) {
|
|
98
|
+
return /* @__PURE__ */ new Error(`semantic memory was built for ${existing}-dimensional embeddings but this embedding model produced ${got}. Vectors from different models are not comparable, so the index can't be shared: point this store at a fresh database file when you change embedding models.`);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Load the `sqlite-vec` extension into `db`. `false` (not a throw) when it can't load — an
|
|
102
|
+
* unsupported platform, or a host that blocks native extensions — because the provider has a
|
|
103
|
+
* correct JS fallback for that case.
|
|
104
|
+
*
|
|
105
|
+
* Exported so anything else that has to open this provider's database (a backup tool, say: a
|
|
106
|
+
* `vec0` virtual table can't be read, or `VACUUM`ed, without its module) loads the extension the
|
|
107
|
+
* same way the provider does, rather than carrying a second copy of how.
|
|
108
|
+
*/
|
|
109
|
+
function loadSqliteVec(db) {
|
|
110
|
+
try {
|
|
111
|
+
db.loadExtension(sqliteVec.getLoadablePath());
|
|
112
|
+
return true;
|
|
113
|
+
} catch {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
79
117
|
/**
|
|
80
118
|
* Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
|
|
81
119
|
*
|
|
@@ -109,12 +147,7 @@ var SQLiteSemanticMemoryProvider = class {
|
|
|
109
147
|
this.db = new Database(path, { create: true });
|
|
110
148
|
this.db.exec(SEMANTIC_MEMORY_MIGRATION_SQL);
|
|
111
149
|
this.db.exec("PRAGMA journal_mode = WAL;");
|
|
112
|
-
|
|
113
|
-
this.db.loadExtension(sqliteVec.getLoadablePath());
|
|
114
|
-
this.vecAvailable = true;
|
|
115
|
-
} catch {
|
|
116
|
-
this.vecAvailable = false;
|
|
117
|
-
}
|
|
150
|
+
this.vecAvailable = loadSqliteVec(this.db);
|
|
118
151
|
}
|
|
119
152
|
/** Whether `recall()` is using the native `sqlite-vec` index (true) or the in-JS cosine
|
|
120
153
|
* fallback scan (false, either because the extension didn't load or no fact has been
|
|
@@ -122,21 +155,36 @@ var SQLiteSemanticMemoryProvider = class {
|
|
|
122
155
|
get hasVectorIndex() {
|
|
123
156
|
return this.vecAvailable && this.vecDimensions != null;
|
|
124
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* Make sure the `vec0` index exists and was built for vectors of `dimensions` — and throw,
|
|
160
|
+
* before anything is written, when it was not.
|
|
161
|
+
*
|
|
162
|
+
* `CREATE VIRTUAL TABLE IF NOT EXISTS` is a silent no-op against a table that already exists
|
|
163
|
+
* with a different width, and the dimension this instance tracks in memory starts out null on
|
|
164
|
+
* every process start. So after switching to an embedding model of another size, nothing here
|
|
165
|
+
* would object: the index would be reported as usable (`hasVectorIndex`) and the first insert
|
|
166
|
+
* would fail deep in the native extension, after the fact's metadata row was already written.
|
|
167
|
+
* Compare against the width actually on disk instead, and name the way out.
|
|
168
|
+
*/
|
|
125
169
|
ensureVecTable(dimensions) {
|
|
126
|
-
if (this.vecDimensions != null)
|
|
170
|
+
if (this.vecDimensions != null) {
|
|
171
|
+
if (dimensions !== this.vecDimensions) throw dimensionMismatch(this.vecDimensions, dimensions);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE name = ?").get(VEC_TABLE);
|
|
175
|
+
const onDisk = existing ? /float\[(\d+)\]/.exec(existing.sql)?.[1] : void 0;
|
|
176
|
+
if (onDisk !== void 0 && Number(onDisk) !== dimensions) throw dimensionMismatch(Number(onDisk), dimensions);
|
|
127
177
|
this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VEC_TABLE} USING vec0(scope TEXT partition key, embedding float[${dimensions}] distance_metric=cosine)`);
|
|
128
178
|
this.vecDimensions = dimensions;
|
|
129
179
|
}
|
|
130
180
|
async remember(ref, fact) {
|
|
131
181
|
const [vector] = await this.embeddings.embed([fact.content], { type: "passage" });
|
|
132
182
|
if (!vector) return;
|
|
183
|
+
if (this.vecAvailable) this.ensureVecTable(vector.length);
|
|
133
184
|
const result = this.db.prepare(`INSERT INTO alineo_semantic_memory
|
|
134
185
|
(id, scope, content, vector, source_sandbox_id, source_entry_index, remembered_at)
|
|
135
186
|
VALUES (?, ?, ?, ?, ?, ?, ?)`).run(crypto.randomUUID(), scopeKey(ref), fact.content, JSON.stringify(vector), fact.sourceRef?.sandboxId ?? null, fact.sourceRef?.entryIndex ?? null, Date.now());
|
|
136
|
-
if (this.vecAvailable) {
|
|
137
|
-
this.ensureVecTable(vector.length);
|
|
138
|
-
this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`).run(result.lastInsertRowid, scopeKey(ref), new Float32Array(vector));
|
|
139
|
-
}
|
|
187
|
+
if (this.vecAvailable) this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`).run(result.lastInsertRowid, scopeKey(ref), new Float32Array(vector));
|
|
140
188
|
}
|
|
141
189
|
/** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
|
|
142
190
|
* insert in a single transaction — see `IBulkSemanticMemoryProvider`. */
|
|
@@ -185,6 +233,14 @@ var SQLiteSemanticMemoryProvider = class {
|
|
|
185
233
|
async listAll(ref) {
|
|
186
234
|
return this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ?").all(scopeKey(ref)).map(factFromRow);
|
|
187
235
|
}
|
|
236
|
+
/**
|
|
237
|
+
* The `limit` most recently remembered facts, newest first — a bounded read for callers that
|
|
238
|
+
* page through a resource's facts, so a large store isn't deserialized just to show ten.
|
|
239
|
+
* `listAll()` is the unbounded form (and what compaction and `Memory.fork()` need).
|
|
240
|
+
*/
|
|
241
|
+
async listRecent(ref, limit) {
|
|
242
|
+
return this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ? ORDER BY remembered_at DESC, rowid_key DESC LIMIT ?").all(scopeKey(ref), Math.max(0, Math.floor(limit))).map(factFromRow);
|
|
243
|
+
}
|
|
188
244
|
async forget(ref, ids) {
|
|
189
245
|
if (ids.length === 0) return 0;
|
|
190
246
|
const placeholders = ids.map(() => "?").join(", ");
|
|
@@ -199,4 +255,4 @@ var SQLiteSemanticMemoryProvider = class {
|
|
|
199
255
|
}
|
|
200
256
|
};
|
|
201
257
|
//#endregion
|
|
202
|
-
export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider };
|
|
258
|
+
export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider, loadSqliteVec };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alineo-labs/sqlite-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
"test": "bun test"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@alineo-labs/memory": "0.
|
|
31
|
+
"@alineo-labs/memory": "0.3.1",
|
|
32
32
|
"sqlite-vec": "^0.1.9"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|