@alineo-labs/sqlite-memory 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md ADDED
@@ -0,0 +1,47 @@
1
+ # @alineo-labs/sqlite-memory
2
+
3
+ File-based `@alineo-labs/memory` backend for [alineo](https://alineo.tech), via `bun:sqlite`. Zero infrastructure, survives process restarts.
4
+
5
+ ```bash
6
+ bun add @alineo-labs/sqlite-memory
7
+ ```
8
+
9
+ For a shared, multi-process backend, use [`@alineo-labs/postgres-memory`](https://github.com/DrejT/alineo/tree/main/packages/adapters/postgres-memory) instead.
10
+
11
+ ---
12
+
13
+ ## Usage
14
+
15
+ ```ts
16
+ import { Memory } from "@alineo-labs/memory";
17
+ import {
18
+ SQLiteWorkingMemoryProvider,
19
+ SQLiteSemanticMemoryProvider,
20
+ } from "@alineo-labs/sqlite-memory";
21
+
22
+ const memory = new Memory({
23
+ workingMemory: new SQLiteWorkingMemoryProvider("./alineo-memory.db"),
24
+ semantic: new SQLiteSemanticMemoryProvider("./alineo-memory.db", myEmbeddingProvider),
25
+ });
26
+ ```
27
+
28
+ Both providers migrate and enable WAL mode on construction — no separate `connect()` step.
29
+ Pass the same file path to both to keep working and semantic memory in one file, or different
30
+ paths to keep them separate.
31
+
32
+ Semantic recall ranks using [`sqlite-vec`](https://github.com/asg017/sqlite-vec)'s native
33
+ `vec0` virtual table — a real vector index, not a JS-level scan — whenever the extension loads
34
+ successfully on the current platform (`sqlite-vec` ships prebuilt binaries for the common
35
+ platforms; verified working here on win32/x64). Each resource's vectors are isolated via
36
+ `vec0`'s own partition-key mechanism, applied natively during the KNN search itself rather
37
+ than as a filter afterward — the latter would silently return too few or the wrong results
38
+ whenever another resource's facts happen to be nearer to the query. Check
39
+ `provider.hasVectorIndex` to see which path is active; if the extension fails to load, this
40
+ falls back to the same in-JS cosine scan `InMemorySemanticMemoryProvider` uses — correct, just
41
+ slower.
42
+
43
+ ---
44
+
45
+ ## License
46
+
47
+ Apache 2.0
@@ -0,0 +1,63 @@
1
+ import { EmbeddingProvider, IBulkSemanticMemoryProvider, IPrunableSemanticMemoryProvider, IWorkingMemoryProvider, MemoryFact, RememberedFact, ResourceRef } from "@alineo-labs/memory";
2
+
3
+ //#region src/working.d.ts
4
+ /**
5
+ * Persisted `IWorkingMemoryProvider` backed by `bun:sqlite` — a real, file-based backend with
6
+ * zero external services, same "just a file" story as `@alineo-labs/sqlite`'s ledger adapter.
7
+ * Survives process restarts, unlike `InMemoryWorkingMemoryProvider`.
8
+ */
9
+ declare class SQLiteWorkingMemoryProvider implements IWorkingMemoryProvider {
10
+ private readonly db;
11
+ constructor(path: string);
12
+ get(ref: ResourceRef, key: string): Promise<unknown | undefined>;
13
+ set(ref: ResourceRef, key: string, value: unknown): Promise<void>;
14
+ list(ref: ResourceRef): Promise<Record<string, unknown>>;
15
+ delete(ref: ResourceRef, key: string): Promise<void>;
16
+ /** Release the underlying SQLite connection. */
17
+ close(): void;
18
+ }
19
+ //#endregion
20
+ //#region src/semantic.d.ts
21
+ /**
22
+ * Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
23
+ *
24
+ * Ranks `recall()` with a real native vector index — the `sqlite-vec` extension's `vec0`
25
+ * virtual table — not a JS-level scan, whenever the extension loads successfully for the
26
+ * current platform (verified working here on win32/x64; `sqlite-vec` ships prebuilt binaries
27
+ * for the common platforms via optional npm deps). The vector index is created lazily, sized
28
+ * to the first embedding's dimension actually seen — every subsequent `remember()` must
29
+ * produce vectors of that same dimension (mixing embedding models with different output sizes
30
+ * on one instance will throw from `sqlite-vec` itself, not silently misbehave).
31
+ *
32
+ * If the extension fails to load (e.g. an unsupported platform, or a sandboxed environment
33
+ * that blocks native extension loading), this falls back to the same in-JS cosine-similarity
34
+ * scan `InMemorySemanticMemoryProvider` uses — slower, but still correct. `hasVectorIndex`
35
+ * reports which path is active. The plain `vector` (JSON) column is always populated
36
+ * regardless, so nothing about the fallback is a degraded schema — it's a genuinely
37
+ * lower-performance code path over the same data.
38
+ */
39
+ declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider {
40
+ private readonly embeddings;
41
+ private readonly db;
42
+ private vecAvailable;
43
+ private vecDimensions;
44
+ constructor(path: string, embeddings: EmbeddingProvider);
45
+ /** Whether `recall()` is using the native `sqlite-vec` index (true) or the in-JS cosine
46
+ * fallback scan (false, either because the extension didn't load or no fact has been
47
+ * remembered yet to size the index from). */
48
+ get hasVectorIndex(): boolean;
49
+ private ensureVecTable;
50
+ remember(ref: ResourceRef, fact: MemoryFact): Promise<void>;
51
+ /** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
52
+ * insert in a single transaction — see `IBulkSemanticMemoryProvider`. */
53
+ rememberMany(ref: ResourceRef, facts: MemoryFact[]): Promise<void>;
54
+ recall(ref: ResourceRef, query: string, opts?: {
55
+ topK?: number;
56
+ }): Promise<MemoryFact[]>;
57
+ listAll(ref: ResourceRef): Promise<RememberedFact[]>;
58
+ forget(ref: ResourceRef, ids: string[]): Promise<number>;
59
+ /** Release the underlying SQLite connection. */
60
+ close(): void;
61
+ }
62
+ //#endregion
63
+ export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider };
package/dist/index.mjs ADDED
@@ -0,0 +1,202 @@
1
+ import { Database } from "bun:sqlite";
2
+ import { mkdirSync } from "node:fs";
3
+ import { dirname } from "node:path";
4
+ import { cosineSimilarity, factFromRow, scopeKey } from "@alineo-labs/memory";
5
+ import * as sqliteVec from "sqlite-vec";
6
+ //#region src/migrations.ts
7
+ const WORKING_MEMORY_MIGRATION_SQL = `
8
+ CREATE TABLE IF NOT EXISTS alineo_working_memory (
9
+ scope TEXT NOT NULL,
10
+ key TEXT NOT NULL,
11
+ value TEXT NOT NULL,
12
+ PRIMARY KEY (scope, key)
13
+ );
14
+ `;
15
+ /**
16
+ * `rowid_key` is a plain autoincrementing integer, separate from the public `id` (a UUID) —
17
+ * it exists purely so a row here can be joined to its matching row in the `vec0` virtual
18
+ * table created lazily by `SQLiteSemanticMemoryProvider` (`vec0` keys rows by integer
19
+ * `rowid`, and `id` isn't guaranteed to be representable as one). `vector` (JSON-encoded)
20
+ * stays on this table even when the `vec0` index is active — it's the fallback path used if
21
+ * the `sqlite-vec` native extension fails to load on a given platform.
22
+ */
23
+ const SEMANTIC_MEMORY_MIGRATION_SQL = `
24
+ CREATE TABLE IF NOT EXISTS alineo_semantic_memory (
25
+ rowid_key INTEGER PRIMARY KEY AUTOINCREMENT,
26
+ id TEXT NOT NULL UNIQUE,
27
+ scope TEXT NOT NULL,
28
+ content TEXT NOT NULL,
29
+ vector TEXT NOT NULL,
30
+ source_sandbox_id TEXT,
31
+ source_entry_index INTEGER,
32
+ remembered_at INTEGER NOT NULL
33
+ );
34
+
35
+ CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope ON alineo_semantic_memory(scope);
36
+ `;
37
+ //#endregion
38
+ //#region src/working.ts
39
+ /**
40
+ * Persisted `IWorkingMemoryProvider` backed by `bun:sqlite` — a real, file-based backend with
41
+ * zero external services, same "just a file" story as `@alineo-labs/sqlite`'s ledger adapter.
42
+ * Survives process restarts, unlike `InMemoryWorkingMemoryProvider`.
43
+ */
44
+ var SQLiteWorkingMemoryProvider = class {
45
+ db;
46
+ constructor(path) {
47
+ try {
48
+ mkdirSync(dirname(path), { recursive: true });
49
+ } catch (e) {
50
+ if (e.code !== "EEXIST") throw e;
51
+ }
52
+ this.db = new Database(path, { create: true });
53
+ this.db.exec(WORKING_MEMORY_MIGRATION_SQL);
54
+ this.db.exec("PRAGMA journal_mode = WAL;");
55
+ }
56
+ async get(ref, key) {
57
+ const row = this.db.prepare("SELECT value FROM alineo_working_memory WHERE scope = ? AND key = ?").get(scopeKey(ref), key);
58
+ return row ? JSON.parse(row.value) : void 0;
59
+ }
60
+ async set(ref, key, value) {
61
+ this.db.prepare(`INSERT INTO alineo_working_memory (scope, key, value) VALUES (?, ?, ?)
62
+ ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`).run(scopeKey(ref), key, JSON.stringify(value));
63
+ }
64
+ async list(ref) {
65
+ const rows = this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ?").all(scopeKey(ref));
66
+ return Object.fromEntries(rows.map((r) => [r.key, JSON.parse(r.value)]));
67
+ }
68
+ async delete(ref, key) {
69
+ this.db.prepare("DELETE FROM alineo_working_memory WHERE scope = ? AND key = ?").run(scopeKey(ref), key);
70
+ }
71
+ /** Release the underlying SQLite connection. */
72
+ close() {
73
+ this.db.close();
74
+ }
75
+ };
76
+ //#endregion
77
+ //#region src/semantic.ts
78
+ const VEC_TABLE = "alineo_semantic_vec";
79
+ /**
80
+ * Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
81
+ *
82
+ * Ranks `recall()` with a real native vector index — the `sqlite-vec` extension's `vec0`
83
+ * virtual table — not a JS-level scan, whenever the extension loads successfully for the
84
+ * current platform (verified working here on win32/x64; `sqlite-vec` ships prebuilt binaries
85
+ * for the common platforms via optional npm deps). The vector index is created lazily, sized
86
+ * to the first embedding's dimension actually seen — every subsequent `remember()` must
87
+ * produce vectors of that same dimension (mixing embedding models with different output sizes
88
+ * on one instance will throw from `sqlite-vec` itself, not silently misbehave).
89
+ *
90
+ * If the extension fails to load (e.g. an unsupported platform, or a sandboxed environment
91
+ * that blocks native extension loading), this falls back to the same in-JS cosine-similarity
92
+ * scan `InMemorySemanticMemoryProvider` uses — slower, but still correct. `hasVectorIndex`
93
+ * reports which path is active. The plain `vector` (JSON) column is always populated
94
+ * regardless, so nothing about the fallback is a degraded schema — it's a genuinely
95
+ * lower-performance code path over the same data.
96
+ */
97
+ var SQLiteSemanticMemoryProvider = class {
98
+ embeddings;
99
+ db;
100
+ vecAvailable;
101
+ vecDimensions = null;
102
+ constructor(path, embeddings) {
103
+ this.embeddings = embeddings;
104
+ try {
105
+ mkdirSync(dirname(path), { recursive: true });
106
+ } catch (e) {
107
+ if (e.code !== "EEXIST") throw e;
108
+ }
109
+ this.db = new Database(path, { create: true });
110
+ this.db.exec(SEMANTIC_MEMORY_MIGRATION_SQL);
111
+ this.db.exec("PRAGMA journal_mode = WAL;");
112
+ try {
113
+ this.db.loadExtension(sqliteVec.getLoadablePath());
114
+ this.vecAvailable = true;
115
+ } catch {
116
+ this.vecAvailable = false;
117
+ }
118
+ }
119
+ /** Whether `recall()` is using the native `sqlite-vec` index (true) or the in-JS cosine
120
+ * fallback scan (false, either because the extension didn't load or no fact has been
121
+ * remembered yet to size the index from). */
122
+ get hasVectorIndex() {
123
+ return this.vecAvailable && this.vecDimensions != null;
124
+ }
125
+ ensureVecTable(dimensions) {
126
+ if (this.vecDimensions != null) return;
127
+ this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VEC_TABLE} USING vec0(scope TEXT partition key, embedding float[${dimensions}] distance_metric=cosine)`);
128
+ this.vecDimensions = dimensions;
129
+ }
130
+ async remember(ref, fact) {
131
+ const [vector] = await this.embeddings.embed([fact.content], { type: "passage" });
132
+ if (!vector) return;
133
+ const result = this.db.prepare(`INSERT INTO alineo_semantic_memory
134
+ (id, scope, content, vector, source_sandbox_id, source_entry_index, remembered_at)
135
+ VALUES (?, ?, ?, ?, ?, ?, ?)`).run(crypto.randomUUID(), scopeKey(ref), fact.content, JSON.stringify(vector), fact.sourceRef?.sandboxId ?? null, fact.sourceRef?.entryIndex ?? null, Date.now());
136
+ if (this.vecAvailable) {
137
+ this.ensureVecTable(vector.length);
138
+ this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`).run(result.lastInsertRowid, scopeKey(ref), new Float32Array(vector));
139
+ }
140
+ }
141
+ /** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
142
+ * insert in a single transaction — see `IBulkSemanticMemoryProvider`. */
143
+ async rememberMany(ref, facts) {
144
+ if (facts.length === 0) return;
145
+ const vectors = await this.embeddings.embed(facts.map((f) => f.content), { type: "passage" });
146
+ if (this.vecAvailable) {
147
+ const firstVector = vectors.find((v) => v != null);
148
+ if (firstVector) this.ensureVecTable(firstVector.length);
149
+ }
150
+ const scope = scopeKey(ref);
151
+ const insertMeta = this.db.prepare(`INSERT INTO alineo_semantic_memory
152
+ (id, scope, content, vector, source_sandbox_id, source_entry_index, remembered_at)
153
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
154
+ const insertVec = this.vecAvailable && this.vecDimensions != null ? this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`) : null;
155
+ const now = Date.now();
156
+ this.db.transaction(() => {
157
+ for (let i = 0; i < facts.length; i++) {
158
+ const vector = vectors[i];
159
+ const fact = facts[i];
160
+ if (!vector) continue;
161
+ const result = insertMeta.run(crypto.randomUUID(), scope, fact.content, JSON.stringify(vector), fact.sourceRef?.sandboxId ?? null, fact.sourceRef?.entryIndex ?? null, now);
162
+ insertVec?.run(result.lastInsertRowid, scope, new Float32Array(vector));
163
+ }
164
+ })();
165
+ }
166
+ async recall(ref, query, opts = {}) {
167
+ const topK = opts.topK ?? 5;
168
+ const [queryVector] = await this.embeddings.embed([query], { type: "query" });
169
+ if (!queryVector) return [];
170
+ if (this.hasVectorIndex) return this.db.prepare(`
171
+ SELECT m.rowid_key, m.id, m.content, m.vector, m.source_sandbox_id,
172
+ m.source_entry_index, m.remembered_at
173
+ FROM ${VEC_TABLE} v
174
+ JOIN alineo_semantic_memory m ON m.rowid_key = v.rowid
175
+ WHERE v.embedding MATCH ? AND v.scope = ? AND k = ?
176
+ ORDER BY v.distance
177
+ `).all(new Float32Array(queryVector), scopeKey(ref), topK).map(factFromRow);
178
+ const rows = this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ?").all(scopeKey(ref));
179
+ if (rows.length === 0) return [];
180
+ return rows.map((row) => ({
181
+ row,
182
+ score: cosineSimilarity(queryVector, JSON.parse(row.vector))
183
+ })).sort((a, b) => b.score - a.score).slice(0, topK).map(({ row }) => factFromRow(row));
184
+ }
185
+ async listAll(ref) {
186
+ return this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ?").all(scopeKey(ref)).map(factFromRow);
187
+ }
188
+ async forget(ref, ids) {
189
+ if (ids.length === 0) return 0;
190
+ const placeholders = ids.map(() => "?").join(", ");
191
+ if (this.vecAvailable) this.db.prepare(`DELETE FROM ${VEC_TABLE} WHERE rowid IN (
192
+ SELECT rowid_key FROM alineo_semantic_memory WHERE scope = ? AND id IN (${placeholders})
193
+ )`).run(scopeKey(ref), ...ids);
194
+ return this.db.prepare(`DELETE FROM alineo_semantic_memory WHERE scope = ? AND id IN (${placeholders})`).run(scopeKey(ref), ...ids).changes;
195
+ }
196
+ /** Release the underlying SQLite connection. */
197
+ close() {
198
+ this.db.close();
199
+ }
200
+ };
201
+ //#endregion
202
+ export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider };
package/package.json ADDED
@@ -0,0 +1,39 @@
1
+ {
2
+ "name": "@alineo-labs/sqlite-memory",
3
+ "version": "0.2.0",
4
+ "license": "Apache-2.0",
5
+ "repository": {
6
+ "type": "git",
7
+ "url": "git+https://github.com/DrejT/alineo.git",
8
+ "directory": "packages/adapters/sqlite-memory"
9
+ },
10
+ "files": [
11
+ "dist"
12
+ ],
13
+ "type": "module",
14
+ "main": "./dist/index.mjs",
15
+ "types": "./dist/index.d.mts",
16
+ "exports": {
17
+ ".": {
18
+ "import": "./dist/index.mjs",
19
+ "types": "./dist/index.d.mts"
20
+ }
21
+ },
22
+ "publishConfig": {
23
+ "access": "public",
24
+ "provenance": true
25
+ },
26
+ "scripts": {
27
+ "build": "tsdown",
28
+ "test": "bun test"
29
+ },
30
+ "dependencies": {
31
+ "@alineo-labs/memory": "0.2.0",
32
+ "sqlite-vec": "^0.1.9"
33
+ },
34
+ "devDependencies": {
35
+ "bun-types": "1.3.14",
36
+ "tsdown": "0.22.3",
37
+ "typescript": "6.0.3"
38
+ }
39
+ }