@alineo-labs/sqlite-memory 0.2.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -1,22 +1,45 @@
1
- import { EmbeddingProvider, IBulkSemanticMemoryProvider, IPrunableSemanticMemoryProvider, IWorkingMemoryProvider, MemoryFact, RememberedFact, ResourceRef } from "@alineo-labs/memory";
1
+ import { Database } from "bun:sqlite";
2
+ import { EmbeddingProvider, IBulkSemanticMemoryProvider, IPagedWorkingMemoryProvider, IPrunableSemanticMemoryProvider, IRecentSemanticMemoryProvider, MemoryFact, RememberedFact, ResourceRef } from "@alineo-labs/memory";
2
3
  //#region src/working.d.ts
3
4
  /**
4
5
  * Persisted `IWorkingMemoryProvider` backed by `bun:sqlite` — a real, file-based backend with
5
6
  * zero external services, same "just a file" story as `@alineo-labs/sqlite`'s ledger adapter.
6
7
  * Survives process restarts, unlike `InMemoryWorkingMemoryProvider`.
7
8
  */
8
- export declare class SQLiteWorkingMemoryProvider implements IWorkingMemoryProvider {
9
+ export declare class SQLiteWorkingMemoryProvider implements IPagedWorkingMemoryProvider {
9
10
  private readonly db;
10
11
  constructor(path: string);
11
12
  get(ref: ResourceRef, key: string): Promise<unknown | undefined>;
12
13
  set(ref: ResourceRef, key: string, value: unknown): Promise<void>;
13
14
  list(ref: ResourceRef): Promise<Record<string, unknown>>;
15
+ /**
16
+ * One page, walking the `(scope, key)` primary-key index — cost follows `limit`, not how many
17
+ * keys the resource has. Ascending UTF-8 byte order (SQLite's BINARY collation), which is the
18
+ * order `listPage`'s contract names.
19
+ */
20
+ listPage(ref: ResourceRef, opts: {
21
+ after?: string;
22
+ limit: number;
23
+ }): Promise<{
24
+ entries: Array<[string, unknown]>;
25
+ more: boolean;
26
+ }>;
14
27
  delete(ref: ResourceRef, key: string): Promise<void>;
15
28
  /** Release the underlying SQLite connection. */
16
29
  close(): void;
17
30
  }
18
31
  //#endregion
19
32
  //#region src/semantic.d.ts
33
+ /**
34
+ * Load the `sqlite-vec` extension into `db`. `false` (not a throw) when it can't load — an
35
+ * unsupported platform, or a host that blocks native extensions — because the provider has a
36
+ * correct JS fallback for that case.
37
+ *
38
+ * Exported so anything else that has to open this provider's database (a backup tool, say: a
39
+ * `vec0` virtual table can't be read, or `VACUUM`ed, without its module) loads the extension the
40
+ * same way the provider does, rather than carrying a second copy of how.
41
+ */
42
+ export declare function loadSqliteVec(db: Database): boolean;
20
43
  /**
21
44
  * Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
22
45
  *
@@ -35,7 +58,7 @@ export declare class SQLiteWorkingMemoryProvider implements IWorkingMemoryProvid
35
58
  * regardless, so nothing about the fallback is a degraded schema — it's a genuinely
36
59
  * lower-performance code path over the same data.
37
60
  */
38
- export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider {
61
+ export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider, IRecentSemanticMemoryProvider {
39
62
  private readonly embeddings;
40
63
  private readonly db;
41
64
  private vecAvailable;
@@ -45,6 +68,17 @@ export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMe
45
68
  * fallback scan (false, either because the extension didn't load or no fact has been
46
69
  * remembered yet to size the index from). */
47
70
  get hasVectorIndex(): boolean;
71
+ /**
72
+ * Make sure the `vec0` index exists and was built for vectors of `dimensions` — and throw,
73
+ * before anything is written, when it was not.
74
+ *
75
+ * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a silent no-op against a table that already exists
76
+ * with a different width, and the dimension this instance tracks in memory starts out null on
77
+ * every process start. So after switching to an embedding model of another size, nothing here
78
+ * would object: the index would be reported as usable (`hasVectorIndex`) and the first insert
79
+ * would fail deep in the native extension, after the fact's metadata row was already written.
80
+ * Compare against the width actually on disk instead, and name the way out.
81
+ */
48
82
  private ensureVecTable;
49
83
  remember(ref: ResourceRef, fact: MemoryFact): Promise<void>;
50
84
  /** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
@@ -54,6 +88,12 @@ export declare class SQLiteSemanticMemoryProvider implements IPrunableSemanticMe
54
88
  topK?: number;
55
89
  }): Promise<MemoryFact[]>;
56
90
  listAll(ref: ResourceRef): Promise<RememberedFact[]>;
91
+ /**
92
+ * The `limit` most recently remembered facts, newest first — a bounded read for callers that
93
+ * page through a resource's facts, so a large store isn't deserialized just to show ten.
94
+ * `listAll()` is the unbounded form (and what compaction and `Memory.fork()` need).
95
+ */
96
+ listRecent(ref: ResourceRef, limit: number): Promise<RememberedFact[]>;
57
97
  forget(ref: ResourceRef, ids: string[]): Promise<number>;
58
98
  /** Release the underlying SQLite connection. */
59
99
  close(): void;
package/dist/index.mjs CHANGED
@@ -33,6 +33,10 @@ CREATE TABLE IF NOT EXISTS alineo_semantic_memory (
33
33
  );
34
34
 
35
35
  CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope ON alineo_semantic_memory(scope);
36
+ -- Lets listRecent() walk one resource's facts newest-first and stop at its LIMIT, instead of
37
+ -- sorting everything the resource ever remembered.
38
+ CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope_time
39
+ ON alineo_semantic_memory(scope, remembered_at);
36
40
  `;
37
41
  //#endregion
38
42
  //#region src/working.ts
@@ -65,6 +69,20 @@ var SQLiteWorkingMemoryProvider = class {
65
69
  const rows = this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ?").all(scopeKey(ref));
66
70
  return Object.fromEntries(rows.map((r) => [r.key, JSON.parse(r.value)]));
67
71
  }
72
+ /**
73
+ * One page, walking the `(scope, key)` primary-key index — cost follows `limit`, not how many
74
+ * keys the resource has. Ascending UTF-8 byte order (SQLite's BINARY collation), which is the
75
+ * order `listPage`'s contract names.
76
+ */
77
+ async listPage(ref, opts) {
78
+ const limit = Math.max(0, Math.floor(opts.limit));
79
+ const scope = scopeKey(ref);
80
+ const rows = opts.after === void 0 ? this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ? ORDER BY key LIMIT ?").all(scope, limit + 1) : this.db.prepare("SELECT key, value FROM alineo_working_memory WHERE scope = ? AND key > ? ORDER BY key LIMIT ?").all(scope, opts.after, limit + 1);
81
+ return {
82
+ entries: rows.slice(0, limit).map((r) => [r.key, JSON.parse(r.value)]),
83
+ more: rows.length > limit
84
+ };
85
+ }
68
86
  async delete(ref, key) {
69
87
  this.db.prepare("DELETE FROM alineo_working_memory WHERE scope = ? AND key = ?").run(scopeKey(ref), key);
70
88
  }
@@ -76,6 +94,26 @@ var SQLiteWorkingMemoryProvider = class {
76
94
  //#endregion
77
95
  //#region src/semantic.ts
78
96
  const VEC_TABLE = "alineo_semantic_vec";
97
+ function dimensionMismatch(existing, got) {
98
+ return /* @__PURE__ */ new Error(`semantic memory was built for ${existing}-dimensional embeddings but this embedding model produced ${got}. Vectors from different models are not comparable, so the index can't be shared: point this store at a fresh database file when you change embedding models.`);
99
+ }
100
+ /**
101
+ * Load the `sqlite-vec` extension into `db`. `false` (not a throw) when it can't load — an
102
+ * unsupported platform, or a host that blocks native extensions — because the provider has a
103
+ * correct JS fallback for that case.
104
+ *
105
+ * Exported so anything else that has to open this provider's database (a backup tool, say: a
106
+ * `vec0` virtual table can't be read, or `VACUUM`ed, without its module) loads the extension the
107
+ * same way the provider does, rather than carrying a second copy of how.
108
+ */
109
+ function loadSqliteVec(db) {
110
+ try {
111
+ db.loadExtension(sqliteVec.getLoadablePath());
112
+ return true;
113
+ } catch {
114
+ return false;
115
+ }
116
+ }
79
117
  /**
80
118
  * Persisted `ISemanticMemoryProvider` (+ pruning) backed by `bun:sqlite`.
81
119
  *
@@ -109,12 +147,7 @@ var SQLiteSemanticMemoryProvider = class {
109
147
  this.db = new Database(path, { create: true });
110
148
  this.db.exec(SEMANTIC_MEMORY_MIGRATION_SQL);
111
149
  this.db.exec("PRAGMA journal_mode = WAL;");
112
- try {
113
- this.db.loadExtension(sqliteVec.getLoadablePath());
114
- this.vecAvailable = true;
115
- } catch {
116
- this.vecAvailable = false;
117
- }
150
+ this.vecAvailable = loadSqliteVec(this.db);
118
151
  }
119
152
  /** Whether `recall()` is using the native `sqlite-vec` index (true) or the in-JS cosine
120
153
  * fallback scan (false, either because the extension didn't load or no fact has been
@@ -122,21 +155,36 @@ var SQLiteSemanticMemoryProvider = class {
122
155
  get hasVectorIndex() {
123
156
  return this.vecAvailable && this.vecDimensions != null;
124
157
  }
158
+ /**
159
+ * Make sure the `vec0` index exists and was built for vectors of `dimensions` — and throw,
160
+ * before anything is written, when it was not.
161
+ *
162
+ * `CREATE VIRTUAL TABLE IF NOT EXISTS` is a silent no-op against a table that already exists
163
+ * with a different width, and the dimension this instance tracks in memory starts out null on
164
+ * every process start. So after switching to an embedding model of another size, nothing here
165
+ * would object: the index would be reported as usable (`hasVectorIndex`) and the first insert
166
+ * would fail deep in the native extension, after the fact's metadata row was already written.
167
+ * Compare against the width actually on disk instead, and name the way out.
168
+ */
125
169
  ensureVecTable(dimensions) {
126
- if (this.vecDimensions != null) return;
170
+ if (this.vecDimensions != null) {
171
+ if (dimensions !== this.vecDimensions) throw dimensionMismatch(this.vecDimensions, dimensions);
172
+ return;
173
+ }
174
+ const existing = this.db.prepare("SELECT sql FROM sqlite_master WHERE name = ?").get(VEC_TABLE);
175
+ const onDisk = existing ? /float\[(\d+)\]/.exec(existing.sql)?.[1] : void 0;
176
+ if (onDisk !== void 0 && Number(onDisk) !== dimensions) throw dimensionMismatch(Number(onDisk), dimensions);
127
177
  this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${VEC_TABLE} USING vec0(scope TEXT partition key, embedding float[${dimensions}] distance_metric=cosine)`);
128
178
  this.vecDimensions = dimensions;
129
179
  }
130
180
  async remember(ref, fact) {
131
181
  const [vector] = await this.embeddings.embed([fact.content], { type: "passage" });
132
182
  if (!vector) return;
183
+ if (this.vecAvailable) this.ensureVecTable(vector.length);
133
184
  const result = this.db.prepare(`INSERT INTO alineo_semantic_memory
134
185
  (id, scope, content, vector, source_sandbox_id, source_entry_index, remembered_at)
135
186
  VALUES (?, ?, ?, ?, ?, ?, ?)`).run(crypto.randomUUID(), scopeKey(ref), fact.content, JSON.stringify(vector), fact.sourceRef?.sandboxId ?? null, fact.sourceRef?.entryIndex ?? null, Date.now());
136
- if (this.vecAvailable) {
137
- this.ensureVecTable(vector.length);
138
- this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`).run(result.lastInsertRowid, scopeKey(ref), new Float32Array(vector));
139
- }
187
+ if (this.vecAvailable) this.db.prepare(`INSERT INTO ${VEC_TABLE}(rowid, scope, embedding) VALUES (?, ?, ?)`).run(result.lastInsertRowid, scopeKey(ref), new Float32Array(vector));
140
188
  }
141
189
  /** Batches the embedding call for all `facts` into one `embed()` invocation, and wraps every
142
190
  * insert in a single transaction — see `IBulkSemanticMemoryProvider`. */
@@ -185,6 +233,14 @@ var SQLiteSemanticMemoryProvider = class {
185
233
  async listAll(ref) {
186
234
  return this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ?").all(scopeKey(ref)).map(factFromRow);
187
235
  }
236
+ /**
237
+ * The `limit` most recently remembered facts, newest first — a bounded read for callers that
238
+ * page through a resource's facts, so a large store isn't deserialized just to show ten.
239
+ * `listAll()` is the unbounded form (and what compaction and `Memory.fork()` need).
240
+ */
241
+ async listRecent(ref, limit) {
242
+ return this.db.prepare("SELECT * FROM alineo_semantic_memory WHERE scope = ? ORDER BY remembered_at DESC, rowid_key DESC LIMIT ?").all(scopeKey(ref), Math.max(0, Math.floor(limit))).map(factFromRow);
243
+ }
188
244
  async forget(ref, ids) {
189
245
  if (ids.length === 0) return 0;
190
246
  const placeholders = ids.map(() => "?").join(", ");
@@ -199,4 +255,4 @@ var SQLiteSemanticMemoryProvider = class {
199
255
  }
200
256
  };
201
257
  //#endregion
202
- export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider };
258
+ export { SQLiteSemanticMemoryProvider, SQLiteWorkingMemoryProvider, loadSqliteVec };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alineo-labs/sqlite-memory",
3
- "version": "0.2.2",
3
+ "version": "0.3.0",
4
4
  "license": "Apache-2.0",
5
5
  "repository": {
6
6
  "type": "git",
@@ -28,7 +28,7 @@
28
28
  "test": "bun test"
29
29
  },
30
30
  "dependencies": {
31
- "@alineo-labs/memory": "0.2.2",
31
+ "@alineo-labs/memory": "0.3.0",
32
32
  "sqlite-vec": "^0.1.9"
33
33
  },
34
34
  "devDependencies": {