alvin-bot 4.20.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,499 @@
1
+ /**
2
+ * Embeddings Facade — provider-agnostic memory search.
3
+ *
4
+ * Manages the SQLite DB, picks an active provider via auto-detect, handles
5
+ * schema migrations on provider switch, and exposes the legacy public API
6
+ * (initEmbeddings, searchMemory, reindexMemory, getIndexStats) so callers
7
+ * outside this module don't need to know which backend is running.
8
+ *
9
+ * Provider tiers (auto-detected in this order, override via EMBEDDINGS_PROVIDER):
10
+ * 1. Gemini (3072-dim, GOOGLE_API_KEY) — free tier
11
+ * 2. OpenAI (1536-dim, OPENAI_API_KEY) — ~$0.02/1M tokens
12
+ * 3. Ollama (768-dim default, local) — free, private
13
+ * 4. FTS5 (BM25 keyword, no key needed) — universal fallback
14
+ *
15
+ * Schema-mismatch handling: when meta.embedding_model differs from the active
16
+ * provider's name (e.g. user added GOOGLE_API_KEY after running on FTS5), we
17
+ * drop the previous provider's tables, clear file_mtimes, and initialise the
18
+ * new provider's schema. The next reindexMemory() call repopulates everything
19
+ * from disk. This is what makes the "user adds key later" flow seamless.
20
+ */
21
+ import fs from "fs";
22
+ import path from "path";
23
+ import { resolve } from "path";
24
+ import os from "os";
25
+ import { createRequire } from "module";
26
+ import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_DB } from "../../paths.js";
27
+ import { ASSETS_DIR, ASSETS_INDEX_MD } from "../../paths.js";
28
+ import { detectProvider, parseProviderKey } from "./auto-detect.js";
29
+ let SqliteClass = null;
30
+ let sqliteLoadAttempted = false;
31
+ let sqliteLoadError = null;
32
+ const cjsRequire = createRequire(import.meta.url);
33
+ function loadSqlite() {
34
+ if (sqliteLoadAttempted)
35
+ return SqliteClass;
36
+ sqliteLoadAttempted = true;
37
+ try {
38
+ SqliteClass = cjsRequire("better-sqlite3");
39
+ return SqliteClass;
40
+ }
41
+ catch (err) {
42
+ sqliteLoadError = err instanceof Error ? err : new Error(String(err));
43
+ console.warn("⚠️ better-sqlite3 native binary unavailable — embeddings disabled. " +
44
+ "Bot continues without semantic memory search. Fix: rebuild deps with " +
45
+ "`cd $(npm root -g)/alvin-bot && npm rebuild better-sqlite3` or reinstall " +
46
+ "alvin-bot. Underlying error: " +
47
+ sqliteLoadError.message);
48
+ return null;
49
+ }
50
+ }
51
+ // ── State ────────────────────────────────────────────────
52
+ const HUB_MEMORY_DIR = resolve(os.homedir(), ".claude", "hub", "MEMORY");
53
+ const SCHEMA_VERSION = "2"; // bumped from 1 when introducing multi-provider
54
+ let dbInstance = null;
55
+ let activeProvider = null;
56
+ let initialised = false;
57
+ let initInFlight = null;
58
+ // ── DB lifecycle ────────────────────────────────────────
59
+ function openDb() {
60
+ if (dbInstance)
61
+ return dbInstance;
62
+ const Database = loadSqlite();
63
+ if (!Database)
64
+ return null;
65
+ fs.mkdirSync(path.dirname(EMBEDDINGS_DB), { recursive: true });
66
+ dbInstance = new Database(EMBEDDINGS_DB);
67
+ dbInstance.pragma("journal_mode = WAL");
68
+ dbInstance.pragma("synchronous = NORMAL");
69
+ dbInstance.pragma("temp_store = MEMORY");
70
+ dbInstance.pragma("mmap_size = 268435456"); // 256 MB
71
+ // Shared tables — owned by the facade, not by any single provider.
72
+ dbInstance.exec(`
73
+ CREATE TABLE IF NOT EXISTS meta (
74
+ key TEXT PRIMARY KEY,
75
+ value TEXT NOT NULL
76
+ );
77
+ CREATE TABLE IF NOT EXISTS file_mtimes (
78
+ source TEXT PRIMARY KEY,
79
+ mtime_ms REAL NOT NULL
80
+ );
81
+ `);
82
+ return dbInstance;
83
+ }
84
+ export function closeEmbeddingsDb() {
85
+ if (dbInstance) {
86
+ dbInstance.close();
87
+ dbInstance = null;
88
+ }
89
+ activeProvider = null;
90
+ initialised = false;
91
+ initInFlight = null;
92
+ }
93
+ // ── Meta helpers ────────────────────────────────────────
94
+ function getMeta(db, key) {
95
+ const row = db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
96
+ return row?.value ?? null;
97
+ }
98
+ function setMeta(db, key, value) {
99
+ db.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(key, value);
100
+ }
101
+ function clearAllProviderSchemas(db) {
102
+ // Drop both possible provider-owned tables. Defensive — covers any past
103
+ // schema regardless of which provider wrote it.
104
+ db.exec("DROP TABLE IF EXISTS entries; DROP TABLE IF EXISTS entries_fts;");
105
+ db.exec("DELETE FROM file_mtimes;");
106
+ }
107
+ // ── File mtime tracking ─────────────────────────────────
108
+ function getFileMtimes(db) {
109
+ const rows = db.prepare("SELECT source, mtime_ms FROM file_mtimes").all();
110
+ const out = {};
111
+ for (const r of rows)
112
+ out[r.source] = r.mtime_ms;
113
+ return out;
114
+ }
115
+ function setFileMtime(db, source, mtimeMs) {
116
+ db.prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms").run(source, mtimeMs);
117
+ }
118
+ // ── File discovery ──────────────────────────────────────
119
+ const TEXT_EXTENSIONS = new Set([".md", ".html", ".txt", ".css", ".ts"]);
120
+ function walkAssetDir(dir) {
121
+ const results = [];
122
+ function walk(currentDir) {
123
+ let entries;
124
+ try {
125
+ entries = fs.readdirSync(currentDir, { withFileTypes: true });
126
+ }
127
+ catch {
128
+ return;
129
+ }
130
+ for (const entry of entries) {
131
+ const fullPath = resolve(currentDir, entry.name);
132
+ if (entry.isDirectory()) {
133
+ walk(fullPath);
134
+ }
135
+ else if (entry.isFile()) {
136
+ if (currentDir === dir && (entry.name === "INDEX.json" || entry.name === "INDEX.md"))
137
+ continue;
138
+ results.push({ name: entry.name, path: fullPath });
139
+ }
140
+ }
141
+ }
142
+ walk(dir);
143
+ return results;
144
+ }
145
+ function getIndexableFiles() {
146
+ const files = [];
147
+ if (fs.existsSync(MEMORY_FILE)) {
148
+ files.push({ path: MEMORY_FILE, relativePath: "MEMORY.md" });
149
+ }
150
+ if (fs.existsSync(MEMORY_DIR)) {
151
+ const entries = fs.readdirSync(MEMORY_DIR);
152
+ for (const entry of entries) {
153
+ if (entry.endsWith(".md") && !entry.startsWith(".")) {
154
+ files.push({
155
+ path: resolve(MEMORY_DIR, entry),
156
+ relativePath: `memory/${entry}`,
157
+ });
158
+ }
159
+ }
160
+ }
161
+ if (fs.existsSync(HUB_MEMORY_DIR)) {
162
+ try {
163
+ const entries = fs.readdirSync(HUB_MEMORY_DIR);
164
+ for (const entry of entries) {
165
+ if (entry.endsWith(".md") && !entry.startsWith(".")) {
166
+ files.push({
167
+ path: resolve(HUB_MEMORY_DIR, entry),
168
+ relativePath: `hub/${entry}`,
169
+ });
170
+ }
171
+ }
172
+ }
173
+ catch {
174
+ /* hub dir not available */
175
+ }
176
+ }
177
+ if (fs.existsSync(ASSETS_INDEX_MD)) {
178
+ files.push({ path: ASSETS_INDEX_MD, relativePath: "assets/INDEX.md" });
179
+ }
180
+ if (fs.existsSync(ASSETS_DIR)) {
181
+ for (const entry of walkAssetDir(ASSETS_DIR)) {
182
+ if (TEXT_EXTENSIONS.has(path.extname(entry.name))) {
183
+ files.push({
184
+ path: entry.path,
185
+ relativePath: `assets/${path.relative(ASSETS_DIR, entry.path)}`,
186
+ });
187
+ }
188
+ }
189
+ }
190
+ return files;
191
+ }
192
+ function getStaleFiles(db) {
193
+ const all = getIndexableFiles();
194
+ const known = getFileMtimes(db);
195
+ const stale = [];
196
+ for (const f of all) {
197
+ try {
198
+ const mtime = fs.statSync(f.path).mtimeMs;
199
+ if (!known[f.relativePath] || known[f.relativePath] < mtime) {
200
+ stale.push(f);
201
+ }
202
+ }
203
+ catch {
204
+ /* file disappeared */
205
+ }
206
+ }
207
+ return stale;
208
+ }
209
+ // ── Chunking ────────────────────────────────────────────
210
+ function chunkMarkdown(content, source) {
211
+ const chunks = [];
212
+ const sections = content.split(/^(?=## )/gm);
213
+ for (let i = 0; i < sections.length; i++) {
214
+ const section = sections[i].trim();
215
+ if (!section || section.length < 20)
216
+ continue;
217
+ if (section.length > 1000) {
218
+ const paragraphs = section.split(/\n\n+/);
219
+ let cur = "";
220
+ let chunkIdx = 0;
221
+ for (const p of paragraphs) {
222
+ if (cur.length + p.length > 800 && cur.length > 100) {
223
+ chunks.push({ id: `${source}:${i}:${chunkIdx}`, source, text: cur.trim() });
224
+ cur = "";
225
+ chunkIdx++;
226
+ }
227
+ cur += p + "\n\n";
228
+ }
229
+ if (cur.trim().length > 20) {
230
+ chunks.push({ id: `${source}:${i}:${chunkIdx}`, source, text: cur.trim() });
231
+ }
232
+ }
233
+ else {
234
+ chunks.push({ id: `${source}:${i}`, source, text: section });
235
+ }
236
+ }
237
+ return chunks;
238
+ }
239
+ // ── Provider sync ───────────────────────────────────────
240
+ /**
241
+ * Ensure the DB schema matches the active provider. If the stored model name
242
+ * differs from the active provider's, wipe provider-owned tables + file_mtimes
243
+ * so the next reindex repopulates from disk against the new schema. Idempotent.
244
+ */
245
+ function syncProviderSchema(db, provider) {
246
+ // Legacy v4.20 DBs only have meta.model (set by embeddings-migration.ts).
247
+ // Treat that as the previous embedding_model so we don't accidentally
248
+ // wipe a 49 MB vector store just because the meta key was renamed.
249
+ const storedModel = getMeta(db, "embedding_model") ?? getMeta(db, "model");
250
+ // Schema mismatch is detected by provider-name change ONLY. Bumping
251
+ // SCHEMA_VERSION alone must NOT trigger a drop — vector providers (Gemini,
252
+ // OpenAI, Ollama) all share the same `entries` table layout, so a refactor
253
+ // version bump shouldn't cost users a full re-embed against the API.
254
+ const switched = storedModel !== null && storedModel !== provider.name;
255
+ if (switched) {
256
+ clearAllProviderSchemas(db);
257
+ }
258
+ // Initialise the active provider's schema (idempotent — IF NOT EXISTS guards).
259
+ provider.initSchema(db);
260
+ setMeta(db, "embedding_model", provider.name);
261
+ setMeta(db, "embedding_dim", String(provider.dim));
262
+ setMeta(db, "embedding_tier", provider.tier);
263
+ setMeta(db, "schemaVersion", SCHEMA_VERSION);
264
+ return { switched, previous: storedModel };
265
+ }
266
+ // ── Internal init ───────────────────────────────────────
267
+ async function ensureInit() {
268
+ if (initialised && dbInstance && activeProvider) {
269
+ return { db: dbInstance, provider: activeProvider };
270
+ }
271
+ if (initInFlight) {
272
+ await initInFlight;
273
+ return initialised && dbInstance && activeProvider
274
+ ? { db: dbInstance, provider: activeProvider }
275
+ : null;
276
+ }
277
+ initInFlight = (async () => {
278
+ const db = openDb();
279
+ if (!db)
280
+ return; // sqlite unavailable — leave initialised=false
281
+ const overrideKey = parseProviderKey(process.env.EMBEDDINGS_PROVIDER);
282
+ const provider = await detectProvider(overrideKey);
283
+ const sync = syncProviderSchema(db, provider);
284
+ if (sync.switched) {
285
+ console.log(`ℹ️ Memory provider changed: ${sync.previous ?? "none"} → ${provider.name} (${provider.tier}). Reindex on next access.`);
286
+ }
287
+ else {
288
+ // Quiet info log on first startup of a new install.
289
+ const total = provider.countEntries(db);
290
+ if (total === 0) {
291
+ console.log(`ℹ️ Memory provider: ${provider.name} (${provider.tier}). Initial index will run on first use.`);
292
+ }
293
+ }
294
+ activeProvider = provider;
295
+ initialised = true;
296
+ })();
297
+ try {
298
+ await initInFlight;
299
+ }
300
+ finally {
301
+ initInFlight = null;
302
+ }
303
+ return initialised && dbInstance && activeProvider
304
+ ? { db: dbInstance, provider: activeProvider }
305
+ : null;
306
+ }
307
+ // ── Public API ──────────────────────────────────────────
308
+ export async function reindexMemory(force = false) {
309
+ const ctx = await ensureInit();
310
+ if (!ctx)
311
+ return { indexed: 0, total: 0 };
312
+ const { db, provider } = ctx;
313
+ const filesToIndex = force ? getIndexableFiles() : getStaleFiles(db);
314
+ if (filesToIndex.length === 0) {
315
+ return { indexed: 0, total: provider.countEntries(db) };
316
+ }
317
+ // Drop existing entries for these files (per-source DELETE).
318
+ provider.dropEntriesForSources(db, filesToIndex.map(f => f.relativePath));
319
+ // Chunk all files.
320
+ const allChunks = [];
321
+ const fileMtimeMap = [];
322
+ for (const file of filesToIndex) {
323
+ try {
324
+ const content = fs.readFileSync(file.path, "utf-8");
325
+ const chunks = chunkMarkdown(content, file.relativePath);
326
+ for (const c of chunks)
327
+ allChunks.push(c);
328
+ fileMtimeMap.push(file);
329
+ }
330
+ catch (err) {
331
+ console.error(`Failed to chunk ${file.relativePath}:`, err);
332
+ }
333
+ }
334
+ if (allChunks.length === 0) {
335
+ // No content to embed — but DO update mtimes so we don't re-walk these
336
+ // files every startup.
337
+ const updMtime = db.transaction((files) => {
338
+ for (const f of files) {
339
+ try {
340
+ setFileMtime(db, f.relativePath, fs.statSync(f.path).mtimeMs);
341
+ }
342
+ catch {
343
+ /* file vanished */
344
+ }
345
+ }
346
+ });
347
+ updMtime(fileMtimeMap);
348
+ return { indexed: 0, total: provider.countEntries(db) };
349
+ }
350
+ await provider.indexChunks(db, allChunks);
351
+ // Update mtimes for indexed files.
352
+ const updMtime = db.transaction((files) => {
353
+ for (const f of files) {
354
+ try {
355
+ setFileMtime(db, f.relativePath, fs.statSync(f.path).mtimeMs);
356
+ }
357
+ catch {
358
+ /* file vanished */
359
+ }
360
+ }
361
+ });
362
+ updMtime(fileMtimeMap);
363
+ setMeta(db, "lastReindex", String(Date.now()));
364
+ // For Ollama with unknown dim, set it now from the actual vector size.
365
+ if (provider.dim === 0) {
366
+ // Probe one entry; vector providers store as BLOB, FTS5 doesn't have one.
367
+ try {
368
+ const row = db.prepare("SELECT vector FROM entries LIMIT 1").get();
369
+ if (row?.vector) {
370
+ const detectedDim = row.vector.byteLength / 4;
371
+ setMeta(db, "embedding_dim", String(detectedDim));
372
+ }
373
+ }
374
+ catch {
375
+ /* not a vector provider */
376
+ }
377
+ }
378
+ return { indexed: allChunks.length, total: provider.countEntries(db) };
379
+ }
380
+ export async function searchMemory(query, topK = 5, minScore = 0.3) {
381
+ const ctx = await ensureInit();
382
+ if (!ctx)
383
+ return [];
384
+ const { db, provider } = ctx;
385
+ // Lazy first-time index if empty.
386
+ if (provider.countEntries(db) === 0) {
387
+ try {
388
+ await reindexMemory();
389
+ }
390
+ catch (err) {
391
+ // Reindex failure (e.g. API key missing for vector provider) — return
392
+ // empty so the caller can degrade gracefully.
393
+ console.log(`ℹ️ Memory search unavailable: ${err instanceof Error ? err.message : String(err)}`);
394
+ return [];
395
+ }
396
+ if (provider.countEntries(db) === 0)
397
+ return [];
398
+ }
399
+ try {
400
+ return await provider.search(db, query, topK, minScore);
401
+ }
402
+ catch (err) {
403
+ console.log(`ℹ️ Memory search failed (${provider.name}): ${err instanceof Error ? err.message : String(err)}`);
404
+ return [];
405
+ }
406
+ }
407
+ export async function initEmbeddings() {
408
+ const ctx = await ensureInit();
409
+ if (!ctx)
410
+ return; // sqlite unavailable — already warned in loadSqlite
411
+ const { db, provider } = ctx;
412
+ try {
413
+ const stale = getStaleFiles(db);
414
+ if (stale.length === 0 && provider.countEntries(db) > 0) {
415
+ return; // already up to date
416
+ }
417
+ const result = await reindexMemory();
418
+ if (result.indexed > 0) {
419
+ console.log(`🔍 Memory indexed: ${result.indexed} chunks via ${provider.name} (${result.total} total)`);
420
+ }
421
+ }
422
+ catch (err) {
423
+ // Don't crash the bot if reindexing fails (e.g. API down). Log INFO not
424
+ // WARN — bot keeps running, search just returns empty until conditions
425
+ // recover. Public users without keys hit the FTS5 path which never throws.
426
+ console.log(`ℹ️ Memory init deferred: ${err instanceof Error ? err.message : String(err)}`);
427
+ }
428
+ }
429
+ export function getIndexStats() {
430
+ const stats = {
431
+ entries: 0,
432
+ files: 0,
433
+ lastReindex: 0,
434
+ sizeBytes: 0,
435
+ provider: "unavailable",
436
+ tier: "none",
437
+ dim: 0,
438
+ };
439
+ if (!loadSqlite())
440
+ return stats;
441
+ const db = openDb();
442
+ if (!db)
443
+ return stats;
444
+ try {
445
+ if (activeProvider) {
446
+ stats.entries = activeProvider.countEntries(db);
447
+ stats.provider = activeProvider.name;
448
+ stats.tier = activeProvider.tier;
449
+ stats.dim = activeProvider.dim;
450
+ }
451
+ else {
452
+ // Fall back to stored meta if init never ran.
453
+ stats.provider = getMeta(db, "embedding_model") ?? "unknown";
454
+ stats.tier = getMeta(db, "embedding_tier") ?? "unknown";
455
+ stats.dim = Number(getMeta(db, "embedding_dim") ?? 0);
456
+ }
457
+ stats.files = db.prepare("SELECT COUNT(*) AS c FROM file_mtimes").get().c;
458
+ const lr = getMeta(db, "lastReindex");
459
+ if (lr)
460
+ stats.lastReindex = Number(lr);
461
+ if (fs.existsSync(EMBEDDINGS_DB))
462
+ stats.sizeBytes = fs.statSync(EMBEDDINGS_DB).size;
463
+ }
464
+ catch {
465
+ /* DB not initialised or partial */
466
+ }
467
+ return stats;
468
+ }
469
+ export function getEmbeddingsBackendStatus() {
470
+ loadSqlite();
471
+ return { available: SqliteClass !== null, error: sqliteLoadError?.message ?? null };
472
+ }
473
+ /**
474
+ * Synchronous probe: does the SQLite memory store have at least one indexed
475
+ * entry, regardless of which provider wrote it? Used by the inject-mode
476
+ * resolver to decide between legacy plain-text and SQLite-backed search at
477
+ * system-prompt build time (which is sync).
478
+ *
479
+ * Cheap: opens the DB if needed (idempotent), runs a single COUNT on whichever
480
+ * provider table exists. Does NOT call out to embedding APIs.
481
+ */
482
+ export function isSqliteMemoryReady() {
483
+ if (!loadSqlite())
484
+ return false;
485
+ const db = openDb();
486
+ if (!db)
487
+ return false;
488
+ for (const tbl of ["entries", "entries_fts"]) {
489
+ try {
490
+ const r = db.prepare(`SELECT COUNT(*) AS c FROM ${tbl}`).get();
491
+ if (r && r.c > 0)
492
+ return true;
493
+ }
494
+ catch {
495
+ /* table missing — try next */
496
+ }
497
+ }
498
+ return false;
499
+ }
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Ollama Memory Provider — local, private, free embeddings via Ollama.
3
+ *
4
+ * Default model: nomic-embed-text (768-dim, ~270 MB pull).
5
+ * Alternatives via OLLAMA_EMBEDDING_MODEL: mxbai-embed-large (1024-dim),
6
+ * all-minilm (384-dim, fast), bge-large (1024-dim).
7
+ *
8
+ * Uses /api/embed (newer batched endpoint). Detects host via OLLAMA_HOST or
9
+ * OLLAMA_BASE_URL env, defaults to http://localhost:11434.
10
+ */
11
+ import { VectorProviderBase } from "./vector-base.js";
12
+ const DEFAULT_MODEL = "nomic-embed-text";
13
+ const DEFAULT_HOST = "http://localhost:11434";
14
+ // Hardcoded dims for common models — saves a probe call. Unknown models fall
15
+ // through to dynamic detection on first embed().
16
+ const KNOWN_DIMS = {
17
+ "nomic-embed-text": 768,
18
+ "mxbai-embed-large": 1024,
19
+ "all-minilm": 384,
20
+ "bge-large": 1024,
21
+ "bge-small-en-v1.5": 384,
22
+ "snowflake-arctic-embed": 1024,
23
+ };
24
+ function ollamaHost() {
25
+ return process.env.OLLAMA_HOST || process.env.OLLAMA_BASE_URL || DEFAULT_HOST;
26
+ }
27
+ function ollamaModel() {
28
+ return process.env.OLLAMA_EMBEDDING_MODEL || DEFAULT_MODEL;
29
+ }
30
+ export class OllamaProvider extends VectorProviderBase {
31
+ name;
32
+ dim;
33
+ tier = "vector-local";
34
+ constructor() {
35
+ super();
36
+ const model = ollamaModel();
37
+ // Strip any tag like `:latest` for the dim lookup.
38
+ const baseModel = model.split(":")[0];
39
+ this.name = `ollama:${model}`;
40
+ this.dim = KNOWN_DIMS[baseModel] ?? 0; // 0 means "discover dynamically on first embed"
41
+ }
42
+ async isAvailable() {
43
+ try {
44
+ const res = await fetch(`${ollamaHost()}/api/tags`, {
45
+ signal: AbortSignal.timeout(2000),
46
+ });
47
+ if (!res.ok)
48
+ return false;
49
+ const data = (await res.json());
50
+ const wanted = ollamaModel();
51
+ const wantedBase = wanted.split(":")[0];
52
+ // Match either the exact tag or the base name.
53
+ return Boolean(data.models?.some(m => m.name === wanted || m.name.startsWith(`${wantedBase}:`)));
54
+ }
55
+ catch {
56
+ return false;
57
+ }
58
+ }
59
+ async embed(texts) {
60
+ const res = await fetch(`${ollamaHost()}/api/embed`, {
61
+ method: "POST",
62
+ headers: { "Content-Type": "application/json" },
63
+ body: JSON.stringify({ model: ollamaModel(), input: texts }),
64
+ });
65
+ if (!res.ok) {
66
+ throw new Error(`Ollama embed error: ${res.status} — ${await res.text()}`);
67
+ }
68
+ const data = (await res.json());
69
+ if (!Array.isArray(data.embeddings) || data.embeddings.length !== texts.length) {
70
+ throw new Error(`Ollama embed returned ${data.embeddings?.length ?? 0} vectors, expected ${texts.length}`);
71
+ }
72
+ return data.embeddings;
73
+ }
74
+ async embedQuery(text) {
75
+ const [v] = await this.embed([text]);
76
+ return v;
77
+ }
78
+ }
@@ -0,0 +1,49 @@
1
+ /**
2
+ * OpenAI Memory Provider — text-embedding-3-small (1536-dim, $0.02/1M tokens).
3
+ *
4
+ * Most public users already have OPENAI_API_KEY set for the LLM, so this is a
5
+ * near-zero-friction upgrade from FTS5. Reasonably priced even at heavy use.
6
+ */
7
+ import { config } from "../../config.js";
8
+ import { VectorProviderBase } from "./vector-base.js";
9
+ const MODEL = "text-embedding-3-small";
10
+ const DIM = 1536;
11
+ const BATCH_SIZE = 100;
12
+ export class OpenAIProvider extends VectorProviderBase {
13
+ name = MODEL;
14
+ dim = DIM;
15
+ tier = "vector-cloud";
16
+ async isAvailable() {
17
+ return Boolean(config.apiKeys.openai);
18
+ }
19
+ async embed(texts) {
20
+ const apiKey = config.apiKeys.openai;
21
+ if (!apiKey)
22
+ throw new Error("OPENAI_API_KEY not configured");
23
+ const out = [];
24
+ for (let i = 0; i < texts.length; i += BATCH_SIZE) {
25
+ const batch = texts.slice(i, i + BATCH_SIZE);
26
+ const res = await fetch("https://api.openai.com/v1/embeddings", {
27
+ method: "POST",
28
+ headers: {
29
+ "Content-Type": "application/json",
30
+ Authorization: `Bearer ${apiKey}`,
31
+ },
32
+ body: JSON.stringify({ model: MODEL, input: batch }),
33
+ });
34
+ if (!res.ok) {
35
+ throw new Error(`OpenAI embeddings API error: ${res.status} — ${await res.text()}`);
36
+ }
37
+ const data = (await res.json());
38
+ // Sort by index to keep order stable across the batch.
39
+ data.data.sort((a, b) => a.index - b.index);
40
+ for (const e of data.data)
41
+ out.push(e.embedding);
42
+ }
43
+ return out;
44
+ }
45
+ async embedQuery(text) {
46
+ const [v] = await this.embed([text]);
47
+ return v;
48
+ }
49
+ }
@@ -0,0 +1,22 @@
1
+ /**
2
+ * Memory Provider interface — abstracts vector + keyword backends.
3
+ *
4
+ * The embeddings service supports four providers (Gemini, OpenAI, Ollama, FTS5)
5
+ * behind a single facade. Vector providers (Gemini/OpenAI/Ollama) share an
6
+ * `entries` table with a Float32 BLOB column. The FTS5 provider uses an
7
+ * `entries_fts` virtual table for BM25 keyword ranking — no embeddings, no
8
+ * keys, no API calls. Universal zero-config fallback.
9
+ *
10
+ * Common to all providers:
11
+ * meta(key, value) — model name, dim, lastReindex, pending_reindex
12
+ * file_mtimes(source, mtime_ms) — staleness tracking
13
+ *
14
+ * Provider-owned tables:
15
+ * Vector providers → entries(id, source, text, vector BLOB, indexed_at)
16
+ * FTS5 provider → entries_fts(text, source UNINDEXED, id UNINDEXED) VIRTUAL
17
+ *
18
+ * When the active provider changes (e.g. user adds GOOGLE_API_KEY), the facade
19
+ * detects the schema mismatch via meta.embedding_model and triggers a clean
20
+ * reindex against the new provider's schema.
21
+ */
22
+ export {};