alvin-bot 4.20.2 → 4.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Shared base for vector-based providers (Gemini, OpenAI, Ollama).
3
+ *
4
+ * Owns the `entries` table schema, vector BLOB encoding, cosine-similarity
5
+ * search, and transactional indexing. Subclasses implement only the embedding
6
+ * API calls (embed for documents, embedQuery for the search query).
7
+ *
8
+ * Vectors are stored as Float32 BLOBs (4 bytes per dim). For a 1536-dim model
9
+ * that's 6 KB per chunk; 3072-dim is 12 KB. Reading is mmap-cheap; cosine sim
10
+ * runs in JS over the in-memory result set — fast enough for tens of thousands
11
+ * of chunks.
12
+ */
13
+ function vectorToBlob(v) {
14
+ const f32 = new Float32Array(v);
15
+ return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
16
+ }
17
+ function blobToVector(b) {
18
+ // better-sqlite3 buffers may be unaligned; copy via DataView guarantees alignment.
19
+ const f32 = new Float32Array(b.byteLength / 4);
20
+ const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
21
+ for (let i = 0; i < f32.length; i++) {
22
+ f32[i] = dv.getFloat32(i * 4, true);
23
+ }
24
+ return f32;
25
+ }
26
+ function cosineSimilarity(a, b) {
27
+ if (a.length !== b.length)
28
+ return 0;
29
+ let dot = 0;
30
+ let na = 0;
31
+ let nb = 0;
32
+ for (let i = 0; i < a.length; i++) {
33
+ dot += a[i] * b[i];
34
+ na += a[i] * a[i];
35
+ nb += b[i] * b[i];
36
+ }
37
+ const denom = Math.sqrt(na) * Math.sqrt(nb);
38
+ return denom === 0 ? 0 : dot / denom;
39
+ }
40
+ export class VectorProviderBase {
41
+ initSchema(db) {
42
+ db.exec(`
43
+ CREATE TABLE IF NOT EXISTS entries (
44
+ id TEXT PRIMARY KEY,
45
+ source TEXT NOT NULL,
46
+ text TEXT NOT NULL,
47
+ vector BLOB NOT NULL,
48
+ indexed_at INTEGER NOT NULL
49
+ );
50
+ CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
51
+ `);
52
+ }
53
+ dropSchema(db) {
54
+ db.exec(`DROP TABLE IF EXISTS entries;`);
55
+ }
56
+ async indexChunks(db, chunks) {
57
+ if (chunks.length === 0)
58
+ return;
59
+ const vectors = await this.embed(chunks.map(c => c.text));
60
+ if (vectors.length !== chunks.length) {
61
+ throw new Error(`Embedding count mismatch: requested ${chunks.length}, got ${vectors.length}`);
62
+ }
63
+ const insertStmt = db.prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?) " +
64
+ "ON CONFLICT(id) DO UPDATE SET source=excluded.source, text=excluded.text, " +
65
+ "vector=excluded.vector, indexed_at=excluded.indexed_at");
66
+ const now = Date.now();
67
+ const writeAll = db.transaction((rows) => {
68
+ for (const r of rows)
69
+ insertStmt.run(r.id, r.source, r.text, r.vector, now);
70
+ });
71
+ writeAll(chunks.map((c, i) => ({
72
+ id: c.id,
73
+ source: c.source,
74
+ text: c.text,
75
+ vector: vectorToBlob(vectors[i]),
76
+ })));
77
+ }
78
+ dropEntriesForSources(db, sources) {
79
+ if (sources.length === 0)
80
+ return;
81
+ const del = db.prepare("DELETE FROM entries WHERE source = ?");
82
+ const dropAll = db.transaction((srcs) => {
83
+ for (const s of srcs)
84
+ del.run(s);
85
+ });
86
+ dropAll(sources);
87
+ }
88
+ async search(db, query, topK, minScore) {
89
+ const qv = Float32Array.from(await this.embedQuery(query));
90
+ const rows = db
91
+ .prepare("SELECT source, text, vector FROM entries")
92
+ .all();
93
+ const scored = [];
94
+ for (const row of rows) {
95
+ const v = blobToVector(row.vector);
96
+ const score = cosineSimilarity(qv, v);
97
+ if (score >= minScore) {
98
+ scored.push({ text: row.text, source: row.source, score });
99
+ }
100
+ }
101
+ scored.sort((a, b) => b.score - a.score);
102
+ return scored.slice(0, topK);
103
+ }
104
+ countEntries(db) {
105
+ try {
106
+ const row = db.prepare("SELECT COUNT(*) AS c FROM entries").get();
107
+ return row?.c ?? 0;
108
+ }
109
+ catch {
110
+ return 0;
111
+ }
112
+ }
113
+ }
@@ -1,505 +1,9 @@
1
1
  /**
2
- * Embeddings Service — Vector-based semantic memory search.
2
+ * Embeddings Service — public API shim.
3
3
  *
4
- * Uses Google's gemini-embedding-001 model for generating embeddings.
5
- * Stores embeddings in a SQLite database (.embeddings.db) replaces the
6
- * older .embeddings.json index since v4.20. The migration runs once
7
- * automatically on startup (see src/migrate.ts).
8
- *
9
- * Architecture:
10
- * - Each memory entry (paragraph/section) gets a 3072-dim Float32 vector.
11
- * - Vectors are stored as raw BLOB (4 bytes × 3072 = 12 KB each) instead of
12
- * JSON-encoded Float64 arrays (~24 KB each) — halves disk footprint.
13
- * - Cosine similarity runs in-memory: SQLite has no native vector ops, but
14
- * reading the BLOBs is mmap-cheap and JS does the dot product fast enough
15
- * for the current corpus (a few thousand entries).
16
- * - Reindexing is per-chunk INSERT/UPDATE — no full-file rewrite.
17
- */
18
- import fs from "fs";
19
- import path from "path";
20
- import { resolve } from "path";
21
- import os from "os";
22
- import { createRequire } from "module";
23
- import { config } from "../config.js";
24
- import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_DB } from "../paths.js";
25
- import { ASSETS_DIR, ASSETS_INDEX_MD } from "../paths.js";
26
- let SqliteClass = null;
27
- let sqliteLoadAttempted = false;
28
- let sqliteLoadError = null;
29
- const cjsRequire = createRequire(import.meta.url);
30
- function loadSqlite() {
31
- if (sqliteLoadAttempted)
32
- return SqliteClass;
33
- sqliteLoadAttempted = true;
34
- try {
35
- SqliteClass = cjsRequire("better-sqlite3");
36
- return SqliteClass;
37
- }
38
- catch (err) {
39
- sqliteLoadError = err instanceof Error ? err : new Error(String(err));
40
- console.warn("⚠️ better-sqlite3 native binary unavailable — embeddings disabled. " +
41
- "Bot continues without semantic memory search. Fix: rebuild deps with " +
42
- "`cd $(npm root -g)/alvin-bot && npm rebuild better-sqlite3` or reinstall " +
43
- "alvin-bot. Underlying error: " +
44
- sqliteLoadError.message);
45
- return null;
46
- }
47
- }
48
- export function getEmbeddingsBackendStatus() {
49
- loadSqlite();
50
- return { available: SqliteClass !== null, error: sqliteLoadError?.message ?? null };
51
- }
52
- // Hub memory directory (Claude Hub — read-only, additional context)
53
- const HUB_MEMORY_DIR = resolve(os.homedir(), ".claude", "hub", "MEMORY");
54
- // ── Constants ───────────────────────────────────────────
55
- const EMBEDDING_MODEL = "gemini-embedding-001";
56
- const EMBEDDING_DIMENSION = 3072;
57
- const SCHEMA_VERSION = "1";
58
- // ── Vector encoding (Float32Array ↔ Buffer) ─────────────
59
- function vectorToBlob(v) {
60
- const f32 = new Float32Array(v);
61
- // Buffer.from(arrayBuffer, byteOffset, length) preserves the underlying memory.
62
- return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
63
- }
64
- function blobToVector(b) {
65
- // Buffers from better-sqlite3 own their memory and may not be aligned to 4 bytes.
66
- // Copying into a fresh Float32Array guarantees alignment.
67
- const f32 = new Float32Array(b.byteLength / 4);
68
- const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
69
- for (let i = 0; i < f32.length; i++) {
70
- f32[i] = dv.getFloat32(i * 4, true /* little-endian */);
71
- }
72
- return f32;
73
- }
74
- // ── DB lifecycle ────────────────────────────────────────
75
- let dbInstance = null;
76
- /**
77
- * Returns the live DB handle, or null when better-sqlite3 isn't loadable.
78
- * Callers must handle the null case (treat as "search unavailable").
4
+ * v4.22.0 refactor: the implementation moved to src/services/embeddings/ with
5
+ * pluggable providers (Gemini, OpenAI, Ollama, FTS5). This file re-exports the
6
+ * facade so existing callers (memory.ts, personality.ts, self-search.ts,
7
+ * commands.ts, index.ts) keep working without import changes.
79
8
  */
80
- function db() {
81
- if (dbInstance)
82
- return dbInstance;
83
- const Database = loadSqlite();
84
- if (!Database)
85
- return null;
86
- // Ensure directory exists (handles fresh installs).
87
- fs.mkdirSync(path.dirname(EMBEDDINGS_DB), { recursive: true });
88
- dbInstance = new Database(EMBEDDINGS_DB);
89
- dbInstance.pragma("journal_mode = WAL");
90
- dbInstance.pragma("synchronous = NORMAL");
91
- dbInstance.pragma("temp_store = MEMORY");
92
- dbInstance.pragma("mmap_size = 268435456"); // 256 MB
93
- dbInstance.exec(`
94
- CREATE TABLE IF NOT EXISTS meta (
95
- key TEXT PRIMARY KEY,
96
- value TEXT NOT NULL
97
- );
98
- CREATE TABLE IF NOT EXISTS file_mtimes (
99
- source TEXT PRIMARY KEY,
100
- mtime_ms REAL NOT NULL
101
- );
102
- CREATE TABLE IF NOT EXISTS entries (
103
- id TEXT PRIMARY KEY,
104
- source TEXT NOT NULL,
105
- text TEXT NOT NULL,
106
- vector BLOB NOT NULL,
107
- indexed_at INTEGER NOT NULL
108
- );
109
- CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source);
110
- `);
111
- // Initialise meta if absent.
112
- const set = dbInstance.prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO NOTHING");
113
- set.run("model", EMBEDDING_MODEL);
114
- set.run("schemaVersion", SCHEMA_VERSION);
115
- return dbInstance;
116
- }
117
- /** Close handle (used by tests / shutdown). */
118
- export function closeEmbeddingsDb() {
119
- if (dbInstance) {
120
- dbInstance.close();
121
- dbInstance = null;
122
- }
123
- }
124
- /** Sharper assertion for use inside helpers that require an open DB. */
125
- function dbOrThrow() {
126
- const d = db();
127
- if (!d) {
128
- throw new Error("Embeddings DB unavailable — better-sqlite3 native module not loaded");
129
- }
130
- return d;
131
- }
132
- // ── Meta helpers ────────────────────────────────────────
133
- function getMeta(key) {
134
- const row = dbOrThrow().prepare("SELECT value FROM meta WHERE key = ?").get(key);
135
- return row?.value ?? null;
136
- }
137
- function setMeta(key, value) {
138
- dbOrThrow()
139
- .prepare("INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
140
- .run(key, value);
141
- }
142
- function getFileMtimes() {
143
- const rows = dbOrThrow()
144
- .prepare("SELECT source, mtime_ms FROM file_mtimes")
145
- .all();
146
- const out = {};
147
- for (const r of rows)
148
- out[r.source] = r.mtime_ms;
149
- return out;
150
- }
151
- function setFileMtime(source, mtimeMs) {
152
- dbOrThrow()
153
- .prepare("INSERT INTO file_mtimes (source, mtime_ms) VALUES (?, ?) ON CONFLICT(source) DO UPDATE SET mtime_ms = excluded.mtime_ms")
154
- .run(source, mtimeMs);
155
- }
156
- // ── Google Embeddings API ───────────────────────────────
157
- async function getEmbeddings(texts) {
158
- const apiKey = config.apiKeys.google;
159
- if (!apiKey) {
160
- throw new Error("Google API key not configured. Set GOOGLE_API_KEY in .env");
161
- }
162
- const results = [];
163
- const batchSize = 100;
164
- for (let i = 0; i < texts.length; i += batchSize) {
165
- const batch = texts.slice(i, i + batchSize);
166
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${EMBEDDING_MODEL}:batchEmbedContents?key=${apiKey}`, {
167
- method: "POST",
168
- headers: { "Content-Type": "application/json" },
169
- body: JSON.stringify({
170
- requests: batch.map(text => ({
171
- model: `models/${EMBEDDING_MODEL}`,
172
- content: { parts: [{ text }] },
173
- taskType: "RETRIEVAL_DOCUMENT",
174
- })),
175
- }),
176
- });
177
- if (!response.ok) {
178
- const err = await response.text();
179
- throw new Error(`Embedding API error: ${response.status} — ${err}`);
180
- }
181
- const data = (await response.json());
182
- for (const emb of data.embeddings) {
183
- results.push(emb.values);
184
- }
185
- }
186
- return results;
187
- }
188
- async function getQueryEmbedding(text) {
189
- const apiKey = config.apiKeys.google;
190
- if (!apiKey) {
191
- throw new Error("Google API key not configured");
192
- }
193
- const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${EMBEDDING_MODEL}:embedContent?key=${apiKey}`, {
194
- method: "POST",
195
- headers: { "Content-Type": "application/json" },
196
- body: JSON.stringify({
197
- model: `models/${EMBEDDING_MODEL}`,
198
- content: { parts: [{ text }] },
199
- taskType: "RETRIEVAL_QUERY",
200
- }),
201
- });
202
- if (!response.ok) {
203
- const err = await response.text();
204
- throw new Error(`Embedding API error: ${response.status} — ${err}`);
205
- }
206
- const data = (await response.json());
207
- return data.embedding.values;
208
- }
209
- // ── Vector Math ─────────────────────────────────────────
210
- function cosineSimilarityF32(a, b) {
211
- if (a.length !== b.length)
212
- return 0;
213
- let dotProduct = 0;
214
- let normA = 0;
215
- let normB = 0;
216
- for (let i = 0; i < a.length; i++) {
217
- dotProduct += a[i] * b[i];
218
- normA += a[i] * a[i];
219
- normB += b[i] * b[i];
220
- }
221
- const denom = Math.sqrt(normA) * Math.sqrt(normB);
222
- return denom === 0 ? 0 : dotProduct / denom;
223
- }
224
- // ── Text Chunking ───────────────────────────────────────
225
- function chunkMarkdown(content, source) {
226
- const chunks = [];
227
- const sections = content.split(/^(?=## )/gm);
228
- for (let i = 0; i < sections.length; i++) {
229
- const section = sections[i].trim();
230
- if (!section || section.length < 20)
231
- continue;
232
- if (section.length > 1000) {
233
- const paragraphs = section.split(/\n\n+/);
234
- let currentChunk = "";
235
- let chunkIdx = 0;
236
- for (const para of paragraphs) {
237
- if (currentChunk.length + para.length > 800 && currentChunk.length > 100) {
238
- chunks.push({
239
- id: `${source}:${i}:${chunkIdx}`,
240
- text: currentChunk.trim(),
241
- });
242
- currentChunk = "";
243
- chunkIdx++;
244
- }
245
- currentChunk += para + "\n\n";
246
- }
247
- if (currentChunk.trim().length > 20) {
248
- chunks.push({
249
- id: `${source}:${i}:${chunkIdx}`,
250
- text: currentChunk.trim(),
251
- });
252
- }
253
- }
254
- else {
255
- chunks.push({
256
- id: `${source}:${i}`,
257
- text: section,
258
- });
259
- }
260
- }
261
- return chunks;
262
- }
263
- // ── Indexable file discovery ────────────────────────────
264
- function walkAssetDir(dir) {
265
- const results = [];
266
- function walk(currentDir) {
267
- let entries;
268
- try {
269
- entries = fs.readdirSync(currentDir, { withFileTypes: true });
270
- }
271
- catch {
272
- return;
273
- }
274
- for (const entry of entries) {
275
- const fullPath = resolve(currentDir, entry.name);
276
- if (entry.isDirectory()) {
277
- walk(fullPath);
278
- }
279
- else if (entry.isFile()) {
280
- if (currentDir === dir && (entry.name === "INDEX.json" || entry.name === "INDEX.md"))
281
- continue;
282
- results.push({ name: entry.name, path: fullPath });
283
- }
284
- }
285
- }
286
- walk(dir);
287
- return results;
288
- }
289
- const TEXT_EXTENSIONS = new Set([".md", ".html", ".txt", ".css", ".ts"]);
290
- function getIndexableFiles() {
291
- const files = [];
292
- if (fs.existsSync(MEMORY_FILE)) {
293
- files.push({ path: MEMORY_FILE, relativePath: "MEMORY.md" });
294
- }
295
- if (fs.existsSync(MEMORY_DIR)) {
296
- const entries = fs.readdirSync(MEMORY_DIR);
297
- for (const entry of entries) {
298
- if (entry.endsWith(".md") && !entry.startsWith(".")) {
299
- files.push({
300
- path: resolve(MEMORY_DIR, entry),
301
- relativePath: `memory/${entry}`,
302
- });
303
- }
304
- }
305
- }
306
- if (fs.existsSync(HUB_MEMORY_DIR)) {
307
- try {
308
- const entries = fs.readdirSync(HUB_MEMORY_DIR);
309
- for (const entry of entries) {
310
- if (entry.endsWith(".md") && !entry.startsWith(".")) {
311
- files.push({
312
- path: resolve(HUB_MEMORY_DIR, entry),
313
- relativePath: `hub/${entry}`,
314
- });
315
- }
316
- }
317
- }
318
- catch {
319
- /* Hub not available — skip */
320
- }
321
- }
322
- if (fs.existsSync(ASSETS_INDEX_MD)) {
323
- files.push({ path: ASSETS_INDEX_MD, relativePath: "assets/INDEX.md" });
324
- }
325
- if (fs.existsSync(ASSETS_DIR)) {
326
- for (const entry of walkAssetDir(ASSETS_DIR)) {
327
- if (TEXT_EXTENSIONS.has(path.extname(entry.name))) {
328
- files.push({
329
- path: entry.path,
330
- relativePath: `assets/${path.relative(ASSETS_DIR, entry.path)}`,
331
- });
332
- }
333
- }
334
- }
335
- return files;
336
- }
337
- function getStaleFiles() {
338
- const allFiles = getIndexableFiles();
339
- const known = getFileMtimes();
340
- const stale = [];
341
- for (const file of allFiles) {
342
- try {
343
- const mtime = fs.statSync(file.path).mtimeMs;
344
- if (!known[file.relativePath] || known[file.relativePath] < mtime) {
345
- stale.push(file);
346
- }
347
- }
348
- catch {
349
- /* file disappeared */
350
- }
351
- }
352
- return stale;
353
- }
354
- // ── Public API ──────────────────────────────────────────
355
- export async function reindexMemory(force = false) {
356
- if (!loadSqlite()) {
357
- return { indexed: 0, total: 0 };
358
- }
359
- const filesToIndex = force ? getIndexableFiles() : getStaleFiles();
360
- if (filesToIndex.length === 0) {
361
- const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
362
- return { indexed: 0, total };
363
- }
364
- // Drop existing entries for files being reindexed (per-source DELETE is O(log n) thanks to idx).
365
- const delStmt = dbOrThrow().prepare("DELETE FROM entries WHERE source = ?");
366
- const dropOld = dbOrThrow().transaction((sources) => {
367
- for (const s of sources)
368
- delStmt.run(s);
369
- });
370
- dropOld(filesToIndex.map(f => f.relativePath));
371
- // Chunk all files.
372
- const allChunks = [];
373
- for (const file of filesToIndex) {
374
- try {
375
- const content = fs.readFileSync(file.path, "utf-8");
376
- const chunks = chunkMarkdown(content, file.relativePath);
377
- const mtime = fs.statSync(file.path).mtimeMs;
378
- for (const chunk of chunks) {
379
- allChunks.push({ ...chunk, source: file.relativePath, mtime });
380
- }
381
- }
382
- catch (err) {
383
- console.error(`Failed to chunk ${file.relativePath}:`, err);
384
- }
385
- }
386
- if (allChunks.length === 0) {
387
- // Even with zero chunks, keep mtimes in sync so we don't re-walk on next run.
388
- const updMtime = dbOrThrow().transaction((files) => {
389
- for (const f of files) {
390
- try {
391
- setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
392
- }
393
- catch {
394
- /* file disappeared */
395
- }
396
- }
397
- });
398
- updMtime(filesToIndex);
399
- const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
400
- return { indexed: 0, total };
401
- }
402
- // Get embeddings for all chunks (network).
403
- const texts = allChunks.map(c => c.text);
404
- const vectors = await getEmbeddings(texts);
405
- // Single transaction for all writes.
406
- const insertStmt = dbOrThrow().prepare("INSERT INTO entries (id, source, text, vector, indexed_at) VALUES (?, ?, ?, ?, ?) " +
407
- "ON CONFLICT(id) DO UPDATE SET source=excluded.source, text=excluded.text, vector=excluded.vector, indexed_at=excluded.indexed_at");
408
- const writeAll = dbOrThrow().transaction((rows) => {
409
- for (const r of rows) {
410
- insertStmt.run(r.id, r.source, r.text, r.vector, r.indexedAt);
411
- }
412
- });
413
- const now = Date.now();
414
- writeAll(allChunks.map((c, i) => ({
415
- id: c.id,
416
- source: c.source,
417
- text: c.text,
418
- vector: vectorToBlob(vectors[i]),
419
- indexedAt: now,
420
- })));
421
- // Update mtimes for the files we just (re-)indexed.
422
- const updMtime = dbOrThrow().transaction((files) => {
423
- for (const f of files) {
424
- try {
425
- setFileMtime(f.relativePath, fs.statSync(f.path).mtimeMs);
426
- }
427
- catch {
428
- /* file disappeared */
429
- }
430
- }
431
- });
432
- updMtime(filesToIndex);
433
- setMeta("lastReindex", String(now));
434
- const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
435
- return { indexed: allChunks.length, total };
436
- }
437
- export async function searchMemory(query, topK = 5, minScore = 0.3) {
438
- if (!loadSqlite()) {
439
- return [];
440
- }
441
- // Auto-index if empty.
442
- const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
443
- if (total === 0) {
444
- await reindexMemory();
445
- const after = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
446
- if (after === 0)
447
- return [];
448
- }
449
- const queryVector = Float32Array.from(await getQueryEmbedding(query));
450
- const rows = dbOrThrow().prepare("SELECT id, source, text, vector FROM entries").all();
451
- const scored = [];
452
- for (const row of rows) {
453
- const v = blobToVector(row.vector);
454
- const score = cosineSimilarityF32(queryVector, v);
455
- if (score >= minScore) {
456
- scored.push({ text: row.text, source: row.source, score });
457
- }
458
- }
459
- scored.sort((a, b) => b.score - a.score);
460
- return scored.slice(0, topK);
461
- }
462
- export async function initEmbeddings() {
463
- if (!loadSqlite()) {
464
- return; // already warned via loadSqlite
465
- }
466
- try {
467
- db(); // Open & migrate schema.
468
- const stale = getStaleFiles();
469
- if (stale.length === 0) {
470
- const total = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
471
- if (total > 0)
472
- return;
473
- }
474
- const result = await reindexMemory();
475
- if (result.indexed > 0) {
476
- console.log(`🔍 Embeddings: indexed ${result.indexed} chunks (${result.total} total)`);
477
- }
478
- }
479
- catch (err) {
480
- console.warn("⚠️ Embeddings init failed:", err instanceof Error ? err.message : err);
481
- }
482
- }
483
- export function getIndexStats() {
484
- let entries = 0;
485
- let files = 0;
486
- let lastReindex = 0;
487
- let sizeBytes = 0;
488
- if (!loadSqlite()) {
489
- return { entries, files, lastReindex, sizeBytes };
490
- }
491
- try {
492
- entries = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM entries").get().c;
493
- files = dbOrThrow().prepare("SELECT COUNT(*) AS c FROM file_mtimes").get().c;
494
- const meta = getMeta("lastReindex");
495
- if (meta)
496
- lastReindex = Number(meta);
497
- sizeBytes = fs.statSync(EMBEDDINGS_DB).size;
498
- }
499
- catch {
500
- /* DB not yet initialised */
501
- }
502
- return { entries, files, lastReindex, sizeBytes };
503
- }
504
- // ── Re-export embedding dim for tests / debugging ──────
505
- export { EMBEDDING_DIMENSION, EMBEDDING_MODEL };
9
+ export { initEmbeddings, searchMemory, reindexMemory, getIndexStats, getEmbeddingsBackendStatus, closeEmbeddingsDb, isSqliteMemoryReady, } from "./embeddings/index.js";
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Memory inject-mode resolver.
3
+ *
4
+ * v4.22 introduces three modes for how curated long-term memory is added to
5
+ * the system prompt:
6
+ *
7
+ * legacy — inject MEMORY.md + daily logs as plain text on every turn.
8
+ * Pre-v4.22 behaviour. Tokens-heavy but works without any API key
9
+ * or SQLite. The fallback when nothing else is configured.
10
+ *
11
+ * sqlite — DON'T bulk-inject MEMORY.md or daily logs. Trust the SQLite
12
+ * memory store (vector or FTS5) + searchMemory() to surface
13
+ * relevant chunks on demand. identity.md and preferences.md are
14
+ * still always plain-text injected because they're tiny and
15
+ * always-on by design.
16
+ *
17
+ * auto — (default) sqlite if the SQLite store has at least one indexed
18
+ * entry, otherwise legacy. This is the seamless-upgrade path:
19
+ * public users keep the legacy behaviour until they've actually
20
+ * populated the SQLite store, then automatically benefit from
21
+ * smaller prompts + targeted retrieval.
22
+ *
23
+ * Override via MEMORY_INJECT_MODE=auto|legacy|sqlite. The bot logs the
24
+ * resolved mode at startup.
25
+ */
26
+ import { isSqliteMemoryReady } from "./embeddings.js";
27
+ export function getInjectModeRaw() {
28
+ const v = (process.env.MEMORY_INJECT_MODE || "auto").trim().toLowerCase();
29
+ if (v === "legacy" || v === "sqlite" || v === "auto")
30
+ return v;
31
+ return "auto";
32
+ }
33
+ /**
34
+ * Resolve the effective mode. In auto, defer to whether the SQLite store
35
+ * actually has indexed entries — falls back to legacy on a fresh install or
36
+ * when reindex hasn't run yet.
37
+ */
38
+ export function getEffectiveInjectMode() {
39
+ const raw = getInjectModeRaw();
40
+ if (raw === "legacy" || raw === "sqlite")
41
+ return raw;
42
+ return isSqliteMemoryReady() ? "sqlite" : "legacy";
43
+ }