mixgram 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/.mixgram/index.db +0 -0
  2. package/.mixgram/test-esm.db +0 -0
  3. package/.mixgram/test-index.db +0 -0
  4. package/.mixgram/test-stage1.db +0 -0
  5. package/README.md +335 -0
  6. package/bin/mixgram.js +309 -0
  7. package/index.js +9 -0
  8. package/package.json +51 -0
  9. package/projects/castlebravo/memory/decisions/decision--architecture/sqlite-derived-index.md +17 -0
  10. package/projects/test/memory/decisions/decision--architecture/sqlite-derived-index.md +17 -0
  11. package/spec/fixtures/realistic-content.js +97 -0
  12. package/spec/reporter.js +127 -0
  13. package/spec/run-tests-runner.js +40 -0
  14. package/spec/run-tests.js +160 -0
  15. package/spec/scenarios/scenarios.js +520 -0
  16. package/spec/sdd.md +1002 -0
  17. package/spec/tdd.md +1237 -0
  18. package/src/config.js +56 -0
  19. package/src/core/documents/documents.js +207 -0
  20. package/src/core/embeddings/embedder.js +58 -0
  21. package/src/core/embeddings/queue.js +118 -0
  22. package/src/core/embeddings/vectorStore.js +127 -0
  23. package/src/core/embeddings/worker.js +27 -0
  24. package/src/core/indexing/indexer.js +131 -0
  25. package/src/core/indexing/parser.js +78 -0
  26. package/src/core/indexing/reindex.js +101 -0
  27. package/src/core/prompts/prompts.js +15 -0
  28. package/src/core/search/search.js +223 -0
  29. package/src/core/sessions/sessions.js +27 -0
  30. package/src/db/migrations/001_initial.sql +86 -0
  31. package/src/db/sqlite.js +39 -0
  32. package/src/fs/paths.js +79 -0
  33. package/src/fs/watcher.js +49 -0
  34. package/src/mcp/cli-adapter.js +118 -0
  35. package/src/mcp/server.js +47 -0
  36. package/src/mcp/tool-registry.js +138 -0
  37. package/src/mcp/tools.js +159 -0
  38. package/src/utils/hash.js +6 -0
  39. package/src/utils/ids.js +22 -0
  40. package/src/utils/markdown.js +49 -0
package/src/config.js ADDED
@@ -0,0 +1,56 @@
1
+ import path from 'path';
2
+
3
+ const DEFAULT_CONFIG = {
4
+ homeMemoryRoot: './home/memory',
5
+ projectsRoot: './projects',
6
+ sqlitePath: './.mixgram/index.db',
7
+ watch: false,
8
+ indexing: {
9
+ chunkSize: 1200,
10
+ chunkOverlap: 120,
11
+ reindexOnStartup: true,
12
+ ftsWeights: {
13
+ title: 10.0,
14
+ topicKey: 8.0,
15
+ heading: 5.0,
16
+ body: 1.0
17
+ }
18
+ },
19
+ embeddings: {
20
+ enabled: false,
21
+ mode: 'optional',
22
+ provider: 'local-huggingface',
23
+ vectorStore: 'sqlite-vec',
24
+ model: 'Xenova/multilingual-e5-large',
25
+ dimensions: 1024,
26
+ dtype: 'q8',
27
+ workerConcurrency: 2,
28
+ maxRetries: 3,
29
+ queueOnWrite: true,
30
+ similarityThreshold: 0.87
31
+ },
32
+ search: {
33
+ mode: 'fts-only',
34
+ defaultLimit: 10,
35
+ ftsWeight: 0.7,
36
+ semanticWeight: 0.3,
37
+ defaultScopeMode: 'merged'
38
+ }
39
+ };
40
+
41
+ function resolvePaths(config, baseDir) {
42
+ const base = baseDir || process.cwd();
43
+ return {
44
+ ...config,
45
+ homeMemoryRoot: path.resolve(base, config.homeMemoryRoot),
46
+ projectsRoot: path.resolve(base, config.projectsRoot),
47
+ sqlitePath: path.resolve(base, config.sqlitePath)
48
+ };
49
+ }
50
+
51
+ function loadConfig(overrides = {}, baseDir) {
52
+ const config = { ...DEFAULT_CONFIG, ...overrides };
53
+ return resolvePaths(config, baseDir);
54
+ }
55
+
56
+ export { DEFAULT_CONFIG, loadConfig, resolvePaths };
@@ -0,0 +1,207 @@
1
+ import fs from 'fs';
2
+ import { documentPath, ensureDir } from '../../fs/paths.js';
3
+ import { toMarkdown, parse } from '../../utils/markdown.js';
4
+ import { observationId } from '../../utils/ids.js';
5
+ import { contentHash } from '../../utils/hash.js';
6
+ import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
7
+ import { getDb } from '../../db/sqlite.js';
8
+
9
+ /**
10
+ * Resolve existing document by id (takes precedence) or by topic_key + scope + project.
11
+ * @returns {{ path: string, frontmatter: object, body: string } | null}
12
+ */
13
+ function resolveDocument(config, { id, topic_key, scope, project }) {
14
+ const db = getDb(config);
15
+ if (id) {
16
+ const row = db.prepare('SELECT path FROM documents WHERE id = ? AND deleted_at IS NULL').get(id);
17
+ if (!row) return null;
18
+ const fullPath = row.path;
19
+ if (!fs.existsSync(fullPath)) return null;
20
+ const raw = fs.readFileSync(fullPath, 'utf8');
21
+ const parsed = parse(raw);
22
+ return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
23
+ }
24
+ if (topic_key != null && scope != null) {
25
+ const row = db.prepare(
26
+ 'SELECT id, path FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
27
+ ).get(topic_key, scope, project ?? null, project ?? null);
28
+ if (!row) return null;
29
+ const fullPath = row.path;
30
+ if (!fs.existsSync(fullPath)) return null;
31
+ const raw = fs.readFileSync(fullPath, 'utf8');
32
+ const parsed = parse(raw);
33
+ return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
34
+ }
35
+ return null;
36
+ }
37
+
38
+ /**
39
+ * Build frontmatter for a new or updated document.
40
+ */
41
+ function buildFrontmatter(overrides = {}, existing = null) {
42
+ const now = new Date().toISOString();
43
+ if (existing) {
44
+ const rev = (existing.revision_count ?? 1) + 1;
45
+ return {
46
+ ...existing,
47
+ ...overrides,
48
+ updated_at: now,
49
+ revision_count: rev,
50
+ id: existing.id
51
+ };
52
+ }
53
+ return {
54
+ id: overrides.id || observationId(),
55
+ title: overrides.title ?? '',
56
+ type: overrides.type ?? 'generated_note',
57
+ scope: overrides.scope ?? 'project',
58
+ project: overrides.project ?? null,
59
+ topic_key: overrides.topic_key ?? null,
60
+ session_id: overrides.session_id ?? null,
61
+ tool_name: overrides.tool_name ?? null,
62
+ created_at: overrides.created_at ?? now,
63
+ updated_at: now,
64
+ revision_count: 1,
65
+ duplicate_count: 0,
66
+ deleted: false,
67
+ tags: Array.isArray(overrides.tags) ? overrides.tags : [],
68
+ embedding_status: 'disabled'
69
+ };
70
+ }
71
+
72
+ /**
73
+ * Save or update a memory document (mem_save semantics).
74
+ * Resolution: 1) by id, 2) by topic_key in scope, 3) create new.
75
+ * @returns {{ id: string, path: string, created: boolean }}
76
+ */
77
+ function saveDocument(config, payload) {
78
+ const { title, type, scope, project, topic_key, content, session_id, id: givenId, tags } = payload;
79
+ const resolved = givenId
80
+ ? resolveDocument(config, { id: givenId })
81
+ : topic_key != null && scope != null
82
+ ? resolveDocument(config, { topic_key, scope, project })
83
+ : null;
84
+
85
+ let docPath;
86
+ let frontmatter;
87
+ let body = typeof content === 'string' ? content : (content && content.text) || '';
88
+
89
+ if (resolved) {
90
+ docPath = resolved.path;
91
+ frontmatter = buildFrontmatter(
92
+ { title, type, scope, project, topic_key, session_id, tags },
93
+ { ...resolved.frontmatter, revision_count: resolved.frontmatter.revision_count ?? 1 }
94
+ );
95
+ body = body || resolved.body;
96
+ } else {
97
+ frontmatter = buildFrontmatter({
98
+ id: givenId || observationId(),
99
+ title,
100
+ type: type || 'generated_note',
101
+ scope: scope || 'project',
102
+ project: project || null,
103
+ topic_key: topic_key || null,
104
+ session_id: session_id || null,
105
+ tool_name: 'mem_save',
106
+ tags: tags || []
107
+ });
108
+ docPath = documentPath(config, frontmatter);
109
+ ensureDir(docPath);
110
+ }
111
+
112
+ const raw = toMarkdown(frontmatter, body);
113
+ fs.writeFileSync(docPath, raw, 'utf8');
114
+ const stats = fs.statSync(docPath);
115
+ indexDocument(config, docPath, raw, stats.mtimeMs);
116
+ return {
117
+ id: frontmatter.id,
118
+ path: docPath,
119
+ created: !resolved
120
+ };
121
+ }
122
+
123
+ /**
124
+ * Update document by id (mem_update semantics).
125
+ * @returns {{ id: string } | null}
126
+ */
127
+ function updateDocument(config, { id, title, content, tags }) {
128
+ const resolved = resolveDocument(config, { id });
129
+ if (!resolved) return null;
130
+ const body = typeof content === 'string' ? content : (content && content.text) || resolved.body;
131
+ const frontmatter = buildFrontmatter(
132
+ { title: title ?? resolved.frontmatter.title, tags },
133
+ resolved.frontmatter
134
+ );
135
+ const raw = toMarkdown(frontmatter, body);
136
+ fs.writeFileSync(resolved.path, raw, 'utf8');
137
+ const stats = fs.statSync(resolved.path);
138
+ indexDocument(config, resolved.path, raw, stats.mtimeMs);
139
+ return { id: frontmatter.id };
140
+ }
141
+
142
+ /**
143
+ * Soft or hard delete (mem_delete semantics).
144
+ * @param {{ hardDelete?: boolean }} options
145
+ * @returns {boolean} true if document existed and was handled
146
+ */
147
+ function deleteDocument(config, documentId, options = {}) {
148
+ const resolved = resolveDocument(config, { id: documentId });
149
+ if (!resolved) return false;
150
+ if (options.hardDelete) {
151
+ try { fs.unlinkSync(resolved.path); } catch (_) {}
152
+ removeDocumentFromIndex(config, documentId);
153
+ } else {
154
+ const now = new Date().toISOString();
155
+ const frontmatter = { ...resolved.frontmatter, deleted_at: now, deleted: true };
156
+ const raw = toMarkdown(frontmatter, resolved.body);
157
+ fs.writeFileSync(resolved.path, raw, 'utf8');
158
+ const db = getDb(config);
159
+ db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
160
+ const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
161
+ for (const { rowid } of ftsRowids) {
162
+ db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
163
+ }
164
+ }
165
+ return true;
166
+ }
167
+
168
+ /**
169
+ * Get full document content by id (mem_get_observation).
170
+ * @returns {{ id: string, title: string, type: string, scope: string, project: string | null, topic_key: string | null, content: string } | null}
171
+ */
172
+ function getObservation(config, documentId) {
173
+ const resolved = resolveDocument(config, { id: documentId });
174
+ if (!resolved) return null;
175
+ const fullContent = toMarkdown(resolved.frontmatter, resolved.body);
176
+ return {
177
+ id: resolved.frontmatter.id,
178
+ title: resolved.frontmatter.title,
179
+ type: resolved.frontmatter.type,
180
+ scope: resolved.frontmatter.scope,
181
+ project: resolved.frontmatter.project ?? null,
182
+ topic_key: resolved.frontmatter.topic_key ?? null,
183
+ session_id: resolved.frontmatter.session_id ?? null,
184
+ content: fullContent
185
+ };
186
+ }
187
+
188
+ /**
189
+ * List document ids in a session ordered by updated_at (for mem_timeline).
190
+ */
191
+ function getObservationsBySession(config, sessionId) {
192
+ const db = getDb(config);
193
+ const rows = db.prepare(
194
+ 'SELECT id FROM documents WHERE session_id = ? AND deleted_at IS NULL ORDER BY updated_at ASC'
195
+ ).all(sessionId);
196
+ return rows.map((r) => r.id);
197
+ }
198
+
199
+ export {
200
+ resolveDocument,
201
+ buildFrontmatter,
202
+ saveDocument,
203
+ updateDocument,
204
+ deleteDocument,
205
+ getObservation,
206
+ getObservationsBySession
207
+ };
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Optional embedding provider using @huggingface/transformers (Xenova/multilingual-e5-large).
3
+ * Dynamically imported so the app runs without it when embeddings are disabled.
4
+ */
5
+ const MODEL = 'Xenova/multilingual-e5-large';
6
+ const DIM = 1024;
7
+ const DTYPE = 'q8';
8
+
9
+ const pipelineCache = new Map();
10
+
11
+ async function loadPipeline(options = {}) {
12
+ const model = options.model ?? MODEL;
13
+ const dtype = options.dtype ?? DTYPE;
14
+ const key = `${model}:${dtype}`;
15
+ if (pipelineCache.has(key)) return pipelineCache.get(key);
16
+ try {
17
+ const { pipeline } = await import('@huggingface/transformers');
18
+ const pipe = pipeline('feature-extraction', model, {
19
+ pooling: 'mean',
20
+ normalize: true,
21
+ dtype
22
+ });
23
+ pipelineCache.set(key, pipe);
24
+ return pipe;
25
+ } catch (err) {
26
+ throw err;
27
+ }
28
+ }
29
+
30
+ /**
31
+ * Embed a single text with the given pipeline. Returns Float32Array (L2-normalized).
32
+ */
33
+ async function embedWith(extractor, text, dimensions) {
34
+ if (!text || typeof text !== 'string') text = '';
35
+ const output = await extractor(text, { pooling: 'mean', normalize: true });
36
+ const arr = output.data || output.arraySync?.() || Array.from(output);
37
+ return new Float32Array(arr.slice(0, dimensions));
38
+ }
39
+
40
+ export async function getEmbedder(config) {
41
+ if (!config?.embeddings?.enabled) return null;
42
+ try {
43
+ const opts = config.embeddings;
44
+ const extractor = await loadPipeline({
45
+ model: opts.model ?? MODEL,
46
+ dtype: opts.dtype ?? DTYPE
47
+ });
48
+ const dimensions = opts.dimensions ?? DIM;
49
+ return {
50
+ embed: (text) => embedWith(extractor, text, dimensions),
51
+ dimensions
52
+ };
53
+ } catch {
54
+ return null;
55
+ }
56
+ }
57
+
58
+ export { embedWith as embed, DIM, MODEL, DTYPE };
@@ -0,0 +1,118 @@
1
+ /**
2
+ * Embedding job queue: enqueue chunk ids after indexing; worker processes pending jobs.
3
+ */
4
+ import { getDb } from '../../db/sqlite.js';
5
+
6
+ const MODEL_NAME = 'Xenova_multilingual_e5_large';
7
+ const STATUS = { PENDING: 'pending', PROCESSING: 'processing', COMPLETED: 'completed', FAILED: 'failed', STALE: 'stale' };
8
+
9
+ function getModelName(config) {
10
+ return config?.embeddings?.model || MODEL_NAME;
11
+ }
12
+
13
+ /**
14
+ * Enqueue chunks for embedding. Idempotent per chunk: existing pending/processing left as-is; completed/failed/stale replaced by pending.
15
+ */
16
+ function enqueueChunks(config, chunkIds) {
17
+ if (!config?.embeddings?.enabled || !chunkIds?.length) return;
18
+ const db = getDb(config);
19
+ const model = getModelName(config);
20
+ const now = new Date().toISOString();
21
+ const insert = db.prepare(`
22
+ INSERT INTO embedding_jobs (id, chunk_id, model_name, status, attempts, created_at, updated_at)
23
+ VALUES (?, ?, ?, ?, 0, ?, ?)
24
+ ON CONFLICT(id) DO UPDATE SET status = 'pending', attempts = 0, last_error = NULL, updated_at = ?
25
+ WHERE embedding_jobs.status IN ('completed', 'failed', 'stale')
26
+ `);
27
+ for (const cid of chunkIds) {
28
+ const id = `${cid}:${model}`;
29
+ insert.run(id, cid, model, STATUS.PENDING, now, now, now);
30
+ }
31
+ }
32
+
33
+ /**
34
+ * Mark jobs for these chunk ids as stale (e.g. before re-indexing the document).
35
+ */
36
+ function markStale(config, chunkIds) {
37
+ if (!chunkIds?.length) return;
38
+ const db = getDb(config);
39
+ const model = getModelName(config);
40
+ const now = new Date().toISOString();
41
+ const update = db.prepare(`
42
+ UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE chunk_id = ? AND model_name = ?
43
+ `);
44
+ for (const cid of chunkIds) {
45
+ update.run(STATUS.STALE, now, cid, model);
46
+ }
47
+ }
48
+
49
+ /**
50
+ * Process one pending job: load chunk content, embed, write to vector store, mark completed/failed.
51
+ */
52
+ async function processNextJob(config) {
53
+ if (!config?.embeddings?.enabled) return false;
54
+ const db = getDb(config);
55
+ const model = getModelName(config);
56
+ const job = db.prepare(`
57
+ SELECT id, chunk_id FROM embedding_jobs
58
+ WHERE status = ? AND model_name = ?
59
+ ORDER BY created_at ASC LIMIT 1
60
+ `).get(STATUS.PENDING, model);
61
+ if (!job) return false;
62
+
63
+ const now = new Date().toISOString();
64
+ db.prepare(`UPDATE embedding_jobs SET status = ?, attempts = attempts + 1, updated_at = ? WHERE id = ?`)
65
+ .run(STATUS.PROCESSING, now, job.id);
66
+
67
+ let embedder;
68
+ let vectorStore;
69
+ try {
70
+ const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
71
+ import('./embedder.js'),
72
+ import('./vectorStore.js')
73
+ ]);
74
+ embedder = await getEmbedder(config);
75
+ vectorStore = await getVectorStore(config);
76
+ if (!embedder || !vectorStore) {
77
+ db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
78
+ .run(STATUS.FAILED, 'embeddings not available', now, job.id);
79
+ return true;
80
+ }
81
+ } catch (err) {
82
+ db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
83
+ .run(STATUS.FAILED, String(err?.message || err), now, job.id);
84
+ return true;
85
+ }
86
+
87
+ const chunkRow = db.prepare(`
88
+ SELECT c.id, c.document_id, c.content_hash, c.content
89
+ FROM document_chunks c
90
+ WHERE c.id = ?
91
+ `).get(job.chunk_id);
92
+ if (!chunkRow) {
93
+ db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
94
+ .run(STATUS.FAILED, 'chunk not found', now, job.id);
95
+ return true;
96
+ }
97
+
98
+ try {
99
+ const embedding = await embedder.embed(chunkRow.content);
100
+ await vectorStore.insert(config, {
101
+ chunk_id: chunkRow.id,
102
+ document_id: chunkRow.document_id,
103
+ content_hash: chunkRow.content_hash,
104
+ embedding
105
+ });
106
+ db.prepare(`UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE id = ?`)
107
+ .run(STATUS.COMPLETED, now, job.id);
108
+ } catch (err) {
109
+ const maxRetries = config?.embeddings?.maxRetries ?? 3;
110
+ const attempts = db.prepare('SELECT attempts FROM embedding_jobs WHERE id = ?').get(job.id)?.attempts ?? 1;
111
+ const nextStatus = attempts >= maxRetries ? STATUS.FAILED : STATUS.PENDING;
112
+ db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
113
+ .run(nextStatus, String(err?.message || err), now, job.id);
114
+ }
115
+ return true;
116
+ }
117
+
118
+ export { enqueueChunks, markStale, processNextJob, getModelName, STATUS };
@@ -0,0 +1,127 @@
1
+ /**
2
+ * Vector storage using sqlite-vec. Loaded dynamically so it's optional.
3
+ */
4
+ import { getDb } from '../../db/sqlite.js';
5
+
6
+ const MODEL_SUFFIX = 'Xenova_multilingual_e5_large';
7
+
8
+ function getCacheTableName() {
9
+ return `vec_cache_${MODEL_SUFFIX}`;
10
+ }
11
+
12
+ function getVecTableName() {
13
+ return `vec_${MODEL_SUFFIX}`;
14
+ }
15
+
16
+ let sqliteVecLoaded = false;
17
+
18
+ async function ensureSqliteVecLoaded(config) {
19
+ if (sqliteVecLoaded) return;
20
+ try {
21
+ const sqliteVec = await import('sqlite-vec');
22
+ const db = getDb(config);
23
+ sqliteVec.load(db);
24
+ sqliteVecLoaded = true;
25
+ } catch (_) {
26
+ throw new Error('sqlite-vec not available');
27
+ }
28
+ }
29
+
30
+ async function ensureTables(config) {
31
+ const db = getDb(config);
32
+ const cacheTable = getCacheTableName();
33
+ const vecTable = getVecTableName();
34
+ const exists = db.prepare(
35
+ "SELECT name FROM sqlite_master WHERE type='table' AND name=?"
36
+ ).get(cacheTable);
37
+ if (exists) return;
38
+ await ensureSqliteVecLoaded(config);
39
+ db.exec(`
40
+ CREATE TABLE IF NOT EXISTS ${cacheTable} (
41
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
42
+ chunk_id TEXT NOT NULL UNIQUE,
43
+ document_id TEXT NOT NULL,
44
+ content_hash TEXT NOT NULL,
45
+ embedding_model TEXT NOT NULL,
46
+ created_at TEXT NOT NULL,
47
+ FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
48
+ )
49
+ `);
50
+ db.exec(`
51
+ CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTable} USING vec0(
52
+ embedding float[1024] distance_metric=cosine
53
+ )
54
+ `);
55
+ }
56
+
57
+ /**
58
+ * Insert a vector for a chunk. Call after embedding the chunk content.
59
+ */
60
+ async function insert(config, { chunk_id, document_id, content_hash, embedding }) {
61
+ if (!config?.embeddings?.enabled) return;
62
+ await ensureTables(config);
63
+ const db = getDb(config);
64
+ const cacheTable = getCacheTableName();
65
+ const vecTable = getVecTableName();
66
+ const now = new Date().toISOString();
67
+ const model = config.embeddings?.model || MODEL_SUFFIX;
68
+ const emb = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
69
+ db.prepare(`INSERT INTO ${vecTable} (embedding) VALUES (?)`).run(emb);
70
+ const rawId = db.prepare('SELECT last_insert_rowid() AS id').get().id;
71
+ const id = typeof rawId === 'bigint' ? Number(rawId) : parseInt(rawId, 10);
72
+ db.prepare(
73
+ `INSERT INTO ${cacheTable} (id, chunk_id, document_id, content_hash, embedding_model, created_at) VALUES (?, ?, ?, ?, ?, ?)`
74
+ ).run(id, chunk_id, document_id, content_hash, model, now);
75
+ }
76
+
77
+ /**
78
+ * KNN search. Returns [{ chunk_id, document_id, distance }]. distance is cosine distance (0 = identical).
79
+ */
80
+ async function search(config, queryEmbedding, k = 20, options = {}) {
81
+ if (!config?.embeddings?.enabled) return [];
82
+ await ensureTables(config);
83
+ const db = getDb(config);
84
+ const cacheTable = getCacheTableName();
85
+ const vecTable = getVecTableName();
86
+ const emb = queryEmbedding instanceof Float32Array ? queryEmbedding : new Float32Array(queryEmbedding);
87
+ const rows = db.prepare(
88
+ `SELECT v.rowid, v.distance, c.chunk_id, c.document_id
89
+ FROM ${vecTable} v
90
+ JOIN ${cacheTable} c ON c.id = v.rowid
91
+ WHERE v.embedding MATCH ? AND k = ?`
92
+ ).all(emb, Math.min(k, 100));
93
+ const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.87;
94
+ return rows
95
+ .filter((r) => (1 - r.distance) >= threshold)
96
+ .map((r) => ({ chunk_id: r.chunk_id, document_id: r.document_id, distance: r.distance }));
97
+ }
98
+
99
+ /**
100
+ * Remove vectors for given chunk ids (e.g. on document update/invalidation).
101
+ */
102
+ async function removeChunks(config, chunkIds) {
103
+ if (!chunkIds?.length || !config?.embeddings?.enabled) return;
104
+ await ensureTables(config);
105
+ const db = getDb(config);
106
+ const cacheTable = getCacheTableName();
107
+ const vecTable = getVecTableName();
108
+ for (const cid of chunkIds) {
109
+ const row = db.prepare(`SELECT id FROM ${cacheTable} WHERE chunk_id = ?`).get(cid);
110
+ if (row) {
111
+ db.prepare(`DELETE FROM ${vecTable} WHERE rowid = ?`).run(row.id);
112
+ db.prepare(`DELETE FROM ${cacheTable} WHERE id = ?`).run(row.id);
113
+ }
114
+ }
115
+ }
116
+
117
+ export async function getVectorStore(config) {
118
+ if (!config?.embeddings?.enabled) return null;
119
+ try {
120
+ await ensureSqliteVecLoaded(config);
121
+ return { insert, search, removeChunks, ensureTables };
122
+ } catch {
123
+ return null;
124
+ }
125
+ }
126
+
127
+ export { insert, search, removeChunks, ensureTables, getCacheTableName, getVecTableName };
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Background worker that processes embedding jobs when embeddings are enabled.
3
+ */
4
+ import { processNextJob } from './queue.js';
5
+
6
+ const DEFAULT_POLL_MS = 2000;
7
+ let intervalId = null;
8
+
9
+ function startWorker(config) {
10
+ if (!config?.embeddings?.enabled || intervalId != null) return;
11
+ const pollMs = config.embeddings?.workerPollMs ?? DEFAULT_POLL_MS;
12
+ intervalId = setInterval(async () => {
13
+ try {
14
+ const processed = await processNextJob(config);
15
+ if (!processed) return;
16
+ } catch (_) {}
17
+ }, pollMs);
18
+ }
19
+
20
+ function stopWorker() {
21
+ if (intervalId != null) {
22
+ clearInterval(intervalId);
23
+ intervalId = null;
24
+ }
25
+ }
26
+
27
+ export { startWorker, stopWorker };