mixgram 1.3.6 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +68 -345
  2. package/bin/mixgram.js +51 -160
  3. package/examples/embedding-demo.js +22 -142
  4. package/package.json +19 -19
  5. package/spec/bench-indexing.js +23 -211
  6. package/spec/bench-search.js +26 -212
  7. package/spec/run-tests.js +1 -214
  8. package/spec/scenarios/scenarios.js +277 -607
  9. package/spec/sdd.md +14 -988
  10. package/spec/tdd.md +15 -1232
  11. package/spec/test-large-document.js +20 -91
  12. package/src/config.js +57 -48
  13. package/src/core/documents/documents.js +62 -224
  14. package/src/core/embeddings/embedder.js +14 -54
  15. package/src/core/embeddings/queue.js +33 -106
  16. package/src/core/embeddings/vectorStore.js +30 -111
  17. package/src/core/embeddings/worker-process.js +23 -74
  18. package/src/core/indexing/indexer.js +51 -237
  19. package/src/core/indexing/parser.js +13 -114
  20. package/src/core/indexing/reindex.js +28 -137
  21. package/src/core/search/search.js +64 -158
  22. package/src/db/migrations/001_initial.sql +14 -63
  23. package/src/db/sqlite.js +19 -29
  24. package/src/fs/paths.js +11 -101
  25. package/src/fs/watcher.js +15 -48
  26. package/src/mcp/cli-adapter.js +32 -110
  27. package/src/mcp/server.js +64 -141
  28. package/src/mcp/tool-registry.js +21 -132
  29. package/src/mcp/tools.js +27 -199
  30. package/src/utils/markdown.js +15 -82
  31. package/spec/fixtures/realistic-content.js +0 -106
  32. package/spec/reporter.js +0 -127
  33. package/spec/run-tests-runner.js +0 -40
  34. package/src/core/embeddings/worker.js +0 -27
  35. package/src/core/prompts/prompts.js +0 -15
  36. package/src/core/sessions/sessions.js +0 -27
  37. package/src/utils/ids.js +0 -27
@@ -1,119 +1,46 @@
1
- /**
2
- * Embedding job queue: enqueue chunk ids after indexing; worker processes pending jobs.
3
- */
1
+ import { randomUUID } from 'crypto';
4
2
  import { getDb } from '../../db/sqlite.js';
5
3
 
6
- const MODEL_NAME = 'Xenova_multilingual_e5_large';
7
- const STATUS = { PENDING: 'pending', PROCESSING: 'processing', COMPLETED: 'completed', FAILED: 'failed', STALE: 'stale' };
8
-
9
- /** Queue key for jobs; use a single normalized name so enqueue and process always match. */
10
- function getModelName(config) {
11
- return MODEL_NAME;
12
- }
13
-
14
- /**
15
- * Enqueue chunks for embedding. Idempotent per chunk: existing pending/processing left as-is; completed/failed/stale replaced by pending.
16
- */
17
- function enqueueChunks(config, chunkIds) {
18
- if (!config?.embeddings?.enabled || !chunkIds?.length) return;
4
+ export function enqueueDocuments(config, ids) {
5
+ if (!config.embeddings.enabled) return;
19
6
  const db = getDb(config);
20
- const model = getModelName(config);
21
7
  const now = new Date().toISOString();
22
- const insert = db.prepare(`
23
- INSERT INTO embedding_jobs (id, chunk_id, model_name, status, attempts, created_at, updated_at)
24
- VALUES (?, ?, ?, ?, 0, ?, ?)
25
- ON CONFLICT(id) DO UPDATE SET status = 'pending', attempts = 0, last_error = NULL, updated_at = ?
26
- WHERE embedding_jobs.status IN ('completed', 'failed', 'stale')
27
- `);
28
- for (const cid of chunkIds) {
29
- const id = `${cid}:${model}`;
30
- insert.run(id, cid, model, STATUS.PENDING, now, now, now);
31
- }
8
+ const insert = db.prepare(`INSERT INTO embedding_jobs (id,document_id,content_hash,model_name,status,attempts,created_at,updated_at)
9
+ SELECT ?,id,content_hash,?,'pending',0,?,? FROM documents WHERE id=? AND deleted_at IS NULL
10
+ ON CONFLICT(id) DO UPDATE SET content_hash=excluded.content_hash,status='pending',attempts=0,last_error=NULL,updated_at=excluded.updated_at
11
+ WHERE embedding_jobs.content_hash != excluded.content_hash`);
12
+ for (const id of ids) insert.run(`${id}:${config.embeddings.model}`, config.embeddings.model, now, now, id);
32
13
  }
33
14
 
34
- /**
35
- * Mark jobs for these chunk ids as stale (e.g. before re-indexing the document).
36
- */
37
- function markStale(config, chunkIds) {
38
- if (!chunkIds?.length) return;
15
+ export async function processNextJob(config) {
16
+ if (!config.embeddings.enabled) return false;
39
17
  const db = getDb(config);
40
- const model = getModelName(config);
41
18
  const now = new Date().toISOString();
42
- const update = db.prepare(`
43
- UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE chunk_id = ? AND model_name = ?
44
- `);
45
- for (const cid of chunkIds) {
46
- update.run(STATUS.STALE, now, cid, model);
47
- }
48
- }
49
-
50
- /**
51
- * Process one pending job: load chunk content, embed, write to vector store, mark completed/failed.
52
- */
53
- async function processNextJob(config) {
54
- if (!config?.embeddings?.enabled) return false;
55
- const db = getDb(config);
56
- const model = getModelName(config);
57
- const job = db.prepare(`
58
- SELECT id, chunk_id FROM embedding_jobs
59
- WHERE status = ? AND model_name = ?
60
- ORDER BY created_at ASC LIMIT 1
61
- `).get(STATUS.PENDING, model);
19
+ const expired = new Date(Date.now() - config.embeddings.jobLeaseMs).toISOString();
20
+ const token = randomUUID();
21
+ const job = db.transaction(() => {
22
+ const next = db.prepare(`SELECT * FROM embedding_jobs WHERE model_name=? AND
23
+ (status='pending' OR (status='processing' AND updated_at < ?)) ORDER BY created_at,id LIMIT 1`).get(config.embeddings.model, expired);
24
+ if (next) db.prepare("UPDATE embedding_jobs SET status='processing',attempts=attempts+1,updated_at=?,claim=? WHERE id=?").run(now, token, next.id);
25
+ return next;
26
+ })();
62
27
  if (!job) return false;
63
-
64
- const now = new Date().toISOString();
65
- db.prepare(`UPDATE embedding_jobs SET status = ?, attempts = attempts + 1, updated_at = ? WHERE id = ?`)
66
- .run(STATUS.PROCESSING, now, job.id);
67
-
68
- let embedder;
69
- let vectorStore;
70
- try {
71
- const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
72
- import('./embedder.js'),
73
- import('./vectorStore.js')
74
- ]);
75
- embedder = await getEmbedder(config);
76
- vectorStore = await getVectorStore(config);
77
- if (!embedder || !vectorStore) {
78
- db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
79
- .run(STATUS.FAILED, 'embeddings not available', now, job.id);
80
- return true;
81
- }
82
- } catch (err) {
83
- db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
84
- .run(STATUS.FAILED, String(err?.message || err), now, job.id);
85
- return true;
86
- }
87
-
88
- const docRow = db.prepare(`
89
- SELECT id, id AS document_id, content_hash, body AS content
90
- FROM documents
91
- WHERE id = ?
92
- `).get(job.chunk_id);
93
- if (!docRow?.content) {
94
- db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
95
- .run(STATUS.FAILED, 'document not found or empty body', now, job.id);
96
- return true;
97
- }
98
-
28
+ const doc = db.prepare('SELECT id,content_hash,body FROM documents WHERE id=? AND deleted_at IS NULL').get(job.document_id);
99
29
  try {
100
- const embedding = await embedder.embed(docRow.content);
101
- await vectorStore.insert(config, {
102
- chunk_id: docRow.id,
103
- document_id: docRow.document_id,
104
- content_hash: docRow.content_hash,
105
- embedding
106
- });
107
- db.prepare(`UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE id = ?`)
108
- .run(STATUS.COMPLETED, now, job.id);
109
- } catch (err) {
110
- const maxRetries = config?.embeddings?.maxRetries ?? 3;
111
- const attempts = db.prepare('SELECT attempts FROM embedding_jobs WHERE id = ?').get(job.id)?.attempts ?? 1;
112
- const nextStatus = attempts >= maxRetries ? STATUS.FAILED : STATUS.PENDING;
113
- db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
114
- .run(nextStatus, String(err?.message || err), now, job.id);
30
+ if (!doc) { db.prepare('DELETE FROM embedding_jobs WHERE id=? AND claim=?').run(job.id, token); return true; }
31
+ const [{ getEmbedder }, { getVectorStore }] = await Promise.all([import('./embedder.js'), import('./vectorStore.js')]);
32
+ const embedder = await getEmbedder(config);
33
+ const store = await getVectorStore(config);
34
+ const embedding = await embedder.embed(doc.body);
35
+ const current = db.prepare('SELECT content_hash,deleted_at FROM documents WHERE id=?').get(doc.id);
36
+ if (!current || current.deleted_at || current.content_hash !== doc.content_hash) return true;
37
+ await store.insert(config, { id: doc.id, content_hash: doc.content_hash, embedding });
38
+ db.prepare("UPDATE embedding_jobs SET status='completed',last_error=NULL WHERE id=? AND claim=? AND content_hash=? AND status='processing'")
39
+ .run(job.id, token, doc.content_hash);
40
+ } catch (error) {
41
+ const status = job.attempts + 1 >= config.embeddings.maxRetries ? 'failed' : 'pending';
42
+ db.prepare("UPDATE embedding_jobs SET status=?,last_error=? WHERE id=? AND claim=? AND content_hash=? AND status='processing'")
43
+ .run(status, error.message, job.id, token, job.content_hash);
115
44
  }
116
45
  return true;
117
46
  }
118
-
119
- export { enqueueChunks, markStale, processNextJob, getModelName, STATUS };
@@ -1,126 +1,45 @@
1
- /**
2
- * Vector storage using sqlite-vec. Loaded dynamically so it's optional.
3
- */
4
1
  import { getDb } from '../../db/sqlite.js';
2
+ import { documentFilter } from '../search/search.js';
5
3
 
6
- const MODEL_SUFFIX = 'Xenova_multilingual_e5_large';
7
-
8
- function getCacheTableName() {
9
- return `vec_cache_${MODEL_SUFFIX}`;
10
- }
11
-
12
- function getVecTableName() {
13
- return `vec_${MODEL_SUFFIX}`;
14
- }
15
-
16
- let sqliteVecLoaded = false;
17
-
18
- async function ensureSqliteVecLoaded(config) {
19
- if (sqliteVecLoaded) return;
20
- try {
21
- const sqliteVec = await import('sqlite-vec');
22
- const db = getDb(config);
23
- sqliteVec.load(db);
24
- sqliteVecLoaded = true;
25
- } catch (_) {
26
- throw new Error('sqlite-vec not available');
27
- }
28
- }
4
+ const loaded = new WeakSet();
29
5
 
30
6
  async function ensureTables(config) {
31
7
  const db = getDb(config);
32
- const cacheTable = getCacheTableName();
33
- const vecTable = getVecTableName();
34
- const exists = db.prepare(
35
- "SELECT name FROM sqlite_master WHERE type='table' AND name=?"
36
- ).get(cacheTable);
37
- if (exists) return;
38
- await ensureSqliteVecLoaded(config);
39
- db.exec(`
40
- CREATE TABLE IF NOT EXISTS ${cacheTable} (
41
- id INTEGER PRIMARY KEY AUTOINCREMENT,
42
- chunk_id TEXT NOT NULL UNIQUE,
43
- document_id TEXT NOT NULL,
44
- content_hash TEXT NOT NULL,
45
- embedding_model TEXT NOT NULL,
46
- created_at TEXT NOT NULL
47
- )
48
- `);
49
- db.exec(`
50
- CREATE VIRTUAL TABLE IF NOT EXISTS ${vecTable} USING vec0(
51
- embedding float[1024] distance_metric=cosine
52
- )
53
- `);
8
+ if (!loaded.has(db)) {
9
+ const vec = await import('sqlite-vec');
10
+ vec.load(db);
11
+ loaded.add(db);
12
+ }
13
+ db.exec(`CREATE TABLE IF NOT EXISTS document_vectors (
14
+ document_id TEXT NOT NULL, model TEXT NOT NULL, content_hash TEXT NOT NULL, embedding BLOB NOT NULL,
15
+ PRIMARY KEY(document_id,model)
16
+ )`);
17
+ return db;
54
18
  }
55
19
 
56
- /**
57
- * Insert a vector for a chunk. Call after embedding the chunk content.
58
- */
59
- async function insert(config, { chunk_id, document_id, content_hash, embedding }) {
60
- if (!config?.embeddings?.enabled) return;
61
- await ensureTables(config);
62
- const db = getDb(config);
63
- const cacheTable = getCacheTableName();
64
- const vecTable = getVecTableName();
65
- const now = new Date().toISOString();
66
- const model = config.embeddings?.model || MODEL_SUFFIX;
67
- const emb = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
68
- db.prepare(`INSERT INTO ${vecTable} (embedding) VALUES (?)`).run(emb);
69
- const rawId = db.prepare('SELECT last_insert_rowid() AS id').get().id;
70
- const id = typeof rawId === 'bigint' ? Number(rawId) : parseInt(rawId, 10);
71
- db.prepare(
72
- `INSERT INTO ${cacheTable} (id, chunk_id, document_id, content_hash, embedding_model, created_at) VALUES (?, ?, ?, ?, ?, ?)`
73
- ).run(id, chunk_id, document_id, content_hash, model, now);
20
+ function vectorBytes(config, vector) {
21
+ if (vector.length !== config.embeddings.dimensions || !Array.from(vector).every(Number.isFinite)) throw new Error('Invalid embedding dimensions or values');
22
+ return new Float32Array(vector);
74
23
  }
75
24
 
76
- /**
77
- * KNN search. Returns [{ chunk_id, document_id, distance }]. distance is cosine distance (0 = identical).
78
- */
79
- async function search(config, queryEmbedding, k = 20, options = {}) {
80
- if (!config?.embeddings?.enabled) return [];
81
- await ensureTables(config);
82
- const db = getDb(config);
83
- const cacheTable = getCacheTableName();
84
- const vecTable = getVecTableName();
85
- const emb = queryEmbedding instanceof Float32Array ? queryEmbedding : new Float32Array(queryEmbedding);
86
- const rows = db.prepare(
87
- `SELECT v.rowid, v.distance, c.chunk_id, c.document_id
88
- FROM ${vecTable} v
89
- JOIN ${cacheTable} c ON c.id = v.rowid
90
- WHERE v.embedding MATCH ? AND k = ?`
91
- ).all(emb, Math.min(k, 100));
92
- const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.80;
93
- return rows
94
- .filter((r) => (1 - r.distance) >= threshold)
95
- .map((r) => ({ chunk_id: r.chunk_id, document_id: r.document_id, distance: r.distance }));
25
+ async function insert(config, { id, content_hash, embedding }) {
26
+ const db = await ensureTables(config);
27
+ db.prepare(`INSERT INTO document_vectors VALUES (?,?,?,?) ON CONFLICT(document_id,model)
28
+ DO UPDATE SET content_hash=excluded.content_hash,embedding=excluded.embedding`)
29
+ .run(id, config.embeddings.model, content_hash, vectorBytes(config, embedding));
96
30
  }
97
31
 
98
- /**
99
- * Remove vectors for given chunk ids (e.g. on document update/invalidation).
100
- */
101
- async function removeChunks(config, chunkIds) {
102
- if (!chunkIds?.length || !config?.embeddings?.enabled) return;
103
- await ensureTables(config);
104
- const db = getDb(config);
105
- const cacheTable = getCacheTableName();
106
- const vecTable = getVecTableName();
107
- for (const cid of chunkIds) {
108
- const row = db.prepare(`SELECT id FROM ${cacheTable} WHERE chunk_id = ?`).get(cid);
109
- if (row) {
110
- db.prepare(`DELETE FROM ${vecTable} WHERE rowid = ?`).run(row.id);
111
- db.prepare(`DELETE FROM ${cacheTable} WHERE id = ?`).run(row.id);
112
- }
113
- }
32
+ async function search(config, vector, options = {}) {
33
+ const db = await ensureTables(config);
34
+ const filter = documentFilter(config, options);
35
+ return db.prepare(`SELECT d.*,vec_distance_cosine(v.embedding,?) AS distance
36
+ FROM document_vectors v JOIN documents d ON d.id=v.document_id AND d.content_hash=v.content_hash
37
+ WHERE v.model=? AND ${filter.sql} AND distance <= ? ORDER BY distance,d.id`)
38
+ .all(vectorBytes(config, vector), config.embeddings.model, ...filter.params, 1 - config.embeddings.similarityThreshold);
114
39
  }
115
40
 
116
41
  export async function getVectorStore(config) {
117
- if (!config?.embeddings?.enabled) return null;
118
- try {
119
- await ensureSqliteVecLoaded(config);
120
- return { insert, search, removeChunks, ensureTables };
121
- } catch {
122
- return null;
123
- }
42
+ if (!config.embeddings.enabled) return null;
43
+ await ensureTables(config);
44
+ return { insert, search };
124
45
  }
125
-
126
- export { insert, search, removeChunks, ensureTables, getCacheTableName, getVecTableName };
@@ -1,79 +1,28 @@
1
- /**
2
- * Standalone process for embedding jobs and query embeddings. Isolates
3
- * @huggingface/transformers (and any native/ONNX crashes) from the MCP server.
4
- * Invoked via child_process.fork() with env MIXGRAM_WORKER_CONFIG pointing to
5
- * a JSON file with the resolved config. Listens for IPC 'embed' to embed query
6
- * text for hybrid search so the main process never loads transformers.
7
- */
8
1
  import fs from 'fs';
9
- import path from 'path';
10
- import { fileURLToPath } from 'url';
11
-
12
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
13
- const DEFAULT_POLL_MS = 2000;
14
-
15
- function loadWorkerConfig() {
16
- const configPath = process.env.MIXGRAM_WORKER_CONFIG;
17
- if (!configPath || !fs.existsSync(configPath)) return null;
18
- const raw = fs.readFileSync(configPath, 'utf8');
19
- return JSON.parse(raw);
20
- }
21
-
22
- let embedderInstance = null;
23
-
24
- async function getEmbedder(config) {
25
- if (embedderInstance) return embedderInstance;
26
- const { getEmbedder: getEmbedderFn } = await import('./embedder.js');
27
- embedderInstance = await getEmbedderFn(config);
28
- return embedderInstance;
29
- }
30
-
31
- function setupIpc(config) {
32
- process.on('message', async (msg) => {
33
- if (msg?.type !== 'embed' || typeof msg.text !== 'string') return;
34
- try {
35
- const embedder = await getEmbedder(config);
36
- if (!embedder) {
37
- process.send?.({ type: 'embedResult', id: msg.id, err: 'embeddings not available' });
38
- return;
39
- }
40
- const vector = await embedder.embed(msg.text);
41
- process.send?.({ type: 'embedResult', id: msg.id, vector: Array.from(vector) });
42
- } catch (err) {
43
- process.send?.({ type: 'embedResult', id: msg.id, err: String(err?.message ?? err) });
44
- }
45
- });
46
- }
2
+ import { getEmbedder } from './embedder.js';
3
+ import { processNextJob } from './queue.js';
4
+ import { closeDb } from '../../db/sqlite.js';
5
+
6
+ const config = JSON.parse(fs.readFileSync(process.env.MIXGRAM_WORKER_CONFIG, 'utf8'));
7
+ let stopped = false;
8
+ process.on('disconnect', () => { stopped = true; closeDb(config); process.exit(0); });
9
+ process.on('message', async msg => {
10
+ if (msg?.type !== 'embed' || typeof msg.text !== 'string') return;
11
+ try {
12
+ const embedder = await getEmbedder(config);
13
+ const vector = await embedder.embed(msg.text);
14
+ if (process.connected) process.send({ type: 'embedResult', id: msg.id, vector: Array.from(vector) });
15
+ } catch (error) {
16
+ if (process.connected) process.send({ type: 'embedResult', id: msg.id, err: error.message });
17
+ }
18
+ });
47
19
 
48
- async function runLoop(config) {
49
- const { processNextJob } = await import('./queue.js');
50
- const pollMs = config?.embeddings?.workerPollMs ?? DEFAULT_POLL_MS;
51
- const run = async () => {
20
+ async function loop() {
21
+ while (!stopped) {
52
22
  try {
53
- await processNextJob(config);
54
- } catch (err) {
55
- if (process.stderr) {
56
- process.stderr.write(`[mixgram-worker] job error: ${err?.message ?? err}\n`);
57
- }
58
- }
59
- };
60
- await run();
61
- setInterval(run, pollMs);
62
- }
63
-
64
- function main() {
65
- const config = loadWorkerConfig();
66
- if (!config?.embeddings?.enabled) {
67
- process.exit(0);
68
- return;
23
+ if (await processNextJob(config)) continue;
24
+ } catch (error) { process.stderr.write(`[mixgram-worker] ${error.message}\n`); }
25
+ await new Promise(resolve => setTimeout(resolve, config.embeddings.workerPollMs));
69
26
  }
70
- if (process.send) setupIpc(config);
71
- runLoop(config).catch((err) => {
72
- if (process.stderr) {
73
- process.stderr.write(`[mixgram-worker] fatal: ${err?.message ?? err}\n`);
74
- }
75
- process.exit(1);
76
- });
77
27
  }
78
-
79
- main();
28
+ loop().catch(error => { process.stderr.write(`${error.message}\n`); process.exit(1); });