mixgram 1.3.6 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +68 -345
  2. package/bin/mixgram.js +51 -160
  3. package/examples/embedding-demo.js +22 -142
  4. package/package.json +19 -19
  5. package/spec/bench-indexing.js +23 -211
  6. package/spec/bench-search.js +26 -212
  7. package/spec/run-tests.js +1 -214
  8. package/spec/scenarios/scenarios.js +277 -607
  9. package/spec/sdd.md +14 -988
  10. package/spec/tdd.md +15 -1232
  11. package/spec/test-large-document.js +20 -91
  12. package/src/config.js +57 -48
  13. package/src/core/documents/documents.js +62 -224
  14. package/src/core/embeddings/embedder.js +14 -54
  15. package/src/core/embeddings/queue.js +33 -106
  16. package/src/core/embeddings/vectorStore.js +30 -111
  17. package/src/core/embeddings/worker-process.js +23 -74
  18. package/src/core/indexing/indexer.js +51 -237
  19. package/src/core/indexing/parser.js +13 -114
  20. package/src/core/indexing/reindex.js +28 -137
  21. package/src/core/search/search.js +64 -158
  22. package/src/db/migrations/001_initial.sql +14 -63
  23. package/src/db/sqlite.js +19 -29
  24. package/src/fs/paths.js +11 -101
  25. package/src/fs/watcher.js +15 -48
  26. package/src/mcp/cli-adapter.js +32 -110
  27. package/src/mcp/server.js +64 -141
  28. package/src/mcp/tool-registry.js +21 -132
  29. package/src/mcp/tools.js +27 -199
  30. package/src/utils/markdown.js +15 -82
  31. package/spec/fixtures/realistic-content.js +0 -106
  32. package/spec/reporter.js +0 -127
  33. package/spec/run-tests-runner.js +0 -40
  34. package/src/core/embeddings/worker.js +0 -27
  35. package/src/core/prompts/prompts.js +0 -15
  36. package/src/core/sessions/sessions.js +0 -27
  37. package/src/utils/ids.js +0 -27
@@ -1,170 +1,76 @@
1
1
  import { getDb } from '../../db/sqlite.js';
2
2
  import { normalizeForFts } from '../indexing/indexer.js';
3
3
 
4
- function getBm25Weights(config) {
5
- const w = config.indexing?.ftsWeights || {};
6
- return [
7
- w.title ?? 10,
8
- w.h1 ?? 8,
9
- w.h2 ?? 6,
10
- w.h3 ?? 5,
11
- w.h4 ?? 4,
12
- w.h5 ?? 3,
13
- w.h6 ?? 2,
14
- w.body ?? 1
15
- ];
16
- }
17
-
18
- // FTS5 uses "/" for NEAR (e.g. term/5) and "-" for NOT; dash-like chars between words break parsing.
19
- const FTS5_NEAR_SLASH = /\//g;
20
- const FTS5_NOT_HYPHEN = /(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g; // hyphen, en/em-dash, minus
21
- // FTS5 operators are case-sensitive; normalizeForFts lowercases the string, so we restore them after.
22
- const FTS5_OPERATOR_RESTORE = [
23
- [/\band\b/g, 'AND'],
24
- [/\bor\b/g, 'OR'],
25
- [/\bnot\b/g, 'NOT']
26
- ];
27
-
28
- /**
29
- * Normalize query for FTS5: same as indexed text (NFD, remove diacritics, lowercase).
30
- * Replaces "/" with space so it is not interpreted as FTS5 NEAR syntax (e.g. term/5).
31
- * Replaces "-" (and Unicode dashes) between word chars with space so "alchemy-tycoon" is not parsed as "alchemy" NOT "tycoon".
32
- * Restores FTS5 operators AND, OR, NOT to uppercase after normalization (FTS5 requires them case-sensitive).
33
- */
34
- function normalizeQuery(q) {
35
- if (q == null || typeof q !== 'string') return '';
36
- const trimmed = q.trim();
37
- const slashSafe = trimmed.replace(FTS5_NEAR_SLASH, ' ');
38
- const hyphenSafe = slashSafe.replace(FTS5_NOT_HYPHEN, ' ');
39
- let out = normalizeForFts(hyphenSafe);
40
- for (const [re, replacement] of FTS5_OPERATOR_RESTORE) {
41
- out = out.replace(re, replacement);
4
+ export function documentFilter(config, { collection, type, tags = [] } = {}) {
5
+ if (collection && !Object.hasOwn(config.collections, collection)) throw new Error('Unknown collection');
6
+ const conditions = ['d.deleted_at IS NULL'];
7
+ const params = [];
8
+ const collections = collection ? [collection] : Object.keys(config.collections);
9
+ conditions.push(`d.collection IN (${collections.map(() => '?').join(',')})`);
10
+ params.push(...collections);
11
+ if (type) { conditions.push('d.type = ?'); params.push(type); }
12
+ for (const tag of new Set(tags)) {
13
+ conditions.push('EXISTS (SELECT 1 FROM json_each(d.tags) WHERE value = ?)');
14
+ params.push(tag);
42
15
  }
43
- return out;
16
+ return { sql: conditions.join(' AND '), params };
44
17
  }
45
18
 
46
- /**
47
- * @param {object} config - resolved config
48
- * @param {object} options
49
- * @param {string} options.query - search query (will be normalized for token match)
50
- * @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
51
- * @param {string} [options.project] - project name (required for project-only and merged)
52
- * @param {number} [options.limit] - max results
53
- * @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, created: string | null, snippet: string, score: number }>}
54
- */
55
- function search(config, options = {}) {
56
- const {
57
- query,
58
- scopeMode = config.search?.defaultScopeMode || 'merged',
59
- project = null,
60
- limit = config.search?.defaultLimit ?? 10
61
- } = options;
62
-
63
- if (!query || !String(query).trim()) {
64
- return [];
65
- }
66
-
67
- const db = getDb(config);
68
- const q = normalizeQuery(query);
69
- const weights = getBm25Weights(config);
70
- const bm25Args = weights.map((w, i) => (i === 0 ? w : `, ${w}`)).join('');
71
-
72
- let scopeCondition = '';
73
- const params = [q];
74
-
75
- if (scopeMode === 'project-only') {
76
- if (!project) return [];
77
- scopeCondition = ' AND d.scope = ? AND d.project = ? AND d.deleted_at IS NULL';
78
- params.push('project', project);
79
- } else if (scopeMode === 'home-only') {
80
- scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
81
- params.push('home');
82
- } else {
83
- if (project) {
84
- scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
85
- params.push('home', 'project', project);
86
- } else {
87
- scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR d.scope = ?)';
88
- params.push('home', 'project');
89
- }
90
- }
91
-
92
- params.push(project || '');
93
- params.push(limit);
94
-
95
- const sql = `
96
- SELECT
97
- f.document_id AS documentId,
98
- d.title AS title,
99
- d.topic_key AS topicKey,
100
- d.type AS type,
101
- d.scope AS scope,
102
- d.project AS project,
103
- d.created_at AS created,
104
- d.body AS body,
105
- snippet(document_fts, 1, '**', '**', '...', 24) AS snippetTitle,
106
- snippet(document_fts, 8, '**', '**', '...', 64) AS snippetBody,
107
- bm25(document_fts, ${bm25Args}) AS rank
108
- FROM document_fts f
109
- JOIN documents d ON d.id = f.document_id
110
- WHERE document_fts MATCH ? ${scopeCondition}
111
- ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
112
- LIMIT ?
113
- `;
114
-
115
- const stmt = db.prepare(sql);
116
- const rows = stmt.all(...params);
19
+ function normalizeQuery(query) {
20
+ let text = normalizeForFts(query.trim().replace(/\//g, ' ').replace(/(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g, ' '));
21
+ for (const op of ['and', 'or', 'not']) text = text.replace(new RegExp(`\\b${op}\\b`, 'g'), op.toUpperCase());
22
+ return text;
23
+ }
117
24
 
118
- return rows.map((r) => {
119
- const fromFts = [r.snippetTitle, r.snippetBody].filter((x) => x != null && x !== '').join(' … ');
120
- const fallback = r.body != null ? String(r.body).slice(0, 200) : '';
121
- const snippet = (fromFts && fromFts.trim()) || fallback || '';
122
- return {
123
- documentId: r.documentId,
124
- title: r.title,
125
- topicKey: r.topicKey,
126
- type: r.type,
127
- scope: r.scope,
128
- project: r.project,
129
- created: r.created || null,
130
- snippet: String(snippet).trim(),
131
- score: -Number(r.rank)
132
- };
133
- });
25
+ function resultRow(config, row, score) {
26
+ return {
27
+ id: row.id, title: row.title, collection: row.collection, type: row.type, key: row.key,
28
+ tags: JSON.parse(row.tags), created_at: row.created_at, updated_at: row.updated_at,
29
+ snippet: row.body.slice(0, config.search.snippetLength), ...(score === undefined ? {} : { score })
30
+ };
134
31
  }
135
32
 
136
- /**
137
- * Get recent context without FTS (for mem_context when no query).
138
- */
139
- function getRecentContext(config, options = {}) {
140
- const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
33
+ export async function search(config, options = {}) {
141
34
  const db = getDb(config);
142
- let sql = `
143
- SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.created_at AS created, d.body
144
- FROM documents d
145
- WHERE d.deleted_at IS NULL
146
- `;
147
- const params = [];
148
- if (project) {
149
- sql += ' AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
150
- params.push('home', 'project', project);
151
- } else {
152
- sql += ' AND d.scope = ?';
153
- params.push('home');
35
+ const filter = documentFilter(config, options);
36
+ const query = options.query?.trim();
37
+ const limit = options.limit ?? config.search.defaultLimit;
38
+ const offset = options.offset ?? 0;
39
+ if (!Number.isInteger(limit) || limit < 1 || limit > config.search.maxLimit || !Number.isInteger(offset) || offset < 0) throw new Error('Invalid pagination');
40
+ if (!query) {
41
+ const rows = db.prepare(`SELECT d.* FROM documents d WHERE ${filter.sql} ORDER BY d.updated_at DESC,d.id ASC LIMIT ? OFFSET ?`).all(...filter.params, limit, offset);
42
+ return { results: rows.map(row => resultRow(config, row)), mode: 'recent', degraded: false };
43
+ }
44
+ const weights = [0, ...Object.values(config.indexing.ftsWeights)].join(',');
45
+ const textRows = db.prepare(`SELECT d.*,bm25(document_fts,${weights}) AS rank FROM document_fts
46
+ JOIN documents d ON d.id=document_fts.document_id WHERE document_fts MATCH ? AND ${filter.sql}
47
+ ORDER BY rank,d.id`).all(normalizeQuery(query), ...filter.params);
48
+ let semanticRows = [];
49
+ let degraded = false;
50
+ let reason;
51
+ if (config.embeddings.enabled) {
52
+ try {
53
+ if (!config.getQueryEmbedding) throw new Error('Embedding worker unavailable');
54
+ const { getVectorStore } = await import('../embeddings/vectorStore.js');
55
+ const store = await getVectorStore(config);
56
+ if (!store) throw new Error('Vector store unavailable');
57
+ const vector = await config.getQueryEmbedding(query);
58
+ if (!vector) throw new Error('Query embedding unavailable');
59
+ semanticRows = await store.search(config, vector, options);
60
+ } catch (error) { degraded = true; reason = error.message; }
154
61
  }
155
- sql += ' ORDER BY d.updated_at DESC LIMIT ?';
156
- params.push(limit);
157
- const rows = db.prepare(sql).all(...params);
158
- return rows.map((r) => ({
159
- documentId: r.documentId,
160
- title: r.title,
161
- topicKey: r.topicKey,
162
- type: r.type,
163
- scope: r.scope,
164
- project: r.project,
165
- created: r.created || null,
166
- snippet: (r.body || '').slice(0, 300).trim()
167
- }));
62
+ const scores = new Map();
63
+ const add = (rows, weight) => rows.forEach((row, i) => {
64
+ const entry = scores.get(row.id) ?? { row, score: 0 };
65
+ entry.score += weight / (config.search.fusionConstant + i + 1);
66
+ scores.set(row.id, entry);
67
+ });
68
+ add(textRows, config.search.ftsWeight);
69
+ add(semanticRows, config.search.semanticWeight);
70
+ const ranked = [...scores.values()].sort((a,b) => b.score - a.score || a.row.id.localeCompare(b.row.id));
71
+ return {
72
+ results: ranked.slice(offset, offset + limit).map(({ row, score }) => resultRow(config, row, score)),
73
+ mode: config.embeddings.enabled && !degraded ? 'hybrid' : 'text', degraded,
74
+ ...(reason ? { reason } : {})
75
+ };
168
76
  }
169
-
170
- export { search, getRecentContext, normalizeQuery, getBm25Weights };
@@ -1,74 +1,25 @@
1
- -- documents (one row per doc; body and section tree stored here)
2
1
  CREATE TABLE IF NOT EXISTS documents (
3
2
  id TEXT PRIMARY KEY,
4
3
  path TEXT NOT NULL UNIQUE,
5
- title TEXT,
4
+ collection TEXT NOT NULL,
5
+ title TEXT NOT NULL,
6
6
  type TEXT NOT NULL,
7
- scope TEXT NOT NULL,
8
- project TEXT,
9
- topic_key TEXT,
10
- session_id TEXT,
11
- tool_name TEXT,
12
- content_hash TEXT NOT NULL,
13
- file_mtime_ms INTEGER,
7
+ key TEXT,
8
+ tags TEXT NOT NULL,
14
9
  created_at TEXT NOT NULL,
15
10
  updated_at TEXT NOT NULL,
16
- indexed_at TEXT,
17
11
  deleted_at TEXT,
18
- revision_count INTEGER NOT NULL DEFAULT 1,
19
- duplicate_count INTEGER NOT NULL DEFAULT 0,
20
- embedding_status TEXT NOT NULL DEFAULT 'disabled',
21
- body TEXT,
22
- structure TEXT,
23
- sections_index TEXT
24
- );
25
-
26
- CREATE TABLE IF NOT EXISTS document_tags (
27
- document_id TEXT NOT NULL,
28
- tag TEXT NOT NULL,
29
- PRIMARY KEY (document_id, tag),
30
- FOREIGN KEY(document_id) REFERENCES documents(id)
31
- );
32
-
33
- CREATE TABLE IF NOT EXISTS sessions (
34
- id TEXT PRIMARY KEY,
35
- project TEXT NOT NULL,
36
- directory TEXT,
37
- started_at TEXT NOT NULL,
38
- ended_at TEXT,
39
- summary TEXT
40
- );
41
-
42
- CREATE TABLE IF NOT EXISTS prompts (
43
- id TEXT PRIMARY KEY,
44
- session_id TEXT NOT NULL,
45
- project TEXT,
46
- content TEXT NOT NULL,
47
- created_at TEXT NOT NULL,
48
- FOREIGN KEY(session_id) REFERENCES sessions(id)
49
- );
50
-
51
- CREATE TABLE IF NOT EXISTS embedding_jobs (
52
- id TEXT PRIMARY KEY,
53
- chunk_id TEXT NOT NULL,
54
- model_name TEXT NOT NULL,
55
- status TEXT NOT NULL,
56
- attempts INTEGER NOT NULL DEFAULT 0,
57
- last_error TEXT,
58
- created_at TEXT NOT NULL,
59
- updated_at TEXT NOT NULL
12
+ content_hash TEXT NOT NULL,
13
+ file_mtime_ms REAL,
14
+ body TEXT NOT NULL
60
15
  );
61
-
62
- -- FTS5: one row per document (title, h1–h6, body)
16
+ CREATE UNIQUE INDEX IF NOT EXISTS documents_key ON documents(collection, key) WHERE key IS NOT NULL AND deleted_at IS NULL;
63
17
  CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
64
- document_id UNINDEXED,
65
- title,
66
- h1,
67
- h2,
68
- h3,
69
- h4,
70
- h5,
71
- h6,
72
- body,
18
+ document_id UNINDEXED, title, h1, h2, h3, h4, h5, h6, body,
73
19
  tokenize = 'unicode61 remove_diacritics 2'
74
20
  );
21
+ CREATE TABLE IF NOT EXISTS embedding_jobs (
22
+ id TEXT PRIMARY KEY, document_id TEXT NOT NULL, content_hash TEXT NOT NULL, model_name TEXT NOT NULL,
23
+ status TEXT NOT NULL, claim TEXT, attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT,
24
+ created_at TEXT NOT NULL, updated_at TEXT NOT NULL
25
+ );
package/src/db/sqlite.js CHANGED
@@ -1,39 +1,29 @@
1
1
  import Database from 'better-sqlite3';
2
2
  import fs from 'fs';
3
3
  import path from 'path';
4
- import { fileURLToPath } from 'url';
5
4
 
6
- const __dirname = path.dirname(fileURLToPath(import.meta.url));
7
-
8
- let dbInstance = null;
5
+ const databases = new WeakMap();
9
6
 
10
7
  export function getDb(config) {
11
- if (dbInstance) return dbInstance;
12
- const dbPath = config.sqlitePath;
13
- const dir = path.dirname(dbPath);
14
- if (!fs.existsSync(dir)) {
15
- fs.mkdirSync(dir, { recursive: true });
16
- }
17
- dbInstance = new Database(dbPath);
18
- dbInstance.pragma('journal_mode = WAL');
19
- runMigrations(dbInstance);
20
- return dbInstance;
21
- }
22
-
23
- function runMigrations(db) {
24
- const migrationsDir = path.join(__dirname, 'migrations');
25
- const files = fs.readdirSync(migrationsDir).filter(f => f.endsWith('.sql')).sort();
26
- for (const file of files) {
27
- const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8');
28
- db.exec(sql);
8
+ if (databases.has(config)) return databases.get(config);
9
+ fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
10
+ const db = new Database(config.sqlitePath);
11
+ try {
12
+ const existing = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='documents'").get();
13
+ if (existing && !db.prepare('PRAGMA table_info(documents)').all().some(c => c.name === 'collection')) {
14
+ throw new Error('Legacy database: Mixgram 2 requires a new SQLite path');
15
+ }
16
+ db.pragma('journal_mode = WAL');
17
+ db.exec(fs.readFileSync(new URL('./migrations/001_initial.sql', import.meta.url), 'utf8'));
18
+ databases.set(config, db);
19
+ return db;
20
+ } catch (error) {
21
+ db.close();
22
+ throw error;
29
23
  }
30
24
  }
31
25
 
32
- export function closeDb() {
33
- if (dbInstance) {
34
- dbInstance.close();
35
- dbInstance = null;
36
- }
26
+ export function closeDb(config) {
27
+ databases.get(config)?.close();
28
+ databases.delete(config);
37
29
  }
38
-
39
- export { runMigrations };
package/src/fs/paths.js CHANGED
@@ -1,108 +1,18 @@
1
1
  import path from 'path';
2
2
  import fs from 'fs';
3
3
 
4
- const TYPE_TO_DIR = {
5
- architecture: 'architecture',
6
- decision: 'decisions',
7
- decisions: 'decisions',
8
- bug: 'bugs',
9
- bugs: 'bugs',
10
- learning: 'learnings',
11
- learnings: 'learnings',
12
- discovery: 'discoveries',
13
- discoveries: 'discoveries',
14
- pattern: 'patterns',
15
- patterns: 'patterns',
16
- reference: 'reference',
17
- session_summary: 'sessions',
18
- sessions: 'sessions',
19
- prompt: 'prompts',
20
- prompts: 'prompts',
21
- generated_note: 'generated',
22
- generated: 'generated'
23
- };
24
-
25
- function slugFromTopicKey(topicKey) {
26
- if (!topicKey || typeof topicKey !== 'string') return 'note';
27
- return topicKey
28
- .replace(/\//g, '-')
29
- .replace(/[^a-z0-9-]/gi, '-')
30
- .replace(/-+/g, '-')
31
- .toLowerCase()
32
- .replace(/^-|-$/g, '') || 'note';
33
- }
34
-
35
- function filenameSlugFromTopicKey(topicKey, { project = null, type = null } = {}) {
36
- if (!topicKey || typeof topicKey !== 'string') return 'note';
37
-
38
- const segments = topicKey
39
- .split('/')
40
- .map((segment) => segment.trim())
41
- .filter(Boolean);
42
-
43
- if (segments.length === 0) return 'note';
44
-
45
- const normalizedProject = slugFromTopicKey(project);
46
- if (normalizedProject !== 'note' && slugFromTopicKey(segments[0]) === normalizedProject) {
47
- segments.shift();
48
- }
49
-
50
- const typeDir = TYPE_TO_DIR[type] || 'generated';
51
- const redundantPrefixes = new Set([
52
- slugFromTopicKey(type),
53
- slugFromTopicKey(typeDir)
54
- ].filter((value) => value !== 'note'));
55
-
56
- if (segments.length > 1 && redundantPrefixes.has(slugFromTopicKey(segments[0]))) {
57
- segments.shift();
58
- }
59
-
60
- return slugFromTopicKey(segments.join('-'));
61
- }
62
-
63
- function documentPath(config, { scope, project, type, topic_key, id }) {
64
- const typeDir = TYPE_TO_DIR[type] || 'generated';
65
- const slug = filenameSlugFromTopicKey(topic_key, { project, type });
66
- const baseName = `${slug}.md`;
67
-
68
- if (scope === 'home') {
69
- const root = config.homeMemoryRoot;
70
- return path.join(root, typeDir, baseName);
71
- }
72
-
73
- if (scope === 'project') {
74
- const root = config.projectMemoryRoot;
75
- return path.join(root, typeDir, baseName);
76
- }
77
-
78
- const root = config.projectMemoryRoot;
79
- return path.join(root, typeDir, baseName);
80
- }
81
-
82
- function ensureDir(filePath) {
83
- const dir = path.dirname(filePath);
84
- if (!fs.existsSync(dir)) {
85
- fs.mkdirSync(dir, { recursive: true });
4
+ export function collectionForPath(config, filePath) {
5
+ const absolute = path.resolve(filePath);
6
+ const real = fs.existsSync(absolute) ? fs.realpathSync(absolute) : absolute;
7
+ for (const [name, root] of Object.entries(config.collections)) {
8
+ if (absolute.startsWith(root + path.sep) && real.startsWith(root + path.sep)) return name;
86
9
  }
10
+ throw new Error('Document is outside configured collections');
87
11
  }
88
12
 
89
- function mixgramDir(config) {
90
- return path.dirname(config.sqlitePath);
91
- }
92
-
93
- function ensureMixgramDir(config) {
94
- const dir = mixgramDir(config);
95
- if (!fs.existsSync(dir)) {
96
- fs.mkdirSync(dir, { recursive: true });
97
- }
13
+ export function documentPath(config, { collection, title, id }) {
14
+ const root = config.collections[collection];
15
+ if (!root) throw new Error('Unknown collection');
16
+ const slug = title.normalize('NFD').replace(/\p{Diacritic}/gu, '').replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').slice(0, 80).toLowerCase() || 'note';
17
+ return path.join(root, `${slug}-${id}.md`);
98
18
  }
99
-
100
- export {
101
- documentPath,
102
- ensureDir,
103
- mixgramDir,
104
- ensureMixgramDir,
105
- TYPE_TO_DIR,
106
- slugFromTopicKey,
107
- filenameSlugFromTopicKey
108
- };
package/src/fs/watcher.js CHANGED
@@ -1,55 +1,22 @@
1
- import path from 'path';
2
1
  import chokidar from 'chokidar';
3
- import fs from 'fs';
2
+ import { getDb } from '../db/sqlite.js';
4
3
  import { indexDocument, removeDocumentFromIndex } from '../core/indexing/indexer.js';
5
- import { getDocumentIdByPath, getDocumentMetaByPath } from '../core/indexing/reindex.js';
6
4
 
7
- /**
8
- * Start watching home (cross-project) and project memory root (repo-local) for .md changes.
9
- * On add/change: index the file (using DB metadata when path already known). On unlink: remove from index by path.
10
- * @param {object} config - resolved config
11
- * @returns {import('chokidar').FSWatcher} watcher instance (call .close() to stop)
12
- */
13
- function startWatcher(config) {
14
- const homeRoot = config.homeMemoryRoot;
15
- const projectRoot = config.projectMemoryRoot;
16
- const patterns = [];
17
- if (homeRoot) patterns.push(path.join(homeRoot, '**/*.md'));
18
- if (projectRoot) patterns.push(path.join(projectRoot, '**/*.md'));
19
- if (patterns.length === 0) return null;
20
-
21
- const watcher = chokidar.watch(patterns, {
5
+ export function startWatcher(config, onError = error => process.stderr.write(`[mixgram] watch: ${error.message}\n`)) {
6
+ const watcher = chokidar.watch(Object.values(config.collections), {
22
7
  ignoreInitial: true,
23
- awaitWriteFinish: { stabilityThreshold: 200 }
8
+ awaitWriteFinish: { stabilityThreshold: config.indexing.watchStabilityMs }
24
9
  });
25
-
26
- watcher
27
- .on('add', (filePath) => {
28
- try {
29
- const raw = fs.readFileSync(filePath, 'utf8');
30
- const stat = fs.statSync(filePath);
31
- const dbMeta = getDocumentMetaByPath(config, filePath);
32
- const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
33
- indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
34
- } catch (_) {}
35
- })
36
- .on('change', (filePath) => {
37
- try {
38
- const raw = fs.readFileSync(filePath, 'utf8');
39
- const stat = fs.statSync(filePath);
40
- const dbMeta = getDocumentMetaByPath(config, filePath);
41
- const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
42
- indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
43
- } catch (_) {}
44
- })
45
- .on('unlink', (filePath) => {
46
- try {
47
- const id = getDocumentIdByPath(config, filePath);
48
- if (id) removeDocumentFromIndex(config, id);
49
- } catch (_) {}
50
- });
51
-
10
+ const update = filePath => {
11
+ if (!filePath.endsWith('.md')) return;
12
+ try { indexDocument(config, filePath); }
13
+ catch (error) { onError(error); }
14
+ };
15
+ watcher.on('add', update).on('change', update).on('unlink', filePath => {
16
+ try {
17
+ const row = getDb(config).prepare('SELECT id FROM documents WHERE path = ?').get(filePath);
18
+ if (row) removeDocumentFromIndex(config, row.id);
19
+ } catch (error) { onError(error); }
20
+ }).on('error', onError);
52
21
  return watcher;
53
22
  }
54
-
55
- export { startWatcher };