mixgram 1.3.7 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +68 -345
  2. package/bin/mixgram.js +51 -160
  3. package/examples/embedding-demo.js +22 -142
  4. package/package.json +2 -3
  5. package/pnpm-workspace.yaml +2 -2
  6. package/spec/bench-indexing.js +23 -211
  7. package/spec/bench-search.js +26 -212
  8. package/spec/run-tests.js +1 -214
  9. package/spec/scenarios/scenarios.js +277 -607
  10. package/spec/sdd.md +14 -988
  11. package/spec/tdd.md +15 -1232
  12. package/spec/test-large-document.js +20 -91
  13. package/src/config.js +57 -48
  14. package/src/core/documents/documents.js +62 -224
  15. package/src/core/embeddings/embedder.js +14 -54
  16. package/src/core/embeddings/queue.js +33 -106
  17. package/src/core/embeddings/vectorStore.js +30 -111
  18. package/src/core/embeddings/worker-process.js +23 -74
  19. package/src/core/indexing/indexer.js +51 -237
  20. package/src/core/indexing/parser.js +13 -114
  21. package/src/core/indexing/reindex.js +28 -137
  22. package/src/core/search/search.js +64 -158
  23. package/src/db/migrations/001_initial.sql +14 -63
  24. package/src/db/sqlite.js +19 -29
  25. package/src/fs/paths.js +11 -101
  26. package/src/fs/watcher.js +15 -48
  27. package/src/mcp/cli-adapter.js +32 -110
  28. package/src/mcp/server.js +64 -141
  29. package/src/mcp/tool-registry.js +21 -132
  30. package/src/mcp/tools.js +27 -199
  31. package/src/utils/markdown.js +15 -82
  32. package/patches/sharp@0.34.5.patch +0 -13
  33. package/spec/fixtures/realistic-content.js +0 -106
  34. package/spec/reporter.js +0 -127
  35. package/spec/run-tests-runner.js +0 -40
  36. package/src/core/embeddings/worker.js +0 -27
  37. package/src/core/prompts/prompts.js +0 -15
  38. package/src/core/sessions/sessions.js +0 -27
  39. package/src/utils/ids.js +0 -27
@@ -1,257 +1,71 @@
1
- import path from 'path';
2
1
  import fs from 'fs';
2
+ import path from 'path';
3
3
  import { getDb } from '../../db/sqlite.js';
4
- import { parseMarkdown } from './parser.js';
4
+ import { parse } from '../../utils/markdown.js';
5
+ import { extractMarkdownFields } from './parser.js';
5
6
  import { contentHash } from '../../utils/hash.js';
7
+ import { collectionForPath } from '../../fs/paths.js';
8
+ import { enqueueDocuments } from '../embeddings/queue.js';
6
9
 
7
- function normalizeForFts(str) {
8
- if (str == null || str === undefined) return '';
9
- return String(str)
10
- .normalize('NFD')
11
- .replace(/\p{Diacritic}/gu, '')
12
- .toLowerCase()
13
- .trim();
14
- }
15
-
16
- /** Infer scope, type and project from file path. Path shape: <project>/docs/<type>/<title>.md → project = dirname(docs). */
17
- function inferMetadataFromPath(config, docPath) {
18
- let norm = path.normalize(path.resolve(docPath));
19
- let homeRoot = config.homeMemoryRoot ? path.normalize(path.resolve(config.homeMemoryRoot)) : '';
20
- let projectRoot = config.projectMemoryRoot ? path.normalize(path.resolve(config.projectMemoryRoot)) : '';
21
- try {
22
- if (norm && fs.existsSync(norm)) norm = fs.realpathSync(norm);
23
- if (homeRoot && fs.existsSync(homeRoot)) homeRoot = fs.realpathSync(homeRoot);
24
- if (projectRoot && fs.existsSync(projectRoot)) projectRoot = fs.realpathSync(projectRoot);
25
- } catch (_) {}
26
-
27
- const dirToType = {
28
- architecture: 'architecture',
29
- decisions: 'decision',
30
- bugs: 'bug',
31
- learnings: 'learning',
32
- discoveries: 'discovery',
33
- patterns: 'pattern',
34
- reference: 'reference',
35
- sessions: 'session_summary',
36
- prompts: 'prompt',
37
- generated: 'generated_note'
38
- };
39
- let scope = 'project';
40
- let type = 'generated_note';
41
- let project = null;
42
-
43
- const underHome = homeRoot && (() => {
44
- if (norm === homeRoot) return true;
45
- const sep = path.sep;
46
- return norm.startsWith(homeRoot + sep);
47
- })();
48
- const underProject = projectRoot && (() => {
49
- if (norm === projectRoot) return true;
50
- const sep = path.sep;
51
- return norm.startsWith(projectRoot + sep);
52
- })();
53
-
54
- if (underHome) {
55
- scope = 'home';
56
- const rel = path.relative(homeRoot, path.dirname(norm));
57
- const firstDir = rel.split(path.sep)[0];
58
- type = dirToType[firstDir] ?? 'generated_note';
59
- } else if (underProject) {
60
- scope = 'project';
61
- const rel = path.relative(projectRoot, path.dirname(norm));
62
- const firstDir = rel.split(path.sep)[0];
63
- type = dirToType[firstDir] ?? 'generated_note';
64
- const projectDir = path.dirname(projectRoot);
65
- if (projectDir && projectDir !== path.dirname(norm)) project = path.basename(projectDir);
66
- }
67
- return { scope, type, project };
10
+ export function normalizeForFts(text) {
11
+ return text.normalize('NFD').replace(/\p{Diacritic}/gu, '').toLowerCase();
68
12
  }
69
13
 
70
- /** Load metadata from DB by path when no override provided. */
71
- function loadMetadataByPath(db, docPath) {
72
- const norm = path.normalize(docPath);
73
- const row = db.prepare(
74
- 'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
75
- ).get(norm);
76
- if (!row) return null;
77
- const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
78
- const tags = tagRows.map((r) => r.tag);
14
+ export function readDocument(config, filePath, raw = fs.readFileSync(filePath, 'utf8')) {
15
+ const { frontmatter, body } = parse(raw);
16
+ const collection = collectionForPath(config, filePath);
17
+ const stat = fs.statSync(filePath);
18
+ const fields = extractMarkdownFields(body, config.indexing);
19
+ const created = frontmatter.created_at ?? stat.birthtime.toISOString();
79
20
  return {
80
- id: row.id,
81
- title: row.title ?? '',
82
- type: row.type ?? 'generated_note',
83
- scope: row.scope ?? 'project',
84
- project: row.project ?? null,
85
- topic_key: row.topic_key ?? null,
86
- session_id: row.session_id ?? null,
87
- tool_name: row.tool_name ?? null,
88
- created_at: row.created_at,
89
- updated_at: row.updated_at,
90
- revision_count: row.revision_count ?? 1,
91
- duplicate_count: row.duplicate_count ?? 0,
92
- deleted: !!row.deleted_at,
93
- tags: Array.isArray(tags) ? tags : [],
94
- embedding_status: row.embedding_status ?? 'disabled',
95
- deleted_at: row.deleted_at ?? null
21
+ metadata: {
22
+ id: frontmatter.id ?? contentHash(path.resolve(filePath)).slice(0, 32),
23
+ title: frontmatter.title ?? fields.title ?? '',
24
+ type: frontmatter.type ?? 'note',
25
+ key: frontmatter.key ?? null,
26
+ tags: [...new Set(frontmatter.tags ?? [])],
27
+ created_at: created,
28
+ updated_at: frontmatter.updated_at ?? stat.mtime.toISOString(),
29
+ deleted_at: frontmatter.deleted_at ?? null
30
+ },
31
+ collection, body, fields, raw, mtime: stat.mtimeMs, path: path.resolve(filePath)
96
32
  };
97
33
  }
98
34
 
99
- function indexDocument(config, docPath, rawContent, fileMtimeMs = null, options = {}) {
35
+ export function indexDocument(config, filePath, raw) {
36
+ const doc = readDocument(config, filePath, raw);
100
37
  const db = getDb(config);
101
- const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
102
- const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
103
-
104
- const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
105
- /** Prefer caller override, then DB row by path, then parsed front matter. */
106
- let fm = options.overrideFrontmatter;
107
- let byPath = null;
108
- if (!fm) {
109
- byPath = loadMetadataByPath(db, docPath);
110
- fm = byPath ? { ...frontmatter, ...byPath } : frontmatter;
111
- } else {
112
- fm = { ...frontmatter, ...fm };
113
- }
114
-
115
- /** When metadata did not come from DB (full reindex or new file), infer scope/type/project from path. */
116
- if (!options.overrideFrontmatter && !byPath) {
117
- const inferred = inferMetadataFromPath(config, docPath);
118
- const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null);
119
- fm = {
120
- ...fm,
121
- scope: inferred.scope,
122
- type: fm.type ?? inferred.type,
123
- project: inferred.project,
124
- created_at: fm.created_at || fileCreatedAt || undefined,
125
- updated_at: fm.updated_at || fileCreatedAt || undefined
126
- };
127
- }
128
-
129
- /** If still no id (e.g. full reindex of file with no front matter), generate from path and set created/updated. */
130
- if (!fm.id) {
131
- const inferred = inferMetadataFromPath(config, docPath);
132
- const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null) ?? new Date().toISOString();
133
- fm = {
134
- ...fm,
135
- id: contentHash(docPath).slice(0, 10),
136
- scope: inferred.scope,
137
- type: fm.type ?? inferred.type,
138
- project: inferred.project,
139
- created_at: fm.created_at || fileCreatedAt,
140
- updated_at: fm.updated_at || fileCreatedAt
141
- };
142
- }
143
-
144
- const id = fm.id;
145
- const now = new Date().toISOString();
146
- const docHash = contentHash(rawContent);
147
- const docTitle = fm.title || title || '';
148
-
149
- const existing = db.prepare('SELECT content_hash, file_mtime_ms, path FROM documents WHERE id = ?').get(id);
150
- if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
151
- /** If path changed (move), update row so path stays in sync. */
152
- const normPath = path.normalize(docPath);
153
- if (existing.path && path.normalize(existing.path) !== normPath) {
154
- db.prepare('UPDATE documents SET path = ?, file_mtime_ms = ? WHERE id = ?').run(path.normalize(docPath), fileMtimeMs, id);
155
- }
156
- return { id };
157
- }
158
-
159
- const docRow = {
160
- id,
161
- path: path.normalize(docPath),
162
- title: docTitle,
163
- type: fm.type || 'generated_note',
164
- scope: fm.scope || 'project',
165
- project: fm.project || null,
166
- topic_key: fm.topic_key || null,
167
- session_id: fm.session_id || null,
168
- tool_name: fm.tool_name || null,
169
- content_hash: docHash,
170
- file_mtime_ms: fileMtimeMs,
171
- created_at: fm.created_at || now,
172
- updated_at: fm.updated_at || now,
173
- indexed_at: now,
174
- deleted_at: fm.deleted_at || null,
175
- revision_count: fm.revision_count ?? 1,
176
- duplicate_count: fm.duplicate_count ?? 0,
177
- embedding_status: fm.embedding_status || 'disabled',
178
- body: body || '',
179
- structure: JSON.stringify(structure),
180
- sections_index: JSON.stringify(sectionsIndex)
181
- };
182
-
183
- let wasExisting = false;
38
+ const m = doc.metadata;
39
+ const hash = contentHash(doc.raw);
184
40
  db.transaction(() => {
185
- const existingDoc = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
186
- if (existingDoc) {
187
- wasExisting = true;
188
- const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(id);
189
- for (const { rowid } of ftsRowids) {
190
- db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
191
- }
192
- db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(id);
193
- }
194
-
195
- db.prepare(`
196
- INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status, body, structure, sections_index)
197
- VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status, @body, @structure, @sections_index)
198
- `).run(docRow);
199
-
200
- const titleNorm = normalizeForFts(docTitle);
201
- const h1Norm = normalizeForFts(h1);
202
- const h2Norm = normalizeForFts(h2);
203
- const h3Norm = normalizeForFts(h3);
204
- const h4Norm = normalizeForFts(h4);
205
- const h5Norm = normalizeForFts(h5);
206
- const h6Norm = normalizeForFts(h6);
207
- const bodyNorm = normalizeForFts(body);
208
-
209
- db.prepare(`
210
- INSERT INTO document_fts (document_id, title, h1, h2, h3, h4, h5, h6, body)
211
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
212
- `).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
213
-
214
- const tags = fm.tags || [];
215
- for (const tag of tags) {
216
- db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
41
+ const existing = db.prepare('SELECT path FROM documents WHERE id = ?').get(m.id);
42
+ if (existing && existing.path !== doc.path && fs.existsSync(existing.path)) throw new Error(`Duplicate document id: ${m.id}`);
43
+ const byPath = db.prepare('SELECT id FROM documents WHERE path = ?').get(doc.path);
44
+ if (byPath && byPath.id !== m.id) removeDocumentFromIndex(config, byPath.id);
45
+ db.prepare(`INSERT INTO documents (id,path,collection,title,type,key,tags,created_at,updated_at,deleted_at,content_hash,file_mtime_ms,body)
46
+ VALUES (@id,@path,@collection,@title,@type,@key,@tags,@created_at,@updated_at,@deleted_at,@content_hash,@file_mtime_ms,@body)
47
+ ON CONFLICT(id) DO UPDATE SET path=excluded.path,collection=excluded.collection,title=excluded.title,type=excluded.type,
48
+ key=excluded.key,tags=excluded.tags,created_at=excluded.created_at,updated_at=excluded.updated_at,deleted_at=excluded.deleted_at,
49
+ content_hash=excluded.content_hash,file_mtime_ms=excluded.file_mtime_ms,body=excluded.body`).run({
50
+ ...m, tags: JSON.stringify(m.tags), path: doc.path, collection: doc.collection, content_hash: hash, file_mtime_ms: doc.mtime, body: doc.body
51
+ });
52
+ db.prepare('DELETE FROM document_fts WHERE document_id = ?').run(m.id);
53
+ if (!m.deleted_at) {
54
+ const fields = [m.title, ...['h1','h2','h3','h4','h5','h6','body'].map(k => doc.fields[k])].map(normalizeForFts);
55
+ db.prepare('INSERT INTO document_fts VALUES (?,?,?,?,?,?,?,?,?)').run(m.id, ...fields);
56
+ enqueueDocuments(config, [m.id]);
57
+ } else {
58
+ db.prepare('DELETE FROM embedding_jobs WHERE document_id = ?').run(m.id);
217
59
  }
218
60
  })();
219
-
220
- if (config?.embeddings?.enabled) {
221
- import('../embeddings/queue.js')
222
- .then(({ enqueueChunks, markStale }) => {
223
- if (wasExisting) markStale(config, [id]);
224
- enqueueChunks(config, [id]);
225
- })
226
- .catch((err) => {
227
- if (typeof process !== 'undefined' && process.stderr) {
228
- process.stderr.write(`[mixgram] embeddings queue error: ${err?.message ?? err}\n`);
229
- }
230
- });
231
- }
232
-
233
- return { id };
61
+ return { id: m.id };
234
62
  }
235
63
 
236
- function removeDocumentFromIndex(config, documentId) {
64
+ export function removeDocumentFromIndex(config, id) {
237
65
  const db = getDb(config);
238
- if (config?.embeddings?.enabled) {
239
- import('../embeddings/queue.js')
240
- .then(({ markStale }) => markStale(config, [documentId]))
241
- .catch(() => {});
242
- import('../embeddings/vectorStore.js')
243
- .then(({ removeChunks }) => removeChunks(config, [documentId]))
244
- .catch(() => {});
245
- }
246
- db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(documentId);
247
66
  db.transaction(() => {
248
- const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
249
- for (const { rowid } of ftsRowids) {
250
- db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
251
- }
252
- db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
253
- db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
67
+ db.prepare('DELETE FROM document_fts WHERE document_id = ?').run(id);
68
+ db.prepare('DELETE FROM embedding_jobs WHERE document_id = ?').run(id);
69
+ db.prepare('DELETE FROM documents WHERE id = ?').run(id);
254
70
  })();
255
71
  }
256
-
257
- export { indexDocument, removeDocumentFromIndex, normalizeForFts };
@@ -1,125 +1,24 @@
1
1
  import { unified } from 'unified';
2
2
  import remarkParse from 'remark-parse';
3
3
  import { visit } from 'unist-util-visit';
4
- import { parse as parseMarkdownWithFrontmatter } from '../../utils/markdown.js';
5
4
 
6
- /**
7
- * Recursively extract plain text from an mdast node (strips emphasis, links, etc.).
8
- */
9
- function extractTextFromNode(node) {
10
- if (!node) return '';
11
- if (node.type === 'text' || node.type === 'inlineCode') {
12
- return node.value || '';
13
- }
14
- if (node.children && Array.isArray(node.children)) {
15
- return node.children.map((child) => extractTextFromNode(child)).join('');
16
- }
17
- return '';
5
+ function extractText(node) {
6
+ if (node.type === 'text' || node.type === 'inlineCode') return node.value;
7
+ return (node.children ?? []).map(extractText).join('');
18
8
  }
19
9
 
20
- /**
21
- * Build mdast from raw markdown and extract title, h1–h6, body, structure, sectionsIndex.
22
- * One parse, one tree walk; no chunking. Compatible with the remark-parse indexing model.
23
- *
24
- * @param {string} markdown - Full markdown body (no frontmatter)
25
- * @param {{ includeCodeBlocks?: boolean }} options
26
- * @returns {{ title: string, h1: string, h2: string, h3: string, h4: string, h5: string, h6: string, body: string, structure: object[], sectionsIndex: object }}
27
- */
28
- export function extractMarkdownFields(markdown, { includeCodeBlocks = false } = {}) {
10
+ export function extractMarkdownFields(markdown, { includeCodeBlocks }) {
29
11
  const tree = unified().use(remarkParse).parse(markdown);
30
-
31
- const h1 = [];
32
- const h2 = [];
33
- const h3 = [];
34
- const h4 = [];
35
- const h5 = [];
36
- const h6 = [];
37
- const bodyParts = [];
38
- const structure = [];
39
- const stack = [];
40
- const sectionsIndex = {};
41
- let currentSection = null;
42
- let sectionIdCounter = 0;
43
- let title = '';
44
-
45
- visit(tree, (node) => {
46
- if (node.type === 'heading') {
47
- const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
48
- const depth = node.depth;
49
-
50
- switch (depth) {
51
- case 1: h1.push(text); break;
52
- case 2: h2.push(text); break;
53
- case 3: h3.push(text); break;
54
- case 4: h4.push(text); break;
55
- case 5: h5.push(text); break;
56
- case 6: h6.push(text); break;
57
- default: break;
58
- }
59
- if (!title && depth === 1) title = text;
60
-
61
- const section = {
62
- id: `s${++sectionIdCounter}`,
63
- depth,
64
- heading: text,
65
- parentId: null,
66
- childrenIds: []
67
- };
68
-
69
- while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
70
- stack.pop();
71
- }
72
- if (stack.length > 0) {
73
- const parent = stack[stack.length - 1];
74
- section.parentId = parent.id;
75
- parent.childrenIds.push(section.id);
76
- } else {
77
- structure.push(section);
78
- }
79
- stack.push(section);
80
- sectionsIndex[section.id] = section;
81
- currentSection = section;
82
- return;
83
- }
84
-
85
- if (node.type === 'paragraph') {
86
- const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
87
- if (text) bodyParts.push(text);
88
- return;
89
- }
90
-
91
- if (includeCodeBlocks && node.type === 'code') {
92
- const text = node.value?.trim();
93
- if (text) bodyParts.push(text);
94
- }
12
+ const headings = Array.from({ length: 6 }, () => []);
13
+ const paragraphs = [];
14
+ visit(tree, node => {
15
+ if (node.type === 'heading') headings[node.depth - 1].push(extractText(node));
16
+ else if (node.type === 'paragraph') paragraphs.push(extractText(node));
17
+ else if (includeCodeBlocks && node.type === 'code') paragraphs.push(node.value);
95
18
  });
96
-
97
- const body = bodyParts.join('\n\n');
98
-
99
19
  return {
100
- title: title || (h1[0] ?? ''),
101
- h1: h1.join('\n'),
102
- h2: h2.join('\n'),
103
- h3: h3.join('\n'),
104
- h4: h4.join('\n'),
105
- h5: h5.join('\n'),
106
- h6: h6.join('\n'),
107
- body,
108
- structure,
109
- sectionsIndex
20
+ title: headings[0][0] ?? '',
21
+ ...Object.fromEntries(headings.map((values,i) => [`h${i+1}`, values.join('\n')])),
22
+ body: paragraphs.join('\n\n')
110
23
  };
111
24
  }
112
-
113
- /**
114
- * Parse raw content: frontmatter (gray-matter) + AST-based extraction of body fields.
115
- * Returns one object per document (no chunking).
116
- *
117
- * @param {string} rawContent - Full file content including optional frontmatter
118
- * @param {{ includeCodeBlocks?: boolean }} options
119
- */
120
- export function parseMarkdown(rawContent, options = {}) {
121
- const { frontmatter, body: rawBody } = parseMarkdownWithFrontmatter(rawContent);
122
- const fields = extractMarkdownFields(rawBody, options);
123
- const body = (fields.body && fields.body.trim()) ? fields.body : (rawBody || '').trim();
124
- return { frontmatter, ...fields, body };
125
- }
@@ -1,151 +1,42 @@
1
- import path from 'path';
2
1
  import fs from 'fs';
2
+ import path from 'path';
3
3
  import { globSync } from 'glob';
4
4
  import { getDb } from '../../db/sqlite.js';
5
+ import { enqueueDocuments } from '../embeddings/queue.js';
6
+ import { contentHash } from '../../utils/hash.js';
5
7
  import { indexDocument, removeDocumentFromIndex } from './indexer.js';
6
- import { TYPE_TO_DIR } from '../../fs/paths.js';
7
-
8
- /**
9
- * Load document metadata from DB by path (for reindex/watcher).
10
- * Returns frontmatter-like object or null.
11
- */
12
- function getDocumentMetaByPath(config, filePath) {
13
- const db = getDb(config);
14
- const norm = path.normalize(filePath);
15
- const row = db.prepare(
16
- 'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
17
- ).get(norm);
18
- if (!row) return null;
19
- const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
20
- const tags = tagRows.map((r) => r.tag);
21
- return rowToFrontmatter(row, tags);
22
- }
23
-
24
- function rowToFrontmatter(row, tags = []) {
25
- return {
26
- id: row.id,
27
- title: row.title ?? '',
28
- type: row.type ?? 'generated_note',
29
- scope: row.scope ?? 'project',
30
- project: row.project ?? null,
31
- topic_key: row.topic_key ?? null,
32
- session_id: row.session_id ?? null,
33
- tool_name: row.tool_name ?? null,
34
- created_at: row.created_at,
35
- updated_at: row.updated_at,
36
- revision_count: row.revision_count ?? 1,
37
- duplicate_count: row.duplicate_count ?? 0,
38
- deleted: !!row.deleted_at,
39
- tags: Array.isArray(tags) ? tags : [],
40
- embedding_status: row.embedding_status ?? 'disabled',
41
- deleted_at: row.deleted_at ?? null
42
- };
43
- }
44
8
 
45
- /**
46
- * List all Markdown file paths under home (cross-project) and project memory root (repo-local).
47
- */
48
9
  function listMarkdownPaths(config) {
49
- const homeRoot = config.homeMemoryRoot;
50
- const projectRoot = config.projectMemoryRoot;
51
- const out = [];
52
- if (fs.existsSync(homeRoot)) {
53
- const homeFiles = globSync('**/*.md', { cwd: homeRoot });
54
- out.push(...homeFiles.map((p) => path.join(homeRoot, p)));
55
- }
56
- if (projectRoot && fs.existsSync(projectRoot)) {
57
- const projectFiles = globSync('**/*.md', { cwd: projectRoot });
58
- out.push(...projectFiles.map((p) => path.join(projectRoot, p)));
59
- }
60
- return out;
10
+ return Object.values(config.collections).flatMap(root => fs.existsSync(root)
11
+ ? globSync('**/*.md', { cwd: root, nodir: true, follow: false }).map(p => path.resolve(root, p)) : []);
61
12
  }
62
13
 
63
- /**
64
- * Clear entire text index (documents, chunks, tags, FTS). Use before full rebuild.
65
- */
66
- function clearIndex(config) {
14
+ export function reindex(config, { full = false } = {}) {
15
+ const paths = listMarkdownPaths(config);
67
16
  const db = getDb(config);
17
+ const result = { scanned: paths.length, indexed: 0, skipped: 0, removed: 0, errors: [] };
68
18
  db.transaction(() => {
69
- const ftsRowids = db.prepare('SELECT rowid FROM document_fts').all();
70
- for (const { rowid } of ftsRowids) {
71
- db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
72
- }
73
- db.prepare('DELETE FROM document_tags').run();
74
- db.prepare('DELETE FROM documents').run();
75
- })();
76
- }
77
-
78
- /**
79
- * Full reindex: clear index then index every .md file under home and projects.
80
- * Use for rebuild-from-scratch (e.g. after deleting SQLite).
81
- */
82
- function fullReindex(config) {
83
- clearIndex(config);
84
- const paths = listMarkdownPaths(config);
85
- let indexed = 0;
86
- for (const filePath of paths) {
87
- try {
88
- const raw = fs.readFileSync(filePath, 'utf8');
89
- const stat = fs.statSync(filePath);
90
- const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
91
- indexDocument(config, filePath, raw, stat.mtimeMs, { fileCreatedAt });
92
- indexed++;
93
- } catch (err) {
94
- // skip invalid or unreadable files
19
+ if (full) {
20
+ db.exec('DELETE FROM document_fts; DELETE FROM embedding_jobs; DELETE FROM documents;');
95
21
  }
96
- }
97
- return { indexed, total: paths.length };
98
- }
99
-
100
- /**
101
- * Incremental reindex: scan all .md files; skip unchanged (same path + file_mtime_ms);
102
- * index new or changed; remove from index documents whose path is no longer on disk.
103
- * Does not clear the index first.
104
- * @returns {{ scanned: number, skipped: number, indexed: number, removed: number, totalPaths: number }}
105
- */
106
- function incrementalReindex(config) {
107
- const paths = listMarkdownPaths(config);
108
- const pathsSet = new Set(paths.map((p) => path.normalize(p)));
109
- const db = getDb(config);
110
- const getStored = db.prepare('SELECT id, file_mtime_ms FROM documents WHERE path = ?');
111
- let indexed = 0;
112
- let skipped = 0;
113
- for (const filePath of paths) {
114
- try {
115
- const norm = path.normalize(filePath);
116
- const stat = fs.statSync(filePath);
117
- const row = getStored.get(norm);
118
- if (row != null && row.file_mtime_ms != null && row.file_mtime_ms === stat.mtimeMs) {
119
- skipped++;
120
- continue;
22
+ for (const filePath of paths) {
23
+ try {
24
+ const raw = fs.readFileSync(filePath, 'utf8');
25
+ const stored = db.prepare('SELECT content_hash FROM documents WHERE path = ?').get(filePath);
26
+ if (!full && stored?.content_hash === contentHash(raw)) { result.skipped++; continue; }
27
+ indexDocument(config, filePath, raw);
28
+ result.indexed++;
29
+ } catch (error) {
30
+ result.errors.push({ path: filePath, error: error.message });
121
31
  }
122
- const raw = fs.readFileSync(filePath, 'utf8');
123
- const dbMeta = getDocumentMetaByPath(config, filePath);
124
- const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
125
- indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
126
- indexed++;
127
- } catch (_) {}
128
- }
129
- const allDocs = db.prepare('SELECT id, path FROM documents').all();
130
- let removed = 0;
131
- for (const row of allDocs) {
132
- const norm = path.normalize(row.path);
133
- if (!pathsSet.has(norm)) {
134
- removeDocumentFromIndex(config, row.id);
135
- removed++;
136
32
  }
137
- }
138
- return { scanned: paths.length, skipped, indexed, removed, totalPaths: paths.length };
139
- }
140
-
141
- /**
142
- * Get document id by file path (for watcher on delete).
143
- */
144
- function getDocumentIdByPath(config, filePath) {
145
- const db = getDb(config);
146
- const norm = path.normalize(filePath);
147
- const row = db.prepare('SELECT id FROM documents WHERE path = ?').get(norm);
148
- return row?.id ?? null;
33
+ if (config.embeddings.enabled) {
34
+ enqueueDocuments(config, db.prepare('SELECT id FROM documents WHERE deleted_at IS NULL').all().map(row => row.id));
35
+ }
36
+ const present = new Set(paths);
37
+ for (const row of db.prepare('SELECT id,path FROM documents').all()) {
38
+ if (!present.has(row.path)) { removeDocumentFromIndex(config, row.id); result.removed++; }
39
+ }
40
+ })();
41
+ return result;
149
42
  }
150
-
151
- export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath, getDocumentMetaByPath };