mixgram 1.3.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -345
- package/bin/mixgram.js +51 -160
- package/examples/embedding-demo.js +22 -142
- package/package.json +19 -19
- package/spec/bench-indexing.js +23 -211
- package/spec/bench-search.js +26 -212
- package/spec/run-tests.js +1 -214
- package/spec/scenarios/scenarios.js +277 -607
- package/spec/sdd.md +14 -988
- package/spec/tdd.md +15 -1232
- package/spec/test-large-document.js +20 -91
- package/src/config.js +57 -48
- package/src/core/documents/documents.js +62 -224
- package/src/core/embeddings/embedder.js +14 -54
- package/src/core/embeddings/queue.js +33 -106
- package/src/core/embeddings/vectorStore.js +30 -111
- package/src/core/embeddings/worker-process.js +23 -74
- package/src/core/indexing/indexer.js +51 -237
- package/src/core/indexing/parser.js +13 -114
- package/src/core/indexing/reindex.js +28 -137
- package/src/core/search/search.js +64 -158
- package/src/db/migrations/001_initial.sql +14 -63
- package/src/db/sqlite.js +19 -29
- package/src/fs/paths.js +11 -101
- package/src/fs/watcher.js +15 -48
- package/src/mcp/cli-adapter.js +32 -110
- package/src/mcp/server.js +64 -141
- package/src/mcp/tool-registry.js +21 -132
- package/src/mcp/tools.js +27 -199
- package/src/utils/markdown.js +15 -82
- package/spec/fixtures/realistic-content.js +0 -106
- package/spec/reporter.js +0 -127
- package/spec/run-tests-runner.js +0 -40
- package/src/core/embeddings/worker.js +0 -27
- package/src/core/prompts/prompts.js +0 -15
- package/src/core/sessions/sessions.js +0 -27
- package/src/utils/ids.js +0 -27
|
@@ -1,257 +1,71 @@
|
|
|
1
|
-
import path from 'path';
|
|
2
1
|
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
3
|
import { getDb } from '../../db/sqlite.js';
|
|
4
|
-
import {
|
|
4
|
+
import { parse } from '../../utils/markdown.js';
|
|
5
|
+
import { extractMarkdownFields } from './parser.js';
|
|
5
6
|
import { contentHash } from '../../utils/hash.js';
|
|
7
|
+
import { collectionForPath } from '../../fs/paths.js';
|
|
8
|
+
import { enqueueDocuments } from '../embeddings/queue.js';
|
|
6
9
|
|
|
7
|
-
function normalizeForFts(
|
|
8
|
-
|
|
9
|
-
return String(str)
|
|
10
|
-
.normalize('NFD')
|
|
11
|
-
.replace(/\p{Diacritic}/gu, '')
|
|
12
|
-
.toLowerCase()
|
|
13
|
-
.trim();
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Infer scope, type and project from file path. Path shape: <project>/docs/<type>/<title>.md → project = dirname(docs). */
|
|
17
|
-
function inferMetadataFromPath(config, docPath) {
|
|
18
|
-
let norm = path.normalize(path.resolve(docPath));
|
|
19
|
-
let homeRoot = config.homeMemoryRoot ? path.normalize(path.resolve(config.homeMemoryRoot)) : '';
|
|
20
|
-
let projectRoot = config.projectMemoryRoot ? path.normalize(path.resolve(config.projectMemoryRoot)) : '';
|
|
21
|
-
try {
|
|
22
|
-
if (norm && fs.existsSync(norm)) norm = fs.realpathSync(norm);
|
|
23
|
-
if (homeRoot && fs.existsSync(homeRoot)) homeRoot = fs.realpathSync(homeRoot);
|
|
24
|
-
if (projectRoot && fs.existsSync(projectRoot)) projectRoot = fs.realpathSync(projectRoot);
|
|
25
|
-
} catch (_) {}
|
|
26
|
-
|
|
27
|
-
const dirToType = {
|
|
28
|
-
architecture: 'architecture',
|
|
29
|
-
decisions: 'decision',
|
|
30
|
-
bugs: 'bug',
|
|
31
|
-
learnings: 'learning',
|
|
32
|
-
discoveries: 'discovery',
|
|
33
|
-
patterns: 'pattern',
|
|
34
|
-
reference: 'reference',
|
|
35
|
-
sessions: 'session_summary',
|
|
36
|
-
prompts: 'prompt',
|
|
37
|
-
generated: 'generated_note'
|
|
38
|
-
};
|
|
39
|
-
let scope = 'project';
|
|
40
|
-
let type = 'generated_note';
|
|
41
|
-
let project = null;
|
|
42
|
-
|
|
43
|
-
const underHome = homeRoot && (() => {
|
|
44
|
-
if (norm === homeRoot) return true;
|
|
45
|
-
const sep = path.sep;
|
|
46
|
-
return norm.startsWith(homeRoot + sep);
|
|
47
|
-
})();
|
|
48
|
-
const underProject = projectRoot && (() => {
|
|
49
|
-
if (norm === projectRoot) return true;
|
|
50
|
-
const sep = path.sep;
|
|
51
|
-
return norm.startsWith(projectRoot + sep);
|
|
52
|
-
})();
|
|
53
|
-
|
|
54
|
-
if (underHome) {
|
|
55
|
-
scope = 'home';
|
|
56
|
-
const rel = path.relative(homeRoot, path.dirname(norm));
|
|
57
|
-
const firstDir = rel.split(path.sep)[0];
|
|
58
|
-
type = dirToType[firstDir] ?? 'generated_note';
|
|
59
|
-
} else if (underProject) {
|
|
60
|
-
scope = 'project';
|
|
61
|
-
const rel = path.relative(projectRoot, path.dirname(norm));
|
|
62
|
-
const firstDir = rel.split(path.sep)[0];
|
|
63
|
-
type = dirToType[firstDir] ?? 'generated_note';
|
|
64
|
-
const projectDir = path.dirname(projectRoot);
|
|
65
|
-
if (projectDir && projectDir !== path.dirname(norm)) project = path.basename(projectDir);
|
|
66
|
-
}
|
|
67
|
-
return { scope, type, project };
|
|
10
|
+
export function normalizeForFts(text) {
|
|
11
|
+
return text.normalize('NFD').replace(/\p{Diacritic}/gu, '').toLowerCase();
|
|
68
12
|
}
|
|
69
13
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
if (!row) return null;
|
|
77
|
-
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
78
|
-
const tags = tagRows.map((r) => r.tag);
|
|
14
|
+
export function readDocument(config, filePath, raw = fs.readFileSync(filePath, 'utf8')) {
|
|
15
|
+
const { frontmatter, body } = parse(raw);
|
|
16
|
+
const collection = collectionForPath(config, filePath);
|
|
17
|
+
const stat = fs.statSync(filePath);
|
|
18
|
+
const fields = extractMarkdownFields(body, config.indexing);
|
|
19
|
+
const created = frontmatter.created_at ?? stat.birthtime.toISOString();
|
|
79
20
|
return {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
duplicate_count: row.duplicate_count ?? 0,
|
|
92
|
-
deleted: !!row.deleted_at,
|
|
93
|
-
tags: Array.isArray(tags) ? tags : [],
|
|
94
|
-
embedding_status: row.embedding_status ?? 'disabled',
|
|
95
|
-
deleted_at: row.deleted_at ?? null
|
|
21
|
+
metadata: {
|
|
22
|
+
id: frontmatter.id ?? contentHash(path.resolve(filePath)).slice(0, 32),
|
|
23
|
+
title: frontmatter.title ?? fields.title ?? '',
|
|
24
|
+
type: frontmatter.type ?? 'note',
|
|
25
|
+
key: frontmatter.key ?? null,
|
|
26
|
+
tags: [...new Set(frontmatter.tags ?? [])],
|
|
27
|
+
created_at: created,
|
|
28
|
+
updated_at: frontmatter.updated_at ?? stat.mtime.toISOString(),
|
|
29
|
+
deleted_at: frontmatter.deleted_at ?? null
|
|
30
|
+
},
|
|
31
|
+
collection, body, fields, raw, mtime: stat.mtimeMs, path: path.resolve(filePath)
|
|
96
32
|
};
|
|
97
33
|
}
|
|
98
34
|
|
|
99
|
-
function indexDocument(config,
|
|
35
|
+
export function indexDocument(config, filePath, raw) {
|
|
36
|
+
const doc = readDocument(config, filePath, raw);
|
|
100
37
|
const db = getDb(config);
|
|
101
|
-
const
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
105
|
-
/** Prefer caller override, then DB row by path, then parsed front matter. */
|
|
106
|
-
let fm = options.overrideFrontmatter;
|
|
107
|
-
let byPath = null;
|
|
108
|
-
if (!fm) {
|
|
109
|
-
byPath = loadMetadataByPath(db, docPath);
|
|
110
|
-
fm = byPath ? { ...frontmatter, ...byPath } : frontmatter;
|
|
111
|
-
} else {
|
|
112
|
-
fm = { ...frontmatter, ...fm };
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
/** When metadata did not come from DB (full reindex or new file), infer scope/type/project from path. */
|
|
116
|
-
if (!options.overrideFrontmatter && !byPath) {
|
|
117
|
-
const inferred = inferMetadataFromPath(config, docPath);
|
|
118
|
-
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null);
|
|
119
|
-
fm = {
|
|
120
|
-
...fm,
|
|
121
|
-
scope: inferred.scope,
|
|
122
|
-
type: fm.type ?? inferred.type,
|
|
123
|
-
project: inferred.project,
|
|
124
|
-
created_at: fm.created_at || fileCreatedAt || undefined,
|
|
125
|
-
updated_at: fm.updated_at || fileCreatedAt || undefined
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
/** If still no id (e.g. full reindex of file with no front matter), generate from path and set created/updated. */
|
|
130
|
-
if (!fm.id) {
|
|
131
|
-
const inferred = inferMetadataFromPath(config, docPath);
|
|
132
|
-
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null) ?? new Date().toISOString();
|
|
133
|
-
fm = {
|
|
134
|
-
...fm,
|
|
135
|
-
id: contentHash(docPath).slice(0, 10),
|
|
136
|
-
scope: inferred.scope,
|
|
137
|
-
type: fm.type ?? inferred.type,
|
|
138
|
-
project: inferred.project,
|
|
139
|
-
created_at: fm.created_at || fileCreatedAt,
|
|
140
|
-
updated_at: fm.updated_at || fileCreatedAt
|
|
141
|
-
};
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
const id = fm.id;
|
|
145
|
-
const now = new Date().toISOString();
|
|
146
|
-
const docHash = contentHash(rawContent);
|
|
147
|
-
const docTitle = fm.title || title || '';
|
|
148
|
-
|
|
149
|
-
const existing = db.prepare('SELECT content_hash, file_mtime_ms, path FROM documents WHERE id = ?').get(id);
|
|
150
|
-
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
151
|
-
/** If path changed (move), update row so path stays in sync. */
|
|
152
|
-
const normPath = path.normalize(docPath);
|
|
153
|
-
if (existing.path && path.normalize(existing.path) !== normPath) {
|
|
154
|
-
db.prepare('UPDATE documents SET path = ?, file_mtime_ms = ? WHERE id = ?').run(path.normalize(docPath), fileMtimeMs, id);
|
|
155
|
-
}
|
|
156
|
-
return { id };
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
const docRow = {
|
|
160
|
-
id,
|
|
161
|
-
path: path.normalize(docPath),
|
|
162
|
-
title: docTitle,
|
|
163
|
-
type: fm.type || 'generated_note',
|
|
164
|
-
scope: fm.scope || 'project',
|
|
165
|
-
project: fm.project || null,
|
|
166
|
-
topic_key: fm.topic_key || null,
|
|
167
|
-
session_id: fm.session_id || null,
|
|
168
|
-
tool_name: fm.tool_name || null,
|
|
169
|
-
content_hash: docHash,
|
|
170
|
-
file_mtime_ms: fileMtimeMs,
|
|
171
|
-
created_at: fm.created_at || now,
|
|
172
|
-
updated_at: fm.updated_at || now,
|
|
173
|
-
indexed_at: now,
|
|
174
|
-
deleted_at: fm.deleted_at || null,
|
|
175
|
-
revision_count: fm.revision_count ?? 1,
|
|
176
|
-
duplicate_count: fm.duplicate_count ?? 0,
|
|
177
|
-
embedding_status: fm.embedding_status || 'disabled',
|
|
178
|
-
body: body || '',
|
|
179
|
-
structure: JSON.stringify(structure),
|
|
180
|
-
sections_index: JSON.stringify(sectionsIndex)
|
|
181
|
-
};
|
|
182
|
-
|
|
183
|
-
let wasExisting = false;
|
|
38
|
+
const m = doc.metadata;
|
|
39
|
+
const hash = contentHash(doc.raw);
|
|
184
40
|
db.transaction(() => {
|
|
185
|
-
const
|
|
186
|
-
if (
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
const h3Norm = normalizeForFts(h3);
|
|
204
|
-
const h4Norm = normalizeForFts(h4);
|
|
205
|
-
const h5Norm = normalizeForFts(h5);
|
|
206
|
-
const h6Norm = normalizeForFts(h6);
|
|
207
|
-
const bodyNorm = normalizeForFts(body);
|
|
208
|
-
|
|
209
|
-
db.prepare(`
|
|
210
|
-
INSERT INTO document_fts (document_id, title, h1, h2, h3, h4, h5, h6, body)
|
|
211
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
212
|
-
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
213
|
-
|
|
214
|
-
const tags = fm.tags || [];
|
|
215
|
-
for (const tag of tags) {
|
|
216
|
-
db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
|
|
41
|
+
const existing = db.prepare('SELECT path FROM documents WHERE id = ?').get(m.id);
|
|
42
|
+
if (existing && existing.path !== doc.path && fs.existsSync(existing.path)) throw new Error(`Duplicate document id: ${m.id}`);
|
|
43
|
+
const byPath = db.prepare('SELECT id FROM documents WHERE path = ?').get(doc.path);
|
|
44
|
+
if (byPath && byPath.id !== m.id) removeDocumentFromIndex(config, byPath.id);
|
|
45
|
+
db.prepare(`INSERT INTO documents (id,path,collection,title,type,key,tags,created_at,updated_at,deleted_at,content_hash,file_mtime_ms,body)
|
|
46
|
+
VALUES (@id,@path,@collection,@title,@type,@key,@tags,@created_at,@updated_at,@deleted_at,@content_hash,@file_mtime_ms,@body)
|
|
47
|
+
ON CONFLICT(id) DO UPDATE SET path=excluded.path,collection=excluded.collection,title=excluded.title,type=excluded.type,
|
|
48
|
+
key=excluded.key,tags=excluded.tags,created_at=excluded.created_at,updated_at=excluded.updated_at,deleted_at=excluded.deleted_at,
|
|
49
|
+
content_hash=excluded.content_hash,file_mtime_ms=excluded.file_mtime_ms,body=excluded.body`).run({
|
|
50
|
+
...m, tags: JSON.stringify(m.tags), path: doc.path, collection: doc.collection, content_hash: hash, file_mtime_ms: doc.mtime, body: doc.body
|
|
51
|
+
});
|
|
52
|
+
db.prepare('DELETE FROM document_fts WHERE document_id = ?').run(m.id);
|
|
53
|
+
if (!m.deleted_at) {
|
|
54
|
+
const fields = [m.title, ...['h1','h2','h3','h4','h5','h6','body'].map(k => doc.fields[k])].map(normalizeForFts);
|
|
55
|
+
db.prepare('INSERT INTO document_fts VALUES (?,?,?,?,?,?,?,?,?)').run(m.id, ...fields);
|
|
56
|
+
enqueueDocuments(config, [m.id]);
|
|
57
|
+
} else {
|
|
58
|
+
db.prepare('DELETE FROM embedding_jobs WHERE document_id = ?').run(m.id);
|
|
217
59
|
}
|
|
218
60
|
})();
|
|
219
|
-
|
|
220
|
-
if (config?.embeddings?.enabled) {
|
|
221
|
-
import('../embeddings/queue.js')
|
|
222
|
-
.then(({ enqueueChunks, markStale }) => {
|
|
223
|
-
if (wasExisting) markStale(config, [id]);
|
|
224
|
-
enqueueChunks(config, [id]);
|
|
225
|
-
})
|
|
226
|
-
.catch((err) => {
|
|
227
|
-
if (typeof process !== 'undefined' && process.stderr) {
|
|
228
|
-
process.stderr.write(`[mixgram] embeddings queue error: ${err?.message ?? err}\n`);
|
|
229
|
-
}
|
|
230
|
-
});
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
return { id };
|
|
61
|
+
return { id: m.id };
|
|
234
62
|
}
|
|
235
63
|
|
|
236
|
-
function removeDocumentFromIndex(config,
|
|
64
|
+
export function removeDocumentFromIndex(config, id) {
|
|
237
65
|
const db = getDb(config);
|
|
238
|
-
if (config?.embeddings?.enabled) {
|
|
239
|
-
import('../embeddings/queue.js')
|
|
240
|
-
.then(({ markStale }) => markStale(config, [documentId]))
|
|
241
|
-
.catch(() => {});
|
|
242
|
-
import('../embeddings/vectorStore.js')
|
|
243
|
-
.then(({ removeChunks }) => removeChunks(config, [documentId]))
|
|
244
|
-
.catch(() => {});
|
|
245
|
-
}
|
|
246
|
-
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(documentId);
|
|
247
66
|
db.transaction(() => {
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
|
|
253
|
-
db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
|
|
67
|
+
db.prepare('DELETE FROM document_fts WHERE document_id = ?').run(id);
|
|
68
|
+
db.prepare('DELETE FROM embedding_jobs WHERE document_id = ?').run(id);
|
|
69
|
+
db.prepare('DELETE FROM documents WHERE id = ?').run(id);
|
|
254
70
|
})();
|
|
255
71
|
}
|
|
256
|
-
|
|
257
|
-
export { indexDocument, removeDocumentFromIndex, normalizeForFts };
|
|
@@ -1,125 +1,24 @@
|
|
|
1
1
|
import { unified } from 'unified';
|
|
2
2
|
import remarkParse from 'remark-parse';
|
|
3
3
|
import { visit } from 'unist-util-visit';
|
|
4
|
-
import { parse as parseMarkdownWithFrontmatter } from '../../utils/markdown.js';
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
function extractTextFromNode(node) {
|
|
10
|
-
if (!node) return '';
|
|
11
|
-
if (node.type === 'text' || node.type === 'inlineCode') {
|
|
12
|
-
return node.value || '';
|
|
13
|
-
}
|
|
14
|
-
if (node.children && Array.isArray(node.children)) {
|
|
15
|
-
return node.children.map((child) => extractTextFromNode(child)).join('');
|
|
16
|
-
}
|
|
17
|
-
return '';
|
|
5
|
+
function extractText(node) {
|
|
6
|
+
if (node.type === 'text' || node.type === 'inlineCode') return node.value;
|
|
7
|
+
return (node.children ?? []).map(extractText).join('');
|
|
18
8
|
}
|
|
19
9
|
|
|
20
|
-
|
|
21
|
-
* Build mdast from raw markdown and extract title, h1–h6, body, structure, sectionsIndex.
|
|
22
|
-
* One parse, one tree walk; no chunking. Compatible with the remark-parse indexing model.
|
|
23
|
-
*
|
|
24
|
-
* @param {string} markdown - Full markdown body (no frontmatter)
|
|
25
|
-
* @param {{ includeCodeBlocks?: boolean }} options
|
|
26
|
-
* @returns {{ title: string, h1: string, h2: string, h3: string, h4: string, h5: string, h6: string, body: string, structure: object[], sectionsIndex: object }}
|
|
27
|
-
*/
|
|
28
|
-
export function extractMarkdownFields(markdown, { includeCodeBlocks = false } = {}) {
|
|
10
|
+
export function extractMarkdownFields(markdown, { includeCodeBlocks }) {
|
|
29
11
|
const tree = unified().use(remarkParse).parse(markdown);
|
|
30
|
-
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const h6 = [];
|
|
37
|
-
const bodyParts = [];
|
|
38
|
-
const structure = [];
|
|
39
|
-
const stack = [];
|
|
40
|
-
const sectionsIndex = {};
|
|
41
|
-
let currentSection = null;
|
|
42
|
-
let sectionIdCounter = 0;
|
|
43
|
-
let title = '';
|
|
44
|
-
|
|
45
|
-
visit(tree, (node) => {
|
|
46
|
-
if (node.type === 'heading') {
|
|
47
|
-
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
48
|
-
const depth = node.depth;
|
|
49
|
-
|
|
50
|
-
switch (depth) {
|
|
51
|
-
case 1: h1.push(text); break;
|
|
52
|
-
case 2: h2.push(text); break;
|
|
53
|
-
case 3: h3.push(text); break;
|
|
54
|
-
case 4: h4.push(text); break;
|
|
55
|
-
case 5: h5.push(text); break;
|
|
56
|
-
case 6: h6.push(text); break;
|
|
57
|
-
default: break;
|
|
58
|
-
}
|
|
59
|
-
if (!title && depth === 1) title = text;
|
|
60
|
-
|
|
61
|
-
const section = {
|
|
62
|
-
id: `s${++sectionIdCounter}`,
|
|
63
|
-
depth,
|
|
64
|
-
heading: text,
|
|
65
|
-
parentId: null,
|
|
66
|
-
childrenIds: []
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
|
|
70
|
-
stack.pop();
|
|
71
|
-
}
|
|
72
|
-
if (stack.length > 0) {
|
|
73
|
-
const parent = stack[stack.length - 1];
|
|
74
|
-
section.parentId = parent.id;
|
|
75
|
-
parent.childrenIds.push(section.id);
|
|
76
|
-
} else {
|
|
77
|
-
structure.push(section);
|
|
78
|
-
}
|
|
79
|
-
stack.push(section);
|
|
80
|
-
sectionsIndex[section.id] = section;
|
|
81
|
-
currentSection = section;
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
if (node.type === 'paragraph') {
|
|
86
|
-
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
87
|
-
if (text) bodyParts.push(text);
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
if (includeCodeBlocks && node.type === 'code') {
|
|
92
|
-
const text = node.value?.trim();
|
|
93
|
-
if (text) bodyParts.push(text);
|
|
94
|
-
}
|
|
12
|
+
const headings = Array.from({ length: 6 }, () => []);
|
|
13
|
+
const paragraphs = [];
|
|
14
|
+
visit(tree, node => {
|
|
15
|
+
if (node.type === 'heading') headings[node.depth - 1].push(extractText(node));
|
|
16
|
+
else if (node.type === 'paragraph') paragraphs.push(extractText(node));
|
|
17
|
+
else if (includeCodeBlocks && node.type === 'code') paragraphs.push(node.value);
|
|
95
18
|
});
|
|
96
|
-
|
|
97
|
-
const body = bodyParts.join('\n\n');
|
|
98
|
-
|
|
99
19
|
return {
|
|
100
|
-
title:
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
h3: h3.join('\n'),
|
|
104
|
-
h4: h4.join('\n'),
|
|
105
|
-
h5: h5.join('\n'),
|
|
106
|
-
h6: h6.join('\n'),
|
|
107
|
-
body,
|
|
108
|
-
structure,
|
|
109
|
-
sectionsIndex
|
|
20
|
+
title: headings[0][0] ?? '',
|
|
21
|
+
...Object.fromEntries(headings.map((values,i) => [`h${i+1}`, values.join('\n')])),
|
|
22
|
+
body: paragraphs.join('\n\n')
|
|
110
23
|
};
|
|
111
24
|
}
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Parse raw content: frontmatter (gray-matter) + AST-based extraction of body fields.
|
|
115
|
-
* Returns one object per document (no chunking).
|
|
116
|
-
*
|
|
117
|
-
* @param {string} rawContent - Full file content including optional frontmatter
|
|
118
|
-
* @param {{ includeCodeBlocks?: boolean }} options
|
|
119
|
-
*/
|
|
120
|
-
export function parseMarkdown(rawContent, options = {}) {
|
|
121
|
-
const { frontmatter, body: rawBody } = parseMarkdownWithFrontmatter(rawContent);
|
|
122
|
-
const fields = extractMarkdownFields(rawBody, options);
|
|
123
|
-
const body = (fields.body && fields.body.trim()) ? fields.body : (rawBody || '').trim();
|
|
124
|
-
return { frontmatter, ...fields, body };
|
|
125
|
-
}
|
|
@@ -1,151 +1,42 @@
|
|
|
1
|
-
import path from 'path';
|
|
2
1
|
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
3
|
import { globSync } from 'glob';
|
|
4
4
|
import { getDb } from '../../db/sqlite.js';
|
|
5
|
+
import { enqueueDocuments } from '../embeddings/queue.js';
|
|
6
|
+
import { contentHash } from '../../utils/hash.js';
|
|
5
7
|
import { indexDocument, removeDocumentFromIndex } from './indexer.js';
|
|
6
|
-
import { TYPE_TO_DIR } from '../../fs/paths.js';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Load document metadata from DB by path (for reindex/watcher).
|
|
10
|
-
* Returns frontmatter-like object or null.
|
|
11
|
-
*/
|
|
12
|
-
function getDocumentMetaByPath(config, filePath) {
|
|
13
|
-
const db = getDb(config);
|
|
14
|
-
const norm = path.normalize(filePath);
|
|
15
|
-
const row = db.prepare(
|
|
16
|
-
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
17
|
-
).get(norm);
|
|
18
|
-
if (!row) return null;
|
|
19
|
-
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
20
|
-
const tags = tagRows.map((r) => r.tag);
|
|
21
|
-
return rowToFrontmatter(row, tags);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function rowToFrontmatter(row, tags = []) {
|
|
25
|
-
return {
|
|
26
|
-
id: row.id,
|
|
27
|
-
title: row.title ?? '',
|
|
28
|
-
type: row.type ?? 'generated_note',
|
|
29
|
-
scope: row.scope ?? 'project',
|
|
30
|
-
project: row.project ?? null,
|
|
31
|
-
topic_key: row.topic_key ?? null,
|
|
32
|
-
session_id: row.session_id ?? null,
|
|
33
|
-
tool_name: row.tool_name ?? null,
|
|
34
|
-
created_at: row.created_at,
|
|
35
|
-
updated_at: row.updated_at,
|
|
36
|
-
revision_count: row.revision_count ?? 1,
|
|
37
|
-
duplicate_count: row.duplicate_count ?? 0,
|
|
38
|
-
deleted: !!row.deleted_at,
|
|
39
|
-
tags: Array.isArray(tags) ? tags : [],
|
|
40
|
-
embedding_status: row.embedding_status ?? 'disabled',
|
|
41
|
-
deleted_at: row.deleted_at ?? null
|
|
42
|
-
};
|
|
43
|
-
}
|
|
44
8
|
|
|
45
|
-
/**
|
|
46
|
-
* List all Markdown file paths under home (cross-project) and project memory root (repo-local).
|
|
47
|
-
*/
|
|
48
9
|
function listMarkdownPaths(config) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const out = [];
|
|
52
|
-
if (fs.existsSync(homeRoot)) {
|
|
53
|
-
const homeFiles = globSync('**/*.md', { cwd: homeRoot });
|
|
54
|
-
out.push(...homeFiles.map((p) => path.join(homeRoot, p)));
|
|
55
|
-
}
|
|
56
|
-
if (projectRoot && fs.existsSync(projectRoot)) {
|
|
57
|
-
const projectFiles = globSync('**/*.md', { cwd: projectRoot });
|
|
58
|
-
out.push(...projectFiles.map((p) => path.join(projectRoot, p)));
|
|
59
|
-
}
|
|
60
|
-
return out;
|
|
10
|
+
return Object.values(config.collections).flatMap(root => fs.existsSync(root)
|
|
11
|
+
? globSync('**/*.md', { cwd: root, nodir: true, follow: false }).map(p => path.resolve(root, p)) : []);
|
|
61
12
|
}
|
|
62
13
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
*/
|
|
66
|
-
function clearIndex(config) {
|
|
14
|
+
export function reindex(config, { full = false } = {}) {
|
|
15
|
+
const paths = listMarkdownPaths(config);
|
|
67
16
|
const db = getDb(config);
|
|
17
|
+
const result = { scanned: paths.length, indexed: 0, skipped: 0, removed: 0, errors: [] };
|
|
68
18
|
db.transaction(() => {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
72
|
-
}
|
|
73
|
-
db.prepare('DELETE FROM document_tags').run();
|
|
74
|
-
db.prepare('DELETE FROM documents').run();
|
|
75
|
-
})();
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Full reindex: clear index then index every .md file under home and projects.
|
|
80
|
-
* Use for rebuild-from-scratch (e.g. after deleting SQLite).
|
|
81
|
-
*/
|
|
82
|
-
function fullReindex(config) {
|
|
83
|
-
clearIndex(config);
|
|
84
|
-
const paths = listMarkdownPaths(config);
|
|
85
|
-
let indexed = 0;
|
|
86
|
-
for (const filePath of paths) {
|
|
87
|
-
try {
|
|
88
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
89
|
-
const stat = fs.statSync(filePath);
|
|
90
|
-
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
91
|
-
indexDocument(config, filePath, raw, stat.mtimeMs, { fileCreatedAt });
|
|
92
|
-
indexed++;
|
|
93
|
-
} catch (err) {
|
|
94
|
-
// skip invalid or unreadable files
|
|
19
|
+
if (full) {
|
|
20
|
+
db.exec('DELETE FROM document_fts; DELETE FROM embedding_jobs; DELETE FROM documents;');
|
|
95
21
|
}
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
*/
|
|
106
|
-
function incrementalReindex(config) {
|
|
107
|
-
const paths = listMarkdownPaths(config);
|
|
108
|
-
const pathsSet = new Set(paths.map((p) => path.normalize(p)));
|
|
109
|
-
const db = getDb(config);
|
|
110
|
-
const getStored = db.prepare('SELECT id, file_mtime_ms FROM documents WHERE path = ?');
|
|
111
|
-
let indexed = 0;
|
|
112
|
-
let skipped = 0;
|
|
113
|
-
for (const filePath of paths) {
|
|
114
|
-
try {
|
|
115
|
-
const norm = path.normalize(filePath);
|
|
116
|
-
const stat = fs.statSync(filePath);
|
|
117
|
-
const row = getStored.get(norm);
|
|
118
|
-
if (row != null && row.file_mtime_ms != null && row.file_mtime_ms === stat.mtimeMs) {
|
|
119
|
-
skipped++;
|
|
120
|
-
continue;
|
|
22
|
+
for (const filePath of paths) {
|
|
23
|
+
try {
|
|
24
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
25
|
+
const stored = db.prepare('SELECT content_hash FROM documents WHERE path = ?').get(filePath);
|
|
26
|
+
if (!full && stored?.content_hash === contentHash(raw)) { result.skipped++; continue; }
|
|
27
|
+
indexDocument(config, filePath, raw);
|
|
28
|
+
result.indexed++;
|
|
29
|
+
} catch (error) {
|
|
30
|
+
result.errors.push({ path: filePath, error: error.message });
|
|
121
31
|
}
|
|
122
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
123
|
-
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
124
|
-
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
125
|
-
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
126
|
-
indexed++;
|
|
127
|
-
} catch (_) {}
|
|
128
|
-
}
|
|
129
|
-
const allDocs = db.prepare('SELECT id, path FROM documents').all();
|
|
130
|
-
let removed = 0;
|
|
131
|
-
for (const row of allDocs) {
|
|
132
|
-
const norm = path.normalize(row.path);
|
|
133
|
-
if (!pathsSet.has(norm)) {
|
|
134
|
-
removeDocumentFromIndex(config, row.id);
|
|
135
|
-
removed++;
|
|
136
32
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const norm = path.normalize(filePath);
|
|
147
|
-
const row = db.prepare('SELECT id FROM documents WHERE path = ?').get(norm);
|
|
148
|
-
return row?.id ?? null;
|
|
33
|
+
if (config.embeddings.enabled) {
|
|
34
|
+
enqueueDocuments(config, db.prepare('SELECT id FROM documents WHERE deleted_at IS NULL').all().map(row => row.id));
|
|
35
|
+
}
|
|
36
|
+
const present = new Set(paths);
|
|
37
|
+
for (const row of db.prepare('SELECT id,path FROM documents').all()) {
|
|
38
|
+
if (!present.has(row.path)) { removeDocumentFromIndex(config, row.id); result.removed++; }
|
|
39
|
+
}
|
|
40
|
+
})();
|
|
41
|
+
return result;
|
|
149
42
|
}
|
|
150
|
-
|
|
151
|
-
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath, getDocumentMetaByPath };
|