mixgram 1.1.6 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mixgram.js +22 -0
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +12 -7
- package/spec/scenarios/scenarios.js +5 -3
- package/spec/test-large-document.js +98 -0
- package/src/config.js +7 -4
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -126
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/utils/markdown.js +42 -2
package/src/config.js
CHANGED
|
@@ -15,13 +15,16 @@ const DEFAULT_CONFIG = {
|
|
|
15
15
|
sqlitePath: '~/.mixgram/index.db',
|
|
16
16
|
watch: true,
|
|
17
17
|
indexing: {
|
|
18
|
-
chunkSize: 1200,
|
|
19
|
-
chunkOverlap: 120,
|
|
20
18
|
reindexOnStartup: true,
|
|
19
|
+
includeCodeBlocks: false,
|
|
21
20
|
ftsWeights: {
|
|
22
21
|
title: 10.0,
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
h1: 8.0,
|
|
23
|
+
h2: 6.0,
|
|
24
|
+
h3: 5.0,
|
|
25
|
+
h4: 4.0,
|
|
26
|
+
h5: 3.0,
|
|
27
|
+
h6: 2.0,
|
|
25
28
|
body: 1.0
|
|
26
29
|
}
|
|
27
30
|
},
|
|
@@ -157,9 +157,9 @@ function deleteDocument(config, documentId, options = {}) {
|
|
|
157
157
|
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
158
158
|
const db = getDb(config);
|
|
159
159
|
db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
|
|
160
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
160
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
|
|
161
161
|
for (const { rowid } of ftsRowids) {
|
|
162
|
-
db.prepare('DELETE FROM
|
|
162
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
165
|
return true;
|
|
@@ -84,23 +84,23 @@ async function processNextJob(config) {
|
|
|
84
84
|
return true;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
const
|
|
88
|
-
SELECT
|
|
89
|
-
FROM
|
|
90
|
-
WHERE
|
|
87
|
+
const docRow = db.prepare(`
|
|
88
|
+
SELECT id, id AS document_id, content_hash, body AS content
|
|
89
|
+
FROM documents
|
|
90
|
+
WHERE id = ?
|
|
91
91
|
`).get(job.chunk_id);
|
|
92
|
-
if (!
|
|
92
|
+
if (!docRow?.content) {
|
|
93
93
|
db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
|
|
94
|
-
.run(STATUS.FAILED, '
|
|
94
|
+
.run(STATUS.FAILED, 'document not found or empty body', now, job.id);
|
|
95
95
|
return true;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
try {
|
|
99
|
-
const embedding = await embedder.embed(
|
|
99
|
+
const embedding = await embedder.embed(docRow.content);
|
|
100
100
|
await vectorStore.insert(config, {
|
|
101
|
-
chunk_id:
|
|
102
|
-
document_id:
|
|
103
|
-
content_hash:
|
|
101
|
+
chunk_id: docRow.id,
|
|
102
|
+
document_id: docRow.document_id,
|
|
103
|
+
content_hash: docRow.content_hash,
|
|
104
104
|
embedding
|
|
105
105
|
});
|
|
106
106
|
db.prepare(`UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE id = ?`)
|
|
@@ -43,8 +43,7 @@ async function ensureTables(config) {
|
|
|
43
43
|
document_id TEXT NOT NULL,
|
|
44
44
|
content_hash TEXT NOT NULL,
|
|
45
45
|
embedding_model TEXT NOT NULL,
|
|
46
|
-
created_at TEXT NOT NULL
|
|
47
|
-
FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
|
|
46
|
+
created_at TEXT NOT NULL
|
|
48
47
|
)
|
|
49
48
|
`);
|
|
50
49
|
db.exec(`
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { getDb } from '../../db/sqlite.js';
|
|
2
|
-
import {
|
|
2
|
+
import { parseMarkdown } from './parser.js';
|
|
3
3
|
import { contentHash } from '../../utils/hash.js';
|
|
4
|
-
import { chunkId } from '../../utils/ids.js';
|
|
5
4
|
|
|
6
5
|
function normalizeForFts(str) {
|
|
7
6
|
if (str == null || str === undefined) return '';
|
|
@@ -14,21 +13,26 @@ function normalizeForFts(str) {
|
|
|
14
13
|
|
|
15
14
|
function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
16
15
|
const db = getDb(config);
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
chunkOverlap: config.indexing.chunkOverlap
|
|
20
|
-
});
|
|
16
|
+
const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
|
|
17
|
+
const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
|
|
21
18
|
|
|
19
|
+
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
22
20
|
const id = frontmatter.id;
|
|
23
21
|
if (!id) throw new Error('Document frontmatter must include id');
|
|
24
22
|
|
|
25
23
|
const now = new Date().toISOString();
|
|
26
24
|
const docHash = contentHash(rawContent);
|
|
25
|
+
const docTitle = frontmatter.title || title || '';
|
|
26
|
+
|
|
27
|
+
const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
|
|
28
|
+
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
29
|
+
return { id };
|
|
30
|
+
}
|
|
27
31
|
|
|
28
32
|
const docRow = {
|
|
29
33
|
id,
|
|
30
34
|
path: docPath,
|
|
31
|
-
title:
|
|
35
|
+
title: docTitle,
|
|
32
36
|
type: frontmatter.type || 'generated_note',
|
|
33
37
|
scope: frontmatter.scope || 'project',
|
|
34
38
|
project: frontmatter.project || null,
|
|
@@ -43,49 +47,42 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
43
47
|
deleted_at: frontmatter.deleted_at || null,
|
|
44
48
|
revision_count: frontmatter.revision_count ?? 1,
|
|
45
49
|
duplicate_count: frontmatter.duplicate_count ?? 0,
|
|
46
|
-
embedding_status: frontmatter.embedding_status || 'disabled'
|
|
50
|
+
embedding_status: frontmatter.embedding_status || 'disabled',
|
|
51
|
+
body: body || '',
|
|
52
|
+
structure: JSON.stringify(structure),
|
|
53
|
+
sections_index: JSON.stringify(sectionsIndex)
|
|
47
54
|
};
|
|
48
55
|
|
|
49
|
-
let
|
|
56
|
+
let wasExisting = false;
|
|
50
57
|
db.transaction(() => {
|
|
51
|
-
const
|
|
52
|
-
if (
|
|
53
|
-
|
|
54
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
58
|
+
const existingDoc = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
|
|
59
|
+
if (existingDoc) {
|
|
60
|
+
wasExisting = true;
|
|
61
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(id);
|
|
55
62
|
for (const { rowid } of ftsRowids) {
|
|
56
|
-
db.prepare('DELETE FROM
|
|
63
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
57
64
|
}
|
|
58
|
-
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(id);
|
|
59
65
|
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(id);
|
|
60
66
|
}
|
|
61
67
|
|
|
62
68
|
db.prepare(`
|
|
63
|
-
INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status)
|
|
64
|
-
VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status)
|
|
69
|
+
INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status, body, structure, sections_index)
|
|
70
|
+
VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status, @body, @structure, @sections_index)
|
|
65
71
|
`).run(docRow);
|
|
66
72
|
|
|
67
|
-
const titleNorm = normalizeForFts(
|
|
68
|
-
const
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const chunkHash = contentHash(chunk.content);
|
|
76
|
-
const headingPathNorm = normalizeForFts(chunk.headingPath);
|
|
77
|
-
const bodyNorm = normalizeForFts(chunk.content);
|
|
78
|
-
|
|
79
|
-
db.prepare(`
|
|
80
|
-
INSERT INTO document_chunks (id, document_id, chunk_index, heading_path, heading_level, content, content_hash, created_at, updated_at)
|
|
81
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
82
|
-
`).run(cid, id, idx, chunk.headingPath || '', chunk.headingLevel || 0, chunk.content, chunkHash, now, now);
|
|
73
|
+
const titleNorm = normalizeForFts(docTitle);
|
|
74
|
+
const h1Norm = normalizeForFts(h1);
|
|
75
|
+
const h2Norm = normalizeForFts(h2);
|
|
76
|
+
const h3Norm = normalizeForFts(h3);
|
|
77
|
+
const h4Norm = normalizeForFts(h4);
|
|
78
|
+
const h5Norm = normalizeForFts(h5);
|
|
79
|
+
const h6Norm = normalizeForFts(h6);
|
|
80
|
+
const bodyNorm = normalizeForFts(body);
|
|
83
81
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
});
|
|
82
|
+
db.prepare(`
|
|
83
|
+
INSERT INTO document_fts (document_id, title, h1, h2, h3, h4, h5, h6, body)
|
|
84
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
|
+
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
89
86
|
|
|
90
87
|
const tags = frontmatter.tags || [];
|
|
91
88
|
for (const tag of tags) {
|
|
@@ -94,51 +91,37 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
94
91
|
})();
|
|
95
92
|
|
|
96
93
|
if (config?.embeddings?.enabled) {
|
|
97
|
-
const newChunkIds = Array.from({ length: chunks.length }, (_, idx) => chunkId(id, idx));
|
|
98
94
|
import('../embeddings/queue.js')
|
|
99
95
|
.then(({ enqueueChunks, markStale }) => {
|
|
100
|
-
if (
|
|
101
|
-
enqueueChunks(config,
|
|
96
|
+
if (wasExisting) markStale(config, [id]);
|
|
97
|
+
enqueueChunks(config, [id]);
|
|
102
98
|
})
|
|
103
99
|
.catch((err) => {
|
|
104
100
|
if (typeof process !== 'undefined' && process.stderr) {
|
|
105
101
|
process.stderr.write(`[mixgram] embeddings queue error: ${err?.message ?? err}\n`);
|
|
106
102
|
}
|
|
107
103
|
});
|
|
108
|
-
if (oldChunkIds.length) {
|
|
109
|
-
import('../embeddings/vectorStore.js')
|
|
110
|
-
.then(({ removeChunks }) => removeChunks(config, oldChunkIds))
|
|
111
|
-
.catch(() => {});
|
|
112
|
-
}
|
|
113
104
|
}
|
|
114
105
|
|
|
115
|
-
return { id
|
|
106
|
+
return { id };
|
|
116
107
|
}
|
|
117
108
|
|
|
118
109
|
function removeDocumentFromIndex(config, documentId) {
|
|
119
110
|
const db = getDb(config);
|
|
120
|
-
|
|
121
|
-
if (config?.embeddings?.enabled && chunkIds.length) {
|
|
111
|
+
if (config?.embeddings?.enabled) {
|
|
122
112
|
import('../embeddings/queue.js')
|
|
123
|
-
.then(({ markStale }) => markStale(config,
|
|
124
|
-
.catch((
|
|
125
|
-
if (typeof process !== 'undefined' && process.stderr) {
|
|
126
|
-
process.stderr.write(`[mixgram] embeddings markStale error: ${err?.message ?? err}\n`);
|
|
127
|
-
}
|
|
128
|
-
});
|
|
113
|
+
.then(({ markStale }) => markStale(config, [documentId]))
|
|
114
|
+
.catch(() => {});
|
|
129
115
|
import('../embeddings/vectorStore.js')
|
|
130
|
-
.then(({ removeChunks }) => removeChunks(config,
|
|
116
|
+
.then(({ removeChunks }) => removeChunks(config, [documentId]))
|
|
131
117
|
.catch(() => {});
|
|
132
118
|
}
|
|
133
|
-
|
|
134
|
-
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(cid);
|
|
135
|
-
}
|
|
136
|
-
const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
|
|
119
|
+
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(documentId);
|
|
137
120
|
db.transaction(() => {
|
|
121
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
|
|
138
122
|
for (const { rowid } of ftsRowids) {
|
|
139
|
-
db.prepare('DELETE FROM
|
|
123
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
140
124
|
}
|
|
141
|
-
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(documentId);
|
|
142
125
|
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
|
|
143
126
|
db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
|
|
144
127
|
})();
|
|
@@ -1,78 +1,125 @@
|
|
|
1
1
|
import { unified } from 'unified';
|
|
2
2
|
import remarkParse from 'remark-parse';
|
|
3
3
|
import { visit } from 'unist-util-visit';
|
|
4
|
-
import { parse as
|
|
4
|
+
import { parse as parseMarkdownWithFrontmatter } from '../../utils/markdown.js';
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Recursively extract plain text from an mdast node (strips emphasis, links, etc.).
|
|
8
|
+
*/
|
|
9
|
+
function extractTextFromNode(node) {
|
|
10
|
+
if (!node) return '';
|
|
11
|
+
if (node.type === 'text' || node.type === 'inlineCode') {
|
|
12
|
+
return node.value || '';
|
|
13
|
+
}
|
|
14
|
+
if (node.children && Array.isArray(node.children)) {
|
|
15
|
+
return node.children.map((child) => extractTextFromNode(child)).join('');
|
|
16
|
+
}
|
|
9
17
|
return '';
|
|
10
18
|
}
|
|
11
19
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Build mdast from raw markdown and extract title, h1–h6, body, structure, sectionsIndex.
|
|
22
|
+
* One parse, one tree walk; no chunking. Compatible with the remark-parse indexing model.
|
|
23
|
+
*
|
|
24
|
+
* @param {string} markdown - Full markdown body (no frontmatter)
|
|
25
|
+
* @param {{ includeCodeBlocks?: boolean }} options
|
|
26
|
+
* @returns {{ title: string, h1: string, h2: string, h3: string, h4: string, h5: string, h6: string, body: string, structure: object[], sectionsIndex: object }}
|
|
27
|
+
*/
|
|
28
|
+
export function extractMarkdownFields(markdown, { includeCodeBlocks = false } = {}) {
|
|
29
|
+
const tree = unified().use(remarkParse).parse(markdown);
|
|
18
30
|
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
31
|
+
const h1 = [];
|
|
32
|
+
const h2 = [];
|
|
33
|
+
const h3 = [];
|
|
34
|
+
const h4 = [];
|
|
35
|
+
const h5 = [];
|
|
36
|
+
const h6 = [];
|
|
37
|
+
const bodyParts = [];
|
|
38
|
+
const structure = [];
|
|
39
|
+
const stack = [];
|
|
40
|
+
const sectionsIndex = {};
|
|
41
|
+
let currentSection = null;
|
|
42
|
+
let sectionIdCounter = 0;
|
|
43
|
+
let title = '';
|
|
25
44
|
|
|
26
45
|
visit(tree, (node) => {
|
|
27
46
|
if (node.type === 'heading') {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
47
|
+
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
48
|
+
const depth = node.depth;
|
|
49
|
+
|
|
50
|
+
switch (depth) {
|
|
51
|
+
case 1: h1.push(text); break;
|
|
52
|
+
case 2: h2.push(text); break;
|
|
53
|
+
case 3: h3.push(text); break;
|
|
54
|
+
case 4: h4.push(text); break;
|
|
55
|
+
case 5: h5.push(text); break;
|
|
56
|
+
case 6: h6.push(text); break;
|
|
57
|
+
default: break;
|
|
58
|
+
}
|
|
59
|
+
if (!title && depth === 1) title = text;
|
|
60
|
+
|
|
61
|
+
const section = {
|
|
62
|
+
id: `s${++sectionIdCounter}`,
|
|
63
|
+
depth,
|
|
64
|
+
heading: text,
|
|
65
|
+
parentId: null,
|
|
66
|
+
childrenIds: []
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
|
|
70
|
+
stack.pop();
|
|
71
|
+
}
|
|
72
|
+
if (stack.length > 0) {
|
|
73
|
+
const parent = stack[stack.length - 1];
|
|
74
|
+
section.parentId = parent.id;
|
|
75
|
+
parent.childrenIds.push(section.id);
|
|
76
|
+
} else {
|
|
77
|
+
structure.push(section);
|
|
78
|
+
}
|
|
79
|
+
stack.push(section);
|
|
80
|
+
sectionsIndex[section.id] = section;
|
|
81
|
+
currentSection = section;
|
|
33
82
|
return;
|
|
34
83
|
}
|
|
35
|
-
|
|
36
|
-
|
|
84
|
+
|
|
85
|
+
if (node.type === 'paragraph') {
|
|
86
|
+
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
87
|
+
if (text) bodyParts.push(text);
|
|
88
|
+
return;
|
|
37
89
|
}
|
|
38
|
-
});
|
|
39
90
|
|
|
40
|
-
|
|
91
|
+
if (includeCodeBlocks && node.type === 'code') {
|
|
92
|
+
const text = node.value?.trim();
|
|
93
|
+
if (text) bodyParts.push(text);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
41
96
|
|
|
42
|
-
|
|
43
|
-
sections.push({ headingPath: '', headingLevel: 0, content: mdBody.trim() });
|
|
44
|
-
}
|
|
45
|
-
return sections;
|
|
46
|
-
}
|
|
97
|
+
const body = bodyParts.join('\n\n');
|
|
47
98
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const lastSpace = content.lastIndexOf(' ', end);
|
|
61
|
-
if (lastSpace > start) end = lastSpace;
|
|
62
|
-
}
|
|
63
|
-
chunks.push({ headingPath, headingLevel, content: content.slice(start, end).trim() });
|
|
64
|
-
start = end - (end - start < content.length ? chunkOverlap : 0);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
return chunks;
|
|
99
|
+
return {
|
|
100
|
+
title: title || (h1[0] ?? ''),
|
|
101
|
+
h1: h1.join('\n'),
|
|
102
|
+
h2: h2.join('\n'),
|
|
103
|
+
h3: h3.join('\n'),
|
|
104
|
+
h4: h4.join('\n'),
|
|
105
|
+
h5: h5.join('\n'),
|
|
106
|
+
h6: h6.join('\n'),
|
|
107
|
+
body,
|
|
108
|
+
structure,
|
|
109
|
+
sectionsIndex
|
|
110
|
+
};
|
|
68
111
|
}
|
|
69
112
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Parse raw content: frontmatter (gray-matter) + AST-based extraction of body fields.
|
|
115
|
+
* Returns one object per document (no chunking).
|
|
116
|
+
*
|
|
117
|
+
* @param {string} rawContent - Full file content including optional frontmatter
|
|
118
|
+
* @param {{ includeCodeBlocks?: boolean }} options
|
|
119
|
+
*/
|
|
120
|
+
export function parseMarkdown(rawContent, options = {}) {
|
|
121
|
+
const { frontmatter, body: rawBody } = parseMarkdownWithFrontmatter(rawContent);
|
|
122
|
+
const fields = extractMarkdownFields(rawBody, options);
|
|
123
|
+
const body = (fields.body && fields.body.trim()) ? fields.body : (rawBody || '').trim();
|
|
124
|
+
return { frontmatter, ...fields, body };
|
|
76
125
|
}
|
|
77
|
-
|
|
78
|
-
export { extractSections, chunkSections, parseAndChunk };
|
|
@@ -28,11 +28,10 @@ function listMarkdownPaths(config) {
|
|
|
28
28
|
function clearIndex(config) {
|
|
29
29
|
const db = getDb(config);
|
|
30
30
|
db.transaction(() => {
|
|
31
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
31
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts').all();
|
|
32
32
|
for (const { rowid } of ftsRowids) {
|
|
33
|
-
db.prepare('DELETE FROM
|
|
33
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
34
34
|
}
|
|
35
|
-
db.prepare('DELETE FROM document_chunks').run();
|
|
36
35
|
db.prepare('DELETE FROM document_tags').run();
|
|
37
36
|
db.prepare('DELETE FROM documents').run();
|
|
38
37
|
})();
|
|
@@ -60,22 +59,32 @@ function fullReindex(config) {
|
|
|
60
59
|
}
|
|
61
60
|
|
|
62
61
|
/**
|
|
63
|
-
* Incremental reindex:
|
|
64
|
-
*
|
|
62
|
+
* Incremental reindex: scan all .md files; skip unchanged (same path + file_mtime_ms);
|
|
63
|
+
* index new or changed; remove from index documents whose path is no longer on disk.
|
|
64
|
+
* Does not clear the index first.
|
|
65
|
+
* @returns {{ scanned: number, skipped: number, indexed: number, removed: number, totalPaths: number }}
|
|
65
66
|
*/
|
|
66
67
|
function incrementalReindex(config) {
|
|
67
68
|
const paths = listMarkdownPaths(config);
|
|
68
69
|
const pathsSet = new Set(paths.map((p) => path.normalize(p)));
|
|
70
|
+
const db = getDb(config);
|
|
71
|
+
const getStored = db.prepare('SELECT id, file_mtime_ms FROM documents WHERE path = ?');
|
|
69
72
|
let indexed = 0;
|
|
73
|
+
let skipped = 0;
|
|
70
74
|
for (const filePath of paths) {
|
|
71
75
|
try {
|
|
72
|
-
const
|
|
76
|
+
const norm = path.normalize(filePath);
|
|
73
77
|
const stat = fs.statSync(filePath);
|
|
78
|
+
const row = getStored.get(norm);
|
|
79
|
+
if (row != null && row.file_mtime_ms != null && row.file_mtime_ms === stat.mtimeMs) {
|
|
80
|
+
skipped++;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
74
84
|
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
75
85
|
indexed++;
|
|
76
86
|
} catch (_) {}
|
|
77
87
|
}
|
|
78
|
-
const db = getDb(config);
|
|
79
88
|
const allDocs = db.prepare('SELECT id, path FROM documents').all();
|
|
80
89
|
let removed = 0;
|
|
81
90
|
for (const row of allDocs) {
|
|
@@ -85,7 +94,7 @@ function incrementalReindex(config) {
|
|
|
85
94
|
removed++;
|
|
86
95
|
}
|
|
87
96
|
}
|
|
88
|
-
return { indexed, removed, totalPaths: paths.length };
|
|
97
|
+
return { scanned: paths.length, skipped, indexed, removed, totalPaths: paths.length };
|
|
89
98
|
}
|
|
90
99
|
|
|
91
100
|
/**
|