mixgram 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/bin/mixgram.js +16 -9
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +16 -12
- package/spec/scenarios/scenarios.js +6 -4
- package/spec/test-large-document.js +98 -0
- package/src/config.js +8 -5
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/embeddings/worker-process.js +79 -0
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -106
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/mcp/server.js +67 -2
- package/src/utils/markdown.js +42 -2
|
@@ -1,78 +1,125 @@
|
|
|
1
1
|
import { unified } from 'unified';
|
|
2
2
|
import remarkParse from 'remark-parse';
|
|
3
3
|
import { visit } from 'unist-util-visit';
|
|
4
|
-
import { parse as
|
|
4
|
+
import { parse as parseMarkdownWithFrontmatter } from '../../utils/markdown.js';
|
|
5
5
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
/**
|
|
7
|
+
* Recursively extract plain text from an mdast node (strips emphasis, links, etc.).
|
|
8
|
+
*/
|
|
9
|
+
function extractTextFromNode(node) {
|
|
10
|
+
if (!node) return '';
|
|
11
|
+
if (node.type === 'text' || node.type === 'inlineCode') {
|
|
12
|
+
return node.value || '';
|
|
13
|
+
}
|
|
14
|
+
if (node.children && Array.isArray(node.children)) {
|
|
15
|
+
return node.children.map((child) => extractTextFromNode(child)).join('');
|
|
16
|
+
}
|
|
9
17
|
return '';
|
|
10
18
|
}
|
|
11
19
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Build mdast from raw markdown and extract title, h1–h6, body, structure, sectionsIndex.
|
|
22
|
+
* One parse, one tree walk; no chunking. Compatible with the remark-parse indexing model.
|
|
23
|
+
*
|
|
24
|
+
* @param {string} markdown - Full markdown body (no frontmatter)
|
|
25
|
+
* @param {{ includeCodeBlocks?: boolean }} options
|
|
26
|
+
* @returns {{ title: string, h1: string, h2: string, h3: string, h4: string, h5: string, h6: string, body: string, structure: object[], sectionsIndex: object }}
|
|
27
|
+
*/
|
|
28
|
+
export function extractMarkdownFields(markdown, { includeCodeBlocks = false } = {}) {
|
|
29
|
+
const tree = unified().use(remarkParse).parse(markdown);
|
|
18
30
|
|
|
19
|
-
const
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
31
|
+
const h1 = [];
|
|
32
|
+
const h2 = [];
|
|
33
|
+
const h3 = [];
|
|
34
|
+
const h4 = [];
|
|
35
|
+
const h5 = [];
|
|
36
|
+
const h6 = [];
|
|
37
|
+
const bodyParts = [];
|
|
38
|
+
const structure = [];
|
|
39
|
+
const stack = [];
|
|
40
|
+
const sectionsIndex = {};
|
|
41
|
+
let currentSection = null;
|
|
42
|
+
let sectionIdCounter = 0;
|
|
43
|
+
let title = '';
|
|
25
44
|
|
|
26
45
|
visit(tree, (node) => {
|
|
27
46
|
if (node.type === 'heading') {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
47
|
+
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
48
|
+
const depth = node.depth;
|
|
49
|
+
|
|
50
|
+
switch (depth) {
|
|
51
|
+
case 1: h1.push(text); break;
|
|
52
|
+
case 2: h2.push(text); break;
|
|
53
|
+
case 3: h3.push(text); break;
|
|
54
|
+
case 4: h4.push(text); break;
|
|
55
|
+
case 5: h5.push(text); break;
|
|
56
|
+
case 6: h6.push(text); break;
|
|
57
|
+
default: break;
|
|
58
|
+
}
|
|
59
|
+
if (!title && depth === 1) title = text;
|
|
60
|
+
|
|
61
|
+
const section = {
|
|
62
|
+
id: `s${++sectionIdCounter}`,
|
|
63
|
+
depth,
|
|
64
|
+
heading: text,
|
|
65
|
+
parentId: null,
|
|
66
|
+
childrenIds: []
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
|
|
70
|
+
stack.pop();
|
|
71
|
+
}
|
|
72
|
+
if (stack.length > 0) {
|
|
73
|
+
const parent = stack[stack.length - 1];
|
|
74
|
+
section.parentId = parent.id;
|
|
75
|
+
parent.childrenIds.push(section.id);
|
|
76
|
+
} else {
|
|
77
|
+
structure.push(section);
|
|
78
|
+
}
|
|
79
|
+
stack.push(section);
|
|
80
|
+
sectionsIndex[section.id] = section;
|
|
81
|
+
currentSection = section;
|
|
33
82
|
return;
|
|
34
83
|
}
|
|
35
|
-
|
|
36
|
-
|
|
84
|
+
|
|
85
|
+
if (node.type === 'paragraph') {
|
|
86
|
+
const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
|
|
87
|
+
if (text) bodyParts.push(text);
|
|
88
|
+
return;
|
|
37
89
|
}
|
|
38
|
-
});
|
|
39
90
|
|
|
40
|
-
|
|
91
|
+
if (includeCodeBlocks && node.type === 'code') {
|
|
92
|
+
const text = node.value?.trim();
|
|
93
|
+
if (text) bodyParts.push(text);
|
|
94
|
+
}
|
|
95
|
+
});
|
|
41
96
|
|
|
42
|
-
|
|
43
|
-
sections.push({ headingPath: '', headingLevel: 0, content: mdBody.trim() });
|
|
44
|
-
}
|
|
45
|
-
return sections;
|
|
46
|
-
}
|
|
97
|
+
const body = bodyParts.join('\n\n');
|
|
47
98
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const lastSpace = content.lastIndexOf(' ', end);
|
|
61
|
-
if (lastSpace > start) end = lastSpace;
|
|
62
|
-
}
|
|
63
|
-
chunks.push({ headingPath, headingLevel, content: content.slice(start, end).trim() });
|
|
64
|
-
start = end - (end - start < content.length ? chunkOverlap : 0);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
return chunks;
|
|
99
|
+
return {
|
|
100
|
+
title: title || (h1[0] ?? ''),
|
|
101
|
+
h1: h1.join('\n'),
|
|
102
|
+
h2: h2.join('\n'),
|
|
103
|
+
h3: h3.join('\n'),
|
|
104
|
+
h4: h4.join('\n'),
|
|
105
|
+
h5: h5.join('\n'),
|
|
106
|
+
h6: h6.join('\n'),
|
|
107
|
+
body,
|
|
108
|
+
structure,
|
|
109
|
+
sectionsIndex
|
|
110
|
+
};
|
|
68
111
|
}
|
|
69
112
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
113
|
+
/**
|
|
114
|
+
* Parse raw content: frontmatter (gray-matter) + AST-based extraction of body fields.
|
|
115
|
+
* Returns one object per document (no chunking).
|
|
116
|
+
*
|
|
117
|
+
* @param {string} rawContent - Full file content including optional frontmatter
|
|
118
|
+
* @param {{ includeCodeBlocks?: boolean }} options
|
|
119
|
+
*/
|
|
120
|
+
export function parseMarkdown(rawContent, options = {}) {
|
|
121
|
+
const { frontmatter, body: rawBody } = parseMarkdownWithFrontmatter(rawContent);
|
|
122
|
+
const fields = extractMarkdownFields(rawBody, options);
|
|
123
|
+
const body = (fields.body && fields.body.trim()) ? fields.body : (rawBody || '').trim();
|
|
124
|
+
return { frontmatter, ...fields, body };
|
|
76
125
|
}
|
|
77
|
-
|
|
78
|
-
export { extractSections, chunkSections, parseAndChunk };
|
|
@@ -28,11 +28,10 @@ function listMarkdownPaths(config) {
|
|
|
28
28
|
function clearIndex(config) {
|
|
29
29
|
const db = getDb(config);
|
|
30
30
|
db.transaction(() => {
|
|
31
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
31
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts').all();
|
|
32
32
|
for (const { rowid } of ftsRowids) {
|
|
33
|
-
db.prepare('DELETE FROM
|
|
33
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
34
34
|
}
|
|
35
|
-
db.prepare('DELETE FROM document_chunks').run();
|
|
36
35
|
db.prepare('DELETE FROM document_tags').run();
|
|
37
36
|
db.prepare('DELETE FROM documents').run();
|
|
38
37
|
})();
|
|
@@ -60,22 +59,32 @@ function fullReindex(config) {
|
|
|
60
59
|
}
|
|
61
60
|
|
|
62
61
|
/**
|
|
63
|
-
* Incremental reindex:
|
|
64
|
-
*
|
|
62
|
+
* Incremental reindex: scan all .md files; skip unchanged (same path + file_mtime_ms);
|
|
63
|
+
* index new or changed; remove from index documents whose path is no longer on disk.
|
|
64
|
+
* Does not clear the index first.
|
|
65
|
+
* @returns {{ scanned: number, skipped: number, indexed: number, removed: number, totalPaths: number }}
|
|
65
66
|
*/
|
|
66
67
|
function incrementalReindex(config) {
|
|
67
68
|
const paths = listMarkdownPaths(config);
|
|
68
69
|
const pathsSet = new Set(paths.map((p) => path.normalize(p)));
|
|
70
|
+
const db = getDb(config);
|
|
71
|
+
const getStored = db.prepare('SELECT id, file_mtime_ms FROM documents WHERE path = ?');
|
|
69
72
|
let indexed = 0;
|
|
73
|
+
let skipped = 0;
|
|
70
74
|
for (const filePath of paths) {
|
|
71
75
|
try {
|
|
72
|
-
const
|
|
76
|
+
const norm = path.normalize(filePath);
|
|
73
77
|
const stat = fs.statSync(filePath);
|
|
78
|
+
const row = getStored.get(norm);
|
|
79
|
+
if (row != null && row.file_mtime_ms != null && row.file_mtime_ms === stat.mtimeMs) {
|
|
80
|
+
skipped++;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
74
84
|
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
75
85
|
indexed++;
|
|
76
86
|
} catch (_) {}
|
|
77
87
|
}
|
|
78
|
-
const db = getDb(config);
|
|
79
88
|
const allDocs = db.prepare('SELECT id, path FROM documents').all();
|
|
80
89
|
let removed = 0;
|
|
81
90
|
for (const row of allDocs) {
|
|
@@ -85,7 +94,7 @@ function incrementalReindex(config) {
|
|
|
85
94
|
removed++;
|
|
86
95
|
}
|
|
87
96
|
}
|
|
88
|
-
return { indexed, removed, totalPaths: paths.length };
|
|
97
|
+
return { scanned: paths.length, skipped, indexed, removed, totalPaths: paths.length };
|
|
89
98
|
}
|
|
90
99
|
|
|
91
100
|
/**
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { getDb } from '../../db/sqlite.js';
|
|
2
2
|
import { normalizeForFts } from '../indexing/indexer.js';
|
|
3
|
+
|
|
3
4
|
function getBm25Weights(config) {
|
|
4
5
|
const w = config.indexing?.ftsWeights || {};
|
|
5
6
|
return [
|
|
6
7
|
w.title ?? 10,
|
|
7
|
-
w.
|
|
8
|
-
|
|
9
|
-
w.
|
|
8
|
+
w.h1 ?? 8,
|
|
9
|
+
w.h2 ?? 6,
|
|
10
|
+
w.h3 ?? 5,
|
|
11
|
+
w.h4 ?? 4,
|
|
12
|
+
w.h5 ?? 3,
|
|
13
|
+
w.h6 ?? 2,
|
|
10
14
|
w.body ?? 1
|
|
11
15
|
];
|
|
12
16
|
}
|
|
@@ -17,7 +21,7 @@ function getBm25Weights(config) {
|
|
|
17
21
|
*/
|
|
18
22
|
function normalizeQuery(q) {
|
|
19
23
|
if (q == null || typeof q !== 'string') return '';
|
|
20
|
-
return q.trim();
|
|
24
|
+
return normalizeForFts(q.trim());
|
|
21
25
|
}
|
|
22
26
|
|
|
23
27
|
/**
|
|
@@ -27,7 +31,7 @@ function normalizeQuery(q) {
|
|
|
27
31
|
* @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
|
|
28
32
|
* @param {string} [options.project] - project name (required for project-only and merged)
|
|
29
33
|
* @param {number} [options.limit] - max results
|
|
30
|
-
* @returns {Array<{ documentId: string,
|
|
34
|
+
* @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, snippet: string, score: number }>}
|
|
31
35
|
*/
|
|
32
36
|
function search(config, options = {}) {
|
|
33
37
|
const {
|
|
@@ -57,33 +61,28 @@ function search(config, options = {}) {
|
|
|
57
61
|
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
58
62
|
params.push('home');
|
|
59
63
|
} else {
|
|
60
|
-
// merged: project or home; when project given, prefer project
|
|
61
64
|
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
62
65
|
params.push('home', 'project', project || '');
|
|
63
66
|
}
|
|
64
67
|
|
|
65
|
-
params.push(project || '');
|
|
66
|
-
|
|
67
|
-
params.push(ftsLimit);
|
|
68
|
+
params.push(project || '');
|
|
69
|
+
params.push(limit);
|
|
68
70
|
|
|
69
71
|
const sql = `
|
|
70
72
|
SELECT
|
|
71
|
-
f.chunk_id AS chunkId,
|
|
72
73
|
f.document_id AS documentId,
|
|
73
74
|
d.title AS title,
|
|
74
75
|
d.topic_key AS topicKey,
|
|
75
76
|
d.type AS type,
|
|
76
77
|
d.scope AS scope,
|
|
77
78
|
d.project AS project,
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
snippet(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
FROM document_chunks_fts f
|
|
79
|
+
d.body AS body,
|
|
80
|
+
snippet(document_fts, 1, '<b>', '</b>', '...', 24) AS snippetTitle,
|
|
81
|
+
snippet(document_fts, 8, '<b>', '</b>', '...', 64) AS snippetBody,
|
|
82
|
+
bm25(document_fts, ${bm25Args}) AS rank
|
|
83
|
+
FROM document_fts f
|
|
84
84
|
JOIN documents d ON d.id = f.document_id
|
|
85
|
-
|
|
86
|
-
WHERE document_chunks_fts MATCH ? ${scopeCondition}
|
|
85
|
+
WHERE document_fts MATCH ? ${scopeCondition}
|
|
87
86
|
ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
|
|
88
87
|
LIMIT ?
|
|
89
88
|
`;
|
|
@@ -91,97 +90,21 @@ function search(config, options = {}) {
|
|
|
91
90
|
const stmt = db.prepare(sql);
|
|
92
91
|
const rows = stmt.all(...params);
|
|
93
92
|
|
|
94
|
-
|
|
95
|
-
const
|
|
93
|
+
return rows.map((r) => {
|
|
94
|
+
const fromFts = [r.snippetTitle, r.snippetBody].filter((x) => x != null && x !== '').join(' … ');
|
|
95
|
+
const fallback = r.body != null ? String(r.body).slice(0, 200) : '';
|
|
96
|
+
const snippet = (fromFts && fromFts.trim()) || fallback || '';
|
|
96
97
|
return {
|
|
97
98
|
documentId: r.documentId,
|
|
98
|
-
chunkId: r.chunkId,
|
|
99
99
|
title: r.title,
|
|
100
100
|
topicKey: r.topicKey,
|
|
101
101
|
type: r.type,
|
|
102
102
|
scope: r.scope,
|
|
103
103
|
project: r.project,
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
score: -Number(r.rank), // bm25 returns negative (lower = better), we expose positive
|
|
107
|
-
_rank: r.rank
|
|
104
|
+
snippet: String(snippet).trim(),
|
|
105
|
+
score: -Number(r.rank)
|
|
108
106
|
};
|
|
109
107
|
});
|
|
110
|
-
|
|
111
|
-
if (!config.embeddings?.enabled || mapped.length === 0) {
|
|
112
|
-
return mapped.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return hybridSearch(config, { query, scopeMode, project, limit, ftsRows: mapped });
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
async function hybridSearch(config, { query, scopeMode, project, limit, ftsRows }) {
|
|
119
|
-
const ftsWeight = config.search?.ftsWeight ?? 0.7;
|
|
120
|
-
const semanticWeight = config.search?.semanticWeight ?? 0.3;
|
|
121
|
-
let embedder;
|
|
122
|
-
let vectorStore;
|
|
123
|
-
try {
|
|
124
|
-
const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
|
|
125
|
-
import('../embeddings/embedder.js'),
|
|
126
|
-
import('../embeddings/vectorStore.js')
|
|
127
|
-
]);
|
|
128
|
-
embedder = await getEmbedder(config);
|
|
129
|
-
vectorStore = await getVectorStore(config);
|
|
130
|
-
} catch {
|
|
131
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
132
|
-
}
|
|
133
|
-
if (!embedder || !vectorStore) {
|
|
134
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
135
|
-
}
|
|
136
|
-
let semanticHits;
|
|
137
|
-
try {
|
|
138
|
-
const queryEmbedding = await embedder.embed(query);
|
|
139
|
-
semanticHits = await vectorStore.search(config, queryEmbedding, Math.max(limit * 2, 30));
|
|
140
|
-
} catch {
|
|
141
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
142
|
-
}
|
|
143
|
-
const byChunk = new Map(ftsRows.map((r) => [r.chunkId, { ...r }]));
|
|
144
|
-
const ftsScores = ftsRows.map((r) => -Number(r._rank));
|
|
145
|
-
const minFts = Math.min(...ftsScores);
|
|
146
|
-
const maxFts = Math.max(...ftsScores);
|
|
147
|
-
const rangeFts = maxFts - minFts || 1;
|
|
148
|
-
for (const row of ftsRows) {
|
|
149
|
-
const normFts = (minFts === maxFts) ? 1 : ((-Number(row._rank)) - minFts) / rangeFts;
|
|
150
|
-
const sem = semanticHits.find((h) => h.chunk_id === row.chunkId);
|
|
151
|
-
const similarity = sem ? 1 - sem.distance : 0;
|
|
152
|
-
const combined = ftsWeight * normFts + semanticWeight * similarity;
|
|
153
|
-
byChunk.get(row.chunkId).score = combined;
|
|
154
|
-
}
|
|
155
|
-
for (const sem of semanticHits) {
|
|
156
|
-
if (!byChunk.has(sem.chunk_id)) {
|
|
157
|
-
const db = getDb(config);
|
|
158
|
-
const r = db.prepare(`
|
|
159
|
-
SELECT c.id AS chunkId, c.document_id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
|
|
160
|
-
FROM document_chunks c
|
|
161
|
-
JOIN documents d ON d.id = c.document_id
|
|
162
|
-
WHERE c.id = ?
|
|
163
|
-
`).get(sem.chunk_id);
|
|
164
|
-
if (r) {
|
|
165
|
-
const similarity = 1 - sem.distance;
|
|
166
|
-
byChunk.set(sem.chunk_id, {
|
|
167
|
-
documentId: r.documentId,
|
|
168
|
-
chunkId: r.chunkId,
|
|
169
|
-
title: r.title,
|
|
170
|
-
topicKey: r.topicKey,
|
|
171
|
-
type: r.type,
|
|
172
|
-
scope: r.scope,
|
|
173
|
-
project: r.project,
|
|
174
|
-
headingPath: r.headingPath,
|
|
175
|
-
snippet: (r.content || '').slice(0, 200).trim(),
|
|
176
|
-
score: semanticWeight * similarity
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
|
-
return Array.from(byChunk.values())
|
|
182
|
-
.sort((a, b) => b.score - a.score)
|
|
183
|
-
.slice(0, limit)
|
|
184
|
-
.map(({ _rank, ...r }) => r);
|
|
185
108
|
}
|
|
186
109
|
|
|
187
110
|
/**
|
|
@@ -191,9 +114,8 @@ function getRecentContext(config, options = {}) {
|
|
|
191
114
|
const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
|
|
192
115
|
const db = getDb(config);
|
|
193
116
|
let sql = `
|
|
194
|
-
SELECT
|
|
195
|
-
FROM
|
|
196
|
-
JOIN documents d ON d.id = c.document_id
|
|
117
|
+
SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.body
|
|
118
|
+
FROM documents d
|
|
197
119
|
WHERE d.deleted_at IS NULL
|
|
198
120
|
`;
|
|
199
121
|
const params = [];
|
|
@@ -209,14 +131,12 @@ function getRecentContext(config, options = {}) {
|
|
|
209
131
|
const rows = db.prepare(sql).all(...params);
|
|
210
132
|
return rows.map((r) => ({
|
|
211
133
|
documentId: r.documentId,
|
|
212
|
-
chunkId: r.chunkId,
|
|
213
134
|
title: r.title,
|
|
214
135
|
topicKey: r.topicKey,
|
|
215
136
|
type: r.type,
|
|
216
137
|
scope: r.scope,
|
|
217
138
|
project: r.project,
|
|
218
|
-
|
|
219
|
-
snippet: (r.content || '').slice(0, 300).trim()
|
|
139
|
+
snippet: (r.body || '').slice(0, 300).trim()
|
|
220
140
|
}));
|
|
221
141
|
}
|
|
222
142
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
-- documents
|
|
1
|
+
-- documents (one row per doc; body and section tree stored here)
|
|
2
2
|
CREATE TABLE IF NOT EXISTS documents (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
path TEXT NOT NULL UNIQUE,
|
|
@@ -17,21 +17,10 @@ CREATE TABLE IF NOT EXISTS documents (
|
|
|
17
17
|
deleted_at TEXT,
|
|
18
18
|
revision_count INTEGER NOT NULL DEFAULT 1,
|
|
19
19
|
duplicate_count INTEGER NOT NULL DEFAULT 0,
|
|
20
|
-
embedding_status TEXT NOT NULL DEFAULT 'disabled'
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
id TEXT PRIMARY KEY,
|
|
25
|
-
document_id TEXT NOT NULL,
|
|
26
|
-
chunk_index INTEGER NOT NULL,
|
|
27
|
-
heading_path TEXT,
|
|
28
|
-
heading_level INTEGER,
|
|
29
|
-
content TEXT NOT NULL,
|
|
30
|
-
token_count INTEGER,
|
|
31
|
-
content_hash TEXT NOT NULL,
|
|
32
|
-
created_at TEXT NOT NULL,
|
|
33
|
-
updated_at TEXT NOT NULL,
|
|
34
|
-
FOREIGN KEY(document_id) REFERENCES documents(id)
|
|
20
|
+
embedding_status TEXT NOT NULL DEFAULT 'disabled',
|
|
21
|
+
body TEXT,
|
|
22
|
+
structure TEXT,
|
|
23
|
+
sections_index TEXT
|
|
35
24
|
);
|
|
36
25
|
|
|
37
26
|
CREATE TABLE IF NOT EXISTS document_tags (
|
|
@@ -67,20 +56,19 @@ CREATE TABLE IF NOT EXISTS embedding_jobs (
|
|
|
67
56
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
68
57
|
last_error TEXT,
|
|
69
58
|
created_at TEXT NOT NULL,
|
|
70
|
-
updated_at TEXT NOT NULL
|
|
71
|
-
FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
|
|
59
|
+
updated_at TEXT NOT NULL
|
|
72
60
|
);
|
|
73
61
|
|
|
74
|
-
-- FTS5:
|
|
75
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS
|
|
76
|
-
chunk_id UNINDEXED,
|
|
62
|
+
-- FTS5: one row per document (title, h1–h6, body)
|
|
63
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
|
|
77
64
|
document_id UNINDEXED,
|
|
78
65
|
title,
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
66
|
+
h1,
|
|
67
|
+
h2,
|
|
68
|
+
h3,
|
|
69
|
+
h4,
|
|
70
|
+
h5,
|
|
71
|
+
h6,
|
|
84
72
|
body,
|
|
85
73
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
86
74
|
);
|
package/src/mcp/server.js
CHANGED
|
@@ -1,9 +1,49 @@
|
|
|
1
|
+
import { fork } from 'child_process';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
import { randomUUID } from 'crypto';
|
|
6
|
+
import { fileURLToPath } from 'url';
|
|
1
7
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
8
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
3
9
|
import { loadConfig } from '../config.js';
|
|
4
10
|
import { createToolHandlers } from './tools.js';
|
|
5
11
|
import { getToolDefinitions } from './tool-registry.js';
|
|
6
12
|
|
|
13
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
14
|
+
let embeddingWorkerChild = null;
|
|
15
|
+
let embeddingWorkerConfigPath = null;
|
|
16
|
+
|
|
17
|
+
const EMBED_TIMEOUT_MS = 60000;
|
|
18
|
+
|
|
19
|
+
function makeQueryEmbedder(child) {
|
|
20
|
+
return function getQueryEmbedding(text) {
|
|
21
|
+
if (!child?.connected) return Promise.resolve(null);
|
|
22
|
+
const id = randomUUID();
|
|
23
|
+
return new Promise((resolve, reject) => {
|
|
24
|
+
const timeout = setTimeout(() => {
|
|
25
|
+
cleanup();
|
|
26
|
+
reject(new Error('embedding timeout'));
|
|
27
|
+
}, EMBED_TIMEOUT_MS);
|
|
28
|
+
const onMessage = (msg) => {
|
|
29
|
+
if (msg?.type !== 'embedResult' || msg.id !== id) return;
|
|
30
|
+
cleanup();
|
|
31
|
+
if (msg.err) {
|
|
32
|
+
reject(new Error(msg.err));
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
resolve(msg.vector ? new Float32Array(msg.vector) : null);
|
|
36
|
+
};
|
|
37
|
+
const cleanup = () => {
|
|
38
|
+
clearTimeout(timeout);
|
|
39
|
+
child.off('message', onMessage);
|
|
40
|
+
};
|
|
41
|
+
child.on('message', onMessage);
|
|
42
|
+
child.send({ type: 'embed', id, text });
|
|
43
|
+
});
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
7
47
|
function createServer(configOverrides = {}, baseDir = null, projectBaseDir = null) {
|
|
8
48
|
const config = loadConfig(configOverrides, baseDir, projectBaseDir);
|
|
9
49
|
const mcpServer = new McpServer(
|
|
@@ -59,8 +99,33 @@ async function run(configOverrides = {}, baseDir = null, projectBaseDir = null)
|
|
|
59
99
|
}
|
|
60
100
|
if (config.embeddings?.enabled) {
|
|
61
101
|
try {
|
|
62
|
-
|
|
63
|
-
|
|
102
|
+
embeddingWorkerConfigPath = path.join(
|
|
103
|
+
os.tmpdir(),
|
|
104
|
+
`mixgram-worker-config-${process.pid}-${Date.now()}.json`
|
|
105
|
+
);
|
|
106
|
+
fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
|
|
107
|
+
const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
|
|
108
|
+
embeddingWorkerChild = fork(workerPath, [], {
|
|
109
|
+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
110
|
+
env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
|
|
111
|
+
});
|
|
112
|
+
config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
|
|
113
|
+
embeddingWorkerChild.on('exit', (code, signal) => {
|
|
114
|
+
if (code !== 0 && code != null && process.stderr) {
|
|
115
|
+
process.stderr.write(`[mixgram] embeddings worker exited: code=${code} signal=${signal}\n`);
|
|
116
|
+
}
|
|
117
|
+
config.getQueryEmbedding = null;
|
|
118
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
119
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
120
|
+
embeddingWorkerConfigPath = null;
|
|
121
|
+
}
|
|
122
|
+
});
|
|
123
|
+
process.on('exit', () => {
|
|
124
|
+
if (embeddingWorkerChild) embeddingWorkerChild.kill();
|
|
125
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
126
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
127
|
+
}
|
|
128
|
+
});
|
|
64
129
|
} catch (err) {
|
|
65
130
|
if (typeof process !== 'undefined' && process.stderr) {
|
|
66
131
|
process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
|
package/src/utils/markdown.js
CHANGED
|
@@ -6,6 +6,29 @@ export const FRONTMATTER_KEYS = [
|
|
|
6
6
|
'embedding_status', 'indexed_at', 'deleted_at'
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
+
/** Keys written to .md frontmatter (needed for re-index, scope/project filtering, timeline). */
|
|
10
|
+
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'scope', 'project', 'topic_key', 'session_id', 'created'];
|
|
11
|
+
|
|
12
|
+
function formatCreated(createdAt) {
|
|
13
|
+
if (!createdAt) return null;
|
|
14
|
+
const d = new Date(createdAt);
|
|
15
|
+
if (Number.isNaN(d.getTime())) return createdAt;
|
|
16
|
+
const y = d.getFullYear();
|
|
17
|
+
const m = String(d.getMonth() + 1).padStart(2, '0');
|
|
18
|
+
const day = String(d.getDate()).padStart(2, '0');
|
|
19
|
+
const h = String(d.getHours()).padStart(2, '0');
|
|
20
|
+
const min = String(d.getMinutes()).padStart(2, '0');
|
|
21
|
+
const s = String(d.getSeconds()).padStart(2, '0');
|
|
22
|
+
return `${y}-${m}-${day} ${h}:${min}:${s}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function parseCreated(value) {
|
|
26
|
+
if (!value) return null;
|
|
27
|
+
if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value)) return value;
|
|
28
|
+
const d = new Date(value);
|
|
29
|
+
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
|
30
|
+
}
|
|
31
|
+
|
|
9
32
|
function serializeFrontmatter(fm) {
|
|
10
33
|
const obj = {};
|
|
11
34
|
for (const k of FRONTMATTER_KEYS) {
|
|
@@ -16,9 +39,26 @@ function serializeFrontmatter(fm) {
|
|
|
16
39
|
return obj;
|
|
17
40
|
}
|
|
18
41
|
|
|
42
|
+
/** Minimal frontmatter for .md (id required for re-index; created in friendly format). */
|
|
43
|
+
function serializeDisplayFrontmatter(fm) {
|
|
44
|
+
const obj = {};
|
|
45
|
+
for (const k of DISPLAY_FRONTMATTER_KEYS) {
|
|
46
|
+
if (k === 'created') {
|
|
47
|
+
const v = fm.created_at != null ? formatCreated(fm.created_at) : (fm.created ?? null);
|
|
48
|
+
if (v) obj[k] = v;
|
|
49
|
+
} else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
|
|
50
|
+
obj[k] = fm[k];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (fm.deleted_at) obj.deleted_at = fm.deleted_at;
|
|
54
|
+
if (fm.deleted === true) obj.deleted = true;
|
|
55
|
+
return obj;
|
|
56
|
+
}
|
|
57
|
+
|
|
19
58
|
export function parse(content) {
|
|
20
59
|
const parsed = matter(content);
|
|
21
60
|
const data = parsed.data || {};
|
|
61
|
+
const createdAt = data.created_at ?? parseCreated(data.created);
|
|
22
62
|
return {
|
|
23
63
|
frontmatter: {
|
|
24
64
|
id: data.id,
|
|
@@ -29,7 +69,7 @@ export function parse(content) {
|
|
|
29
69
|
topic_key: data.topic_key,
|
|
30
70
|
session_id: data.session_id,
|
|
31
71
|
tool_name: data.tool_name,
|
|
32
|
-
created_at:
|
|
72
|
+
created_at: createdAt,
|
|
33
73
|
updated_at: data.updated_at,
|
|
34
74
|
revision_count: data.revision_count ?? 1,
|
|
35
75
|
duplicate_count: data.duplicate_count ?? 0,
|
|
@@ -44,6 +84,6 @@ export function parse(content) {
|
|
|
44
84
|
}
|
|
45
85
|
|
|
46
86
|
export function toMarkdown(frontmatter, body) {
|
|
47
|
-
const fm =
|
|
87
|
+
const fm = serializeDisplayFrontmatter(frontmatter);
|
|
48
88
|
return matter.stringify(body || '', fm, { lineWidth: -1 });
|
|
49
89
|
}
|