mixgram 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/bin/mixgram.js +16 -9
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +16 -12
- package/spec/scenarios/scenarios.js +6 -4
- package/spec/test-large-document.js +98 -0
- package/src/config.js +8 -5
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/embeddings/worker-process.js +79 -0
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -106
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/mcp/server.js +67 -2
- package/src/utils/markdown.js +42 -2
package/spec/run-tests.js
CHANGED
|
@@ -14,6 +14,7 @@ import { createToolHandlers } from '../src/mcp/tools.js';
|
|
|
14
14
|
import { getToolDefinitions } from '../src/mcp/tool-registry.js';
|
|
15
15
|
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
16
16
|
import { scenarios } from './scenarios/scenarios.js';
|
|
17
|
+
import { runLargeDocumentTest } from './test-large-document.js';
|
|
17
18
|
import * as reporter from './reporter.js';
|
|
18
19
|
|
|
19
20
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -22,7 +23,7 @@ const __dirname = path.dirname(__filename);
|
|
|
22
23
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-test-'));
|
|
23
24
|
const config = loadConfig({
|
|
24
25
|
homeMemoryRoot: path.join(tmpDir, 'home', 'memory'),
|
|
25
|
-
projectMemoryRoot: path.join(tmpDir, '
|
|
26
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
26
27
|
sqlitePath: path.join(tmpDir, '.mixgram', 'index.db')
|
|
27
28
|
});
|
|
28
29
|
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
@@ -57,7 +58,7 @@ function runCliSetupChecks() {
|
|
|
57
58
|
const repoRoot = path.dirname(__dirname);
|
|
58
59
|
|
|
59
60
|
if (reporter.VISUAL) {
|
|
60
|
-
reporter.startScenario('CLI setup cursor config', 'Verify Cursor setup writes
|
|
61
|
+
reporter.startScenario('CLI setup cursor config', 'Verify Cursor setup writes project-memory arg for project-scoped memory.');
|
|
61
62
|
}
|
|
62
63
|
|
|
63
64
|
let p = 0;
|
|
@@ -83,9 +84,8 @@ function runCliSetupChecks() {
|
|
|
83
84
|
const cursorConfig = JSON.parse(fs.readFileSync(cursorConfigPath, 'utf8'));
|
|
84
85
|
const entry = cursorConfig?.mcpServers?.mixgram;
|
|
85
86
|
ok(entry?.command === 'mixgram', 'cursor entry command preserved');
|
|
86
|
-
|
|
87
|
-
ok(
|
|
88
|
-
ok(entry?.args?.[2] === '${workspaceFolder}/mixgram', 'cursor entry uses workspaceFolder in args');
|
|
87
|
+
const hasProjectMemory = Array.isArray(entry?.args) && entry.args.includes('--project-memory') && entry.args.includes('${workspaceFolder}/docs');
|
|
88
|
+
ok(hasProjectMemory, 'cursor entry uses --project-memory ${workspaceFolder}/docs');
|
|
89
89
|
} catch (err) {
|
|
90
90
|
if (reporter.VISUAL) reporter.check('CLI setup cursor config threw', false, err.message);
|
|
91
91
|
f += 1;
|
|
@@ -168,15 +168,15 @@ async function runLegacySemanticKnn() {
|
|
|
168
168
|
const persistId = parse(savePersist).id;
|
|
169
169
|
ok(!!persistId, 'saved persistence doc');
|
|
170
170
|
const db = getDb(configSemantic);
|
|
171
|
-
const
|
|
172
|
-
ok(!!
|
|
173
|
-
if (embedder && vectorStore &&
|
|
174
|
-
const embedding = await embedder.embed(
|
|
171
|
+
const docRow = db.prepare('SELECT id, body, content_hash FROM documents WHERE id = ?').get(persistId);
|
|
172
|
+
ok(!!docRow, 'document exists for persistence doc');
|
|
173
|
+
if (embedder && vectorStore && docRow?.body) {
|
|
174
|
+
const embedding = await embedder.embed(docRow.body);
|
|
175
175
|
await vectorStore.ensureTables(configSemantic);
|
|
176
176
|
await vectorStore.insert(configSemantic, {
|
|
177
|
-
chunk_id:
|
|
178
|
-
document_id:
|
|
179
|
-
content_hash:
|
|
177
|
+
chunk_id: docRow.id,
|
|
178
|
+
document_id: docRow.id,
|
|
179
|
+
content_hash: docRow.content_hash,
|
|
180
180
|
embedding
|
|
181
181
|
});
|
|
182
182
|
const queryEmb = await embedder.embed('dónde se persisten los datos');
|
|
@@ -196,6 +196,10 @@ async function run() {
|
|
|
196
196
|
await runScenarios();
|
|
197
197
|
await runLegacySemanticKnn();
|
|
198
198
|
|
|
199
|
+
const largeResult = runLargeDocumentTest(config);
|
|
200
|
+
totalPassed += largeResult.passed;
|
|
201
|
+
totalFailed += largeResult.failed;
|
|
202
|
+
|
|
199
203
|
closeDb();
|
|
200
204
|
try { fs.rmSync(tmpDir, { recursive: true }); } catch (_) {}
|
|
201
205
|
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
semanticFallbackDoc,
|
|
22
22
|
PROJECT_NAME
|
|
23
23
|
} from '../fixtures/realistic-content.js';
|
|
24
|
+
import { toMarkdown } from '../../src/utils/markdown.js';
|
|
24
25
|
|
|
25
26
|
function parse(res) {
|
|
26
27
|
return JSON.parse(res.content[0].text);
|
|
@@ -45,7 +46,7 @@ export const scenarios = [
|
|
|
45
46
|
reporter.step('mem_save', { title: decisionSqliteIndex.title, topic_key: decisionSqliteIndex.topic_key }, { id: s1.id, path: s1.path, created: s1.created });
|
|
46
47
|
ok(s1.success && s1.id && s1.created === true, 'save success');
|
|
47
48
|
ok(fs.existsSync(s1.path), 'file created');
|
|
48
|
-
ok(s1.path.includes('
|
|
49
|
+
ok(s1.path.includes('docs') || s1.path.replace(/\\/g, '/').includes('docs'), 'project doc under project memory root');
|
|
49
50
|
ok(path.basename(s1.path) === 'architecture-sqlite-derived-index.md', 'project doc filename omits redundant prefixes');
|
|
50
51
|
|
|
51
52
|
const search1 = await h.mem_search({ query: 'derived SQLite index', project: PROJECT_NAME });
|
|
@@ -156,11 +157,12 @@ export const scenarios = [
|
|
|
156
157
|
|
|
157
158
|
reporter.startScenario(this.name, this.goal);
|
|
158
159
|
|
|
159
|
-
const search2 = await h.mem_search({ query: '
|
|
160
|
+
const search2 = await h.mem_search({ query: 'updated body', project: PROJECT_NAME, limit: 5 });
|
|
160
161
|
const res = parseRes(search2).results;
|
|
161
|
-
reporter.step('mem_search', { query: '
|
|
162
|
+
reporter.step('mem_search', { query: 'updated body', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
|
|
162
163
|
ok(Array.isArray(res), 'results array');
|
|
163
|
-
ok(res.length >= 1
|
|
164
|
+
ok(res.length >= 1, 'has results');
|
|
165
|
+
ok(res.length === 0 || res[0].snippet != null, 'snippets present');
|
|
164
166
|
|
|
165
167
|
reporter.endScenario(p, f);
|
|
166
168
|
return { passed: p, failed: f };
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
|
|
4
|
+
* Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
|
|
5
|
+
*/
|
|
6
|
+
import fs from 'fs';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
import { loadConfig } from '../src/config.js';
|
|
10
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
11
|
+
import { search } from '../src/core/search/search.js';
|
|
12
|
+
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
13
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
14
|
+
|
|
15
|
+
const TARGET_SIZE = 100_000;
|
|
16
|
+
const NEEDLE = 'NeedleInHaystack100k';
|
|
17
|
+
|
|
18
|
+
function buildLargeBody(targetSize) {
|
|
19
|
+
const intro = `# Documento grande\n\nPárrafo inicial. ${NEEDLE} aquí.\n\n`;
|
|
20
|
+
const paragraph = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(20) + '\n\n';
|
|
21
|
+
const repeat = Math.ceil((targetSize - intro.length) / paragraph.length);
|
|
22
|
+
return intro + paragraph.repeat(Math.max(0, repeat));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {object} config - resolved config (e.g. from run-tests)
|
|
27
|
+
* @returns {{ passed: number, failed: number }}
|
|
28
|
+
*/
|
|
29
|
+
export function runLargeDocumentTest(config) {
|
|
30
|
+
let passed = 0;
|
|
31
|
+
let failed = 0;
|
|
32
|
+
const ok = (cond, msg) => {
|
|
33
|
+
if (cond) passed++;
|
|
34
|
+
else failed++;
|
|
35
|
+
if (!cond) console.error('FAIL:', msg);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const docsDir = path.join(config.projectMemoryRoot, 'generated');
|
|
39
|
+
fs.mkdirSync(docsDir, { recursive: true });
|
|
40
|
+
|
|
41
|
+
const body = buildLargeBody(TARGET_SIZE);
|
|
42
|
+
const id = 'obs_large_100k';
|
|
43
|
+
const now = new Date().toISOString();
|
|
44
|
+
const frontmatter = {
|
|
45
|
+
id,
|
|
46
|
+
title: 'Documento 100k',
|
|
47
|
+
type: 'generated_note',
|
|
48
|
+
scope: 'project',
|
|
49
|
+
project: 'test',
|
|
50
|
+
topic_key: 'test/large-100k',
|
|
51
|
+
created_at: now
|
|
52
|
+
};
|
|
53
|
+
const raw = toMarkdown(frontmatter, body);
|
|
54
|
+
const docPath = path.join(docsDir, 'large-100k.md');
|
|
55
|
+
fs.writeFileSync(docPath, raw, 'utf8');
|
|
56
|
+
const stat = fs.statSync(docPath);
|
|
57
|
+
|
|
58
|
+
const size = raw.length;
|
|
59
|
+
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
|
+
|
|
61
|
+
indexDocument(config, docPath, raw, stat.mtimeMs);
|
|
62
|
+
|
|
63
|
+
const db = getDb(config);
|
|
64
|
+
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
|
65
|
+
ok(!!doc, 'documento en tabla documents');
|
|
66
|
+
ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
|
|
67
|
+
|
|
68
|
+
const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
|
|
69
|
+
ok(!!ftsRow, 'documento en FTS');
|
|
70
|
+
ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
|
|
71
|
+
|
|
72
|
+
const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
|
|
73
|
+
ok(results.length >= 1, 'búsqueda encuentra el documento');
|
|
74
|
+
ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
|
|
75
|
+
ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
|
|
76
|
+
|
|
77
|
+
return { passed, failed };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function main() {
|
|
81
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
|
|
82
|
+
const config = loadConfig({
|
|
83
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
84
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
85
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
86
|
+
});
|
|
87
|
+
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
88
|
+
|
|
89
|
+
const { passed, failed } = runLargeDocumentTest(config);
|
|
90
|
+
console.log(`Large document test: ${passed} passed, ${failed} failed`);
|
|
91
|
+
closeDb();
|
|
92
|
+
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
93
|
+
process.exitCode = failed > 0 ? 1 : 0;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (process.argv[1]?.endsWith('test-large-document.js')) {
|
|
97
|
+
main();
|
|
98
|
+
}
|
package/src/config.js
CHANGED
|
@@ -15,13 +15,16 @@ const DEFAULT_CONFIG = {
|
|
|
15
15
|
sqlitePath: '~/.mixgram/index.db',
|
|
16
16
|
watch: true,
|
|
17
17
|
indexing: {
|
|
18
|
-
chunkSize: 1200,
|
|
19
|
-
chunkOverlap: 120,
|
|
20
18
|
reindexOnStartup: true,
|
|
19
|
+
includeCodeBlocks: false,
|
|
21
20
|
ftsWeights: {
|
|
22
21
|
title: 10.0,
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
h1: 8.0,
|
|
23
|
+
h2: 6.0,
|
|
24
|
+
h3: 5.0,
|
|
25
|
+
h4: 4.0,
|
|
26
|
+
h5: 3.0,
|
|
27
|
+
h6: 2.0,
|
|
25
28
|
body: 1.0
|
|
26
29
|
}
|
|
27
30
|
},
|
|
@@ -55,7 +58,7 @@ function resolvePaths(config, baseDir, projectBaseDir = null) {
|
|
|
55
58
|
return {
|
|
56
59
|
...config,
|
|
57
60
|
homeMemoryRoot: path.isAbsolute(homeRoot) ? homeRoot : path.resolve(base, homeRoot),
|
|
58
|
-
projectMemoryRoot: path.resolve(projectBase, config.projectMemoryRoot ?? './
|
|
61
|
+
projectMemoryRoot: path.resolve(projectBase, config.projectMemoryRoot ?? './docs'),
|
|
59
62
|
sqlitePath: path.isAbsolute(sqlite) ? sqlite : path.resolve(base, sqlite)
|
|
60
63
|
};
|
|
61
64
|
}
|
|
@@ -157,9 +157,9 @@ function deleteDocument(config, documentId, options = {}) {
|
|
|
157
157
|
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
158
158
|
const db = getDb(config);
|
|
159
159
|
db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
|
|
160
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
160
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
|
|
161
161
|
for (const { rowid } of ftsRowids) {
|
|
162
|
-
db.prepare('DELETE FROM
|
|
162
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
163
163
|
}
|
|
164
164
|
}
|
|
165
165
|
return true;
|
|
@@ -84,23 +84,23 @@ async function processNextJob(config) {
|
|
|
84
84
|
return true;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
-
const
|
|
88
|
-
SELECT
|
|
89
|
-
FROM
|
|
90
|
-
WHERE
|
|
87
|
+
const docRow = db.prepare(`
|
|
88
|
+
SELECT id, id AS document_id, content_hash, body AS content
|
|
89
|
+
FROM documents
|
|
90
|
+
WHERE id = ?
|
|
91
91
|
`).get(job.chunk_id);
|
|
92
|
-
if (!
|
|
92
|
+
if (!docRow?.content) {
|
|
93
93
|
db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
|
|
94
|
-
.run(STATUS.FAILED, '
|
|
94
|
+
.run(STATUS.FAILED, 'document not found or empty body', now, job.id);
|
|
95
95
|
return true;
|
|
96
96
|
}
|
|
97
97
|
|
|
98
98
|
try {
|
|
99
|
-
const embedding = await embedder.embed(
|
|
99
|
+
const embedding = await embedder.embed(docRow.content);
|
|
100
100
|
await vectorStore.insert(config, {
|
|
101
|
-
chunk_id:
|
|
102
|
-
document_id:
|
|
103
|
-
content_hash:
|
|
101
|
+
chunk_id: docRow.id,
|
|
102
|
+
document_id: docRow.document_id,
|
|
103
|
+
content_hash: docRow.content_hash,
|
|
104
104
|
embedding
|
|
105
105
|
});
|
|
106
106
|
db.prepare(`UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE id = ?`)
|
|
@@ -43,8 +43,7 @@ async function ensureTables(config) {
|
|
|
43
43
|
document_id TEXT NOT NULL,
|
|
44
44
|
content_hash TEXT NOT NULL,
|
|
45
45
|
embedding_model TEXT NOT NULL,
|
|
46
|
-
created_at TEXT NOT NULL
|
|
47
|
-
FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
|
|
46
|
+
created_at TEXT NOT NULL
|
|
48
47
|
)
|
|
49
48
|
`);
|
|
50
49
|
db.exec(`
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standalone process for embedding jobs and query embeddings. Isolates
|
|
3
|
+
* @huggingface/transformers (and any native/ONNX crashes) from the MCP server.
|
|
4
|
+
* Invoked via child_process.fork() with env MIXGRAM_WORKER_CONFIG pointing to
|
|
5
|
+
* a JSON file with the resolved config. Listens for IPC 'embed' to embed query
|
|
6
|
+
* text for hybrid search so the main process never loads transformers.
|
|
7
|
+
*/
|
|
8
|
+
import fs from 'fs';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import { fileURLToPath } from 'url';
|
|
11
|
+
|
|
12
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
|
+
const DEFAULT_POLL_MS = 2000;
|
|
14
|
+
|
|
15
|
+
function loadWorkerConfig() {
|
|
16
|
+
const configPath = process.env.MIXGRAM_WORKER_CONFIG;
|
|
17
|
+
if (!configPath || !fs.existsSync(configPath)) return null;
|
|
18
|
+
const raw = fs.readFileSync(configPath, 'utf8');
|
|
19
|
+
return JSON.parse(raw);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
let embedderInstance = null;
|
|
23
|
+
|
|
24
|
+
async function getEmbedder(config) {
|
|
25
|
+
if (embedderInstance) return embedderInstance;
|
|
26
|
+
const { getEmbedder: getEmbedderFn } = await import('./embedder.js');
|
|
27
|
+
embedderInstance = await getEmbedderFn(config);
|
|
28
|
+
return embedderInstance;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function setupIpc(config) {
|
|
32
|
+
process.on('message', async (msg) => {
|
|
33
|
+
if (msg?.type !== 'embed' || typeof msg.text !== 'string') return;
|
|
34
|
+
try {
|
|
35
|
+
const embedder = await getEmbedder(config);
|
|
36
|
+
if (!embedder) {
|
|
37
|
+
process.send?.({ type: 'embedResult', id: msg.id, err: 'embeddings not available' });
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const vector = await embedder.embed(msg.text);
|
|
41
|
+
process.send?.({ type: 'embedResult', id: msg.id, vector: Array.from(vector) });
|
|
42
|
+
} catch (err) {
|
|
43
|
+
process.send?.({ type: 'embedResult', id: msg.id, err: String(err?.message ?? err) });
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function runLoop(config) {
|
|
49
|
+
const { processNextJob } = await import('./queue.js');
|
|
50
|
+
const pollMs = config?.embeddings?.workerPollMs ?? DEFAULT_POLL_MS;
|
|
51
|
+
const run = async () => {
|
|
52
|
+
try {
|
|
53
|
+
await processNextJob(config);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
if (process.stderr) {
|
|
56
|
+
process.stderr.write(`[mixgram-worker] job error: ${err?.message ?? err}\n`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
await run();
|
|
61
|
+
setInterval(run, pollMs);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function main() {
|
|
65
|
+
const config = loadWorkerConfig();
|
|
66
|
+
if (!config?.embeddings?.enabled) {
|
|
67
|
+
process.exit(0);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (process.send) setupIpc(config);
|
|
71
|
+
runLoop(config).catch((err) => {
|
|
72
|
+
if (process.stderr) {
|
|
73
|
+
process.stderr.write(`[mixgram-worker] fatal: ${err?.message ?? err}\n`);
|
|
74
|
+
}
|
|
75
|
+
process.exit(1);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
main();
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { getDb } from '../../db/sqlite.js';
|
|
2
|
-
import {
|
|
2
|
+
import { parseMarkdown } from './parser.js';
|
|
3
3
|
import { contentHash } from '../../utils/hash.js';
|
|
4
|
-
import { chunkId } from '../../utils/ids.js';
|
|
5
4
|
|
|
6
5
|
function normalizeForFts(str) {
|
|
7
6
|
if (str == null || str === undefined) return '';
|
|
@@ -14,21 +13,26 @@ function normalizeForFts(str) {
|
|
|
14
13
|
|
|
15
14
|
function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
16
15
|
const db = getDb(config);
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
chunkOverlap: config.indexing.chunkOverlap
|
|
20
|
-
});
|
|
16
|
+
const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
|
|
17
|
+
const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
|
|
21
18
|
|
|
19
|
+
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
22
20
|
const id = frontmatter.id;
|
|
23
21
|
if (!id) throw new Error('Document frontmatter must include id');
|
|
24
22
|
|
|
25
23
|
const now = new Date().toISOString();
|
|
26
24
|
const docHash = contentHash(rawContent);
|
|
25
|
+
const docTitle = frontmatter.title || title || '';
|
|
26
|
+
|
|
27
|
+
const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
|
|
28
|
+
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
29
|
+
return { id };
|
|
30
|
+
}
|
|
27
31
|
|
|
28
32
|
const docRow = {
|
|
29
33
|
id,
|
|
30
34
|
path: docPath,
|
|
31
|
-
title:
|
|
35
|
+
title: docTitle,
|
|
32
36
|
type: frontmatter.type || 'generated_note',
|
|
33
37
|
scope: frontmatter.scope || 'project',
|
|
34
38
|
project: frontmatter.project || null,
|
|
@@ -43,49 +47,42 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
43
47
|
deleted_at: frontmatter.deleted_at || null,
|
|
44
48
|
revision_count: frontmatter.revision_count ?? 1,
|
|
45
49
|
duplicate_count: frontmatter.duplicate_count ?? 0,
|
|
46
|
-
embedding_status: frontmatter.embedding_status || 'disabled'
|
|
50
|
+
embedding_status: frontmatter.embedding_status || 'disabled',
|
|
51
|
+
body: body || '',
|
|
52
|
+
structure: JSON.stringify(structure),
|
|
53
|
+
sections_index: JSON.stringify(sectionsIndex)
|
|
47
54
|
};
|
|
48
55
|
|
|
49
|
-
let
|
|
56
|
+
let wasExisting = false;
|
|
50
57
|
db.transaction(() => {
|
|
51
|
-
const
|
|
52
|
-
if (
|
|
53
|
-
|
|
54
|
-
const ftsRowids = db.prepare('SELECT rowid FROM
|
|
58
|
+
const existingDoc = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
|
|
59
|
+
if (existingDoc) {
|
|
60
|
+
wasExisting = true;
|
|
61
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(id);
|
|
55
62
|
for (const { rowid } of ftsRowids) {
|
|
56
|
-
db.prepare('DELETE FROM
|
|
63
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
57
64
|
}
|
|
58
|
-
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(id);
|
|
59
65
|
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(id);
|
|
60
66
|
}
|
|
61
67
|
|
|
62
68
|
db.prepare(`
|
|
63
|
-
INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status)
|
|
64
|
-
VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status)
|
|
69
|
+
INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status, body, structure, sections_index)
|
|
70
|
+
VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status, @body, @structure, @sections_index)
|
|
65
71
|
`).run(docRow);
|
|
66
72
|
|
|
67
|
-
const titleNorm = normalizeForFts(
|
|
68
|
-
const
|
|
69
|
-
const
|
|
70
|
-
const
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const chunkHash = contentHash(chunk.content);
|
|
76
|
-
const headingPathNorm = normalizeForFts(chunk.headingPath);
|
|
77
|
-
const bodyNorm = normalizeForFts(chunk.content);
|
|
78
|
-
|
|
79
|
-
db.prepare(`
|
|
80
|
-
INSERT INTO document_chunks (id, document_id, chunk_index, heading_path, heading_level, content, content_hash, created_at, updated_at)
|
|
81
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
82
|
-
`).run(cid, id, idx, chunk.headingPath || '', chunk.headingLevel || 0, chunk.content, chunkHash, now, now);
|
|
73
|
+
const titleNorm = normalizeForFts(docTitle);
|
|
74
|
+
const h1Norm = normalizeForFts(h1);
|
|
75
|
+
const h2Norm = normalizeForFts(h2);
|
|
76
|
+
const h3Norm = normalizeForFts(h3);
|
|
77
|
+
const h4Norm = normalizeForFts(h4);
|
|
78
|
+
const h5Norm = normalizeForFts(h5);
|
|
79
|
+
const h6Norm = normalizeForFts(h6);
|
|
80
|
+
const bodyNorm = normalizeForFts(body);
|
|
83
81
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
});
|
|
82
|
+
db.prepare(`
|
|
83
|
+
INSERT INTO document_fts (document_id, title, h1, h2, h3, h4, h5, h6, body)
|
|
84
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
|
+
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
89
86
|
|
|
90
87
|
const tags = frontmatter.tags || [];
|
|
91
88
|
for (const tag of tags) {
|
|
@@ -94,51 +91,37 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
94
91
|
})();
|
|
95
92
|
|
|
96
93
|
if (config?.embeddings?.enabled) {
|
|
97
|
-
const newChunkIds = Array.from({ length: chunks.length }, (_, idx) => chunkId(id, idx));
|
|
98
94
|
import('../embeddings/queue.js')
|
|
99
95
|
.then(({ enqueueChunks, markStale }) => {
|
|
100
|
-
if (
|
|
101
|
-
enqueueChunks(config,
|
|
96
|
+
if (wasExisting) markStale(config, [id]);
|
|
97
|
+
enqueueChunks(config, [id]);
|
|
102
98
|
})
|
|
103
99
|
.catch((err) => {
|
|
104
100
|
if (typeof process !== 'undefined' && process.stderr) {
|
|
105
101
|
process.stderr.write(`[mixgram] embeddings queue error: ${err?.message ?? err}\n`);
|
|
106
102
|
}
|
|
107
103
|
});
|
|
108
|
-
if (oldChunkIds.length) {
|
|
109
|
-
import('../embeddings/vectorStore.js')
|
|
110
|
-
.then(({ removeChunks }) => removeChunks(config, oldChunkIds))
|
|
111
|
-
.catch(() => {});
|
|
112
|
-
}
|
|
113
104
|
}
|
|
114
105
|
|
|
115
|
-
return { id
|
|
106
|
+
return { id };
|
|
116
107
|
}
|
|
117
108
|
|
|
118
109
|
function removeDocumentFromIndex(config, documentId) {
|
|
119
110
|
const db = getDb(config);
|
|
120
|
-
|
|
121
|
-
if (config?.embeddings?.enabled && chunkIds.length) {
|
|
111
|
+
if (config?.embeddings?.enabled) {
|
|
122
112
|
import('../embeddings/queue.js')
|
|
123
|
-
.then(({ markStale }) => markStale(config,
|
|
124
|
-
.catch((
|
|
125
|
-
if (typeof process !== 'undefined' && process.stderr) {
|
|
126
|
-
process.stderr.write(`[mixgram] embeddings markStale error: ${err?.message ?? err}\n`);
|
|
127
|
-
}
|
|
128
|
-
});
|
|
113
|
+
.then(({ markStale }) => markStale(config, [documentId]))
|
|
114
|
+
.catch(() => {});
|
|
129
115
|
import('../embeddings/vectorStore.js')
|
|
130
|
-
.then(({ removeChunks }) => removeChunks(config,
|
|
116
|
+
.then(({ removeChunks }) => removeChunks(config, [documentId]))
|
|
131
117
|
.catch(() => {});
|
|
132
118
|
}
|
|
133
|
-
|
|
134
|
-
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(cid);
|
|
135
|
-
}
|
|
136
|
-
const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
|
|
119
|
+
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(documentId);
|
|
137
120
|
db.transaction(() => {
|
|
121
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
|
|
138
122
|
for (const { rowid } of ftsRowids) {
|
|
139
|
-
db.prepare('DELETE FROM
|
|
123
|
+
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
140
124
|
}
|
|
141
|
-
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(documentId);
|
|
142
125
|
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
|
|
143
126
|
db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
|
|
144
127
|
})();
|