mixgram 1.1.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/spec/run-tests.js CHANGED
@@ -14,6 +14,7 @@ import { createToolHandlers } from '../src/mcp/tools.js';
14
14
  import { getToolDefinitions } from '../src/mcp/tool-registry.js';
15
15
  import { closeDb, getDb } from '../src/db/sqlite.js';
16
16
  import { scenarios } from './scenarios/scenarios.js';
17
+ import { runLargeDocumentTest } from './test-large-document.js';
17
18
  import * as reporter from './reporter.js';
18
19
 
19
20
  const __filename = fileURLToPath(import.meta.url);
@@ -22,7 +23,7 @@ const __dirname = path.dirname(__filename);
22
23
  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-test-'));
23
24
  const config = loadConfig({
24
25
  homeMemoryRoot: path.join(tmpDir, 'home', 'memory'),
25
- projectMemoryRoot: path.join(tmpDir, 'mixgram'),
26
+ projectMemoryRoot: path.join(tmpDir, 'docs'),
26
27
  sqlitePath: path.join(tmpDir, '.mixgram', 'index.db')
27
28
  });
28
29
  fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
@@ -57,7 +58,7 @@ function runCliSetupChecks() {
57
58
  const repoRoot = path.dirname(__dirname);
58
59
 
59
60
  if (reporter.VISUAL) {
60
- reporter.startScenario('CLI setup cursor config', 'Verify Cursor setup writes workspace cwd for project-scoped memory.');
61
+ reporter.startScenario('CLI setup cursor config', 'Verify Cursor setup writes project-memory arg for project-scoped memory.');
61
62
  }
62
63
 
63
64
  let p = 0;
@@ -83,9 +84,8 @@ function runCliSetupChecks() {
83
84
  const cursorConfig = JSON.parse(fs.readFileSync(cursorConfigPath, 'utf8'));
84
85
  const entry = cursorConfig?.mcpServers?.mixgram;
85
86
  ok(entry?.command === 'mixgram', 'cursor entry command preserved');
86
- ok(Array.isArray(entry?.args) && entry.args[0] === 'mcp', 'cursor entry args preserved');
87
- ok(entry?.args?.[1] === '--project-memory', 'cursor entry overrides project memory');
88
- ok(entry?.args?.[2] === '${workspaceFolder}/mixgram', 'cursor entry uses workspaceFolder in args');
87
+ const hasProjectMemory = Array.isArray(entry?.args) && entry.args.includes('--project-memory') && entry.args.includes('${workspaceFolder}/docs');
88
+ ok(hasProjectMemory, 'cursor entry uses --project-memory ${workspaceFolder}/docs');
89
89
  } catch (err) {
90
90
  if (reporter.VISUAL) reporter.check('CLI setup cursor config threw', false, err.message);
91
91
  f += 1;
@@ -168,15 +168,15 @@ async function runLegacySemanticKnn() {
168
168
  const persistId = parse(savePersist).id;
169
169
  ok(!!persistId, 'saved persistence doc');
170
170
  const db = getDb(configSemantic);
171
- const chunkRow = db.prepare('SELECT id, document_id, content_hash, content FROM document_chunks WHERE document_id = ? LIMIT 1').get(persistId);
172
- ok(!!chunkRow, 'chunk exists for persistence doc');
173
- if (embedder && vectorStore && chunkRow) {
174
- const embedding = await embedder.embed(chunkRow.content);
171
+ const docRow = db.prepare('SELECT id, body, content_hash FROM documents WHERE id = ?').get(persistId);
172
+ ok(!!docRow, 'document exists for persistence doc');
173
+ if (embedder && vectorStore && docRow?.body) {
174
+ const embedding = await embedder.embed(docRow.body);
175
175
  await vectorStore.ensureTables(configSemantic);
176
176
  await vectorStore.insert(configSemantic, {
177
- chunk_id: chunkRow.id,
178
- document_id: chunkRow.document_id,
179
- content_hash: chunkRow.content_hash,
177
+ chunk_id: docRow.id,
178
+ document_id: docRow.id,
179
+ content_hash: docRow.content_hash,
180
180
  embedding
181
181
  });
182
182
  const queryEmb = await embedder.embed('dónde se persisten los datos');
@@ -196,6 +196,10 @@ async function run() {
196
196
  await runScenarios();
197
197
  await runLegacySemanticKnn();
198
198
 
199
+ const largeResult = runLargeDocumentTest(config);
200
+ totalPassed += largeResult.passed;
201
+ totalFailed += largeResult.failed;
202
+
199
203
  closeDb();
200
204
  try { fs.rmSync(tmpDir, { recursive: true }); } catch (_) {}
201
205
 
@@ -21,6 +21,7 @@ import {
21
21
  semanticFallbackDoc,
22
22
  PROJECT_NAME
23
23
  } from '../fixtures/realistic-content.js';
24
+ import { toMarkdown } from '../../src/utils/markdown.js';
24
25
 
25
26
  function parse(res) {
26
27
  return JSON.parse(res.content[0].text);
@@ -45,7 +46,7 @@ export const scenarios = [
45
46
  reporter.step('mem_save', { title: decisionSqliteIndex.title, topic_key: decisionSqliteIndex.topic_key }, { id: s1.id, path: s1.path, created: s1.created });
46
47
  ok(s1.success && s1.id && s1.created === true, 'save success');
47
48
  ok(fs.existsSync(s1.path), 'file created');
48
- ok(s1.path.includes('mixgram') || s1.path.replace(/\\/g, '/').includes('mixgram'), 'project doc under mixgram');
49
+ ok(s1.path.includes('docs') || s1.path.replace(/\\/g, '/').includes('docs'), 'project doc under project memory root');
49
50
  ok(path.basename(s1.path) === 'architecture-sqlite-derived-index.md', 'project doc filename omits redundant prefixes');
50
51
 
51
52
  const search1 = await h.mem_search({ query: 'derived SQLite index', project: PROJECT_NAME });
@@ -156,11 +157,12 @@ export const scenarios = [
156
157
 
157
158
  reporter.startScenario(this.name, this.goal);
158
159
 
159
- const search2 = await h.mem_search({ query: 'derived index', project: PROJECT_NAME, limit: 5 });
160
+ const search2 = await h.mem_search({ query: 'updated body', project: PROJECT_NAME, limit: 5 });
160
161
  const res = parseRes(search2).results;
161
- reporter.step('mem_search', { query: 'derived index', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
162
+ reporter.step('mem_search', { query: 'updated body', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
162
163
  ok(Array.isArray(res), 'results array');
163
- ok(res.length >= 1 && res[0].snippet != null, 'snippets present');
164
+ ok(res.length >= 1, 'has results');
165
+ ok(res.length === 0 || res[0].snippet != null, 'snippets present');
164
166
 
165
167
  reporter.endScenario(p, f);
166
168
  return { passed: p, failed: f };
@@ -0,0 +1,98 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
4
+ * Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
5
+ */
6
+ import fs from 'fs';
7
+ import path from 'path';
8
+ import os from 'os';
9
+ import { loadConfig } from '../src/config.js';
10
+ import { indexDocument } from '../src/core/indexing/indexer.js';
11
+ import { search } from '../src/core/search/search.js';
12
+ import { getDb, closeDb } from '../src/db/sqlite.js';
13
+ import { toMarkdown } from '../src/utils/markdown.js';
14
+
15
+ const TARGET_SIZE = 100_000;
16
+ const NEEDLE = 'NeedleInHaystack100k';
17
+
18
+ function buildLargeBody(targetSize) {
19
+ const intro = `# Documento grande\n\nPárrafo inicial. ${NEEDLE} aquí.\n\n`;
20
+ const paragraph = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(20) + '\n\n';
21
+ const repeat = Math.ceil((targetSize - intro.length) / paragraph.length);
22
+ return intro + paragraph.repeat(Math.max(0, repeat));
23
+ }
24
+
25
+ /**
26
+ * @param {object} config - resolved config (e.g. from run-tests)
27
+ * @returns {{ passed: number, failed: number }}
28
+ */
29
+ export function runLargeDocumentTest(config) {
30
+ let passed = 0;
31
+ let failed = 0;
32
+ const ok = (cond, msg) => {
33
+ if (cond) passed++;
34
+ else failed++;
35
+ if (!cond) console.error('FAIL:', msg);
36
+ };
37
+
38
+ const docsDir = path.join(config.projectMemoryRoot, 'generated');
39
+ fs.mkdirSync(docsDir, { recursive: true });
40
+
41
+ const body = buildLargeBody(TARGET_SIZE);
42
+ const id = 'obs_large_100k';
43
+ const now = new Date().toISOString();
44
+ const frontmatter = {
45
+ id,
46
+ title: 'Documento 100k',
47
+ type: 'generated_note',
48
+ scope: 'project',
49
+ project: 'test',
50
+ topic_key: 'test/large-100k',
51
+ created_at: now
52
+ };
53
+ const raw = toMarkdown(frontmatter, body);
54
+ const docPath = path.join(docsDir, 'large-100k.md');
55
+ fs.writeFileSync(docPath, raw, 'utf8');
56
+ const stat = fs.statSync(docPath);
57
+
58
+ const size = raw.length;
59
+ ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
60
+
61
+ indexDocument(config, docPath, raw, stat.mtimeMs);
62
+
63
+ const db = getDb(config);
64
+ const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
65
+ ok(!!doc, 'documento en tabla documents');
66
+ ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
67
+
68
+ const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
69
+ ok(!!ftsRow, 'documento en FTS');
70
+ ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
71
+
72
+ const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
73
+ ok(results.length >= 1, 'búsqueda encuentra el documento');
74
+ ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
75
+ ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
76
+
77
+ return { passed, failed };
78
+ }
79
+
80
+ function main() {
81
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
82
+ const config = loadConfig({
83
+ homeMemoryRoot: path.join(tmpDir, 'home'),
84
+ projectMemoryRoot: path.join(tmpDir, 'docs'),
85
+ sqlitePath: path.join(tmpDir, 'index.db')
86
+ });
87
+ fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
88
+
89
+ const { passed, failed } = runLargeDocumentTest(config);
90
+ console.log(`Large document test: ${passed} passed, ${failed} failed`);
91
+ closeDb();
92
+ try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
93
+ process.exitCode = failed > 0 ? 1 : 0;
94
+ }
95
+
96
+ if (process.argv[1]?.endsWith('test-large-document.js')) {
97
+ main();
98
+ }
package/src/config.js CHANGED
@@ -15,13 +15,16 @@ const DEFAULT_CONFIG = {
15
15
  sqlitePath: '~/.mixgram/index.db',
16
16
  watch: true,
17
17
  indexing: {
18
- chunkSize: 1200,
19
- chunkOverlap: 120,
20
18
  reindexOnStartup: true,
19
+ includeCodeBlocks: false,
21
20
  ftsWeights: {
22
21
  title: 10.0,
23
- topicKey: 8.0,
24
- heading: 5.0,
22
+ h1: 8.0,
23
+ h2: 6.0,
24
+ h3: 5.0,
25
+ h4: 4.0,
26
+ h5: 3.0,
27
+ h6: 2.0,
25
28
  body: 1.0
26
29
  }
27
30
  },
@@ -55,7 +58,7 @@ function resolvePaths(config, baseDir, projectBaseDir = null) {
55
58
  return {
56
59
  ...config,
57
60
  homeMemoryRoot: path.isAbsolute(homeRoot) ? homeRoot : path.resolve(base, homeRoot),
58
- projectMemoryRoot: path.resolve(projectBase, config.projectMemoryRoot ?? './mixgram'),
61
+ projectMemoryRoot: path.resolve(projectBase, config.projectMemoryRoot ?? './docs'),
59
62
  sqlitePath: path.isAbsolute(sqlite) ? sqlite : path.resolve(base, sqlite)
60
63
  };
61
64
  }
@@ -157,9 +157,9 @@ function deleteDocument(config, documentId, options = {}) {
157
157
  fs.writeFileSync(resolved.path, raw, 'utf8');
158
158
  const db = getDb(config);
159
159
  db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
160
- const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
160
+ const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
161
161
  for (const { rowid } of ftsRowids) {
162
- db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
162
+ db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
163
163
  }
164
164
  }
165
165
  return true;
@@ -84,23 +84,23 @@ async function processNextJob(config) {
84
84
  return true;
85
85
  }
86
86
 
87
- const chunkRow = db.prepare(`
88
- SELECT c.id, c.document_id, c.content_hash, c.content
89
- FROM document_chunks c
90
- WHERE c.id = ?
87
+ const docRow = db.prepare(`
88
+ SELECT id, id AS document_id, content_hash, body AS content
89
+ FROM documents
90
+ WHERE id = ?
91
91
  `).get(job.chunk_id);
92
- if (!chunkRow) {
92
+ if (!docRow?.content) {
93
93
  db.prepare(`UPDATE embedding_jobs SET status = ?, last_error = ?, updated_at = ? WHERE id = ?`)
94
- .run(STATUS.FAILED, 'chunk not found', now, job.id);
94
+ .run(STATUS.FAILED, 'document not found or empty body', now, job.id);
95
95
  return true;
96
96
  }
97
97
 
98
98
  try {
99
- const embedding = await embedder.embed(chunkRow.content);
99
+ const embedding = await embedder.embed(docRow.content);
100
100
  await vectorStore.insert(config, {
101
- chunk_id: chunkRow.id,
102
- document_id: chunkRow.document_id,
103
- content_hash: chunkRow.content_hash,
101
+ chunk_id: docRow.id,
102
+ document_id: docRow.document_id,
103
+ content_hash: docRow.content_hash,
104
104
  embedding
105
105
  });
106
106
  db.prepare(`UPDATE embedding_jobs SET status = ?, updated_at = ? WHERE id = ?`)
@@ -43,8 +43,7 @@ async function ensureTables(config) {
43
43
  document_id TEXT NOT NULL,
44
44
  content_hash TEXT NOT NULL,
45
45
  embedding_model TEXT NOT NULL,
46
- created_at TEXT NOT NULL,
47
- FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
46
+ created_at TEXT NOT NULL
48
47
  )
49
48
  `);
50
49
  db.exec(`
@@ -0,0 +1,79 @@
1
+ /**
2
+ * Standalone process for embedding jobs and query embeddings. Isolates
3
+ * @huggingface/transformers (and any native/ONNX crashes) from the MCP server.
4
+ * Invoked via child_process.fork() with env MIXGRAM_WORKER_CONFIG pointing to
5
+ * a JSON file with the resolved config. Listens for IPC 'embed' to embed query
6
+ * text for hybrid search so the main process never loads transformers.
7
+ */
8
+ import fs from 'fs';
9
+ import path from 'path';
10
+ import { fileURLToPath } from 'url';
11
+
12
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
13
+ const DEFAULT_POLL_MS = 2000;
14
+
15
+ function loadWorkerConfig() {
16
+ const configPath = process.env.MIXGRAM_WORKER_CONFIG;
17
+ if (!configPath || !fs.existsSync(configPath)) return null;
18
+ const raw = fs.readFileSync(configPath, 'utf8');
19
+ return JSON.parse(raw);
20
+ }
21
+
22
+ let embedderInstance = null;
23
+
24
+ async function getEmbedder(config) {
25
+ if (embedderInstance) return embedderInstance;
26
+ const { getEmbedder: getEmbedderFn } = await import('./embedder.js');
27
+ embedderInstance = await getEmbedderFn(config);
28
+ return embedderInstance;
29
+ }
30
+
31
+ function setupIpc(config) {
32
+ process.on('message', async (msg) => {
33
+ if (msg?.type !== 'embed' || typeof msg.text !== 'string') return;
34
+ try {
35
+ const embedder = await getEmbedder(config);
36
+ if (!embedder) {
37
+ process.send?.({ type: 'embedResult', id: msg.id, err: 'embeddings not available' });
38
+ return;
39
+ }
40
+ const vector = await embedder.embed(msg.text);
41
+ process.send?.({ type: 'embedResult', id: msg.id, vector: Array.from(vector) });
42
+ } catch (err) {
43
+ process.send?.({ type: 'embedResult', id: msg.id, err: String(err?.message ?? err) });
44
+ }
45
+ });
46
+ }
47
+
48
+ async function runLoop(config) {
49
+ const { processNextJob } = await import('./queue.js');
50
+ const pollMs = config?.embeddings?.workerPollMs ?? DEFAULT_POLL_MS;
51
+ const run = async () => {
52
+ try {
53
+ await processNextJob(config);
54
+ } catch (err) {
55
+ if (process.stderr) {
56
+ process.stderr.write(`[mixgram-worker] job error: ${err?.message ?? err}\n`);
57
+ }
58
+ }
59
+ };
60
+ await run();
61
+ setInterval(run, pollMs);
62
+ }
63
+
64
+ function main() {
65
+ const config = loadWorkerConfig();
66
+ if (!config?.embeddings?.enabled) {
67
+ process.exit(0);
68
+ return;
69
+ }
70
+ if (process.send) setupIpc(config);
71
+ runLoop(config).catch((err) => {
72
+ if (process.stderr) {
73
+ process.stderr.write(`[mixgram-worker] fatal: ${err?.message ?? err}\n`);
74
+ }
75
+ process.exit(1);
76
+ });
77
+ }
78
+
79
+ main();
@@ -1,7 +1,6 @@
1
1
  import { getDb } from '../../db/sqlite.js';
2
- import { parseAndChunk } from './parser.js';
2
+ import { parseMarkdown } from './parser.js';
3
3
  import { contentHash } from '../../utils/hash.js';
4
- import { chunkId } from '../../utils/ids.js';
5
4
 
6
5
  function normalizeForFts(str) {
7
6
  if (str == null || str === undefined) return '';
@@ -14,21 +13,26 @@ function normalizeForFts(str) {
14
13
 
15
14
  function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
16
15
  const db = getDb(config);
17
- const { frontmatter, chunks } = parseAndChunk(rawContent, {
18
- chunkSize: config.indexing.chunkSize,
19
- chunkOverlap: config.indexing.chunkOverlap
20
- });
16
+ const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
17
+ const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
21
18
 
19
+ const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
22
20
  const id = frontmatter.id;
23
21
  if (!id) throw new Error('Document frontmatter must include id');
24
22
 
25
23
  const now = new Date().toISOString();
26
24
  const docHash = contentHash(rawContent);
25
+ const docTitle = frontmatter.title || title || '';
26
+
27
+ const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
28
+ if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
29
+ return { id };
30
+ }
27
31
 
28
32
  const docRow = {
29
33
  id,
30
34
  path: docPath,
31
- title: frontmatter.title || '',
35
+ title: docTitle,
32
36
  type: frontmatter.type || 'generated_note',
33
37
  scope: frontmatter.scope || 'project',
34
38
  project: frontmatter.project || null,
@@ -43,49 +47,42 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
43
47
  deleted_at: frontmatter.deleted_at || null,
44
48
  revision_count: frontmatter.revision_count ?? 1,
45
49
  duplicate_count: frontmatter.duplicate_count ?? 0,
46
- embedding_status: frontmatter.embedding_status || 'disabled'
50
+ embedding_status: frontmatter.embedding_status || 'disabled',
51
+ body: body || '',
52
+ structure: JSON.stringify(structure),
53
+ sections_index: JSON.stringify(sectionsIndex)
47
54
  };
48
55
 
49
- let oldChunkIds = [];
56
+ let wasExisting = false;
50
57
  db.transaction(() => {
51
- const existing = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
52
- if (existing) {
53
- oldChunkIds = db.prepare('SELECT id FROM document_chunks WHERE document_id = ?').all(id).map((r) => r.id);
54
- const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(id);
58
+ const existingDoc = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
59
+ if (existingDoc) {
60
+ wasExisting = true;
61
+ const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(id);
55
62
  for (const { rowid } of ftsRowids) {
56
- db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
63
+ db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
57
64
  }
58
- db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(id);
59
65
  db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(id);
60
66
  }
61
67
 
62
68
  db.prepare(`
63
- INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status)
64
- VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status)
69
+ INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status, body, structure, sections_index)
70
+ VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status, @body, @structure, @sections_index)
65
71
  `).run(docRow);
66
72
 
67
- const titleNorm = normalizeForFts(docRow.title);
68
- const topicKeyNorm = normalizeForFts(docRow.topic_key);
69
- const typeNorm = normalizeForFts(docRow.type);
70
- const scopeNorm = normalizeForFts(docRow.scope);
71
- const projectNorm = normalizeForFts(docRow.project);
72
-
73
- chunks.forEach((chunk, idx) => {
74
- const cid = chunkId(id, idx);
75
- const chunkHash = contentHash(chunk.content);
76
- const headingPathNorm = normalizeForFts(chunk.headingPath);
77
- const bodyNorm = normalizeForFts(chunk.content);
78
-
79
- db.prepare(`
80
- INSERT INTO document_chunks (id, document_id, chunk_index, heading_path, heading_level, content, content_hash, created_at, updated_at)
81
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
82
- `).run(cid, id, idx, chunk.headingPath || '', chunk.headingLevel || 0, chunk.content, chunkHash, now, now);
73
+ const titleNorm = normalizeForFts(docTitle);
74
+ const h1Norm = normalizeForFts(h1);
75
+ const h2Norm = normalizeForFts(h2);
76
+ const h3Norm = normalizeForFts(h3);
77
+ const h4Norm = normalizeForFts(h4);
78
+ const h5Norm = normalizeForFts(h5);
79
+ const h6Norm = normalizeForFts(h6);
80
+ const bodyNorm = normalizeForFts(body);
83
81
 
84
- db.prepare(`
85
- INSERT INTO document_chunks_fts (chunk_id, document_id, title, topic_key, type, scope, project, heading_path, body)
86
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
87
- `).run(cid, id, titleNorm, topicKeyNorm, typeNorm, scopeNorm, projectNorm, headingPathNorm, bodyNorm);
88
- });
82
+ db.prepare(`
83
+ INSERT INTO document_fts (document_id, title, h1, h2, h3, h4, h5, h6, body)
84
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
85
+ `).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
89
86
 
90
87
  const tags = frontmatter.tags || [];
91
88
  for (const tag of tags) {
@@ -94,51 +91,37 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
94
91
  })();
95
92
 
96
93
  if (config?.embeddings?.enabled) {
97
- const newChunkIds = Array.from({ length: chunks.length }, (_, idx) => chunkId(id, idx));
98
94
  import('../embeddings/queue.js')
99
95
  .then(({ enqueueChunks, markStale }) => {
100
- if (oldChunkIds.length) markStale(config, oldChunkIds);
101
- enqueueChunks(config, newChunkIds);
96
+ if (wasExisting) markStale(config, [id]);
97
+ enqueueChunks(config, [id]);
102
98
  })
103
99
  .catch((err) => {
104
100
  if (typeof process !== 'undefined' && process.stderr) {
105
101
  process.stderr.write(`[mixgram] embeddings queue error: ${err?.message ?? err}\n`);
106
102
  }
107
103
  });
108
- if (oldChunkIds.length) {
109
- import('../embeddings/vectorStore.js')
110
- .then(({ removeChunks }) => removeChunks(config, oldChunkIds))
111
- .catch(() => {});
112
- }
113
104
  }
114
105
 
115
- return { id, chunkCount: chunks.length };
106
+ return { id };
116
107
  }
117
108
 
118
109
  function removeDocumentFromIndex(config, documentId) {
119
110
  const db = getDb(config);
120
- const chunkIds = db.prepare('SELECT id FROM document_chunks WHERE document_id = ?').all(documentId).map((r) => r.id);
121
- if (config?.embeddings?.enabled && chunkIds.length) {
111
+ if (config?.embeddings?.enabled) {
122
112
  import('../embeddings/queue.js')
123
- .then(({ markStale }) => markStale(config, chunkIds))
124
- .catch((err) => {
125
- if (typeof process !== 'undefined' && process.stderr) {
126
- process.stderr.write(`[mixgram] embeddings markStale error: ${err?.message ?? err}\n`);
127
- }
128
- });
113
+ .then(({ markStale }) => markStale(config, [documentId]))
114
+ .catch(() => {});
129
115
  import('../embeddings/vectorStore.js')
130
- .then(({ removeChunks }) => removeChunks(config, chunkIds))
116
+ .then(({ removeChunks }) => removeChunks(config, [documentId]))
131
117
  .catch(() => {});
132
118
  }
133
- for (const cid of chunkIds) {
134
- db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(cid);
135
- }
136
- const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
119
+ db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(documentId);
137
120
  db.transaction(() => {
121
+ const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
138
122
  for (const { rowid } of ftsRowids) {
139
- db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
123
+ db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
140
124
  }
141
- db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(documentId);
142
125
  db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
143
126
  db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
144
127
  })();