mixgram 1.1.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,78 +1,125 @@
1
1
  import { unified } from 'unified';
2
2
  import remarkParse from 'remark-parse';
3
3
  import { visit } from 'unist-util-visit';
4
- import { parse as parseFrontmatter } from '../../utils/markdown.js';
4
+ import { parse as parseMarkdownWithFrontmatter } from '../../utils/markdown.js';
5
5
 
6
- function stringifyNode(node) {
7
- if (node.value) return node.value;
8
- if (node.children) return node.children.map(stringifyNode).join('');
6
+ /**
7
+ * Recursively extract plain text from an mdast node (strips emphasis, links, etc.).
8
+ */
9
+ function extractTextFromNode(node) {
10
+ if (!node) return '';
11
+ if (node.type === 'text' || node.type === 'inlineCode') {
12
+ return node.value || '';
13
+ }
14
+ if (node.children && Array.isArray(node.children)) {
15
+ return node.children.map((child) => extractTextFromNode(child)).join('');
16
+ }
9
17
  return '';
10
18
  }
11
19
 
12
- function extractSections(mdBody) {
13
- const tree = unified().use(remarkParse).parse(mdBody);
14
- const sections = [];
15
- let currentHeading = null;
16
- let currentDepth = 0;
17
- let currentContent = [];
20
+ /**
21
+ * Build mdast from raw markdown and extract title, h1–h6, body, structure, sectionsIndex.
22
+ * One parse, one tree walk; no chunking. Compatible with the remark-parse indexing model.
23
+ *
24
+ * @param {string} markdown - Full markdown body (no frontmatter)
25
+ * @param {{ includeCodeBlocks?: boolean }} options
26
+ * @returns {{ title: string, h1: string, h2: string, h3: string, h4: string, h5: string, h6: string, body: string, structure: object[], sectionsIndex: object }}
27
+ */
28
+ export function extractMarkdownFields(markdown, { includeCodeBlocks = false } = {}) {
29
+ const tree = unified().use(remarkParse).parse(markdown);
18
30
 
19
- const flush = () => {
20
- const text = currentContent.join('\n').trim();
21
- if (text) {
22
- sections.push({ headingPath: currentHeading || '', headingLevel: currentDepth, content: text });
23
- }
24
- };
31
+ const h1 = [];
32
+ const h2 = [];
33
+ const h3 = [];
34
+ const h4 = [];
35
+ const h5 = [];
36
+ const h6 = [];
37
+ const bodyParts = [];
38
+ const structure = [];
39
+ const stack = [];
40
+ const sectionsIndex = {};
41
+ let currentSection = null;
42
+ let sectionIdCounter = 0;
43
+ let title = '';
25
44
 
26
45
  visit(tree, (node) => {
27
46
  if (node.type === 'heading') {
28
- flush();
29
- currentDepth = node.depth;
30
- const title = stringifyNode(node);
31
- currentHeading = currentHeading ? `${currentHeading} > ${title}` : title;
32
- currentContent = [title ? `#${'#'.repeat(node.depth - 1)} ${title}` : ''];
47
+ const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
48
+ const depth = node.depth;
49
+
50
+ switch (depth) {
51
+ case 1: h1.push(text); break;
52
+ case 2: h2.push(text); break;
53
+ case 3: h3.push(text); break;
54
+ case 4: h4.push(text); break;
55
+ case 5: h5.push(text); break;
56
+ case 6: h6.push(text); break;
57
+ default: break;
58
+ }
59
+ if (!title && depth === 1) title = text;
60
+
61
+ const section = {
62
+ id: `s${++sectionIdCounter}`,
63
+ depth,
64
+ heading: text,
65
+ parentId: null,
66
+ childrenIds: []
67
+ };
68
+
69
+ while (stack.length > 0 && stack[stack.length - 1].depth >= depth) {
70
+ stack.pop();
71
+ }
72
+ if (stack.length > 0) {
73
+ const parent = stack[stack.length - 1];
74
+ section.parentId = parent.id;
75
+ parent.childrenIds.push(section.id);
76
+ } else {
77
+ structure.push(section);
78
+ }
79
+ stack.push(section);
80
+ sectionsIndex[section.id] = section;
81
+ currentSection = section;
33
82
  return;
34
83
  }
35
- if (node.type === 'paragraph' || node.type === 'code' || node.type === 'blockquote' || node.type === 'list') {
36
- currentContent.push(stringifyNode(node));
84
+
85
+ if (node.type === 'paragraph') {
86
+ const text = (node.children || []).map((child) => extractTextFromNode(child)).join('').trim();
87
+ if (text) bodyParts.push(text);
88
+ return;
37
89
  }
38
- });
39
90
 
40
- flush();
91
+ if (includeCodeBlocks && node.type === 'code') {
92
+ const text = node.value?.trim();
93
+ if (text) bodyParts.push(text);
94
+ }
95
+ });
41
96
 
42
- if (sections.length === 0 && mdBody.trim()) {
43
- sections.push({ headingPath: '', headingLevel: 0, content: mdBody.trim() });
44
- }
45
- return sections;
46
- }
97
+ const body = bodyParts.join('\n\n');
47
98
 
48
- function chunkSections(sections, chunkSize = 1200, chunkOverlap = 120) {
49
- const chunks = [];
50
- for (const sec of sections) {
51
- const { headingPath, headingLevel, content } = sec;
52
- if (content.length <= chunkSize) {
53
- chunks.push({ headingPath, headingLevel, content });
54
- continue;
55
- }
56
- let start = 0;
57
- while (start < content.length) {
58
- let end = Math.min(start + chunkSize, content.length);
59
- if (end < content.length) {
60
- const lastSpace = content.lastIndexOf(' ', end);
61
- if (lastSpace > start) end = lastSpace;
62
- }
63
- chunks.push({ headingPath, headingLevel, content: content.slice(start, end).trim() });
64
- start = end - (end - start < content.length ? chunkOverlap : 0);
65
- }
66
- }
67
- return chunks;
99
+ return {
100
+ title: title || (h1[0] ?? ''),
101
+ h1: h1.join('\n'),
102
+ h2: h2.join('\n'),
103
+ h3: h3.join('\n'),
104
+ h4: h4.join('\n'),
105
+ h5: h5.join('\n'),
106
+ h6: h6.join('\n'),
107
+ body,
108
+ structure,
109
+ sectionsIndex
110
+ };
68
111
  }
69
112
 
70
- function parseAndChunk(rawContent, options = {}) {
71
- const { chunkSize = 1200, chunkOverlap = 120 } = options;
72
- const { frontmatter, body } = parseFrontmatter(rawContent);
73
- const sections = extractSections(body);
74
- const chunks = chunkSections(sections, chunkSize, chunkOverlap);
75
- return { frontmatter, body, chunks };
113
+ /**
114
+ * Parse raw content: frontmatter (gray-matter) + AST-based extraction of body fields.
115
+ * Returns one object per document (no chunking).
116
+ *
117
+ * @param {string} rawContent - Full file content including optional frontmatter
118
+ * @param {{ includeCodeBlocks?: boolean }} options
119
+ */
120
+ export function parseMarkdown(rawContent, options = {}) {
121
+ const { frontmatter, body: rawBody } = parseMarkdownWithFrontmatter(rawContent);
122
+ const fields = extractMarkdownFields(rawBody, options);
123
+ const body = (fields.body && fields.body.trim()) ? fields.body : (rawBody || '').trim();
124
+ return { frontmatter, ...fields, body };
76
125
  }
77
-
78
- export { extractSections, chunkSections, parseAndChunk };
@@ -28,11 +28,10 @@ function listMarkdownPaths(config) {
28
28
  function clearIndex(config) {
29
29
  const db = getDb(config);
30
30
  db.transaction(() => {
31
- const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts').all();
31
+ const ftsRowids = db.prepare('SELECT rowid FROM document_fts').all();
32
32
  for (const { rowid } of ftsRowids) {
33
- db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
33
+ db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
34
34
  }
35
- db.prepare('DELETE FROM document_chunks').run();
36
35
  db.prepare('DELETE FROM document_tags').run();
37
36
  db.prepare('DELETE FROM documents').run();
38
37
  })();
@@ -60,22 +59,32 @@ function fullReindex(config) {
60
59
  }
61
60
 
62
61
  /**
63
- * Incremental reindex: index all found .md files, then remove from index any document
64
- * whose path is no longer present on disk (manual delete). Does not clear the index first.
62
+ * Incremental reindex: scan all .md files; skip unchanged (same path + file_mtime_ms);
63
+ * index new or changed; remove from index documents whose path is no longer on disk.
64
+ * Does not clear the index first.
65
+ * @returns {{ scanned: number, skipped: number, indexed: number, removed: number, totalPaths: number }}
65
66
  */
66
67
  function incrementalReindex(config) {
67
68
  const paths = listMarkdownPaths(config);
68
69
  const pathsSet = new Set(paths.map((p) => path.normalize(p)));
70
+ const db = getDb(config);
71
+ const getStored = db.prepare('SELECT id, file_mtime_ms FROM documents WHERE path = ?');
69
72
  let indexed = 0;
73
+ let skipped = 0;
70
74
  for (const filePath of paths) {
71
75
  try {
72
- const raw = fs.readFileSync(filePath, 'utf8');
76
+ const norm = path.normalize(filePath);
73
77
  const stat = fs.statSync(filePath);
78
+ const row = getStored.get(norm);
79
+ if (row != null && row.file_mtime_ms != null && row.file_mtime_ms === stat.mtimeMs) {
80
+ skipped++;
81
+ continue;
82
+ }
83
+ const raw = fs.readFileSync(filePath, 'utf8');
74
84
  indexDocument(config, filePath, raw, stat.mtimeMs);
75
85
  indexed++;
76
86
  } catch (_) {}
77
87
  }
78
- const db = getDb(config);
79
88
  const allDocs = db.prepare('SELECT id, path FROM documents').all();
80
89
  let removed = 0;
81
90
  for (const row of allDocs) {
@@ -85,7 +94,7 @@ function incrementalReindex(config) {
85
94
  removed++;
86
95
  }
87
96
  }
88
- return { indexed, removed, totalPaths: paths.length };
97
+ return { scanned: paths.length, skipped, indexed, removed, totalPaths: paths.length };
89
98
  }
90
99
 
91
100
  /**
@@ -1,12 +1,16 @@
1
1
  import { getDb } from '../../db/sqlite.js';
2
2
  import { normalizeForFts } from '../indexing/indexer.js';
3
+
3
4
  function getBm25Weights(config) {
4
5
  const w = config.indexing?.ftsWeights || {};
5
6
  return [
6
7
  w.title ?? 10,
7
- w.topicKey ?? 8,
8
- 1, 1, 1,
9
- w.heading ?? 5,
8
+ w.h1 ?? 8,
9
+ w.h2 ?? 6,
10
+ w.h3 ?? 5,
11
+ w.h4 ?? 4,
12
+ w.h5 ?? 3,
13
+ w.h6 ?? 2,
10
14
  w.body ?? 1
11
15
  ];
12
16
  }
@@ -17,7 +21,7 @@ function getBm25Weights(config) {
17
21
  */
18
22
  function normalizeQuery(q) {
19
23
  if (q == null || typeof q !== 'string') return '';
20
- return q.trim();
24
+ return normalizeForFts(q.trim());
21
25
  }
22
26
 
23
27
  /**
@@ -27,7 +31,7 @@ function normalizeQuery(q) {
27
31
  * @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
28
32
  * @param {string} [options.project] - project name (required for project-only and merged)
29
33
  * @param {number} [options.limit] - max results
30
- * @returns {Array<{ documentId: string, chunkId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, headingPath: string, snippet: string, score: number }>}
34
+ * @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, snippet: string, score: number }>}
31
35
  */
32
36
  function search(config, options = {}) {
33
37
  const {
@@ -57,33 +61,28 @@ function search(config, options = {}) {
57
61
  scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
58
62
  params.push('home');
59
63
  } else {
60
- // merged: project or home; when project given, prefer project
61
64
  scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
62
65
  params.push('home', 'project', project || '');
63
66
  }
64
67
 
65
- params.push(project || ''); // ORDER BY: project first in merged, else no match
66
- const ftsLimit = config.embeddings?.enabled ? Math.max(limit * 3, 50) : limit;
67
- params.push(ftsLimit);
68
+ params.push(project || '');
69
+ params.push(limit);
68
70
 
69
71
  const sql = `
70
72
  SELECT
71
- f.chunk_id AS chunkId,
72
73
  f.document_id AS documentId,
73
74
  d.title AS title,
74
75
  d.topic_key AS topicKey,
75
76
  d.type AS type,
76
77
  d.scope AS scope,
77
78
  d.project AS project,
78
- c.heading_path AS headingPath,
79
- c.content AS content,
80
- snippet(document_chunks_fts, 0, '<b>', '</b>', '...', 24) AS snippetTitle,
81
- snippet(document_chunks_fts, 6, '<b>', '</b>', '...', 32) AS snippetBody,
82
- bm25(document_chunks_fts, ${bm25Args}) AS rank
83
- FROM document_chunks_fts f
79
+ d.body AS body,
80
+ snippet(document_fts, 1, '<b>', '</b>', '...', 24) AS snippetTitle,
81
+ snippet(document_fts, 8, '<b>', '</b>', '...', 64) AS snippetBody,
82
+ bm25(document_fts, ${bm25Args}) AS rank
83
+ FROM document_fts f
84
84
  JOIN documents d ON d.id = f.document_id
85
- JOIN document_chunks c ON c.id = f.chunk_id
86
- WHERE document_chunks_fts MATCH ? ${scopeCondition}
85
+ WHERE document_fts MATCH ? ${scopeCondition}
87
86
  ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
88
87
  LIMIT ?
89
88
  `;
@@ -91,97 +90,21 @@ function search(config, options = {}) {
91
90
  const stmt = db.prepare(sql);
92
91
  const rows = stmt.all(...params);
93
92
 
94
- const mapped = rows.map((r) => {
95
- const snippet = [r.snippetTitle, r.snippetBody].filter(Boolean).join(' … ') || r.content?.slice(0, 200) || '';
93
+ return rows.map((r) => {
94
+ const fromFts = [r.snippetTitle, r.snippetBody].filter((x) => x != null && x !== '').join(' … ');
95
+ const fallback = r.body != null ? String(r.body).slice(0, 200) : '';
96
+ const snippet = (fromFts && fromFts.trim()) || fallback || '';
96
97
  return {
97
98
  documentId: r.documentId,
98
- chunkId: r.chunkId,
99
99
  title: r.title,
100
100
  topicKey: r.topicKey,
101
101
  type: r.type,
102
102
  scope: r.scope,
103
103
  project: r.project,
104
- headingPath: r.headingPath,
105
- snippet: snippet.trim(),
106
- score: -Number(r.rank), // bm25 returns negative (lower = better), we expose positive
107
- _rank: r.rank
104
+ snippet: String(snippet).trim(),
105
+ score: -Number(r.rank)
108
106
  };
109
107
  });
110
-
111
- if (!config.embeddings?.enabled || mapped.length === 0) {
112
- return mapped.slice(0, limit).map(({ _rank, ...r }) => r);
113
- }
114
-
115
- return hybridSearch(config, { query, scopeMode, project, limit, ftsRows: mapped });
116
- }
117
-
118
- async function hybridSearch(config, { query, scopeMode, project, limit, ftsRows }) {
119
- const ftsWeight = config.search?.ftsWeight ?? 0.7;
120
- const semanticWeight = config.search?.semanticWeight ?? 0.3;
121
- let embedder;
122
- let vectorStore;
123
- try {
124
- const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
125
- import('../embeddings/embedder.js'),
126
- import('../embeddings/vectorStore.js')
127
- ]);
128
- embedder = await getEmbedder(config);
129
- vectorStore = await getVectorStore(config);
130
- } catch {
131
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
132
- }
133
- if (!embedder || !vectorStore) {
134
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
135
- }
136
- let semanticHits;
137
- try {
138
- const queryEmbedding = await embedder.embed(query);
139
- semanticHits = await vectorStore.search(config, queryEmbedding, Math.max(limit * 2, 30));
140
- } catch {
141
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
142
- }
143
- const byChunk = new Map(ftsRows.map((r) => [r.chunkId, { ...r }]));
144
- const ftsScores = ftsRows.map((r) => -Number(r._rank));
145
- const minFts = Math.min(...ftsScores);
146
- const maxFts = Math.max(...ftsScores);
147
- const rangeFts = maxFts - minFts || 1;
148
- for (const row of ftsRows) {
149
- const normFts = (minFts === maxFts) ? 1 : ((-Number(row._rank)) - minFts) / rangeFts;
150
- const sem = semanticHits.find((h) => h.chunk_id === row.chunkId);
151
- const similarity = sem ? 1 - sem.distance : 0;
152
- const combined = ftsWeight * normFts + semanticWeight * similarity;
153
- byChunk.get(row.chunkId).score = combined;
154
- }
155
- for (const sem of semanticHits) {
156
- if (!byChunk.has(sem.chunk_id)) {
157
- const db = getDb(config);
158
- const r = db.prepare(`
159
- SELECT c.id AS chunkId, c.document_id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
160
- FROM document_chunks c
161
- JOIN documents d ON d.id = c.document_id
162
- WHERE c.id = ?
163
- `).get(sem.chunk_id);
164
- if (r) {
165
- const similarity = 1 - sem.distance;
166
- byChunk.set(sem.chunk_id, {
167
- documentId: r.documentId,
168
- chunkId: r.chunkId,
169
- title: r.title,
170
- topicKey: r.topicKey,
171
- type: r.type,
172
- scope: r.scope,
173
- project: r.project,
174
- headingPath: r.headingPath,
175
- snippet: (r.content || '').slice(0, 200).trim(),
176
- score: semanticWeight * similarity
177
- });
178
- }
179
- }
180
- }
181
- return Array.from(byChunk.values())
182
- .sort((a, b) => b.score - a.score)
183
- .slice(0, limit)
184
- .map(({ _rank, ...r }) => r);
185
108
  }
186
109
 
187
110
  /**
@@ -191,9 +114,8 @@ function getRecentContext(config, options = {}) {
191
114
  const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
192
115
  const db = getDb(config);
193
116
  let sql = `
194
- SELECT c.id AS chunkId, d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
195
- FROM document_chunks c
196
- JOIN documents d ON d.id = c.document_id
117
+ SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.body
118
+ FROM documents d
197
119
  WHERE d.deleted_at IS NULL
198
120
  `;
199
121
  const params = [];
@@ -209,14 +131,12 @@ function getRecentContext(config, options = {}) {
209
131
  const rows = db.prepare(sql).all(...params);
210
132
  return rows.map((r) => ({
211
133
  documentId: r.documentId,
212
- chunkId: r.chunkId,
213
134
  title: r.title,
214
135
  topicKey: r.topicKey,
215
136
  type: r.type,
216
137
  scope: r.scope,
217
138
  project: r.project,
218
- headingPath: r.headingPath,
219
- snippet: (r.content || '').slice(0, 300).trim()
139
+ snippet: (r.body || '').slice(0, 300).trim()
220
140
  }));
221
141
  }
222
142
 
@@ -1,4 +1,4 @@
1
- -- documents
1
+ -- documents (one row per doc; body and section tree stored here)
2
2
  CREATE TABLE IF NOT EXISTS documents (
3
3
  id TEXT PRIMARY KEY,
4
4
  path TEXT NOT NULL UNIQUE,
@@ -17,21 +17,10 @@ CREATE TABLE IF NOT EXISTS documents (
17
17
  deleted_at TEXT,
18
18
  revision_count INTEGER NOT NULL DEFAULT 1,
19
19
  duplicate_count INTEGER NOT NULL DEFAULT 0,
20
- embedding_status TEXT NOT NULL DEFAULT 'disabled'
21
- );
22
-
23
- CREATE TABLE IF NOT EXISTS document_chunks (
24
- id TEXT PRIMARY KEY,
25
- document_id TEXT NOT NULL,
26
- chunk_index INTEGER NOT NULL,
27
- heading_path TEXT,
28
- heading_level INTEGER,
29
- content TEXT NOT NULL,
30
- token_count INTEGER,
31
- content_hash TEXT NOT NULL,
32
- created_at TEXT NOT NULL,
33
- updated_at TEXT NOT NULL,
34
- FOREIGN KEY(document_id) REFERENCES documents(id)
20
+ embedding_status TEXT NOT NULL DEFAULT 'disabled',
21
+ body TEXT,
22
+ structure TEXT,
23
+ sections_index TEXT
35
24
  );
36
25
 
37
26
  CREATE TABLE IF NOT EXISTS document_tags (
@@ -67,20 +56,19 @@ CREATE TABLE IF NOT EXISTS embedding_jobs (
67
56
  attempts INTEGER NOT NULL DEFAULT 0,
68
57
  last_error TEXT,
69
58
  created_at TEXT NOT NULL,
70
- updated_at TEXT NOT NULL,
71
- FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
59
+ updated_at TEXT NOT NULL
72
60
  );
73
61
 
74
- -- FTS5: content must be not null for columns; use empty string for missing
75
- CREATE VIRTUAL TABLE IF NOT EXISTS document_chunks_fts USING fts5(
76
- chunk_id UNINDEXED,
62
+ -- FTS5: one row per document (title, h1–h6, body)
63
+ CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
77
64
  document_id UNINDEXED,
78
65
  title,
79
- topic_key,
80
- type,
81
- scope,
82
- project,
83
- heading_path,
66
+ h1,
67
+ h2,
68
+ h3,
69
+ h4,
70
+ h5,
71
+ h6,
84
72
  body,
85
73
  tokenize = 'unicode61 remove_diacritics 2'
86
74
  );
package/src/mcp/server.js CHANGED
@@ -1,9 +1,49 @@
1
+ import { fork } from 'child_process';
2
+ import fs from 'fs';
3
+ import path from 'path';
4
+ import os from 'os';
5
+ import { randomUUID } from 'crypto';
6
+ import { fileURLToPath } from 'url';
1
7
  import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
8
  import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
3
9
  import { loadConfig } from '../config.js';
4
10
  import { createToolHandlers } from './tools.js';
5
11
  import { getToolDefinitions } from './tool-registry.js';
6
12
 
13
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
14
+ let embeddingWorkerChild = null;
15
+ let embeddingWorkerConfigPath = null;
16
+
17
+ const EMBED_TIMEOUT_MS = 60000;
18
+
19
+ function makeQueryEmbedder(child) {
20
+ return function getQueryEmbedding(text) {
21
+ if (!child?.connected) return Promise.resolve(null);
22
+ const id = randomUUID();
23
+ return new Promise((resolve, reject) => {
24
+ const timeout = setTimeout(() => {
25
+ cleanup();
26
+ reject(new Error('embedding timeout'));
27
+ }, EMBED_TIMEOUT_MS);
28
+ const onMessage = (msg) => {
29
+ if (msg?.type !== 'embedResult' || msg.id !== id) return;
30
+ cleanup();
31
+ if (msg.err) {
32
+ reject(new Error(msg.err));
33
+ return;
34
+ }
35
+ resolve(msg.vector ? new Float32Array(msg.vector) : null);
36
+ };
37
+ const cleanup = () => {
38
+ clearTimeout(timeout);
39
+ child.off('message', onMessage);
40
+ };
41
+ child.on('message', onMessage);
42
+ child.send({ type: 'embed', id, text });
43
+ });
44
+ };
45
+ }
46
+
7
47
  function createServer(configOverrides = {}, baseDir = null, projectBaseDir = null) {
8
48
  const config = loadConfig(configOverrides, baseDir, projectBaseDir);
9
49
  const mcpServer = new McpServer(
@@ -59,8 +99,33 @@ async function run(configOverrides = {}, baseDir = null, projectBaseDir = null)
59
99
  }
60
100
  if (config.embeddings?.enabled) {
61
101
  try {
62
- const { startWorker } = await import('../core/embeddings/worker.js');
63
- startWorker(config);
102
+ embeddingWorkerConfigPath = path.join(
103
+ os.tmpdir(),
104
+ `mixgram-worker-config-${process.pid}-${Date.now()}.json`
105
+ );
106
+ fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
107
+ const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
108
+ embeddingWorkerChild = fork(workerPath, [], {
109
+ stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
110
+ env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
111
+ });
112
+ config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
113
+ embeddingWorkerChild.on('exit', (code, signal) => {
114
+ if (code !== 0 && code != null && process.stderr) {
115
+ process.stderr.write(`[mixgram] embeddings worker exited: code=${code} signal=${signal}\n`);
116
+ }
117
+ config.getQueryEmbedding = null;
118
+ if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
119
+ try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
120
+ embeddingWorkerConfigPath = null;
121
+ }
122
+ });
123
+ process.on('exit', () => {
124
+ if (embeddingWorkerChild) embeddingWorkerChild.kill();
125
+ if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
126
+ try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
127
+ }
128
+ });
64
129
  } catch (err) {
65
130
  if (typeof process !== 'undefined' && process.stderr) {
66
131
  process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
@@ -6,6 +6,29 @@ export const FRONTMATTER_KEYS = [
6
6
  'embedding_status', 'indexed_at', 'deleted_at'
7
7
  ];
8
8
 
9
+ /** Keys written to .md frontmatter (needed for re-index, scope/project filtering, timeline). */
10
+ const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'scope', 'project', 'topic_key', 'session_id', 'created'];
11
+
12
+ function formatCreated(createdAt) {
13
+ if (!createdAt) return null;
14
+ const d = new Date(createdAt);
15
+ if (Number.isNaN(d.getTime())) return createdAt;
16
+ const y = d.getFullYear();
17
+ const m = String(d.getMonth() + 1).padStart(2, '0');
18
+ const day = String(d.getDate()).padStart(2, '0');
19
+ const h = String(d.getHours()).padStart(2, '0');
20
+ const min = String(d.getMinutes()).padStart(2, '0');
21
+ const s = String(d.getSeconds()).padStart(2, '0');
22
+ return `${y}-${m}-${day} ${h}:${min}:${s}`;
23
+ }
24
+
25
+ function parseCreated(value) {
26
+ if (!value) return null;
27
+ if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value)) return value;
28
+ const d = new Date(value);
29
+ return Number.isNaN(d.getTime()) ? null : d.toISOString();
30
+ }
31
+
9
32
  function serializeFrontmatter(fm) {
10
33
  const obj = {};
11
34
  for (const k of FRONTMATTER_KEYS) {
@@ -16,9 +39,26 @@ function serializeFrontmatter(fm) {
16
39
  return obj;
17
40
  }
18
41
 
42
+ /** Minimal frontmatter for .md (id required for re-index; created in friendly format). */
43
+ function serializeDisplayFrontmatter(fm) {
44
+ const obj = {};
45
+ for (const k of DISPLAY_FRONTMATTER_KEYS) {
46
+ if (k === 'created') {
47
+ const v = fm.created_at != null ? formatCreated(fm.created_at) : (fm.created ?? null);
48
+ if (v) obj[k] = v;
49
+ } else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
50
+ obj[k] = fm[k];
51
+ }
52
+ }
53
+ if (fm.deleted_at) obj.deleted_at = fm.deleted_at;
54
+ if (fm.deleted === true) obj.deleted = true;
55
+ return obj;
56
+ }
57
+
19
58
  export function parse(content) {
20
59
  const parsed = matter(content);
21
60
  const data = parsed.data || {};
61
+ const createdAt = data.created_at ?? parseCreated(data.created);
22
62
  return {
23
63
  frontmatter: {
24
64
  id: data.id,
@@ -29,7 +69,7 @@ export function parse(content) {
29
69
  topic_key: data.topic_key,
30
70
  session_id: data.session_id,
31
71
  tool_name: data.tool_name,
32
- created_at: data.created_at,
72
+ created_at: createdAt,
33
73
  updated_at: data.updated_at,
34
74
  revision_count: data.revision_count ?? 1,
35
75
  duplicate_count: data.duplicate_count ?? 0,
@@ -44,6 +84,6 @@ export function parse(content) {
44
84
  }
45
85
 
46
86
  export function toMarkdown(frontmatter, body) {
47
- const fm = serializeFrontmatter(frontmatter);
87
+ const fm = serializeDisplayFrontmatter(frontmatter);
48
88
  return matter.stringify(body || '', fm, { lineWidth: -1 });
49
89
  }