mixgram 1.1.6 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/bin/mixgram.js CHANGED
@@ -7,6 +7,7 @@
7
7
  */
8
8
  import { run } from '../src/mcp/server.js';
9
9
  import { loadConfig } from '../src/config.js';
10
+ import { closeDb } from '../src/db/sqlite.js';
10
11
  import { createToolHandlers } from '../src/mcp/tools.js';
11
12
  import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../src/mcp/cli-adapter.js';
12
13
  import fs from 'fs';
@@ -217,6 +218,7 @@ Usage: mixgram <command> [options]
217
218
  Commands:
218
219
  mcp [options] Run the MCP server (stdio). Use this in your client config.
219
220
  setup <client> Register Mixgram with an MCP client.
221
+ flushdb Delete the SQLite database file (and WAL/shm).
220
222
  help [tool] Show help; with optional tool name, show options for that tool.
221
223
  -v, --version Print version from package and exit.
222
224
  <tool> [options] Run an MCP tool by name. Tools:
@@ -263,6 +265,26 @@ async function main() {
263
265
  return;
264
266
  }
265
267
 
268
+ if (SUBCOMMAND === 'flushdb') {
269
+ const { overrides, baseDir, projectBaseDir } = loadCliConfig();
270
+ const config = loadConfig(overrides, baseDir, projectBaseDir);
271
+ const dbPath = config.sqlitePath;
272
+ closeDb();
273
+ const removed = [];
274
+ for (const p of [dbPath, dbPath + '-wal', dbPath + '-shm']) {
275
+ if (fs.existsSync(p)) {
276
+ fs.unlinkSync(p);
277
+ removed.push(p);
278
+ }
279
+ }
280
+ if (removed.length) {
281
+ console.log('Removed:', removed.join(', '));
282
+ } else {
283
+ console.log('No database file at', dbPath);
284
+ }
285
+ return;
286
+ }
287
+
266
288
  if (SUBCOMMAND === 'setup') {
267
289
  if (!ARG) {
268
290
  console.error('Usage: mixgram setup <cursor|gemini-cli|codex>');
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mixgram",
3
- "version": "1.1.6",
3
+ "version": "1.2.2",
4
4
  "description": "MCP memory server with Markdown source of truth and SQLite FTS5.",
5
5
  "keywords": [
6
6
  "mcp",
@@ -33,19 +33,32 @@
33
33
  "scripts": {
34
34
  "start": "node index.js",
35
35
  "test": "node spec/run-tests.js",
36
- "test:visual": "node spec/run-tests.js --visual"
36
+ "test:visual": "node spec/run-tests.js --visual",
37
+ "test:large": "node spec/test-large-document.js",
38
+ "bench:indexing": "node spec/bench-indexing.js",
39
+ "bench:embed": "node spec/bench-indexing.js --embed",
40
+ "bench:search": "node spec/bench-search.js",
41
+ "bench:search:embed": "node spec/bench-search.js --embed",
42
+ "bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
43
+ "duplication": "jscpd src",
44
+ "deadcode": "knip",
45
+ "quality": "npm run duplication && npm run deadcode"
37
46
  },
38
47
  "dependencies": {
39
48
  "@huggingface/transformers": "^3.4.2",
40
49
  "@modelcontextprotocol/sdk": "^1.0.0",
41
50
  "better-sqlite3": "^12.6.2",
42
51
  "chokidar": "^4.0.1",
52
+ "glob": "^13.0.6",
43
53
  "gray-matter": "^4.0.3",
44
- "glob": "^13.0.0",
45
54
  "remark-parse": "^11.0.0",
46
55
  "sqlite-vec": "^0.1.6",
47
56
  "unified": "^11.0.5",
48
57
  "unist-util-visit": "^5.0.0",
49
58
  "zod": "^3.23.0"
59
+ },
60
+ "devDependencies": {
61
+ "jscpd": "^4.0.8",
62
+ "knip": "^5.86.0"
50
63
  }
51
64
  }
@@ -0,0 +1,219 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Benchmark: indexing and reindexing with synthetic corpus.
4
+ * Breaks down timings (clear, listPaths, indexing) and compares with/without embeddings.
5
+ * Opt-in; not run by npm test.
6
+ *
7
+ * Usage:
8
+ * npm run bench:indexing # indexing only
9
+ * npm run bench:embed # indexing + embedding queue processing (slow)
10
+ */
11
+ import fs from 'fs';
12
+ import path from 'path';
13
+ import os from 'os';
14
+ import { loadConfig } from '../src/config.js';
15
+ import {
16
+ fullReindex,
17
+ incrementalReindex,
18
+ clearIndex,
19
+ listMarkdownPaths
20
+ } from '../src/core/indexing/reindex.js';
21
+ import { indexDocument } from '../src/core/indexing/indexer.js';
22
+ import { processNextJob } from '../src/core/embeddings/queue.js';
23
+ import { closeDb, getDb } from '../src/db/sqlite.js';
24
+ import { toMarkdown } from '../src/utils/markdown.js';
25
+
26
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-'));
27
+ const baseConfig = loadConfig({
28
+ homeMemoryRoot: path.join(tmpDir, 'home'),
29
+ projectMemoryRoot: path.join(tmpDir, 'docs'),
30
+ sqlitePath: path.join(tmpDir, 'index.db')
31
+ });
32
+ fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
33
+ fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
34
+
35
+ const configNoEmb = {
36
+ ...baseConfig,
37
+ embeddings: { ...baseConfig.embeddings, enabled: false }
38
+ };
39
+ const configEmb = {
40
+ ...baseConfig,
41
+ embeddings: { ...baseConfig.embeddings, enabled: true }
42
+ };
43
+
44
+ /** Frontmatter with same keys the app writes so the indexer parses the same. */
45
+ function writeDoc(dir, id, title, body) {
46
+ const now = new Date().toISOString();
47
+ const frontmatter = {
48
+ id,
49
+ title,
50
+ type: 'generated_note',
51
+ scope: 'project',
52
+ project: 'bench',
53
+ topic_key: `bench/${id}`,
54
+ created_at: now
55
+ };
56
+ const raw = toMarkdown(frontmatter, body);
57
+ const filePath = path.join(dir, 'generated', `${id}.md`);
58
+ fs.writeFileSync(filePath, raw, 'utf8');
59
+ return filePath;
60
+ }
61
+
62
+ function buildLargeBody(size) {
63
+ const intro = '# Big\n\nNeedleInHaystack\n\n';
64
+ const filler = 'x'.repeat(Math.max(0, size - intro.length));
65
+ return intro + filler;
66
+ }
67
+
68
+ const NUM_SMALL = 3;
69
+ const LARGE_SIZE = 8_000;
70
+ const RUN_EMBED_PROCESSING = process.argv.includes('--embed');
71
+
72
+ console.log('Corpus: %d small .md + 1 large .md (~%d chars)', NUM_SMALL, LARGE_SIZE);
73
+ console.log('Mode: %s\n', RUN_EMBED_PROCESSING ? 'indexing + embedding queue processing' : 'indexing only');
74
+
75
+ // --- Create corpus ---
76
+ for (let i = 0; i < NUM_SMALL; i++) {
77
+ writeDoc(baseConfig.projectMemoryRoot, `obs_small_${i}`, `Small ${i}`, `Content for small doc ${i}.\n`);
78
+ }
79
+ const largePath = writeDoc(baseConfig.projectMemoryRoot, 'obs_large', 'Large doc', buildLargeBody(LARGE_SIZE));
80
+
81
+ /**
82
+ * Full reindex with breakdown: clearIndex, listMarkdownPaths, index loop.
83
+ * @returns {{ total: number, clear: number, list: number, indexLoop: number, indexed: number }}
84
+ */
85
+ function fullReindexWithBreakdown(config) {
86
+ let t0, t1;
87
+ t0 = performance.now();
88
+ clearIndex(config);
89
+ t1 = performance.now();
90
+ const clearMs = t1 - t0;
91
+
92
+ t0 = performance.now();
93
+ const paths = listMarkdownPaths(config);
94
+ t1 = performance.now();
95
+ const listMs = t1 - t0;
96
+
97
+ let indexed = 0;
98
+ t0 = performance.now();
99
+ for (const filePath of paths) {
100
+ try {
101
+ const raw = fs.readFileSync(filePath, 'utf8');
102
+ const stat = fs.statSync(filePath);
103
+ indexDocument(config, filePath, raw, stat.mtimeMs);
104
+ indexed++;
105
+ } catch (_) {}
106
+ }
107
+ t1 = performance.now();
108
+ const indexLoopMs = t1 - t0;
109
+ const totalMs = clearMs + listMs + indexLoopMs;
110
+
111
+ return { total: totalMs, clear: clearMs, list: listMs, indexLoop: indexLoopMs, indexed };
112
+ }
113
+
114
+ function ms(n) {
115
+ return Math.round(n);
116
+ }
117
+
118
+ function section(title) {
119
+ console.log('\n--- %s ---', title);
120
+ }
121
+
122
+ async function runBenchmark() {
123
+ // ========== Without embeddings ==========
124
+ section('Without embeddings');
125
+
126
+ let b = fullReindexWithBreakdown(configNoEmb);
127
+ console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
128
+ ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
129
+
130
+ let t0 = performance.now();
131
+ const inc0 = incrementalReindex(configNoEmb);
132
+ let t1 = performance.now();
133
+ console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
134
+ ms(t1 - t0), inc0.scanned, inc0.skipped, inc0.indexed, inc0.removed);
135
+
136
+ fs.appendFileSync(largePath, '\n\nAppended line.');
137
+
138
+ t0 = performance.now();
139
+ const inc1 = incrementalReindex(configNoEmb);
140
+ t1 = performance.now();
141
+ console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
142
+ ms(t1 - t0), inc1.scanned, inc1.skipped, inc1.indexed, inc1.removed);
143
+
144
+ // Final state without embeddings
145
+ const db = getDb(baseConfig);
146
+ const docsCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
147
+ const ftsCount = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
148
+ console.log('State: %d documents, %d FTS rows.', docsCount, ftsCount);
149
+
150
+ // ========== With embeddings (indexing + queue) ==========
151
+ section('With embeddings (indexing + queue)');
152
+
153
+ clearIndex(configEmb);
154
+ b = fullReindexWithBreakdown(configEmb);
155
+ // Allow enqueue promises to settle
156
+ await new Promise((r) => setImmediate(r));
157
+ await new Promise((r) => setImmediate(r));
158
+
159
+ const jobsEnqueued = db.prepare('SELECT COUNT(*) AS c FROM embedding_jobs').get()?.c ?? 0;
160
+ console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
161
+ ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
162
+ console.log('Jobs in queue: %d', jobsEnqueued);
163
+
164
+ t0 = performance.now();
165
+ const inc0emb = incrementalReindex(configEmb);
166
+ t1 = performance.now();
167
+ console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
168
+ ms(t1 - t0), inc0emb.scanned, inc0emb.skipped, inc0emb.indexed, inc0emb.removed);
169
+
170
+ fs.appendFileSync(largePath, '\n\nAnother line.');
171
+
172
+ t0 = performance.now();
173
+ const inc1emb = incrementalReindex(configEmb);
174
+ t1 = performance.now();
175
+ console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
176
+ ms(t1 - t0), inc1emb.scanned, inc1emb.skipped, inc1emb.indexed, inc1emb.removed);
177
+
178
+ // ========== Embedding processing (optional) ==========
179
+ if (RUN_EMBED_PROCESSING && configEmb.embeddings?.enabled) {
180
+ section('Embedding processing (queue → model → vector store)');
181
+
182
+ const pendingBefore = db.prepare("SELECT COUNT(*) AS c FROM embedding_jobs WHERE status = 'pending'").get()?.c ?? 0;
183
+ t0 = performance.now();
184
+ let processed = 0;
185
+ while (await processNextJob(configEmb)) {
186
+ processed++;
187
+ }
188
+ t1 = performance.now();
189
+ console.log('Process queue: %d ms (jobs processed=%d, pending before=%d)',
190
+ ms(t1 - t0), processed, pendingBefore);
191
+ }
192
+
193
+ // Final summary
194
+ section('Summary');
195
+ const docsFinal = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
196
+ const ftsFinal = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
197
+ const jobsFinal = db.prepare('SELECT status, COUNT(*) AS c FROM embedding_jobs GROUP BY status').all();
198
+ console.log('Documents: %d, FTS rows: %d', docsFinal, ftsFinal);
199
+ if (jobsFinal.length) {
200
+ console.log('Embedding jobs: %s', jobsFinal.map((r) => `${r.status}=${r.c}`).join(', '));
201
+ }
202
+ }
203
+
204
+ runBenchmark()
205
+ .then(() => {
206
+ closeDb();
207
+ try {
208
+ fs.rmSync(tmpDir, { recursive: true, force: true });
209
+ } catch (_) {}
210
+ console.log('\nDone.\n');
211
+ })
212
+ .catch((err) => {
213
+ closeDb();
214
+ try {
215
+ fs.rmSync(tmpDir, { recursive: true, force: true });
216
+ } catch (_) {}
217
+ console.error(err);
218
+ process.exit(1);
219
+ });
@@ -0,0 +1,221 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Benchmark: search (FTS and optional semantic) over synthetic corpus.
4
+ * Compares short vs long documents, FTS-only vs with embeddings.
5
+ * Opt-in; not run by npm test.
6
+ *
7
+ * Usage:
8
+ * npm run bench:search # FTS only
9
+ * npm run bench:search:embed # FTS + semantic search (needs embed model)
10
+ */
11
+ import fs from 'fs';
12
+ import path from 'path';
13
+ import os from 'os';
14
+ import { loadConfig } from '../src/config.js';
15
+ import { fullReindex, clearIndex, listMarkdownPaths } from '../src/core/indexing/reindex.js';
16
+ import { indexDocument } from '../src/core/indexing/indexer.js';
17
+ import { processNextJob } from '../src/core/embeddings/queue.js';
18
+ import { search as searchFts, getRecentContext } from '../src/core/search/search.js';
19
+ import { closeDb, getDb } from '../src/db/sqlite.js';
20
+ import { toMarkdown } from '../src/utils/markdown.js';
21
+
22
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-search-'));
23
+ const baseConfig = loadConfig({
24
+ homeMemoryRoot: path.join(tmpDir, 'home'),
25
+ projectMemoryRoot: path.join(tmpDir, 'docs'),
26
+ sqlitePath: path.join(tmpDir, 'index.db')
27
+ });
28
+ fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
29
+ fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
30
+
31
+ const configNoEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: false } };
32
+ const configEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: true } };
33
+
34
+ const RUN_SEMANTIC = process.argv.includes('--embed');
35
+
36
+ function writeDoc(dir, id, title, body) {
37
+ const now = new Date().toISOString();
38
+ const frontmatter = {
39
+ id,
40
+ title,
41
+ type: 'generated_note',
42
+ scope: 'project',
43
+ project: 'bench',
44
+ topic_key: `bench/${id}`,
45
+ created_at: now
46
+ };
47
+ const raw = toMarkdown(frontmatter, body);
48
+ const filePath = path.join(dir, 'generated', `${id}.md`);
49
+ fs.writeFileSync(filePath, raw, 'utf8');
50
+ return filePath;
51
+ }
52
+
53
+ function buildLargeBody(size, needle = 'NeedleInHaystack') {
54
+ const intro = `# Big\n\n${needle}\n\n`;
55
+ const filler = 'x'.repeat(Math.max(0, size - intro.length));
56
+ return intro + filler;
57
+ }
58
+
59
+ const NUM_SMALL = 5;
60
+ const LARGE_SIZE = 10_000;
61
+ const WARMUP = 2;
62
+ const RUNS = 5;
63
+
64
+ function ms(n) {
65
+ return Math.round(n);
66
+ }
67
+
68
+ function section(title) {
69
+ console.log('\n--- %s ---', title);
70
+ }
71
+
72
+ function median(arr) {
73
+ const a = [...arr].sort((x, y) => x - y);
74
+ const m = Math.floor(a.length / 2);
75
+ return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
76
+ }
77
+
78
+ /**
79
+ * Time a sync function over multiple runs; report median ms.
80
+ */
81
+ function timeSync(fn, runs = RUNS, warmup = WARMUP) {
82
+ for (let i = 0; i < warmup; i++) fn();
83
+ const times = [];
84
+ for (let i = 0; i < runs; i++) {
85
+ const t0 = performance.now();
86
+ fn();
87
+ times.push(performance.now() - t0);
88
+ }
89
+ return median(times);
90
+ }
91
+
92
+ /**
93
+ * Time an async function over multiple runs; report median ms.
94
+ */
95
+ async function timeAsync(fn, runs = RUNS, warmup = WARMUP) {
96
+ for (let i = 0; i < warmup; i++) await fn();
97
+ const times = [];
98
+ for (let i = 0; i < runs; i++) {
99
+ const t0 = performance.now();
100
+ await fn();
101
+ times.push(performance.now() - t0);
102
+ }
103
+ return median(times);
104
+ }
105
+
106
+ async function runBenchmark() {
107
+ console.log('Corpus: %d short .md + 1 large .md (~%d chars). Mode: %s\n', NUM_SMALL, LARGE_SIZE, RUN_SEMANTIC ? 'FTS + semantic' : 'FTS only');
108
+
109
+ // --- Corpus: short docs + one large doc with needle ---
110
+ for (let i = 0; i < NUM_SMALL; i++) {
111
+ writeDoc(baseConfig.projectMemoryRoot, `short_${i}`, `Short ${i}`, `Content for short doc ${i}. Some terms: alpha beta gamma.\n`);
112
+ }
113
+ writeDoc(baseConfig.projectMemoryRoot, 'large', 'Large doc', buildLargeBody(LARGE_SIZE));
114
+
115
+ fullReindex(configNoEmb);
116
+ const db = getDb(baseConfig);
117
+ const docCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
118
+ console.log('Indexed %d documents (FTS only).\n', docCount);
119
+
120
+ // ========== FTS search ==========
121
+ section('FTS search (no embeddings)');
122
+
123
+ const ftsShort = timeSync(() => {
124
+ searchFts(configNoEmb, { query: 'Content', limit: 10 });
125
+ });
126
+ console.log('Query "Content" (short, matches short docs): median %d ms', ms(ftsShort));
127
+
128
+ const ftsNeedle = timeSync(() => {
129
+ searchFts(configNoEmb, { query: 'NeedleInHaystack', limit: 10 });
130
+ });
131
+ console.log('Query "NeedleInHaystack" (needle in large): median %d ms', ms(ftsNeedle));
132
+
133
+ const ftsPhrase = timeSync(() => {
134
+ searchFts(configNoEmb, { query: 'alpha beta gamma', limit: 10 });
135
+ });
136
+ console.log('Query "alpha beta gamma" (phrase in short): median %d ms', ms(ftsPhrase));
137
+
138
+ const ftsRecent = timeSync(() => {
139
+ getRecentContext(configNoEmb, { limit: 10 });
140
+ });
141
+ console.log('getRecentContext(limit=10): median %d ms', ms(ftsRecent));
142
+
143
+ // ========== Semantic search (optional) ==========
144
+ if (RUN_SEMANTIC) {
145
+ section('With embeddings: index + process queue');
146
+
147
+ clearIndex(configEmb);
148
+ const paths = listMarkdownPaths(configEmb);
149
+ for (const filePath of paths) {
150
+ try {
151
+ const raw = fs.readFileSync(filePath, 'utf8');
152
+ const stat = fs.statSync(filePath);
153
+ indexDocument(configEmb, filePath, raw, stat.mtimeMs);
154
+ } catch (_) {}
155
+ }
156
+ await new Promise((r) => setImmediate(r));
157
+ await new Promise((r) => setImmediate(r));
158
+
159
+ let processed = 0;
160
+ while (await processNextJob(configEmb)) processed++;
161
+ const vecCount = db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name LIKE 'vec_%'").get()?.c ?? 0;
162
+ const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
163
+ const vecRows = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
164
+ console.log('Embedding jobs processed: %d. Vector rows: %d.', processed, vecRows);
165
+
166
+ section('Semantic search (query embed + vector KNN)');
167
+
168
+ const { getEmbedder } = await import('../src/core/embeddings/embedder.js');
169
+ const { search: searchVector } = await import('../src/core/embeddings/vectorStore.js');
170
+ const embedder = await getEmbedder(configEmb);
171
+ if (!embedder) {
172
+ console.log('Embedder not available; skipping semantic timings.');
173
+ } else {
174
+ const embedTime = await timeAsync(async () => {
175
+ await embedder.embed('NeedleInHaystack');
176
+ });
177
+ console.log('Query embedding (single phrase): median %d ms', ms(embedTime));
178
+
179
+ const vectorSearchTime = await timeAsync(async () => {
180
+ const vec = await embedder.embed('NeedleInHaystack');
181
+ await searchVector(configEmb, vec, 20);
182
+ });
183
+ console.log('Embed + vector search (k=20): median %d ms', ms(vectorSearchTime));
184
+
185
+ const precomputedVec = await embedder.embed('query');
186
+ const vectorOnlyTime = await timeAsync(async () => {
187
+ await searchVector(configEmb, precomputedVec, 10);
188
+ });
189
+ console.log('Vector search only (k=10, no embed): median %d ms', ms(vectorOnlyTime));
190
+ }
191
+
192
+ section('FTS with embeddings enabled (same as FTS-only path)');
193
+ const ftsWithEmb = timeSync(() => {
194
+ searchFts(configEmb, { query: 'Content', limit: 10 });
195
+ });
196
+ console.log('Query "Content": median %d ms', ms(ftsWithEmb));
197
+ }
198
+
199
+ section('Summary');
200
+ console.log('FTS: short query, needle-in-large, phrase, getRecentContext.');
201
+ if (RUN_SEMANTIC) {
202
+ console.log('Semantic: query embed + vector KNN (npm run bench:search:embed).');
203
+ }
204
+ }
205
+
206
+ runBenchmark()
207
+ .then(() => {
208
+ closeDb();
209
+ try {
210
+ fs.rmSync(tmpDir, { recursive: true, force: true });
211
+ } catch (_) {}
212
+ console.log('\nDone.\n');
213
+ })
214
+ .catch((err) => {
215
+ closeDb();
216
+ try {
217
+ fs.rmSync(tmpDir, { recursive: true, force: true });
218
+ } catch (_) {}
219
+ console.error(err);
220
+ process.exit(1);
221
+ });
package/spec/run-tests.js CHANGED
@@ -14,6 +14,7 @@ import { createToolHandlers } from '../src/mcp/tools.js';
14
14
  import { getToolDefinitions } from '../src/mcp/tool-registry.js';
15
15
  import { closeDb, getDb } from '../src/db/sqlite.js';
16
16
  import { scenarios } from './scenarios/scenarios.js';
17
+ import { runLargeDocumentTest } from './test-large-document.js';
17
18
  import * as reporter from './reporter.js';
18
19
 
19
20
  const __filename = fileURLToPath(import.meta.url);
@@ -167,15 +168,15 @@ async function runLegacySemanticKnn() {
167
168
  const persistId = parse(savePersist).id;
168
169
  ok(!!persistId, 'saved persistence doc');
169
170
  const db = getDb(configSemantic);
170
- const chunkRow = db.prepare('SELECT id, document_id, content_hash, content FROM document_chunks WHERE document_id = ? LIMIT 1').get(persistId);
171
- ok(!!chunkRow, 'chunk exists for persistence doc');
172
- if (embedder && vectorStore && chunkRow) {
173
- const embedding = await embedder.embed(chunkRow.content);
171
+ const docRow = db.prepare('SELECT id, body, content_hash FROM documents WHERE id = ?').get(persistId);
172
+ ok(!!docRow, 'document exists for persistence doc');
173
+ if (embedder && vectorStore && docRow?.body) {
174
+ const embedding = await embedder.embed(docRow.body);
174
175
  await vectorStore.ensureTables(configSemantic);
175
176
  await vectorStore.insert(configSemantic, {
176
- chunk_id: chunkRow.id,
177
- document_id: chunkRow.document_id,
178
- content_hash: chunkRow.content_hash,
177
+ chunk_id: docRow.id,
178
+ document_id: docRow.id,
179
+ content_hash: docRow.content_hash,
179
180
  embedding
180
181
  });
181
182
  const queryEmb = await embedder.embed('dónde se persisten los datos');
@@ -195,6 +196,10 @@ async function run() {
195
196
  await runScenarios();
196
197
  await runLegacySemanticKnn();
197
198
 
199
+ const largeResult = runLargeDocumentTest(config);
200
+ totalPassed += largeResult.passed;
201
+ totalFailed += largeResult.failed;
202
+
198
203
  closeDb();
199
204
  try { fs.rmSync(tmpDir, { recursive: true }); } catch (_) {}
200
205
 
@@ -21,6 +21,7 @@ import {
21
21
  semanticFallbackDoc,
22
22
  PROJECT_NAME
23
23
  } from '../fixtures/realistic-content.js';
24
+ import { toMarkdown } from '../../src/utils/markdown.js';
24
25
 
25
26
  function parse(res) {
26
27
  return JSON.parse(res.content[0].text);
@@ -156,11 +157,12 @@ export const scenarios = [
156
157
 
157
158
  reporter.startScenario(this.name, this.goal);
158
159
 
159
- const search2 = await h.mem_search({ query: 'derived index', project: PROJECT_NAME, limit: 5 });
160
+ const search2 = await h.mem_search({ query: 'updated body', project: PROJECT_NAME, limit: 5 });
160
161
  const res = parseRes(search2).results;
161
- reporter.step('mem_search', { query: 'derived index', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
162
+ reporter.step('mem_search', { query: 'updated body', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
162
163
  ok(Array.isArray(res), 'results array');
163
- ok(res.length >= 1 && res[0].snippet != null, 'snippets present');
164
+ ok(res.length >= 1, 'has results');
165
+ ok(res.length === 0 || res[0].snippet != null, 'snippets present');
164
166
 
165
167
  reporter.endScenario(p, f);
166
168
  return { passed: p, failed: f };
@@ -0,0 +1,98 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
4
+ * Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
5
+ */
6
+ import fs from 'fs';
7
+ import path from 'path';
8
+ import os from 'os';
9
+ import { loadConfig } from '../src/config.js';
10
+ import { indexDocument } from '../src/core/indexing/indexer.js';
11
+ import { search } from '../src/core/search/search.js';
12
+ import { getDb, closeDb } from '../src/db/sqlite.js';
13
+ import { toMarkdown } from '../src/utils/markdown.js';
14
+
15
+ const TARGET_SIZE = 100_000;
16
+ const NEEDLE = 'NeedleInHaystack100k';
17
+
18
+ function buildLargeBody(targetSize) {
19
+ const intro = `# Documento grande\n\nPárrafo inicial. ${NEEDLE} aquí.\n\n`;
20
+ const paragraph = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(20) + '\n\n';
21
+ const repeat = Math.ceil((targetSize - intro.length) / paragraph.length);
22
+ return intro + paragraph.repeat(Math.max(0, repeat));
23
+ }
24
+
25
+ /**
26
+ * @param {object} config - resolved config (e.g. from run-tests)
27
+ * @returns {{ passed: number, failed: number }}
28
+ */
29
+ export function runLargeDocumentTest(config) {
30
+ let passed = 0;
31
+ let failed = 0;
32
+ const ok = (cond, msg) => {
33
+ if (cond) passed++;
34
+ else failed++;
35
+ if (!cond) console.error('FAIL:', msg);
36
+ };
37
+
38
+ const docsDir = path.join(config.projectMemoryRoot, 'generated');
39
+ fs.mkdirSync(docsDir, { recursive: true });
40
+
41
+ const body = buildLargeBody(TARGET_SIZE);
42
+ const id = 'obs_large_100k';
43
+ const now = new Date().toISOString();
44
+ const frontmatter = {
45
+ id,
46
+ title: 'Documento 100k',
47
+ type: 'generated_note',
48
+ scope: 'project',
49
+ project: 'test',
50
+ topic_key: 'test/large-100k',
51
+ created_at: now
52
+ };
53
+ const raw = toMarkdown(frontmatter, body);
54
+ const docPath = path.join(docsDir, 'large-100k.md');
55
+ fs.writeFileSync(docPath, raw, 'utf8');
56
+ const stat = fs.statSync(docPath);
57
+
58
+ const size = raw.length;
59
+ ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
60
+
61
+ indexDocument(config, docPath, raw, stat.mtimeMs);
62
+
63
+ const db = getDb(config);
64
+ const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
65
+ ok(!!doc, 'documento en tabla documents');
66
+ ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
67
+
68
+ const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
69
+ ok(!!ftsRow, 'documento en FTS');
70
+ ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
71
+
72
+ const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
73
+ ok(results.length >= 1, 'búsqueda encuentra el documento');
74
+ ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
75
+ ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
76
+
77
+ return { passed, failed };
78
+ }
79
+
80
+ function main() {
81
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
82
+ const config = loadConfig({
83
+ homeMemoryRoot: path.join(tmpDir, 'home'),
84
+ projectMemoryRoot: path.join(tmpDir, 'docs'),
85
+ sqlitePath: path.join(tmpDir, 'index.db')
86
+ });
87
+ fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
88
+
89
+ const { passed, failed } = runLargeDocumentTest(config);
90
+ console.log(`Large document test: ${passed} passed, ${failed} failed`);
91
+ closeDb();
92
+ try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
93
+ process.exitCode = failed > 0 ? 1 : 0;
94
+ }
95
+
96
+ if (process.argv[1]?.endsWith('test-large-document.js')) {
97
+ main();
98
+ }