mixgram 1.1.6 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/mixgram.js +22 -0
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +12 -7
- package/spec/scenarios/scenarios.js +5 -3
- package/spec/test-large-document.js +98 -0
- package/src/config.js +7 -4
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -126
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/utils/markdown.js +42 -2
package/bin/mixgram.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { run } from '../src/mcp/server.js';
|
|
9
9
|
import { loadConfig } from '../src/config.js';
|
|
10
|
+
import { closeDb } from '../src/db/sqlite.js';
|
|
10
11
|
import { createToolHandlers } from '../src/mcp/tools.js';
|
|
11
12
|
import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../src/mcp/cli-adapter.js';
|
|
12
13
|
import fs from 'fs';
|
|
@@ -217,6 +218,7 @@ Usage: mixgram <command> [options]
|
|
|
217
218
|
Commands:
|
|
218
219
|
mcp [options] Run the MCP server (stdio). Use this in your client config.
|
|
219
220
|
setup <client> Register Mixgram with an MCP client.
|
|
221
|
+
flushdb Delete the SQLite database file (and WAL/shm).
|
|
220
222
|
help [tool] Show help; with optional tool name, show options for that tool.
|
|
221
223
|
-v, --version Print version from package and exit.
|
|
222
224
|
<tool> [options] Run an MCP tool by name. Tools:
|
|
@@ -263,6 +265,26 @@ async function main() {
|
|
|
263
265
|
return;
|
|
264
266
|
}
|
|
265
267
|
|
|
268
|
+
if (SUBCOMMAND === 'flushdb') {
|
|
269
|
+
const { overrides, baseDir, projectBaseDir } = loadCliConfig();
|
|
270
|
+
const config = loadConfig(overrides, baseDir, projectBaseDir);
|
|
271
|
+
const dbPath = config.sqlitePath;
|
|
272
|
+
closeDb();
|
|
273
|
+
const removed = [];
|
|
274
|
+
for (const p of [dbPath, dbPath + '-wal', dbPath + '-shm']) {
|
|
275
|
+
if (fs.existsSync(p)) {
|
|
276
|
+
fs.unlinkSync(p);
|
|
277
|
+
removed.push(p);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
if (removed.length) {
|
|
281
|
+
console.log('Removed:', removed.join(', '));
|
|
282
|
+
} else {
|
|
283
|
+
console.log('No database file at', dbPath);
|
|
284
|
+
}
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
|
|
266
288
|
if (SUBCOMMAND === 'setup') {
|
|
267
289
|
if (!ARG) {
|
|
268
290
|
console.error('Usage: mixgram setup <cursor|gemini-cli|codex>');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixgram",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.2",
|
|
4
4
|
"description": "MCP memory server with Markdown source of truth and SQLite FTS5.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
@@ -33,19 +33,32 @@
|
|
|
33
33
|
"scripts": {
|
|
34
34
|
"start": "node index.js",
|
|
35
35
|
"test": "node spec/run-tests.js",
|
|
36
|
-
"test:visual": "node spec/run-tests.js --visual"
|
|
36
|
+
"test:visual": "node spec/run-tests.js --visual",
|
|
37
|
+
"test:large": "node spec/test-large-document.js",
|
|
38
|
+
"bench:indexing": "node spec/bench-indexing.js",
|
|
39
|
+
"bench:embed": "node spec/bench-indexing.js --embed",
|
|
40
|
+
"bench:search": "node spec/bench-search.js",
|
|
41
|
+
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
42
|
+
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
43
|
+
"duplication": "jscpd src",
|
|
44
|
+
"deadcode": "knip",
|
|
45
|
+
"quality": "npm run duplication && npm run deadcode"
|
|
37
46
|
},
|
|
38
47
|
"dependencies": {
|
|
39
48
|
"@huggingface/transformers": "^3.4.2",
|
|
40
49
|
"@modelcontextprotocol/sdk": "^1.0.0",
|
|
41
50
|
"better-sqlite3": "^12.6.2",
|
|
42
51
|
"chokidar": "^4.0.1",
|
|
52
|
+
"glob": "^13.0.6",
|
|
43
53
|
"gray-matter": "^4.0.3",
|
|
44
|
-
"glob": "^13.0.0",
|
|
45
54
|
"remark-parse": "^11.0.0",
|
|
46
55
|
"sqlite-vec": "^0.1.6",
|
|
47
56
|
"unified": "^11.0.5",
|
|
48
57
|
"unist-util-visit": "^5.0.0",
|
|
49
58
|
"zod": "^3.23.0"
|
|
59
|
+
},
|
|
60
|
+
"devDependencies": {
|
|
61
|
+
"jscpd": "^4.0.8",
|
|
62
|
+
"knip": "^5.86.0"
|
|
50
63
|
}
|
|
51
64
|
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Benchmark: indexing and reindexing with synthetic corpus.
|
|
4
|
+
* Breaks down timings (clear, listPaths, indexing) and compares with/without embeddings.
|
|
5
|
+
* Opt-in; not run by npm test.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* npm run bench:indexing # indexing only
|
|
9
|
+
* npm run bench:embed # indexing + embedding queue processing (slow)
|
|
10
|
+
*/
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
import { loadConfig } from '../src/config.js';
|
|
15
|
+
import {
|
|
16
|
+
fullReindex,
|
|
17
|
+
incrementalReindex,
|
|
18
|
+
clearIndex,
|
|
19
|
+
listMarkdownPaths
|
|
20
|
+
} from '../src/core/indexing/reindex.js';
|
|
21
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
22
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
23
|
+
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
24
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
25
|
+
|
|
26
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-'));
|
|
27
|
+
const baseConfig = loadConfig({
|
|
28
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
29
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
30
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
31
|
+
});
|
|
32
|
+
fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
|
|
33
|
+
fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
|
|
34
|
+
|
|
35
|
+
const configNoEmb = {
|
|
36
|
+
...baseConfig,
|
|
37
|
+
embeddings: { ...baseConfig.embeddings, enabled: false }
|
|
38
|
+
};
|
|
39
|
+
const configEmb = {
|
|
40
|
+
...baseConfig,
|
|
41
|
+
embeddings: { ...baseConfig.embeddings, enabled: true }
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** Frontmatter with same keys the app writes so the indexer parses the same. */
|
|
45
|
+
function writeDoc(dir, id, title, body) {
|
|
46
|
+
const now = new Date().toISOString();
|
|
47
|
+
const frontmatter = {
|
|
48
|
+
id,
|
|
49
|
+
title,
|
|
50
|
+
type: 'generated_note',
|
|
51
|
+
scope: 'project',
|
|
52
|
+
project: 'bench',
|
|
53
|
+
topic_key: `bench/${id}`,
|
|
54
|
+
created_at: now
|
|
55
|
+
};
|
|
56
|
+
const raw = toMarkdown(frontmatter, body);
|
|
57
|
+
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
58
|
+
fs.writeFileSync(filePath, raw, 'utf8');
|
|
59
|
+
return filePath;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function buildLargeBody(size) {
|
|
63
|
+
const intro = '# Big\n\nNeedleInHaystack\n\n';
|
|
64
|
+
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
65
|
+
return intro + filler;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const NUM_SMALL = 3;
|
|
69
|
+
const LARGE_SIZE = 8_000;
|
|
70
|
+
const RUN_EMBED_PROCESSING = process.argv.includes('--embed');
|
|
71
|
+
|
|
72
|
+
console.log('Corpus: %d small .md + 1 large .md (~%d chars)', NUM_SMALL, LARGE_SIZE);
|
|
73
|
+
console.log('Mode: %s\n', RUN_EMBED_PROCESSING ? 'indexing + embedding queue processing' : 'indexing only');
|
|
74
|
+
|
|
75
|
+
// --- Create corpus ---
|
|
76
|
+
for (let i = 0; i < NUM_SMALL; i++) {
|
|
77
|
+
writeDoc(baseConfig.projectMemoryRoot, `obs_small_${i}`, `Small ${i}`, `Content for small doc ${i}.\n`);
|
|
78
|
+
}
|
|
79
|
+
const largePath = writeDoc(baseConfig.projectMemoryRoot, 'obs_large', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Full reindex with breakdown: clearIndex, listMarkdownPaths, index loop.
|
|
83
|
+
* @returns {{ total: number, clear: number, list: number, indexLoop: number, indexed: number }}
|
|
84
|
+
*/
|
|
85
|
+
function fullReindexWithBreakdown(config) {
|
|
86
|
+
let t0, t1;
|
|
87
|
+
t0 = performance.now();
|
|
88
|
+
clearIndex(config);
|
|
89
|
+
t1 = performance.now();
|
|
90
|
+
const clearMs = t1 - t0;
|
|
91
|
+
|
|
92
|
+
t0 = performance.now();
|
|
93
|
+
const paths = listMarkdownPaths(config);
|
|
94
|
+
t1 = performance.now();
|
|
95
|
+
const listMs = t1 - t0;
|
|
96
|
+
|
|
97
|
+
let indexed = 0;
|
|
98
|
+
t0 = performance.now();
|
|
99
|
+
for (const filePath of paths) {
|
|
100
|
+
try {
|
|
101
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
102
|
+
const stat = fs.statSync(filePath);
|
|
103
|
+
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
104
|
+
indexed++;
|
|
105
|
+
} catch (_) {}
|
|
106
|
+
}
|
|
107
|
+
t1 = performance.now();
|
|
108
|
+
const indexLoopMs = t1 - t0;
|
|
109
|
+
const totalMs = clearMs + listMs + indexLoopMs;
|
|
110
|
+
|
|
111
|
+
return { total: totalMs, clear: clearMs, list: listMs, indexLoop: indexLoopMs, indexed };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function ms(n) {
|
|
115
|
+
return Math.round(n);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function section(title) {
|
|
119
|
+
console.log('\n--- %s ---', title);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function runBenchmark() {
|
|
123
|
+
// ========== Without embeddings ==========
|
|
124
|
+
section('Without embeddings');
|
|
125
|
+
|
|
126
|
+
let b = fullReindexWithBreakdown(configNoEmb);
|
|
127
|
+
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
128
|
+
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
129
|
+
|
|
130
|
+
let t0 = performance.now();
|
|
131
|
+
const inc0 = incrementalReindex(configNoEmb);
|
|
132
|
+
let t1 = performance.now();
|
|
133
|
+
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
134
|
+
ms(t1 - t0), inc0.scanned, inc0.skipped, inc0.indexed, inc0.removed);
|
|
135
|
+
|
|
136
|
+
fs.appendFileSync(largePath, '\n\nAppended line.');
|
|
137
|
+
|
|
138
|
+
t0 = performance.now();
|
|
139
|
+
const inc1 = incrementalReindex(configNoEmb);
|
|
140
|
+
t1 = performance.now();
|
|
141
|
+
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
142
|
+
ms(t1 - t0), inc1.scanned, inc1.skipped, inc1.indexed, inc1.removed);
|
|
143
|
+
|
|
144
|
+
// Final state without embeddings
|
|
145
|
+
const db = getDb(baseConfig);
|
|
146
|
+
const docsCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
147
|
+
const ftsCount = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
|
|
148
|
+
console.log('State: %d documents, %d FTS rows.', docsCount, ftsCount);
|
|
149
|
+
|
|
150
|
+
// ========== With embeddings (indexing + queue) ==========
|
|
151
|
+
section('With embeddings (indexing + queue)');
|
|
152
|
+
|
|
153
|
+
clearIndex(configEmb);
|
|
154
|
+
b = fullReindexWithBreakdown(configEmb);
|
|
155
|
+
// Allow enqueue promises to settle
|
|
156
|
+
await new Promise((r) => setImmediate(r));
|
|
157
|
+
await new Promise((r) => setImmediate(r));
|
|
158
|
+
|
|
159
|
+
const jobsEnqueued = db.prepare('SELECT COUNT(*) AS c FROM embedding_jobs').get()?.c ?? 0;
|
|
160
|
+
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
161
|
+
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
162
|
+
console.log('Jobs in queue: %d', jobsEnqueued);
|
|
163
|
+
|
|
164
|
+
t0 = performance.now();
|
|
165
|
+
const inc0emb = incrementalReindex(configEmb);
|
|
166
|
+
t1 = performance.now();
|
|
167
|
+
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
168
|
+
ms(t1 - t0), inc0emb.scanned, inc0emb.skipped, inc0emb.indexed, inc0emb.removed);
|
|
169
|
+
|
|
170
|
+
fs.appendFileSync(largePath, '\n\nAnother line.');
|
|
171
|
+
|
|
172
|
+
t0 = performance.now();
|
|
173
|
+
const inc1emb = incrementalReindex(configEmb);
|
|
174
|
+
t1 = performance.now();
|
|
175
|
+
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
176
|
+
ms(t1 - t0), inc1emb.scanned, inc1emb.skipped, inc1emb.indexed, inc1emb.removed);
|
|
177
|
+
|
|
178
|
+
// ========== Embedding processing (optional) ==========
|
|
179
|
+
if (RUN_EMBED_PROCESSING && configEmb.embeddings?.enabled) {
|
|
180
|
+
section('Embedding processing (queue → model → vector store)');
|
|
181
|
+
|
|
182
|
+
const pendingBefore = db.prepare("SELECT COUNT(*) AS c FROM embedding_jobs WHERE status = 'pending'").get()?.c ?? 0;
|
|
183
|
+
t0 = performance.now();
|
|
184
|
+
let processed = 0;
|
|
185
|
+
while (await processNextJob(configEmb)) {
|
|
186
|
+
processed++;
|
|
187
|
+
}
|
|
188
|
+
t1 = performance.now();
|
|
189
|
+
console.log('Process queue: %d ms (jobs processed=%d, pending before=%d)',
|
|
190
|
+
ms(t1 - t0), processed, pendingBefore);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Final summary
|
|
194
|
+
section('Summary');
|
|
195
|
+
const docsFinal = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
196
|
+
const ftsFinal = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
|
|
197
|
+
const jobsFinal = db.prepare('SELECT status, COUNT(*) AS c FROM embedding_jobs GROUP BY status').all();
|
|
198
|
+
console.log('Documents: %d, FTS rows: %d', docsFinal, ftsFinal);
|
|
199
|
+
if (jobsFinal.length) {
|
|
200
|
+
console.log('Embedding jobs: %s', jobsFinal.map((r) => `${r.status}=${r.c}`).join(', '));
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
runBenchmark()
|
|
205
|
+
.then(() => {
|
|
206
|
+
closeDb();
|
|
207
|
+
try {
|
|
208
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
209
|
+
} catch (_) {}
|
|
210
|
+
console.log('\nDone.\n');
|
|
211
|
+
})
|
|
212
|
+
.catch((err) => {
|
|
213
|
+
closeDb();
|
|
214
|
+
try {
|
|
215
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
216
|
+
} catch (_) {}
|
|
217
|
+
console.error(err);
|
|
218
|
+
process.exit(1);
|
|
219
|
+
});
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Benchmark: search (FTS and optional semantic) over synthetic corpus.
|
|
4
|
+
* Compares short vs long documents, FTS-only vs with embeddings.
|
|
5
|
+
* Opt-in; not run by npm test.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* npm run bench:search # FTS only
|
|
9
|
+
* npm run bench:search:embed # FTS + semantic search (needs embed model)
|
|
10
|
+
*/
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
import { loadConfig } from '../src/config.js';
|
|
15
|
+
import { fullReindex, clearIndex, listMarkdownPaths } from '../src/core/indexing/reindex.js';
|
|
16
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
17
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
18
|
+
import { search as searchFts, getRecentContext } from '../src/core/search/search.js';
|
|
19
|
+
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
20
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
21
|
+
|
|
22
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-search-'));
|
|
23
|
+
const baseConfig = loadConfig({
|
|
24
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
25
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
26
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
27
|
+
});
|
|
28
|
+
fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
|
|
29
|
+
fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
|
|
30
|
+
|
|
31
|
+
const configNoEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: false } };
|
|
32
|
+
const configEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: true } };
|
|
33
|
+
|
|
34
|
+
const RUN_SEMANTIC = process.argv.includes('--embed');
|
|
35
|
+
|
|
36
|
+
function writeDoc(dir, id, title, body) {
|
|
37
|
+
const now = new Date().toISOString();
|
|
38
|
+
const frontmatter = {
|
|
39
|
+
id,
|
|
40
|
+
title,
|
|
41
|
+
type: 'generated_note',
|
|
42
|
+
scope: 'project',
|
|
43
|
+
project: 'bench',
|
|
44
|
+
topic_key: `bench/${id}`,
|
|
45
|
+
created_at: now
|
|
46
|
+
};
|
|
47
|
+
const raw = toMarkdown(frontmatter, body);
|
|
48
|
+
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
49
|
+
fs.writeFileSync(filePath, raw, 'utf8');
|
|
50
|
+
return filePath;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function buildLargeBody(size, needle = 'NeedleInHaystack') {
|
|
54
|
+
const intro = `# Big\n\n${needle}\n\n`;
|
|
55
|
+
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
56
|
+
return intro + filler;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const NUM_SMALL = 5;
|
|
60
|
+
const LARGE_SIZE = 10_000;
|
|
61
|
+
const WARMUP = 2;
|
|
62
|
+
const RUNS = 5;
|
|
63
|
+
|
|
64
|
+
function ms(n) {
|
|
65
|
+
return Math.round(n);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function section(title) {
|
|
69
|
+
console.log('\n--- %s ---', title);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function median(arr) {
|
|
73
|
+
const a = [...arr].sort((x, y) => x - y);
|
|
74
|
+
const m = Math.floor(a.length / 2);
|
|
75
|
+
return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Time a sync function over multiple runs; report median ms.
|
|
80
|
+
*/
|
|
81
|
+
function timeSync(fn, runs = RUNS, warmup = WARMUP) {
|
|
82
|
+
for (let i = 0; i < warmup; i++) fn();
|
|
83
|
+
const times = [];
|
|
84
|
+
for (let i = 0; i < runs; i++) {
|
|
85
|
+
const t0 = performance.now();
|
|
86
|
+
fn();
|
|
87
|
+
times.push(performance.now() - t0);
|
|
88
|
+
}
|
|
89
|
+
return median(times);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Time an async function over multiple runs; report median ms.
|
|
94
|
+
*/
|
|
95
|
+
async function timeAsync(fn, runs = RUNS, warmup = WARMUP) {
|
|
96
|
+
for (let i = 0; i < warmup; i++) await fn();
|
|
97
|
+
const times = [];
|
|
98
|
+
for (let i = 0; i < runs; i++) {
|
|
99
|
+
const t0 = performance.now();
|
|
100
|
+
await fn();
|
|
101
|
+
times.push(performance.now() - t0);
|
|
102
|
+
}
|
|
103
|
+
return median(times);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function runBenchmark() {
|
|
107
|
+
console.log('Corpus: %d short .md + 1 large .md (~%d chars). Mode: %s\n', NUM_SMALL, LARGE_SIZE, RUN_SEMANTIC ? 'FTS + semantic' : 'FTS only');
|
|
108
|
+
|
|
109
|
+
// --- Corpus: short docs + one large doc with needle ---
|
|
110
|
+
for (let i = 0; i < NUM_SMALL; i++) {
|
|
111
|
+
writeDoc(baseConfig.projectMemoryRoot, `short_${i}`, `Short ${i}`, `Content for short doc ${i}. Some terms: alpha beta gamma.\n`);
|
|
112
|
+
}
|
|
113
|
+
writeDoc(baseConfig.projectMemoryRoot, 'large', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
114
|
+
|
|
115
|
+
fullReindex(configNoEmb);
|
|
116
|
+
const db = getDb(baseConfig);
|
|
117
|
+
const docCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
118
|
+
console.log('Indexed %d documents (FTS only).\n', docCount);
|
|
119
|
+
|
|
120
|
+
// ========== FTS search ==========
|
|
121
|
+
section('FTS search (no embeddings)');
|
|
122
|
+
|
|
123
|
+
const ftsShort = timeSync(() => {
|
|
124
|
+
searchFts(configNoEmb, { query: 'Content', limit: 10 });
|
|
125
|
+
});
|
|
126
|
+
console.log('Query "Content" (short, matches short docs): median %d ms', ms(ftsShort));
|
|
127
|
+
|
|
128
|
+
const ftsNeedle = timeSync(() => {
|
|
129
|
+
searchFts(configNoEmb, { query: 'NeedleInHaystack', limit: 10 });
|
|
130
|
+
});
|
|
131
|
+
console.log('Query "NeedleInHaystack" (needle in large): median %d ms', ms(ftsNeedle));
|
|
132
|
+
|
|
133
|
+
const ftsPhrase = timeSync(() => {
|
|
134
|
+
searchFts(configNoEmb, { query: 'alpha beta gamma', limit: 10 });
|
|
135
|
+
});
|
|
136
|
+
console.log('Query "alpha beta gamma" (phrase in short): median %d ms', ms(ftsPhrase));
|
|
137
|
+
|
|
138
|
+
const ftsRecent = timeSync(() => {
|
|
139
|
+
getRecentContext(configNoEmb, { limit: 10 });
|
|
140
|
+
});
|
|
141
|
+
console.log('getRecentContext(limit=10): median %d ms', ms(ftsRecent));
|
|
142
|
+
|
|
143
|
+
// ========== Semantic search (optional) ==========
|
|
144
|
+
if (RUN_SEMANTIC) {
|
|
145
|
+
section('With embeddings: index + process queue');
|
|
146
|
+
|
|
147
|
+
clearIndex(configEmb);
|
|
148
|
+
const paths = listMarkdownPaths(configEmb);
|
|
149
|
+
for (const filePath of paths) {
|
|
150
|
+
try {
|
|
151
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
152
|
+
const stat = fs.statSync(filePath);
|
|
153
|
+
indexDocument(configEmb, filePath, raw, stat.mtimeMs);
|
|
154
|
+
} catch (_) {}
|
|
155
|
+
}
|
|
156
|
+
await new Promise((r) => setImmediate(r));
|
|
157
|
+
await new Promise((r) => setImmediate(r));
|
|
158
|
+
|
|
159
|
+
let processed = 0;
|
|
160
|
+
while (await processNextJob(configEmb)) processed++;
|
|
161
|
+
const vecCount = db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name LIKE 'vec_%'").get()?.c ?? 0;
|
|
162
|
+
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
163
|
+
const vecRows = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
164
|
+
console.log('Embedding jobs processed: %d. Vector rows: %d.', processed, vecRows);
|
|
165
|
+
|
|
166
|
+
section('Semantic search (query embed + vector KNN)');
|
|
167
|
+
|
|
168
|
+
const { getEmbedder } = await import('../src/core/embeddings/embedder.js');
|
|
169
|
+
const { search: searchVector } = await import('../src/core/embeddings/vectorStore.js');
|
|
170
|
+
const embedder = await getEmbedder(configEmb);
|
|
171
|
+
if (!embedder) {
|
|
172
|
+
console.log('Embedder not available; skipping semantic timings.');
|
|
173
|
+
} else {
|
|
174
|
+
const embedTime = await timeAsync(async () => {
|
|
175
|
+
await embedder.embed('NeedleInHaystack');
|
|
176
|
+
});
|
|
177
|
+
console.log('Query embedding (single phrase): median %d ms', ms(embedTime));
|
|
178
|
+
|
|
179
|
+
const vectorSearchTime = await timeAsync(async () => {
|
|
180
|
+
const vec = await embedder.embed('NeedleInHaystack');
|
|
181
|
+
await searchVector(configEmb, vec, 20);
|
|
182
|
+
});
|
|
183
|
+
console.log('Embed + vector search (k=20): median %d ms', ms(vectorSearchTime));
|
|
184
|
+
|
|
185
|
+
const precomputedVec = await embedder.embed('query');
|
|
186
|
+
const vectorOnlyTime = await timeAsync(async () => {
|
|
187
|
+
await searchVector(configEmb, precomputedVec, 10);
|
|
188
|
+
});
|
|
189
|
+
console.log('Vector search only (k=10, no embed): median %d ms', ms(vectorOnlyTime));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
section('FTS with embeddings enabled (same as FTS-only path)');
|
|
193
|
+
const ftsWithEmb = timeSync(() => {
|
|
194
|
+
searchFts(configEmb, { query: 'Content', limit: 10 });
|
|
195
|
+
});
|
|
196
|
+
console.log('Query "Content": median %d ms', ms(ftsWithEmb));
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
section('Summary');
|
|
200
|
+
console.log('FTS: short query, needle-in-large, phrase, getRecentContext.');
|
|
201
|
+
if (RUN_SEMANTIC) {
|
|
202
|
+
console.log('Semantic: query embed + vector KNN (npm run bench:search:embed).');
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
runBenchmark()
|
|
207
|
+
.then(() => {
|
|
208
|
+
closeDb();
|
|
209
|
+
try {
|
|
210
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
211
|
+
} catch (_) {}
|
|
212
|
+
console.log('\nDone.\n');
|
|
213
|
+
})
|
|
214
|
+
.catch((err) => {
|
|
215
|
+
closeDb();
|
|
216
|
+
try {
|
|
217
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
218
|
+
} catch (_) {}
|
|
219
|
+
console.error(err);
|
|
220
|
+
process.exit(1);
|
|
221
|
+
});
|
package/spec/run-tests.js
CHANGED
|
@@ -14,6 +14,7 @@ import { createToolHandlers } from '../src/mcp/tools.js';
|
|
|
14
14
|
import { getToolDefinitions } from '../src/mcp/tool-registry.js';
|
|
15
15
|
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
16
16
|
import { scenarios } from './scenarios/scenarios.js';
|
|
17
|
+
import { runLargeDocumentTest } from './test-large-document.js';
|
|
17
18
|
import * as reporter from './reporter.js';
|
|
18
19
|
|
|
19
20
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -167,15 +168,15 @@ async function runLegacySemanticKnn() {
|
|
|
167
168
|
const persistId = parse(savePersist).id;
|
|
168
169
|
ok(!!persistId, 'saved persistence doc');
|
|
169
170
|
const db = getDb(configSemantic);
|
|
170
|
-
const
|
|
171
|
-
ok(!!
|
|
172
|
-
if (embedder && vectorStore &&
|
|
173
|
-
const embedding = await embedder.embed(
|
|
171
|
+
const docRow = db.prepare('SELECT id, body, content_hash FROM documents WHERE id = ?').get(persistId);
|
|
172
|
+
ok(!!docRow, 'document exists for persistence doc');
|
|
173
|
+
if (embedder && vectorStore && docRow?.body) {
|
|
174
|
+
const embedding = await embedder.embed(docRow.body);
|
|
174
175
|
await vectorStore.ensureTables(configSemantic);
|
|
175
176
|
await vectorStore.insert(configSemantic, {
|
|
176
|
-
chunk_id:
|
|
177
|
-
document_id:
|
|
178
|
-
content_hash:
|
|
177
|
+
chunk_id: docRow.id,
|
|
178
|
+
document_id: docRow.id,
|
|
179
|
+
content_hash: docRow.content_hash,
|
|
179
180
|
embedding
|
|
180
181
|
});
|
|
181
182
|
const queryEmb = await embedder.embed('dónde se persisten los datos');
|
|
@@ -195,6 +196,10 @@ async function run() {
|
|
|
195
196
|
await runScenarios();
|
|
196
197
|
await runLegacySemanticKnn();
|
|
197
198
|
|
|
199
|
+
const largeResult = runLargeDocumentTest(config);
|
|
200
|
+
totalPassed += largeResult.passed;
|
|
201
|
+
totalFailed += largeResult.failed;
|
|
202
|
+
|
|
198
203
|
closeDb();
|
|
199
204
|
try { fs.rmSync(tmpDir, { recursive: true }); } catch (_) {}
|
|
200
205
|
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
semanticFallbackDoc,
|
|
22
22
|
PROJECT_NAME
|
|
23
23
|
} from '../fixtures/realistic-content.js';
|
|
24
|
+
import { toMarkdown } from '../../src/utils/markdown.js';
|
|
24
25
|
|
|
25
26
|
function parse(res) {
|
|
26
27
|
return JSON.parse(res.content[0].text);
|
|
@@ -156,11 +157,12 @@ export const scenarios = [
|
|
|
156
157
|
|
|
157
158
|
reporter.startScenario(this.name, this.goal);
|
|
158
159
|
|
|
159
|
-
const search2 = await h.mem_search({ query: '
|
|
160
|
+
const search2 = await h.mem_search({ query: 'updated body', project: PROJECT_NAME, limit: 5 });
|
|
160
161
|
const res = parseRes(search2).results;
|
|
161
|
-
reporter.step('mem_search', { query: '
|
|
162
|
+
reporter.step('mem_search', { query: 'updated body', limit: 5 }, { results: res }, { highlight: ['title', 'snippet', 'score'] });
|
|
162
163
|
ok(Array.isArray(res), 'results array');
|
|
163
|
-
ok(res.length >= 1
|
|
164
|
+
ok(res.length >= 1, 'has results');
|
|
165
|
+
ok(res.length === 0 || res[0].snippet != null, 'snippets present');
|
|
164
166
|
|
|
165
167
|
reporter.endScenario(p, f);
|
|
166
168
|
return { passed: p, failed: f };
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
|
|
4
|
+
* Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
|
|
5
|
+
*/
|
|
6
|
+
import fs from 'fs';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import os from 'os';
|
|
9
|
+
import { loadConfig } from '../src/config.js';
|
|
10
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
11
|
+
import { search } from '../src/core/search/search.js';
|
|
12
|
+
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
13
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
14
|
+
|
|
15
|
+
const TARGET_SIZE = 100_000;
|
|
16
|
+
const NEEDLE = 'NeedleInHaystack100k';
|
|
17
|
+
|
|
18
|
+
function buildLargeBody(targetSize) {
|
|
19
|
+
const intro = `# Documento grande\n\nPárrafo inicial. ${NEEDLE} aquí.\n\n`;
|
|
20
|
+
const paragraph = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(20) + '\n\n';
|
|
21
|
+
const repeat = Math.ceil((targetSize - intro.length) / paragraph.length);
|
|
22
|
+
return intro + paragraph.repeat(Math.max(0, repeat));
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {object} config - resolved config (e.g. from run-tests)
|
|
27
|
+
* @returns {{ passed: number, failed: number }}
|
|
28
|
+
*/
|
|
29
|
+
export function runLargeDocumentTest(config) {
|
|
30
|
+
let passed = 0;
|
|
31
|
+
let failed = 0;
|
|
32
|
+
const ok = (cond, msg) => {
|
|
33
|
+
if (cond) passed++;
|
|
34
|
+
else failed++;
|
|
35
|
+
if (!cond) console.error('FAIL:', msg);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const docsDir = path.join(config.projectMemoryRoot, 'generated');
|
|
39
|
+
fs.mkdirSync(docsDir, { recursive: true });
|
|
40
|
+
|
|
41
|
+
const body = buildLargeBody(TARGET_SIZE);
|
|
42
|
+
const id = 'obs_large_100k';
|
|
43
|
+
const now = new Date().toISOString();
|
|
44
|
+
const frontmatter = {
|
|
45
|
+
id,
|
|
46
|
+
title: 'Documento 100k',
|
|
47
|
+
type: 'generated_note',
|
|
48
|
+
scope: 'project',
|
|
49
|
+
project: 'test',
|
|
50
|
+
topic_key: 'test/large-100k',
|
|
51
|
+
created_at: now
|
|
52
|
+
};
|
|
53
|
+
const raw = toMarkdown(frontmatter, body);
|
|
54
|
+
const docPath = path.join(docsDir, 'large-100k.md');
|
|
55
|
+
fs.writeFileSync(docPath, raw, 'utf8');
|
|
56
|
+
const stat = fs.statSync(docPath);
|
|
57
|
+
|
|
58
|
+
const size = raw.length;
|
|
59
|
+
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
|
+
|
|
61
|
+
indexDocument(config, docPath, raw, stat.mtimeMs);
|
|
62
|
+
|
|
63
|
+
const db = getDb(config);
|
|
64
|
+
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
|
65
|
+
ok(!!doc, 'documento en tabla documents');
|
|
66
|
+
ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
|
|
67
|
+
|
|
68
|
+
const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
|
|
69
|
+
ok(!!ftsRow, 'documento en FTS');
|
|
70
|
+
ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
|
|
71
|
+
|
|
72
|
+
const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
|
|
73
|
+
ok(results.length >= 1, 'búsqueda encuentra el documento');
|
|
74
|
+
ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
|
|
75
|
+
ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
|
|
76
|
+
|
|
77
|
+
return { passed, failed };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function main() {
|
|
81
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
|
|
82
|
+
const config = loadConfig({
|
|
83
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
84
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
85
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
86
|
+
});
|
|
87
|
+
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
88
|
+
|
|
89
|
+
const { passed, failed } = runLargeDocumentTest(config);
|
|
90
|
+
console.log(`Large document test: ${passed} passed, ${failed} failed`);
|
|
91
|
+
closeDb();
|
|
92
|
+
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
93
|
+
process.exitCode = failed > 0 ? 1 : 0;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (process.argv[1]?.endsWith('test-large-document.js')) {
|
|
97
|
+
main();
|
|
98
|
+
}
|