mixgram 1.3.7 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -345
- package/bin/mixgram.js +51 -160
- package/examples/embedding-demo.js +22 -142
- package/package.json +19 -19
- package/spec/bench-indexing.js +23 -211
- package/spec/bench-search.js +26 -212
- package/spec/run-tests.js +1 -214
- package/spec/scenarios/scenarios.js +277 -607
- package/spec/sdd.md +14 -988
- package/spec/tdd.md +15 -1232
- package/spec/test-large-document.js +20 -91
- package/src/config.js +57 -48
- package/src/core/documents/documents.js +62 -224
- package/src/core/embeddings/embedder.js +14 -54
- package/src/core/embeddings/queue.js +33 -106
- package/src/core/embeddings/vectorStore.js +30 -111
- package/src/core/embeddings/worker-process.js +23 -74
- package/src/core/indexing/indexer.js +51 -237
- package/src/core/indexing/parser.js +13 -114
- package/src/core/indexing/reindex.js +28 -137
- package/src/core/search/search.js +64 -158
- package/src/db/migrations/001_initial.sql +14 -63
- package/src/db/sqlite.js +19 -29
- package/src/fs/paths.js +11 -101
- package/src/fs/watcher.js +15 -48
- package/src/mcp/cli-adapter.js +32 -110
- package/src/mcp/server.js +64 -141
- package/src/mcp/tool-registry.js +21 -132
- package/src/mcp/tools.js +27 -199
- package/src/utils/markdown.js +15 -82
- package/spec/fixtures/realistic-content.js +0 -106
- package/spec/reporter.js +0 -127
- package/spec/run-tests-runner.js +0 -40
- package/src/core/embeddings/worker.js +0 -27
- package/src/core/prompts/prompts.js +0 -15
- package/src/core/sessions/sessions.js +0 -27
- package/src/utils/ids.js +0 -27
package/spec/bench-indexing.js
CHANGED
|
@@ -1,219 +1,31 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Benchmark: indexing and reindexing with synthetic corpus.
|
|
4
|
-
* Breaks down timings (clear, listPaths, indexing) and compares with/without embeddings.
|
|
5
|
-
* Opt-in; not run by npm test.
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* npm run bench:indexing # indexing only
|
|
9
|
-
* npm run bench:embed # indexing + embedding queue processing (slow)
|
|
10
|
-
*/
|
|
11
2
|
import fs from 'fs';
|
|
12
|
-
import path from 'path';
|
|
13
3
|
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
14
5
|
import { loadConfig } from '../src/config.js';
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
incrementalReindex,
|
|
18
|
-
clearIndex,
|
|
19
|
-
listMarkdownPaths
|
|
20
|
-
} from '../src/core/indexing/reindex.js';
|
|
21
|
-
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
6
|
+
import { saveDocument } from '../src/core/documents/documents.js';
|
|
7
|
+
import { reindex } from '../src/core/indexing/reindex.js';
|
|
22
8
|
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
23
|
-
import {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const configNoEmb = {
|
|
36
|
-
...baseConfig,
|
|
37
|
-
embeddings: { ...baseConfig.embeddings, enabled: false }
|
|
38
|
-
};
|
|
39
|
-
const configEmb = {
|
|
40
|
-
...baseConfig,
|
|
41
|
-
embeddings: { ...baseConfig.embeddings, enabled: true }
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
/** Frontmatter with same keys the app writes so the indexer parses the same. */
|
|
45
|
-
function writeDoc(dir, id, title, body) {
|
|
46
|
-
const now = new Date().toISOString();
|
|
47
|
-
const frontmatter = {
|
|
48
|
-
id,
|
|
49
|
-
title,
|
|
50
|
-
type: 'generated_note',
|
|
51
|
-
scope: 'project',
|
|
52
|
-
project: 'bench',
|
|
53
|
-
topic_key: `bench/${id}`,
|
|
54
|
-
created_at: now
|
|
55
|
-
};
|
|
56
|
-
const raw = toMarkdown(frontmatter, body);
|
|
57
|
-
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
58
|
-
fs.writeFileSync(filePath, raw, 'utf8');
|
|
59
|
-
return filePath;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
function buildLargeBody(size) {
|
|
63
|
-
const intro = '# Big\n\nNeedleInHaystack\n\n';
|
|
64
|
-
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
65
|
-
return intro + filler;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
const NUM_SMALL = 3;
|
|
69
|
-
const LARGE_SIZE = 8_000;
|
|
70
|
-
const RUN_EMBED_PROCESSING = process.argv.includes('--embed');
|
|
71
|
-
|
|
72
|
-
console.log('Corpus: %d small .md + 1 large .md (~%d chars)', NUM_SMALL, LARGE_SIZE);
|
|
73
|
-
console.log('Mode: %s\n', RUN_EMBED_PROCESSING ? 'indexing + embedding queue processing' : 'indexing only');
|
|
74
|
-
|
|
75
|
-
// --- Create corpus ---
|
|
76
|
-
for (let i = 0; i < NUM_SMALL; i++) {
|
|
77
|
-
writeDoc(baseConfig.projectMemoryRoot, `bench${String(i).padStart(5, '0')}`, `Small ${i}`, `Content for small doc ${i}.\n`);
|
|
78
|
-
}
|
|
79
|
-
const largePath = writeDoc(baseConfig.projectMemoryRoot, 'bench99999', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Full reindex with breakdown: clearIndex, listMarkdownPaths, index loop.
|
|
83
|
-
* @returns {{ total: number, clear: number, list: number, indexLoop: number, indexed: number }}
|
|
84
|
-
*/
|
|
85
|
-
function fullReindexWithBreakdown(config) {
|
|
86
|
-
let t0, t1;
|
|
87
|
-
t0 = performance.now();
|
|
88
|
-
clearIndex(config);
|
|
89
|
-
t1 = performance.now();
|
|
90
|
-
const clearMs = t1 - t0;
|
|
91
|
-
|
|
92
|
-
t0 = performance.now();
|
|
93
|
-
const paths = listMarkdownPaths(config);
|
|
94
|
-
t1 = performance.now();
|
|
95
|
-
const listMs = t1 - t0;
|
|
96
|
-
|
|
97
|
-
let indexed = 0;
|
|
98
|
-
t0 = performance.now();
|
|
99
|
-
for (const filePath of paths) {
|
|
100
|
-
try {
|
|
101
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
102
|
-
const stat = fs.statSync(filePath);
|
|
103
|
-
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
104
|
-
indexed++;
|
|
105
|
-
} catch (_) {}
|
|
106
|
-
}
|
|
107
|
-
t1 = performance.now();
|
|
108
|
-
const indexLoopMs = t1 - t0;
|
|
109
|
-
const totalMs = clearMs + listMs + indexLoopMs;
|
|
110
|
-
|
|
111
|
-
return { total: totalMs, clear: clearMs, list: listMs, indexLoop: indexLoopMs, indexed };
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
function ms(n) {
|
|
115
|
-
return Math.round(n);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
function section(title) {
|
|
119
|
-
console.log('\n--- %s ---', title);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
async function runBenchmark() {
|
|
123
|
-
// ========== Without embeddings ==========
|
|
124
|
-
section('Without embeddings');
|
|
125
|
-
|
|
126
|
-
let b = fullReindexWithBreakdown(configNoEmb);
|
|
127
|
-
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
128
|
-
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
129
|
-
|
|
130
|
-
let t0 = performance.now();
|
|
131
|
-
const inc0 = incrementalReindex(configNoEmb);
|
|
132
|
-
let t1 = performance.now();
|
|
133
|
-
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
134
|
-
ms(t1 - t0), inc0.scanned, inc0.skipped, inc0.indexed, inc0.removed);
|
|
135
|
-
|
|
136
|
-
fs.appendFileSync(largePath, '\n\nAppended line.');
|
|
137
|
-
|
|
138
|
-
t0 = performance.now();
|
|
139
|
-
const inc1 = incrementalReindex(configNoEmb);
|
|
140
|
-
t1 = performance.now();
|
|
141
|
-
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
142
|
-
ms(t1 - t0), inc1.scanned, inc1.skipped, inc1.indexed, inc1.removed);
|
|
143
|
-
|
|
144
|
-
// Final state without embeddings
|
|
145
|
-
const db = getDb(baseConfig);
|
|
146
|
-
const docsCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
147
|
-
const ftsCount = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
|
|
148
|
-
console.log('State: %d documents, %d FTS rows.', docsCount, ftsCount);
|
|
149
|
-
|
|
150
|
-
// ========== With embeddings (indexing + queue) ==========
|
|
151
|
-
section('With embeddings (indexing + queue)');
|
|
152
|
-
|
|
153
|
-
clearIndex(configEmb);
|
|
154
|
-
b = fullReindexWithBreakdown(configEmb);
|
|
155
|
-
// Allow enqueue promises to settle
|
|
156
|
-
await new Promise((r) => setImmediate(r));
|
|
157
|
-
await new Promise((r) => setImmediate(r));
|
|
158
|
-
|
|
159
|
-
const jobsEnqueued = db.prepare('SELECT COUNT(*) AS c FROM embedding_jobs').get()?.c ?? 0;
|
|
160
|
-
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
161
|
-
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
162
|
-
console.log('Jobs in queue: %d', jobsEnqueued);
|
|
163
|
-
|
|
164
|
-
t0 = performance.now();
|
|
165
|
-
const inc0emb = incrementalReindex(configEmb);
|
|
166
|
-
t1 = performance.now();
|
|
167
|
-
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
168
|
-
ms(t1 - t0), inc0emb.scanned, inc0emb.skipped, inc0emb.indexed, inc0emb.removed);
|
|
169
|
-
|
|
170
|
-
fs.appendFileSync(largePath, '\n\nAnother line.');
|
|
171
|
-
|
|
172
|
-
t0 = performance.now();
|
|
173
|
-
const inc1emb = incrementalReindex(configEmb);
|
|
174
|
-
t1 = performance.now();
|
|
175
|
-
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
176
|
-
ms(t1 - t0), inc1emb.scanned, inc1emb.skipped, inc1emb.indexed, inc1emb.removed);
|
|
177
|
-
|
|
178
|
-
// ========== Embedding processing (optional) ==========
|
|
179
|
-
if (RUN_EMBED_PROCESSING && configEmb.embeddings?.enabled) {
|
|
180
|
-
section('Embedding processing (queue → model → vector store)');
|
|
181
|
-
|
|
182
|
-
const pendingBefore = db.prepare("SELECT COUNT(*) AS c FROM embedding_jobs WHERE status = 'pending'").get()?.c ?? 0;
|
|
183
|
-
t0 = performance.now();
|
|
184
|
-
let processed = 0;
|
|
185
|
-
while (await processNextJob(configEmb)) {
|
|
186
|
-
processed++;
|
|
187
|
-
}
|
|
188
|
-
t1 = performance.now();
|
|
189
|
-
console.log('Process queue: %d ms (jobs processed=%d, pending before=%d)',
|
|
190
|
-
ms(t1 - t0), processed, pendingBefore);
|
|
9
|
+
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
10
|
+
|
|
11
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-index-bench-'));
|
|
12
|
+
const config = loadConfig({ collections: { general: path.join(root, 'docs') }, sqlitePath: path.join(root, 'index.db'), embeddings: { enabled: process.argv.includes('--embed') } });
|
|
13
|
+
try {
|
|
14
|
+
for (let i = 0; i < 100; i++) saveDocument(config, { title: `Document ${i}`, content: 'Example knowledge. '.repeat(i === 99 ? 6000 : 30) });
|
|
15
|
+
for (const full of [true, false]) {
|
|
16
|
+
const start = performance.now();
|
|
17
|
+
const result = reindex(config, { full });
|
|
18
|
+
console.log(JSON.stringify({ full, milliseconds: performance.now() - start, ...result }));
|
|
19
|
+
if (result.errors.length) throw new Error('Indexing failed');
|
|
191
20
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
console.log('Documents: %d, FTS rows: %d', docsFinal, ftsFinal);
|
|
199
|
-
if (jobsFinal.length) {
|
|
200
|
-
console.log('Embedding jobs: %s', jobsFinal.map((r) => `${r.status}=${r.c}`).join(', '));
|
|
21
|
+
if (config.embeddings.enabled) {
|
|
22
|
+
const start = performance.now();
|
|
23
|
+
while (await processNextJob(config)) {}
|
|
24
|
+
console.log(JSON.stringify({ embeddingMilliseconds: performance.now() - start,
|
|
25
|
+
jobs: getDb(config).prepare('SELECT status,COUNT(*) AS count FROM embedding_jobs GROUP BY status').all() }));
|
|
26
|
+
if (getDb(config).prepare("SELECT 1 FROM embedding_jobs WHERE status='failed'").get()) throw new Error('Embedding jobs failed');
|
|
201
27
|
}
|
|
28
|
+
} finally {
|
|
29
|
+
closeDb(config);
|
|
30
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
202
31
|
}
|
|
203
|
-
|
|
204
|
-
runBenchmark()
|
|
205
|
-
.then(() => {
|
|
206
|
-
closeDb();
|
|
207
|
-
try {
|
|
208
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
209
|
-
} catch (_) {}
|
|
210
|
-
console.log('\nDone.\n');
|
|
211
|
-
})
|
|
212
|
-
.catch((err) => {
|
|
213
|
-
closeDb();
|
|
214
|
-
try {
|
|
215
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
216
|
-
} catch (_) {}
|
|
217
|
-
console.error(err);
|
|
218
|
-
process.exit(1);
|
|
219
|
-
});
|
package/spec/bench-search.js
CHANGED
|
@@ -1,221 +1,35 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Benchmark: search (FTS and optional semantic) over synthetic corpus.
|
|
4
|
-
* Compares short vs long documents, FTS-only vs with embeddings.
|
|
5
|
-
* Opt-in; not run by npm test.
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* npm run bench:search # FTS only
|
|
9
|
-
* npm run bench:search:embed # FTS + semantic search (needs embed model)
|
|
10
|
-
*/
|
|
11
2
|
import fs from 'fs';
|
|
12
|
-
import path from 'path';
|
|
13
3
|
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
14
5
|
import { loadConfig } from '../src/config.js';
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
6
|
+
import { saveDocument } from '../src/core/documents/documents.js';
|
|
7
|
+
import { search } from '../src/core/search/search.js';
|
|
17
8
|
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
18
|
-
import {
|
|
19
|
-
import { closeDb
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
|
|
30
|
-
|
|
31
|
-
const configNoEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: false } };
|
|
32
|
-
const configEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: true } };
|
|
33
|
-
|
|
34
|
-
const RUN_SEMANTIC = process.argv.includes('--embed');
|
|
35
|
-
|
|
36
|
-
function writeDoc(dir, id, title, body) {
|
|
37
|
-
const now = new Date().toISOString();
|
|
38
|
-
const frontmatter = {
|
|
39
|
-
id,
|
|
40
|
-
title,
|
|
41
|
-
type: 'generated_note',
|
|
42
|
-
scope: 'project',
|
|
43
|
-
project: 'bench',
|
|
44
|
-
topic_key: `bench/${id}`,
|
|
45
|
-
created_at: now
|
|
46
|
-
};
|
|
47
|
-
const raw = toMarkdown(frontmatter, body);
|
|
48
|
-
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
49
|
-
fs.writeFileSync(filePath, raw, 'utf8');
|
|
50
|
-
return filePath;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function buildLargeBody(size, needle = 'NeedleInHaystack') {
|
|
54
|
-
const intro = `# Big\n\n${needle}\n\n`;
|
|
55
|
-
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
56
|
-
return intro + filler;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
const NUM_SMALL = 5;
|
|
60
|
-
const LARGE_SIZE = 10_000;
|
|
61
|
-
const WARMUP = 2;
|
|
62
|
-
const RUNS = 5;
|
|
63
|
-
|
|
64
|
-
function ms(n) {
|
|
65
|
-
return Math.round(n);
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function section(title) {
|
|
69
|
-
console.log('\n--- %s ---', title);
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function median(arr) {
|
|
73
|
-
const a = [...arr].sort((x, y) => x - y);
|
|
74
|
-
const m = Math.floor(a.length / 2);
|
|
75
|
-
return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
/**
|
|
79
|
-
* Time a sync function over multiple runs; report median ms.
|
|
80
|
-
*/
|
|
81
|
-
function timeSync(fn, runs = RUNS, warmup = WARMUP) {
|
|
82
|
-
for (let i = 0; i < warmup; i++) fn();
|
|
83
|
-
const times = [];
|
|
84
|
-
for (let i = 0; i < runs; i++) {
|
|
85
|
-
const t0 = performance.now();
|
|
86
|
-
fn();
|
|
87
|
-
times.push(performance.now() - t0);
|
|
88
|
-
}
|
|
89
|
-
return median(times);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Time an async function over multiple runs; report median ms.
|
|
94
|
-
*/
|
|
95
|
-
async function timeAsync(fn, runs = RUNS, warmup = WARMUP) {
|
|
96
|
-
for (let i = 0; i < warmup; i++) await fn();
|
|
97
|
-
const times = [];
|
|
98
|
-
for (let i = 0; i < runs; i++) {
|
|
99
|
-
const t0 = performance.now();
|
|
100
|
-
await fn();
|
|
101
|
-
times.push(performance.now() - t0);
|
|
102
|
-
}
|
|
103
|
-
return median(times);
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
async function runBenchmark() {
|
|
107
|
-
console.log('Corpus: %d short .md + 1 large .md (~%d chars). Mode: %s\n', NUM_SMALL, LARGE_SIZE, RUN_SEMANTIC ? 'FTS + semantic' : 'FTS only');
|
|
108
|
-
|
|
109
|
-
// --- Corpus: short docs + one large doc with needle ---
|
|
110
|
-
for (let i = 0; i < NUM_SMALL; i++) {
|
|
111
|
-
writeDoc(baseConfig.projectMemoryRoot, `short_${i}`, `Short ${i}`, `Content for short doc ${i}. Some terms: alpha beta gamma.\n`);
|
|
9
|
+
import { getEmbedder } from '../src/core/embeddings/embedder.js';
|
|
10
|
+
import { closeDb } from '../src/db/sqlite.js';
|
|
11
|
+
|
|
12
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-search-bench-'));
|
|
13
|
+
const config = loadConfig({ collections: { general: path.join(root, 'docs') }, sqlitePath: path.join(root, 'index.db'), embeddings: { enabled: process.argv.includes('--embed') } });
|
|
14
|
+
try {
|
|
15
|
+
for (let i = 0; i < 100; i++) saveDocument(config, { title: `Document ${i}`, content: 'Memory contains reusable knowledge. '.repeat(i === 99 ? 3000 : 10), tags: [i % 2 ? 'odd' : 'even'] });
|
|
16
|
+
if (config.embeddings.enabled) {
|
|
17
|
+
while (await processNextJob(config)) {}
|
|
18
|
+
const embedder = await getEmbedder(config);
|
|
19
|
+
config.getQueryEmbedding = text => embedder.embed(text);
|
|
112
20
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
// ========== FTS search ==========
|
|
121
|
-
section('FTS search (no embeddings)');
|
|
122
|
-
|
|
123
|
-
const ftsShort = timeSync(() => {
|
|
124
|
-
searchFts(configNoEmb, { query: 'Content', limit: 10 });
|
|
125
|
-
});
|
|
126
|
-
console.log('Query "Content" (short, matches short docs): median %d ms', ms(ftsShort));
|
|
127
|
-
|
|
128
|
-
const ftsNeedle = timeSync(() => {
|
|
129
|
-
searchFts(configNoEmb, { query: 'NeedleInHaystack', limit: 10 });
|
|
130
|
-
});
|
|
131
|
-
console.log('Query "NeedleInHaystack" (needle in large): median %d ms', ms(ftsNeedle));
|
|
132
|
-
|
|
133
|
-
const ftsPhrase = timeSync(() => {
|
|
134
|
-
searchFts(configNoEmb, { query: 'alpha beta gamma', limit: 10 });
|
|
135
|
-
});
|
|
136
|
-
console.log('Query "alpha beta gamma" (phrase in short): median %d ms', ms(ftsPhrase));
|
|
137
|
-
|
|
138
|
-
const ftsRecent = timeSync(() => {
|
|
139
|
-
getRecentContext(configNoEmb, { limit: 10 });
|
|
140
|
-
});
|
|
141
|
-
console.log('getRecentContext(limit=10): median %d ms', ms(ftsRecent));
|
|
142
|
-
|
|
143
|
-
// ========== Semantic search (optional) ==========
|
|
144
|
-
if (RUN_SEMANTIC) {
|
|
145
|
-
section('With embeddings: index + process queue');
|
|
146
|
-
|
|
147
|
-
clearIndex(configEmb);
|
|
148
|
-
const paths = listMarkdownPaths(configEmb);
|
|
149
|
-
for (const filePath of paths) {
|
|
150
|
-
try {
|
|
151
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
152
|
-
const stat = fs.statSync(filePath);
|
|
153
|
-
indexDocument(configEmb, filePath, raw, stat.mtimeMs);
|
|
154
|
-
} catch (_) {}
|
|
21
|
+
for (const options of [{ query: 'knowledge' }, { query: 'knowledge', tags: ['even'], offset: 10 }, {}]) {
|
|
22
|
+
const samples = [];
|
|
23
|
+
for (let i = 0; i < 7; i++) {
|
|
24
|
+
const start = performance.now();
|
|
25
|
+
const result = await search(config, options);
|
|
26
|
+
if (result.degraded) throw new Error(result.reason);
|
|
27
|
+
if (i >= 2) samples.push(performance.now() - start);
|
|
155
28
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
let processed = 0;
|
|
160
|
-
while (await processNextJob(configEmb)) processed++;
|
|
161
|
-
const vecCount = db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name LIKE 'vec_%'").get()?.c ?? 0;
|
|
162
|
-
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
163
|
-
const vecRows = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
164
|
-
console.log('Embedding jobs processed: %d. Vector rows: %d.', processed, vecRows);
|
|
165
|
-
|
|
166
|
-
section('Semantic search (query embed + vector KNN)');
|
|
167
|
-
|
|
168
|
-
const { getEmbedder } = await import('../src/core/embeddings/embedder.js');
|
|
169
|
-
const { search: searchVector } = await import('../src/core/embeddings/vectorStore.js');
|
|
170
|
-
const embedder = await getEmbedder(configEmb);
|
|
171
|
-
if (!embedder) {
|
|
172
|
-
console.log('Embedder not available; skipping semantic timings.');
|
|
173
|
-
} else {
|
|
174
|
-
const embedTime = await timeAsync(async () => {
|
|
175
|
-
await embedder.embed('NeedleInHaystack');
|
|
176
|
-
});
|
|
177
|
-
console.log('Query embedding (single phrase): median %d ms', ms(embedTime));
|
|
178
|
-
|
|
179
|
-
const vectorSearchTime = await timeAsync(async () => {
|
|
180
|
-
const vec = await embedder.embed('NeedleInHaystack');
|
|
181
|
-
await searchVector(configEmb, vec, 20);
|
|
182
|
-
});
|
|
183
|
-
console.log('Embed + vector search (k=20): median %d ms', ms(vectorSearchTime));
|
|
184
|
-
|
|
185
|
-
const precomputedVec = await embedder.embed('query');
|
|
186
|
-
const vectorOnlyTime = await timeAsync(async () => {
|
|
187
|
-
await searchVector(configEmb, precomputedVec, 10);
|
|
188
|
-
});
|
|
189
|
-
console.log('Vector search only (k=10, no embed): median %d ms', ms(vectorOnlyTime));
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
section('FTS with embeddings enabled (same as FTS-only path)');
|
|
193
|
-
const ftsWithEmb = timeSync(() => {
|
|
194
|
-
searchFts(configEmb, { query: 'Content', limit: 10 });
|
|
195
|
-
});
|
|
196
|
-
console.log('Query "Content": median %d ms', ms(ftsWithEmb));
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
section('Summary');
|
|
200
|
-
console.log('FTS: short query, needle-in-large, phrase, getRecentContext.');
|
|
201
|
-
if (RUN_SEMANTIC) {
|
|
202
|
-
console.log('Semantic: query embed + vector KNN (npm run bench:search:embed).');
|
|
29
|
+
samples.sort((a,b) => a-b);
|
|
30
|
+
console.log(JSON.stringify({ options, medianMilliseconds: samples[2] }));
|
|
203
31
|
}
|
|
32
|
+
} finally {
|
|
33
|
+
closeDb(config);
|
|
34
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
204
35
|
}
|
|
205
|
-
|
|
206
|
-
runBenchmark()
|
|
207
|
-
.then(() => {
|
|
208
|
-
closeDb();
|
|
209
|
-
try {
|
|
210
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
211
|
-
} catch (_) {}
|
|
212
|
-
console.log('\nDone.\n');
|
|
213
|
-
})
|
|
214
|
-
.catch((err) => {
|
|
215
|
-
closeDb();
|
|
216
|
-
try {
|
|
217
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
218
|
-
} catch (_) {}
|
|
219
|
-
console.error(err);
|
|
220
|
-
process.exit(1);
|
|
221
|
-
});
|