mixgram 1.1.4 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -14
- package/bin/mixgram.js +16 -9
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +16 -12
- package/spec/scenarios/scenarios.js +6 -4
- package/spec/test-large-document.js +98 -0
- package/src/config.js +8 -5
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/embeddings/worker-process.js +79 -0
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -106
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/mcp/server.js +67 -2
- package/src/utils/markdown.js +42 -2
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ MCP memory server compatible with [Engram](https://github.com/gentleman-programm
|
|
|
9
9
|
### Advantages over Engram
|
|
10
10
|
|
|
11
11
|
- **Visible, editable docs** — As you advance with agentic development you see the actual documentation files on disk. You can open them, read them, and edit them manually; the agent’s memory is not hidden in a black box.
|
|
12
|
-
- **Versioned** — Memory lives in Markdown under your repo (e.g. `
|
|
12
|
+
- **Versioned** — Memory lives in Markdown under your repo (e.g. `docs/<type>/` by default for project scope) or in a shared home (e.g. `home/memory/` for cross-project). Commit, branch, and diff as with any other docs.
|
|
13
13
|
- **No semantic deps required** — Core experience is text-only (FTS5). No need to install embeddings or vector libs unless you opt in.
|
|
14
14
|
|
|
15
15
|
---
|
|
@@ -57,35 +57,35 @@ Arguments are passed as `--key value`. Booleans use `--flag` or `--no-flag`. Arr
|
|
|
57
57
|
| `--embeddings` | `MIXGRAM_EMBEDDINGS_ENABLED` | Enable optional semantic (hybrid) search. |
|
|
58
58
|
| `--watch` | `MIXGRAM_WATCH` | Watch files and reindex on change. |
|
|
59
59
|
| `--home <path>` | `MIXGRAM_HOME` | Home memory root (cross-project). |
|
|
60
|
-
| `--project-memory <path>` | `MIXGRAM_PROJECT_MEMORY` | Project memory root (default: `./
|
|
60
|
+
| `--project-memory <path>` | `MIXGRAM_PROJECT_MEMORY` | Project memory root (default: `./docs`, relative to repo). |
|
|
61
61
|
| `--projects <path>` | `MIXGRAM_PROJECTS` | Projects root (legacy). |
|
|
62
62
|
| `--sqlite-path <path>` | `MIXGRAM_SQLITE_PATH` | SQLite index path. |
|
|
63
63
|
|
|
64
64
|
**Example — Cursor**
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
Create `.cursor/mcp.json` in your repo (recommended, and required for Cursor Cloud Agents) or add the same config to `~/.cursor/mcp.json` (or run `mixgram setup cursor`):
|
|
67
67
|
|
|
68
68
|
```json
|
|
69
69
|
{
|
|
70
70
|
"mcpServers": {
|
|
71
71
|
"mixgram": {
|
|
72
72
|
"command": "mixgram",
|
|
73
|
-
"args": ["mcp", "--project-memory", "${workspaceFolder}/
|
|
73
|
+
"args": ["mcp", "--project-memory", "${workspaceFolder}/docs"]
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
}
|
|
77
77
|
```
|
|
78
78
|
|
|
79
|
-
With embeddings and custom paths
|
|
79
|
+
With embeddings and custom paths:
|
|
80
80
|
|
|
81
81
|
```json
|
|
82
82
|
"mixgram": {
|
|
83
83
|
"command": "mixgram",
|
|
84
|
-
"args": ["mcp", "--
|
|
84
|
+
"args": ["mcp", "--project-memory", "${workspaceFolder}/docs", "--embeddings", "--home", "/data/memory"]
|
|
85
85
|
}
|
|
86
86
|
```
|
|
87
87
|
|
|
88
|
-
|
|
88
|
+
Restart Cursor after changing config.
|
|
89
89
|
|
|
90
90
|
---
|
|
91
91
|
|
|
@@ -115,7 +115,7 @@ Example **`.mixgram/config.json`** (project or home):
|
|
|
115
115
|
```json
|
|
116
116
|
{
|
|
117
117
|
"homeMemoryRoot": "~/.mixgram/docs",
|
|
118
|
-
"projectMemoryRoot": "./
|
|
118
|
+
"projectMemoryRoot": "./docs",
|
|
119
119
|
"sqlitePath": "~/.mixgram/index.db",
|
|
120
120
|
"embeddings": {
|
|
121
121
|
"enabled": true
|
|
@@ -128,7 +128,7 @@ Paths in the config file are relative to the config file’s directory (project)
|
|
|
128
128
|
| Option | Default | Description |
|
|
129
129
|
|--------|---------|-------------|
|
|
130
130
|
| `homeMemoryRoot` | `~/.mixgram/docs` | Cross-project memory (Markdown files). |
|
|
131
|
-
| `projectMemoryRoot` | `./
|
|
131
|
+
| `projectMemoryRoot` | `./docs` | Project memory in the current repo (e.g. `docs/architecture/`, `docs/decisions/`). You can use another folder name, e.g. `./specs` for `specs/architecture/`. |
|
|
132
132
|
| `sqlitePath` | `~/.mixgram/index.db` | SQLite index (FTS5 + optional vectors). |
|
|
133
133
|
| `watch` | `true` | Watch files and reindex on change. |
|
|
134
134
|
| `indexing.reindexOnStartup` | `true` | Full reindex when server starts. |
|
|
@@ -143,7 +143,7 @@ import { run } from 'mixgram/src/mcp/server.js';
|
|
|
143
143
|
|
|
144
144
|
await run({
|
|
145
145
|
homeMemoryRoot: '/data/memory',
|
|
146
|
-
projectMemoryRoot: '/path/to/repo/
|
|
146
|
+
projectMemoryRoot: '/path/to/repo/docs',
|
|
147
147
|
sqlitePath: '/data/.mixgram/index.db',
|
|
148
148
|
watch: true
|
|
149
149
|
});
|
|
@@ -169,7 +169,7 @@ Core flow: save Markdown-backed documents, search with FTS5, get context.
|
|
|
169
169
|
}
|
|
170
170
|
```
|
|
171
171
|
|
|
172
|
-
- **scope** `project` → file under `<repo>/
|
|
172
|
+
- **scope** `project` → file under `<repo>/docs/<type>/...` (e.g. `docs/architecture/`, `docs/decisions/`).
|
|
173
173
|
- **scope** `home` → file under `home/memory/...` (cross-project).
|
|
174
174
|
- Same `topic_key` + scope + project → **update** existing doc; otherwise create.
|
|
175
175
|
|
|
@@ -238,12 +238,12 @@ Useful after cloning a repo or editing Markdown by hand:
|
|
|
238
238
|
{ "full": true }
|
|
239
239
|
```
|
|
240
240
|
|
|
241
|
-
- `full: true` — rebuild entire index from `home/**/*.md` and `
|
|
241
|
+
- `full: true` — rebuild entire index from `home/**/*.md` and `docs/**/*.md` (project memory in repo).
|
|
242
242
|
- `full: false` (default) — incremental (by mtime).
|
|
243
243
|
|
|
244
244
|
Manual edit example:
|
|
245
245
|
|
|
246
|
-
1. Edit a `.md` file under `home/memory` or `
|
|
246
|
+
1. Edit a `.md` file under `home/memory` or `docs/<type>/` in your repo.
|
|
247
247
|
2. Call `mem_reindex({ full: true })` (or run with `watch: true` so changes are picked up).
|
|
248
248
|
3. `mem_search` and `mem_get_observation` reflect the new content.
|
|
249
249
|
|
|
@@ -280,7 +280,7 @@ When `embeddings.enabled` is `true` and the optional embedding stack is availabl
|
|
|
280
280
|
|
|
281
281
|
- **Saves** are durable and searchable by text immediately; embeddings are queued and processed in the background.
|
|
282
282
|
- **Search** becomes **hybrid**: FTS + vector similarity, blended with `search.ftsWeight` and `search.semanticWeight` (e.g. 0.7 and 0.3).
|
|
283
|
-
- A **worker** runs in
|
|
283
|
+
- A **worker** runs in a separate process and processes the embedding queue (e.g. every 2 seconds). Query embeddings for hybrid search are also computed in that process via IPC, so the main MCP server never loads `@huggingface/transformers` and is not affected by native/ONNX crashes in the embedding stack.
|
|
284
284
|
|
|
285
285
|
### Enable embeddings
|
|
286
286
|
|
package/bin/mixgram.js
CHANGED
|
@@ -12,6 +12,10 @@ import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../
|
|
|
12
12
|
import fs from 'fs';
|
|
13
13
|
import path from 'path';
|
|
14
14
|
import os from 'os';
|
|
15
|
+
import { fileURLToPath } from 'url';
|
|
16
|
+
|
|
17
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
18
|
+
const PKG = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8'));
|
|
15
19
|
|
|
16
20
|
const SUBCOMMAND = process.argv[2];
|
|
17
21
|
const ARG = process.argv[3];
|
|
@@ -102,8 +106,6 @@ const MIXGRAM_ENTRY = {
|
|
|
102
106
|
|
|
103
107
|
const CURSOR_MIXGRAM_ENTRY = {
|
|
104
108
|
command: 'mixgram',
|
|
105
|
-
// Cursor supports interpolation in args, so force project-scoped docs into
|
|
106
|
-
// the active workspace without relying on cwd resolution.
|
|
107
109
|
args: ['mcp', '--project-memory', '${workspaceFolder}/docs']
|
|
108
110
|
};
|
|
109
111
|
|
|
@@ -216,6 +218,7 @@ Commands:
|
|
|
216
218
|
mcp [options] Run the MCP server (stdio). Use this in your client config.
|
|
217
219
|
setup <client> Register Mixgram with an MCP client.
|
|
218
220
|
help [tool] Show help; with optional tool name, show options for that tool.
|
|
221
|
+
-v, --version Print version from package and exit.
|
|
219
222
|
<tool> [options] Run an MCP tool by name. Tools:
|
|
220
223
|
${toolLines}
|
|
221
224
|
|
|
@@ -223,21 +226,20 @@ mcp options (and env / config file):
|
|
|
223
226
|
--config <path> Config file (default: ./.mixgram/config.json or ~/.mixgram/config.json)
|
|
224
227
|
--embeddings Enable semantic search (or embeddings.enabled in config)
|
|
225
228
|
--watch Watch files and reindex on change
|
|
226
|
-
--home <path> Home memory root (default:
|
|
227
|
-
--project-memory <path> Project memory root (default: ./
|
|
228
|
-
--
|
|
229
|
-
--sqlite-path <path> SQLite index path (default: ./.mixgram/index.db)
|
|
229
|
+
--home <path> Home memory root (default: ~/.mixgram/docs)
|
|
230
|
+
--project-memory <path> Project memory root (default: ./docs, relative to repo)
|
|
231
|
+
--sqlite-path <path> SQLite index path (default: ~/.mixgram/index.db)
|
|
230
232
|
|
|
231
233
|
Env: MIXGRAM_CONFIG, MIXGRAM_EMBEDDINGS_ENABLED, MIXGRAM_HOME, MIXGRAM_PROJECT_MEMORY,
|
|
232
|
-
|
|
234
|
+
MIXGRAM_SQLITE_PATH, MIXGRAM_WATCH
|
|
233
235
|
|
|
234
236
|
Setup targets:
|
|
235
237
|
cursor Cursor IDE
|
|
236
238
|
gemini-cli Gemini CLI
|
|
237
239
|
codex Codex
|
|
238
240
|
|
|
239
|
-
Example (Cursor): "mixgram": { "command": "mixgram", "args": ["mcp"] }
|
|
240
|
-
With embeddings: "args": ["mcp", "--embeddings"]
|
|
241
|
+
Example (Cursor): "mixgram": { "command": "mixgram", "args": ["mcp", "--project-memory", "${workspaceFolder}/docs"] }
|
|
242
|
+
With embeddings: "args": ["mcp", "--project-memory", "${workspaceFolder}/docs", "--embeddings"]
|
|
241
243
|
Config file: .mixgram/config.json or ~/.mixgram/config.json
|
|
242
244
|
|
|
243
245
|
npm install -g mixgram
|
|
@@ -245,6 +247,11 @@ Config file: .mixgram/config.json or ~/.mixgram/config.json
|
|
|
245
247
|
}
|
|
246
248
|
|
|
247
249
|
async function main() {
|
|
250
|
+
if (SUBCOMMAND === '-v' || SUBCOMMAND === '--version') {
|
|
251
|
+
console.log(PKG.version);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
|
|
248
255
|
if (SUBCOMMAND === 'mcp') {
|
|
249
256
|
const { overrides, baseDir, projectBaseDir } = loadCliConfig();
|
|
250
257
|
await run(overrides, baseDir, projectBaseDir)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixgram",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "MCP memory server with Markdown source of truth and SQLite FTS5.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
@@ -33,19 +33,32 @@
|
|
|
33
33
|
"scripts": {
|
|
34
34
|
"start": "node index.js",
|
|
35
35
|
"test": "node spec/run-tests.js",
|
|
36
|
-
"test:visual": "node spec/run-tests.js --visual"
|
|
36
|
+
"test:visual": "node spec/run-tests.js --visual",
|
|
37
|
+
"test:large": "node spec/test-large-document.js",
|
|
38
|
+
"bench:indexing": "node spec/bench-indexing.js",
|
|
39
|
+
"bench:embed": "node spec/bench-indexing.js --embed",
|
|
40
|
+
"bench:search": "node spec/bench-search.js",
|
|
41
|
+
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
42
|
+
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
43
|
+
"duplication": "jscpd src",
|
|
44
|
+
"deadcode": "knip",
|
|
45
|
+
"quality": "npm run duplication && npm run deadcode"
|
|
37
46
|
},
|
|
38
47
|
"dependencies": {
|
|
39
48
|
"@huggingface/transformers": "^3.4.2",
|
|
40
49
|
"@modelcontextprotocol/sdk": "^1.0.0",
|
|
41
50
|
"better-sqlite3": "^12.6.2",
|
|
42
51
|
"chokidar": "^4.0.1",
|
|
52
|
+
"glob": "^13.0.6",
|
|
43
53
|
"gray-matter": "^4.0.3",
|
|
44
|
-
"glob": "^13.0.0",
|
|
45
54
|
"remark-parse": "^11.0.0",
|
|
46
55
|
"sqlite-vec": "^0.1.6",
|
|
47
56
|
"unified": "^11.0.5",
|
|
48
57
|
"unist-util-visit": "^5.0.0",
|
|
49
58
|
"zod": "^3.23.0"
|
|
59
|
+
},
|
|
60
|
+
"devDependencies": {
|
|
61
|
+
"jscpd": "^4.0.8",
|
|
62
|
+
"knip": "^5.86.0"
|
|
50
63
|
}
|
|
51
64
|
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Benchmark: indexing and reindexing with synthetic corpus.
|
|
4
|
+
* Breaks down timings (clear, listPaths, indexing) and compares with/without embeddings.
|
|
5
|
+
* Opt-in; not run by npm test.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* npm run bench:indexing # indexing only
|
|
9
|
+
* npm run bench:embed # indexing + embedding queue processing (slow)
|
|
10
|
+
*/
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
import { loadConfig } from '../src/config.js';
|
|
15
|
+
import {
|
|
16
|
+
fullReindex,
|
|
17
|
+
incrementalReindex,
|
|
18
|
+
clearIndex,
|
|
19
|
+
listMarkdownPaths
|
|
20
|
+
} from '../src/core/indexing/reindex.js';
|
|
21
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
22
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
23
|
+
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
24
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
25
|
+
|
|
26
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-'));
|
|
27
|
+
const baseConfig = loadConfig({
|
|
28
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
29
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
30
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
31
|
+
});
|
|
32
|
+
fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
|
|
33
|
+
fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
|
|
34
|
+
|
|
35
|
+
const configNoEmb = {
|
|
36
|
+
...baseConfig,
|
|
37
|
+
embeddings: { ...baseConfig.embeddings, enabled: false }
|
|
38
|
+
};
|
|
39
|
+
const configEmb = {
|
|
40
|
+
...baseConfig,
|
|
41
|
+
embeddings: { ...baseConfig.embeddings, enabled: true }
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
/** Frontmatter with same keys the app writes so the indexer parses the same. */
|
|
45
|
+
function writeDoc(dir, id, title, body) {
|
|
46
|
+
const now = new Date().toISOString();
|
|
47
|
+
const frontmatter = {
|
|
48
|
+
id,
|
|
49
|
+
title,
|
|
50
|
+
type: 'generated_note',
|
|
51
|
+
scope: 'project',
|
|
52
|
+
project: 'bench',
|
|
53
|
+
topic_key: `bench/${id}`,
|
|
54
|
+
created_at: now
|
|
55
|
+
};
|
|
56
|
+
const raw = toMarkdown(frontmatter, body);
|
|
57
|
+
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
58
|
+
fs.writeFileSync(filePath, raw, 'utf8');
|
|
59
|
+
return filePath;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function buildLargeBody(size) {
|
|
63
|
+
const intro = '# Big\n\nNeedleInHaystack\n\n';
|
|
64
|
+
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
65
|
+
return intro + filler;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const NUM_SMALL = 3;
|
|
69
|
+
const LARGE_SIZE = 8_000;
|
|
70
|
+
const RUN_EMBED_PROCESSING = process.argv.includes('--embed');
|
|
71
|
+
|
|
72
|
+
console.log('Corpus: %d small .md + 1 large .md (~%d chars)', NUM_SMALL, LARGE_SIZE);
|
|
73
|
+
console.log('Mode: %s\n', RUN_EMBED_PROCESSING ? 'indexing + embedding queue processing' : 'indexing only');
|
|
74
|
+
|
|
75
|
+
// --- Create corpus ---
|
|
76
|
+
for (let i = 0; i < NUM_SMALL; i++) {
|
|
77
|
+
writeDoc(baseConfig.projectMemoryRoot, `obs_small_${i}`, `Small ${i}`, `Content for small doc ${i}.\n`);
|
|
78
|
+
}
|
|
79
|
+
const largePath = writeDoc(baseConfig.projectMemoryRoot, 'obs_large', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Full reindex with breakdown: clearIndex, listMarkdownPaths, index loop.
|
|
83
|
+
* @returns {{ total: number, clear: number, list: number, indexLoop: number, indexed: number }}
|
|
84
|
+
*/
|
|
85
|
+
function fullReindexWithBreakdown(config) {
|
|
86
|
+
let t0, t1;
|
|
87
|
+
t0 = performance.now();
|
|
88
|
+
clearIndex(config);
|
|
89
|
+
t1 = performance.now();
|
|
90
|
+
const clearMs = t1 - t0;
|
|
91
|
+
|
|
92
|
+
t0 = performance.now();
|
|
93
|
+
const paths = listMarkdownPaths(config);
|
|
94
|
+
t1 = performance.now();
|
|
95
|
+
const listMs = t1 - t0;
|
|
96
|
+
|
|
97
|
+
let indexed = 0;
|
|
98
|
+
t0 = performance.now();
|
|
99
|
+
for (const filePath of paths) {
|
|
100
|
+
try {
|
|
101
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
102
|
+
const stat = fs.statSync(filePath);
|
|
103
|
+
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
104
|
+
indexed++;
|
|
105
|
+
} catch (_) {}
|
|
106
|
+
}
|
|
107
|
+
t1 = performance.now();
|
|
108
|
+
const indexLoopMs = t1 - t0;
|
|
109
|
+
const totalMs = clearMs + listMs + indexLoopMs;
|
|
110
|
+
|
|
111
|
+
return { total: totalMs, clear: clearMs, list: listMs, indexLoop: indexLoopMs, indexed };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function ms(n) {
|
|
115
|
+
return Math.round(n);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function section(title) {
|
|
119
|
+
console.log('\n--- %s ---', title);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function runBenchmark() {
|
|
123
|
+
// ========== Without embeddings ==========
|
|
124
|
+
section('Without embeddings');
|
|
125
|
+
|
|
126
|
+
let b = fullReindexWithBreakdown(configNoEmb);
|
|
127
|
+
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
128
|
+
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
129
|
+
|
|
130
|
+
let t0 = performance.now();
|
|
131
|
+
const inc0 = incrementalReindex(configNoEmb);
|
|
132
|
+
let t1 = performance.now();
|
|
133
|
+
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
134
|
+
ms(t1 - t0), inc0.scanned, inc0.skipped, inc0.indexed, inc0.removed);
|
|
135
|
+
|
|
136
|
+
fs.appendFileSync(largePath, '\n\nAppended line.');
|
|
137
|
+
|
|
138
|
+
t0 = performance.now();
|
|
139
|
+
const inc1 = incrementalReindex(configNoEmb);
|
|
140
|
+
t1 = performance.now();
|
|
141
|
+
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
142
|
+
ms(t1 - t0), inc1.scanned, inc1.skipped, inc1.indexed, inc1.removed);
|
|
143
|
+
|
|
144
|
+
// Final state without embeddings
|
|
145
|
+
const db = getDb(baseConfig);
|
|
146
|
+
const docsCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
147
|
+
const ftsCount = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
|
|
148
|
+
console.log('State: %d documents, %d FTS rows.', docsCount, ftsCount);
|
|
149
|
+
|
|
150
|
+
// ========== With embeddings (indexing + queue) ==========
|
|
151
|
+
section('With embeddings (indexing + queue)');
|
|
152
|
+
|
|
153
|
+
clearIndex(configEmb);
|
|
154
|
+
b = fullReindexWithBreakdown(configEmb);
|
|
155
|
+
// Allow enqueue promises to settle
|
|
156
|
+
await new Promise((r) => setImmediate(r));
|
|
157
|
+
await new Promise((r) => setImmediate(r));
|
|
158
|
+
|
|
159
|
+
const jobsEnqueued = db.prepare('SELECT COUNT(*) AS c FROM embedding_jobs').get()?.c ?? 0;
|
|
160
|
+
console.log('fullReindex: %d ms (clear %d ms, list %d ms, indexing %d ms, docs=%d)',
|
|
161
|
+
ms(b.total), ms(b.clear), ms(b.list), ms(b.indexLoop), b.indexed);
|
|
162
|
+
console.log('Jobs in queue: %d', jobsEnqueued);
|
|
163
|
+
|
|
164
|
+
t0 = performance.now();
|
|
165
|
+
const inc0emb = incrementalReindex(configEmb);
|
|
166
|
+
t1 = performance.now();
|
|
167
|
+
console.log('incremental (0 changes): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
168
|
+
ms(t1 - t0), inc0emb.scanned, inc0emb.skipped, inc0emb.indexed, inc0emb.removed);
|
|
169
|
+
|
|
170
|
+
fs.appendFileSync(largePath, '\n\nAnother line.');
|
|
171
|
+
|
|
172
|
+
t0 = performance.now();
|
|
173
|
+
const inc1emb = incrementalReindex(configEmb);
|
|
174
|
+
t1 = performance.now();
|
|
175
|
+
console.log('incremental (1 file changed): %d ms → scanned=%d skipped=%d indexed=%d removed=%d',
|
|
176
|
+
ms(t1 - t0), inc1emb.scanned, inc1emb.skipped, inc1emb.indexed, inc1emb.removed);
|
|
177
|
+
|
|
178
|
+
// ========== Embedding processing (optional) ==========
|
|
179
|
+
if (RUN_EMBED_PROCESSING && configEmb.embeddings?.enabled) {
|
|
180
|
+
section('Embedding processing (queue → model → vector store)');
|
|
181
|
+
|
|
182
|
+
const pendingBefore = db.prepare("SELECT COUNT(*) AS c FROM embedding_jobs WHERE status = 'pending'").get()?.c ?? 0;
|
|
183
|
+
t0 = performance.now();
|
|
184
|
+
let processed = 0;
|
|
185
|
+
while (await processNextJob(configEmb)) {
|
|
186
|
+
processed++;
|
|
187
|
+
}
|
|
188
|
+
t1 = performance.now();
|
|
189
|
+
console.log('Process queue: %d ms (jobs processed=%d, pending before=%d)',
|
|
190
|
+
ms(t1 - t0), processed, pendingBefore);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Final summary
|
|
194
|
+
section('Summary');
|
|
195
|
+
const docsFinal = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
196
|
+
const ftsFinal = db.prepare('SELECT COUNT(*) AS c FROM document_fts').get()?.c ?? 0;
|
|
197
|
+
const jobsFinal = db.prepare('SELECT status, COUNT(*) AS c FROM embedding_jobs GROUP BY status').all();
|
|
198
|
+
console.log('Documents: %d, FTS rows: %d', docsFinal, ftsFinal);
|
|
199
|
+
if (jobsFinal.length) {
|
|
200
|
+
console.log('Embedding jobs: %s', jobsFinal.map((r) => `${r.status}=${r.c}`).join(', '));
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
runBenchmark()
|
|
205
|
+
.then(() => {
|
|
206
|
+
closeDb();
|
|
207
|
+
try {
|
|
208
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
209
|
+
} catch (_) {}
|
|
210
|
+
console.log('\nDone.\n');
|
|
211
|
+
})
|
|
212
|
+
.catch((err) => {
|
|
213
|
+
closeDb();
|
|
214
|
+
try {
|
|
215
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
216
|
+
} catch (_) {}
|
|
217
|
+
console.error(err);
|
|
218
|
+
process.exit(1);
|
|
219
|
+
});
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Benchmark: search (FTS and optional semantic) over synthetic corpus.
|
|
4
|
+
* Compares short vs long documents, FTS-only vs with embeddings.
|
|
5
|
+
* Opt-in; not run by npm test.
|
|
6
|
+
*
|
|
7
|
+
* Usage:
|
|
8
|
+
* npm run bench:search # FTS only
|
|
9
|
+
* npm run bench:search:embed # FTS + semantic search (needs embed model)
|
|
10
|
+
*/
|
|
11
|
+
import fs from 'fs';
|
|
12
|
+
import path from 'path';
|
|
13
|
+
import os from 'os';
|
|
14
|
+
import { loadConfig } from '../src/config.js';
|
|
15
|
+
import { fullReindex, clearIndex, listMarkdownPaths } from '../src/core/indexing/reindex.js';
|
|
16
|
+
import { indexDocument } from '../src/core/indexing/indexer.js';
|
|
17
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
18
|
+
import { search as searchFts, getRecentContext } from '../src/core/search/search.js';
|
|
19
|
+
import { closeDb, getDb } from '../src/db/sqlite.js';
|
|
20
|
+
import { toMarkdown } from '../src/utils/markdown.js';
|
|
21
|
+
|
|
22
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-bench-search-'));
|
|
23
|
+
const baseConfig = loadConfig({
|
|
24
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
25
|
+
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
26
|
+
sqlitePath: path.join(tmpDir, 'index.db')
|
|
27
|
+
});
|
|
28
|
+
fs.mkdirSync(path.dirname(baseConfig.sqlitePath), { recursive: true });
|
|
29
|
+
fs.mkdirSync(path.join(tmpDir, 'docs', 'generated'), { recursive: true });
|
|
30
|
+
|
|
31
|
+
const configNoEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: false } };
|
|
32
|
+
const configEmb = { ...baseConfig, embeddings: { ...baseConfig.embeddings, enabled: true } };
|
|
33
|
+
|
|
34
|
+
const RUN_SEMANTIC = process.argv.includes('--embed');
|
|
35
|
+
|
|
36
|
+
function writeDoc(dir, id, title, body) {
|
|
37
|
+
const now = new Date().toISOString();
|
|
38
|
+
const frontmatter = {
|
|
39
|
+
id,
|
|
40
|
+
title,
|
|
41
|
+
type: 'generated_note',
|
|
42
|
+
scope: 'project',
|
|
43
|
+
project: 'bench',
|
|
44
|
+
topic_key: `bench/${id}`,
|
|
45
|
+
created_at: now
|
|
46
|
+
};
|
|
47
|
+
const raw = toMarkdown(frontmatter, body);
|
|
48
|
+
const filePath = path.join(dir, 'generated', `${id}.md`);
|
|
49
|
+
fs.writeFileSync(filePath, raw, 'utf8');
|
|
50
|
+
return filePath;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function buildLargeBody(size, needle = 'NeedleInHaystack') {
|
|
54
|
+
const intro = `# Big\n\n${needle}\n\n`;
|
|
55
|
+
const filler = 'x'.repeat(Math.max(0, size - intro.length));
|
|
56
|
+
return intro + filler;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const NUM_SMALL = 5;
|
|
60
|
+
const LARGE_SIZE = 10_000;
|
|
61
|
+
const WARMUP = 2;
|
|
62
|
+
const RUNS = 5;
|
|
63
|
+
|
|
64
|
+
function ms(n) {
|
|
65
|
+
return Math.round(n);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function section(title) {
|
|
69
|
+
console.log('\n--- %s ---', title);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function median(arr) {
|
|
73
|
+
const a = [...arr].sort((x, y) => x - y);
|
|
74
|
+
const m = Math.floor(a.length / 2);
|
|
75
|
+
return a.length % 2 ? a[m] : (a[m - 1] + a[m]) / 2;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Time a sync function over multiple runs; report median ms.
|
|
80
|
+
*/
|
|
81
|
+
function timeSync(fn, runs = RUNS, warmup = WARMUP) {
|
|
82
|
+
for (let i = 0; i < warmup; i++) fn();
|
|
83
|
+
const times = [];
|
|
84
|
+
for (let i = 0; i < runs; i++) {
|
|
85
|
+
const t0 = performance.now();
|
|
86
|
+
fn();
|
|
87
|
+
times.push(performance.now() - t0);
|
|
88
|
+
}
|
|
89
|
+
return median(times);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Time an async function over multiple runs; report median ms.
|
|
94
|
+
*/
|
|
95
|
+
async function timeAsync(fn, runs = RUNS, warmup = WARMUP) {
|
|
96
|
+
for (let i = 0; i < warmup; i++) await fn();
|
|
97
|
+
const times = [];
|
|
98
|
+
for (let i = 0; i < runs; i++) {
|
|
99
|
+
const t0 = performance.now();
|
|
100
|
+
await fn();
|
|
101
|
+
times.push(performance.now() - t0);
|
|
102
|
+
}
|
|
103
|
+
return median(times);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
async function runBenchmark() {
|
|
107
|
+
console.log('Corpus: %d short .md + 1 large .md (~%d chars). Mode: %s\n', NUM_SMALL, LARGE_SIZE, RUN_SEMANTIC ? 'FTS + semantic' : 'FTS only');
|
|
108
|
+
|
|
109
|
+
// --- Corpus: short docs + one large doc with needle ---
|
|
110
|
+
for (let i = 0; i < NUM_SMALL; i++) {
|
|
111
|
+
writeDoc(baseConfig.projectMemoryRoot, `short_${i}`, `Short ${i}`, `Content for short doc ${i}. Some terms: alpha beta gamma.\n`);
|
|
112
|
+
}
|
|
113
|
+
writeDoc(baseConfig.projectMemoryRoot, 'large', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
114
|
+
|
|
115
|
+
fullReindex(configNoEmb);
|
|
116
|
+
const db = getDb(baseConfig);
|
|
117
|
+
const docCount = db.prepare('SELECT COUNT(*) AS c FROM documents').get()?.c ?? 0;
|
|
118
|
+
console.log('Indexed %d documents (FTS only).\n', docCount);
|
|
119
|
+
|
|
120
|
+
// ========== FTS search ==========
|
|
121
|
+
section('FTS search (no embeddings)');
|
|
122
|
+
|
|
123
|
+
const ftsShort = timeSync(() => {
|
|
124
|
+
searchFts(configNoEmb, { query: 'Content', limit: 10 });
|
|
125
|
+
});
|
|
126
|
+
console.log('Query "Content" (short, matches short docs): median %d ms', ms(ftsShort));
|
|
127
|
+
|
|
128
|
+
const ftsNeedle = timeSync(() => {
|
|
129
|
+
searchFts(configNoEmb, { query: 'NeedleInHaystack', limit: 10 });
|
|
130
|
+
});
|
|
131
|
+
console.log('Query "NeedleInHaystack" (needle in large): median %d ms', ms(ftsNeedle));
|
|
132
|
+
|
|
133
|
+
const ftsPhrase = timeSync(() => {
|
|
134
|
+
searchFts(configNoEmb, { query: 'alpha beta gamma', limit: 10 });
|
|
135
|
+
});
|
|
136
|
+
console.log('Query "alpha beta gamma" (phrase in short): median %d ms', ms(ftsPhrase));
|
|
137
|
+
|
|
138
|
+
const ftsRecent = timeSync(() => {
|
|
139
|
+
getRecentContext(configNoEmb, { limit: 10 });
|
|
140
|
+
});
|
|
141
|
+
console.log('getRecentContext(limit=10): median %d ms', ms(ftsRecent));
|
|
142
|
+
|
|
143
|
+
// ========== Semantic search (optional) ==========
|
|
144
|
+
if (RUN_SEMANTIC) {
|
|
145
|
+
section('With embeddings: index + process queue');
|
|
146
|
+
|
|
147
|
+
clearIndex(configEmb);
|
|
148
|
+
const paths = listMarkdownPaths(configEmb);
|
|
149
|
+
for (const filePath of paths) {
|
|
150
|
+
try {
|
|
151
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
152
|
+
const stat = fs.statSync(filePath);
|
|
153
|
+
indexDocument(configEmb, filePath, raw, stat.mtimeMs);
|
|
154
|
+
} catch (_) {}
|
|
155
|
+
}
|
|
156
|
+
await new Promise((r) => setImmediate(r));
|
|
157
|
+
await new Promise((r) => setImmediate(r));
|
|
158
|
+
|
|
159
|
+
let processed = 0;
|
|
160
|
+
while (await processNextJob(configEmb)) processed++;
|
|
161
|
+
const vecCount = db.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type='table' AND name LIKE 'vec_%'").get()?.c ?? 0;
|
|
162
|
+
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
163
|
+
const vecRows = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
164
|
+
console.log('Embedding jobs processed: %d. Vector rows: %d.', processed, vecRows);
|
|
165
|
+
|
|
166
|
+
section('Semantic search (query embed + vector KNN)');
|
|
167
|
+
|
|
168
|
+
const { getEmbedder } = await import('../src/core/embeddings/embedder.js');
|
|
169
|
+
const { search: searchVector } = await import('../src/core/embeddings/vectorStore.js');
|
|
170
|
+
const embedder = await getEmbedder(configEmb);
|
|
171
|
+
if (!embedder) {
|
|
172
|
+
console.log('Embedder not available; skipping semantic timings.');
|
|
173
|
+
} else {
|
|
174
|
+
const embedTime = await timeAsync(async () => {
|
|
175
|
+
await embedder.embed('NeedleInHaystack');
|
|
176
|
+
});
|
|
177
|
+
console.log('Query embedding (single phrase): median %d ms', ms(embedTime));
|
|
178
|
+
|
|
179
|
+
const vectorSearchTime = await timeAsync(async () => {
|
|
180
|
+
const vec = await embedder.embed('NeedleInHaystack');
|
|
181
|
+
await searchVector(configEmb, vec, 20);
|
|
182
|
+
});
|
|
183
|
+
console.log('Embed + vector search (k=20): median %d ms', ms(vectorSearchTime));
|
|
184
|
+
|
|
185
|
+
const precomputedVec = await embedder.embed('query');
|
|
186
|
+
const vectorOnlyTime = await timeAsync(async () => {
|
|
187
|
+
await searchVector(configEmb, precomputedVec, 10);
|
|
188
|
+
});
|
|
189
|
+
console.log('Vector search only (k=10, no embed): median %d ms', ms(vectorOnlyTime));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
section('FTS with embeddings enabled (same as FTS-only path)');
|
|
193
|
+
const ftsWithEmb = timeSync(() => {
|
|
194
|
+
searchFts(configEmb, { query: 'Content', limit: 10 });
|
|
195
|
+
});
|
|
196
|
+
console.log('Query "Content": median %d ms', ms(ftsWithEmb));
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
section('Summary');
|
|
200
|
+
console.log('FTS: short query, needle-in-large, phrase, getRecentContext.');
|
|
201
|
+
if (RUN_SEMANTIC) {
|
|
202
|
+
console.log('Semantic: query embed + vector KNN (npm run bench:search:embed).');
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
runBenchmark()
|
|
207
|
+
.then(() => {
|
|
208
|
+
closeDb();
|
|
209
|
+
try {
|
|
210
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
211
|
+
} catch (_) {}
|
|
212
|
+
console.log('\nDone.\n');
|
|
213
|
+
})
|
|
214
|
+
.catch((err) => {
|
|
215
|
+
closeDb();
|
|
216
|
+
try {
|
|
217
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
218
|
+
} catch (_) {}
|
|
219
|
+
console.error(err);
|
|
220
|
+
process.exit(1);
|
|
221
|
+
});
|