mixgram 1.3.0 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -3
- package/bin/mixgram.js +17 -1
- package/examples/embedding-demo.js +154 -0
- package/package.json +21 -20
- package/patches/sharp@0.34.5.patch +13 -0
- package/pnpm-workspace.yaml +35 -0
- package/spec/fixtures/realistic-content.js +9 -0
- package/spec/scenarios/scenarios.js +31 -0
- package/spec/sdd.md +3 -3
- package/src/config.js +1 -1
- package/src/core/embeddings/queue.js +2 -1
- package/src/core/embeddings/vectorStore.js +1 -1
- package/src/core/search/search.js +12 -2
- package/src/mcp/server.js +54 -29
- package/src/mcp/tools.js +54 -7
package/README.md
CHANGED
|
@@ -38,17 +38,54 @@ Then use the `mixgram` command in your client config (see below). No extra depen
|
|
|
38
38
|
| `mixgram help <tool>` | Show options for a specific MCP tool. |
|
|
39
39
|
| `mixgram <tool> [--key value ...]` | Run any MCP tool from the command line (see below). |
|
|
40
40
|
|
|
41
|
-
**MCP tools from the CLI** — Every MCP tool is automatically available as a subcommand. No extra wiring is needed: if a tool is added to the internal registry, it is exposed both to MCP clients and to the CLI. Use `mixgram help` to list tools and `mixgram help <tool>` (or `mixgram <tool> --help`) to see a tool’s options.
|
|
41
|
+
**MCP tools from the CLI** — Every MCP tool is automatically available as a subcommand. No extra wiring is needed: if a tool is added to the internal registry, it is exposed both to MCP clients and to the CLI. Use `mixgram help` to list tools and `mixgram help <tool>` (or `mixgram <tool> --help`) to see a tool’s options.
|
|
42
|
+
|
|
43
|
+
### Command-line examples
|
|
42
44
|
|
|
43
45
|
```bash
|
|
46
|
+
# Stats and health
|
|
44
47
|
mixgram mem_stats
|
|
48
|
+
mixgram mem_stats --embeddings # enable embeddings for this run (or use env/config, see below)
|
|
49
|
+
|
|
50
|
+
# Save a note
|
|
51
|
+
mixgram mem_save --title "My note" --content "Search" --type decision --project my-app
|
|
52
|
+
mixgram mem_save --title "Other note" --content "Model Context Protocol" --type architecture
|
|
53
|
+
|
|
54
|
+
# Search (default: merged scope, limit 10)
|
|
55
|
+
mixgram mem_search --query "index"
|
|
45
56
|
mixgram mem_search --query "index" --limit 5
|
|
46
|
-
mixgram
|
|
57
|
+
mixgram mem_search --query "indexing" --scope-mode project-only --project my-app
|
|
58
|
+
|
|
59
|
+
# OR-style query: use FTS5 keyword OR (and optional quoted phrases)
|
|
60
|
+
mixgram mem_search --query 'mcp OR "model context protocol"' --limit 10
|
|
61
|
+
mixgram mem_search --query 'cursor OR "model context protocol"' --limit 5
|
|
62
|
+
|
|
63
|
+
# Reindex from disk (e.g. after editing Markdown by hand)
|
|
47
64
|
mixgram mem_reindex --full
|
|
65
|
+
|
|
66
|
+
# Context (recent or by query)
|
|
67
|
+
mixgram mem_context --query "indexing decisions" --project my-app --limit 5
|
|
68
|
+
mixgram mem_context --limit 5
|
|
48
69
|
```
|
|
49
70
|
|
|
71
|
+
**Example: index and find by embedding (not FTS)** — Save a note with one wording, then search with different words that mean the same. FTS finds nothing; semantic search returns the doc:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# 1. Index a document (content about storing memory)
|
|
75
|
+
mixgram mem_save --title "Where we store memory" --content "We keep agent memory in Markdown under docs/." --type decision --project demo --embeddings
|
|
76
|
+
|
|
77
|
+
# 2. Search by meaning: query has no literal overlap with the text
|
|
78
|
+
mixgram mem_search --query "where is the agent memory stored" --project demo --embeddings --limit 5
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
The first run embeds the document (worker runs in-process and processes the queue after save; the first time can take a few seconds while the embedding model loads). The second run uses the embedding worker to turn the query into a vector and returns the document by similarity. You can also use `npm run example:embedding` for a self-contained demo in a temp dir.
|
|
82
|
+
|
|
83
|
+
**Query syntax (FTS5):** Search uses SQLite FTS5. You can use the operators `OR`, `AND`, `NOT` and quoted phrases. For an OR-style search (e.g. “mcp” or “model context protocol”), use the word `OR` in the query, not the pipe character: `--query 'mcp OR "model context protocol"'`. Unquoted terms are ANDed by default; use `OR` explicitly to match any of several terms.
|
|
84
|
+
|
|
50
85
|
Arguments are passed as `--key value`. Booleans use `--flag` or `--no-flag`. Arrays use repeated flags: `--tags a --tags b`.
|
|
51
86
|
|
|
87
|
+
**Enabling embeddings from the CLI:** For the MCP server use `mixgram mcp --embeddings`. For one-off tool runs (e.g. `mixgram mem_stats`, `mixgram mem_search`) you can pass `--embeddings` after the tool name, set the env var `MIXGRAM_EMBEDDINGS_ENABLED=1` (or `true`), or enable in `.mixgram/config.json` with `"embeddings": { "enabled": true }`. Example: `mixgram mem_search --query "something" --embeddings --limit 5`.
|
|
88
|
+
|
|
52
89
|
**`mcp` options** (and env / config file):
|
|
53
90
|
|
|
54
91
|
| Option | Env | Description |
|
|
@@ -291,7 +328,7 @@ When `embeddings.enabled` is `true` and the optional embedding stack is availabl
|
|
|
291
328
|
await run({
|
|
292
329
|
embeddings: {
|
|
293
330
|
enabled: true,
|
|
294
|
-
similarityThreshold: 0.
|
|
331
|
+
similarityThreshold: 0.80,
|
|
295
332
|
maxRetries: 3
|
|
296
333
|
},
|
|
297
334
|
search: {
|
|
@@ -321,6 +358,38 @@ mem_search({
|
|
|
321
358
|
|
|
322
359
|
With embeddings enabled and vectors ready, results combine literal matches (FTS) and semantic similarity (e.g. paraphrases).
|
|
323
360
|
|
|
361
|
+
### Example: indexing and search (embedding effectiveness)
|
|
362
|
+
|
|
363
|
+
To see the difference between **FTS-only** and **semantic** search in one run, run the demo script (uses a temp dir; first run may download the embedding model):
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
node examples/embedding-demo.js
|
|
367
|
+
# or
|
|
368
|
+
npm run example:embedding
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
The script:
|
|
372
|
+
|
|
373
|
+
1. **Indexes** three short documents (e.g. “memoria en Markdown”, “índice derivado”, “persistencia de sesión”).
|
|
374
|
+
2. **Processes the embedding queue** so each document gets a vector.
|
|
375
|
+
3. **Runs FTS search** for a query that does *not* appear literally in the text (e.g. “dónde se almacena la memoria del agente”) — FTS returns nothing.
|
|
376
|
+
4. **Runs semantic (vector) search** for the same query — returns the relevant document by meaning.
|
|
377
|
+
|
|
378
|
+
Example output:
|
|
379
|
+
|
|
380
|
+
```text
|
|
381
|
+
--- 3. FTS search (text only) ---
|
|
382
|
+
Query: "where is the agent memory stored"
|
|
383
|
+
(FTS finds no literal match; these words are not in the documents.)
|
|
384
|
+
|
|
385
|
+
--- 4. Semantic search (embeddings) ---
|
|
386
|
+
Query: "where is the agent memory stored"
|
|
387
|
+
- [Session persistence] similarity=0.856
|
|
388
|
+
Persist the session context across agent restarts. The important state...
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
So: **FTS** only matches when the query words appear in the document; **embeddings** can retrieve documents that share the same meaning without sharing the same words.
|
|
392
|
+
|
|
324
393
|
---
|
|
325
394
|
|
|
326
395
|
## Tests
|
package/bin/mixgram.js
CHANGED
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* mixgram mcp [options] — run MCP server (stdio)
|
|
6
6
|
* mixgram setup <client> — add Mixgram to Cursor / Gemini CLI / Codex config
|
|
7
7
|
*/
|
|
8
|
-
import { run } from '../src/mcp/server.js';
|
|
8
|
+
import { run, startEmbeddingWorker } from '../src/mcp/server.js';
|
|
9
9
|
import { loadConfig } from '../src/config.js';
|
|
10
10
|
import { closeDb } from '../src/db/sqlite.js';
|
|
11
11
|
import { createToolHandlers } from '../src/mcp/tools.js';
|
|
12
12
|
import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../src/mcp/cli-adapter.js';
|
|
13
|
+
import { processNextJob, enqueueChunks } from '../src/core/embeddings/queue.js';
|
|
13
14
|
import fs from 'fs';
|
|
14
15
|
import path from 'path';
|
|
15
16
|
import os from 'os';
|
|
@@ -327,14 +328,29 @@ async function main() {
|
|
|
327
328
|
const args = parseToolArgs(toolDef, toolArgv);
|
|
328
329
|
const { overrides, baseDir, projectBaseDir } = loadCliConfig();
|
|
329
330
|
const config = loadConfig(overrides, baseDir, projectBaseDir);
|
|
331
|
+
let cleanupEmbeddingWorker = () => {};
|
|
332
|
+
if (config.embeddings?.enabled) {
|
|
333
|
+
cleanupEmbeddingWorker = startEmbeddingWorker(config);
|
|
334
|
+
}
|
|
330
335
|
const handlers = createToolHandlers(config);
|
|
331
336
|
try {
|
|
332
337
|
const result = await handlers[SUBCOMMAND](args);
|
|
338
|
+
if ((SUBCOMMAND === 'mem_save' || SUBCOMMAND === 'mem_update') && config.embeddings?.enabled) {
|
|
339
|
+
let docId = null;
|
|
340
|
+
try {
|
|
341
|
+
const out = JSON.parse(result?.content?.[0]?.text || '{}');
|
|
342
|
+
if (out.id) docId = out.id;
|
|
343
|
+
} catch (_) {}
|
|
344
|
+
if (docId) enqueueChunks(config, [docId]);
|
|
345
|
+
while (await processNextJob(config)) {}
|
|
346
|
+
}
|
|
333
347
|
const text = result?.content?.[0]?.text;
|
|
334
348
|
if (text != null) console.log(text);
|
|
335
349
|
} catch (err) {
|
|
336
350
|
console.error(err);
|
|
337
351
|
process.exit(1);
|
|
352
|
+
} finally {
|
|
353
|
+
cleanupEmbeddingWorker();
|
|
338
354
|
}
|
|
339
355
|
return;
|
|
340
356
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Example: indexing + search showing embedding effectiveness.
|
|
4
|
+
*
|
|
5
|
+
* Concept:
|
|
6
|
+
* - FTS (full-text search) only matches when query words appear in the document.
|
|
7
|
+
* - Semantic search (embeddings) matches by meaning: different wording, same idea.
|
|
8
|
+
* This script indexes a few documents, then runs the same query with FTS (no match)
|
|
9
|
+
* and with vector search (finds the relevant doc by similarity).
|
|
10
|
+
*
|
|
11
|
+
* Steps:
|
|
12
|
+
* 1. Index several short documents with related content but different wording.
|
|
13
|
+
* 2. Process the embedding queue so each document gets a vector.
|
|
14
|
+
* 3. Compare FTS (text-only) vs semantic (vector) search.
|
|
15
|
+
*
|
|
16
|
+
* Requires: npm install (includes @huggingface/transformers and sqlite-vec).
|
|
17
|
+
* Usage: node examples/embedding-demo.js
|
|
18
|
+
*/
|
|
19
|
+
import fs from 'fs';
|
|
20
|
+
import path from 'path';
|
|
21
|
+
import os from 'os';
|
|
22
|
+
import { loadConfig } from '../src/config.js';
|
|
23
|
+
import { saveDocument } from '../src/core/documents/documents.js';
|
|
24
|
+
import { search as searchFts } from '../src/core/search/search.js';
|
|
25
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
26
|
+
import { getEmbedder } from '../src/core/embeddings/embedder.js';
|
|
27
|
+
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
28
|
+
|
|
29
|
+
async function run() {
|
|
30
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-embedding-demo-'));
|
|
31
|
+
const projectRoot = path.join(tmpDir, 'repo');
|
|
32
|
+
const docsDir = path.join(projectRoot, 'docs');
|
|
33
|
+
fs.mkdirSync(docsDir, { recursive: true });
|
|
34
|
+
|
|
35
|
+
const overrides = {
|
|
36
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
37
|
+
projectMemoryRoot: path.join(projectRoot, 'docs'),
|
|
38
|
+
sqlitePath: path.join(tmpDir, 'index.db'),
|
|
39
|
+
watch: false,
|
|
40
|
+
indexing: { reindexOnStartup: false },
|
|
41
|
+
embeddings: {
|
|
42
|
+
enabled: true,
|
|
43
|
+
similarityThreshold: 0.80 // more permissive so the demo shows more results
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const config = loadConfig(overrides, tmpDir, projectRoot);
|
|
48
|
+
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
49
|
+
fs.mkdirSync(config.homeMemoryRoot, { recursive: true });
|
|
50
|
+
|
|
51
|
+
const db = getDb(config);
|
|
52
|
+
|
|
53
|
+
console.log('--- 1. Indexing: save documents ---\n');
|
|
54
|
+
|
|
55
|
+
const docs = [
|
|
56
|
+
{
|
|
57
|
+
title: 'Memory in Markdown',
|
|
58
|
+
type: 'decision',
|
|
59
|
+
content: 'We store the agent\'s memory in Markdown files under docs/. That way the documentation is visible and editable by hand.'
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
title: 'Derived index',
|
|
63
|
+
type: 'decision',
|
|
64
|
+
content: 'The search index is derived from the Markdown files using SQLite FTS5. There is no database as source of truth other than the .md files.'
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
title: 'Session persistence',
|
|
68
|
+
type: 'decision',
|
|
69
|
+
content: 'Persist the session context across agent restarts. The important state lives in the memory documents.'
|
|
70
|
+
}
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
for (const d of docs) {
|
|
74
|
+
const out = saveDocument(config, {
|
|
75
|
+
title: d.title,
|
|
76
|
+
type: d.type,
|
|
77
|
+
scope: 'project',
|
|
78
|
+
project: 'demo',
|
|
79
|
+
content: d.content
|
|
80
|
+
});
|
|
81
|
+
console.log(` Saved: ${d.title} → ${out.id}`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
await new Promise((r) => setImmediate(r));
|
|
85
|
+
await new Promise((r) => setImmediate(r));
|
|
86
|
+
|
|
87
|
+
console.log('\n--- 2. Process embedding queue ---\n');
|
|
88
|
+
let processed = 0;
|
|
89
|
+
while (await processNextJob(config)) processed++;
|
|
90
|
+
console.log(` Jobs processed: ${processed}`);
|
|
91
|
+
|
|
92
|
+
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
93
|
+
const vecCount = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
94
|
+
console.log(` Vectors in index: ${vecCount}\n`);
|
|
95
|
+
|
|
96
|
+
console.log('--- 3. FTS search (text only) ---\n');
|
|
97
|
+
|
|
98
|
+
const queryLiteral = 'Markdown';
|
|
99
|
+
const ftsResults = searchFts(config, { query: queryLiteral, project: 'demo', limit: 5 });
|
|
100
|
+
console.log(` Query: "${queryLiteral}"`);
|
|
101
|
+
ftsResults.forEach((r, i) => {
|
|
102
|
+
console.log(` ${i + 1}. [${r.title}] score=${r.score.toFixed(2)} — ${r.snippet.slice(0, 60)}...`);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
const querySemantic = 'where is the agent memory stored';
|
|
106
|
+
const ftsSemantic = searchFts(config, { query: querySemantic, project: 'demo', limit: 5 });
|
|
107
|
+
console.log(`\n Query: "${querySemantic}"`);
|
|
108
|
+
if (ftsSemantic.length === 0) {
|
|
109
|
+
console.log(' (FTS finds no literal match; these words are not in the documents.)');
|
|
110
|
+
} else {
|
|
111
|
+
ftsSemantic.forEach((r, i) => {
|
|
112
|
+
console.log(` ${i + 1}. [${r.title}] — ${r.snippet.slice(0, 60)}...`);
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
console.log('\n--- 4. Semantic search (embeddings) ---\n');
|
|
117
|
+
|
|
118
|
+
const embedder = await getEmbedder(config);
|
|
119
|
+
const { getVectorStore } = await import('../src/core/embeddings/vectorStore.js');
|
|
120
|
+
const vectorStore = await getVectorStore(config);
|
|
121
|
+
|
|
122
|
+
if (!embedder || !vectorStore) {
|
|
123
|
+
console.log(' Embeddings not available (missing @huggingface/transformers or sqlite-vec).');
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const queryVec = await embedder.embed(querySemantic);
|
|
128
|
+
const vectorResults = await vectorStore.search(config, queryVec, 10, { similarityThreshold: 0.80 });
|
|
129
|
+
|
|
130
|
+
console.log(` Query: "${querySemantic}"`);
|
|
131
|
+
console.log(' Results by semantic similarity:\n');
|
|
132
|
+
|
|
133
|
+
for (const v of vectorResults) {
|
|
134
|
+
const row = db.prepare('SELECT id, title, body FROM documents WHERE id = ?').get(v.document_id);
|
|
135
|
+
const similarity = (1 - v.distance).toFixed(3);
|
|
136
|
+
const snippet = (row?.body || '').slice(0, 80);
|
|
137
|
+
console.log(` - [${row?.title ?? v.document_id}] similarity=${similarity}`);
|
|
138
|
+
console.log(` ${snippet}...`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
console.log('\n--- Summary ---');
|
|
142
|
+
console.log('FTS only finds when the query words appear in the text.');
|
|
143
|
+
console.log('Embedding search finds documents that share the same meaning even without shared words.\n');
|
|
144
|
+
|
|
145
|
+
closeDb();
|
|
146
|
+
try {
|
|
147
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
148
|
+
} catch (_) {}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
run().catch((err) => {
|
|
152
|
+
console.error(err);
|
|
153
|
+
process.exit(1);
|
|
154
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixgram",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.4",
|
|
4
4
|
"description": "MCP memory server with Markdown source of truth and SQLite FTS5.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
@@ -30,6 +30,23 @@
|
|
|
30
30
|
"bin": {
|
|
31
31
|
"mixgram": "./bin/mixgram.js"
|
|
32
32
|
},
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"@huggingface/transformers": "^3.8.1",
|
|
35
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
36
|
+
"better-sqlite3": "^12.6.2",
|
|
37
|
+
"chokidar": "^4.0.3",
|
|
38
|
+
"glob": "^13.0.6",
|
|
39
|
+
"gray-matter": "^4.0.3",
|
|
40
|
+
"remark-parse": "^11.0.0",
|
|
41
|
+
"sqlite-vec": "^0.1.6",
|
|
42
|
+
"unified": "^11.0.5",
|
|
43
|
+
"unist-util-visit": "^5.1.0",
|
|
44
|
+
"zod": "^3.25.76"
|
|
45
|
+
},
|
|
46
|
+
"devDependencies": {
|
|
47
|
+
"jscpd": "^4.0.8",
|
|
48
|
+
"knip": "^5.86.0"
|
|
49
|
+
},
|
|
33
50
|
"scripts": {
|
|
34
51
|
"start": "node index.js",
|
|
35
52
|
"test": "node spec/run-tests.js",
|
|
@@ -40,25 +57,9 @@
|
|
|
40
57
|
"bench:search": "node spec/bench-search.js",
|
|
41
58
|
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
42
59
|
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
60
|
+
"example:embedding": "node examples/embedding-demo.js",
|
|
43
61
|
"duplication": "jscpd src",
|
|
44
62
|
"deadcode": "knip",
|
|
45
|
-
"quality": "npm run duplication && npm run deadcode"
|
|
46
|
-
},
|
|
47
|
-
"dependencies": {
|
|
48
|
-
"@huggingface/transformers": "^3.4.2",
|
|
49
|
-
"@modelcontextprotocol/sdk": "^1.0.0",
|
|
50
|
-
"better-sqlite3": "^12.6.2",
|
|
51
|
-
"chokidar": "^4.0.1",
|
|
52
|
-
"glob": "^13.0.6",
|
|
53
|
-
"gray-matter": "^4.0.3",
|
|
54
|
-
"remark-parse": "^11.0.0",
|
|
55
|
-
"sqlite-vec": "^0.1.6",
|
|
56
|
-
"unified": "^11.0.5",
|
|
57
|
-
"unist-util-visit": "^5.0.0",
|
|
58
|
-
"zod": "^3.23.0"
|
|
59
|
-
},
|
|
60
|
-
"devDependencies": {
|
|
61
|
-
"jscpd": "^4.0.8",
|
|
62
|
-
"knip": "^5.86.0"
|
|
63
|
+
"quality": "npm run duplication && npm run deadcode"
|
|
63
64
|
}
|
|
64
|
-
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
diff --git a/package.json b/package.json
|
|
2
|
+
index 0c0d009888ec50a444c930c31da0b57160739591..9dbf4c35e01e2855f2f30b573288e61bedf0ae07 100644
|
|
3
|
+
--- a/package.json
|
|
4
|
+
+++ b/package.json
|
|
5
|
+
@@ -93,7 +93,7 @@
|
|
6
|
+
],
|
|
7
|
+
"scripts": {
|
|
8
|
+
"build": "node install/build.js",
|
|
9
|
+
- "install": "node install/check.js || npm run build",
|
|
10
|
+
+ "install": "SHARP_IGNORE_GLOBAL_LIBVIPS=true node install/check.js || npm run build",
|
|
11
|
+
"clean": "rm -rf src/build/ .nyc_output/ coverage/ test/fixtures/output.*",
|
|
12
|
+
"test": "npm run lint && npm run test-unit",
|
|
13
|
+
"lint": "npm run lint-cpp && npm run lint-js && npm run lint-types",
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
allowBuilds:
|
|
2
|
+
better-sqlite3: true
|
|
3
|
+
onnxruntime-node: true
|
|
4
|
+
protobufjs: true
|
|
5
|
+
sharp: true
|
|
6
|
+
minimumReleaseAgeExclude:
|
|
7
|
+
- hono@4.12.7
|
|
8
|
+
- smol-toml@1.6.1
|
|
9
|
+
- brace-expansion@5.0.5
|
|
10
|
+
- picomatch@2.3.2
|
|
11
|
+
- picomatch@4.0.4
|
|
12
|
+
- yaml@2.8.3
|
|
13
|
+
- path-to-regexp@8.4.0
|
|
14
|
+
- hono@4.12.12
|
|
15
|
+
- '@hono/node-server@1.19.13'
|
|
16
|
+
- protobufjs@7.5.5
|
|
17
|
+
- fast-uri@3.1.1
|
|
18
|
+
- fast-uri@3.1.2
|
|
19
|
+
- hono@4.12.18
|
|
20
|
+
- protobufjs@7.5.6
|
|
21
|
+
- ip-address@10.1.1
|
|
22
|
+
- '@protobufjs/utf8@1.1.1'
|
|
23
|
+
- hono@4.12.16
|
|
24
|
+
- protobufjs@7.5.8
|
|
25
|
+
- qs@6.15.2
|
|
26
|
+
- hono@4.12.21
|
|
27
|
+
- brace-expansion@5.0.6
|
|
28
|
+
- protobufjs@7.6.3
|
|
29
|
+
- tar@7.5.16
|
|
30
|
+
- protobufjs@7.6.1
|
|
31
|
+
- hono@4.12.25
|
|
32
|
+
- hono@4.12.14
|
|
33
|
+
- js-yaml@3.15.0
|
|
34
|
+
patchedDependencies:
|
|
35
|
+
sharp@0.34.5: patches/sharp@0.34.5.patch
|
|
@@ -95,3 +95,12 @@ export const semanticFallbackDoc = {
|
|
|
95
95
|
topic_key: 'test/semantic-fallback',
|
|
96
96
|
content: 'Text for semantic fallback test.'
|
|
97
97
|
};
|
|
98
|
+
|
|
99
|
+
/** Used to test FTS5 OR query: save then search with "mcp OR \"model context protocol\"" */
|
|
100
|
+
export const noteMcpArchitecture = {
|
|
101
|
+
title: 'Other note',
|
|
102
|
+
type: 'architecture',
|
|
103
|
+
content: 'Model Context Protocol',
|
|
104
|
+
scope: 'project',
|
|
105
|
+
project: null
|
|
106
|
+
};
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
semanticPersistenceDoc,
|
|
20
20
|
semanticPersistenceQuery,
|
|
21
21
|
semanticFallbackDoc,
|
|
22
|
+
noteMcpArchitecture,
|
|
22
23
|
PROJECT_NAME
|
|
23
24
|
} from '../fixtures/realistic-content.js';
|
|
24
25
|
import { toMarkdown } from '../../src/utils/markdown.js';
|
|
@@ -198,6 +199,36 @@ export const scenarios = [
|
|
|
198
199
|
return { passed: p, failed: f };
|
|
199
200
|
}
|
|
200
201
|
},
|
|
202
|
+
{
|
|
203
|
+
name: 'Search with OR query (FTS5 operator case)',
|
|
204
|
+
goal: 'mem_save then mem_search with query using OR (e.g. mcp OR "model context protocol") returns the saved document.',
|
|
205
|
+
run: async function (ctx) {
|
|
206
|
+
const { h, reporter } = ctx;
|
|
207
|
+
const parseRes = ctx.parse || parse;
|
|
208
|
+
let p = 0, f = 0;
|
|
209
|
+
const ok = (cond, msg) => { if (cond) p++; else f++; reporter.check(msg, cond); };
|
|
210
|
+
|
|
211
|
+
reporter.startScenario(this.name, this.goal);
|
|
212
|
+
|
|
213
|
+
const saveRes = await h.mem_save(noteMcpArchitecture);
|
|
214
|
+
const saveData = parseRes(saveRes);
|
|
215
|
+
reporter.step('mem_save', { title: noteMcpArchitecture.title, type: noteMcpArchitecture.type }, { id: saveData.id, created: saveData.created });
|
|
216
|
+
ok(saveData.success && saveData.id, 'mem_save success');
|
|
217
|
+
|
|
218
|
+
const searchRes = await h.mem_search({ query: 'mcp OR "model context protocol"', limit: 10 });
|
|
219
|
+
const searchData = parseRes(searchRes);
|
|
220
|
+
reporter.step('mem_search', { query: 'mcp OR "model context protocol"' }, searchData, { highlight: ['title', 'snippet'] });
|
|
221
|
+
ok(Array.isArray(searchData.results), 'results is array');
|
|
222
|
+
ok(searchData.results.length >= 1, 'OR query returns at least one result');
|
|
223
|
+
const found = searchData.results.some(
|
|
224
|
+
(r) => (r.title && r.title.includes('Other note')) || (r.snippet && (r.snippet.toLowerCase().includes('mcp') || r.snippet.toLowerCase().includes('model context protocol')))
|
|
225
|
+
);
|
|
226
|
+
ok(found, 'saved note is in OR query results');
|
|
227
|
+
|
|
228
|
+
reporter.endScenario(p, f);
|
|
229
|
+
return { passed: p, failed: f };
|
|
230
|
+
}
|
|
231
|
+
},
|
|
201
232
|
{
|
|
202
233
|
name: 'Context for project',
|
|
203
234
|
goal: 'mem_context returns usable context (recent or query-based).',
|
package/spec/sdd.md
CHANGED
|
@@ -768,7 +768,7 @@ const embedding = output.tolist(); // 1024-dim, L2-normalized
|
|
|
768
768
|
```
|
|
769
769
|
For tag/scope filtering, `k` is increased and results are filtered in application code.
|
|
770
770
|
3. **Join** with `cache_<suffix>` on `cache.id = vec.rowid` to get chunk_id, document_id, etc., then join to `document_chunks` / `documents` for full metadata and snippets.
|
|
771
|
-
4. **Similarity threshold:** sqlite-vec returns **cosine distance** (0 = identical). Convert to similarity as `similarity = 1 - distance`. Accept a hit only if `similarity >= similarityThreshold` (default `0.
|
|
771
|
+
4. **Similarity threshold:** sqlite-vec returns **cosine distance** (0 = identical). Convert to similarity as `similarity = 1 - distance`. Accept a hit only if `similarity >= similarityThreshold` (default `0.80`, from config `embeddings.similarityThreshold`).
|
|
772
772
|
5. **Optional:** TTL, tag/scope filters (e.g. require all requested tags or scope).
|
|
773
773
|
|
|
774
774
|
### 13.7.6 Dependencies for replication
|
|
@@ -792,7 +792,7 @@ Node must support native addons for `better-sqlite3` and `sqlite-vec`. First run
|
|
|
792
792
|
| Pooling/norm | mean pooling, normalize: true |
|
|
793
793
|
| DB | SQLite (better-sqlite3) |
|
|
794
794
|
| Vector index | sqlite-vec vec0, cosine |
|
|
795
|
-
| Match rule | cosine similarity ≥ 0.
|
|
795
|
+
| Match rule | cosine similarity ≥ 0.80 (configurable via `embeddings.similarityThreshold`) |
|
|
796
796
|
|
|
797
797
|
---
|
|
798
798
|
|
|
@@ -951,7 +951,7 @@ src/
|
|
|
951
951
|
"workerConcurrency": 2,
|
|
952
952
|
"maxRetries": 3,
|
|
953
953
|
"queueOnWrite": true,
|
|
954
|
-
"similarityThreshold": 0.
|
|
954
|
+
"similarityThreshold": 0.80
|
|
955
955
|
},
|
|
956
956
|
"search": {
|
|
957
957
|
"mode": "fts-only",
|
package/src/config.js
CHANGED
|
@@ -6,8 +6,9 @@ import { getDb } from '../../db/sqlite.js';
|
|
|
6
6
|
const MODEL_NAME = 'Xenova_multilingual_e5_large';
|
|
7
7
|
const STATUS = { PENDING: 'pending', PROCESSING: 'processing', COMPLETED: 'completed', FAILED: 'failed', STALE: 'stale' };
|
|
8
8
|
|
|
9
|
+
/** Queue key for jobs; use a single normalized name so enqueue and process always match. */
|
|
9
10
|
function getModelName(config) {
|
|
10
|
-
return
|
|
11
|
+
return MODEL_NAME;
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -89,7 +89,7 @@ async function search(config, queryEmbedding, k = 20, options = {}) {
|
|
|
89
89
|
JOIN ${cacheTable} c ON c.id = v.rowid
|
|
90
90
|
WHERE v.embedding MATCH ? AND k = ?`
|
|
91
91
|
).all(emb, Math.min(k, 100));
|
|
92
|
-
const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.
|
|
92
|
+
const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.80;
|
|
93
93
|
return rows
|
|
94
94
|
.filter((r) => (1 - r.distance) >= threshold)
|
|
95
95
|
.map((r) => ({ chunk_id: r.chunk_id, document_id: r.document_id, distance: r.distance }));
|
|
@@ -18,19 +18,29 @@ function getBm25Weights(config) {
|
|
|
18
18
|
// FTS5 uses "/" for NEAR (e.g. term/5) and "-" for NOT; dash-like chars between words break parsing.
|
|
19
19
|
const FTS5_NEAR_SLASH = /\//g;
|
|
20
20
|
const FTS5_NOT_HYPHEN = /(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g; // hyphen, en/em-dash, minus
|
|
21
|
+
// FTS5 operators are case-sensitive; normalizeForFts lowercases the string, so we restore them after.
|
|
22
|
+
const FTS5_OPERATOR_RESTORE = [
|
|
23
|
+
[/\band\b/g, 'AND'],
|
|
24
|
+
[/\bor\b/g, 'OR'],
|
|
25
|
+
[/\bnot\b/g, 'NOT']
|
|
26
|
+
];
|
|
21
27
|
|
|
22
28
|
/**
|
|
23
29
|
* Normalize query for FTS5: same as indexed text (NFD, remove diacritics, lowercase).
|
|
24
30
|
* Replaces "/" with space so it is not interpreted as FTS5 NEAR syntax (e.g. term/5).
|
|
25
31
|
* Replaces "-" (and Unicode dashes) between word chars with space so "alchemy-tycoon" is not parsed as "alchemy" NOT "tycoon".
|
|
26
|
-
*
|
|
32
|
+
* Restores FTS5 operators AND, OR, NOT to uppercase after normalization (FTS5 requires them case-sensitive).
|
|
27
33
|
*/
|
|
28
34
|
function normalizeQuery(q) {
|
|
29
35
|
if (q == null || typeof q !== 'string') return '';
|
|
30
36
|
const trimmed = q.trim();
|
|
31
37
|
const slashSafe = trimmed.replace(FTS5_NEAR_SLASH, ' ');
|
|
32
38
|
const hyphenSafe = slashSafe.replace(FTS5_NOT_HYPHEN, ' ');
|
|
33
|
-
|
|
39
|
+
let out = normalizeForFts(hyphenSafe);
|
|
40
|
+
for (const [re, replacement] of FTS5_OPERATOR_RESTORE) {
|
|
41
|
+
out = out.replace(re, replacement);
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
/**
|
package/src/mcp/server.js
CHANGED
|
@@ -44,6 +44,54 @@ function makeQueryEmbedder(child) {
|
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Start the embedding worker and set config.getQueryEmbedding for use by mem_search.
|
|
49
|
+
* Used by the CLI when running tools with --embeddings so that semantic search works.
|
|
50
|
+
* Returns a cleanup function to kill the worker and remove the temp config file.
|
|
51
|
+
*/
|
|
52
|
+
function startEmbeddingWorker(config) {
|
|
53
|
+
if (!config?.embeddings?.enabled || config.getQueryEmbedding) {
|
|
54
|
+
return () => {};
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
embeddingWorkerConfigPath = path.join(
|
|
58
|
+
os.tmpdir(),
|
|
59
|
+
`mixgram-worker-config-${process.pid}-${Date.now()}.json`
|
|
60
|
+
);
|
|
61
|
+
fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
|
|
62
|
+
const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
|
|
63
|
+
embeddingWorkerChild = fork(workerPath, [], {
|
|
64
|
+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
65
|
+
env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
|
|
66
|
+
});
|
|
67
|
+
config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
|
|
68
|
+
const cleanup = () => {
|
|
69
|
+
if (embeddingWorkerChild) {
|
|
70
|
+
try { embeddingWorkerChild.kill(); } catch (_) {}
|
|
71
|
+
embeddingWorkerChild = null;
|
|
72
|
+
}
|
|
73
|
+
config.getQueryEmbedding = null;
|
|
74
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
75
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
76
|
+
embeddingWorkerConfigPath = null;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
embeddingWorkerChild.on('exit', () => {
|
|
80
|
+
config.getQueryEmbedding = null;
|
|
81
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
82
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
83
|
+
embeddingWorkerConfigPath = null;
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
return cleanup;
|
|
87
|
+
} catch (err) {
|
|
88
|
+
if (typeof process !== 'undefined' && process.stderr) {
|
|
89
|
+
process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
|
|
90
|
+
}
|
|
91
|
+
return () => {};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
47
95
|
function createServer(configOverrides = {}, baseDir = null, projectBaseDir = null) {
|
|
48
96
|
const config = loadConfig(configOverrides, baseDir, projectBaseDir);
|
|
49
97
|
const mcpServer = new McpServer(
|
|
@@ -98,42 +146,19 @@ async function run(configOverrides = {}, baseDir = null, projectBaseDir = null)
|
|
|
98
146
|
}
|
|
99
147
|
}
|
|
100
148
|
if (config.embeddings?.enabled) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
`mixgram-worker-config-${process.pid}-${Date.now()}.json`
|
|
105
|
-
);
|
|
106
|
-
fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
|
|
107
|
-
const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
|
|
108
|
-
embeddingWorkerChild = fork(workerPath, [], {
|
|
109
|
-
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
110
|
-
env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
|
|
111
|
-
});
|
|
112
|
-
config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
|
|
113
|
-
embeddingWorkerChild.on('exit', (code, signal) => {
|
|
149
|
+
const cleanup = startEmbeddingWorker(config);
|
|
150
|
+
if (cleanup) {
|
|
151
|
+
embeddingWorkerChild?.on('exit', (code, signal) => {
|
|
114
152
|
if (code !== 0 && code != null && process.stderr) {
|
|
115
153
|
process.stderr.write(`[mixgram] embeddings worker exited: code=${code} signal=${signal}\n`);
|
|
116
154
|
}
|
|
117
|
-
|
|
118
|
-
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
119
|
-
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
120
|
-
embeddingWorkerConfigPath = null;
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
process.on('exit', () => {
|
|
124
|
-
if (embeddingWorkerChild) embeddingWorkerChild.kill();
|
|
125
|
-
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
126
|
-
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
127
|
-
}
|
|
155
|
+
cleanup();
|
|
128
156
|
});
|
|
129
|
-
|
|
130
|
-
if (typeof process !== 'undefined' && process.stderr) {
|
|
131
|
-
process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
|
|
132
|
-
}
|
|
157
|
+
process.on('exit', () => cleanup());
|
|
133
158
|
}
|
|
134
159
|
}
|
|
135
160
|
const transport = new StdioServerTransport();
|
|
136
161
|
await mcpServer.connect(transport);
|
|
137
162
|
}
|
|
138
163
|
|
|
139
|
-
export { createServer, run };
|
|
164
|
+
export { createServer, run, startEmbeddingWorker };
|
package/src/mcp/tools.js
CHANGED
|
@@ -17,12 +17,13 @@ function suggestTopicKey(title, content = '', type = 'generated_note') {
|
|
|
17
17
|
function createToolHandlers(config) {
|
|
18
18
|
return {
|
|
19
19
|
mem_save: async (args) => {
|
|
20
|
+
const topicKey = args.topic_key ?? suggestTopicKey(args.title, args.content ?? '', args.type || 'generated_note');
|
|
20
21
|
const result = saveDocument(config, {
|
|
21
22
|
title: args.title,
|
|
22
23
|
type: args.type || 'generated_note',
|
|
23
24
|
scope: args.scope || 'project',
|
|
24
25
|
project: args.project ?? null,
|
|
25
|
-
topic_key:
|
|
26
|
+
topic_key: topicKey,
|
|
26
27
|
content: args.content ?? '',
|
|
27
28
|
session_id: args.session_id ?? null,
|
|
28
29
|
id: args.id ?? null,
|
|
@@ -54,12 +55,58 @@ function createToolHandlers(config) {
|
|
|
54
55
|
},
|
|
55
56
|
|
|
56
57
|
mem_search: async (args) => {
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
58
|
+
const scopeMode = args.scope_mode || config.search?.defaultScopeMode || 'merged';
|
|
59
|
+
const project = args.project ?? null;
|
|
60
|
+
const limit = args.limit ?? config.search?.defaultLimit ?? 10;
|
|
61
|
+
const query = args.query?.trim();
|
|
62
|
+
if (!query) {
|
|
63
|
+
return { content: [{ type: 'text', text: JSON.stringify({ results: [] }) }] };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const ftsResults = await search(config, { query, scopeMode, project, limit });
|
|
67
|
+
let results = [...ftsResults];
|
|
68
|
+
|
|
69
|
+
if (config.embeddings?.enabled && config.getQueryEmbedding) {
|
|
70
|
+
try {
|
|
71
|
+
const { getVectorStore } = await import('../core/embeddings/vectorStore.js');
|
|
72
|
+
const vectorStore = await getVectorStore(config);
|
|
73
|
+
if (vectorStore) {
|
|
74
|
+
const queryVec = await config.getQueryEmbedding(query);
|
|
75
|
+
if (queryVec) {
|
|
76
|
+
const semanticThreshold = config.embeddings?.similarityThreshold ?? 0.75;
|
|
77
|
+
const vecResults = await vectorStore.search(config, queryVec, Math.max(limit * 2, 20), { similarityThreshold: semanticThreshold });
|
|
78
|
+
const db = getDb(config);
|
|
79
|
+
const seen = new Set(results.map((r) => r.documentId));
|
|
80
|
+
const semanticList = [];
|
|
81
|
+
for (const v of vecResults) {
|
|
82
|
+
if (seen.has(v.document_id)) continue;
|
|
83
|
+
const row = db.prepare(
|
|
84
|
+
'SELECT id, title, topic_key, type, scope, project, created_at, body FROM documents WHERE id = ? AND deleted_at IS NULL'
|
|
85
|
+
).get(v.document_id);
|
|
86
|
+
if (!row) continue;
|
|
87
|
+
if (scopeMode === 'project-only' && (row.scope !== 'project' || row.project !== project)) continue;
|
|
88
|
+
if (scopeMode === 'home-only' && row.scope !== 'home') continue;
|
|
89
|
+
if (scopeMode === 'merged' && project && row.scope === 'project' && row.project !== project) continue;
|
|
90
|
+
seen.add(v.document_id);
|
|
91
|
+
const snippet = (row.body || '').slice(0, 200).trim();
|
|
92
|
+
semanticList.push({
|
|
93
|
+
documentId: row.id,
|
|
94
|
+
title: row.title,
|
|
95
|
+
topicKey: row.topic_key,
|
|
96
|
+
type: row.type,
|
|
97
|
+
scope: row.scope,
|
|
98
|
+
project: row.project,
|
|
99
|
+
created: row.created_at || null,
|
|
100
|
+
snippet: snippet || '',
|
|
101
|
+
score: 1 - v.distance
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
results = [...semanticList, ...results].slice(0, limit);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
} catch (_) {}
|
|
108
|
+
}
|
|
109
|
+
|
|
63
110
|
return { content: [{ type: 'text', text: JSON.stringify({ results }) }] };
|
|
64
111
|
},
|
|
65
112
|
|