mixgram 1.2.6 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mixgram/index.db +0 -0
- package/README.md +72 -3
- package/bin/mixgram.js +17 -1
- package/docs/architecture/note.md +7 -0
- package/docs/decisions/note.md +7 -0
- package/docs/decisions/where-we-store-memory.md +7 -0
- package/docs/decisions/x.md +7 -0
- package/examples/embedding-demo.js +154 -0
- package/package.json +2 -1
- package/spec/fixtures/realistic-content.js +9 -0
- package/spec/scenarios/scenarios.js +72 -3
- package/spec/sdd.md +3 -3
- package/spec/test-large-document.js +1 -1
- package/src/config.js +1 -1
- package/src/core/documents/documents.js +56 -23
- package/src/core/embeddings/queue.js +2 -1
- package/src/core/embeddings/vectorStore.js +1 -1
- package/src/core/indexing/indexer.js +146 -19
- package/src/core/indexing/reindex.js +44 -3
- package/src/core/search/search.js +19 -4
- package/src/fs/watcher.js +8 -4
- package/src/mcp/server.js +54 -29
- package/src/mcp/tools.js +54 -7
- package/src/utils/markdown.js +5 -4
|
Binary file
|
package/README.md
CHANGED
|
@@ -38,17 +38,54 @@ Then use the `mixgram` command in your client config (see below). No extra depen
|
|
|
38
38
|
| `mixgram help <tool>` | Show options for a specific MCP tool. |
|
|
39
39
|
| `mixgram <tool> [--key value ...]` | Run any MCP tool from the command line (see below). |
|
|
40
40
|
|
|
41
|
-
**MCP tools from the CLI** — Every MCP tool is automatically available as a subcommand. No extra wiring is needed: if a tool is added to the internal registry, it is exposed both to MCP clients and to the CLI. Use `mixgram help` to list tools and `mixgram help <tool>` (or `mixgram <tool> --help`) to see a tool’s options.
|
|
41
|
+
**MCP tools from the CLI** — Every MCP tool is automatically available as a subcommand. No extra wiring is needed: if a tool is added to the internal registry, it is exposed both to MCP clients and to the CLI. Use `mixgram help` to list tools and `mixgram help <tool>` (or `mixgram <tool> --help`) to see a tool’s options.
|
|
42
|
+
|
|
43
|
+
### Command-line examples
|
|
42
44
|
|
|
43
45
|
```bash
|
|
46
|
+
# Stats and health
|
|
44
47
|
mixgram mem_stats
|
|
48
|
+
mixgram mem_stats --embeddings # enable embeddings for this run (or use env/config, see below)
|
|
49
|
+
|
|
50
|
+
# Save a note
|
|
51
|
+
mixgram mem_save --title "My note" --content "Search" --type decision --project my-app
|
|
52
|
+
mixgram mem_save --title "Other note" --content "Model Context Protocol" --type architecture
|
|
53
|
+
|
|
54
|
+
# Search (default: merged scope, limit 10)
|
|
55
|
+
mixgram mem_search --query "index"
|
|
45
56
|
mixgram mem_search --query "index" --limit 5
|
|
46
|
-
mixgram
|
|
57
|
+
mixgram mem_search --query "indexing" --scope-mode project-only --project my-app
|
|
58
|
+
|
|
59
|
+
# OR-style query: use FTS5 keyword OR (and optional quoted phrases)
|
|
60
|
+
mixgram mem_search --query 'mcp OR "model context protocol"' --limit 10
|
|
61
|
+
mixgram mem_search --query 'cursor OR "model context protocol"' --limit 5
|
|
62
|
+
|
|
63
|
+
# Reindex from disk (e.g. after editing Markdown by hand)
|
|
47
64
|
mixgram mem_reindex --full
|
|
65
|
+
|
|
66
|
+
# Context (recent or by query)
|
|
67
|
+
mixgram mem_context --query "indexing decisions" --project my-app --limit 5
|
|
68
|
+
mixgram mem_context --limit 5
|
|
48
69
|
```
|
|
49
70
|
|
|
71
|
+
**Example: index and find by embedding (not FTS)** — Save a note with one wording, then search with different words that mean the same. FTS finds nothing; semantic search returns the doc:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# 1. Index a document (content about storing memory)
|
|
75
|
+
mixgram mem_save --title "Where we store memory" --content "We keep agent memory in Markdown under docs/." --type decision --project demo --embeddings
|
|
76
|
+
|
|
77
|
+
# 2. Search by meaning: query has no literal overlap with the text
|
|
78
|
+
mixgram mem_search --query "where is the agent memory stored" --project demo --embeddings --limit 5
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
The first run embeds the document (worker runs in-process and processes the queue after save; the first time can take a few seconds while the embedding model loads). The second run uses the embedding worker to turn the query into a vector and returns the document by similarity. You can also use `npm run example:embedding` for a self-contained demo in a temp dir.
|
|
82
|
+
|
|
83
|
+
**Query syntax (FTS5):** Search uses SQLite FTS5. You can use the operators `OR`, `AND`, `NOT` and quoted phrases. For an OR-style search (e.g. “mcp” or “model context protocol”), use the word `OR` in the query, not the pipe character: `--query 'mcp OR "model context protocol"'`. Unquoted terms are ANDed by default; use `OR` explicitly to match any of several terms.
|
|
84
|
+
|
|
50
85
|
Arguments are passed as `--key value`. Booleans use `--flag` or `--no-flag`. Arrays use repeated flags: `--tags a --tags b`.
|
|
51
86
|
|
|
87
|
+
**Enabling embeddings from the CLI:** For the MCP server use `mixgram mcp --embeddings`. For one-off tool runs (e.g. `mixgram mem_stats`, `mixgram mem_search`) you can pass `--embeddings` after the tool name, set the env var `MIXGRAM_EMBEDDINGS_ENABLED=1` (or `true`), or enable in `.mixgram/config.json` with `"embeddings": { "enabled": true }`. Example: `mixgram mem_search --query "something" --embeddings --limit 5`.
|
|
88
|
+
|
|
52
89
|
**`mcp` options** (and env / config file):
|
|
53
90
|
|
|
54
91
|
| Option | Env | Description |
|
|
@@ -291,7 +328,7 @@ When `embeddings.enabled` is `true` and the optional embedding stack is availabl
|
|
|
291
328
|
await run({
|
|
292
329
|
embeddings: {
|
|
293
330
|
enabled: true,
|
|
294
|
-
similarityThreshold: 0.
|
|
331
|
+
similarityThreshold: 0.80,
|
|
295
332
|
maxRetries: 3
|
|
296
333
|
},
|
|
297
334
|
search: {
|
|
@@ -321,6 +358,38 @@ mem_search({
|
|
|
321
358
|
|
|
322
359
|
With embeddings enabled and vectors ready, results combine literal matches (FTS) and semantic similarity (e.g. paraphrases).
|
|
323
360
|
|
|
361
|
+
### Example: indexing and search (embedding effectiveness)
|
|
362
|
+
|
|
363
|
+
To see the difference between **FTS-only** and **semantic** search in one run, run the demo script (uses a temp dir; first run may download the embedding model):
|
|
364
|
+
|
|
365
|
+
```bash
|
|
366
|
+
node examples/embedding-demo.js
|
|
367
|
+
# or
|
|
368
|
+
npm run example:embedding
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
The script:
|
|
372
|
+
|
|
373
|
+
1. **Indexes** three short documents (e.g. “memoria en Markdown”, “índice derivado”, “persistencia de sesión”).
|
|
374
|
+
2. **Processes the embedding queue** so each document gets a vector.
|
|
375
|
+
3. **Runs FTS search** for a query that does *not* appear literally in the text (e.g. “dónde se almacena la memoria del agente”) — FTS returns nothing.
|
|
376
|
+
4. **Runs semantic (vector) search** for the same query — returns the relevant document by meaning.
|
|
377
|
+
|
|
378
|
+
Example output:
|
|
379
|
+
|
|
380
|
+
```text
|
|
381
|
+
--- 3. FTS search (text only) ---
|
|
382
|
+
Query: "where is the agent memory stored"
|
|
383
|
+
(FTS finds no literal match; these words are not in the documents.)
|
|
384
|
+
|
|
385
|
+
--- 4. Semantic search (embeddings) ---
|
|
386
|
+
Query: "where is the agent memory stored"
|
|
387
|
+
- [Session persistence] similarity=0.856
|
|
388
|
+
Persist the session context across agent restarts. The important state...
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
So: **FTS** only matches when the query words appear in the document; **embeddings** can retrieve documents that share the same meaning without sharing the same words.
|
|
392
|
+
|
|
324
393
|
---
|
|
325
394
|
|
|
326
395
|
## Tests
|
package/bin/mixgram.js
CHANGED
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* mixgram mcp [options] — run MCP server (stdio)
|
|
6
6
|
* mixgram setup <client> — add Mixgram to Cursor / Gemini CLI / Codex config
|
|
7
7
|
*/
|
|
8
|
-
import { run } from '../src/mcp/server.js';
|
|
8
|
+
import { run, startEmbeddingWorker } from '../src/mcp/server.js';
|
|
9
9
|
import { loadConfig } from '../src/config.js';
|
|
10
10
|
import { closeDb } from '../src/db/sqlite.js';
|
|
11
11
|
import { createToolHandlers } from '../src/mcp/tools.js';
|
|
12
12
|
import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../src/mcp/cli-adapter.js';
|
|
13
|
+
import { processNextJob, enqueueChunks } from '../src/core/embeddings/queue.js';
|
|
13
14
|
import fs from 'fs';
|
|
14
15
|
import path from 'path';
|
|
15
16
|
import os from 'os';
|
|
@@ -327,14 +328,29 @@ async function main() {
|
|
|
327
328
|
const args = parseToolArgs(toolDef, toolArgv);
|
|
328
329
|
const { overrides, baseDir, projectBaseDir } = loadCliConfig();
|
|
329
330
|
const config = loadConfig(overrides, baseDir, projectBaseDir);
|
|
331
|
+
let cleanupEmbeddingWorker = () => {};
|
|
332
|
+
if (config.embeddings?.enabled) {
|
|
333
|
+
cleanupEmbeddingWorker = startEmbeddingWorker(config);
|
|
334
|
+
}
|
|
330
335
|
const handlers = createToolHandlers(config);
|
|
331
336
|
try {
|
|
332
337
|
const result = await handlers[SUBCOMMAND](args);
|
|
338
|
+
if ((SUBCOMMAND === 'mem_save' || SUBCOMMAND === 'mem_update') && config.embeddings?.enabled) {
|
|
339
|
+
let docId = null;
|
|
340
|
+
try {
|
|
341
|
+
const out = JSON.parse(result?.content?.[0]?.text || '{}');
|
|
342
|
+
if (out.id) docId = out.id;
|
|
343
|
+
} catch (_) {}
|
|
344
|
+
if (docId) enqueueChunks(config, [docId]);
|
|
345
|
+
while (await processNextJob(config)) {}
|
|
346
|
+
}
|
|
333
347
|
const text = result?.content?.[0]?.text;
|
|
334
348
|
if (text != null) console.log(text);
|
|
335
349
|
} catch (err) {
|
|
336
350
|
console.error(err);
|
|
337
351
|
process.exit(1);
|
|
352
|
+
} finally {
|
|
353
|
+
cleanupEmbeddingWorker();
|
|
338
354
|
}
|
|
339
355
|
return;
|
|
340
356
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Example: indexing + search showing embedding effectiveness.
|
|
4
|
+
*
|
|
5
|
+
* Concept:
|
|
6
|
+
* - FTS (full-text search) only matches when query words appear in the document.
|
|
7
|
+
* - Semantic search (embeddings) matches by meaning: different wording, same idea.
|
|
8
|
+
* This script indexes a few documents, then runs the same query with FTS (no match)
|
|
9
|
+
* and with vector search (finds the relevant doc by similarity).
|
|
10
|
+
*
|
|
11
|
+
* Steps:
|
|
12
|
+
* 1. Index several short documents with related content but different wording.
|
|
13
|
+
* 2. Process the embedding queue so each document gets a vector.
|
|
14
|
+
* 3. Compare FTS (text-only) vs semantic (vector) search.
|
|
15
|
+
*
|
|
16
|
+
* Requires: npm install (includes @huggingface/transformers and sqlite-vec).
|
|
17
|
+
* Usage: node examples/embedding-demo.js
|
|
18
|
+
*/
|
|
19
|
+
import fs from 'fs';
|
|
20
|
+
import path from 'path';
|
|
21
|
+
import os from 'os';
|
|
22
|
+
import { loadConfig } from '../src/config.js';
|
|
23
|
+
import { saveDocument } from '../src/core/documents/documents.js';
|
|
24
|
+
import { search as searchFts } from '../src/core/search/search.js';
|
|
25
|
+
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
26
|
+
import { getEmbedder } from '../src/core/embeddings/embedder.js';
|
|
27
|
+
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
28
|
+
|
|
29
|
+
async function run() {
|
|
30
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-embedding-demo-'));
|
|
31
|
+
const projectRoot = path.join(tmpDir, 'repo');
|
|
32
|
+
const docsDir = path.join(projectRoot, 'docs');
|
|
33
|
+
fs.mkdirSync(docsDir, { recursive: true });
|
|
34
|
+
|
|
35
|
+
const overrides = {
|
|
36
|
+
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
37
|
+
projectMemoryRoot: path.join(projectRoot, 'docs'),
|
|
38
|
+
sqlitePath: path.join(tmpDir, 'index.db'),
|
|
39
|
+
watch: false,
|
|
40
|
+
indexing: { reindexOnStartup: false },
|
|
41
|
+
embeddings: {
|
|
42
|
+
enabled: true,
|
|
43
|
+
similarityThreshold: 0.80 // more permissive so the demo shows more results
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const config = loadConfig(overrides, tmpDir, projectRoot);
|
|
48
|
+
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
49
|
+
fs.mkdirSync(config.homeMemoryRoot, { recursive: true });
|
|
50
|
+
|
|
51
|
+
const db = getDb(config);
|
|
52
|
+
|
|
53
|
+
console.log('--- 1. Indexing: save documents ---\n');
|
|
54
|
+
|
|
55
|
+
const docs = [
|
|
56
|
+
{
|
|
57
|
+
title: 'Memory in Markdown',
|
|
58
|
+
type: 'decision',
|
|
59
|
+
content: 'We store the agent\'s memory in Markdown files under docs/. That way the documentation is visible and editable by hand.'
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
title: 'Derived index',
|
|
63
|
+
type: 'decision',
|
|
64
|
+
content: 'The search index is derived from the Markdown files using SQLite FTS5. There is no database as source of truth other than the .md files.'
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
title: 'Session persistence',
|
|
68
|
+
type: 'decision',
|
|
69
|
+
content: 'Persist the session context across agent restarts. The important state lives in the memory documents.'
|
|
70
|
+
}
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
for (const d of docs) {
|
|
74
|
+
const out = saveDocument(config, {
|
|
75
|
+
title: d.title,
|
|
76
|
+
type: d.type,
|
|
77
|
+
scope: 'project',
|
|
78
|
+
project: 'demo',
|
|
79
|
+
content: d.content
|
|
80
|
+
});
|
|
81
|
+
console.log(` Saved: ${d.title} → ${out.id}`);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
await new Promise((r) => setImmediate(r));
|
|
85
|
+
await new Promise((r) => setImmediate(r));
|
|
86
|
+
|
|
87
|
+
console.log('\n--- 2. Process embedding queue ---\n');
|
|
88
|
+
let processed = 0;
|
|
89
|
+
while (await processNextJob(config)) processed++;
|
|
90
|
+
console.log(` Jobs processed: ${processed}`);
|
|
91
|
+
|
|
92
|
+
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
93
|
+
const vecCount = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
94
|
+
console.log(` Vectors in index: ${vecCount}\n`);
|
|
95
|
+
|
|
96
|
+
console.log('--- 3. FTS search (text only) ---\n');
|
|
97
|
+
|
|
98
|
+
const queryLiteral = 'Markdown';
|
|
99
|
+
const ftsResults = searchFts(config, { query: queryLiteral, project: 'demo', limit: 5 });
|
|
100
|
+
console.log(` Query: "${queryLiteral}"`);
|
|
101
|
+
ftsResults.forEach((r, i) => {
|
|
102
|
+
console.log(` ${i + 1}. [${r.title}] score=${r.score.toFixed(2)} — ${r.snippet.slice(0, 60)}...`);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
const querySemantic = 'where is the agent memory stored';
|
|
106
|
+
const ftsSemantic = searchFts(config, { query: querySemantic, project: 'demo', limit: 5 });
|
|
107
|
+
console.log(`\n Query: "${querySemantic}"`);
|
|
108
|
+
if (ftsSemantic.length === 0) {
|
|
109
|
+
console.log(' (FTS finds no literal match; these words are not in the documents.)');
|
|
110
|
+
} else {
|
|
111
|
+
ftsSemantic.forEach((r, i) => {
|
|
112
|
+
console.log(` ${i + 1}. [${r.title}] — ${r.snippet.slice(0, 60)}...`);
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
console.log('\n--- 4. Semantic search (embeddings) ---\n');
|
|
117
|
+
|
|
118
|
+
const embedder = await getEmbedder(config);
|
|
119
|
+
const { getVectorStore } = await import('../src/core/embeddings/vectorStore.js');
|
|
120
|
+
const vectorStore = await getVectorStore(config);
|
|
121
|
+
|
|
122
|
+
if (!embedder || !vectorStore) {
|
|
123
|
+
console.log(' Embeddings not available (missing @huggingface/transformers or sqlite-vec).');
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const queryVec = await embedder.embed(querySemantic);
|
|
128
|
+
const vectorResults = await vectorStore.search(config, queryVec, 10, { similarityThreshold: 0.80 });
|
|
129
|
+
|
|
130
|
+
console.log(` Query: "${querySemantic}"`);
|
|
131
|
+
console.log(' Results by semantic similarity:\n');
|
|
132
|
+
|
|
133
|
+
for (const v of vectorResults) {
|
|
134
|
+
const row = db.prepare('SELECT id, title, body FROM documents WHERE id = ?').get(v.document_id);
|
|
135
|
+
const similarity = (1 - v.distance).toFixed(3);
|
|
136
|
+
const snippet = (row?.body || '').slice(0, 80);
|
|
137
|
+
console.log(` - [${row?.title ?? v.document_id}] similarity=${similarity}`);
|
|
138
|
+
console.log(` ${snippet}...`);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
console.log('\n--- Summary ---');
|
|
142
|
+
console.log('FTS only finds when the query words appear in the text.');
|
|
143
|
+
console.log('Embedding search finds documents that share the same meaning even without shared words.\n');
|
|
144
|
+
|
|
145
|
+
closeDb();
|
|
146
|
+
try {
|
|
147
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
148
|
+
} catch (_) {}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
run().catch((err) => {
|
|
152
|
+
console.error(err);
|
|
153
|
+
process.exit(1);
|
|
154
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixgram",
|
|
3
|
-
"version": "1.2
|
|
3
|
+
"version": "1.3.2",
|
|
4
4
|
"description": "MCP memory server with Markdown source of truth and SQLite FTS5.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
"bench:search": "node spec/bench-search.js",
|
|
41
41
|
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
42
42
|
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
43
|
+
"example:embedding": "node examples/embedding-demo.js",
|
|
43
44
|
"duplication": "jscpd src",
|
|
44
45
|
"deadcode": "knip",
|
|
45
46
|
"quality": "npm run duplication && npm run deadcode"
|
|
@@ -95,3 +95,12 @@ export const semanticFallbackDoc = {
|
|
|
95
95
|
topic_key: 'test/semantic-fallback',
|
|
96
96
|
content: 'Text for semantic fallback test.'
|
|
97
97
|
};
|
|
98
|
+
|
|
99
|
+
/** Used to test FTS5 OR query: save then search with "mcp OR \"model context protocol\"" */
|
|
100
|
+
export const noteMcpArchitecture = {
|
|
101
|
+
title: 'Other note',
|
|
102
|
+
type: 'architecture',
|
|
103
|
+
content: 'Model Context Protocol',
|
|
104
|
+
scope: 'project',
|
|
105
|
+
project: null
|
|
106
|
+
};
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
semanticPersistenceDoc,
|
|
20
20
|
semanticPersistenceQuery,
|
|
21
21
|
semanticFallbackDoc,
|
|
22
|
+
noteMcpArchitecture,
|
|
22
23
|
PROJECT_NAME
|
|
23
24
|
} from '../fixtures/realistic-content.js';
|
|
24
25
|
import { toMarkdown } from '../../src/utils/markdown.js';
|
|
@@ -198,6 +199,36 @@ export const scenarios = [
|
|
|
198
199
|
return { passed: p, failed: f };
|
|
199
200
|
}
|
|
200
201
|
},
|
|
202
|
+
{
|
|
203
|
+
name: 'Search with OR query (FTS5 operator case)',
|
|
204
|
+
goal: 'mem_save then mem_search with query using OR (e.g. mcp OR "model context protocol") returns the saved document.',
|
|
205
|
+
run: async function (ctx) {
|
|
206
|
+
const { h, reporter } = ctx;
|
|
207
|
+
const parseRes = ctx.parse || parse;
|
|
208
|
+
let p = 0, f = 0;
|
|
209
|
+
const ok = (cond, msg) => { if (cond) p++; else f++; reporter.check(msg, cond); };
|
|
210
|
+
|
|
211
|
+
reporter.startScenario(this.name, this.goal);
|
|
212
|
+
|
|
213
|
+
const saveRes = await h.mem_save(noteMcpArchitecture);
|
|
214
|
+
const saveData = parseRes(saveRes);
|
|
215
|
+
reporter.step('mem_save', { title: noteMcpArchitecture.title, type: noteMcpArchitecture.type }, { id: saveData.id, created: saveData.created });
|
|
216
|
+
ok(saveData.success && saveData.id, 'mem_save success');
|
|
217
|
+
|
|
218
|
+
const searchRes = await h.mem_search({ query: 'mcp OR "model context protocol"', limit: 10 });
|
|
219
|
+
const searchData = parseRes(searchRes);
|
|
220
|
+
reporter.step('mem_search', { query: 'mcp OR "model context protocol"' }, searchData, { highlight: ['title', 'snippet'] });
|
|
221
|
+
ok(Array.isArray(searchData.results), 'results is array');
|
|
222
|
+
ok(searchData.results.length >= 1, 'OR query returns at least one result');
|
|
223
|
+
const found = searchData.results.some(
|
|
224
|
+
(r) => (r.title && r.title.includes('Other note')) || (r.snippet && (r.snippet.toLowerCase().includes('mcp') || r.snippet.toLowerCase().includes('model context protocol')))
|
|
225
|
+
);
|
|
226
|
+
ok(found, 'saved note is in OR query results');
|
|
227
|
+
|
|
228
|
+
reporter.endScenario(p, f);
|
|
229
|
+
return { passed: p, failed: f };
|
|
230
|
+
}
|
|
231
|
+
},
|
|
201
232
|
{
|
|
202
233
|
name: 'Context for project',
|
|
203
234
|
goal: 'mem_context returns usable context (recent or query-based).',
|
|
@@ -362,7 +393,7 @@ export const scenarios = [
|
|
|
362
393
|
await h.mem_reindex({ full: true });
|
|
363
394
|
reporter.step('mem_reindex', { full: true }, {});
|
|
364
395
|
|
|
365
|
-
const searchA = await h.mem_search({ query: 'Manual edits'
|
|
396
|
+
const searchA = await h.mem_search({ query: 'Manual edits' });
|
|
366
397
|
const searchAParsed = parseRes(searchA);
|
|
367
398
|
reporter.step('mem_search (after edit)', { query: 'Manual edits' }, searchAParsed, { highlight: ['title', 'snippet'] });
|
|
368
399
|
ok(searchAParsed.results.length >= 1, 'edited phrase searchable');
|
|
@@ -392,13 +423,51 @@ export const scenarios = [
|
|
|
392
423
|
reporter.step('fullReindex', {}, reindexResult);
|
|
393
424
|
ok(reindexResult.indexed >= 1, 'full reindex indexed');
|
|
394
425
|
|
|
395
|
-
const searchAfter = await h.mem_search({ query: 'Manual edits'
|
|
426
|
+
const searchAfter = await h.mem_search({ query: 'Manual edits' });
|
|
396
427
|
ok(parseRes(searchAfter).results.length >= 1, 'searchable after rebuild');
|
|
397
428
|
|
|
398
429
|
reporter.endScenario(p, f);
|
|
399
430
|
return { passed: p, failed: f };
|
|
400
431
|
}
|
|
401
432
|
},
|
|
433
|
+
{
|
|
434
|
+
name: 'Move file and reindex (path is source of location)',
|
|
435
|
+
goal: 'Moving a .md file to another folder: incremental reindex desindexa old path and indexa new path; doc remains findable by id.',
|
|
436
|
+
run: async function (ctx) {
|
|
437
|
+
const { h, reporter, config, fs } = ctx;
|
|
438
|
+
const parseRes = ctx.parse || parse;
|
|
439
|
+
let p = 0, f = 0;
|
|
440
|
+
const ok = (cond, msg) => { if (cond) p++; else f++; reporter.check(msg, cond); };
|
|
441
|
+
if (!ctx.shared?.decisionDoc) throw new Error('Requires decisionDoc');
|
|
442
|
+
|
|
443
|
+
reporter.startScenario(this.name, this.goal);
|
|
444
|
+
|
|
445
|
+
const docId = ctx.shared.decisionDoc.id;
|
|
446
|
+
const oldPath = ctx.shared.decisionDoc.path;
|
|
447
|
+
if (!fs.existsSync(oldPath)) throw new Error('decisionDoc path missing');
|
|
448
|
+
const raw = fs.readFileSync(oldPath, 'utf8');
|
|
449
|
+
const rootDir = path.dirname(path.dirname(oldPath));
|
|
450
|
+
const architectureDir = path.join(rootDir, 'architecture');
|
|
451
|
+
if (!fs.existsSync(architectureDir)) fs.mkdirSync(architectureDir, { recursive: true });
|
|
452
|
+
const newPath = path.join(architectureDir, path.basename(oldPath));
|
|
453
|
+
fs.writeFileSync(newPath, raw);
|
|
454
|
+
fs.unlinkSync(oldPath);
|
|
455
|
+
reporter.step('Move file', { from: oldPath, to: newPath }, {});
|
|
456
|
+
|
|
457
|
+
const reindexRes = await h.mem_reindex({ full: false });
|
|
458
|
+
const reindexParsed = parseRes(reindexRes);
|
|
459
|
+
reporter.step('mem_reindex', { full: false }, reindexParsed);
|
|
460
|
+
ok(reindexParsed.indexed >= 1 || reindexParsed.removed >= 1, 'reindex touched index');
|
|
461
|
+
|
|
462
|
+
const obsRes = await h.mem_get_observation({ id: docId });
|
|
463
|
+
const obs = parseRes(obsRes);
|
|
464
|
+
ok(!obs.error && obs.id === docId, 'get_observation by id still works after move');
|
|
465
|
+
ok(obs.title != null || (obs.content && obs.content.length > 0), 'observation has title or content');
|
|
466
|
+
|
|
467
|
+
reporter.endScenario(p, f);
|
|
468
|
+
return { passed: p, failed: f };
|
|
469
|
+
}
|
|
470
|
+
},
|
|
402
471
|
{
|
|
403
472
|
name: 'No duplicates after incremental reindex',
|
|
404
473
|
goal: 'Running incremental reindex without file changes does not duplicate documents.',
|
|
@@ -433,7 +502,7 @@ export const scenarios = [
|
|
|
433
502
|
|
|
434
503
|
reporter.startScenario(this.name, this.goal);
|
|
435
504
|
|
|
436
|
-
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged'
|
|
505
|
+
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged' });
|
|
437
506
|
const homeResults = parseRes(homeSearch).results;
|
|
438
507
|
reporter.step('mem_search (merged)', { query: 'reindex', scope_mode: 'merged' }, { results: homeResults }, { highlight: ['scope', 'title', 'snippet'] });
|
|
439
508
|
const homeInResults = homeResults.some((r) => r.scope === 'home');
|
package/spec/sdd.md
CHANGED
|
@@ -768,7 +768,7 @@ const embedding = output.tolist(); // 1024-dim, L2-normalized
|
|
|
768
768
|
```
|
|
769
769
|
For tag/scope filtering, `k` is increased and results are filtered in application code.
|
|
770
770
|
3. **Join** with `cache_<suffix>` on `cache.id = vec.rowid` to get chunk_id, document_id, etc., then join to `document_chunks` / `documents` for full metadata and snippets.
|
|
771
|
-
4. **Similarity threshold:** sqlite-vec returns **cosine distance** (0 = identical). Convert to similarity as `similarity = 1 - distance`. Accept a hit only if `similarity >= similarityThreshold` (default `0.
|
|
771
|
+
4. **Similarity threshold:** sqlite-vec returns **cosine distance** (0 = identical). Convert to similarity as `similarity = 1 - distance`. Accept a hit only if `similarity >= similarityThreshold` (default `0.80`, from config `embeddings.similarityThreshold`).
|
|
772
772
|
5. **Optional:** TTL, tag/scope filters (e.g. require all requested tags or scope).
|
|
773
773
|
|
|
774
774
|
### 13.7.6 Dependencies for replication
|
|
@@ -792,7 +792,7 @@ Node must support native addons for `better-sqlite3` and `sqlite-vec`. First run
|
|
|
792
792
|
| Pooling/norm | mean pooling, normalize: true |
|
|
793
793
|
| DB | SQLite (better-sqlite3) |
|
|
794
794
|
| Vector index | sqlite-vec vec0, cosine |
|
|
795
|
-
| Match rule | cosine similarity ≥ 0.
|
|
795
|
+
| Match rule | cosine similarity ≥ 0.80 (configurable via `embeddings.similarityThreshold`) |
|
|
796
796
|
|
|
797
797
|
---
|
|
798
798
|
|
|
@@ -951,7 +951,7 @@ src/
|
|
|
951
951
|
"workerConcurrency": 2,
|
|
952
952
|
"maxRetries": 3,
|
|
953
953
|
"queueOnWrite": true,
|
|
954
|
-
"similarityThreshold": 0.
|
|
954
|
+
"similarityThreshold": 0.80
|
|
955
955
|
},
|
|
956
956
|
"search": {
|
|
957
957
|
"mode": "fts-only",
|
|
@@ -58,7 +58,7 @@ export function runLargeDocumentTest(config) {
|
|
|
58
58
|
const size = raw.length;
|
|
59
59
|
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
60
|
|
|
61
|
-
indexDocument(config, docPath, raw, stat.mtimeMs);
|
|
61
|
+
indexDocument(config, docPath, raw, stat.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
62
62
|
|
|
63
63
|
const db = getDb(config);
|
|
64
64
|
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
package/src/config.js
CHANGED
|
@@ -6,32 +6,65 @@ import { contentHash } from '../../utils/hash.js';
|
|
|
6
6
|
import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
|
|
7
7
|
import { getDb } from '../../db/sqlite.js';
|
|
8
8
|
|
|
9
|
+
/** Load document metadata from DB (canonical source). Returns null if not found or file missing. */
|
|
10
|
+
function getDocumentMetaById(config, id) {
|
|
11
|
+
const db = getDb(config);
|
|
12
|
+
const row = db.prepare(
|
|
13
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE id = ? AND deleted_at IS NULL'
|
|
14
|
+
).get(id);
|
|
15
|
+
if (!row) return null;
|
|
16
|
+
if (!fs.existsSync(row.path)) return null;
|
|
17
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(id);
|
|
18
|
+
const tags = tagRows.map((r) => r.tag);
|
|
19
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
20
|
+
const { body } = parse(raw);
|
|
21
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Load document metadata by topic_key + scope + project. */
|
|
25
|
+
function getDocumentMetaByTopicKey(config, topic_key, scope, project) {
|
|
26
|
+
const db = getDb(config);
|
|
27
|
+
const row = db.prepare(
|
|
28
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
29
|
+
).get(topic_key, scope, project ?? null, project ?? null);
|
|
30
|
+
if (!row) return null;
|
|
31
|
+
if (!fs.existsSync(row.path)) return null;
|
|
32
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
33
|
+
const tags = tagRows.map((r) => r.tag);
|
|
34
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
35
|
+
const { body } = parse(raw);
|
|
36
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function rowToFrontmatter(row, tags = []) {
|
|
40
|
+
return {
|
|
41
|
+
id: row.id,
|
|
42
|
+
title: row.title ?? '',
|
|
43
|
+
type: row.type ?? 'generated_note',
|
|
44
|
+
scope: row.scope ?? 'project',
|
|
45
|
+
project: row.project ?? null,
|
|
46
|
+
topic_key: row.topic_key ?? null,
|
|
47
|
+
session_id: row.session_id ?? null,
|
|
48
|
+
tool_name: row.tool_name ?? null,
|
|
49
|
+
created_at: row.created_at,
|
|
50
|
+
updated_at: row.updated_at,
|
|
51
|
+
revision_count: row.revision_count ?? 1,
|
|
52
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
53
|
+
deleted: !!row.deleted_at,
|
|
54
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
55
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
56
|
+
deleted_at: row.deleted_at ?? null
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
9
60
|
/**
|
|
10
61
|
* Resolve existing document by id (takes precedence) or by topic_key + scope + project.
|
|
62
|
+
* Metadata comes from SQLite; body from disk.
|
|
11
63
|
* @returns {{ path: string, frontmatter: object, body: string } | null}
|
|
12
64
|
*/
|
|
13
65
|
function resolveDocument(config, { id, topic_key, scope, project }) {
|
|
14
|
-
|
|
15
|
-
if (
|
|
16
|
-
const row = db.prepare('SELECT path FROM documents WHERE id = ? AND deleted_at IS NULL').get(id);
|
|
17
|
-
if (!row) return null;
|
|
18
|
-
const fullPath = row.path;
|
|
19
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
20
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
21
|
-
const parsed = parse(raw);
|
|
22
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
23
|
-
}
|
|
24
|
-
if (topic_key != null && scope != null) {
|
|
25
|
-
const row = db.prepare(
|
|
26
|
-
'SELECT id, path FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
27
|
-
).get(topic_key, scope, project ?? null, project ?? null);
|
|
28
|
-
if (!row) return null;
|
|
29
|
-
const fullPath = row.path;
|
|
30
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
31
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
32
|
-
const parsed = parse(raw);
|
|
33
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
34
|
-
}
|
|
66
|
+
if (id) return getDocumentMetaById(config, id);
|
|
67
|
+
if (topic_key != null && scope != null) return getDocumentMetaByTopicKey(config, topic_key, scope, project);
|
|
35
68
|
return null;
|
|
36
69
|
}
|
|
37
70
|
|
|
@@ -112,7 +145,7 @@ function saveDocument(config, payload) {
|
|
|
112
145
|
const raw = toMarkdown(frontmatter, body);
|
|
113
146
|
fs.writeFileSync(docPath, raw, 'utf8');
|
|
114
147
|
const stats = fs.statSync(docPath);
|
|
115
|
-
indexDocument(config, docPath, raw, stats.mtimeMs);
|
|
148
|
+
indexDocument(config, docPath, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
116
149
|
return {
|
|
117
150
|
id: frontmatter.id,
|
|
118
151
|
path: docPath,
|
|
@@ -135,7 +168,7 @@ function updateDocument(config, { id, title, content, tags }) {
|
|
|
135
168
|
const raw = toMarkdown(frontmatter, body);
|
|
136
169
|
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
137
170
|
const stats = fs.statSync(resolved.path);
|
|
138
|
-
indexDocument(config, resolved.path, raw, stats.mtimeMs);
|
|
171
|
+
indexDocument(config, resolved.path, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
139
172
|
return { id: frontmatter.id };
|
|
140
173
|
}
|
|
141
174
|
|
|
@@ -6,8 +6,9 @@ import { getDb } from '../../db/sqlite.js';
|
|
|
6
6
|
const MODEL_NAME = 'Xenova_multilingual_e5_large';
|
|
7
7
|
const STATUS = { PENDING: 'pending', PROCESSING: 'processing', COMPLETED: 'completed', FAILED: 'failed', STALE: 'stale' };
|
|
8
8
|
|
|
9
|
+
/** Queue key for jobs; use a single normalized name so enqueue and process always match. */
|
|
9
10
|
function getModelName(config) {
|
|
10
|
-
return
|
|
11
|
+
return MODEL_NAME;
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -89,7 +89,7 @@ async function search(config, queryEmbedding, k = 20, options = {}) {
|
|
|
89
89
|
JOIN ${cacheTable} c ON c.id = v.rowid
|
|
90
90
|
WHERE v.embedding MATCH ? AND k = ?`
|
|
91
91
|
).all(emb, Math.min(k, 100));
|
|
92
|
-
const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.
|
|
92
|
+
const threshold = options.similarityThreshold ?? config.embeddings?.similarityThreshold ?? 0.80;
|
|
93
93
|
return rows
|
|
94
94
|
.filter((r) => (1 - r.distance) >= threshold)
|
|
95
95
|
.map((r) => ({ chunk_id: r.chunk_id, document_id: r.document_id, distance: r.distance }));
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs';
|
|
1
3
|
import { getDb } from '../../db/sqlite.js';
|
|
2
4
|
import { parseMarkdown } from './parser.js';
|
|
3
5
|
import { contentHash } from '../../utils/hash.js';
|
|
@@ -11,43 +13,168 @@ function normalizeForFts(str) {
|
|
|
11
13
|
.trim();
|
|
12
14
|
}
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
/** Infer scope, type and project from file path. Path shape: <project>/docs/<type>/<title>.md → project = dirname(docs). */
|
|
17
|
+
function inferMetadataFromPath(config, docPath) {
|
|
18
|
+
let norm = path.normalize(path.resolve(docPath));
|
|
19
|
+
let homeRoot = config.homeMemoryRoot ? path.normalize(path.resolve(config.homeMemoryRoot)) : '';
|
|
20
|
+
let projectRoot = config.projectMemoryRoot ? path.normalize(path.resolve(config.projectMemoryRoot)) : '';
|
|
21
|
+
try {
|
|
22
|
+
if (norm && fs.existsSync(norm)) norm = fs.realpathSync(norm);
|
|
23
|
+
if (homeRoot && fs.existsSync(homeRoot)) homeRoot = fs.realpathSync(homeRoot);
|
|
24
|
+
if (projectRoot && fs.existsSync(projectRoot)) projectRoot = fs.realpathSync(projectRoot);
|
|
25
|
+
} catch (_) {}
|
|
26
|
+
|
|
27
|
+
const dirToType = {
|
|
28
|
+
architecture: 'architecture',
|
|
29
|
+
decisions: 'decision',
|
|
30
|
+
bugs: 'bug',
|
|
31
|
+
learnings: 'learning',
|
|
32
|
+
discoveries: 'discovery',
|
|
33
|
+
patterns: 'pattern',
|
|
34
|
+
reference: 'reference',
|
|
35
|
+
sessions: 'session_summary',
|
|
36
|
+
prompts: 'prompt',
|
|
37
|
+
generated: 'generated_note'
|
|
38
|
+
};
|
|
39
|
+
let scope = 'project';
|
|
40
|
+
let type = 'generated_note';
|
|
41
|
+
let project = null;
|
|
42
|
+
|
|
43
|
+
const underHome = homeRoot && (() => {
|
|
44
|
+
if (norm === homeRoot) return true;
|
|
45
|
+
const sep = path.sep;
|
|
46
|
+
return norm.startsWith(homeRoot + sep);
|
|
47
|
+
})();
|
|
48
|
+
const underProject = projectRoot && (() => {
|
|
49
|
+
if (norm === projectRoot) return true;
|
|
50
|
+
const sep = path.sep;
|
|
51
|
+
return norm.startsWith(projectRoot + sep);
|
|
52
|
+
})();
|
|
53
|
+
|
|
54
|
+
if (underHome) {
|
|
55
|
+
scope = 'home';
|
|
56
|
+
const rel = path.relative(homeRoot, path.dirname(norm));
|
|
57
|
+
const firstDir = rel.split(path.sep)[0];
|
|
58
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
59
|
+
} else if (underProject) {
|
|
60
|
+
scope = 'project';
|
|
61
|
+
const rel = path.relative(projectRoot, path.dirname(norm));
|
|
62
|
+
const firstDir = rel.split(path.sep)[0];
|
|
63
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
64
|
+
const projectDir = path.dirname(projectRoot);
|
|
65
|
+
if (projectDir && projectDir !== path.dirname(norm)) project = path.basename(projectDir);
|
|
66
|
+
}
|
|
67
|
+
return { scope, type, project };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Load metadata from DB by path when no override provided. */
|
|
71
|
+
function loadMetadataByPath(db, docPath) {
|
|
72
|
+
const norm = path.normalize(docPath);
|
|
73
|
+
const row = db.prepare(
|
|
74
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
75
|
+
).get(norm);
|
|
76
|
+
if (!row) return null;
|
|
77
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
78
|
+
const tags = tagRows.map((r) => r.tag);
|
|
79
|
+
return {
|
|
80
|
+
id: row.id,
|
|
81
|
+
title: row.title ?? '',
|
|
82
|
+
type: row.type ?? 'generated_note',
|
|
83
|
+
scope: row.scope ?? 'project',
|
|
84
|
+
project: row.project ?? null,
|
|
85
|
+
topic_key: row.topic_key ?? null,
|
|
86
|
+
session_id: row.session_id ?? null,
|
|
87
|
+
tool_name: row.tool_name ?? null,
|
|
88
|
+
created_at: row.created_at,
|
|
89
|
+
updated_at: row.updated_at,
|
|
90
|
+
revision_count: row.revision_count ?? 1,
|
|
91
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
92
|
+
deleted: !!row.deleted_at,
|
|
93
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
94
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
95
|
+
deleted_at: row.deleted_at ?? null
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function indexDocument(config, docPath, rawContent, fileMtimeMs = null, options = {}) {
|
|
15
100
|
const db = getDb(config);
|
|
16
101
|
const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
|
|
17
102
|
const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
|
|
18
103
|
|
|
19
104
|
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
20
|
-
|
|
21
|
-
|
|
105
|
+
/** Prefer caller override, then DB row by path, then parsed front matter. */
|
|
106
|
+
let fm = options.overrideFrontmatter;
|
|
107
|
+
let byPath = null;
|
|
108
|
+
if (!fm) {
|
|
109
|
+
byPath = loadMetadataByPath(db, docPath);
|
|
110
|
+
fm = byPath ? { ...frontmatter, ...byPath } : frontmatter;
|
|
111
|
+
} else {
|
|
112
|
+
fm = { ...frontmatter, ...fm };
|
|
113
|
+
}
|
|
22
114
|
|
|
115
|
+
/** When metadata did not come from DB (full reindex or new file), infer scope/type/project from path. */
|
|
116
|
+
if (!options.overrideFrontmatter && !byPath) {
|
|
117
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
118
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null);
|
|
119
|
+
fm = {
|
|
120
|
+
...fm,
|
|
121
|
+
scope: inferred.scope,
|
|
122
|
+
type: fm.type ?? inferred.type,
|
|
123
|
+
project: inferred.project,
|
|
124
|
+
created_at: fm.created_at || fileCreatedAt || undefined,
|
|
125
|
+
updated_at: fm.updated_at || fileCreatedAt || undefined
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** If still no id (e.g. full reindex of file with no front matter), generate from path and set created/updated. */
|
|
130
|
+
if (!fm.id) {
|
|
131
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
132
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null) ?? new Date().toISOString();
|
|
133
|
+
fm = {
|
|
134
|
+
...fm,
|
|
135
|
+
id: contentHash(docPath).slice(0, 10),
|
|
136
|
+
scope: inferred.scope,
|
|
137
|
+
type: fm.type ?? inferred.type,
|
|
138
|
+
project: inferred.project,
|
|
139
|
+
created_at: fm.created_at || fileCreatedAt,
|
|
140
|
+
updated_at: fm.updated_at || fileCreatedAt
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const id = fm.id;
|
|
23
145
|
const now = new Date().toISOString();
|
|
24
146
|
const docHash = contentHash(rawContent);
|
|
25
|
-
const docTitle =
|
|
147
|
+
const docTitle = fm.title || title || '';
|
|
26
148
|
|
|
27
|
-
const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
|
|
149
|
+
const existing = db.prepare('SELECT content_hash, file_mtime_ms, path FROM documents WHERE id = ?').get(id);
|
|
28
150
|
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
151
|
+
/** If path changed (move), update row so path stays in sync. */
|
|
152
|
+
const normPath = path.normalize(docPath);
|
|
153
|
+
if (existing.path && path.normalize(existing.path) !== normPath) {
|
|
154
|
+
db.prepare('UPDATE documents SET path = ?, file_mtime_ms = ? WHERE id = ?').run(path.normalize(docPath), fileMtimeMs, id);
|
|
155
|
+
}
|
|
29
156
|
return { id };
|
|
30
157
|
}
|
|
31
158
|
|
|
32
159
|
const docRow = {
|
|
33
160
|
id,
|
|
34
|
-
path: docPath,
|
|
161
|
+
path: path.normalize(docPath),
|
|
35
162
|
title: docTitle,
|
|
36
|
-
type:
|
|
37
|
-
scope:
|
|
38
|
-
project:
|
|
39
|
-
topic_key:
|
|
40
|
-
session_id:
|
|
41
|
-
tool_name:
|
|
163
|
+
type: fm.type || 'generated_note',
|
|
164
|
+
scope: fm.scope || 'project',
|
|
165
|
+
project: fm.project || null,
|
|
166
|
+
topic_key: fm.topic_key || null,
|
|
167
|
+
session_id: fm.session_id || null,
|
|
168
|
+
tool_name: fm.tool_name || null,
|
|
42
169
|
content_hash: docHash,
|
|
43
170
|
file_mtime_ms: fileMtimeMs,
|
|
44
|
-
created_at:
|
|
45
|
-
updated_at:
|
|
171
|
+
created_at: fm.created_at || now,
|
|
172
|
+
updated_at: fm.updated_at || now,
|
|
46
173
|
indexed_at: now,
|
|
47
|
-
deleted_at:
|
|
48
|
-
revision_count:
|
|
49
|
-
duplicate_count:
|
|
50
|
-
embedding_status:
|
|
174
|
+
deleted_at: fm.deleted_at || null,
|
|
175
|
+
revision_count: fm.revision_count ?? 1,
|
|
176
|
+
duplicate_count: fm.duplicate_count ?? 0,
|
|
177
|
+
embedding_status: fm.embedding_status || 'disabled',
|
|
51
178
|
body: body || '',
|
|
52
179
|
structure: JSON.stringify(structure),
|
|
53
180
|
sections_index: JSON.stringify(sectionsIndex)
|
|
@@ -84,7 +211,7 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
84
211
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
212
|
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
86
213
|
|
|
87
|
-
const tags =
|
|
214
|
+
const tags = fm.tags || [];
|
|
88
215
|
for (const tag of tags) {
|
|
89
216
|
db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
|
|
90
217
|
}
|
|
@@ -3,6 +3,44 @@ import fs from 'fs';
|
|
|
3
3
|
import { globSync } from 'glob';
|
|
4
4
|
import { getDb } from '../../db/sqlite.js';
|
|
5
5
|
import { indexDocument, removeDocumentFromIndex } from './indexer.js';
|
|
6
|
+
import { TYPE_TO_DIR } from '../../fs/paths.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Load document metadata from DB by path (for reindex/watcher).
|
|
10
|
+
* Returns frontmatter-like object or null.
|
|
11
|
+
*/
|
|
12
|
+
function getDocumentMetaByPath(config, filePath) {
|
|
13
|
+
const db = getDb(config);
|
|
14
|
+
const norm = path.normalize(filePath);
|
|
15
|
+
const row = db.prepare(
|
|
16
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
17
|
+
).get(norm);
|
|
18
|
+
if (!row) return null;
|
|
19
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
20
|
+
const tags = tagRows.map((r) => r.tag);
|
|
21
|
+
return rowToFrontmatter(row, tags);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function rowToFrontmatter(row, tags = []) {
|
|
25
|
+
return {
|
|
26
|
+
id: row.id,
|
|
27
|
+
title: row.title ?? '',
|
|
28
|
+
type: row.type ?? 'generated_note',
|
|
29
|
+
scope: row.scope ?? 'project',
|
|
30
|
+
project: row.project ?? null,
|
|
31
|
+
topic_key: row.topic_key ?? null,
|
|
32
|
+
session_id: row.session_id ?? null,
|
|
33
|
+
tool_name: row.tool_name ?? null,
|
|
34
|
+
created_at: row.created_at,
|
|
35
|
+
updated_at: row.updated_at,
|
|
36
|
+
revision_count: row.revision_count ?? 1,
|
|
37
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
38
|
+
deleted: !!row.deleted_at,
|
|
39
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
40
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
41
|
+
deleted_at: row.deleted_at ?? null
|
|
42
|
+
};
|
|
43
|
+
}
|
|
6
44
|
|
|
7
45
|
/**
|
|
8
46
|
* List all Markdown file paths under home (cross-project) and project memory root (repo-local).
|
|
@@ -49,7 +87,8 @@ function fullReindex(config) {
|
|
|
49
87
|
try {
|
|
50
88
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
51
89
|
const stat = fs.statSync(filePath);
|
|
52
|
-
|
|
90
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
91
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { fileCreatedAt });
|
|
53
92
|
indexed++;
|
|
54
93
|
} catch (err) {
|
|
55
94
|
// skip invalid or unreadable files
|
|
@@ -81,7 +120,9 @@ function incrementalReindex(config) {
|
|
|
81
120
|
continue;
|
|
82
121
|
}
|
|
83
122
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
84
|
-
|
|
123
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
124
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
125
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
85
126
|
indexed++;
|
|
86
127
|
} catch (_) {}
|
|
87
128
|
}
|
|
@@ -107,4 +148,4 @@ function getDocumentIdByPath(config, filePath) {
|
|
|
107
148
|
return row?.id ?? null;
|
|
108
149
|
}
|
|
109
150
|
|
|
110
|
-
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath };
|
|
151
|
+
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath, getDocumentMetaByPath };
|
|
@@ -18,19 +18,29 @@ function getBm25Weights(config) {
|
|
|
18
18
|
// FTS5 uses "/" for NEAR (e.g. term/5) and "-" for NOT; dash-like chars between words break parsing.
|
|
19
19
|
const FTS5_NEAR_SLASH = /\//g;
|
|
20
20
|
const FTS5_NOT_HYPHEN = /(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g; // hyphen, en/em-dash, minus
|
|
21
|
+
// FTS5 operators are case-sensitive; normalizeForFts lowercases the string, so we restore them after.
|
|
22
|
+
const FTS5_OPERATOR_RESTORE = [
|
|
23
|
+
[/\band\b/g, 'AND'],
|
|
24
|
+
[/\bor\b/g, 'OR'],
|
|
25
|
+
[/\bnot\b/g, 'NOT']
|
|
26
|
+
];
|
|
21
27
|
|
|
22
28
|
/**
|
|
23
29
|
* Normalize query for FTS5: same as indexed text (NFD, remove diacritics, lowercase).
|
|
24
30
|
* Replaces "/" with space so it is not interpreted as FTS5 NEAR syntax (e.g. term/5).
|
|
25
31
|
* Replaces "-" (and Unicode dashes) between word chars with space so "alchemy-tycoon" is not parsed as "alchemy" NOT "tycoon".
|
|
26
|
-
*
|
|
32
|
+
* Restores FTS5 operators AND, OR, NOT to uppercase after normalization (FTS5 requires them case-sensitive).
|
|
27
33
|
*/
|
|
28
34
|
function normalizeQuery(q) {
|
|
29
35
|
if (q == null || typeof q !== 'string') return '';
|
|
30
36
|
const trimmed = q.trim();
|
|
31
37
|
const slashSafe = trimmed.replace(FTS5_NEAR_SLASH, ' ');
|
|
32
38
|
const hyphenSafe = slashSafe.replace(FTS5_NOT_HYPHEN, ' ');
|
|
33
|
-
|
|
39
|
+
let out = normalizeForFts(hyphenSafe);
|
|
40
|
+
for (const [re, replacement] of FTS5_OPERATOR_RESTORE) {
|
|
41
|
+
out = out.replace(re, replacement);
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
/**
|
|
@@ -70,8 +80,13 @@ function search(config, options = {}) {
|
|
|
70
80
|
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
71
81
|
params.push('home');
|
|
72
82
|
} else {
|
|
73
|
-
|
|
74
|
-
|
|
83
|
+
if (project) {
|
|
84
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
85
|
+
params.push('home', 'project', project);
|
|
86
|
+
} else {
|
|
87
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR d.scope = ?)';
|
|
88
|
+
params.push('home', 'project');
|
|
89
|
+
}
|
|
75
90
|
}
|
|
76
91
|
|
|
77
92
|
params.push(project || '');
|
package/src/fs/watcher.js
CHANGED
|
@@ -2,11 +2,11 @@ import path from 'path';
|
|
|
2
2
|
import chokidar from 'chokidar';
|
|
3
3
|
import fs from 'fs';
|
|
4
4
|
import { indexDocument, removeDocumentFromIndex } from '../core/indexing/indexer.js';
|
|
5
|
-
import { getDocumentIdByPath } from '../core/indexing/reindex.js';
|
|
5
|
+
import { getDocumentIdByPath, getDocumentMetaByPath } from '../core/indexing/reindex.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Start watching home (cross-project) and project memory root (repo-local) for .md changes.
|
|
9
|
-
* On add/change: index the file. On unlink: remove from index by path.
|
|
9
|
+
* On add/change: index the file (using DB metadata when path already known). On unlink: remove from index by path.
|
|
10
10
|
* @param {object} config - resolved config
|
|
11
11
|
* @returns {import('chokidar').FSWatcher} watcher instance (call .close() to stop)
|
|
12
12
|
*/
|
|
@@ -28,14 +28,18 @@ function startWatcher(config) {
|
|
|
28
28
|
try {
|
|
29
29
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
30
30
|
const stat = fs.statSync(filePath);
|
|
31
|
-
|
|
31
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
32
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
33
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
32
34
|
} catch (_) {}
|
|
33
35
|
})
|
|
34
36
|
.on('change', (filePath) => {
|
|
35
37
|
try {
|
|
36
38
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
37
39
|
const stat = fs.statSync(filePath);
|
|
38
|
-
|
|
40
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
41
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
42
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
39
43
|
} catch (_) {}
|
|
40
44
|
})
|
|
41
45
|
.on('unlink', (filePath) => {
|
package/src/mcp/server.js
CHANGED
|
@@ -44,6 +44,54 @@ function makeQueryEmbedder(child) {
|
|
|
44
44
|
};
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
/**
|
|
48
|
+
* Start the embedding worker and set config.getQueryEmbedding for use by mem_search.
|
|
49
|
+
* Used by the CLI when running tools with --embeddings so that semantic search works.
|
|
50
|
+
* Returns a cleanup function to kill the worker and remove the temp config file.
|
|
51
|
+
*/
|
|
52
|
+
function startEmbeddingWorker(config) {
|
|
53
|
+
if (!config?.embeddings?.enabled || config.getQueryEmbedding) {
|
|
54
|
+
return () => {};
|
|
55
|
+
}
|
|
56
|
+
try {
|
|
57
|
+
embeddingWorkerConfigPath = path.join(
|
|
58
|
+
os.tmpdir(),
|
|
59
|
+
`mixgram-worker-config-${process.pid}-${Date.now()}.json`
|
|
60
|
+
);
|
|
61
|
+
fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
|
|
62
|
+
const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
|
|
63
|
+
embeddingWorkerChild = fork(workerPath, [], {
|
|
64
|
+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
65
|
+
env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
|
|
66
|
+
});
|
|
67
|
+
config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
|
|
68
|
+
const cleanup = () => {
|
|
69
|
+
if (embeddingWorkerChild) {
|
|
70
|
+
try { embeddingWorkerChild.kill(); } catch (_) {}
|
|
71
|
+
embeddingWorkerChild = null;
|
|
72
|
+
}
|
|
73
|
+
config.getQueryEmbedding = null;
|
|
74
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
75
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
76
|
+
embeddingWorkerConfigPath = null;
|
|
77
|
+
}
|
|
78
|
+
};
|
|
79
|
+
embeddingWorkerChild.on('exit', () => {
|
|
80
|
+
config.getQueryEmbedding = null;
|
|
81
|
+
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
82
|
+
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
83
|
+
embeddingWorkerConfigPath = null;
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
return cleanup;
|
|
87
|
+
} catch (err) {
|
|
88
|
+
if (typeof process !== 'undefined' && process.stderr) {
|
|
89
|
+
process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
|
|
90
|
+
}
|
|
91
|
+
return () => {};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
47
95
|
function createServer(configOverrides = {}, baseDir = null, projectBaseDir = null) {
|
|
48
96
|
const config = loadConfig(configOverrides, baseDir, projectBaseDir);
|
|
49
97
|
const mcpServer = new McpServer(
|
|
@@ -98,42 +146,19 @@ async function run(configOverrides = {}, baseDir = null, projectBaseDir = null)
|
|
|
98
146
|
}
|
|
99
147
|
}
|
|
100
148
|
if (config.embeddings?.enabled) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
`mixgram-worker-config-${process.pid}-${Date.now()}.json`
|
|
105
|
-
);
|
|
106
|
-
fs.writeFileSync(embeddingWorkerConfigPath, JSON.stringify(config), 'utf8');
|
|
107
|
-
const workerPath = path.join(__dirname, '..', 'core', 'embeddings', 'worker-process.js');
|
|
108
|
-
embeddingWorkerChild = fork(workerPath, [], {
|
|
109
|
-
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
|
|
110
|
-
env: { ...process.env, MIXGRAM_WORKER_CONFIG: embeddingWorkerConfigPath }
|
|
111
|
-
});
|
|
112
|
-
config.getQueryEmbedding = makeQueryEmbedder(embeddingWorkerChild);
|
|
113
|
-
embeddingWorkerChild.on('exit', (code, signal) => {
|
|
149
|
+
const cleanup = startEmbeddingWorker(config);
|
|
150
|
+
if (cleanup) {
|
|
151
|
+
embeddingWorkerChild?.on('exit', (code, signal) => {
|
|
114
152
|
if (code !== 0 && code != null && process.stderr) {
|
|
115
153
|
process.stderr.write(`[mixgram] embeddings worker exited: code=${code} signal=${signal}\n`);
|
|
116
154
|
}
|
|
117
|
-
|
|
118
|
-
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
119
|
-
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
120
|
-
embeddingWorkerConfigPath = null;
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
process.on('exit', () => {
|
|
124
|
-
if (embeddingWorkerChild) embeddingWorkerChild.kill();
|
|
125
|
-
if (embeddingWorkerConfigPath && fs.existsSync(embeddingWorkerConfigPath)) {
|
|
126
|
-
try { fs.unlinkSync(embeddingWorkerConfigPath); } catch (_) {}
|
|
127
|
-
}
|
|
155
|
+
cleanup();
|
|
128
156
|
});
|
|
129
|
-
|
|
130
|
-
if (typeof process !== 'undefined' && process.stderr) {
|
|
131
|
-
process.stderr.write(`[mixgram] embeddings worker start failed: ${err?.message ?? err}\n`);
|
|
132
|
-
}
|
|
157
|
+
process.on('exit', () => cleanup());
|
|
133
158
|
}
|
|
134
159
|
}
|
|
135
160
|
const transport = new StdioServerTransport();
|
|
136
161
|
await mcpServer.connect(transport);
|
|
137
162
|
}
|
|
138
163
|
|
|
139
|
-
export { createServer, run };
|
|
164
|
+
export { createServer, run, startEmbeddingWorker };
|
package/src/mcp/tools.js
CHANGED
|
@@ -17,12 +17,13 @@ function suggestTopicKey(title, content = '', type = 'generated_note') {
|
|
|
17
17
|
function createToolHandlers(config) {
|
|
18
18
|
return {
|
|
19
19
|
mem_save: async (args) => {
|
|
20
|
+
const topicKey = args.topic_key ?? suggestTopicKey(args.title, args.content ?? '', args.type || 'generated_note');
|
|
20
21
|
const result = saveDocument(config, {
|
|
21
22
|
title: args.title,
|
|
22
23
|
type: args.type || 'generated_note',
|
|
23
24
|
scope: args.scope || 'project',
|
|
24
25
|
project: args.project ?? null,
|
|
25
|
-
topic_key:
|
|
26
|
+
topic_key: topicKey,
|
|
26
27
|
content: args.content ?? '',
|
|
27
28
|
session_id: args.session_id ?? null,
|
|
28
29
|
id: args.id ?? null,
|
|
@@ -54,12 +55,58 @@ function createToolHandlers(config) {
|
|
|
54
55
|
},
|
|
55
56
|
|
|
56
57
|
mem_search: async (args) => {
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
58
|
+
const scopeMode = args.scope_mode || config.search?.defaultScopeMode || 'merged';
|
|
59
|
+
const project = args.project ?? null;
|
|
60
|
+
const limit = args.limit ?? config.search?.defaultLimit ?? 10;
|
|
61
|
+
const query = args.query?.trim();
|
|
62
|
+
if (!query) {
|
|
63
|
+
return { content: [{ type: 'text', text: JSON.stringify({ results: [] }) }] };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const ftsResults = await search(config, { query, scopeMode, project, limit });
|
|
67
|
+
let results = [...ftsResults];
|
|
68
|
+
|
|
69
|
+
if (config.embeddings?.enabled && config.getQueryEmbedding) {
|
|
70
|
+
try {
|
|
71
|
+
const { getVectorStore } = await import('../core/embeddings/vectorStore.js');
|
|
72
|
+
const vectorStore = await getVectorStore(config);
|
|
73
|
+
if (vectorStore) {
|
|
74
|
+
const queryVec = await config.getQueryEmbedding(query);
|
|
75
|
+
if (queryVec) {
|
|
76
|
+
const semanticThreshold = config.embeddings?.similarityThreshold ?? 0.75;
|
|
77
|
+
const vecResults = await vectorStore.search(config, queryVec, Math.max(limit * 2, 20), { similarityThreshold: semanticThreshold });
|
|
78
|
+
const db = getDb(config);
|
|
79
|
+
const seen = new Set(results.map((r) => r.documentId));
|
|
80
|
+
const semanticList = [];
|
|
81
|
+
for (const v of vecResults) {
|
|
82
|
+
if (seen.has(v.document_id)) continue;
|
|
83
|
+
const row = db.prepare(
|
|
84
|
+
'SELECT id, title, topic_key, type, scope, project, created_at, body FROM documents WHERE id = ? AND deleted_at IS NULL'
|
|
85
|
+
).get(v.document_id);
|
|
86
|
+
if (!row) continue;
|
|
87
|
+
if (scopeMode === 'project-only' && (row.scope !== 'project' || row.project !== project)) continue;
|
|
88
|
+
if (scopeMode === 'home-only' && row.scope !== 'home') continue;
|
|
89
|
+
if (scopeMode === 'merged' && project && row.scope === 'project' && row.project !== project) continue;
|
|
90
|
+
seen.add(v.document_id);
|
|
91
|
+
const snippet = (row.body || '').slice(0, 200).trim();
|
|
92
|
+
semanticList.push({
|
|
93
|
+
documentId: row.id,
|
|
94
|
+
title: row.title,
|
|
95
|
+
topicKey: row.topic_key,
|
|
96
|
+
type: row.type,
|
|
97
|
+
scope: row.scope,
|
|
98
|
+
project: row.project,
|
|
99
|
+
created: row.created_at || null,
|
|
100
|
+
snippet: snippet || '',
|
|
101
|
+
score: 1 - v.distance
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
results = [...semanticList, ...results].slice(0, limit);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
} catch (_) {}
|
|
108
|
+
}
|
|
109
|
+
|
|
63
110
|
return { content: [{ type: 'text', text: JSON.stringify({ results }) }] };
|
|
64
111
|
},
|
|
65
112
|
|
package/src/utils/markdown.js
CHANGED
|
@@ -6,8 +6,8 @@ export const FRONTMATTER_KEYS = [
|
|
|
6
6
|
'embedding_status', 'indexed_at', 'deleted_at'
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
-
/** Keys written to .md
|
|
10
|
-
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', '
|
|
9
|
+
/** Keys written to .md: id, type, title, created (cosmetic). scope/project stay in DB only; reindex infers scope from path. */
|
|
10
|
+
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'created'];
|
|
11
11
|
|
|
12
12
|
function formatCreated(createdAt) {
|
|
13
13
|
if (!createdAt) return null;
|
|
@@ -39,12 +39,13 @@ function serializeFrontmatter(fm) {
|
|
|
39
39
|
return obj;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
/** Minimal frontmatter for .md (
|
|
42
|
+
/** Minimal frontmatter for .md (cosmetic only; deleted flag for soft-delete visibility). */
|
|
43
43
|
function serializeDisplayFrontmatter(fm) {
|
|
44
44
|
const obj = {};
|
|
45
45
|
for (const k of DISPLAY_FRONTMATTER_KEYS) {
|
|
46
46
|
if (k === 'created') {
|
|
47
|
-
const
|
|
47
|
+
const raw = fm.created_at ?? (fm.created ? parseCreated(fm.created) : null);
|
|
48
|
+
const v = raw ? formatCreated(raw) : null;
|
|
48
49
|
if (v) obj[k] = v;
|
|
49
50
|
} else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
|
|
50
51
|
obj[k] = fm[k];
|