mixgram 1.3.6 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -345
- package/bin/mixgram.js +51 -160
- package/examples/embedding-demo.js +22 -142
- package/package.json +19 -19
- package/spec/bench-indexing.js +23 -211
- package/spec/bench-search.js +26 -212
- package/spec/run-tests.js +1 -214
- package/spec/scenarios/scenarios.js +277 -607
- package/spec/sdd.md +14 -988
- package/spec/tdd.md +15 -1232
- package/spec/test-large-document.js +20 -91
- package/src/config.js +57 -48
- package/src/core/documents/documents.js +62 -224
- package/src/core/embeddings/embedder.js +14 -54
- package/src/core/embeddings/queue.js +33 -106
- package/src/core/embeddings/vectorStore.js +30 -111
- package/src/core/embeddings/worker-process.js +23 -74
- package/src/core/indexing/indexer.js +51 -237
- package/src/core/indexing/parser.js +13 -114
- package/src/core/indexing/reindex.js +28 -137
- package/src/core/search/search.js +64 -158
- package/src/db/migrations/001_initial.sql +14 -63
- package/src/db/sqlite.js +19 -29
- package/src/fs/paths.js +11 -101
- package/src/fs/watcher.js +15 -48
- package/src/mcp/cli-adapter.js +32 -110
- package/src/mcp/server.js +64 -141
- package/src/mcp/tool-registry.js +21 -132
- package/src/mcp/tools.js +27 -199
- package/src/utils/markdown.js +15 -82
- package/spec/fixtures/realistic-content.js +0 -106
- package/spec/reporter.js +0 -127
- package/spec/run-tests-runner.js +0 -40
- package/src/core/embeddings/worker.js +0 -27
- package/src/core/prompts/prompts.js +0 -15
- package/src/core/sessions/sessions.js +0 -27
- package/src/utils/ids.js +0 -27
package/bin/mixgram.js
CHANGED
|
@@ -6,11 +6,11 @@
|
|
|
6
6
|
* mixgram setup <client> — add Mixgram to Cursor / Gemini CLI / Codex config
|
|
7
7
|
*/
|
|
8
8
|
import { run, startEmbeddingWorker } from '../src/mcp/server.js';
|
|
9
|
-
import { loadConfig } from '../src/config.js';
|
|
9
|
+
import { loadConfig, mergeConfig } from '../src/config.js';
|
|
10
10
|
import { closeDb } from '../src/db/sqlite.js';
|
|
11
11
|
import { createToolHandlers } from '../src/mcp/tools.js';
|
|
12
12
|
import { getToolByName, listToolNames, parseToolArgs, formatToolHelp } from '../src/mcp/cli-adapter.js';
|
|
13
|
-
import {
|
|
13
|
+
import { reindex } from '../src/core/indexing/reindex.js';
|
|
14
14
|
import fs from 'fs';
|
|
15
15
|
import path from 'path';
|
|
16
16
|
import os from 'os';
|
|
@@ -22,83 +22,33 @@ const PKG = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'
|
|
|
22
22
|
const SUBCOMMAND = process.argv[2];
|
|
23
23
|
const ARG = process.argv[3];
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
const
|
|
28
|
-
|
|
29
|
-
for (let i = 0; i <
|
|
30
|
-
const
|
|
31
|
-
if (
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
} else if (a === '--home' && args[i + 1]) {
|
|
39
|
-
out.homeMemoryRoot = args[++i];
|
|
40
|
-
} else if (a === '--project-memory' && args[i + 1]) {
|
|
41
|
-
out.projectMemoryRoot = args[++i];
|
|
42
|
-
} else if (a === '--sqlite-path' && args[i + 1]) {
|
|
43
|
-
out.sqlitePath = args[++i];
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
return out;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/** Resolve config file path: --config, MIXGRAM_CONFIG, ./.mixgram/config.json, ~/.mixgram/config.json */
|
|
50
|
-
function resolveConfigPath(explicitPath) {
|
|
51
|
-
if (explicitPath) return path.resolve(explicitPath);
|
|
52
|
-
const envPath = process.env.MIXGRAM_CONFIG;
|
|
53
|
-
if (envPath) return path.resolve(envPath);
|
|
54
|
-
const cwd = process.cwd();
|
|
55
|
-
const local = path.join(cwd, '.mixgram', 'config.json');
|
|
56
|
-
if (fs.existsSync(local)) return local;
|
|
57
|
-
const global = path.join(os.homedir(), '.mixgram', 'config.json');
|
|
58
|
-
if (fs.existsSync(global)) return global;
|
|
59
|
-
return null;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
/** baseDir for path resolution: project root when config is in .mixgram/config.json, else config dir. */
|
|
63
|
-
function getBaseDir(configPath) {
|
|
64
|
-
if (!configPath) return process.cwd();
|
|
65
|
-
const dir = path.dirname(configPath);
|
|
66
|
-
return path.basename(dir) === '.mixgram' ? path.dirname(dir) : dir;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/** Load config from file + env + argv. */
|
|
70
|
-
function loadCliConfig() {
|
|
71
|
-
const argvOverrides = parseMcpArgs();
|
|
72
|
-
const configPath = resolveConfigPath(argvOverrides._configPath);
|
|
73
|
-
const baseDir = getBaseDir(configPath);
|
|
74
|
-
|
|
75
|
-
let fileConfig = {};
|
|
76
|
-
if (configPath) {
|
|
77
|
-
try {
|
|
78
|
-
fileConfig = JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
|
79
|
-
} catch (e) {
|
|
80
|
-
console.error('Could not read config from', configPath, e.message);
|
|
81
|
-
process.exit(1);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
const envOverrides = {};
|
|
86
|
-
if (process.env.MIXGRAM_EMBEDDINGS_ENABLED === '1' || process.env.MIXGRAM_EMBEDDINGS_ENABLED === 'true') {
|
|
87
|
-
envOverrides.embeddings = { ...(fileConfig.embeddings || {}), enabled: true };
|
|
25
|
+
function loadCliConfig(argv = process.argv.slice(3)) {
|
|
26
|
+
const overrides = {};
|
|
27
|
+
const toolArgs = [];
|
|
28
|
+
let explicitPath;
|
|
29
|
+
for (let i = 0; i < argv.length; i++) {
|
|
30
|
+
const flag = argv[i];
|
|
31
|
+
if (['--embeddings', '--no-embeddings'].includes(flag)) overrides.embeddings = { enabled: flag === '--embeddings' };
|
|
32
|
+
else if (['--watch', '--no-watch'].includes(flag)) overrides.watch = flag === '--watch';
|
|
33
|
+
else if (['--config', '--sqlite-path'].includes(flag)) {
|
|
34
|
+
const value = argv[++i];
|
|
35
|
+
if (!value || value.startsWith('--')) throw new Error(`Missing value for ${flag}`);
|
|
36
|
+
if (flag === '--config') explicitPath = path.resolve(value); else overrides.sqlitePath = path.resolve(value);
|
|
37
|
+
} else toolArgs.push(flag);
|
|
88
38
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
39
|
+
const candidates = [path.resolve('.mixgram/v2/config.json'), path.join(os.homedir(), '.mixgram/v2/config.json')];
|
|
40
|
+
const configPath = explicitPath ?? process.env.MIXGRAM_CONFIG ?? candidates.find(p => fs.existsSync(p));
|
|
41
|
+
const file = configPath ? JSON.parse(fs.readFileSync(configPath, 'utf8')) : {};
|
|
42
|
+
const env = {};
|
|
43
|
+
for (const [variable, key] of [['MIXGRAM_WATCH', 'watch'], ['MIXGRAM_EMBEDDINGS_ENABLED', 'embeddings']]) {
|
|
44
|
+
const value = process.env[variable];
|
|
45
|
+
if (value === undefined) continue;
|
|
46
|
+
if (!['1','0','true','false'].includes(value)) throw new Error(`Invalid boolean for ${variable}`);
|
|
47
|
+
const enabled = value === '1' || value === 'true';
|
|
48
|
+
env[key] = key === 'embeddings' ? { enabled } : enabled;
|
|
99
49
|
}
|
|
100
|
-
|
|
101
|
-
return { overrides:
|
|
50
|
+
if (process.env.MIXGRAM_SQLITE_PATH) env.sqlitePath = path.resolve(process.env.MIXGRAM_SQLITE_PATH);
|
|
51
|
+
return { overrides: mergeConfig(file, env, overrides), baseDir: configPath ? path.dirname(path.resolve(configPath)) : process.cwd(), toolArgs };
|
|
102
52
|
}
|
|
103
53
|
|
|
104
54
|
const MIXGRAM_ENTRY = {
|
|
@@ -108,7 +58,7 @@ const MIXGRAM_ENTRY = {
|
|
|
108
58
|
|
|
109
59
|
const CURSOR_MIXGRAM_ENTRY = {
|
|
110
60
|
command: 'mixgram',
|
|
111
|
-
args: ['mcp'
|
|
61
|
+
args: ['mcp']
|
|
112
62
|
};
|
|
113
63
|
|
|
114
64
|
function cursorMcpPath() {
|
|
@@ -211,42 +161,13 @@ args = ["mcp"]
|
|
|
211
161
|
}
|
|
212
162
|
|
|
213
163
|
function printHelp() {
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
setup <client> Register Mixgram with an MCP client.
|
|
222
|
-
flushdb Delete the SQLite database file (and WAL/shm).
|
|
223
|
-
help [tool] Show help; with optional tool name, show options for that tool.
|
|
224
|
-
-v, --version Print version from package and exit.
|
|
225
|
-
<tool> [options] Run an MCP tool by name. Tools:
|
|
226
|
-
${toolLines}
|
|
227
|
-
|
|
228
|
-
mcp options (and env / config file):
|
|
229
|
-
--config <path> Config file (default: ./.mixgram/config.json or ~/.mixgram/config.json)
|
|
230
|
-
--embeddings Enable semantic search (or embeddings.enabled in config)
|
|
231
|
-
--watch Watch files and reindex on change
|
|
232
|
-
--home <path> Home memory root (default: ~/.mixgram/docs)
|
|
233
|
-
--project-memory <path> Project memory root (default: ./docs, relative to repo)
|
|
234
|
-
--sqlite-path <path> SQLite index path (default: ~/.mixgram/index.db)
|
|
235
|
-
|
|
236
|
-
Env: MIXGRAM_CONFIG, MIXGRAM_EMBEDDINGS_ENABLED, MIXGRAM_HOME, MIXGRAM_PROJECT_MEMORY,
|
|
237
|
-
MIXGRAM_SQLITE_PATH, MIXGRAM_WATCH
|
|
238
|
-
|
|
239
|
-
Setup targets:
|
|
240
|
-
cursor Cursor IDE
|
|
241
|
-
gemini-cli Gemini CLI
|
|
242
|
-
codex Codex
|
|
243
|
-
|
|
244
|
-
Example (Cursor): "mixgram": { "command": "mixgram", "args": ["mcp", "--project-memory", "\${workspaceFolder}/docs"] }
|
|
245
|
-
With embeddings: "args": ["mcp", "--project-memory", "\${workspaceFolder}/docs", "--embeddings"]
|
|
246
|
-
Config file: .mixgram/config.json or ~/.mixgram/config.json
|
|
247
|
-
|
|
248
|
-
npm install -g mixgram
|
|
249
|
-
`);
|
|
164
|
+
console.log(`Usage: mixgram <command> [options]
|
|
165
|
+
Commands: mcp, setup <cursor|gemini-cli|codex>, help [tool], --version
|
|
166
|
+
Tools: ${listToolNames().join(', ')}
|
|
167
|
+
Options: --config <path>, --sqlite-path <path>, --embeddings / --no-embeddings, --watch / --no-watch
|
|
168
|
+
Default config: .mixgram/v2/config.json or ~/.mixgram/v2/config.json
|
|
169
|
+
Environment: MIXGRAM_CONFIG, MIXGRAM_SQLITE_PATH, MIXGRAM_EMBEDDINGS_ENABLED, MIXGRAM_WATCH
|
|
170
|
+
Configure collections and defaultCollection in the config file.`);
|
|
250
171
|
}
|
|
251
172
|
|
|
252
173
|
async function main() {
|
|
@@ -256,8 +177,9 @@ async function main() {
|
|
|
256
177
|
}
|
|
257
178
|
|
|
258
179
|
if (SUBCOMMAND === 'mcp') {
|
|
259
|
-
const { overrides, baseDir,
|
|
260
|
-
|
|
180
|
+
const { overrides, baseDir, toolArgs } = loadCliConfig();
|
|
181
|
+
if (toolArgs.length) throw new Error(`Unknown options: ${toolArgs.join(' ')}`);
|
|
182
|
+
await run(overrides, baseDir)
|
|
261
183
|
.then(() => {})
|
|
262
184
|
.catch((err) => {
|
|
263
185
|
console.error(err);
|
|
@@ -266,26 +188,6 @@ async function main() {
|
|
|
266
188
|
return;
|
|
267
189
|
}
|
|
268
190
|
|
|
269
|
-
if (SUBCOMMAND === 'flushdb') {
|
|
270
|
-
const { overrides, baseDir, projectBaseDir } = loadCliConfig();
|
|
271
|
-
const config = loadConfig(overrides, baseDir, projectBaseDir);
|
|
272
|
-
const dbPath = config.sqlitePath;
|
|
273
|
-
closeDb();
|
|
274
|
-
const removed = [];
|
|
275
|
-
for (const p of [dbPath, dbPath + '-wal', dbPath + '-shm']) {
|
|
276
|
-
if (fs.existsSync(p)) {
|
|
277
|
-
fs.unlinkSync(p);
|
|
278
|
-
removed.push(p);
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
if (removed.length) {
|
|
282
|
-
console.log('Removed:', removed.join(', '));
|
|
283
|
-
} else {
|
|
284
|
-
console.log('No database file at', dbPath);
|
|
285
|
-
}
|
|
286
|
-
return;
|
|
287
|
-
}
|
|
288
|
-
|
|
289
191
|
if (SUBCOMMAND === 'setup') {
|
|
290
192
|
if (!ARG) {
|
|
291
193
|
console.error('Usage: mixgram setup <cursor|gemini-cli|codex>');
|
|
@@ -325,32 +227,21 @@ async function main() {
|
|
|
325
227
|
return;
|
|
326
228
|
}
|
|
327
229
|
const toolDef = getToolByName(SUBCOMMAND);
|
|
328
|
-
const
|
|
329
|
-
const
|
|
330
|
-
const config = loadConfig(overrides, baseDir
|
|
331
|
-
|
|
332
|
-
if (config.embeddings?.enabled) {
|
|
333
|
-
cleanupEmbeddingWorker = startEmbeddingWorker(config);
|
|
334
|
-
}
|
|
335
|
-
const handlers = createToolHandlers(config);
|
|
230
|
+
const { overrides, baseDir, toolArgs } = loadCliConfig();
|
|
231
|
+
const args = parseToolArgs(toolDef, toolArgs);
|
|
232
|
+
const config = loadConfig(overrides, baseDir);
|
|
233
|
+
const cleanupWorker = startEmbeddingWorker(config);
|
|
336
234
|
try {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
try {
|
|
341
|
-
const out = JSON.parse(result?.content?.[0]?.text || '{}');
|
|
342
|
-
if (out.id) docId = out.id;
|
|
343
|
-
} catch (_) {}
|
|
344
|
-
if (docId) enqueueChunks(config, [docId]);
|
|
345
|
-
while (await processNextJob(config)) {}
|
|
235
|
+
if (config.indexing.reindexOnStartup && SUBCOMMAND !== 'mem_reindex') {
|
|
236
|
+
const sync = reindex(config);
|
|
237
|
+
for (const error of sync.errors) console.error(`${error.path}: ${error.error}`);
|
|
346
238
|
}
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
console.error(err);
|
|
351
|
-
process.exit(1);
|
|
239
|
+
const result = await createToolHandlers(config)[SUBCOMMAND](args);
|
|
240
|
+
console.log(result.content[0].text);
|
|
241
|
+
if (result.isError) process.exitCode = 1;
|
|
352
242
|
} finally {
|
|
353
|
-
|
|
243
|
+
cleanupWorker();
|
|
244
|
+
closeDb(config);
|
|
354
245
|
}
|
|
355
246
|
return;
|
|
356
247
|
}
|
|
@@ -360,4 +251,4 @@ async function main() {
|
|
|
360
251
|
process.exit(1);
|
|
361
252
|
}
|
|
362
253
|
|
|
363
|
-
main();
|
|
254
|
+
main().catch(error => { console.error(error.message); process.exitCode = 1; });
|
|
@@ -1,154 +1,34 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* Example: indexing + search showing embedding effectiveness.
|
|
4
|
-
*
|
|
5
|
-
* Concept:
|
|
6
|
-
* - FTS (full-text search) only matches when query words appear in the document.
|
|
7
|
-
* - Semantic search (embeddings) matches by meaning: different wording, same idea.
|
|
8
|
-
* This script indexes a few documents, then runs the same query with FTS (no match)
|
|
9
|
-
* and with vector search (finds the relevant doc by similarity).
|
|
10
|
-
*
|
|
11
|
-
* Steps:
|
|
12
|
-
* 1. Index several short documents with related content but different wording.
|
|
13
|
-
* 2. Process the embedding queue so each document gets a vector.
|
|
14
|
-
* 3. Compare FTS (text-only) vs semantic (vector) search.
|
|
15
|
-
*
|
|
16
|
-
* Requires: npm install (includes @huggingface/transformers and sqlite-vec).
|
|
17
|
-
* Usage: node examples/embedding-demo.js
|
|
18
|
-
*/
|
|
19
2
|
import fs from 'fs';
|
|
20
3
|
import path from 'path';
|
|
21
4
|
import os from 'os';
|
|
22
5
|
import { loadConfig } from '../src/config.js';
|
|
23
6
|
import { saveDocument } from '../src/core/documents/documents.js';
|
|
24
|
-
import { search
|
|
7
|
+
import { search } from '../src/core/search/search.js';
|
|
25
8
|
import { processNextJob } from '../src/core/embeddings/queue.js';
|
|
26
9
|
import { getEmbedder } from '../src/core/embeddings/embedder.js';
|
|
27
10
|
import { getDb, closeDb } from '../src/db/sqlite.js';
|
|
28
11
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
};
|
|
46
|
-
|
|
47
|
-
const config = loadConfig(overrides, tmpDir, projectRoot);
|
|
48
|
-
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
49
|
-
fs.mkdirSync(config.homeMemoryRoot, { recursive: true });
|
|
50
|
-
|
|
51
|
-
const db = getDb(config);
|
|
52
|
-
|
|
53
|
-
console.log('--- 1. Indexing: save documents ---\n');
|
|
54
|
-
|
|
55
|
-
const docs = [
|
|
56
|
-
{
|
|
57
|
-
title: 'Memory in Markdown',
|
|
58
|
-
type: 'decision',
|
|
59
|
-
content: 'We store the agent\'s memory in Markdown files under docs/. That way the documentation is visible and editable by hand.'
|
|
60
|
-
},
|
|
61
|
-
{
|
|
62
|
-
title: 'Derived index',
|
|
63
|
-
type: 'decision',
|
|
64
|
-
content: 'The search index is derived from the Markdown files using SQLite FTS5. There is no database as source of truth other than the .md files.'
|
|
65
|
-
},
|
|
66
|
-
{
|
|
67
|
-
title: 'Session persistence',
|
|
68
|
-
type: 'decision',
|
|
69
|
-
content: 'Persist the session context across agent restarts. The important state lives in the memory documents.'
|
|
70
|
-
}
|
|
71
|
-
];
|
|
72
|
-
|
|
73
|
-
for (const d of docs) {
|
|
74
|
-
const out = saveDocument(config, {
|
|
75
|
-
title: d.title,
|
|
76
|
-
type: d.type,
|
|
77
|
-
scope: 'project',
|
|
78
|
-
project: 'demo',
|
|
79
|
-
content: d.content
|
|
80
|
-
});
|
|
81
|
-
console.log(` Saved: ${d.title} → ${out.id}`);
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
await new Promise((r) => setImmediate(r));
|
|
85
|
-
await new Promise((r) => setImmediate(r));
|
|
86
|
-
|
|
87
|
-
console.log('\n--- 2. Process embedding queue ---\n');
|
|
88
|
-
let processed = 0;
|
|
89
|
-
while (await processNextJob(config)) processed++;
|
|
90
|
-
console.log(` Jobs processed: ${processed}`);
|
|
91
|
-
|
|
92
|
-
const cacheTable = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'vec_cache_%'").get()?.name;
|
|
93
|
-
const vecCount = cacheTable ? db.prepare(`SELECT COUNT(*) AS c FROM ${cacheTable}`).get()?.c ?? 0 : 0;
|
|
94
|
-
console.log(` Vectors in index: ${vecCount}\n`);
|
|
95
|
-
|
|
96
|
-
console.log('--- 3. FTS search (text only) ---\n');
|
|
97
|
-
|
|
98
|
-
const queryLiteral = 'Markdown';
|
|
99
|
-
const ftsResults = searchFts(config, { query: queryLiteral, project: 'demo', limit: 5 });
|
|
100
|
-
console.log(` Query: "${queryLiteral}"`);
|
|
101
|
-
ftsResults.forEach((r, i) => {
|
|
102
|
-
console.log(` ${i + 1}. [${r.title}] score=${r.score.toFixed(2)} — ${r.snippet.slice(0, 60)}...`);
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
const querySemantic = 'where is the agent memory stored';
|
|
106
|
-
const ftsSemantic = searchFts(config, { query: querySemantic, project: 'demo', limit: 5 });
|
|
107
|
-
console.log(`\n Query: "${querySemantic}"`);
|
|
108
|
-
if (ftsSemantic.length === 0) {
|
|
109
|
-
console.log(' (FTS finds no literal match; these words are not in the documents.)');
|
|
110
|
-
} else {
|
|
111
|
-
ftsSemantic.forEach((r, i) => {
|
|
112
|
-
console.log(` ${i + 1}. [${r.title}] — ${r.snippet.slice(0, 60)}...`);
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
console.log('\n--- 4. Semantic search (embeddings) ---\n');
|
|
117
|
-
|
|
12
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-demo-'));
|
|
13
|
+
const config = loadConfig({ collections: { general: path.join(root, 'docs') }, sqlitePath: path.join(root, 'index.db') });
|
|
14
|
+
try {
|
|
15
|
+
for (const [title, content] of [
|
|
16
|
+
['Baking', 'Feed the sourdough starter daily and bake bread in a hot oven.'],
|
|
17
|
+
['Research', 'Record sources and methods alongside observations in a research notebook.'],
|
|
18
|
+
['Memory', 'Keep durable knowledge in Markdown files that people can edit.']
|
|
19
|
+
]) saveDocument(config, { title, content });
|
|
20
|
+
const query = 'preserving information for future use';
|
|
21
|
+
console.log('Text only:', JSON.stringify(await search(config, { query }), null, 2));
|
|
22
|
+
config.embeddings.enabled = true;
|
|
23
|
+
const { reindex } = await import('../src/core/indexing/reindex.js');
|
|
24
|
+
reindex(config, { full: true });
|
|
25
|
+
while (await processNextJob(config)) {}
|
|
26
|
+
const failed = getDb(config).prepare("SELECT last_error FROM embedding_jobs WHERE status='failed'").all();
|
|
27
|
+
if (failed.length) throw new Error(JSON.stringify(failed));
|
|
118
28
|
const embedder = await getEmbedder(config);
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
return;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
const queryVec = await embedder.embed(querySemantic);
|
|
128
|
-
const vectorResults = await vectorStore.search(config, queryVec, 10, { similarityThreshold: 0.80 });
|
|
129
|
-
|
|
130
|
-
console.log(` Query: "${querySemantic}"`);
|
|
131
|
-
console.log(' Results by semantic similarity:\n');
|
|
132
|
-
|
|
133
|
-
for (const v of vectorResults) {
|
|
134
|
-
const row = db.prepare('SELECT id, title, body FROM documents WHERE id = ?').get(v.document_id);
|
|
135
|
-
const similarity = (1 - v.distance).toFixed(3);
|
|
136
|
-
const snippet = (row?.body || '').slice(0, 80);
|
|
137
|
-
console.log(` - [${row?.title ?? v.document_id}] similarity=${similarity}`);
|
|
138
|
-
console.log(` ${snippet}...`);
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
console.log('\n--- Summary ---');
|
|
142
|
-
console.log('FTS only finds when the query words appear in the text.');
|
|
143
|
-
console.log('Embedding search finds documents that share the same meaning even without shared words.\n');
|
|
144
|
-
|
|
145
|
-
closeDb();
|
|
146
|
-
try {
|
|
147
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
148
|
-
} catch (_) {}
|
|
29
|
+
config.getQueryEmbedding = text => embedder.embed(text);
|
|
30
|
+
console.log('Hybrid:', JSON.stringify(await search(config, { query }), null, 2));
|
|
31
|
+
} finally {
|
|
32
|
+
closeDb(config);
|
|
33
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
149
34
|
}
|
|
150
|
-
|
|
151
|
-
run().catch((err) => {
|
|
152
|
-
console.error(err);
|
|
153
|
-
process.exit(1);
|
|
154
|
-
});
|
package/package.json
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mixgram",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "General-purpose Markdown memory with collections, tags, text and semantic search over MCP and CLI.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mcp",
|
|
7
7
|
"model-context-protocol",
|
|
8
8
|
"memory",
|
|
9
|
-
"engram",
|
|
10
9
|
"markdown",
|
|
11
10
|
"sqlite",
|
|
12
11
|
"fts5",
|
|
@@ -30,6 +29,21 @@
|
|
|
30
29
|
"bin": {
|
|
31
30
|
"mixgram": "./bin/mixgram.js"
|
|
32
31
|
},
|
|
32
|
+
"scripts": {
|
|
33
|
+
"start": "node index.js",
|
|
34
|
+
"test": "node spec/run-tests.js",
|
|
35
|
+
"test:visual": "node spec/run-tests.js --visual",
|
|
36
|
+
"test:large": "node spec/test-large-document.js",
|
|
37
|
+
"bench:indexing": "node spec/bench-indexing.js",
|
|
38
|
+
"bench:embed": "node spec/bench-indexing.js --embed",
|
|
39
|
+
"bench:search": "node spec/bench-search.js",
|
|
40
|
+
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
41
|
+
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
42
|
+
"example:embedding": "node examples/embedding-demo.js",
|
|
43
|
+
"duplication": "jscpd src",
|
|
44
|
+
"deadcode": "knip",
|
|
45
|
+
"quality": "npm run duplication && npm run deadcode"
|
|
46
|
+
},
|
|
33
47
|
"dependencies": {
|
|
34
48
|
"@huggingface/transformers": "^3.8.1",
|
|
35
49
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
@@ -47,19 +61,5 @@
|
|
|
47
61
|
"jscpd": "^4.0.8",
|
|
48
62
|
"knip": "^5.86.0"
|
|
49
63
|
},
|
|
50
|
-
"
|
|
51
|
-
|
|
52
|
-
"test": "node spec/run-tests.js",
|
|
53
|
-
"test:visual": "node spec/run-tests.js --visual",
|
|
54
|
-
"test:large": "node spec/test-large-document.js",
|
|
55
|
-
"bench:indexing": "node spec/bench-indexing.js",
|
|
56
|
-
"bench:embed": "node spec/bench-indexing.js --embed",
|
|
57
|
-
"bench:search": "node spec/bench-search.js",
|
|
58
|
-
"bench:search:embed": "node spec/bench-search.js --embed",
|
|
59
|
-
"bench:all": "npm run bench:indexing && npm run bench:embed && npm run bench:search && npm run bench:search:embed",
|
|
60
|
-
"example:embedding": "node examples/embedding-demo.js",
|
|
61
|
-
"duplication": "jscpd src",
|
|
62
|
-
"deadcode": "knip",
|
|
63
|
-
"quality": "npm run duplication && npm run deadcode"
|
|
64
|
-
}
|
|
65
|
-
}
|
|
64
|
+
"packageManager": "pnpm@11.3.0"
|
|
65
|
+
}
|