mixgram 1.3.7 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -345
- package/bin/mixgram.js +51 -160
- package/examples/embedding-demo.js +22 -142
- package/package.json +2 -3
- package/pnpm-workspace.yaml +2 -2
- package/spec/bench-indexing.js +23 -211
- package/spec/bench-search.js +26 -212
- package/spec/run-tests.js +1 -214
- package/spec/scenarios/scenarios.js +277 -607
- package/spec/sdd.md +14 -988
- package/spec/tdd.md +15 -1232
- package/spec/test-large-document.js +20 -91
- package/src/config.js +57 -48
- package/src/core/documents/documents.js +62 -224
- package/src/core/embeddings/embedder.js +14 -54
- package/src/core/embeddings/queue.js +33 -106
- package/src/core/embeddings/vectorStore.js +30 -111
- package/src/core/embeddings/worker-process.js +23 -74
- package/src/core/indexing/indexer.js +51 -237
- package/src/core/indexing/parser.js +13 -114
- package/src/core/indexing/reindex.js +28 -137
- package/src/core/search/search.js +64 -158
- package/src/db/migrations/001_initial.sql +14 -63
- package/src/db/sqlite.js +19 -29
- package/src/fs/paths.js +11 -101
- package/src/fs/watcher.js +15 -48
- package/src/mcp/cli-adapter.js +32 -110
- package/src/mcp/server.js +64 -141
- package/src/mcp/tool-registry.js +21 -132
- package/src/mcp/tools.js +27 -199
- package/src/utils/markdown.js +15 -82
- package/patches/sharp@0.34.5.patch +0 -13
- package/spec/fixtures/realistic-content.js +0 -106
- package/spec/reporter.js +0 -127
- package/spec/run-tests-runner.js +0 -40
- package/src/core/embeddings/worker.js +0 -27
- package/src/core/prompts/prompts.js +0 -15
- package/src/core/sessions/sessions.js +0 -27
- package/src/utils/ids.js +0 -27
|
@@ -1,98 +1,27 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
* Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
|
|
4
|
-
* Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
|
|
5
|
-
*/
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
6
3
|
import fs from 'fs';
|
|
7
4
|
import path from 'path';
|
|
8
5
|
import os from 'os';
|
|
9
6
|
import { loadConfig } from '../src/config.js';
|
|
10
|
-
import {
|
|
7
|
+
import { saveDocument, getDocument } from '../src/core/documents/documents.js';
|
|
11
8
|
import { search } from '../src/core/search/search.js';
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
let passed = 0;
|
|
31
|
-
let failed = 0;
|
|
32
|
-
const ok = (cond, msg) => {
|
|
33
|
-
if (cond) passed++;
|
|
34
|
-
else failed++;
|
|
35
|
-
if (!cond) console.error('FAIL:', msg);
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
const docsDir = path.join(config.projectMemoryRoot, 'generated');
|
|
39
|
-
fs.mkdirSync(docsDir, { recursive: true });
|
|
40
|
-
|
|
41
|
-
const body = buildLargeBody(TARGET_SIZE);
|
|
42
|
-
const id = 'large100k00';
|
|
43
|
-
const now = new Date().toISOString();
|
|
44
|
-
const frontmatter = {
|
|
45
|
-
id,
|
|
46
|
-
title: 'Documento 100k',
|
|
47
|
-
type: 'generated_note',
|
|
48
|
-
scope: 'project',
|
|
49
|
-
project: 'test',
|
|
50
|
-
topic_key: 'test/large-100k',
|
|
51
|
-
created_at: now
|
|
52
|
-
};
|
|
53
|
-
const raw = toMarkdown(frontmatter, body);
|
|
54
|
-
const docPath = path.join(docsDir, 'large-100k.md');
|
|
55
|
-
fs.writeFileSync(docPath, raw, 'utf8');
|
|
56
|
-
const stat = fs.statSync(docPath);
|
|
57
|
-
|
|
58
|
-
const size = raw.length;
|
|
59
|
-
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
|
-
|
|
61
|
-
indexDocument(config, docPath, raw, stat.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
62
|
-
|
|
63
|
-
const db = getDb(config);
|
|
64
|
-
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
|
65
|
-
ok(!!doc, 'documento en tabla documents');
|
|
66
|
-
ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
|
|
67
|
-
|
|
68
|
-
const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
|
|
69
|
-
ok(!!ftsRow, 'documento en FTS');
|
|
70
|
-
ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
|
|
71
|
-
|
|
72
|
-
const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
|
|
73
|
-
ok(results.length >= 1, 'búsqueda encuentra el documento');
|
|
74
|
-
ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
|
|
75
|
-
ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
|
|
76
|
-
|
|
77
|
-
return { passed, failed };
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function main() {
|
|
81
|
-
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
|
|
82
|
-
const config = loadConfig({
|
|
83
|
-
homeMemoryRoot: path.join(tmpDir, 'home'),
|
|
84
|
-
projectMemoryRoot: path.join(tmpDir, 'docs'),
|
|
85
|
-
sqlitePath: path.join(tmpDir, 'index.db')
|
|
86
|
-
});
|
|
87
|
-
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
88
|
-
|
|
89
|
-
const { passed, failed } = runLargeDocumentTest(config);
|
|
90
|
-
console.log(`Large document test: ${passed} passed, ${failed} failed`);
|
|
91
|
-
closeDb();
|
|
92
|
-
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
93
|
-
process.exitCode = failed > 0 ? 1 : 0;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
if (process.argv[1]?.endsWith('test-large-document.js')) {
|
|
97
|
-
main();
|
|
9
|
+
import { reindex } from '../src/core/indexing/reindex.js';
|
|
10
|
+
import { closeDb } from '../src/db/sqlite.js';
|
|
11
|
+
|
|
12
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-'));
|
|
13
|
+
const config = loadConfig({ collections: { general: path.join(root, 'docs') }, sqlitePath: path.join(root, 'index.db') });
|
|
14
|
+
try {
|
|
15
|
+
const content = '# Large document\n\n' + 'A paragraph with ordinary text. '.repeat(4000) + '\n\nNeedleInHaystack100k';
|
|
16
|
+
assert.ok(content.length > 100000);
|
|
17
|
+
const saved = saveDocument(config, { title: 'Large', content, tags: ['large'] });
|
|
18
|
+
assert.equal(saved.indexed, true);
|
|
19
|
+
assert.equal(getDocument(config, saved.id).content.trim(), content);
|
|
20
|
+
assert.equal((await search(config, { query: 'NeedleInHaystack100k' })).results[0].id, saved.id);
|
|
21
|
+
assert.deepEqual(reindex(config, { full: true }).errors, []);
|
|
22
|
+
assert.equal((await search(config, { query: 'NeedleInHaystack100k', tags: ['large'] })).results[0].id, saved.id);
|
|
23
|
+
console.log('Large document: save, read, search and rebuild passed (>100k characters).');
|
|
24
|
+
} finally {
|
|
25
|
+
closeDb(config);
|
|
26
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
98
27
|
}
|
package/src/config.js
CHANGED
|
@@ -1,71 +1,80 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import os from 'os';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
if (typeof p !== 'string') return p;
|
|
6
|
-
if (p === '~' || p.startsWith('~/') || p.startsWith('~\\')) {
|
|
7
|
-
return path.join(os.homedir(), p.slice(2) || '');
|
|
8
|
-
}
|
|
9
|
-
return p;
|
|
10
|
-
}
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import { z } from 'zod';
|
|
11
5
|
|
|
12
6
|
const DEFAULT_CONFIG = {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
sqlitePath: '~/.mixgram/index.db',
|
|
7
|
+
collections: { general: '~/.mixgram/v2/general' },
|
|
8
|
+
defaultCollection: 'general',
|
|
9
|
+
sqlitePath: '~/.mixgram/v2/index.db',
|
|
16
10
|
watch: true,
|
|
17
11
|
indexing: {
|
|
18
12
|
reindexOnStartup: true,
|
|
19
13
|
includeCodeBlocks: false,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
h1: 8.0,
|
|
23
|
-
h2: 6.0,
|
|
24
|
-
h3: 5.0,
|
|
25
|
-
h4: 4.0,
|
|
26
|
-
h5: 3.0,
|
|
27
|
-
h6: 2.0,
|
|
28
|
-
body: 1.0
|
|
29
|
-
}
|
|
14
|
+
watchStabilityMs: 200,
|
|
15
|
+
ftsWeights: { title: 10, h1: 8, h2: 6, h3: 5, h4: 4, h5: 3, h6: 2, body: 1 }
|
|
30
16
|
},
|
|
31
17
|
embeddings: {
|
|
32
18
|
enabled: false,
|
|
33
|
-
mode: 'optional',
|
|
34
|
-
provider: 'local-huggingface',
|
|
35
|
-
vectorStore: 'sqlite-vec',
|
|
36
19
|
model: 'Xenova/multilingual-e5-large',
|
|
37
20
|
dimensions: 1024,
|
|
38
21
|
dtype: 'q8',
|
|
39
|
-
workerConcurrency: 2,
|
|
40
22
|
maxRetries: 3,
|
|
41
|
-
|
|
23
|
+
workerPollMs: 2000,
|
|
24
|
+
queryTimeoutMs: 60000,
|
|
25
|
+
jobLeaseMs: 300000,
|
|
42
26
|
similarityThreshold: 0.80
|
|
43
27
|
},
|
|
44
|
-
search: {
|
|
45
|
-
mode: 'fts-only',
|
|
46
|
-
defaultLimit: 10,
|
|
47
|
-
ftsWeight: 0.7,
|
|
48
|
-
semanticWeight: 0.3,
|
|
49
|
-
defaultScopeMode: 'merged'
|
|
50
|
-
}
|
|
28
|
+
search: { defaultLimit: 10, maxLimit: 100, snippetLength: 300, ftsWeight: 0.7, semanticWeight: 0.3, fusionConstant: 60 }
|
|
51
29
|
};
|
|
52
30
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
31
|
+
const positive = z.number().int().positive();
|
|
32
|
+
const schema = z.object({
|
|
33
|
+
collections: z.record(z.string().min(1), z.string().min(1)).refine(v => Object.keys(v).length > 0),
|
|
34
|
+
defaultCollection: z.string().min(1),
|
|
35
|
+
sqlitePath: z.string().min(1),
|
|
36
|
+
watch: z.boolean(),
|
|
37
|
+
indexing: z.object({ reindexOnStartup: z.boolean(), includeCodeBlocks: z.boolean(), watchStabilityMs: positive,
|
|
38
|
+
ftsWeights: z.object(Object.fromEntries(Object.keys(DEFAULT_CONFIG.indexing.ftsWeights).map(k => [k, z.number().nonnegative()]))) }).strict(),
|
|
39
|
+
embeddings: z.object({ enabled: z.boolean(), model: z.string().min(1), dimensions: positive, dtype: z.string().min(1),
|
|
40
|
+
maxRetries: positive, workerPollMs: positive, queryTimeoutMs: positive, jobLeaseMs: positive, similarityThreshold: z.number().min(-1).max(1) }).strict(),
|
|
41
|
+
search: z.object({ defaultLimit: positive, maxLimit: positive, snippetLength: positive, ftsWeight: z.number().nonnegative(),
|
|
42
|
+
semanticWeight: z.number().nonnegative(), fusionConstant: positive }).strict()
|
|
43
|
+
}).strict();
|
|
44
|
+
|
|
45
|
+
export function mergeConfig(...layers) {
|
|
46
|
+
const out = {};
|
|
47
|
+
for (const layer of layers) {
|
|
48
|
+
for (const [key, value] of Object.entries(layer)) {
|
|
49
|
+
out[key] = key !== 'collections' && value && typeof value === 'object' && !Array.isArray(value)
|
|
50
|
+
? mergeConfig(out[key] ?? {}, value) : value;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
64
54
|
}
|
|
65
55
|
|
|
66
|
-
function
|
|
67
|
-
const
|
|
68
|
-
|
|
56
|
+
function resolvePath(value, base) {
|
|
57
|
+
const expanded = value === '~' ? os.homedir() : value.startsWith('~/') ? path.join(os.homedir(), value.slice(2)) : value;
|
|
58
|
+
const absolute = path.resolve(base, expanded);
|
|
59
|
+
if (fs.existsSync(absolute)) return fs.realpathSync(absolute);
|
|
60
|
+
const parent = path.dirname(absolute);
|
|
61
|
+
return parent === absolute ? absolute : path.join(resolvePath(parent, base), path.basename(absolute));
|
|
69
62
|
}
|
|
70
63
|
|
|
71
|
-
export
|
|
64
|
+
export function loadConfig(overrides = {}, baseDir = process.cwd()) {
|
|
65
|
+
const config = schema.parse(mergeConfig(DEFAULT_CONFIG, overrides));
|
|
66
|
+
config.collections = Object.fromEntries(Object.entries(config.collections).map(([name, root]) => [name, resolvePath(root, baseDir)]));
|
|
67
|
+
if (!Object.hasOwn(config.collections, config.defaultCollection)) throw new Error('defaultCollection must name a configured collection');
|
|
68
|
+
const roots = Object.values(config.collections);
|
|
69
|
+
for (let i = 0; i < roots.length; i++) {
|
|
70
|
+
for (let j = i + 1; j < roots.length; j++) {
|
|
71
|
+
if (roots[i] === roots[j] || roots[i].startsWith(roots[j] + path.sep) || roots[j].startsWith(roots[i] + path.sep)) {
|
|
72
|
+
throw new Error('Collection roots must not overlap');
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (config.search.defaultLimit > config.search.maxLimit) throw new Error('defaultLimit exceeds maxLimit');
|
|
77
|
+
if (config.search.ftsWeight + config.search.semanticWeight === 0) throw new Error('At least one search weight must be positive');
|
|
78
|
+
config.sqlitePath = resolvePath(config.sqlitePath, baseDir);
|
|
79
|
+
return config;
|
|
80
|
+
}
|
|
@@ -1,240 +1,78 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { randomUUID } from 'crypto';
|
|
4
|
+
import { documentPath } from '../../fs/paths.js';
|
|
5
|
+
import { toMarkdown } from '../../utils/markdown.js';
|
|
6
|
+
import { readDocument, indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
|
|
7
7
|
import { getDb } from '../../db/sqlite.js';
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
)
|
|
15
|
-
|
|
16
|
-
if (!fs.existsSync(row.path)) return null;
|
|
17
|
-
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(id);
|
|
18
|
-
const tags = tagRows.map((r) => r.tag);
|
|
19
|
-
const raw = fs.readFileSync(row.path, 'utf8');
|
|
20
|
-
const { body } = parse(raw);
|
|
21
|
-
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/** Load document metadata by topic_key + scope + project. */
|
|
25
|
-
function getDocumentMetaByTopicKey(config, topic_key, scope, project) {
|
|
26
|
-
const db = getDb(config);
|
|
27
|
-
const row = db.prepare(
|
|
28
|
-
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
29
|
-
).get(topic_key, scope, project ?? null, project ?? null);
|
|
30
|
-
if (!row) return null;
|
|
31
|
-
if (!fs.existsSync(row.path)) return null;
|
|
32
|
-
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
33
|
-
const tags = tagRows.map((r) => r.tag);
|
|
34
|
-
const raw = fs.readFileSync(row.path, 'utf8');
|
|
35
|
-
const { body } = parse(raw);
|
|
36
|
-
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function rowToFrontmatter(row, tags = []) {
|
|
40
|
-
return {
|
|
41
|
-
id: row.id,
|
|
42
|
-
title: row.title ?? '',
|
|
43
|
-
type: row.type ?? 'generated_note',
|
|
44
|
-
scope: row.scope ?? 'project',
|
|
45
|
-
project: row.project ?? null,
|
|
46
|
-
topic_key: row.topic_key ?? null,
|
|
47
|
-
session_id: row.session_id ?? null,
|
|
48
|
-
tool_name: row.tool_name ?? null,
|
|
49
|
-
created_at: row.created_at,
|
|
50
|
-
updated_at: row.updated_at,
|
|
51
|
-
revision_count: row.revision_count ?? 1,
|
|
52
|
-
duplicate_count: row.duplicate_count ?? 0,
|
|
53
|
-
deleted: !!row.deleted_at,
|
|
54
|
-
tags: Array.isArray(tags) ? tags : [],
|
|
55
|
-
embedding_status: row.embedding_status ?? 'disabled',
|
|
56
|
-
deleted_at: row.deleted_at ?? null
|
|
57
|
-
};
|
|
9
|
+
function resolveDocument(config, id, includeDeleted = false) {
|
|
10
|
+
const row = getDb(config).prepare('SELECT path FROM documents WHERE id = ?').get(id);
|
|
11
|
+
if (!row) throw new Error('Document not found');
|
|
12
|
+
const doc = readDocument(config, row.path);
|
|
13
|
+
if (doc.metadata.id !== id) throw new Error('Document identity changed; reindex required');
|
|
14
|
+
if (!includeDeleted && doc.metadata.deleted_at) throw new Error('Document not found');
|
|
15
|
+
return doc;
|
|
58
16
|
}
|
|
59
17
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
18
|
+
function persist(config, doc, created) {
|
|
19
|
+
fs.mkdirSync(path.dirname(doc.path), { recursive: true });
|
|
20
|
+
const temp = `${doc.path}.${randomUUID()}.tmp`;
|
|
21
|
+
try {
|
|
22
|
+
fs.writeFileSync(temp, toMarkdown(doc.metadata, doc.body), { encoding: 'utf8', flag: 'wx' });
|
|
23
|
+
fs.renameSync(temp, doc.path);
|
|
24
|
+
} finally {
|
|
25
|
+
if (fs.existsSync(temp)) fs.unlinkSync(temp);
|
|
26
|
+
}
|
|
27
|
+
const result = { id: doc.metadata.id, path: doc.path, created, saved: true, indexed: true };
|
|
28
|
+
try { indexDocument(config, doc.path); }
|
|
29
|
+
catch (error) { result.indexed = false; result.error = error.message; }
|
|
30
|
+
return result;
|
|
69
31
|
}
|
|
70
32
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
33
|
+
export function saveDocument(config, payload) {
|
|
34
|
+
const db = getDb(config);
|
|
35
|
+
const collection = payload.collection ?? config.defaultCollection;
|
|
36
|
+
if (!Object.hasOwn(config.collections, collection)) throw new Error('Unknown collection');
|
|
37
|
+
const row = !payload.id && payload.key ? db.prepare('SELECT id FROM documents WHERE collection = ? AND key = ? AND deleted_at IS NULL').get(collection, payload.key) : null;
|
|
38
|
+
const id = payload.id ?? row?.id;
|
|
39
|
+
const existing = id ? resolveDocument(config, id) : null;
|
|
40
|
+
if (existing && payload.collection && payload.collection !== existing.collection) throw new Error('Move the file between collection folders and reindex to change collection');
|
|
75
41
|
const now = new Date().toISOString();
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return {
|
|
79
|
-
...existing,
|
|
80
|
-
...overrides,
|
|
81
|
-
updated_at: now,
|
|
82
|
-
revision_count: rev,
|
|
83
|
-
id: existing.id
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
return {
|
|
87
|
-
id: overrides.id || observationId(),
|
|
88
|
-
title: overrides.title ?? '',
|
|
89
|
-
type: overrides.type ?? 'generated_note',
|
|
90
|
-
scope: overrides.scope ?? 'project',
|
|
91
|
-
project: overrides.project ?? null,
|
|
92
|
-
topic_key: overrides.topic_key ?? null,
|
|
93
|
-
session_id: overrides.session_id ?? null,
|
|
94
|
-
tool_name: overrides.tool_name ?? null,
|
|
95
|
-
created_at: overrides.created_at ?? now,
|
|
96
|
-
updated_at: now,
|
|
97
|
-
revision_count: 1,
|
|
98
|
-
duplicate_count: 0,
|
|
99
|
-
deleted: false,
|
|
100
|
-
tags: Array.isArray(overrides.tags) ? overrides.tags : [],
|
|
101
|
-
embedding_status: 'disabled'
|
|
42
|
+
const metadata = existing ? { ...existing.metadata } : {
|
|
43
|
+
id: randomUUID(), title: '', type: 'note', key: null, tags: [], created_at: now, deleted_at: null
|
|
102
44
|
};
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Save or update a memory document (mem_save semantics).
|
|
107
|
-
* Resolution: 1) by id, 2) by topic_key in scope, 3) create new.
|
|
108
|
-
* @returns {{ id: string, path: string, created: boolean }}
|
|
109
|
-
*/
|
|
110
|
-
function saveDocument(config, payload) {
|
|
111
|
-
const { title, type, scope, project, topic_key, content, session_id, id: givenId, tags } = payload;
|
|
112
|
-
const resolved = givenId
|
|
113
|
-
? resolveDocument(config, { id: givenId })
|
|
114
|
-
: topic_key != null && scope != null
|
|
115
|
-
? resolveDocument(config, { topic_key, scope, project })
|
|
116
|
-
: null;
|
|
117
|
-
|
|
118
|
-
let docPath;
|
|
119
|
-
let frontmatter;
|
|
120
|
-
let body = typeof content === 'string' ? content : (content && content.text) || '';
|
|
121
|
-
|
|
122
|
-
if (resolved) {
|
|
123
|
-
docPath = resolved.path;
|
|
124
|
-
frontmatter = buildFrontmatter(
|
|
125
|
-
{ title, type, scope, project, topic_key, session_id, tags },
|
|
126
|
-
{ ...resolved.frontmatter, revision_count: resolved.frontmatter.revision_count ?? 1 }
|
|
127
|
-
);
|
|
128
|
-
body = body || resolved.body;
|
|
129
|
-
} else {
|
|
130
|
-
frontmatter = buildFrontmatter({
|
|
131
|
-
id: givenId || observationId(),
|
|
132
|
-
title,
|
|
133
|
-
type: type || 'generated_note',
|
|
134
|
-
scope: scope || 'project',
|
|
135
|
-
project: project || null,
|
|
136
|
-
topic_key: topic_key || null,
|
|
137
|
-
session_id: session_id || null,
|
|
138
|
-
tool_name: 'mem_save',
|
|
139
|
-
tags: tags || []
|
|
140
|
-
});
|
|
141
|
-
docPath = documentPath(config, frontmatter);
|
|
142
|
-
ensureDir(docPath);
|
|
45
|
+
for (const key of ['title', 'type', 'key', 'tags']) {
|
|
46
|
+
if (payload[key] !== undefined) metadata[key] = payload[key];
|
|
143
47
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
id: frontmatter.id,
|
|
151
|
-
path: docPath,
|
|
152
|
-
created: !resolved
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Update document by id (mem_update semantics).
|
|
158
|
-
* @returns {{ id: string } | null}
|
|
159
|
-
*/
|
|
160
|
-
function updateDocument(config, { id, title, content, tags }) {
|
|
161
|
-
const resolved = resolveDocument(config, { id });
|
|
162
|
-
if (!resolved) return null;
|
|
163
|
-
const body = typeof content === 'string' ? content : (content && content.text) || resolved.body;
|
|
164
|
-
const frontmatter = buildFrontmatter(
|
|
165
|
-
{ title: title ?? resolved.frontmatter.title, tags },
|
|
166
|
-
resolved.frontmatter
|
|
167
|
-
);
|
|
168
|
-
const raw = toMarkdown(frontmatter, body);
|
|
169
|
-
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
170
|
-
const stats = fs.statSync(resolved.path);
|
|
171
|
-
indexDocument(config, resolved.path, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
172
|
-
return { id: frontmatter.id };
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
/**
|
|
176
|
-
* Soft or hard delete (mem_delete semantics).
|
|
177
|
-
* @param {{ hardDelete?: boolean }} options
|
|
178
|
-
* @returns {boolean} true if document existed and was handled
|
|
179
|
-
*/
|
|
180
|
-
function deleteDocument(config, documentId, options = {}) {
|
|
181
|
-
const resolved = resolveDocument(config, { id: documentId });
|
|
182
|
-
if (!resolved) return false;
|
|
183
|
-
if (options.hardDelete) {
|
|
184
|
-
try { fs.unlinkSync(resolved.path); } catch (_) {}
|
|
185
|
-
removeDocumentFromIndex(config, documentId);
|
|
186
|
-
} else {
|
|
187
|
-
const now = new Date().toISOString();
|
|
188
|
-
const frontmatter = { ...resolved.frontmatter, deleted_at: now, deleted: true };
|
|
189
|
-
const raw = toMarkdown(frontmatter, resolved.body);
|
|
190
|
-
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
191
|
-
const db = getDb(config);
|
|
192
|
-
db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
|
|
193
|
-
const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
|
|
194
|
-
for (const { rowid } of ftsRowids) {
|
|
195
|
-
db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
|
|
196
|
-
}
|
|
48
|
+
metadata.tags = [...new Set(metadata.tags)];
|
|
49
|
+
metadata.updated_at = now;
|
|
50
|
+
const actualCollection = existing?.collection ?? collection;
|
|
51
|
+
if (metadata.key) {
|
|
52
|
+
const conflict = db.prepare('SELECT id FROM documents WHERE collection = ? AND key = ? AND id != ? AND deleted_at IS NULL').get(actualCollection, metadata.key, metadata.id);
|
|
53
|
+
if (conflict) throw new Error('Key already belongs to another document in this collection');
|
|
197
54
|
}
|
|
198
|
-
|
|
55
|
+
const doc = { metadata, body: payload.content ?? existing?.body ?? '',
|
|
56
|
+
path: existing?.path ?? documentPath(config, { collection: actualCollection, ...metadata }) };
|
|
57
|
+
if (!existing && fs.existsSync(doc.path)) throw new Error('Document path already exists');
|
|
58
|
+
return persist(config, doc, !existing);
|
|
199
59
|
}
|
|
200
60
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
*/
|
|
205
|
-
function getObservation(config, documentId) {
|
|
206
|
-
const resolved = resolveDocument(config, { id: documentId });
|
|
207
|
-
if (!resolved) return null;
|
|
208
|
-
const fullContent = toMarkdown(resolved.frontmatter, resolved.body);
|
|
209
|
-
return {
|
|
210
|
-
id: resolved.frontmatter.id,
|
|
211
|
-
title: resolved.frontmatter.title,
|
|
212
|
-
type: resolved.frontmatter.type,
|
|
213
|
-
scope: resolved.frontmatter.scope,
|
|
214
|
-
project: resolved.frontmatter.project ?? null,
|
|
215
|
-
topic_key: resolved.frontmatter.topic_key ?? null,
|
|
216
|
-
session_id: resolved.frontmatter.session_id ?? null,
|
|
217
|
-
content: fullContent
|
|
218
|
-
};
|
|
61
|
+
export function getDocument(config, id) {
|
|
62
|
+
const doc = resolveDocument(config, id);
|
|
63
|
+
return { ...doc.metadata, collection: doc.collection, content: doc.body };
|
|
219
64
|
}
|
|
220
65
|
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
66
|
+
export function deleteDocument(config, id, { hardDelete = false } = {}) {
|
|
67
|
+
const doc = resolveDocument(config, id, true);
|
|
68
|
+
if (hardDelete) {
|
|
69
|
+
fs.unlinkSync(doc.path);
|
|
70
|
+
try { removeDocumentFromIndex(config, id); }
|
|
71
|
+
catch (error) { return { id, deleted: true, indexed: false, error: error.message }; }
|
|
72
|
+
return { id, deleted: true, indexed: true };
|
|
73
|
+
}
|
|
74
|
+
const now = new Date().toISOString();
|
|
75
|
+
doc.metadata.deleted_at = now;
|
|
76
|
+
doc.metadata.updated_at = now;
|
|
77
|
+
return { ...persist(config, doc, false), deleted: true };
|
|
230
78
|
}
|
|
231
|
-
|
|
232
|
-
export {
|
|
233
|
-
resolveDocument,
|
|
234
|
-
buildFrontmatter,
|
|
235
|
-
saveDocument,
|
|
236
|
-
updateDocument,
|
|
237
|
-
deleteDocument,
|
|
238
|
-
getObservation,
|
|
239
|
-
getObservationsBySession
|
|
240
|
-
};
|
|
@@ -1,58 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
* Optional embedding provider using @huggingface/transformers (Xenova/multilingual-e5-large).
|
|
3
|
-
* Dynamically imported so the app runs without it when embeddings are disabled.
|
|
4
|
-
*/
|
|
5
|
-
const MODEL = 'Xenova/multilingual-e5-large';
|
|
6
|
-
const DIM = 1024;
|
|
7
|
-
const DTYPE = 'q8';
|
|
8
|
-
|
|
9
|
-
const pipelineCache = new Map();
|
|
10
|
-
|
|
11
|
-
async function loadPipeline(options = {}) {
|
|
12
|
-
const model = options.model ?? MODEL;
|
|
13
|
-
const dtype = options.dtype ?? DTYPE;
|
|
14
|
-
const key = `${model}:${dtype}`;
|
|
15
|
-
if (pipelineCache.has(key)) return pipelineCache.get(key);
|
|
16
|
-
try {
|
|
17
|
-
const { pipeline } = await import('@huggingface/transformers');
|
|
18
|
-
const pipe = pipeline('feature-extraction', model, {
|
|
19
|
-
pooling: 'mean',
|
|
20
|
-
normalize: true,
|
|
21
|
-
dtype
|
|
22
|
-
});
|
|
23
|
-
pipelineCache.set(key, pipe);
|
|
24
|
-
return pipe;
|
|
25
|
-
} catch (err) {
|
|
26
|
-
throw err;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Embed a single text with the given pipeline. Returns Float32Array (L2-normalized).
|
|
32
|
-
*/
|
|
33
|
-
async function embedWith(extractor, text, dimensions) {
|
|
34
|
-
if (!text || typeof text !== 'string') text = '';
|
|
35
|
-
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
36
|
-
const arr = output.data || output.arraySync?.() || Array.from(output);
|
|
37
|
-
return new Float32Array(arr.slice(0, dimensions));
|
|
38
|
-
}
|
|
1
|
+
const pipelines = new WeakMap();
|
|
39
2
|
|
|
40
3
|
export async function getEmbedder(config) {
|
|
41
|
-
if (!config
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
dtype: opts.dtype ?? DTYPE
|
|
47
|
-
});
|
|
48
|
-
const dimensions = opts.dimensions ?? DIM;
|
|
49
|
-
return {
|
|
50
|
-
embed: (text) => embedWith(extractor, text, dimensions),
|
|
51
|
-
dimensions
|
|
52
|
-
};
|
|
53
|
-
} catch {
|
|
54
|
-
return null;
|
|
4
|
+
if (!config.embeddings.enabled) return null;
|
|
5
|
+
if (!pipelines.has(config)) {
|
|
6
|
+
const loading = import('@huggingface/transformers').then(({ pipeline }) => pipeline('feature-extraction', config.embeddings.model, { dtype: config.embeddings.dtype }));
|
|
7
|
+
pipelines.set(config, loading);
|
|
8
|
+
loading.catch(() => pipelines.delete(config));
|
|
55
9
|
}
|
|
10
|
+
const extractor = await pipelines.get(config);
|
|
11
|
+
return {
|
|
12
|
+
async embed(text) {
|
|
13
|
+
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
14
|
+
if (output.data.length !== config.embeddings.dimensions) throw new Error('Embedding model dimensions do not match configuration');
|
|
15
|
+
return new Float32Array(output.data);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
56
18
|
}
|
|
57
|
-
|
|
58
|
-
export { embedWith as embed, DIM, MODEL, DTYPE };
|