mixgram 1.3.6 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +68 -345
  2. package/bin/mixgram.js +51 -160
  3. package/examples/embedding-demo.js +22 -142
  4. package/package.json +19 -19
  5. package/spec/bench-indexing.js +23 -211
  6. package/spec/bench-search.js +26 -212
  7. package/spec/run-tests.js +1 -214
  8. package/spec/scenarios/scenarios.js +277 -607
  9. package/spec/sdd.md +14 -988
  10. package/spec/tdd.md +15 -1232
  11. package/spec/test-large-document.js +20 -91
  12. package/src/config.js +57 -48
  13. package/src/core/documents/documents.js +62 -224
  14. package/src/core/embeddings/embedder.js +14 -54
  15. package/src/core/embeddings/queue.js +33 -106
  16. package/src/core/embeddings/vectorStore.js +30 -111
  17. package/src/core/embeddings/worker-process.js +23 -74
  18. package/src/core/indexing/indexer.js +51 -237
  19. package/src/core/indexing/parser.js +13 -114
  20. package/src/core/indexing/reindex.js +28 -137
  21. package/src/core/search/search.js +64 -158
  22. package/src/db/migrations/001_initial.sql +14 -63
  23. package/src/db/sqlite.js +19 -29
  24. package/src/fs/paths.js +11 -101
  25. package/src/fs/watcher.js +15 -48
  26. package/src/mcp/cli-adapter.js +32 -110
  27. package/src/mcp/server.js +64 -141
  28. package/src/mcp/tool-registry.js +21 -132
  29. package/src/mcp/tools.js +27 -199
  30. package/src/utils/markdown.js +15 -82
  31. package/spec/fixtures/realistic-content.js +0 -106
  32. package/spec/reporter.js +0 -127
  33. package/spec/run-tests-runner.js +0 -40
  34. package/src/core/embeddings/worker.js +0 -27
  35. package/src/core/prompts/prompts.js +0 -15
  36. package/src/core/sessions/sessions.js +0 -27
  37. package/src/utils/ids.js +0 -27
@@ -1,98 +1,27 @@
1
1
  #!/usr/bin/env node
2
- /**
3
- * Test: indexar un documento de ~100k caracteres y verificar que queda indexado y es buscable.
4
- * Se ejecuta con la suite (npm test) o en solitario: node spec/test-large-document.js
5
- */
2
+ import assert from 'node:assert/strict';
6
3
  import fs from 'fs';
7
4
  import path from 'path';
8
5
  import os from 'os';
9
6
  import { loadConfig } from '../src/config.js';
10
- import { indexDocument } from '../src/core/indexing/indexer.js';
7
+ import { saveDocument, getDocument } from '../src/core/documents/documents.js';
11
8
  import { search } from '../src/core/search/search.js';
12
- import { getDb, closeDb } from '../src/db/sqlite.js';
13
- import { toMarkdown } from '../src/utils/markdown.js';
14
-
15
- const TARGET_SIZE = 100_000;
16
- const NEEDLE = 'NeedleInHaystack100k';
17
-
18
- function buildLargeBody(targetSize) {
19
- const intro = `# Documento grande\n\nPárrafo inicial. ${NEEDLE} aquí.\n\n`;
20
- const paragraph = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. '.repeat(20) + '\n\n';
21
- const repeat = Math.ceil((targetSize - intro.length) / paragraph.length);
22
- return intro + paragraph.repeat(Math.max(0, repeat));
23
- }
24
-
25
- /**
26
- * @param {object} config - resolved config (e.g. from run-tests)
27
- * @returns {{ passed: number, failed: number }}
28
- */
29
- export function runLargeDocumentTest(config) {
30
- let passed = 0;
31
- let failed = 0;
32
- const ok = (cond, msg) => {
33
- if (cond) passed++;
34
- else failed++;
35
- if (!cond) console.error('FAIL:', msg);
36
- };
37
-
38
- const docsDir = path.join(config.projectMemoryRoot, 'generated');
39
- fs.mkdirSync(docsDir, { recursive: true });
40
-
41
- const body = buildLargeBody(TARGET_SIZE);
42
- const id = 'large100k00';
43
- const now = new Date().toISOString();
44
- const frontmatter = {
45
- id,
46
- title: 'Documento 100k',
47
- type: 'generated_note',
48
- scope: 'project',
49
- project: 'test',
50
- topic_key: 'test/large-100k',
51
- created_at: now
52
- };
53
- const raw = toMarkdown(frontmatter, body);
54
- const docPath = path.join(docsDir, 'large-100k.md');
55
- fs.writeFileSync(docPath, raw, 'utf8');
56
- const stat = fs.statSync(docPath);
57
-
58
- const size = raw.length;
59
- ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
60
-
61
- indexDocument(config, docPath, raw, stat.mtimeMs, { overrideFrontmatter: frontmatter });
62
-
63
- const db = getDb(config);
64
- const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
65
- ok(!!doc, 'documento en tabla documents');
66
- ok(doc.body_len >= TARGET_SIZE * 0.9, `body en DB ~100k (${doc?.body_len ?? 0} chars)`);
67
-
68
- const ftsRow = db.prepare('SELECT document_id, length(body) AS body_len FROM document_fts WHERE document_id = ?').get(id);
69
- ok(!!ftsRow, 'documento en FTS');
70
- ok((ftsRow?.body_len ?? 0) >= TARGET_SIZE * 0.9, `body en FTS ~100k (${ftsRow?.body_len ?? 0} chars)`);
71
-
72
- const results = search(config, { query: NEEDLE, project: 'test', limit: 5 });
73
- ok(results.length >= 1, 'búsqueda encuentra el documento');
74
- ok(results.some((r) => r.documentId === id), 'resultado es el doc 100k');
75
- ok(results[0].snippet != null && results[0].snippet.length > 0, 'snippet no vacío');
76
-
77
- return { passed, failed };
78
- }
79
-
80
- function main() {
81
- const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-doc-'));
82
- const config = loadConfig({
83
- homeMemoryRoot: path.join(tmpDir, 'home'),
84
- projectMemoryRoot: path.join(tmpDir, 'docs'),
85
- sqlitePath: path.join(tmpDir, 'index.db')
86
- });
87
- fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
88
-
89
- const { passed, failed } = runLargeDocumentTest(config);
90
- console.log(`Large document test: ${passed} passed, ${failed} failed`);
91
- closeDb();
92
- try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (_) {}
93
- process.exitCode = failed > 0 ? 1 : 0;
94
- }
95
-
96
- if (process.argv[1]?.endsWith('test-large-document.js')) {
97
- main();
9
+ import { reindex } from '../src/core/indexing/reindex.js';
10
+ import { closeDb } from '../src/db/sqlite.js';
11
+
12
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'mixgram-large-'));
13
+ const config = loadConfig({ collections: { general: path.join(root, 'docs') }, sqlitePath: path.join(root, 'index.db') });
14
+ try {
15
+ const content = '# Large document\n\n' + 'A paragraph with ordinary text. '.repeat(4000) + '\n\nNeedleInHaystack100k';
16
+ assert.ok(content.length > 100000);
17
+ const saved = saveDocument(config, { title: 'Large', content, tags: ['large'] });
18
+ assert.equal(saved.indexed, true);
19
+ assert.equal(getDocument(config, saved.id).content.trim(), content);
20
+ assert.equal((await search(config, { query: 'NeedleInHaystack100k' })).results[0].id, saved.id);
21
+ assert.deepEqual(reindex(config, { full: true }).errors, []);
22
+ assert.equal((await search(config, { query: 'NeedleInHaystack100k', tags: ['large'] })).results[0].id, saved.id);
23
+ console.log('Large document: save, read, search and rebuild passed (>100k characters).');
24
+ } finally {
25
+ closeDb(config);
26
+ fs.rmSync(root, { recursive: true, force: true });
98
27
  }
package/src/config.js CHANGED
@@ -1,71 +1,80 @@
1
1
  import path from 'path';
2
2
  import os from 'os';
3
-
4
- function expandHome(p) {
5
- if (typeof p !== 'string') return p;
6
- if (p === '~' || p.startsWith('~/') || p.startsWith('~\\')) {
7
- return path.join(os.homedir(), p.slice(2) || '');
8
- }
9
- return p;
10
- }
3
+ import fs from 'fs';
4
+ import { z } from 'zod';
11
5
 
12
6
  const DEFAULT_CONFIG = {
13
- homeMemoryRoot: '~/.mixgram/docs',
14
- projectMemoryRoot: './docs',
15
- sqlitePath: '~/.mixgram/index.db',
7
+ collections: { general: '~/.mixgram/v2/general' },
8
+ defaultCollection: 'general',
9
+ sqlitePath: '~/.mixgram/v2/index.db',
16
10
  watch: true,
17
11
  indexing: {
18
12
  reindexOnStartup: true,
19
13
  includeCodeBlocks: false,
20
- ftsWeights: {
21
- title: 10.0,
22
- h1: 8.0,
23
- h2: 6.0,
24
- h3: 5.0,
25
- h4: 4.0,
26
- h5: 3.0,
27
- h6: 2.0,
28
- body: 1.0
29
- }
14
+ watchStabilityMs: 200,
15
+ ftsWeights: { title: 10, h1: 8, h2: 6, h3: 5, h4: 4, h5: 3, h6: 2, body: 1 }
30
16
  },
31
17
  embeddings: {
32
18
  enabled: false,
33
- mode: 'optional',
34
- provider: 'local-huggingface',
35
- vectorStore: 'sqlite-vec',
36
19
  model: 'Xenova/multilingual-e5-large',
37
20
  dimensions: 1024,
38
21
  dtype: 'q8',
39
- workerConcurrency: 2,
40
22
  maxRetries: 3,
41
- queueOnWrite: true,
23
+ workerPollMs: 2000,
24
+ queryTimeoutMs: 60000,
25
+ jobLeaseMs: 300000,
42
26
  similarityThreshold: 0.80
43
27
  },
44
- search: {
45
- mode: 'fts-only',
46
- defaultLimit: 10,
47
- ftsWeight: 0.7,
48
- semanticWeight: 0.3,
49
- defaultScopeMode: 'merged'
50
- }
28
+ search: { defaultLimit: 10, maxLimit: 100, snippetLength: 300, ftsWeight: 0.7, semanticWeight: 0.3, fusionConstant: 60 }
51
29
  };
52
30
 
53
- function resolvePaths(config, baseDir, projectBaseDir = null) {
54
- const base = baseDir || process.cwd();
55
- const projectBase = projectBaseDir ?? base;
56
- const homeRoot = expandHome(config.homeMemoryRoot);
57
- const sqlite = expandHome(config.sqlitePath);
58
- return {
59
- ...config,
60
- homeMemoryRoot: path.isAbsolute(homeRoot) ? homeRoot : path.resolve(base, homeRoot),
61
- projectMemoryRoot: path.resolve(projectBase, config.projectMemoryRoot ?? './docs'),
62
- sqlitePath: path.isAbsolute(sqlite) ? sqlite : path.resolve(base, sqlite)
63
- };
31
+ const positive = z.number().int().positive();
32
+ const schema = z.object({
33
+ collections: z.record(z.string().min(1), z.string().min(1)).refine(v => Object.keys(v).length > 0),
34
+ defaultCollection: z.string().min(1),
35
+ sqlitePath: z.string().min(1),
36
+ watch: z.boolean(),
37
+ indexing: z.object({ reindexOnStartup: z.boolean(), includeCodeBlocks: z.boolean(), watchStabilityMs: positive,
38
+ ftsWeights: z.object(Object.fromEntries(Object.keys(DEFAULT_CONFIG.indexing.ftsWeights).map(k => [k, z.number().nonnegative()]))) }).strict(),
39
+ embeddings: z.object({ enabled: z.boolean(), model: z.string().min(1), dimensions: positive, dtype: z.string().min(1),
40
+ maxRetries: positive, workerPollMs: positive, queryTimeoutMs: positive, jobLeaseMs: positive, similarityThreshold: z.number().min(-1).max(1) }).strict(),
41
+ search: z.object({ defaultLimit: positive, maxLimit: positive, snippetLength: positive, ftsWeight: z.number().nonnegative(),
42
+ semanticWeight: z.number().nonnegative(), fusionConstant: positive }).strict()
43
+ }).strict();
44
+
45
+ export function mergeConfig(...layers) {
46
+ const out = {};
47
+ for (const layer of layers) {
48
+ for (const [key, value] of Object.entries(layer)) {
49
+ out[key] = key !== 'collections' && value && typeof value === 'object' && !Array.isArray(value)
50
+ ? mergeConfig(out[key] ?? {}, value) : value;
51
+ }
52
+ }
53
+ return out;
64
54
  }
65
55
 
66
- function loadConfig(overrides = {}, baseDir, projectBaseDir = null) {
67
- const config = { ...DEFAULT_CONFIG, ...overrides };
68
- return resolvePaths(config, baseDir, projectBaseDir);
56
+ function resolvePath(value, base) {
57
+ const expanded = value === '~' ? os.homedir() : value.startsWith('~/') ? path.join(os.homedir(), value.slice(2)) : value;
58
+ const absolute = path.resolve(base, expanded);
59
+ if (fs.existsSync(absolute)) return fs.realpathSync(absolute);
60
+ const parent = path.dirname(absolute);
61
+ return parent === absolute ? absolute : path.join(resolvePath(parent, base), path.basename(absolute));
69
62
  }
70
63
 
71
- export { DEFAULT_CONFIG, loadConfig, resolvePaths };
64
+ export function loadConfig(overrides = {}, baseDir = process.cwd()) {
65
+ const config = schema.parse(mergeConfig(DEFAULT_CONFIG, overrides));
66
+ config.collections = Object.fromEntries(Object.entries(config.collections).map(([name, root]) => [name, resolvePath(root, baseDir)]));
67
+ if (!Object.hasOwn(config.collections, config.defaultCollection)) throw new Error('defaultCollection must name a configured collection');
68
+ const roots = Object.values(config.collections);
69
+ for (let i = 0; i < roots.length; i++) {
70
+ for (let j = i + 1; j < roots.length; j++) {
71
+ if (roots[i] === roots[j] || roots[i].startsWith(roots[j] + path.sep) || roots[j].startsWith(roots[i] + path.sep)) {
72
+ throw new Error('Collection roots must not overlap');
73
+ }
74
+ }
75
+ }
76
+ if (config.search.defaultLimit > config.search.maxLimit) throw new Error('defaultLimit exceeds maxLimit');
77
+ if (config.search.ftsWeight + config.search.semanticWeight === 0) throw new Error('At least one search weight must be positive');
78
+ config.sqlitePath = resolvePath(config.sqlitePath, baseDir);
79
+ return config;
80
+ }
@@ -1,240 +1,78 @@
1
1
  import fs from 'fs';
2
- import { documentPath, ensureDir } from '../../fs/paths.js';
3
- import { toMarkdown, parse } from '../../utils/markdown.js';
4
- import { observationId } from '../../utils/ids.js';
5
- import { contentHash } from '../../utils/hash.js';
6
- import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
2
+ import path from 'path';
3
+ import { randomUUID } from 'crypto';
4
+ import { documentPath } from '../../fs/paths.js';
5
+ import { toMarkdown } from '../../utils/markdown.js';
6
+ import { readDocument, indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
7
7
  import { getDb } from '../../db/sqlite.js';
8
8
 
9
- /** Load document metadata from DB (canonical source). Returns null if not found or file missing. */
10
- function getDocumentMetaById(config, id) {
11
- const db = getDb(config);
12
- const row = db.prepare(
13
- 'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE id = ? AND deleted_at IS NULL'
14
- ).get(id);
15
- if (!row) return null;
16
- if (!fs.existsSync(row.path)) return null;
17
- const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(id);
18
- const tags = tagRows.map((r) => r.tag);
19
- const raw = fs.readFileSync(row.path, 'utf8');
20
- const { body } = parse(raw);
21
- return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
22
- }
23
-
24
- /** Load document metadata by topic_key + scope + project. */
25
- function getDocumentMetaByTopicKey(config, topic_key, scope, project) {
26
- const db = getDb(config);
27
- const row = db.prepare(
28
- 'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
29
- ).get(topic_key, scope, project ?? null, project ?? null);
30
- if (!row) return null;
31
- if (!fs.existsSync(row.path)) return null;
32
- const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
33
- const tags = tagRows.map((r) => r.tag);
34
- const raw = fs.readFileSync(row.path, 'utf8');
35
- const { body } = parse(raw);
36
- return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
37
- }
38
-
39
- function rowToFrontmatter(row, tags = []) {
40
- return {
41
- id: row.id,
42
- title: row.title ?? '',
43
- type: row.type ?? 'generated_note',
44
- scope: row.scope ?? 'project',
45
- project: row.project ?? null,
46
- topic_key: row.topic_key ?? null,
47
- session_id: row.session_id ?? null,
48
- tool_name: row.tool_name ?? null,
49
- created_at: row.created_at,
50
- updated_at: row.updated_at,
51
- revision_count: row.revision_count ?? 1,
52
- duplicate_count: row.duplicate_count ?? 0,
53
- deleted: !!row.deleted_at,
54
- tags: Array.isArray(tags) ? tags : [],
55
- embedding_status: row.embedding_status ?? 'disabled',
56
- deleted_at: row.deleted_at ?? null
57
- };
9
+ function resolveDocument(config, id, includeDeleted = false) {
10
+ const row = getDb(config).prepare('SELECT path FROM documents WHERE id = ?').get(id);
11
+ if (!row) throw new Error('Document not found');
12
+ const doc = readDocument(config, row.path);
13
+ if (doc.metadata.id !== id) throw new Error('Document identity changed; reindex required');
14
+ if (!includeDeleted && doc.metadata.deleted_at) throw new Error('Document not found');
15
+ return doc;
58
16
  }
59
17
 
60
- /**
61
- * Resolve existing document by id (takes precedence) or by topic_key + scope + project.
62
- * Metadata comes from SQLite; body from disk.
63
- * @returns {{ path: string, frontmatter: object, body: string } | null}
64
- */
65
- function resolveDocument(config, { id, topic_key, scope, project }) {
66
- if (id) return getDocumentMetaById(config, id);
67
- if (topic_key != null && scope != null) return getDocumentMetaByTopicKey(config, topic_key, scope, project);
68
- return null;
18
+ function persist(config, doc, created) {
19
+ fs.mkdirSync(path.dirname(doc.path), { recursive: true });
20
+ const temp = `${doc.path}.${randomUUID()}.tmp`;
21
+ try {
22
+ fs.writeFileSync(temp, toMarkdown(doc.metadata, doc.body), { encoding: 'utf8', flag: 'wx' });
23
+ fs.renameSync(temp, doc.path);
24
+ } finally {
25
+ if (fs.existsSync(temp)) fs.unlinkSync(temp);
26
+ }
27
+ const result = { id: doc.metadata.id, path: doc.path, created, saved: true, indexed: true };
28
+ try { indexDocument(config, doc.path); }
29
+ catch (error) { result.indexed = false; result.error = error.message; }
30
+ return result;
69
31
  }
70
32
 
71
- /**
72
- * Build frontmatter for a new or updated document.
73
- */
74
- function buildFrontmatter(overrides = {}, existing = null) {
33
+ export function saveDocument(config, payload) {
34
+ const db = getDb(config);
35
+ const collection = payload.collection ?? config.defaultCollection;
36
+ if (!Object.hasOwn(config.collections, collection)) throw new Error('Unknown collection');
37
+ const row = !payload.id && payload.key ? db.prepare('SELECT id FROM documents WHERE collection = ? AND key = ? AND deleted_at IS NULL').get(collection, payload.key) : null;
38
+ const id = payload.id ?? row?.id;
39
+ const existing = id ? resolveDocument(config, id) : null;
40
+ if (existing && payload.collection && payload.collection !== existing.collection) throw new Error('Move the file between collection folders and reindex to change collection');
75
41
  const now = new Date().toISOString();
76
- if (existing) {
77
- const rev = (existing.revision_count ?? 1) + 1;
78
- return {
79
- ...existing,
80
- ...overrides,
81
- updated_at: now,
82
- revision_count: rev,
83
- id: existing.id
84
- };
85
- }
86
- return {
87
- id: overrides.id || observationId(),
88
- title: overrides.title ?? '',
89
- type: overrides.type ?? 'generated_note',
90
- scope: overrides.scope ?? 'project',
91
- project: overrides.project ?? null,
92
- topic_key: overrides.topic_key ?? null,
93
- session_id: overrides.session_id ?? null,
94
- tool_name: overrides.tool_name ?? null,
95
- created_at: overrides.created_at ?? now,
96
- updated_at: now,
97
- revision_count: 1,
98
- duplicate_count: 0,
99
- deleted: false,
100
- tags: Array.isArray(overrides.tags) ? overrides.tags : [],
101
- embedding_status: 'disabled'
42
+ const metadata = existing ? { ...existing.metadata } : {
43
+ id: randomUUID(), title: '', type: 'note', key: null, tags: [], created_at: now, deleted_at: null
102
44
  };
103
- }
104
-
105
- /**
106
- * Save or update a memory document (mem_save semantics).
107
- * Resolution: 1) by id, 2) by topic_key in scope, 3) create new.
108
- * @returns {{ id: string, path: string, created: boolean }}
109
- */
110
- function saveDocument(config, payload) {
111
- const { title, type, scope, project, topic_key, content, session_id, id: givenId, tags } = payload;
112
- const resolved = givenId
113
- ? resolveDocument(config, { id: givenId })
114
- : topic_key != null && scope != null
115
- ? resolveDocument(config, { topic_key, scope, project })
116
- : null;
117
-
118
- let docPath;
119
- let frontmatter;
120
- let body = typeof content === 'string' ? content : (content && content.text) || '';
121
-
122
- if (resolved) {
123
- docPath = resolved.path;
124
- frontmatter = buildFrontmatter(
125
- { title, type, scope, project, topic_key, session_id, tags },
126
- { ...resolved.frontmatter, revision_count: resolved.frontmatter.revision_count ?? 1 }
127
- );
128
- body = body || resolved.body;
129
- } else {
130
- frontmatter = buildFrontmatter({
131
- id: givenId || observationId(),
132
- title,
133
- type: type || 'generated_note',
134
- scope: scope || 'project',
135
- project: project || null,
136
- topic_key: topic_key || null,
137
- session_id: session_id || null,
138
- tool_name: 'mem_save',
139
- tags: tags || []
140
- });
141
- docPath = documentPath(config, frontmatter);
142
- ensureDir(docPath);
45
+ for (const key of ['title', 'type', 'key', 'tags']) {
46
+ if (payload[key] !== undefined) metadata[key] = payload[key];
143
47
  }
144
-
145
- const raw = toMarkdown(frontmatter, body);
146
- fs.writeFileSync(docPath, raw, 'utf8');
147
- const stats = fs.statSync(docPath);
148
- indexDocument(config, docPath, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
149
- return {
150
- id: frontmatter.id,
151
- path: docPath,
152
- created: !resolved
153
- };
154
- }
155
-
156
- /**
157
- * Update document by id (mem_update semantics).
158
- * @returns {{ id: string } | null}
159
- */
160
- function updateDocument(config, { id, title, content, tags }) {
161
- const resolved = resolveDocument(config, { id });
162
- if (!resolved) return null;
163
- const body = typeof content === 'string' ? content : (content && content.text) || resolved.body;
164
- const frontmatter = buildFrontmatter(
165
- { title: title ?? resolved.frontmatter.title, tags },
166
- resolved.frontmatter
167
- );
168
- const raw = toMarkdown(frontmatter, body);
169
- fs.writeFileSync(resolved.path, raw, 'utf8');
170
- const stats = fs.statSync(resolved.path);
171
- indexDocument(config, resolved.path, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
172
- return { id: frontmatter.id };
173
- }
174
-
175
- /**
176
- * Soft or hard delete (mem_delete semantics).
177
- * @param {{ hardDelete?: boolean }} options
178
- * @returns {boolean} true if document existed and was handled
179
- */
180
- function deleteDocument(config, documentId, options = {}) {
181
- const resolved = resolveDocument(config, { id: documentId });
182
- if (!resolved) return false;
183
- if (options.hardDelete) {
184
- try { fs.unlinkSync(resolved.path); } catch (_) {}
185
- removeDocumentFromIndex(config, documentId);
186
- } else {
187
- const now = new Date().toISOString();
188
- const frontmatter = { ...resolved.frontmatter, deleted_at: now, deleted: true };
189
- const raw = toMarkdown(frontmatter, resolved.body);
190
- fs.writeFileSync(resolved.path, raw, 'utf8');
191
- const db = getDb(config);
192
- db.prepare('UPDATE documents SET deleted_at = ? WHERE id = ?').run(now, documentId);
193
- const ftsRowids = db.prepare('SELECT rowid FROM document_fts WHERE document_id = ?').all(documentId);
194
- for (const { rowid } of ftsRowids) {
195
- db.prepare('DELETE FROM document_fts WHERE rowid = ?').run(rowid);
196
- }
48
+ metadata.tags = [...new Set(metadata.tags)];
49
+ metadata.updated_at = now;
50
+ const actualCollection = existing?.collection ?? collection;
51
+ if (metadata.key) {
52
+ const conflict = db.prepare('SELECT id FROM documents WHERE collection = ? AND key = ? AND id != ? AND deleted_at IS NULL').get(actualCollection, metadata.key, metadata.id);
53
+ if (conflict) throw new Error('Key already belongs to another document in this collection');
197
54
  }
198
- return true;
55
+ const doc = { metadata, body: payload.content ?? existing?.body ?? '',
56
+ path: existing?.path ?? documentPath(config, { collection: actualCollection, ...metadata }) };
57
+ if (!existing && fs.existsSync(doc.path)) throw new Error('Document path already exists');
58
+ return persist(config, doc, !existing);
199
59
  }
200
60
 
201
- /**
202
- * Get full document content by id (mem_get_observation).
203
- * @returns {{ id: string, title: string, type: string, scope: string, project: string | null, topic_key: string | null, content: string } | null}
204
- */
205
- function getObservation(config, documentId) {
206
- const resolved = resolveDocument(config, { id: documentId });
207
- if (!resolved) return null;
208
- const fullContent = toMarkdown(resolved.frontmatter, resolved.body);
209
- return {
210
- id: resolved.frontmatter.id,
211
- title: resolved.frontmatter.title,
212
- type: resolved.frontmatter.type,
213
- scope: resolved.frontmatter.scope,
214
- project: resolved.frontmatter.project ?? null,
215
- topic_key: resolved.frontmatter.topic_key ?? null,
216
- session_id: resolved.frontmatter.session_id ?? null,
217
- content: fullContent
218
- };
61
+ export function getDocument(config, id) {
62
+ const doc = resolveDocument(config, id);
63
+ return { ...doc.metadata, collection: doc.collection, content: doc.body };
219
64
  }
220
65
 
221
- /**
222
- * List document ids in a session ordered by updated_at (for mem_timeline).
223
- */
224
- function getObservationsBySession(config, sessionId) {
225
- const db = getDb(config);
226
- const rows = db.prepare(
227
- 'SELECT id FROM documents WHERE session_id = ? AND deleted_at IS NULL ORDER BY updated_at ASC'
228
- ).all(sessionId);
229
- return rows.map((r) => r.id);
66
+ export function deleteDocument(config, id, { hardDelete = false } = {}) {
67
+ const doc = resolveDocument(config, id, true);
68
+ if (hardDelete) {
69
+ fs.unlinkSync(doc.path);
70
+ try { removeDocumentFromIndex(config, id); }
71
+ catch (error) { return { id, deleted: true, indexed: false, error: error.message }; }
72
+ return { id, deleted: true, indexed: true };
73
+ }
74
+ const now = new Date().toISOString();
75
+ doc.metadata.deleted_at = now;
76
+ doc.metadata.updated_at = now;
77
+ return { ...persist(config, doc, false), deleted: true };
230
78
  }
231
-
232
- export {
233
- resolveDocument,
234
- buildFrontmatter,
235
- saveDocument,
236
- updateDocument,
237
- deleteDocument,
238
- getObservation,
239
- getObservationsBySession
240
- };
@@ -1,58 +1,18 @@
1
- /**
2
- * Optional embedding provider using @huggingface/transformers (Xenova/multilingual-e5-large).
3
- * Dynamically imported so the app runs without it when embeddings are disabled.
4
- */
5
- const MODEL = 'Xenova/multilingual-e5-large';
6
- const DIM = 1024;
7
- const DTYPE = 'q8';
8
-
9
- const pipelineCache = new Map();
10
-
11
- async function loadPipeline(options = {}) {
12
- const model = options.model ?? MODEL;
13
- const dtype = options.dtype ?? DTYPE;
14
- const key = `${model}:${dtype}`;
15
- if (pipelineCache.has(key)) return pipelineCache.get(key);
16
- try {
17
- const { pipeline } = await import('@huggingface/transformers');
18
- const pipe = pipeline('feature-extraction', model, {
19
- pooling: 'mean',
20
- normalize: true,
21
- dtype
22
- });
23
- pipelineCache.set(key, pipe);
24
- return pipe;
25
- } catch (err) {
26
- throw err;
27
- }
28
- }
29
-
30
- /**
31
- * Embed a single text with the given pipeline. Returns Float32Array (L2-normalized).
32
- */
33
- async function embedWith(extractor, text, dimensions) {
34
- if (!text || typeof text !== 'string') text = '';
35
- const output = await extractor(text, { pooling: 'mean', normalize: true });
36
- const arr = output.data || output.arraySync?.() || Array.from(output);
37
- return new Float32Array(arr.slice(0, dimensions));
38
- }
1
+ const pipelines = new WeakMap();
39
2
 
40
3
  export async function getEmbedder(config) {
41
- if (!config?.embeddings?.enabled) return null;
42
- try {
43
- const opts = config.embeddings;
44
- const extractor = await loadPipeline({
45
- model: opts.model ?? MODEL,
46
- dtype: opts.dtype ?? DTYPE
47
- });
48
- const dimensions = opts.dimensions ?? DIM;
49
- return {
50
- embed: (text) => embedWith(extractor, text, dimensions),
51
- dimensions
52
- };
53
- } catch {
54
- return null;
4
+ if (!config.embeddings.enabled) return null;
5
+ if (!pipelines.has(config)) {
6
+ const loading = import('@huggingface/transformers').then(({ pipeline }) => pipeline('feature-extraction', config.embeddings.model, { dtype: config.embeddings.dtype }));
7
+ pipelines.set(config, loading);
8
+ loading.catch(() => pipelines.delete(config));
55
9
  }
10
+ const extractor = await pipelines.get(config);
11
+ return {
12
+ async embed(text) {
13
+ const output = await extractor(text, { pooling: 'mean', normalize: true });
14
+ if (output.data.length !== config.embeddings.dimensions) throw new Error('Embedding model dimensions do not match configuration');
15
+ return new Float32Array(output.data);
16
+ }
17
+ };
56
18
  }
57
-
58
- export { embedWith as embed, DIM, MODEL, DTYPE };