mixgram 1.2.4 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/mixgram.js +2 -2
- package/package.json +1 -1
- package/spec/bench-indexing.js +2 -2
- package/spec/scenarios/scenarios.js +45 -7
- package/spec/sdd.md +2 -2
- package/spec/test-large-document.js +2 -2
- package/src/core/documents/documents.js +56 -23
- package/src/core/indexing/indexer.js +146 -19
- package/src/core/indexing/reindex.js +44 -3
- package/src/core/search/search.js +14 -6
- package/src/fs/watcher.js +8 -4
- package/src/utils/ids.js +12 -7
- package/src/utils/markdown.js +5 -4
package/README.md
CHANGED
|
@@ -220,9 +220,9 @@ Omit `query` or use `*` to get **recent** context only.
|
|
|
220
220
|
### Example: one round-trip
|
|
221
221
|
|
|
222
222
|
```text
|
|
223
|
-
1. mem_save → { success: true, id: "
|
|
223
|
+
1. mem_save → { success: true, id: "<10-char id>", path: "...", created: true }
|
|
224
224
|
2. mem_search({ query: "SQLite", project: "my-app" }) → { results: [ { title, snippet, ... } ] }
|
|
225
|
-
3. mem_get_observation({ id: "
|
|
225
|
+
3. mem_get_observation({ id: "<10-char id>" }) → { title, content, type, scope, project, ... }
|
|
226
226
|
```
|
|
227
227
|
|
|
228
228
|
---
|
package/bin/mixgram.js
CHANGED
|
@@ -240,8 +240,8 @@ Setup targets:
|
|
|
240
240
|
gemini-cli Gemini CLI
|
|
241
241
|
codex Codex
|
|
242
242
|
|
|
243
|
-
Example (Cursor): "mixgram": { "command": "mixgram", "args": ["mcp", "--project-memory", "
|
|
244
|
-
With embeddings: "args": ["mcp", "--project-memory", "
|
|
243
|
+
Example (Cursor): "mixgram": { "command": "mixgram", "args": ["mcp", "--project-memory", "\${workspaceFolder}/docs"] }
|
|
244
|
+
With embeddings: "args": ["mcp", "--project-memory", "\${workspaceFolder}/docs", "--embeddings"]
|
|
245
245
|
Config file: .mixgram/config.json or ~/.mixgram/config.json
|
|
246
246
|
|
|
247
247
|
npm install -g mixgram
|
package/package.json
CHANGED
package/spec/bench-indexing.js
CHANGED
|
@@ -74,9 +74,9 @@ console.log('Mode: %s\n', RUN_EMBED_PROCESSING ? 'indexing + embedding queue pro
|
|
|
74
74
|
|
|
75
75
|
// --- Create corpus ---
|
|
76
76
|
for (let i = 0; i < NUM_SMALL; i++) {
|
|
77
|
-
writeDoc(baseConfig.projectMemoryRoot, `
|
|
77
|
+
writeDoc(baseConfig.projectMemoryRoot, `bench${String(i).padStart(5, '0')}`, `Small ${i}`, `Content for small doc ${i}.\n`);
|
|
78
78
|
}
|
|
79
|
-
const largePath = writeDoc(baseConfig.projectMemoryRoot, '
|
|
79
|
+
const largePath = writeDoc(baseConfig.projectMemoryRoot, 'bench99999', 'Large doc', buildLargeBody(LARGE_SIZE));
|
|
80
80
|
|
|
81
81
|
/**
|
|
82
82
|
* Full reindex with breakdown: clearIndex, listMarkdownPaths, index loop.
|
|
@@ -362,7 +362,7 @@ export const scenarios = [
|
|
|
362
362
|
await h.mem_reindex({ full: true });
|
|
363
363
|
reporter.step('mem_reindex', { full: true }, {});
|
|
364
364
|
|
|
365
|
-
const searchA = await h.mem_search({ query: 'Manual edits'
|
|
365
|
+
const searchA = await h.mem_search({ query: 'Manual edits' });
|
|
366
366
|
const searchAParsed = parseRes(searchA);
|
|
367
367
|
reporter.step('mem_search (after edit)', { query: 'Manual edits' }, searchAParsed, { highlight: ['title', 'snippet'] });
|
|
368
368
|
ok(searchAParsed.results.length >= 1, 'edited phrase searchable');
|
|
@@ -392,13 +392,51 @@ export const scenarios = [
|
|
|
392
392
|
reporter.step('fullReindex', {}, reindexResult);
|
|
393
393
|
ok(reindexResult.indexed >= 1, 'full reindex indexed');
|
|
394
394
|
|
|
395
|
-
const searchAfter = await h.mem_search({ query: 'Manual edits'
|
|
395
|
+
const searchAfter = await h.mem_search({ query: 'Manual edits' });
|
|
396
396
|
ok(parseRes(searchAfter).results.length >= 1, 'searchable after rebuild');
|
|
397
397
|
|
|
398
398
|
reporter.endScenario(p, f);
|
|
399
399
|
return { passed: p, failed: f };
|
|
400
400
|
}
|
|
401
401
|
},
|
|
402
|
+
{
|
|
403
|
+
name: 'Move file and reindex (path is source of location)',
|
|
404
|
+
goal: 'Moving a .md file to another folder: incremental reindex desindexa old path and indexa new path; doc remains findable by id.',
|
|
405
|
+
run: async function (ctx) {
|
|
406
|
+
const { h, reporter, config, fs } = ctx;
|
|
407
|
+
const parseRes = ctx.parse || parse;
|
|
408
|
+
let p = 0, f = 0;
|
|
409
|
+
const ok = (cond, msg) => { if (cond) p++; else f++; reporter.check(msg, cond); };
|
|
410
|
+
if (!ctx.shared?.decisionDoc) throw new Error('Requires decisionDoc');
|
|
411
|
+
|
|
412
|
+
reporter.startScenario(this.name, this.goal);
|
|
413
|
+
|
|
414
|
+
const docId = ctx.shared.decisionDoc.id;
|
|
415
|
+
const oldPath = ctx.shared.decisionDoc.path;
|
|
416
|
+
if (!fs.existsSync(oldPath)) throw new Error('decisionDoc path missing');
|
|
417
|
+
const raw = fs.readFileSync(oldPath, 'utf8');
|
|
418
|
+
const rootDir = path.dirname(path.dirname(oldPath));
|
|
419
|
+
const architectureDir = path.join(rootDir, 'architecture');
|
|
420
|
+
if (!fs.existsSync(architectureDir)) fs.mkdirSync(architectureDir, { recursive: true });
|
|
421
|
+
const newPath = path.join(architectureDir, path.basename(oldPath));
|
|
422
|
+
fs.writeFileSync(newPath, raw);
|
|
423
|
+
fs.unlinkSync(oldPath);
|
|
424
|
+
reporter.step('Move file', { from: oldPath, to: newPath }, {});
|
|
425
|
+
|
|
426
|
+
const reindexRes = await h.mem_reindex({ full: false });
|
|
427
|
+
const reindexParsed = parseRes(reindexRes);
|
|
428
|
+
reporter.step('mem_reindex', { full: false }, reindexParsed);
|
|
429
|
+
ok(reindexParsed.indexed >= 1 || reindexParsed.removed >= 1, 'reindex touched index');
|
|
430
|
+
|
|
431
|
+
const obsRes = await h.mem_get_observation({ id: docId });
|
|
432
|
+
const obs = parseRes(obsRes);
|
|
433
|
+
ok(!obs.error && obs.id === docId, 'get_observation by id still works after move');
|
|
434
|
+
ok(obs.title != null || (obs.content && obs.content.length > 0), 'observation has title or content');
|
|
435
|
+
|
|
436
|
+
reporter.endScenario(p, f);
|
|
437
|
+
return { passed: p, failed: f };
|
|
438
|
+
}
|
|
439
|
+
},
|
|
402
440
|
{
|
|
403
441
|
name: 'No duplicates after incremental reindex',
|
|
404
442
|
goal: 'Running incremental reindex without file changes does not duplicate documents.',
|
|
@@ -433,7 +471,7 @@ export const scenarios = [
|
|
|
433
471
|
|
|
434
472
|
reporter.startScenario(this.name, this.goal);
|
|
435
473
|
|
|
436
|
-
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged'
|
|
474
|
+
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged' });
|
|
437
475
|
const homeResults = parseRes(homeSearch).results;
|
|
438
476
|
reporter.step('mem_search (merged)', { query: 'reindex', scope_mode: 'merged' }, { results: homeResults }, { highlight: ['scope', 'title', 'snippet'] });
|
|
439
477
|
const homeInResults = homeResults.some((r) => r.scope === 'home');
|
|
@@ -532,14 +570,14 @@ export const scenarios = [
|
|
|
532
570
|
|
|
533
571
|
reporter.startScenario(this.name, this.goal);
|
|
534
572
|
|
|
535
|
-
const updBad = await h.mem_update({ id: '
|
|
573
|
+
const updBad = await h.mem_update({ id: 'nonexist01', title: 'No', content: 'No' });
|
|
536
574
|
const updBadParsed = parseRes(updBad);
|
|
537
|
-
reporter.step('mem_update (bad id)', { id: '
|
|
575
|
+
reporter.step('mem_update (bad id)', { id: 'nonexist01' }, updBadParsed);
|
|
538
576
|
ok(updBadParsed.success === false && updBadParsed.error, 'update fails cleanly');
|
|
539
577
|
|
|
540
|
-
const getBad = await h.mem_get_observation({ id: '
|
|
578
|
+
const getBad = await h.mem_get_observation({ id: 'nonexist02' });
|
|
541
579
|
const getBadParsed = parseRes(getBad);
|
|
542
|
-
reporter.step('mem_get_observation (bad id)', { id: '
|
|
580
|
+
reporter.step('mem_get_observation (bad id)', { id: 'nonexist02' }, getBadParsed);
|
|
543
581
|
ok(getBadParsed.error === 'Observation not found' || !!getBadParsed.error, 'get_observation error');
|
|
544
582
|
|
|
545
583
|
reporter.endScenario(p, f);
|
package/spec/sdd.md
CHANGED
|
@@ -338,13 +338,13 @@ Each memory document should include structured frontmatter.
|
|
|
338
338
|
|
|
339
339
|
```md
|
|
340
340
|
---
|
|
341
|
-
id:
|
|
341
|
+
id: a1b2c3d4e5
|
|
342
342
|
title: Use SQLite as derived index
|
|
343
343
|
type: decision
|
|
344
344
|
scope: project
|
|
345
345
|
project: castlebravo
|
|
346
346
|
topic_key: architecture/sqlite-derived-index
|
|
347
|
-
session_id:
|
|
347
|
+
session_id: f6g7h8i9j0
|
|
348
348
|
tool_name: mem_save
|
|
349
349
|
created_at: 2026-03-08T18:00:00Z
|
|
350
350
|
updated_at: 2026-03-08T18:00:00Z
|
|
@@ -39,7 +39,7 @@ export function runLargeDocumentTest(config) {
|
|
|
39
39
|
fs.mkdirSync(docsDir, { recursive: true });
|
|
40
40
|
|
|
41
41
|
const body = buildLargeBody(TARGET_SIZE);
|
|
42
|
-
const id = '
|
|
42
|
+
const id = 'large100k00';
|
|
43
43
|
const now = new Date().toISOString();
|
|
44
44
|
const frontmatter = {
|
|
45
45
|
id,
|
|
@@ -58,7 +58,7 @@ export function runLargeDocumentTest(config) {
|
|
|
58
58
|
const size = raw.length;
|
|
59
59
|
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
60
|
|
|
61
|
-
indexDocument(config, docPath, raw, stat.mtimeMs);
|
|
61
|
+
indexDocument(config, docPath, raw, stat.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
62
62
|
|
|
63
63
|
const db = getDb(config);
|
|
64
64
|
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
|
@@ -6,32 +6,65 @@ import { contentHash } from '../../utils/hash.js';
|
|
|
6
6
|
import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
|
|
7
7
|
import { getDb } from '../../db/sqlite.js';
|
|
8
8
|
|
|
9
|
+
/** Load document metadata from DB (canonical source). Returns null if not found or file missing. */
|
|
10
|
+
function getDocumentMetaById(config, id) {
|
|
11
|
+
const db = getDb(config);
|
|
12
|
+
const row = db.prepare(
|
|
13
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE id = ? AND deleted_at IS NULL'
|
|
14
|
+
).get(id);
|
|
15
|
+
if (!row) return null;
|
|
16
|
+
if (!fs.existsSync(row.path)) return null;
|
|
17
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(id);
|
|
18
|
+
const tags = tagRows.map((r) => r.tag);
|
|
19
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
20
|
+
const { body } = parse(raw);
|
|
21
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Load document metadata by topic_key + scope + project. */
|
|
25
|
+
function getDocumentMetaByTopicKey(config, topic_key, scope, project) {
|
|
26
|
+
const db = getDb(config);
|
|
27
|
+
const row = db.prepare(
|
|
28
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
29
|
+
).get(topic_key, scope, project ?? null, project ?? null);
|
|
30
|
+
if (!row) return null;
|
|
31
|
+
if (!fs.existsSync(row.path)) return null;
|
|
32
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
33
|
+
const tags = tagRows.map((r) => r.tag);
|
|
34
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
35
|
+
const { body } = parse(raw);
|
|
36
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function rowToFrontmatter(row, tags = []) {
|
|
40
|
+
return {
|
|
41
|
+
id: row.id,
|
|
42
|
+
title: row.title ?? '',
|
|
43
|
+
type: row.type ?? 'generated_note',
|
|
44
|
+
scope: row.scope ?? 'project',
|
|
45
|
+
project: row.project ?? null,
|
|
46
|
+
topic_key: row.topic_key ?? null,
|
|
47
|
+
session_id: row.session_id ?? null,
|
|
48
|
+
tool_name: row.tool_name ?? null,
|
|
49
|
+
created_at: row.created_at,
|
|
50
|
+
updated_at: row.updated_at,
|
|
51
|
+
revision_count: row.revision_count ?? 1,
|
|
52
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
53
|
+
deleted: !!row.deleted_at,
|
|
54
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
55
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
56
|
+
deleted_at: row.deleted_at ?? null
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
9
60
|
/**
|
|
10
61
|
* Resolve existing document by id (takes precedence) or by topic_key + scope + project.
|
|
62
|
+
* Metadata comes from SQLite; body from disk.
|
|
11
63
|
* @returns {{ path: string, frontmatter: object, body: string } | null}
|
|
12
64
|
*/
|
|
13
65
|
function resolveDocument(config, { id, topic_key, scope, project }) {
|
|
14
|
-
|
|
15
|
-
if (
|
|
16
|
-
const row = db.prepare('SELECT path FROM documents WHERE id = ? AND deleted_at IS NULL').get(id);
|
|
17
|
-
if (!row) return null;
|
|
18
|
-
const fullPath = row.path;
|
|
19
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
20
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
21
|
-
const parsed = parse(raw);
|
|
22
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
23
|
-
}
|
|
24
|
-
if (topic_key != null && scope != null) {
|
|
25
|
-
const row = db.prepare(
|
|
26
|
-
'SELECT id, path FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
27
|
-
).get(topic_key, scope, project ?? null, project ?? null);
|
|
28
|
-
if (!row) return null;
|
|
29
|
-
const fullPath = row.path;
|
|
30
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
31
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
32
|
-
const parsed = parse(raw);
|
|
33
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
34
|
-
}
|
|
66
|
+
if (id) return getDocumentMetaById(config, id);
|
|
67
|
+
if (topic_key != null && scope != null) return getDocumentMetaByTopicKey(config, topic_key, scope, project);
|
|
35
68
|
return null;
|
|
36
69
|
}
|
|
37
70
|
|
|
@@ -112,7 +145,7 @@ function saveDocument(config, payload) {
|
|
|
112
145
|
const raw = toMarkdown(frontmatter, body);
|
|
113
146
|
fs.writeFileSync(docPath, raw, 'utf8');
|
|
114
147
|
const stats = fs.statSync(docPath);
|
|
115
|
-
indexDocument(config, docPath, raw, stats.mtimeMs);
|
|
148
|
+
indexDocument(config, docPath, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
116
149
|
return {
|
|
117
150
|
id: frontmatter.id,
|
|
118
151
|
path: docPath,
|
|
@@ -135,7 +168,7 @@ function updateDocument(config, { id, title, content, tags }) {
|
|
|
135
168
|
const raw = toMarkdown(frontmatter, body);
|
|
136
169
|
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
137
170
|
const stats = fs.statSync(resolved.path);
|
|
138
|
-
indexDocument(config, resolved.path, raw, stats.mtimeMs);
|
|
171
|
+
indexDocument(config, resolved.path, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
139
172
|
return { id: frontmatter.id };
|
|
140
173
|
}
|
|
141
174
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs';
|
|
1
3
|
import { getDb } from '../../db/sqlite.js';
|
|
2
4
|
import { parseMarkdown } from './parser.js';
|
|
3
5
|
import { contentHash } from '../../utils/hash.js';
|
|
@@ -11,43 +13,168 @@ function normalizeForFts(str) {
|
|
|
11
13
|
.trim();
|
|
12
14
|
}
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
/** Infer scope, type and project from file path. Path shape: <project>/docs/<type>/<title>.md → project = dirname(docs). */
|
|
17
|
+
function inferMetadataFromPath(config, docPath) {
|
|
18
|
+
let norm = path.normalize(path.resolve(docPath));
|
|
19
|
+
let homeRoot = config.homeMemoryRoot ? path.normalize(path.resolve(config.homeMemoryRoot)) : '';
|
|
20
|
+
let projectRoot = config.projectMemoryRoot ? path.normalize(path.resolve(config.projectMemoryRoot)) : '';
|
|
21
|
+
try {
|
|
22
|
+
if (norm && fs.existsSync(norm)) norm = fs.realpathSync(norm);
|
|
23
|
+
if (homeRoot && fs.existsSync(homeRoot)) homeRoot = fs.realpathSync(homeRoot);
|
|
24
|
+
if (projectRoot && fs.existsSync(projectRoot)) projectRoot = fs.realpathSync(projectRoot);
|
|
25
|
+
} catch (_) {}
|
|
26
|
+
|
|
27
|
+
const dirToType = {
|
|
28
|
+
architecture: 'architecture',
|
|
29
|
+
decisions: 'decision',
|
|
30
|
+
bugs: 'bug',
|
|
31
|
+
learnings: 'learning',
|
|
32
|
+
discoveries: 'discovery',
|
|
33
|
+
patterns: 'pattern',
|
|
34
|
+
reference: 'reference',
|
|
35
|
+
sessions: 'session_summary',
|
|
36
|
+
prompts: 'prompt',
|
|
37
|
+
generated: 'generated_note'
|
|
38
|
+
};
|
|
39
|
+
let scope = 'project';
|
|
40
|
+
let type = 'generated_note';
|
|
41
|
+
let project = null;
|
|
42
|
+
|
|
43
|
+
const underHome = homeRoot && (() => {
|
|
44
|
+
if (norm === homeRoot) return true;
|
|
45
|
+
const sep = path.sep;
|
|
46
|
+
return norm.startsWith(homeRoot + sep);
|
|
47
|
+
})();
|
|
48
|
+
const underProject = projectRoot && (() => {
|
|
49
|
+
if (norm === projectRoot) return true;
|
|
50
|
+
const sep = path.sep;
|
|
51
|
+
return norm.startsWith(projectRoot + sep);
|
|
52
|
+
})();
|
|
53
|
+
|
|
54
|
+
if (underHome) {
|
|
55
|
+
scope = 'home';
|
|
56
|
+
const rel = path.relative(homeRoot, path.dirname(norm));
|
|
57
|
+
const firstDir = rel.split(path.sep)[0];
|
|
58
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
59
|
+
} else if (underProject) {
|
|
60
|
+
scope = 'project';
|
|
61
|
+
const rel = path.relative(projectRoot, path.dirname(norm));
|
|
62
|
+
const firstDir = rel.split(path.sep)[0];
|
|
63
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
64
|
+
const projectDir = path.dirname(projectRoot);
|
|
65
|
+
if (projectDir && projectDir !== path.dirname(norm)) project = path.basename(projectDir);
|
|
66
|
+
}
|
|
67
|
+
return { scope, type, project };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Load metadata from DB by path when no override provided. */
|
|
71
|
+
function loadMetadataByPath(db, docPath) {
|
|
72
|
+
const norm = path.normalize(docPath);
|
|
73
|
+
const row = db.prepare(
|
|
74
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
75
|
+
).get(norm);
|
|
76
|
+
if (!row) return null;
|
|
77
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
78
|
+
const tags = tagRows.map((r) => r.tag);
|
|
79
|
+
return {
|
|
80
|
+
id: row.id,
|
|
81
|
+
title: row.title ?? '',
|
|
82
|
+
type: row.type ?? 'generated_note',
|
|
83
|
+
scope: row.scope ?? 'project',
|
|
84
|
+
project: row.project ?? null,
|
|
85
|
+
topic_key: row.topic_key ?? null,
|
|
86
|
+
session_id: row.session_id ?? null,
|
|
87
|
+
tool_name: row.tool_name ?? null,
|
|
88
|
+
created_at: row.created_at,
|
|
89
|
+
updated_at: row.updated_at,
|
|
90
|
+
revision_count: row.revision_count ?? 1,
|
|
91
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
92
|
+
deleted: !!row.deleted_at,
|
|
93
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
94
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
95
|
+
deleted_at: row.deleted_at ?? null
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function indexDocument(config, docPath, rawContent, fileMtimeMs = null, options = {}) {
|
|
15
100
|
const db = getDb(config);
|
|
16
101
|
const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
|
|
17
102
|
const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
|
|
18
103
|
|
|
19
104
|
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
20
|
-
|
|
21
|
-
|
|
105
|
+
/** Prefer caller override, then DB row by path, then parsed front matter. */
|
|
106
|
+
let fm = options.overrideFrontmatter;
|
|
107
|
+
let byPath = null;
|
|
108
|
+
if (!fm) {
|
|
109
|
+
byPath = loadMetadataByPath(db, docPath);
|
|
110
|
+
fm = byPath ? { ...frontmatter, ...byPath } : frontmatter;
|
|
111
|
+
} else {
|
|
112
|
+
fm = { ...frontmatter, ...fm };
|
|
113
|
+
}
|
|
22
114
|
|
|
115
|
+
/** When metadata did not come from DB (full reindex or new file), infer scope/type/project from path. */
|
|
116
|
+
if (!options.overrideFrontmatter && !byPath) {
|
|
117
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
118
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null);
|
|
119
|
+
fm = {
|
|
120
|
+
...fm,
|
|
121
|
+
scope: inferred.scope,
|
|
122
|
+
type: fm.type ?? inferred.type,
|
|
123
|
+
project: inferred.project,
|
|
124
|
+
created_at: fm.created_at || fileCreatedAt || undefined,
|
|
125
|
+
updated_at: fm.updated_at || fileCreatedAt || undefined
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** If still no id (e.g. full reindex of file with no front matter), generate from path and set created/updated. */
|
|
130
|
+
if (!fm.id) {
|
|
131
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
132
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null) ?? new Date().toISOString();
|
|
133
|
+
fm = {
|
|
134
|
+
...fm,
|
|
135
|
+
id: contentHash(docPath).slice(0, 10),
|
|
136
|
+
scope: inferred.scope,
|
|
137
|
+
type: fm.type ?? inferred.type,
|
|
138
|
+
project: inferred.project,
|
|
139
|
+
created_at: fm.created_at || fileCreatedAt,
|
|
140
|
+
updated_at: fm.updated_at || fileCreatedAt
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const id = fm.id;
|
|
23
145
|
const now = new Date().toISOString();
|
|
24
146
|
const docHash = contentHash(rawContent);
|
|
25
|
-
const docTitle =
|
|
147
|
+
const docTitle = fm.title || title || '';
|
|
26
148
|
|
|
27
|
-
const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
|
|
149
|
+
const existing = db.prepare('SELECT content_hash, file_mtime_ms, path FROM documents WHERE id = ?').get(id);
|
|
28
150
|
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
151
|
+
/** If path changed (move), update row so path stays in sync. */
|
|
152
|
+
const normPath = path.normalize(docPath);
|
|
153
|
+
if (existing.path && path.normalize(existing.path) !== normPath) {
|
|
154
|
+
db.prepare('UPDATE documents SET path = ?, file_mtime_ms = ? WHERE id = ?').run(path.normalize(docPath), fileMtimeMs, id);
|
|
155
|
+
}
|
|
29
156
|
return { id };
|
|
30
157
|
}
|
|
31
158
|
|
|
32
159
|
const docRow = {
|
|
33
160
|
id,
|
|
34
|
-
path: docPath,
|
|
161
|
+
path: path.normalize(docPath),
|
|
35
162
|
title: docTitle,
|
|
36
|
-
type:
|
|
37
|
-
scope:
|
|
38
|
-
project:
|
|
39
|
-
topic_key:
|
|
40
|
-
session_id:
|
|
41
|
-
tool_name:
|
|
163
|
+
type: fm.type || 'generated_note',
|
|
164
|
+
scope: fm.scope || 'project',
|
|
165
|
+
project: fm.project || null,
|
|
166
|
+
topic_key: fm.topic_key || null,
|
|
167
|
+
session_id: fm.session_id || null,
|
|
168
|
+
tool_name: fm.tool_name || null,
|
|
42
169
|
content_hash: docHash,
|
|
43
170
|
file_mtime_ms: fileMtimeMs,
|
|
44
|
-
created_at:
|
|
45
|
-
updated_at:
|
|
171
|
+
created_at: fm.created_at || now,
|
|
172
|
+
updated_at: fm.updated_at || now,
|
|
46
173
|
indexed_at: now,
|
|
47
|
-
deleted_at:
|
|
48
|
-
revision_count:
|
|
49
|
-
duplicate_count:
|
|
50
|
-
embedding_status:
|
|
174
|
+
deleted_at: fm.deleted_at || null,
|
|
175
|
+
revision_count: fm.revision_count ?? 1,
|
|
176
|
+
duplicate_count: fm.duplicate_count ?? 0,
|
|
177
|
+
embedding_status: fm.embedding_status || 'disabled',
|
|
51
178
|
body: body || '',
|
|
52
179
|
structure: JSON.stringify(structure),
|
|
53
180
|
sections_index: JSON.stringify(sectionsIndex)
|
|
@@ -84,7 +211,7 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
84
211
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
212
|
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
86
213
|
|
|
87
|
-
const tags =
|
|
214
|
+
const tags = fm.tags || [];
|
|
88
215
|
for (const tag of tags) {
|
|
89
216
|
db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
|
|
90
217
|
}
|
|
@@ -3,6 +3,44 @@ import fs from 'fs';
|
|
|
3
3
|
import { globSync } from 'glob';
|
|
4
4
|
import { getDb } from '../../db/sqlite.js';
|
|
5
5
|
import { indexDocument, removeDocumentFromIndex } from './indexer.js';
|
|
6
|
+
import { TYPE_TO_DIR } from '../../fs/paths.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Load document metadata from DB by path (for reindex/watcher).
|
|
10
|
+
* Returns frontmatter-like object or null.
|
|
11
|
+
*/
|
|
12
|
+
function getDocumentMetaByPath(config, filePath) {
|
|
13
|
+
const db = getDb(config);
|
|
14
|
+
const norm = path.normalize(filePath);
|
|
15
|
+
const row = db.prepare(
|
|
16
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
17
|
+
).get(norm);
|
|
18
|
+
if (!row) return null;
|
|
19
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
20
|
+
const tags = tagRows.map((r) => r.tag);
|
|
21
|
+
return rowToFrontmatter(row, tags);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function rowToFrontmatter(row, tags = []) {
|
|
25
|
+
return {
|
|
26
|
+
id: row.id,
|
|
27
|
+
title: row.title ?? '',
|
|
28
|
+
type: row.type ?? 'generated_note',
|
|
29
|
+
scope: row.scope ?? 'project',
|
|
30
|
+
project: row.project ?? null,
|
|
31
|
+
topic_key: row.topic_key ?? null,
|
|
32
|
+
session_id: row.session_id ?? null,
|
|
33
|
+
tool_name: row.tool_name ?? null,
|
|
34
|
+
created_at: row.created_at,
|
|
35
|
+
updated_at: row.updated_at,
|
|
36
|
+
revision_count: row.revision_count ?? 1,
|
|
37
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
38
|
+
deleted: !!row.deleted_at,
|
|
39
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
40
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
41
|
+
deleted_at: row.deleted_at ?? null
|
|
42
|
+
};
|
|
43
|
+
}
|
|
6
44
|
|
|
7
45
|
/**
|
|
8
46
|
* List all Markdown file paths under home (cross-project) and project memory root (repo-local).
|
|
@@ -49,7 +87,8 @@ function fullReindex(config) {
|
|
|
49
87
|
try {
|
|
50
88
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
51
89
|
const stat = fs.statSync(filePath);
|
|
52
|
-
|
|
90
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
91
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { fileCreatedAt });
|
|
53
92
|
indexed++;
|
|
54
93
|
} catch (err) {
|
|
55
94
|
// skip invalid or unreadable files
|
|
@@ -81,7 +120,9 @@ function incrementalReindex(config) {
|
|
|
81
120
|
continue;
|
|
82
121
|
}
|
|
83
122
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
84
|
-
|
|
123
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
124
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
125
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
85
126
|
indexed++;
|
|
86
127
|
} catch (_) {}
|
|
87
128
|
}
|
|
@@ -107,4 +148,4 @@ function getDocumentIdByPath(config, filePath) {
|
|
|
107
148
|
return row?.id ?? null;
|
|
108
149
|
}
|
|
109
150
|
|
|
110
|
-
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath };
|
|
151
|
+
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath, getDocumentMetaByPath };
|
|
@@ -40,7 +40,7 @@ function normalizeQuery(q) {
|
|
|
40
40
|
* @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
|
|
41
41
|
* @param {string} [options.project] - project name (required for project-only and merged)
|
|
42
42
|
* @param {number} [options.limit] - max results
|
|
43
|
-
* @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, snippet: string, score: number }>}
|
|
43
|
+
* @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, created: string | null, snippet: string, score: number }>}
|
|
44
44
|
*/
|
|
45
45
|
function search(config, options = {}) {
|
|
46
46
|
const {
|
|
@@ -70,8 +70,13 @@ function search(config, options = {}) {
|
|
|
70
70
|
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
71
71
|
params.push('home');
|
|
72
72
|
} else {
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
if (project) {
|
|
74
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
75
|
+
params.push('home', 'project', project);
|
|
76
|
+
} else {
|
|
77
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR d.scope = ?)';
|
|
78
|
+
params.push('home', 'project');
|
|
79
|
+
}
|
|
75
80
|
}
|
|
76
81
|
|
|
77
82
|
params.push(project || '');
|
|
@@ -85,9 +90,10 @@ function search(config, options = {}) {
|
|
|
85
90
|
d.type AS type,
|
|
86
91
|
d.scope AS scope,
|
|
87
92
|
d.project AS project,
|
|
93
|
+
d.created_at AS created,
|
|
88
94
|
d.body AS body,
|
|
89
|
-
snippet(document_fts, 1, '
|
|
90
|
-
snippet(document_fts, 8, '
|
|
95
|
+
snippet(document_fts, 1, '**', '**', '...', 24) AS snippetTitle,
|
|
96
|
+
snippet(document_fts, 8, '**', '**', '...', 64) AS snippetBody,
|
|
91
97
|
bm25(document_fts, ${bm25Args}) AS rank
|
|
92
98
|
FROM document_fts f
|
|
93
99
|
JOIN documents d ON d.id = f.document_id
|
|
@@ -110,6 +116,7 @@ function search(config, options = {}) {
|
|
|
110
116
|
type: r.type,
|
|
111
117
|
scope: r.scope,
|
|
112
118
|
project: r.project,
|
|
119
|
+
created: r.created || null,
|
|
113
120
|
snippet: String(snippet).trim(),
|
|
114
121
|
score: -Number(r.rank)
|
|
115
122
|
};
|
|
@@ -123,7 +130,7 @@ function getRecentContext(config, options = {}) {
|
|
|
123
130
|
const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
|
|
124
131
|
const db = getDb(config);
|
|
125
132
|
let sql = `
|
|
126
|
-
SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.body
|
|
133
|
+
SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.created_at AS created, d.body
|
|
127
134
|
FROM documents d
|
|
128
135
|
WHERE d.deleted_at IS NULL
|
|
129
136
|
`;
|
|
@@ -145,6 +152,7 @@ function getRecentContext(config, options = {}) {
|
|
|
145
152
|
type: r.type,
|
|
146
153
|
scope: r.scope,
|
|
147
154
|
project: r.project,
|
|
155
|
+
created: r.created || null,
|
|
148
156
|
snippet: (r.body || '').slice(0, 300).trim()
|
|
149
157
|
}));
|
|
150
158
|
}
|
package/src/fs/watcher.js
CHANGED
|
@@ -2,11 +2,11 @@ import path from 'path';
|
|
|
2
2
|
import chokidar from 'chokidar';
|
|
3
3
|
import fs from 'fs';
|
|
4
4
|
import { indexDocument, removeDocumentFromIndex } from '../core/indexing/indexer.js';
|
|
5
|
-
import { getDocumentIdByPath } from '../core/indexing/reindex.js';
|
|
5
|
+
import { getDocumentIdByPath, getDocumentMetaByPath } from '../core/indexing/reindex.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Start watching home (cross-project) and project memory root (repo-local) for .md changes.
|
|
9
|
-
* On add/change: index the file. On unlink: remove from index by path.
|
|
9
|
+
* On add/change: index the file (using DB metadata when path already known). On unlink: remove from index by path.
|
|
10
10
|
* @param {object} config - resolved config
|
|
11
11
|
* @returns {import('chokidar').FSWatcher} watcher instance (call .close() to stop)
|
|
12
12
|
*/
|
|
@@ -28,14 +28,18 @@ function startWatcher(config) {
|
|
|
28
28
|
try {
|
|
29
29
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
30
30
|
const stat = fs.statSync(filePath);
|
|
31
|
-
|
|
31
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
32
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
33
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
32
34
|
} catch (_) {}
|
|
33
35
|
})
|
|
34
36
|
.on('change', (filePath) => {
|
|
35
37
|
try {
|
|
36
38
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
37
39
|
const stat = fs.statSync(filePath);
|
|
38
|
-
|
|
40
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
41
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
42
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
39
43
|
} catch (_) {}
|
|
40
44
|
})
|
|
41
45
|
.on('unlink', (filePath) => {
|
package/src/utils/ids.js
CHANGED
|
@@ -1,22 +1,27 @@
|
|
|
1
1
|
import crypto from 'crypto';
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
const ALPHANUM = '0123456789abcdefghijklmnopqrstuvwxyz';
|
|
4
|
+
|
|
5
|
+
export function generateId() {
|
|
6
|
+
const bytes = crypto.randomBytes(10);
|
|
7
|
+
let s = '';
|
|
8
|
+
for (let i = 0; i < 10; i++) s += ALPHANUM[bytes[i] % 36];
|
|
9
|
+
return s;
|
|
6
10
|
}
|
|
7
11
|
|
|
8
12
|
export function observationId() {
|
|
9
|
-
return generateId(
|
|
13
|
+
return generateId();
|
|
10
14
|
}
|
|
11
15
|
|
|
12
16
|
export function sessionId() {
|
|
13
|
-
return generateId(
|
|
17
|
+
return generateId();
|
|
14
18
|
}
|
|
15
19
|
|
|
16
20
|
export function chunkId(documentId, index) {
|
|
17
|
-
return `${documentId}
|
|
21
|
+
return `${documentId}_${index}`;
|
|
18
22
|
}
|
|
19
23
|
|
|
20
24
|
export function promptId() {
|
|
21
|
-
return generateId(
|
|
25
|
+
return generateId();
|
|
22
26
|
}
|
|
27
|
+
|
package/src/utils/markdown.js
CHANGED
|
@@ -6,8 +6,8 @@ export const FRONTMATTER_KEYS = [
|
|
|
6
6
|
'embedding_status', 'indexed_at', 'deleted_at'
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
-
/** Keys written to .md
|
|
10
|
-
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', '
|
|
9
|
+
/** Keys written to .md: id, type, title, created (cosmetic). scope/project stay in DB only; reindex infers scope from path. */
|
|
10
|
+
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'created'];
|
|
11
11
|
|
|
12
12
|
function formatCreated(createdAt) {
|
|
13
13
|
if (!createdAt) return null;
|
|
@@ -39,12 +39,13 @@ function serializeFrontmatter(fm) {
|
|
|
39
39
|
return obj;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
/** Minimal frontmatter for .md (
|
|
42
|
+
/** Minimal frontmatter for .md (cosmetic only; deleted flag for soft-delete visibility). */
|
|
43
43
|
function serializeDisplayFrontmatter(fm) {
|
|
44
44
|
const obj = {};
|
|
45
45
|
for (const k of DISPLAY_FRONTMATTER_KEYS) {
|
|
46
46
|
if (k === 'created') {
|
|
47
|
-
const
|
|
47
|
+
const raw = fm.created_at ?? (fm.created ? parseCreated(fm.created) : null);
|
|
48
|
+
const v = raw ? formatCreated(raw) : null;
|
|
48
49
|
if (v) obj[k] = v;
|
|
49
50
|
} else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
|
|
50
51
|
obj[k] = fm[k];
|