mixgram 1.2.6 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/spec/scenarios/scenarios.js +41 -3
- package/spec/test-large-document.js +1 -1
- package/src/core/documents/documents.js +56 -23
- package/src/core/indexing/indexer.js +146 -19
- package/src/core/indexing/reindex.js +44 -3
- package/src/core/search/search.js +7 -2
- package/src/fs/watcher.js +8 -4
- package/src/utils/markdown.js +5 -4
package/package.json
CHANGED
|
@@ -362,7 +362,7 @@ export const scenarios = [
|
|
|
362
362
|
await h.mem_reindex({ full: true });
|
|
363
363
|
reporter.step('mem_reindex', { full: true }, {});
|
|
364
364
|
|
|
365
|
-
const searchA = await h.mem_search({ query: 'Manual edits'
|
|
365
|
+
const searchA = await h.mem_search({ query: 'Manual edits' });
|
|
366
366
|
const searchAParsed = parseRes(searchA);
|
|
367
367
|
reporter.step('mem_search (after edit)', { query: 'Manual edits' }, searchAParsed, { highlight: ['title', 'snippet'] });
|
|
368
368
|
ok(searchAParsed.results.length >= 1, 'edited phrase searchable');
|
|
@@ -392,13 +392,51 @@ export const scenarios = [
|
|
|
392
392
|
reporter.step('fullReindex', {}, reindexResult);
|
|
393
393
|
ok(reindexResult.indexed >= 1, 'full reindex indexed');
|
|
394
394
|
|
|
395
|
-
const searchAfter = await h.mem_search({ query: 'Manual edits'
|
|
395
|
+
const searchAfter = await h.mem_search({ query: 'Manual edits' });
|
|
396
396
|
ok(parseRes(searchAfter).results.length >= 1, 'searchable after rebuild');
|
|
397
397
|
|
|
398
398
|
reporter.endScenario(p, f);
|
|
399
399
|
return { passed: p, failed: f };
|
|
400
400
|
}
|
|
401
401
|
},
|
|
402
|
+
{
|
|
403
|
+
name: 'Move file and reindex (path is source of location)',
|
|
404
|
+
goal: 'Moving a .md file to another folder: incremental reindex desindexa old path and indexa new path; doc remains findable by id.',
|
|
405
|
+
run: async function (ctx) {
|
|
406
|
+
const { h, reporter, config, fs } = ctx;
|
|
407
|
+
const parseRes = ctx.parse || parse;
|
|
408
|
+
let p = 0, f = 0;
|
|
409
|
+
const ok = (cond, msg) => { if (cond) p++; else f++; reporter.check(msg, cond); };
|
|
410
|
+
if (!ctx.shared?.decisionDoc) throw new Error('Requires decisionDoc');
|
|
411
|
+
|
|
412
|
+
reporter.startScenario(this.name, this.goal);
|
|
413
|
+
|
|
414
|
+
const docId = ctx.shared.decisionDoc.id;
|
|
415
|
+
const oldPath = ctx.shared.decisionDoc.path;
|
|
416
|
+
if (!fs.existsSync(oldPath)) throw new Error('decisionDoc path missing');
|
|
417
|
+
const raw = fs.readFileSync(oldPath, 'utf8');
|
|
418
|
+
const rootDir = path.dirname(path.dirname(oldPath));
|
|
419
|
+
const architectureDir = path.join(rootDir, 'architecture');
|
|
420
|
+
if (!fs.existsSync(architectureDir)) fs.mkdirSync(architectureDir, { recursive: true });
|
|
421
|
+
const newPath = path.join(architectureDir, path.basename(oldPath));
|
|
422
|
+
fs.writeFileSync(newPath, raw);
|
|
423
|
+
fs.unlinkSync(oldPath);
|
|
424
|
+
reporter.step('Move file', { from: oldPath, to: newPath }, {});
|
|
425
|
+
|
|
426
|
+
const reindexRes = await h.mem_reindex({ full: false });
|
|
427
|
+
const reindexParsed = parseRes(reindexRes);
|
|
428
|
+
reporter.step('mem_reindex', { full: false }, reindexParsed);
|
|
429
|
+
ok(reindexParsed.indexed >= 1 || reindexParsed.removed >= 1, 'reindex touched index');
|
|
430
|
+
|
|
431
|
+
const obsRes = await h.mem_get_observation({ id: docId });
|
|
432
|
+
const obs = parseRes(obsRes);
|
|
433
|
+
ok(!obs.error && obs.id === docId, 'get_observation by id still works after move');
|
|
434
|
+
ok(obs.title != null || (obs.content && obs.content.length > 0), 'observation has title or content');
|
|
435
|
+
|
|
436
|
+
reporter.endScenario(p, f);
|
|
437
|
+
return { passed: p, failed: f };
|
|
438
|
+
}
|
|
439
|
+
},
|
|
402
440
|
{
|
|
403
441
|
name: 'No duplicates after incremental reindex',
|
|
404
442
|
goal: 'Running incremental reindex without file changes does not duplicate documents.',
|
|
@@ -433,7 +471,7 @@ export const scenarios = [
|
|
|
433
471
|
|
|
434
472
|
reporter.startScenario(this.name, this.goal);
|
|
435
473
|
|
|
436
|
-
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged'
|
|
474
|
+
const homeSearch = await h.mem_search({ query: 'reindex', scope_mode: 'merged' });
|
|
437
475
|
const homeResults = parseRes(homeSearch).results;
|
|
438
476
|
reporter.step('mem_search (merged)', { query: 'reindex', scope_mode: 'merged' }, { results: homeResults }, { highlight: ['scope', 'title', 'snippet'] });
|
|
439
477
|
const homeInResults = homeResults.some((r) => r.scope === 'home');
|
|
@@ -58,7 +58,7 @@ export function runLargeDocumentTest(config) {
|
|
|
58
58
|
const size = raw.length;
|
|
59
59
|
ok(size >= TARGET_SIZE * 0.95, `documento ~100k (${size} chars)`);
|
|
60
60
|
|
|
61
|
-
indexDocument(config, docPath, raw, stat.mtimeMs);
|
|
61
|
+
indexDocument(config, docPath, raw, stat.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
62
62
|
|
|
63
63
|
const db = getDb(config);
|
|
64
64
|
const doc = db.prepare('SELECT id, length(body) AS body_len FROM documents WHERE id = ?').get(id);
|
|
@@ -6,32 +6,65 @@ import { contentHash } from '../../utils/hash.js';
|
|
|
6
6
|
import { indexDocument, removeDocumentFromIndex } from '../indexing/indexer.js';
|
|
7
7
|
import { getDb } from '../../db/sqlite.js';
|
|
8
8
|
|
|
9
|
+
/** Load document metadata from DB (canonical source). Returns null if not found or file missing. */
|
|
10
|
+
function getDocumentMetaById(config, id) {
|
|
11
|
+
const db = getDb(config);
|
|
12
|
+
const row = db.prepare(
|
|
13
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE id = ? AND deleted_at IS NULL'
|
|
14
|
+
).get(id);
|
|
15
|
+
if (!row) return null;
|
|
16
|
+
if (!fs.existsSync(row.path)) return null;
|
|
17
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(id);
|
|
18
|
+
const tags = tagRows.map((r) => r.tag);
|
|
19
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
20
|
+
const { body } = parse(raw);
|
|
21
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Load document metadata by topic_key + scope + project. */
|
|
25
|
+
function getDocumentMetaByTopicKey(config, topic_key, scope, project) {
|
|
26
|
+
const db = getDb(config);
|
|
27
|
+
const row = db.prepare(
|
|
28
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
29
|
+
).get(topic_key, scope, project ?? null, project ?? null);
|
|
30
|
+
if (!row) return null;
|
|
31
|
+
if (!fs.existsSync(row.path)) return null;
|
|
32
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
33
|
+
const tags = tagRows.map((r) => r.tag);
|
|
34
|
+
const raw = fs.readFileSync(row.path, 'utf8');
|
|
35
|
+
const { body } = parse(raw);
|
|
36
|
+
return { path: row.path, frontmatter: rowToFrontmatter(row, tags), body };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function rowToFrontmatter(row, tags = []) {
|
|
40
|
+
return {
|
|
41
|
+
id: row.id,
|
|
42
|
+
title: row.title ?? '',
|
|
43
|
+
type: row.type ?? 'generated_note',
|
|
44
|
+
scope: row.scope ?? 'project',
|
|
45
|
+
project: row.project ?? null,
|
|
46
|
+
topic_key: row.topic_key ?? null,
|
|
47
|
+
session_id: row.session_id ?? null,
|
|
48
|
+
tool_name: row.tool_name ?? null,
|
|
49
|
+
created_at: row.created_at,
|
|
50
|
+
updated_at: row.updated_at,
|
|
51
|
+
revision_count: row.revision_count ?? 1,
|
|
52
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
53
|
+
deleted: !!row.deleted_at,
|
|
54
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
55
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
56
|
+
deleted_at: row.deleted_at ?? null
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
9
60
|
/**
|
|
10
61
|
* Resolve existing document by id (takes precedence) or by topic_key + scope + project.
|
|
62
|
+
* Metadata comes from SQLite; body from disk.
|
|
11
63
|
* @returns {{ path: string, frontmatter: object, body: string } | null}
|
|
12
64
|
*/
|
|
13
65
|
function resolveDocument(config, { id, topic_key, scope, project }) {
|
|
14
|
-
|
|
15
|
-
if (
|
|
16
|
-
const row = db.prepare('SELECT path FROM documents WHERE id = ? AND deleted_at IS NULL').get(id);
|
|
17
|
-
if (!row) return null;
|
|
18
|
-
const fullPath = row.path;
|
|
19
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
20
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
21
|
-
const parsed = parse(raw);
|
|
22
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
23
|
-
}
|
|
24
|
-
if (topic_key != null && scope != null) {
|
|
25
|
-
const row = db.prepare(
|
|
26
|
-
'SELECT id, path FROM documents WHERE topic_key = ? AND scope = ? AND (project = ? OR (project IS NULL AND ? IS NULL)) AND deleted_at IS NULL'
|
|
27
|
-
).get(topic_key, scope, project ?? null, project ?? null);
|
|
28
|
-
if (!row) return null;
|
|
29
|
-
const fullPath = row.path;
|
|
30
|
-
if (!fs.existsSync(fullPath)) return null;
|
|
31
|
-
const raw = fs.readFileSync(fullPath, 'utf8');
|
|
32
|
-
const parsed = parse(raw);
|
|
33
|
-
return { path: fullPath, frontmatter: parsed.frontmatter, body: parsed.body };
|
|
34
|
-
}
|
|
66
|
+
if (id) return getDocumentMetaById(config, id);
|
|
67
|
+
if (topic_key != null && scope != null) return getDocumentMetaByTopicKey(config, topic_key, scope, project);
|
|
35
68
|
return null;
|
|
36
69
|
}
|
|
37
70
|
|
|
@@ -112,7 +145,7 @@ function saveDocument(config, payload) {
|
|
|
112
145
|
const raw = toMarkdown(frontmatter, body);
|
|
113
146
|
fs.writeFileSync(docPath, raw, 'utf8');
|
|
114
147
|
const stats = fs.statSync(docPath);
|
|
115
|
-
indexDocument(config, docPath, raw, stats.mtimeMs);
|
|
148
|
+
indexDocument(config, docPath, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
116
149
|
return {
|
|
117
150
|
id: frontmatter.id,
|
|
118
151
|
path: docPath,
|
|
@@ -135,7 +168,7 @@ function updateDocument(config, { id, title, content, tags }) {
|
|
|
135
168
|
const raw = toMarkdown(frontmatter, body);
|
|
136
169
|
fs.writeFileSync(resolved.path, raw, 'utf8');
|
|
137
170
|
const stats = fs.statSync(resolved.path);
|
|
138
|
-
indexDocument(config, resolved.path, raw, stats.mtimeMs);
|
|
171
|
+
indexDocument(config, resolved.path, raw, stats.mtimeMs, { overrideFrontmatter: frontmatter });
|
|
139
172
|
return { id: frontmatter.id };
|
|
140
173
|
}
|
|
141
174
|
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs';
|
|
1
3
|
import { getDb } from '../../db/sqlite.js';
|
|
2
4
|
import { parseMarkdown } from './parser.js';
|
|
3
5
|
import { contentHash } from '../../utils/hash.js';
|
|
@@ -11,43 +13,168 @@ function normalizeForFts(str) {
|
|
|
11
13
|
.trim();
|
|
12
14
|
}
|
|
13
15
|
|
|
14
|
-
|
|
16
|
+
/** Infer scope, type and project from file path. Path shape: <project>/docs/<type>/<title>.md → project = dirname(docs). */
|
|
17
|
+
function inferMetadataFromPath(config, docPath) {
|
|
18
|
+
let norm = path.normalize(path.resolve(docPath));
|
|
19
|
+
let homeRoot = config.homeMemoryRoot ? path.normalize(path.resolve(config.homeMemoryRoot)) : '';
|
|
20
|
+
let projectRoot = config.projectMemoryRoot ? path.normalize(path.resolve(config.projectMemoryRoot)) : '';
|
|
21
|
+
try {
|
|
22
|
+
if (norm && fs.existsSync(norm)) norm = fs.realpathSync(norm);
|
|
23
|
+
if (homeRoot && fs.existsSync(homeRoot)) homeRoot = fs.realpathSync(homeRoot);
|
|
24
|
+
if (projectRoot && fs.existsSync(projectRoot)) projectRoot = fs.realpathSync(projectRoot);
|
|
25
|
+
} catch (_) {}
|
|
26
|
+
|
|
27
|
+
const dirToType = {
|
|
28
|
+
architecture: 'architecture',
|
|
29
|
+
decisions: 'decision',
|
|
30
|
+
bugs: 'bug',
|
|
31
|
+
learnings: 'learning',
|
|
32
|
+
discoveries: 'discovery',
|
|
33
|
+
patterns: 'pattern',
|
|
34
|
+
reference: 'reference',
|
|
35
|
+
sessions: 'session_summary',
|
|
36
|
+
prompts: 'prompt',
|
|
37
|
+
generated: 'generated_note'
|
|
38
|
+
};
|
|
39
|
+
let scope = 'project';
|
|
40
|
+
let type = 'generated_note';
|
|
41
|
+
let project = null;
|
|
42
|
+
|
|
43
|
+
const underHome = homeRoot && (() => {
|
|
44
|
+
if (norm === homeRoot) return true;
|
|
45
|
+
const sep = path.sep;
|
|
46
|
+
return norm.startsWith(homeRoot + sep);
|
|
47
|
+
})();
|
|
48
|
+
const underProject = projectRoot && (() => {
|
|
49
|
+
if (norm === projectRoot) return true;
|
|
50
|
+
const sep = path.sep;
|
|
51
|
+
return norm.startsWith(projectRoot + sep);
|
|
52
|
+
})();
|
|
53
|
+
|
|
54
|
+
if (underHome) {
|
|
55
|
+
scope = 'home';
|
|
56
|
+
const rel = path.relative(homeRoot, path.dirname(norm));
|
|
57
|
+
const firstDir = rel.split(path.sep)[0];
|
|
58
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
59
|
+
} else if (underProject) {
|
|
60
|
+
scope = 'project';
|
|
61
|
+
const rel = path.relative(projectRoot, path.dirname(norm));
|
|
62
|
+
const firstDir = rel.split(path.sep)[0];
|
|
63
|
+
type = dirToType[firstDir] ?? 'generated_note';
|
|
64
|
+
const projectDir = path.dirname(projectRoot);
|
|
65
|
+
if (projectDir && projectDir !== path.dirname(norm)) project = path.basename(projectDir);
|
|
66
|
+
}
|
|
67
|
+
return { scope, type, project };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Load metadata from DB by path when no override provided. */
|
|
71
|
+
function loadMetadataByPath(db, docPath) {
|
|
72
|
+
const norm = path.normalize(docPath);
|
|
73
|
+
const row = db.prepare(
|
|
74
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
75
|
+
).get(norm);
|
|
76
|
+
if (!row) return null;
|
|
77
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
78
|
+
const tags = tagRows.map((r) => r.tag);
|
|
79
|
+
return {
|
|
80
|
+
id: row.id,
|
|
81
|
+
title: row.title ?? '',
|
|
82
|
+
type: row.type ?? 'generated_note',
|
|
83
|
+
scope: row.scope ?? 'project',
|
|
84
|
+
project: row.project ?? null,
|
|
85
|
+
topic_key: row.topic_key ?? null,
|
|
86
|
+
session_id: row.session_id ?? null,
|
|
87
|
+
tool_name: row.tool_name ?? null,
|
|
88
|
+
created_at: row.created_at,
|
|
89
|
+
updated_at: row.updated_at,
|
|
90
|
+
revision_count: row.revision_count ?? 1,
|
|
91
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
92
|
+
deleted: !!row.deleted_at,
|
|
93
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
94
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
95
|
+
deleted_at: row.deleted_at ?? null
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function indexDocument(config, docPath, rawContent, fileMtimeMs = null, options = {}) {
|
|
15
100
|
const db = getDb(config);
|
|
16
101
|
const includeCodeBlocks = config.indexing?.includeCodeBlocks ?? false;
|
|
17
102
|
const parsed = parseMarkdown(rawContent, { includeCodeBlocks });
|
|
18
103
|
|
|
19
104
|
const { frontmatter, title, h1, h2, h3, h4, h5, h6, body, structure, sectionsIndex } = parsed;
|
|
20
|
-
|
|
21
|
-
|
|
105
|
+
/** Prefer caller override, then DB row by path, then parsed front matter. */
|
|
106
|
+
let fm = options.overrideFrontmatter;
|
|
107
|
+
let byPath = null;
|
|
108
|
+
if (!fm) {
|
|
109
|
+
byPath = loadMetadataByPath(db, docPath);
|
|
110
|
+
fm = byPath ? { ...frontmatter, ...byPath } : frontmatter;
|
|
111
|
+
} else {
|
|
112
|
+
fm = { ...frontmatter, ...fm };
|
|
113
|
+
}
|
|
22
114
|
|
|
115
|
+
/** When metadata did not come from DB (full reindex or new file), infer scope/type/project from path. */
|
|
116
|
+
if (!options.overrideFrontmatter && !byPath) {
|
|
117
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
118
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null);
|
|
119
|
+
fm = {
|
|
120
|
+
...fm,
|
|
121
|
+
scope: inferred.scope,
|
|
122
|
+
type: fm.type ?? inferred.type,
|
|
123
|
+
project: inferred.project,
|
|
124
|
+
created_at: fm.created_at || fileCreatedAt || undefined,
|
|
125
|
+
updated_at: fm.updated_at || fileCreatedAt || undefined
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** If still no id (e.g. full reindex of file with no front matter), generate from path and set created/updated. */
|
|
130
|
+
if (!fm.id) {
|
|
131
|
+
const inferred = inferMetadataFromPath(config, docPath);
|
|
132
|
+
const fileCreatedAt = options.fileCreatedAt ?? (fileMtimeMs != null ? new Date(fileMtimeMs).toISOString() : null) ?? new Date().toISOString();
|
|
133
|
+
fm = {
|
|
134
|
+
...fm,
|
|
135
|
+
id: contentHash(docPath).slice(0, 10),
|
|
136
|
+
scope: inferred.scope,
|
|
137
|
+
type: fm.type ?? inferred.type,
|
|
138
|
+
project: inferred.project,
|
|
139
|
+
created_at: fm.created_at || fileCreatedAt,
|
|
140
|
+
updated_at: fm.updated_at || fileCreatedAt
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const id = fm.id;
|
|
23
145
|
const now = new Date().toISOString();
|
|
24
146
|
const docHash = contentHash(rawContent);
|
|
25
|
-
const docTitle =
|
|
147
|
+
const docTitle = fm.title || title || '';
|
|
26
148
|
|
|
27
|
-
const existing = db.prepare('SELECT content_hash, file_mtime_ms FROM documents WHERE id = ?').get(id);
|
|
149
|
+
const existing = db.prepare('SELECT content_hash, file_mtime_ms, path FROM documents WHERE id = ?').get(id);
|
|
28
150
|
if (existing && existing.content_hash === docHash && (fileMtimeMs == null || existing.file_mtime_ms === fileMtimeMs)) {
|
|
151
|
+
/** If path changed (move), update row so path stays in sync. */
|
|
152
|
+
const normPath = path.normalize(docPath);
|
|
153
|
+
if (existing.path && path.normalize(existing.path) !== normPath) {
|
|
154
|
+
db.prepare('UPDATE documents SET path = ?, file_mtime_ms = ? WHERE id = ?').run(path.normalize(docPath), fileMtimeMs, id);
|
|
155
|
+
}
|
|
29
156
|
return { id };
|
|
30
157
|
}
|
|
31
158
|
|
|
32
159
|
const docRow = {
|
|
33
160
|
id,
|
|
34
|
-
path: docPath,
|
|
161
|
+
path: path.normalize(docPath),
|
|
35
162
|
title: docTitle,
|
|
36
|
-
type:
|
|
37
|
-
scope:
|
|
38
|
-
project:
|
|
39
|
-
topic_key:
|
|
40
|
-
session_id:
|
|
41
|
-
tool_name:
|
|
163
|
+
type: fm.type || 'generated_note',
|
|
164
|
+
scope: fm.scope || 'project',
|
|
165
|
+
project: fm.project || null,
|
|
166
|
+
topic_key: fm.topic_key || null,
|
|
167
|
+
session_id: fm.session_id || null,
|
|
168
|
+
tool_name: fm.tool_name || null,
|
|
42
169
|
content_hash: docHash,
|
|
43
170
|
file_mtime_ms: fileMtimeMs,
|
|
44
|
-
created_at:
|
|
45
|
-
updated_at:
|
|
171
|
+
created_at: fm.created_at || now,
|
|
172
|
+
updated_at: fm.updated_at || now,
|
|
46
173
|
indexed_at: now,
|
|
47
|
-
deleted_at:
|
|
48
|
-
revision_count:
|
|
49
|
-
duplicate_count:
|
|
50
|
-
embedding_status:
|
|
174
|
+
deleted_at: fm.deleted_at || null,
|
|
175
|
+
revision_count: fm.revision_count ?? 1,
|
|
176
|
+
duplicate_count: fm.duplicate_count ?? 0,
|
|
177
|
+
embedding_status: fm.embedding_status || 'disabled',
|
|
51
178
|
body: body || '',
|
|
52
179
|
structure: JSON.stringify(structure),
|
|
53
180
|
sections_index: JSON.stringify(sectionsIndex)
|
|
@@ -84,7 +211,7 @@ function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
|
84
211
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
85
212
|
`).run(id, titleNorm, h1Norm, h2Norm, h3Norm, h4Norm, h5Norm, h6Norm, bodyNorm);
|
|
86
213
|
|
|
87
|
-
const tags =
|
|
214
|
+
const tags = fm.tags || [];
|
|
88
215
|
for (const tag of tags) {
|
|
89
216
|
db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
|
|
90
217
|
}
|
|
@@ -3,6 +3,44 @@ import fs from 'fs';
|
|
|
3
3
|
import { globSync } from 'glob';
|
|
4
4
|
import { getDb } from '../../db/sqlite.js';
|
|
5
5
|
import { indexDocument, removeDocumentFromIndex } from './indexer.js';
|
|
6
|
+
import { TYPE_TO_DIR } from '../../fs/paths.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Load document metadata from DB by path (for reindex/watcher).
|
|
10
|
+
* Returns frontmatter-like object or null.
|
|
11
|
+
*/
|
|
12
|
+
function getDocumentMetaByPath(config, filePath) {
|
|
13
|
+
const db = getDb(config);
|
|
14
|
+
const norm = path.normalize(filePath);
|
|
15
|
+
const row = db.prepare(
|
|
16
|
+
'SELECT id, path, title, type, scope, project, topic_key, session_id, tool_name, created_at, updated_at, revision_count, duplicate_count, deleted_at, embedding_status FROM documents WHERE path = ?'
|
|
17
|
+
).get(norm);
|
|
18
|
+
if (!row) return null;
|
|
19
|
+
const tagRows = db.prepare('SELECT tag FROM document_tags WHERE document_id = ?').all(row.id);
|
|
20
|
+
const tags = tagRows.map((r) => r.tag);
|
|
21
|
+
return rowToFrontmatter(row, tags);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function rowToFrontmatter(row, tags = []) {
|
|
25
|
+
return {
|
|
26
|
+
id: row.id,
|
|
27
|
+
title: row.title ?? '',
|
|
28
|
+
type: row.type ?? 'generated_note',
|
|
29
|
+
scope: row.scope ?? 'project',
|
|
30
|
+
project: row.project ?? null,
|
|
31
|
+
topic_key: row.topic_key ?? null,
|
|
32
|
+
session_id: row.session_id ?? null,
|
|
33
|
+
tool_name: row.tool_name ?? null,
|
|
34
|
+
created_at: row.created_at,
|
|
35
|
+
updated_at: row.updated_at,
|
|
36
|
+
revision_count: row.revision_count ?? 1,
|
|
37
|
+
duplicate_count: row.duplicate_count ?? 0,
|
|
38
|
+
deleted: !!row.deleted_at,
|
|
39
|
+
tags: Array.isArray(tags) ? tags : [],
|
|
40
|
+
embedding_status: row.embedding_status ?? 'disabled',
|
|
41
|
+
deleted_at: row.deleted_at ?? null
|
|
42
|
+
};
|
|
43
|
+
}
|
|
6
44
|
|
|
7
45
|
/**
|
|
8
46
|
* List all Markdown file paths under home (cross-project) and project memory root (repo-local).
|
|
@@ -49,7 +87,8 @@ function fullReindex(config) {
|
|
|
49
87
|
try {
|
|
50
88
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
51
89
|
const stat = fs.statSync(filePath);
|
|
52
|
-
|
|
90
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
91
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { fileCreatedAt });
|
|
53
92
|
indexed++;
|
|
54
93
|
} catch (err) {
|
|
55
94
|
// skip invalid or unreadable files
|
|
@@ -81,7 +120,9 @@ function incrementalReindex(config) {
|
|
|
81
120
|
continue;
|
|
82
121
|
}
|
|
83
122
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
84
|
-
|
|
123
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
124
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
125
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
85
126
|
indexed++;
|
|
86
127
|
} catch (_) {}
|
|
87
128
|
}
|
|
@@ -107,4 +148,4 @@ function getDocumentIdByPath(config, filePath) {
|
|
|
107
148
|
return row?.id ?? null;
|
|
108
149
|
}
|
|
109
150
|
|
|
110
|
-
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath };
|
|
151
|
+
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath, getDocumentMetaByPath };
|
|
@@ -70,8 +70,13 @@ function search(config, options = {}) {
|
|
|
70
70
|
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
71
71
|
params.push('home');
|
|
72
72
|
} else {
|
|
73
|
-
|
|
74
|
-
|
|
73
|
+
if (project) {
|
|
74
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
75
|
+
params.push('home', 'project', project);
|
|
76
|
+
} else {
|
|
77
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR d.scope = ?)';
|
|
78
|
+
params.push('home', 'project');
|
|
79
|
+
}
|
|
75
80
|
}
|
|
76
81
|
|
|
77
82
|
params.push(project || '');
|
package/src/fs/watcher.js
CHANGED
|
@@ -2,11 +2,11 @@ import path from 'path';
|
|
|
2
2
|
import chokidar from 'chokidar';
|
|
3
3
|
import fs from 'fs';
|
|
4
4
|
import { indexDocument, removeDocumentFromIndex } from '../core/indexing/indexer.js';
|
|
5
|
-
import { getDocumentIdByPath } from '../core/indexing/reindex.js';
|
|
5
|
+
import { getDocumentIdByPath, getDocumentMetaByPath } from '../core/indexing/reindex.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Start watching home (cross-project) and project memory root (repo-local) for .md changes.
|
|
9
|
-
* On add/change: index the file. On unlink: remove from index by path.
|
|
9
|
+
* On add/change: index the file (using DB metadata when path already known). On unlink: remove from index by path.
|
|
10
10
|
* @param {object} config - resolved config
|
|
11
11
|
* @returns {import('chokidar').FSWatcher} watcher instance (call .close() to stop)
|
|
12
12
|
*/
|
|
@@ -28,14 +28,18 @@ function startWatcher(config) {
|
|
|
28
28
|
try {
|
|
29
29
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
30
30
|
const stat = fs.statSync(filePath);
|
|
31
|
-
|
|
31
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
32
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
33
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
32
34
|
} catch (_) {}
|
|
33
35
|
})
|
|
34
36
|
.on('change', (filePath) => {
|
|
35
37
|
try {
|
|
36
38
|
const raw = fs.readFileSync(filePath, 'utf8');
|
|
37
39
|
const stat = fs.statSync(filePath);
|
|
38
|
-
|
|
40
|
+
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
41
|
+
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
42
|
+
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
39
43
|
} catch (_) {}
|
|
40
44
|
})
|
|
41
45
|
.on('unlink', (filePath) => {
|
package/src/utils/markdown.js
CHANGED
|
@@ -6,8 +6,8 @@ export const FRONTMATTER_KEYS = [
|
|
|
6
6
|
'embedding_status', 'indexed_at', 'deleted_at'
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
-
/** Keys written to .md
|
|
10
|
-
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', '
|
|
9
|
+
/** Keys written to .md: id, type, title, created (cosmetic). scope/project stay in DB only; reindex infers scope from path. */
|
|
10
|
+
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'created'];
|
|
11
11
|
|
|
12
12
|
function formatCreated(createdAt) {
|
|
13
13
|
if (!createdAt) return null;
|
|
@@ -39,12 +39,13 @@ function serializeFrontmatter(fm) {
|
|
|
39
39
|
return obj;
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
/** Minimal frontmatter for .md (
|
|
42
|
+
/** Minimal frontmatter for .md (cosmetic only; deleted flag for soft-delete visibility). */
|
|
43
43
|
function serializeDisplayFrontmatter(fm) {
|
|
44
44
|
const obj = {};
|
|
45
45
|
for (const k of DISPLAY_FRONTMATTER_KEYS) {
|
|
46
46
|
if (k === 'created') {
|
|
47
|
-
const
|
|
47
|
+
const raw = fm.created_at ?? (fm.created ? parseCreated(fm.created) : null);
|
|
48
|
+
const v = raw ? formatCreated(raw) : null;
|
|
48
49
|
if (v) obj[k] = v;
|
|
49
50
|
} else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
|
|
50
51
|
obj[k] = fm[k];
|