mixgram 1.3.7 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -345
- package/bin/mixgram.js +51 -160
- package/examples/embedding-demo.js +22 -142
- package/package.json +2 -3
- package/pnpm-workspace.yaml +2 -2
- package/spec/bench-indexing.js +23 -211
- package/spec/bench-search.js +26 -212
- package/spec/run-tests.js +1 -214
- package/spec/scenarios/scenarios.js +277 -607
- package/spec/sdd.md +14 -988
- package/spec/tdd.md +15 -1232
- package/spec/test-large-document.js +20 -91
- package/src/config.js +57 -48
- package/src/core/documents/documents.js +62 -224
- package/src/core/embeddings/embedder.js +14 -54
- package/src/core/embeddings/queue.js +33 -106
- package/src/core/embeddings/vectorStore.js +30 -111
- package/src/core/embeddings/worker-process.js +23 -74
- package/src/core/indexing/indexer.js +51 -237
- package/src/core/indexing/parser.js +13 -114
- package/src/core/indexing/reindex.js +28 -137
- package/src/core/search/search.js +64 -158
- package/src/db/migrations/001_initial.sql +14 -63
- package/src/db/sqlite.js +19 -29
- package/src/fs/paths.js +11 -101
- package/src/fs/watcher.js +15 -48
- package/src/mcp/cli-adapter.js +32 -110
- package/src/mcp/server.js +64 -141
- package/src/mcp/tool-registry.js +21 -132
- package/src/mcp/tools.js +27 -199
- package/src/utils/markdown.js +15 -82
- package/patches/sharp@0.34.5.patch +0 -13
- package/spec/fixtures/realistic-content.js +0 -106
- package/spec/reporter.js +0 -127
- package/spec/run-tests-runner.js +0 -40
- package/src/core/embeddings/worker.js +0 -27
- package/src/core/prompts/prompts.js +0 -15
- package/src/core/sessions/sessions.js +0 -27
- package/src/utils/ids.js +0 -27
|
@@ -1,170 +1,76 @@
|
|
|
1
1
|
import { getDb } from '../../db/sqlite.js';
|
|
2
2
|
import { normalizeForFts } from '../indexing/indexer.js';
|
|
3
3
|
|
|
4
|
-
function
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
];
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// FTS5 uses "/" for NEAR (e.g. term/5) and "-" for NOT; dash-like chars between words break parsing.
|
|
19
|
-
const FTS5_NEAR_SLASH = /\//g;
|
|
20
|
-
const FTS5_NOT_HYPHEN = /(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g; // hyphen, en/em-dash, minus
|
|
21
|
-
// FTS5 operators are case-sensitive; normalizeForFts lowercases the string, so we restore them after.
|
|
22
|
-
const FTS5_OPERATOR_RESTORE = [
|
|
23
|
-
[/\band\b/g, 'AND'],
|
|
24
|
-
[/\bor\b/g, 'OR'],
|
|
25
|
-
[/\bnot\b/g, 'NOT']
|
|
26
|
-
];
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Normalize query for FTS5: same as indexed text (NFD, remove diacritics, lowercase).
|
|
30
|
-
* Replaces "/" with space so it is not interpreted as FTS5 NEAR syntax (e.g. term/5).
|
|
31
|
-
* Replaces "-" (and Unicode dashes) between word chars with space so "alchemy-tycoon" is not parsed as "alchemy" NOT "tycoon".
|
|
32
|
-
* Restores FTS5 operators AND, OR, NOT to uppercase after normalization (FTS5 requires them case-sensitive).
|
|
33
|
-
*/
|
|
34
|
-
function normalizeQuery(q) {
|
|
35
|
-
if (q == null || typeof q !== 'string') return '';
|
|
36
|
-
const trimmed = q.trim();
|
|
37
|
-
const slashSafe = trimmed.replace(FTS5_NEAR_SLASH, ' ');
|
|
38
|
-
const hyphenSafe = slashSafe.replace(FTS5_NOT_HYPHEN, ' ');
|
|
39
|
-
let out = normalizeForFts(hyphenSafe);
|
|
40
|
-
for (const [re, replacement] of FTS5_OPERATOR_RESTORE) {
|
|
41
|
-
out = out.replace(re, replacement);
|
|
4
|
+
export function documentFilter(config, { collection, type, tags = [] } = {}) {
|
|
5
|
+
if (collection && !Object.hasOwn(config.collections, collection)) throw new Error('Unknown collection');
|
|
6
|
+
const conditions = ['d.deleted_at IS NULL'];
|
|
7
|
+
const params = [];
|
|
8
|
+
const collections = collection ? [collection] : Object.keys(config.collections);
|
|
9
|
+
conditions.push(`d.collection IN (${collections.map(() => '?').join(',')})`);
|
|
10
|
+
params.push(...collections);
|
|
11
|
+
if (type) { conditions.push('d.type = ?'); params.push(type); }
|
|
12
|
+
for (const tag of new Set(tags)) {
|
|
13
|
+
conditions.push('EXISTS (SELECT 1 FROM json_each(d.tags) WHERE value = ?)');
|
|
14
|
+
params.push(tag);
|
|
42
15
|
}
|
|
43
|
-
return
|
|
16
|
+
return { sql: conditions.join(' AND '), params };
|
|
44
17
|
}
|
|
45
18
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
* @param {string} [options.project] - project name (required for project-only and merged)
|
|
52
|
-
* @param {number} [options.limit] - max results
|
|
53
|
-
* @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, created: string | null, snippet: string, score: number }>}
|
|
54
|
-
*/
|
|
55
|
-
function search(config, options = {}) {
|
|
56
|
-
const {
|
|
57
|
-
query,
|
|
58
|
-
scopeMode = config.search?.defaultScopeMode || 'merged',
|
|
59
|
-
project = null,
|
|
60
|
-
limit = config.search?.defaultLimit ?? 10
|
|
61
|
-
} = options;
|
|
62
|
-
|
|
63
|
-
if (!query || !String(query).trim()) {
|
|
64
|
-
return [];
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
const db = getDb(config);
|
|
68
|
-
const q = normalizeQuery(query);
|
|
69
|
-
const weights = getBm25Weights(config);
|
|
70
|
-
const bm25Args = weights.map((w, i) => (i === 0 ? w : `, ${w}`)).join('');
|
|
71
|
-
|
|
72
|
-
let scopeCondition = '';
|
|
73
|
-
const params = [q];
|
|
74
|
-
|
|
75
|
-
if (scopeMode === 'project-only') {
|
|
76
|
-
if (!project) return [];
|
|
77
|
-
scopeCondition = ' AND d.scope = ? AND d.project = ? AND d.deleted_at IS NULL';
|
|
78
|
-
params.push('project', project);
|
|
79
|
-
} else if (scopeMode === 'home-only') {
|
|
80
|
-
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
81
|
-
params.push('home');
|
|
82
|
-
} else {
|
|
83
|
-
if (project) {
|
|
84
|
-
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
85
|
-
params.push('home', 'project', project);
|
|
86
|
-
} else {
|
|
87
|
-
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR d.scope = ?)';
|
|
88
|
-
params.push('home', 'project');
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
params.push(project || '');
|
|
93
|
-
params.push(limit);
|
|
94
|
-
|
|
95
|
-
const sql = `
|
|
96
|
-
SELECT
|
|
97
|
-
f.document_id AS documentId,
|
|
98
|
-
d.title AS title,
|
|
99
|
-
d.topic_key AS topicKey,
|
|
100
|
-
d.type AS type,
|
|
101
|
-
d.scope AS scope,
|
|
102
|
-
d.project AS project,
|
|
103
|
-
d.created_at AS created,
|
|
104
|
-
d.body AS body,
|
|
105
|
-
snippet(document_fts, 1, '**', '**', '...', 24) AS snippetTitle,
|
|
106
|
-
snippet(document_fts, 8, '**', '**', '...', 64) AS snippetBody,
|
|
107
|
-
bm25(document_fts, ${bm25Args}) AS rank
|
|
108
|
-
FROM document_fts f
|
|
109
|
-
JOIN documents d ON d.id = f.document_id
|
|
110
|
-
WHERE document_fts MATCH ? ${scopeCondition}
|
|
111
|
-
ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
|
|
112
|
-
LIMIT ?
|
|
113
|
-
`;
|
|
114
|
-
|
|
115
|
-
const stmt = db.prepare(sql);
|
|
116
|
-
const rows = stmt.all(...params);
|
|
19
|
+
function normalizeQuery(query) {
|
|
20
|
+
let text = normalizeForFts(query.trim().replace(/\//g, ' ').replace(/(?<=\w)[-\u2010-\u2015\u2212](?=\w)/g, ' '));
|
|
21
|
+
for (const op of ['and', 'or', 'not']) text = text.replace(new RegExp(`\\b${op}\\b`, 'g'), op.toUpperCase());
|
|
22
|
+
return text;
|
|
23
|
+
}
|
|
117
24
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
title: r.title,
|
|
125
|
-
topicKey: r.topicKey,
|
|
126
|
-
type: r.type,
|
|
127
|
-
scope: r.scope,
|
|
128
|
-
project: r.project,
|
|
129
|
-
created: r.created || null,
|
|
130
|
-
snippet: String(snippet).trim(),
|
|
131
|
-
score: -Number(r.rank)
|
|
132
|
-
};
|
|
133
|
-
});
|
|
25
|
+
function resultRow(config, row, score) {
|
|
26
|
+
return {
|
|
27
|
+
id: row.id, title: row.title, collection: row.collection, type: row.type, key: row.key,
|
|
28
|
+
tags: JSON.parse(row.tags), created_at: row.created_at, updated_at: row.updated_at,
|
|
29
|
+
snippet: row.body.slice(0, config.search.snippetLength), ...(score === undefined ? {} : { score })
|
|
30
|
+
};
|
|
134
31
|
}
|
|
135
32
|
|
|
136
|
-
|
|
137
|
-
* Get recent context without FTS (for mem_context when no query).
|
|
138
|
-
*/
|
|
139
|
-
function getRecentContext(config, options = {}) {
|
|
140
|
-
const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
|
|
33
|
+
export async function search(config, options = {}) {
|
|
141
34
|
const db = getDb(config);
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
35
|
+
const filter = documentFilter(config, options);
|
|
36
|
+
const query = options.query?.trim();
|
|
37
|
+
const limit = options.limit ?? config.search.defaultLimit;
|
|
38
|
+
const offset = options.offset ?? 0;
|
|
39
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > config.search.maxLimit || !Number.isInteger(offset) || offset < 0) throw new Error('Invalid pagination');
|
|
40
|
+
if (!query) {
|
|
41
|
+
const rows = db.prepare(`SELECT d.* FROM documents d WHERE ${filter.sql} ORDER BY d.updated_at DESC,d.id ASC LIMIT ? OFFSET ?`).all(...filter.params, limit, offset);
|
|
42
|
+
return { results: rows.map(row => resultRow(config, row)), mode: 'recent', degraded: false };
|
|
43
|
+
}
|
|
44
|
+
const weights = [0, ...Object.values(config.indexing.ftsWeights)].join(',');
|
|
45
|
+
const textRows = db.prepare(`SELECT d.*,bm25(document_fts,${weights}) AS rank FROM document_fts
|
|
46
|
+
JOIN documents d ON d.id=document_fts.document_id WHERE document_fts MATCH ? AND ${filter.sql}
|
|
47
|
+
ORDER BY rank,d.id`).all(normalizeQuery(query), ...filter.params);
|
|
48
|
+
let semanticRows = [];
|
|
49
|
+
let degraded = false;
|
|
50
|
+
let reason;
|
|
51
|
+
if (config.embeddings.enabled) {
|
|
52
|
+
try {
|
|
53
|
+
if (!config.getQueryEmbedding) throw new Error('Embedding worker unavailable');
|
|
54
|
+
const { getVectorStore } = await import('../embeddings/vectorStore.js');
|
|
55
|
+
const store = await getVectorStore(config);
|
|
56
|
+
if (!store) throw new Error('Vector store unavailable');
|
|
57
|
+
const vector = await config.getQueryEmbedding(query);
|
|
58
|
+
if (!vector) throw new Error('Query embedding unavailable');
|
|
59
|
+
semanticRows = await store.search(config, vector, options);
|
|
60
|
+
} catch (error) { degraded = true; reason = error.message; }
|
|
154
61
|
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
62
|
+
const scores = new Map();
|
|
63
|
+
const add = (rows, weight) => rows.forEach((row, i) => {
|
|
64
|
+
const entry = scores.get(row.id) ?? { row, score: 0 };
|
|
65
|
+
entry.score += weight / (config.search.fusionConstant + i + 1);
|
|
66
|
+
scores.set(row.id, entry);
|
|
67
|
+
});
|
|
68
|
+
add(textRows, config.search.ftsWeight);
|
|
69
|
+
add(semanticRows, config.search.semanticWeight);
|
|
70
|
+
const ranked = [...scores.values()].sort((a,b) => b.score - a.score || a.row.id.localeCompare(b.row.id));
|
|
71
|
+
return {
|
|
72
|
+
results: ranked.slice(offset, offset + limit).map(({ row, score }) => resultRow(config, row, score)),
|
|
73
|
+
mode: config.embeddings.enabled && !degraded ? 'hybrid' : 'text', degraded,
|
|
74
|
+
...(reason ? { reason } : {})
|
|
75
|
+
};
|
|
168
76
|
}
|
|
169
|
-
|
|
170
|
-
export { search, getRecentContext, normalizeQuery, getBm25Weights };
|
|
@@ -1,74 +1,25 @@
|
|
|
1
|
-
-- documents (one row per doc; body and section tree stored here)
|
|
2
1
|
CREATE TABLE IF NOT EXISTS documents (
|
|
3
2
|
id TEXT PRIMARY KEY,
|
|
4
3
|
path TEXT NOT NULL UNIQUE,
|
|
5
|
-
|
|
4
|
+
collection TEXT NOT NULL,
|
|
5
|
+
title TEXT NOT NULL,
|
|
6
6
|
type TEXT NOT NULL,
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
topic_key TEXT,
|
|
10
|
-
session_id TEXT,
|
|
11
|
-
tool_name TEXT,
|
|
12
|
-
content_hash TEXT NOT NULL,
|
|
13
|
-
file_mtime_ms INTEGER,
|
|
7
|
+
key TEXT,
|
|
8
|
+
tags TEXT NOT NULL,
|
|
14
9
|
created_at TEXT NOT NULL,
|
|
15
10
|
updated_at TEXT NOT NULL,
|
|
16
|
-
indexed_at TEXT,
|
|
17
11
|
deleted_at TEXT,
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
body TEXT,
|
|
22
|
-
structure TEXT,
|
|
23
|
-
sections_index TEXT
|
|
24
|
-
);
|
|
25
|
-
|
|
26
|
-
CREATE TABLE IF NOT EXISTS document_tags (
|
|
27
|
-
document_id TEXT NOT NULL,
|
|
28
|
-
tag TEXT NOT NULL,
|
|
29
|
-
PRIMARY KEY (document_id, tag),
|
|
30
|
-
FOREIGN KEY(document_id) REFERENCES documents(id)
|
|
31
|
-
);
|
|
32
|
-
|
|
33
|
-
CREATE TABLE IF NOT EXISTS sessions (
|
|
34
|
-
id TEXT PRIMARY KEY,
|
|
35
|
-
project TEXT NOT NULL,
|
|
36
|
-
directory TEXT,
|
|
37
|
-
started_at TEXT NOT NULL,
|
|
38
|
-
ended_at TEXT,
|
|
39
|
-
summary TEXT
|
|
40
|
-
);
|
|
41
|
-
|
|
42
|
-
CREATE TABLE IF NOT EXISTS prompts (
|
|
43
|
-
id TEXT PRIMARY KEY,
|
|
44
|
-
session_id TEXT NOT NULL,
|
|
45
|
-
project TEXT,
|
|
46
|
-
content TEXT NOT NULL,
|
|
47
|
-
created_at TEXT NOT NULL,
|
|
48
|
-
FOREIGN KEY(session_id) REFERENCES sessions(id)
|
|
49
|
-
);
|
|
50
|
-
|
|
51
|
-
CREATE TABLE IF NOT EXISTS embedding_jobs (
|
|
52
|
-
id TEXT PRIMARY KEY,
|
|
53
|
-
chunk_id TEXT NOT NULL,
|
|
54
|
-
model_name TEXT NOT NULL,
|
|
55
|
-
status TEXT NOT NULL,
|
|
56
|
-
attempts INTEGER NOT NULL DEFAULT 0,
|
|
57
|
-
last_error TEXT,
|
|
58
|
-
created_at TEXT NOT NULL,
|
|
59
|
-
updated_at TEXT NOT NULL
|
|
12
|
+
content_hash TEXT NOT NULL,
|
|
13
|
+
file_mtime_ms REAL,
|
|
14
|
+
body TEXT NOT NULL
|
|
60
15
|
);
|
|
61
|
-
|
|
62
|
-
-- FTS5: one row per document (title, h1–h6, body)
|
|
16
|
+
CREATE UNIQUE INDEX IF NOT EXISTS documents_key ON documents(collection, key) WHERE key IS NOT NULL AND deleted_at IS NULL;
|
|
63
17
|
CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
|
|
64
|
-
document_id UNINDEXED,
|
|
65
|
-
title,
|
|
66
|
-
h1,
|
|
67
|
-
h2,
|
|
68
|
-
h3,
|
|
69
|
-
h4,
|
|
70
|
-
h5,
|
|
71
|
-
h6,
|
|
72
|
-
body,
|
|
18
|
+
document_id UNINDEXED, title, h1, h2, h3, h4, h5, h6, body,
|
|
73
19
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
74
20
|
);
|
|
21
|
+
CREATE TABLE IF NOT EXISTS embedding_jobs (
|
|
22
|
+
id TEXT PRIMARY KEY, document_id TEXT NOT NULL, content_hash TEXT NOT NULL, model_name TEXT NOT NULL,
|
|
23
|
+
status TEXT NOT NULL, claim TEXT, attempts INTEGER NOT NULL DEFAULT 0, last_error TEXT,
|
|
24
|
+
created_at TEXT NOT NULL, updated_at TEXT NOT NULL
|
|
25
|
+
);
|
package/src/db/sqlite.js
CHANGED
|
@@ -1,39 +1,29 @@
|
|
|
1
1
|
import Database from 'better-sqlite3';
|
|
2
2
|
import fs from 'fs';
|
|
3
3
|
import path from 'path';
|
|
4
|
-
import { fileURLToPath } from 'url';
|
|
5
4
|
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
let dbInstance = null;
|
|
5
|
+
const databases = new WeakMap();
|
|
9
6
|
|
|
10
7
|
export function getDb(config) {
|
|
11
|
-
if (
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
for (const file of files) {
|
|
27
|
-
const sql = fs.readFileSync(path.join(migrationsDir, file), 'utf8');
|
|
28
|
-
db.exec(sql);
|
|
8
|
+
if (databases.has(config)) return databases.get(config);
|
|
9
|
+
fs.mkdirSync(path.dirname(config.sqlitePath), { recursive: true });
|
|
10
|
+
const db = new Database(config.sqlitePath);
|
|
11
|
+
try {
|
|
12
|
+
const existing = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='documents'").get();
|
|
13
|
+
if (existing && !db.prepare('PRAGMA table_info(documents)').all().some(c => c.name === 'collection')) {
|
|
14
|
+
throw new Error('Legacy database: Mixgram 2 requires a new SQLite path');
|
|
15
|
+
}
|
|
16
|
+
db.pragma('journal_mode = WAL');
|
|
17
|
+
db.exec(fs.readFileSync(new URL('./migrations/001_initial.sql', import.meta.url), 'utf8'));
|
|
18
|
+
databases.set(config, db);
|
|
19
|
+
return db;
|
|
20
|
+
} catch (error) {
|
|
21
|
+
db.close();
|
|
22
|
+
throw error;
|
|
29
23
|
}
|
|
30
24
|
}
|
|
31
25
|
|
|
32
|
-
export function closeDb() {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
dbInstance = null;
|
|
36
|
-
}
|
|
26
|
+
export function closeDb(config) {
|
|
27
|
+
databases.get(config)?.close();
|
|
28
|
+
databases.delete(config);
|
|
37
29
|
}
|
|
38
|
-
|
|
39
|
-
export { runMigrations };
|
package/src/fs/paths.js
CHANGED
|
@@ -1,108 +1,18 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
2
|
import fs from 'fs';
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
bugs: 'bugs',
|
|
10
|
-
learning: 'learnings',
|
|
11
|
-
learnings: 'learnings',
|
|
12
|
-
discovery: 'discoveries',
|
|
13
|
-
discoveries: 'discoveries',
|
|
14
|
-
pattern: 'patterns',
|
|
15
|
-
patterns: 'patterns',
|
|
16
|
-
reference: 'reference',
|
|
17
|
-
session_summary: 'sessions',
|
|
18
|
-
sessions: 'sessions',
|
|
19
|
-
prompt: 'prompts',
|
|
20
|
-
prompts: 'prompts',
|
|
21
|
-
generated_note: 'generated',
|
|
22
|
-
generated: 'generated'
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
function slugFromTopicKey(topicKey) {
|
|
26
|
-
if (!topicKey || typeof topicKey !== 'string') return 'note';
|
|
27
|
-
return topicKey
|
|
28
|
-
.replace(/\//g, '-')
|
|
29
|
-
.replace(/[^a-z0-9-]/gi, '-')
|
|
30
|
-
.replace(/-+/g, '-')
|
|
31
|
-
.toLowerCase()
|
|
32
|
-
.replace(/^-|-$/g, '') || 'note';
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function filenameSlugFromTopicKey(topicKey, { project = null, type = null } = {}) {
|
|
36
|
-
if (!topicKey || typeof topicKey !== 'string') return 'note';
|
|
37
|
-
|
|
38
|
-
const segments = topicKey
|
|
39
|
-
.split('/')
|
|
40
|
-
.map((segment) => segment.trim())
|
|
41
|
-
.filter(Boolean);
|
|
42
|
-
|
|
43
|
-
if (segments.length === 0) return 'note';
|
|
44
|
-
|
|
45
|
-
const normalizedProject = slugFromTopicKey(project);
|
|
46
|
-
if (normalizedProject !== 'note' && slugFromTopicKey(segments[0]) === normalizedProject) {
|
|
47
|
-
segments.shift();
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
const typeDir = TYPE_TO_DIR[type] || 'generated';
|
|
51
|
-
const redundantPrefixes = new Set([
|
|
52
|
-
slugFromTopicKey(type),
|
|
53
|
-
slugFromTopicKey(typeDir)
|
|
54
|
-
].filter((value) => value !== 'note'));
|
|
55
|
-
|
|
56
|
-
if (segments.length > 1 && redundantPrefixes.has(slugFromTopicKey(segments[0]))) {
|
|
57
|
-
segments.shift();
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
return slugFromTopicKey(segments.join('-'));
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function documentPath(config, { scope, project, type, topic_key, id }) {
|
|
64
|
-
const typeDir = TYPE_TO_DIR[type] || 'generated';
|
|
65
|
-
const slug = filenameSlugFromTopicKey(topic_key, { project, type });
|
|
66
|
-
const baseName = `${slug}.md`;
|
|
67
|
-
|
|
68
|
-
if (scope === 'home') {
|
|
69
|
-
const root = config.homeMemoryRoot;
|
|
70
|
-
return path.join(root, typeDir, baseName);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
if (scope === 'project') {
|
|
74
|
-
const root = config.projectMemoryRoot;
|
|
75
|
-
return path.join(root, typeDir, baseName);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
const root = config.projectMemoryRoot;
|
|
79
|
-
return path.join(root, typeDir, baseName);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
function ensureDir(filePath) {
|
|
83
|
-
const dir = path.dirname(filePath);
|
|
84
|
-
if (!fs.existsSync(dir)) {
|
|
85
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
4
|
+
export function collectionForPath(config, filePath) {
|
|
5
|
+
const absolute = path.resolve(filePath);
|
|
6
|
+
const real = fs.existsSync(absolute) ? fs.realpathSync(absolute) : absolute;
|
|
7
|
+
for (const [name, root] of Object.entries(config.collections)) {
|
|
8
|
+
if (absolute.startsWith(root + path.sep) && real.startsWith(root + path.sep)) return name;
|
|
86
9
|
}
|
|
10
|
+
throw new Error('Document is outside configured collections');
|
|
87
11
|
}
|
|
88
12
|
|
|
89
|
-
function
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
const dir = mixgramDir(config);
|
|
95
|
-
if (!fs.existsSync(dir)) {
|
|
96
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
97
|
-
}
|
|
13
|
+
export function documentPath(config, { collection, title, id }) {
|
|
14
|
+
const root = config.collections[collection];
|
|
15
|
+
if (!root) throw new Error('Unknown collection');
|
|
16
|
+
const slug = title.normalize('NFD').replace(/\p{Diacritic}/gu, '').replace(/[^a-z0-9]+/gi, '-').replace(/^-|-$/g, '').slice(0, 80).toLowerCase() || 'note';
|
|
17
|
+
return path.join(root, `${slug}-${id}.md`);
|
|
98
18
|
}
|
|
99
|
-
|
|
100
|
-
export {
|
|
101
|
-
documentPath,
|
|
102
|
-
ensureDir,
|
|
103
|
-
mixgramDir,
|
|
104
|
-
ensureMixgramDir,
|
|
105
|
-
TYPE_TO_DIR,
|
|
106
|
-
slugFromTopicKey,
|
|
107
|
-
filenameSlugFromTopicKey
|
|
108
|
-
};
|
package/src/fs/watcher.js
CHANGED
|
@@ -1,55 +1,22 @@
|
|
|
1
|
-
import path from 'path';
|
|
2
1
|
import chokidar from 'chokidar';
|
|
3
|
-
import
|
|
2
|
+
import { getDb } from '../db/sqlite.js';
|
|
4
3
|
import { indexDocument, removeDocumentFromIndex } from '../core/indexing/indexer.js';
|
|
5
|
-
import { getDocumentIdByPath, getDocumentMetaByPath } from '../core/indexing/reindex.js';
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
* On add/change: index the file (using DB metadata when path already known). On unlink: remove from index by path.
|
|
10
|
-
* @param {object} config - resolved config
|
|
11
|
-
* @returns {import('chokidar').FSWatcher} watcher instance (call .close() to stop)
|
|
12
|
-
*/
|
|
13
|
-
function startWatcher(config) {
|
|
14
|
-
const homeRoot = config.homeMemoryRoot;
|
|
15
|
-
const projectRoot = config.projectMemoryRoot;
|
|
16
|
-
const patterns = [];
|
|
17
|
-
if (homeRoot) patterns.push(path.join(homeRoot, '**/*.md'));
|
|
18
|
-
if (projectRoot) patterns.push(path.join(projectRoot, '**/*.md'));
|
|
19
|
-
if (patterns.length === 0) return null;
|
|
20
|
-
|
|
21
|
-
const watcher = chokidar.watch(patterns, {
|
|
5
|
+
export function startWatcher(config, onError = error => process.stderr.write(`[mixgram] watch: ${error.message}\n`)) {
|
|
6
|
+
const watcher = chokidar.watch(Object.values(config.collections), {
|
|
22
7
|
ignoreInitial: true,
|
|
23
|
-
awaitWriteFinish: { stabilityThreshold:
|
|
8
|
+
awaitWriteFinish: { stabilityThreshold: config.indexing.watchStabilityMs }
|
|
24
9
|
});
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
.on('change', (filePath) => {
|
|
37
|
-
try {
|
|
38
|
-
const raw = fs.readFileSync(filePath, 'utf8');
|
|
39
|
-
const stat = fs.statSync(filePath);
|
|
40
|
-
const dbMeta = getDocumentMetaByPath(config, filePath);
|
|
41
|
-
const fileCreatedAt = stat.birthtime && !Number.isNaN(stat.birthtime.getTime()) ? stat.birthtime.toISOString() : undefined;
|
|
42
|
-
indexDocument(config, filePath, raw, stat.mtimeMs, { overrideFrontmatter: dbMeta ?? undefined, fileCreatedAt });
|
|
43
|
-
} catch (_) {}
|
|
44
|
-
})
|
|
45
|
-
.on('unlink', (filePath) => {
|
|
46
|
-
try {
|
|
47
|
-
const id = getDocumentIdByPath(config, filePath);
|
|
48
|
-
if (id) removeDocumentFromIndex(config, id);
|
|
49
|
-
} catch (_) {}
|
|
50
|
-
});
|
|
51
|
-
|
|
10
|
+
const update = filePath => {
|
|
11
|
+
if (!filePath.endsWith('.md')) return;
|
|
12
|
+
try { indexDocument(config, filePath); }
|
|
13
|
+
catch (error) { onError(error); }
|
|
14
|
+
};
|
|
15
|
+
watcher.on('add', update).on('change', update).on('unlink', filePath => {
|
|
16
|
+
try {
|
|
17
|
+
const row = getDb(config).prepare('SELECT id FROM documents WHERE path = ?').get(filePath);
|
|
18
|
+
if (row) removeDocumentFromIndex(config, row.id);
|
|
19
|
+
} catch (error) { onError(error); }
|
|
20
|
+
}).on('error', onError);
|
|
52
21
|
return watcher;
|
|
53
22
|
}
|
|
54
|
-
|
|
55
|
-
export { startWatcher };
|