mixgram 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mixgram/index.db +0 -0
- package/.mixgram/test-esm.db +0 -0
- package/.mixgram/test-index.db +0 -0
- package/.mixgram/test-stage1.db +0 -0
- package/README.md +335 -0
- package/bin/mixgram.js +309 -0
- package/index.js +9 -0
- package/package.json +51 -0
- package/projects/castlebravo/memory/decisions/decision--architecture/sqlite-derived-index.md +17 -0
- package/projects/test/memory/decisions/decision--architecture/sqlite-derived-index.md +17 -0
- package/spec/fixtures/realistic-content.js +97 -0
- package/spec/reporter.js +127 -0
- package/spec/run-tests-runner.js +40 -0
- package/spec/run-tests.js +160 -0
- package/spec/scenarios/scenarios.js +520 -0
- package/spec/sdd.md +1002 -0
- package/spec/tdd.md +1237 -0
- package/src/config.js +56 -0
- package/src/core/documents/documents.js +207 -0
- package/src/core/embeddings/embedder.js +58 -0
- package/src/core/embeddings/queue.js +118 -0
- package/src/core/embeddings/vectorStore.js +127 -0
- package/src/core/embeddings/worker.js +27 -0
- package/src/core/indexing/indexer.js +131 -0
- package/src/core/indexing/parser.js +78 -0
- package/src/core/indexing/reindex.js +101 -0
- package/src/core/prompts/prompts.js +15 -0
- package/src/core/search/search.js +223 -0
- package/src/core/sessions/sessions.js +27 -0
- package/src/db/migrations/001_initial.sql +86 -0
- package/src/db/sqlite.js +39 -0
- package/src/fs/paths.js +79 -0
- package/src/fs/watcher.js +49 -0
- package/src/mcp/cli-adapter.js +118 -0
- package/src/mcp/server.js +47 -0
- package/src/mcp/tool-registry.js +138 -0
- package/src/mcp/tools.js +159 -0
- package/src/utils/hash.js +6 -0
- package/src/utils/ids.js +22 -0
- package/src/utils/markdown.js +49 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { getDb } from '../../db/sqlite.js';
|
|
2
|
+
import { parseAndChunk } from './parser.js';
|
|
3
|
+
import { contentHash } from '../../utils/hash.js';
|
|
4
|
+
import { chunkId } from '../../utils/ids.js';
|
|
5
|
+
|
|
6
|
+
function normalizeForFts(str) {
|
|
7
|
+
if (str == null || str === undefined) return '';
|
|
8
|
+
return String(str)
|
|
9
|
+
.normalize('NFD')
|
|
10
|
+
.replace(/\p{Diacritic}/gu, '')
|
|
11
|
+
.toLowerCase()
|
|
12
|
+
.trim();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function indexDocument(config, docPath, rawContent, fileMtimeMs = null) {
|
|
16
|
+
const db = getDb(config);
|
|
17
|
+
const { frontmatter, chunks } = parseAndChunk(rawContent, {
|
|
18
|
+
chunkSize: config.indexing.chunkSize,
|
|
19
|
+
chunkOverlap: config.indexing.chunkOverlap
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const id = frontmatter.id;
|
|
23
|
+
if (!id) throw new Error('Document frontmatter must include id');
|
|
24
|
+
|
|
25
|
+
const now = new Date().toISOString();
|
|
26
|
+
const docHash = contentHash(rawContent);
|
|
27
|
+
|
|
28
|
+
const docRow = {
|
|
29
|
+
id,
|
|
30
|
+
path: docPath,
|
|
31
|
+
title: frontmatter.title || '',
|
|
32
|
+
type: frontmatter.type || 'generated_note',
|
|
33
|
+
scope: frontmatter.scope || 'project',
|
|
34
|
+
project: frontmatter.project || null,
|
|
35
|
+
topic_key: frontmatter.topic_key || null,
|
|
36
|
+
session_id: frontmatter.session_id || null,
|
|
37
|
+
tool_name: frontmatter.tool_name || null,
|
|
38
|
+
content_hash: docHash,
|
|
39
|
+
file_mtime_ms: fileMtimeMs,
|
|
40
|
+
created_at: frontmatter.created_at || now,
|
|
41
|
+
updated_at: frontmatter.updated_at || now,
|
|
42
|
+
indexed_at: now,
|
|
43
|
+
deleted_at: frontmatter.deleted_at || null,
|
|
44
|
+
revision_count: frontmatter.revision_count ?? 1,
|
|
45
|
+
duplicate_count: frontmatter.duplicate_count ?? 0,
|
|
46
|
+
embedding_status: frontmatter.embedding_status || 'disabled'
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
let oldChunkIds = [];
|
|
50
|
+
db.transaction(() => {
|
|
51
|
+
const existing = db.prepare('SELECT id FROM documents WHERE id = ?').get(id);
|
|
52
|
+
if (existing) {
|
|
53
|
+
oldChunkIds = db.prepare('SELECT id FROM document_chunks WHERE document_id = ?').all(id).map((r) => r.id);
|
|
54
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(id);
|
|
55
|
+
for (const { rowid } of ftsRowids) {
|
|
56
|
+
db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
|
|
57
|
+
}
|
|
58
|
+
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(id);
|
|
59
|
+
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(id);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
db.prepare(`
|
|
63
|
+
INSERT OR REPLACE INTO documents (id, path, title, type, scope, project, topic_key, session_id, tool_name, content_hash, file_mtime_ms, created_at, updated_at, indexed_at, deleted_at, revision_count, duplicate_count, embedding_status)
|
|
64
|
+
VALUES (@id, @path, @title, @type, @scope, @project, @topic_key, @session_id, @tool_name, @content_hash, @file_mtime_ms, @created_at, @updated_at, @indexed_at, @deleted_at, @revision_count, @duplicate_count, @embedding_status)
|
|
65
|
+
`).run(docRow);
|
|
66
|
+
|
|
67
|
+
const titleNorm = normalizeForFts(docRow.title);
|
|
68
|
+
const topicKeyNorm = normalizeForFts(docRow.topic_key);
|
|
69
|
+
const typeNorm = normalizeForFts(docRow.type);
|
|
70
|
+
const scopeNorm = normalizeForFts(docRow.scope);
|
|
71
|
+
const projectNorm = normalizeForFts(docRow.project);
|
|
72
|
+
|
|
73
|
+
chunks.forEach((chunk, idx) => {
|
|
74
|
+
const cid = chunkId(id, idx);
|
|
75
|
+
const chunkHash = contentHash(chunk.content);
|
|
76
|
+
const headingPathNorm = normalizeForFts(chunk.headingPath);
|
|
77
|
+
const bodyNorm = normalizeForFts(chunk.content);
|
|
78
|
+
|
|
79
|
+
db.prepare(`
|
|
80
|
+
INSERT INTO document_chunks (id, document_id, chunk_index, heading_path, heading_level, content, content_hash, created_at, updated_at)
|
|
81
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
82
|
+
`).run(cid, id, idx, chunk.headingPath || '', chunk.headingLevel || 0, chunk.content, chunkHash, now, now);
|
|
83
|
+
|
|
84
|
+
db.prepare(`
|
|
85
|
+
INSERT INTO document_chunks_fts (chunk_id, document_id, title, topic_key, type, scope, project, heading_path, body)
|
|
86
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
87
|
+
`).run(cid, id, titleNorm, topicKeyNorm, typeNorm, scopeNorm, projectNorm, headingPathNorm, bodyNorm);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
const tags = frontmatter.tags || [];
|
|
91
|
+
for (const tag of tags) {
|
|
92
|
+
db.prepare('INSERT OR IGNORE INTO document_tags (document_id, tag) VALUES (?, ?)').run(id, tag);
|
|
93
|
+
}
|
|
94
|
+
})();
|
|
95
|
+
|
|
96
|
+
if (config?.embeddings?.enabled) {
|
|
97
|
+
const newChunkIds = Array.from({ length: chunks.length }, (_, idx) => chunkId(id, idx));
|
|
98
|
+
import('../embeddings/queue.js').then(({ enqueueChunks, markStale }) => {
|
|
99
|
+
if (oldChunkIds.length) markStale(config, oldChunkIds);
|
|
100
|
+
enqueueChunks(config, newChunkIds);
|
|
101
|
+
});
|
|
102
|
+
if (oldChunkIds.length) {
|
|
103
|
+
import('../embeddings/vectorStore.js').then(({ removeChunks }) => removeChunks(config, oldChunkIds).catch(() => {}));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return { id, chunkCount: chunks.length };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function removeDocumentFromIndex(config, documentId) {
|
|
111
|
+
const db = getDb(config);
|
|
112
|
+
const chunkIds = db.prepare('SELECT id FROM document_chunks WHERE document_id = ?').all(documentId).map((r) => r.id);
|
|
113
|
+
if (config?.embeddings?.enabled && chunkIds.length) {
|
|
114
|
+
import('../embeddings/queue.js').then(({ markStale }) => markStale(config, chunkIds));
|
|
115
|
+
import('../embeddings/vectorStore.js').then(({ removeChunks }) => removeChunks(config, chunkIds).catch(() => {}));
|
|
116
|
+
}
|
|
117
|
+
for (const cid of chunkIds) {
|
|
118
|
+
db.prepare('DELETE FROM embedding_jobs WHERE chunk_id = ?').run(cid);
|
|
119
|
+
}
|
|
120
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts WHERE document_id = ?').all(documentId);
|
|
121
|
+
db.transaction(() => {
|
|
122
|
+
for (const { rowid } of ftsRowids) {
|
|
123
|
+
db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
|
|
124
|
+
}
|
|
125
|
+
db.prepare('DELETE FROM document_chunks WHERE document_id = ?').run(documentId);
|
|
126
|
+
db.prepare('DELETE FROM document_tags WHERE document_id = ?').run(documentId);
|
|
127
|
+
db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
|
|
128
|
+
})();
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export { indexDocument, removeDocumentFromIndex, normalizeForFts };
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { unified } from 'unified';
|
|
2
|
+
import remarkParse from 'remark-parse';
|
|
3
|
+
import { visit } from 'unist-util-visit';
|
|
4
|
+
import { parse as parseFrontmatter } from '../../utils/markdown.js';
|
|
5
|
+
|
|
6
|
+
function stringifyNode(node) {
|
|
7
|
+
if (node.value) return node.value;
|
|
8
|
+
if (node.children) return node.children.map(stringifyNode).join('');
|
|
9
|
+
return '';
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function extractSections(mdBody) {
|
|
13
|
+
const tree = unified().use(remarkParse).parse(mdBody);
|
|
14
|
+
const sections = [];
|
|
15
|
+
let currentHeading = null;
|
|
16
|
+
let currentDepth = 0;
|
|
17
|
+
let currentContent = [];
|
|
18
|
+
|
|
19
|
+
const flush = () => {
|
|
20
|
+
const text = currentContent.join('\n').trim();
|
|
21
|
+
if (text) {
|
|
22
|
+
sections.push({ headingPath: currentHeading || '', headingLevel: currentDepth, content: text });
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
visit(tree, (node) => {
|
|
27
|
+
if (node.type === 'heading') {
|
|
28
|
+
flush();
|
|
29
|
+
currentDepth = node.depth;
|
|
30
|
+
const title = stringifyNode(node);
|
|
31
|
+
currentHeading = currentHeading ? `${currentHeading} > ${title}` : title;
|
|
32
|
+
currentContent = [title ? `#${'#'.repeat(node.depth - 1)} ${title}` : ''];
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
if (node.type === 'paragraph' || node.type === 'code' || node.type === 'blockquote' || node.type === 'list') {
|
|
36
|
+
currentContent.push(stringifyNode(node));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
flush();
|
|
41
|
+
|
|
42
|
+
if (sections.length === 0 && mdBody.trim()) {
|
|
43
|
+
sections.push({ headingPath: '', headingLevel: 0, content: mdBody.trim() });
|
|
44
|
+
}
|
|
45
|
+
return sections;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function chunkSections(sections, chunkSize = 1200, chunkOverlap = 120) {
|
|
49
|
+
const chunks = [];
|
|
50
|
+
for (const sec of sections) {
|
|
51
|
+
const { headingPath, headingLevel, content } = sec;
|
|
52
|
+
if (content.length <= chunkSize) {
|
|
53
|
+
chunks.push({ headingPath, headingLevel, content });
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
let start = 0;
|
|
57
|
+
while (start < content.length) {
|
|
58
|
+
let end = Math.min(start + chunkSize, content.length);
|
|
59
|
+
if (end < content.length) {
|
|
60
|
+
const lastSpace = content.lastIndexOf(' ', end);
|
|
61
|
+
if (lastSpace > start) end = lastSpace;
|
|
62
|
+
}
|
|
63
|
+
chunks.push({ headingPath, headingLevel, content: content.slice(start, end).trim() });
|
|
64
|
+
start = end - (end - start < content.length ? chunkOverlap : 0);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return chunks;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function parseAndChunk(rawContent, options = {}) {
|
|
71
|
+
const { chunkSize = 1200, chunkOverlap = 120 } = options;
|
|
72
|
+
const { frontmatter, body } = parseFrontmatter(rawContent);
|
|
73
|
+
const sections = extractSections(body);
|
|
74
|
+
const chunks = chunkSections(sections, chunkSize, chunkOverlap);
|
|
75
|
+
return { frontmatter, body, chunks };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export { extractSections, chunkSections, parseAndChunk };
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import { globSync } from 'glob';
|
|
4
|
+
import { getDb } from '../../db/sqlite.js';
|
|
5
|
+
import { indexDocument, removeDocumentFromIndex } from './indexer.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* List all Markdown file paths under home and projects memory roots (absolute).
|
|
9
|
+
*/
|
|
10
|
+
function listMarkdownPaths(config) {
|
|
11
|
+
const homeRoot = config.homeMemoryRoot;
|
|
12
|
+
const projectsRoot = config.projectsRoot;
|
|
13
|
+
const out = [];
|
|
14
|
+
if (fs.existsSync(homeRoot)) {
|
|
15
|
+
const homeFiles = globSync('**/*.md', { cwd: homeRoot });
|
|
16
|
+
out.push(...homeFiles.map((p) => path.join(homeRoot, p)));
|
|
17
|
+
}
|
|
18
|
+
if (fs.existsSync(projectsRoot)) {
|
|
19
|
+
const projectFiles = globSync('**/memory/**/*.md', { cwd: projectsRoot });
|
|
20
|
+
out.push(...projectFiles.map((p) => path.join(projectsRoot, p)));
|
|
21
|
+
}
|
|
22
|
+
return out;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Clear entire text index (documents, chunks, tags, FTS). Use before full rebuild.
|
|
27
|
+
*/
|
|
28
|
+
function clearIndex(config) {
|
|
29
|
+
const db = getDb(config);
|
|
30
|
+
db.transaction(() => {
|
|
31
|
+
const ftsRowids = db.prepare('SELECT rowid FROM document_chunks_fts').all();
|
|
32
|
+
for (const { rowid } of ftsRowids) {
|
|
33
|
+
db.prepare('DELETE FROM document_chunks_fts WHERE rowid = ?').run(rowid);
|
|
34
|
+
}
|
|
35
|
+
db.prepare('DELETE FROM document_chunks').run();
|
|
36
|
+
db.prepare('DELETE FROM document_tags').run();
|
|
37
|
+
db.prepare('DELETE FROM documents').run();
|
|
38
|
+
})();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Full reindex: clear index then index every .md file under home and projects.
|
|
43
|
+
* Use for rebuild-from-scratch (e.g. after deleting SQLite).
|
|
44
|
+
*/
|
|
45
|
+
function fullReindex(config) {
|
|
46
|
+
clearIndex(config);
|
|
47
|
+
const paths = listMarkdownPaths(config);
|
|
48
|
+
let indexed = 0;
|
|
49
|
+
for (const filePath of paths) {
|
|
50
|
+
try {
|
|
51
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
52
|
+
const stat = fs.statSync(filePath);
|
|
53
|
+
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
54
|
+
indexed++;
|
|
55
|
+
} catch (err) {
|
|
56
|
+
// skip invalid or unreadable files
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return { indexed, total: paths.length };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Incremental reindex: index all found .md files, then remove from index any document
|
|
64
|
+
* whose path is no longer present on disk (manual delete). Does not clear the index first.
|
|
65
|
+
*/
|
|
66
|
+
function incrementalReindex(config) {
|
|
67
|
+
const paths = listMarkdownPaths(config);
|
|
68
|
+
const pathsSet = new Set(paths.map((p) => path.normalize(p)));
|
|
69
|
+
let indexed = 0;
|
|
70
|
+
for (const filePath of paths) {
|
|
71
|
+
try {
|
|
72
|
+
const raw = fs.readFileSync(filePath, 'utf8');
|
|
73
|
+
const stat = fs.statSync(filePath);
|
|
74
|
+
indexDocument(config, filePath, raw, stat.mtimeMs);
|
|
75
|
+
indexed++;
|
|
76
|
+
} catch (_) {}
|
|
77
|
+
}
|
|
78
|
+
const db = getDb(config);
|
|
79
|
+
const allDocs = db.prepare('SELECT id, path FROM documents').all();
|
|
80
|
+
let removed = 0;
|
|
81
|
+
for (const row of allDocs) {
|
|
82
|
+
const norm = path.normalize(row.path);
|
|
83
|
+
if (!pathsSet.has(norm)) {
|
|
84
|
+
removeDocumentFromIndex(config, row.id);
|
|
85
|
+
removed++;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return { indexed, removed, totalPaths: paths.length };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Get document id by file path (for watcher on delete).
|
|
93
|
+
*/
|
|
94
|
+
function getDocumentIdByPath(config, filePath) {
|
|
95
|
+
const db = getDb(config);
|
|
96
|
+
const norm = path.normalize(filePath);
|
|
97
|
+
const row = db.prepare('SELECT id FROM documents WHERE path = ?').get(norm);
|
|
98
|
+
return row?.id ?? null;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export { listMarkdownPaths, clearIndex, fullReindex, incrementalReindex, getDocumentIdByPath };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { getDb } from '../../db/sqlite.js';
|
|
2
|
+
import { promptId } from '../../utils/ids.js';
|
|
3
|
+
|
|
4
|
+
function savePrompt(config, options) {
|
|
5
|
+
const { session_id, project = null, content } = options;
|
|
6
|
+
const id = options.id || promptId();
|
|
7
|
+
const db = getDb(config);
|
|
8
|
+
const now = new Date().toISOString();
|
|
9
|
+
db.prepare(
|
|
10
|
+
'INSERT INTO prompts (id, session_id, project, content, created_at) VALUES (?, ?, ?, ?, ?)'
|
|
11
|
+
).run(id, session_id, project, content ?? '', now);
|
|
12
|
+
return { id, created_at: now };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { savePrompt };
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
import { getDb } from '../../db/sqlite.js';
|
|
2
|
+
import { normalizeForFts } from '../indexing/indexer.js';
|
|
3
|
+
function getBm25Weights(config) {
|
|
4
|
+
const w = config.indexing?.ftsWeights || {};
|
|
5
|
+
return [
|
|
6
|
+
w.title ?? 10,
|
|
7
|
+
w.topicKey ?? 8,
|
|
8
|
+
1, 1, 1,
|
|
9
|
+
w.heading ?? 5,
|
|
10
|
+
w.body ?? 1
|
|
11
|
+
];
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Normalize query for FTS5: same as indexed text (NFD, remove diacritics, lowercase).
|
|
16
|
+
* Leaves FTS5 operators (AND, OR, NOT, "phrase") intact.
|
|
17
|
+
*/
|
|
18
|
+
function normalizeQuery(q) {
|
|
19
|
+
if (q == null || typeof q !== 'string') return '';
|
|
20
|
+
return q.trim();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* @param {object} config - resolved config
|
|
25
|
+
* @param {object} options
|
|
26
|
+
* @param {string} options.query - search query (will be normalized for token match)
|
|
27
|
+
* @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
|
|
28
|
+
* @param {string} [options.project] - project name (required for project-only and merged)
|
|
29
|
+
* @param {number} [options.limit] - max results
|
|
30
|
+
* @returns {Array<{ documentId: string, chunkId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, headingPath: string, snippet: string, score: number }>}
|
|
31
|
+
*/
|
|
32
|
+
function search(config, options = {}) {
|
|
33
|
+
const {
|
|
34
|
+
query,
|
|
35
|
+
scopeMode = config.search?.defaultScopeMode || 'merged',
|
|
36
|
+
project = null,
|
|
37
|
+
limit = config.search?.defaultLimit ?? 10
|
|
38
|
+
} = options;
|
|
39
|
+
|
|
40
|
+
if (!query || !String(query).trim()) {
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const db = getDb(config);
|
|
45
|
+
const q = normalizeQuery(query);
|
|
46
|
+
const weights = getBm25Weights(config);
|
|
47
|
+
const bm25Args = weights.map((w, i) => (i === 0 ? w : `, ${w}`)).join('');
|
|
48
|
+
|
|
49
|
+
let scopeCondition = '';
|
|
50
|
+
const params = [q];
|
|
51
|
+
|
|
52
|
+
if (scopeMode === 'project-only') {
|
|
53
|
+
if (!project) return [];
|
|
54
|
+
scopeCondition = ' AND d.scope = ? AND d.project = ? AND d.deleted_at IS NULL';
|
|
55
|
+
params.push('project', project);
|
|
56
|
+
} else if (scopeMode === 'home-only') {
|
|
57
|
+
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
58
|
+
params.push('home');
|
|
59
|
+
} else {
|
|
60
|
+
// merged: project or home; when project given, prefer project
|
|
61
|
+
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
62
|
+
params.push('home', 'project', project || '');
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
params.push(project || ''); // ORDER BY: project first in merged, else no match
|
|
66
|
+
const ftsLimit = config.embeddings?.enabled ? Math.max(limit * 3, 50) : limit;
|
|
67
|
+
params.push(ftsLimit);
|
|
68
|
+
|
|
69
|
+
const sql = `
|
|
70
|
+
SELECT
|
|
71
|
+
f.chunk_id AS chunkId,
|
|
72
|
+
f.document_id AS documentId,
|
|
73
|
+
d.title AS title,
|
|
74
|
+
d.topic_key AS topicKey,
|
|
75
|
+
d.type AS type,
|
|
76
|
+
d.scope AS scope,
|
|
77
|
+
d.project AS project,
|
|
78
|
+
c.heading_path AS headingPath,
|
|
79
|
+
c.content AS content,
|
|
80
|
+
snippet(document_chunks_fts, 0, '<b>', '</b>', '...', 24) AS snippetTitle,
|
|
81
|
+
snippet(document_chunks_fts, 6, '<b>', '</b>', '...', 32) AS snippetBody,
|
|
82
|
+
bm25(document_chunks_fts, ${bm25Args}) AS rank
|
|
83
|
+
FROM document_chunks_fts f
|
|
84
|
+
JOIN documents d ON d.id = f.document_id
|
|
85
|
+
JOIN document_chunks c ON c.id = f.chunk_id
|
|
86
|
+
WHERE document_chunks_fts MATCH ? ${scopeCondition}
|
|
87
|
+
ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
|
|
88
|
+
LIMIT ?
|
|
89
|
+
`;
|
|
90
|
+
|
|
91
|
+
const stmt = db.prepare(sql);
|
|
92
|
+
const rows = stmt.all(...params);
|
|
93
|
+
|
|
94
|
+
const mapped = rows.map((r) => {
|
|
95
|
+
const snippet = [r.snippetTitle, r.snippetBody].filter(Boolean).join(' … ') || r.content?.slice(0, 200) || '';
|
|
96
|
+
return {
|
|
97
|
+
documentId: r.documentId,
|
|
98
|
+
chunkId: r.chunkId,
|
|
99
|
+
title: r.title,
|
|
100
|
+
topicKey: r.topicKey,
|
|
101
|
+
type: r.type,
|
|
102
|
+
scope: r.scope,
|
|
103
|
+
project: r.project,
|
|
104
|
+
headingPath: r.headingPath,
|
|
105
|
+
snippet: snippet.trim(),
|
|
106
|
+
score: -Number(r.rank), // bm25 returns negative (lower = better), we expose positive
|
|
107
|
+
_rank: r.rank
|
|
108
|
+
};
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
if (!config.embeddings?.enabled || mapped.length === 0) {
|
|
112
|
+
return mapped.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return hybridSearch(config, { query, scopeMode, project, limit, ftsRows: mapped });
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function hybridSearch(config, { query, scopeMode, project, limit, ftsRows }) {
|
|
119
|
+
const ftsWeight = config.search?.ftsWeight ?? 0.7;
|
|
120
|
+
const semanticWeight = config.search?.semanticWeight ?? 0.3;
|
|
121
|
+
let embedder;
|
|
122
|
+
let vectorStore;
|
|
123
|
+
try {
|
|
124
|
+
const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
|
|
125
|
+
import('../embeddings/embedder.js'),
|
|
126
|
+
import('../embeddings/vectorStore.js')
|
|
127
|
+
]);
|
|
128
|
+
embedder = await getEmbedder(config);
|
|
129
|
+
vectorStore = await getVectorStore(config);
|
|
130
|
+
} catch {
|
|
131
|
+
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
132
|
+
}
|
|
133
|
+
if (!embedder || !vectorStore) {
|
|
134
|
+
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
135
|
+
}
|
|
136
|
+
let semanticHits;
|
|
137
|
+
try {
|
|
138
|
+
const queryEmbedding = await embedder.embed(query);
|
|
139
|
+
semanticHits = await vectorStore.search(config, queryEmbedding, Math.max(limit * 2, 30));
|
|
140
|
+
} catch {
|
|
141
|
+
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
142
|
+
}
|
|
143
|
+
const byChunk = new Map(ftsRows.map((r) => [r.chunkId, { ...r }]));
|
|
144
|
+
const ftsScores = ftsRows.map((r) => -Number(r._rank));
|
|
145
|
+
const minFts = Math.min(...ftsScores);
|
|
146
|
+
const maxFts = Math.max(...ftsScores);
|
|
147
|
+
const rangeFts = maxFts - minFts || 1;
|
|
148
|
+
for (const row of ftsRows) {
|
|
149
|
+
const normFts = (minFts === maxFts) ? 1 : ((-Number(row._rank)) - minFts) / rangeFts;
|
|
150
|
+
const sem = semanticHits.find((h) => h.chunk_id === row.chunkId);
|
|
151
|
+
const similarity = sem ? 1 - sem.distance : 0;
|
|
152
|
+
const combined = ftsWeight * normFts + semanticWeight * similarity;
|
|
153
|
+
byChunk.get(row.chunkId).score = combined;
|
|
154
|
+
}
|
|
155
|
+
for (const sem of semanticHits) {
|
|
156
|
+
if (!byChunk.has(sem.chunk_id)) {
|
|
157
|
+
const db = getDb(config);
|
|
158
|
+
const r = db.prepare(`
|
|
159
|
+
SELECT c.id AS chunkId, c.document_id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
|
|
160
|
+
FROM document_chunks c
|
|
161
|
+
JOIN documents d ON d.id = c.document_id
|
|
162
|
+
WHERE c.id = ?
|
|
163
|
+
`).get(sem.chunk_id);
|
|
164
|
+
if (r) {
|
|
165
|
+
const similarity = 1 - sem.distance;
|
|
166
|
+
byChunk.set(sem.chunk_id, {
|
|
167
|
+
documentId: r.documentId,
|
|
168
|
+
chunkId: r.chunkId,
|
|
169
|
+
title: r.title,
|
|
170
|
+
topicKey: r.topicKey,
|
|
171
|
+
type: r.type,
|
|
172
|
+
scope: r.scope,
|
|
173
|
+
project: r.project,
|
|
174
|
+
headingPath: r.headingPath,
|
|
175
|
+
snippet: (r.content || '').slice(0, 200).trim(),
|
|
176
|
+
score: semanticWeight * similarity
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return Array.from(byChunk.values())
|
|
182
|
+
.sort((a, b) => b.score - a.score)
|
|
183
|
+
.slice(0, limit)
|
|
184
|
+
.map(({ _rank, ...r }) => r);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Get recent context without FTS (for mem_context when no query).
|
|
189
|
+
*/
|
|
190
|
+
function getRecentContext(config, options = {}) {
|
|
191
|
+
const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
|
|
192
|
+
const db = getDb(config);
|
|
193
|
+
let sql = `
|
|
194
|
+
SELECT c.id AS chunkId, d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
|
|
195
|
+
FROM document_chunks c
|
|
196
|
+
JOIN documents d ON d.id = c.document_id
|
|
197
|
+
WHERE d.deleted_at IS NULL
|
|
198
|
+
`;
|
|
199
|
+
const params = [];
|
|
200
|
+
if (project) {
|
|
201
|
+
sql += ' AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
202
|
+
params.push('home', 'project', project);
|
|
203
|
+
} else {
|
|
204
|
+
sql += ' AND d.scope = ?';
|
|
205
|
+
params.push('home');
|
|
206
|
+
}
|
|
207
|
+
sql += ' ORDER BY d.updated_at DESC LIMIT ?';
|
|
208
|
+
params.push(limit);
|
|
209
|
+
const rows = db.prepare(sql).all(...params);
|
|
210
|
+
return rows.map((r) => ({
|
|
211
|
+
documentId: r.documentId,
|
|
212
|
+
chunkId: r.chunkId,
|
|
213
|
+
title: r.title,
|
|
214
|
+
topicKey: r.topicKey,
|
|
215
|
+
type: r.type,
|
|
216
|
+
scope: r.scope,
|
|
217
|
+
project: r.project,
|
|
218
|
+
headingPath: r.headingPath,
|
|
219
|
+
snippet: (r.content || '').slice(0, 300).trim()
|
|
220
|
+
}));
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export { search, getRecentContext, normalizeQuery, getBm25Weights };
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { getDb } from '../../db/sqlite.js';
|
|
2
|
+
import { sessionId } from '../../utils/ids.js';
|
|
3
|
+
|
|
4
|
+
function startSession(config, options = {}) {
|
|
5
|
+
const id = options.id || sessionId();
|
|
6
|
+
const { project, directory } = options;
|
|
7
|
+
const db = getDb(config);
|
|
8
|
+
const now = new Date().toISOString();
|
|
9
|
+
db.prepare(
|
|
10
|
+
'INSERT INTO sessions (id, project, directory, started_at) VALUES (?, ?, ?, ?)'
|
|
11
|
+
).run(id, project || '', directory ?? null, now);
|
|
12
|
+
return { id, started_at: now };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function endSession(config, sessionIdParam) {
|
|
16
|
+
const db = getDb(config);
|
|
17
|
+
const now = new Date().toISOString();
|
|
18
|
+
const r = db.prepare('UPDATE sessions SET ended_at = ? WHERE id = ?').run(now, sessionIdParam);
|
|
19
|
+
return { success: r.changes > 0, ended_at: now };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function getSession(config, sessionIdParam) {
|
|
23
|
+
const db = getDb(config);
|
|
24
|
+
return db.prepare('SELECT * FROM sessions WHERE id = ?').get(sessionIdParam) ?? null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export { startSession, endSession, getSession };
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
-- documents
|
|
2
|
+
CREATE TABLE IF NOT EXISTS documents (
|
|
3
|
+
id TEXT PRIMARY KEY,
|
|
4
|
+
path TEXT NOT NULL UNIQUE,
|
|
5
|
+
title TEXT,
|
|
6
|
+
type TEXT NOT NULL,
|
|
7
|
+
scope TEXT NOT NULL,
|
|
8
|
+
project TEXT,
|
|
9
|
+
topic_key TEXT,
|
|
10
|
+
session_id TEXT,
|
|
11
|
+
tool_name TEXT,
|
|
12
|
+
content_hash TEXT NOT NULL,
|
|
13
|
+
file_mtime_ms INTEGER,
|
|
14
|
+
created_at TEXT NOT NULL,
|
|
15
|
+
updated_at TEXT NOT NULL,
|
|
16
|
+
indexed_at TEXT,
|
|
17
|
+
deleted_at TEXT,
|
|
18
|
+
revision_count INTEGER NOT NULL DEFAULT 1,
|
|
19
|
+
duplicate_count INTEGER NOT NULL DEFAULT 0,
|
|
20
|
+
embedding_status TEXT NOT NULL DEFAULT 'disabled'
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS document_chunks (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
document_id TEXT NOT NULL,
|
|
26
|
+
chunk_index INTEGER NOT NULL,
|
|
27
|
+
heading_path TEXT,
|
|
28
|
+
heading_level INTEGER,
|
|
29
|
+
content TEXT NOT NULL,
|
|
30
|
+
token_count INTEGER,
|
|
31
|
+
content_hash TEXT NOT NULL,
|
|
32
|
+
created_at TEXT NOT NULL,
|
|
33
|
+
updated_at TEXT NOT NULL,
|
|
34
|
+
FOREIGN KEY(document_id) REFERENCES documents(id)
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
CREATE TABLE IF NOT EXISTS document_tags (
|
|
38
|
+
document_id TEXT NOT NULL,
|
|
39
|
+
tag TEXT NOT NULL,
|
|
40
|
+
PRIMARY KEY (document_id, tag),
|
|
41
|
+
FOREIGN KEY(document_id) REFERENCES documents(id)
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
45
|
+
id TEXT PRIMARY KEY,
|
|
46
|
+
project TEXT NOT NULL,
|
|
47
|
+
directory TEXT,
|
|
48
|
+
started_at TEXT NOT NULL,
|
|
49
|
+
ended_at TEXT,
|
|
50
|
+
summary TEXT
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
CREATE TABLE IF NOT EXISTS prompts (
|
|
54
|
+
id TEXT PRIMARY KEY,
|
|
55
|
+
session_id TEXT NOT NULL,
|
|
56
|
+
project TEXT,
|
|
57
|
+
content TEXT NOT NULL,
|
|
58
|
+
created_at TEXT NOT NULL,
|
|
59
|
+
FOREIGN KEY(session_id) REFERENCES sessions(id)
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
CREATE TABLE IF NOT EXISTS embedding_jobs (
|
|
63
|
+
id TEXT PRIMARY KEY,
|
|
64
|
+
chunk_id TEXT NOT NULL,
|
|
65
|
+
model_name TEXT NOT NULL,
|
|
66
|
+
status TEXT NOT NULL,
|
|
67
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
68
|
+
last_error TEXT,
|
|
69
|
+
created_at TEXT NOT NULL,
|
|
70
|
+
updated_at TEXT NOT NULL,
|
|
71
|
+
FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
-- FTS5: content must be not null for columns; use empty string for missing
|
|
75
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS document_chunks_fts USING fts5(
|
|
76
|
+
chunk_id UNINDEXED,
|
|
77
|
+
document_id UNINDEXED,
|
|
78
|
+
title,
|
|
79
|
+
topic_key,
|
|
80
|
+
type,
|
|
81
|
+
scope,
|
|
82
|
+
project,
|
|
83
|
+
heading_path,
|
|
84
|
+
body,
|
|
85
|
+
tokenize = 'unicode61 remove_diacritics 2'
|
|
86
|
+
);
|