mixgram 1.1.6 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +16 -3
- package/spec/bench-indexing.js +219 -0
- package/spec/bench-search.js +221 -0
- package/spec/run-tests.js +12 -7
- package/spec/scenarios/scenarios.js +5 -3
- package/spec/test-large-document.js +98 -0
- package/src/config.js +7 -4
- package/src/core/documents/documents.js +2 -2
- package/src/core/embeddings/queue.js +10 -10
- package/src/core/embeddings/vectorStore.js +1 -2
- package/src/core/indexing/indexer.js +45 -62
- package/src/core/indexing/parser.js +105 -58
- package/src/core/indexing/reindex.js +17 -8
- package/src/core/search/search.js +26 -126
- package/src/db/migrations/001_initial.sql +14 -26
- package/src/utils/markdown.js +42 -2
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { getDb } from '../../db/sqlite.js';
|
|
2
2
|
import { normalizeForFts } from '../indexing/indexer.js';
|
|
3
|
+
|
|
3
4
|
function getBm25Weights(config) {
|
|
4
5
|
const w = config.indexing?.ftsWeights || {};
|
|
5
6
|
return [
|
|
6
7
|
w.title ?? 10,
|
|
7
|
-
w.
|
|
8
|
-
|
|
9
|
-
w.
|
|
8
|
+
w.h1 ?? 8,
|
|
9
|
+
w.h2 ?? 6,
|
|
10
|
+
w.h3 ?? 5,
|
|
11
|
+
w.h4 ?? 4,
|
|
12
|
+
w.h5 ?? 3,
|
|
13
|
+
w.h6 ?? 2,
|
|
10
14
|
w.body ?? 1
|
|
11
15
|
];
|
|
12
16
|
}
|
|
@@ -17,7 +21,7 @@ function getBm25Weights(config) {
|
|
|
17
21
|
*/
|
|
18
22
|
function normalizeQuery(q) {
|
|
19
23
|
if (q == null || typeof q !== 'string') return '';
|
|
20
|
-
return q.trim();
|
|
24
|
+
return normalizeForFts(q.trim());
|
|
21
25
|
}
|
|
22
26
|
|
|
23
27
|
/**
|
|
@@ -27,7 +31,7 @@ function normalizeQuery(q) {
|
|
|
27
31
|
* @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
|
|
28
32
|
* @param {string} [options.project] - project name (required for project-only and merged)
|
|
29
33
|
* @param {number} [options.limit] - max results
|
|
30
|
-
* @returns {Array<{ documentId: string,
|
|
34
|
+
* @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, snippet: string, score: number }>}
|
|
31
35
|
*/
|
|
32
36
|
function search(config, options = {}) {
|
|
33
37
|
const {
|
|
@@ -57,33 +61,28 @@ function search(config, options = {}) {
|
|
|
57
61
|
scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
|
|
58
62
|
params.push('home');
|
|
59
63
|
} else {
|
|
60
|
-
// merged: project or home; when project given, prefer project
|
|
61
64
|
scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
|
|
62
65
|
params.push('home', 'project', project || '');
|
|
63
66
|
}
|
|
64
67
|
|
|
65
|
-
params.push(project || '');
|
|
66
|
-
|
|
67
|
-
params.push(ftsLimit);
|
|
68
|
+
params.push(project || '');
|
|
69
|
+
params.push(limit);
|
|
68
70
|
|
|
69
71
|
const sql = `
|
|
70
72
|
SELECT
|
|
71
|
-
f.chunk_id AS chunkId,
|
|
72
73
|
f.document_id AS documentId,
|
|
73
74
|
d.title AS title,
|
|
74
75
|
d.topic_key AS topicKey,
|
|
75
76
|
d.type AS type,
|
|
76
77
|
d.scope AS scope,
|
|
77
78
|
d.project AS project,
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
snippet(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
FROM document_chunks_fts f
|
|
79
|
+
d.body AS body,
|
|
80
|
+
snippet(document_fts, 1, '<b>', '</b>', '...', 24) AS snippetTitle,
|
|
81
|
+
snippet(document_fts, 8, '<b>', '</b>', '...', 64) AS snippetBody,
|
|
82
|
+
bm25(document_fts, ${bm25Args}) AS rank
|
|
83
|
+
FROM document_fts f
|
|
84
84
|
JOIN documents d ON d.id = f.document_id
|
|
85
|
-
|
|
86
|
-
WHERE document_chunks_fts MATCH ? ${scopeCondition}
|
|
85
|
+
WHERE document_fts MATCH ? ${scopeCondition}
|
|
87
86
|
ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
|
|
88
87
|
LIMIT ?
|
|
89
88
|
`;
|
|
@@ -91,117 +90,21 @@ function search(config, options = {}) {
|
|
|
91
90
|
const stmt = db.prepare(sql);
|
|
92
91
|
const rows = stmt.all(...params);
|
|
93
92
|
|
|
94
|
-
|
|
95
|
-
const
|
|
93
|
+
return rows.map((r) => {
|
|
94
|
+
const fromFts = [r.snippetTitle, r.snippetBody].filter((x) => x != null && x !== '').join(' … ');
|
|
95
|
+
const fallback = r.body != null ? String(r.body).slice(0, 200) : '';
|
|
96
|
+
const snippet = (fromFts && fromFts.trim()) || fallback || '';
|
|
96
97
|
return {
|
|
97
98
|
documentId: r.documentId,
|
|
98
|
-
chunkId: r.chunkId,
|
|
99
99
|
title: r.title,
|
|
100
100
|
topicKey: r.topicKey,
|
|
101
101
|
type: r.type,
|
|
102
102
|
scope: r.scope,
|
|
103
103
|
project: r.project,
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
score: -Number(r.rank), // bm25 returns negative (lower = better), we expose positive
|
|
107
|
-
_rank: r.rank
|
|
104
|
+
snippet: String(snippet).trim(),
|
|
105
|
+
score: -Number(r.rank)
|
|
108
106
|
};
|
|
109
107
|
});
|
|
110
|
-
|
|
111
|
-
if (!config.embeddings?.enabled || mapped.length === 0) {
|
|
112
|
-
return mapped.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
return hybridSearch(config, { query, scopeMode, project, limit, ftsRows: mapped });
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
async function hybridSearch(config, { query, scopeMode, project, limit, ftsRows }) {
|
|
119
|
-
const ftsWeight = config.search?.ftsWeight ?? 0.7;
|
|
120
|
-
const semanticWeight = config.search?.semanticWeight ?? 0.3;
|
|
121
|
-
let queryEmbedding;
|
|
122
|
-
if (typeof config.getQueryEmbedding === 'function') {
|
|
123
|
-
try {
|
|
124
|
-
queryEmbedding = await config.getQueryEmbedding(query);
|
|
125
|
-
} catch {
|
|
126
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
127
|
-
}
|
|
128
|
-
} else {
|
|
129
|
-
let embedder;
|
|
130
|
-
try {
|
|
131
|
-
const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
|
|
132
|
-
import('../embeddings/embedder.js'),
|
|
133
|
-
import('../embeddings/vectorStore.js')
|
|
134
|
-
]);
|
|
135
|
-
embedder = await getEmbedder(config);
|
|
136
|
-
if (!embedder) {
|
|
137
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
138
|
-
}
|
|
139
|
-
queryEmbedding = await embedder.embed(query);
|
|
140
|
-
} catch {
|
|
141
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
if (!queryEmbedding) {
|
|
145
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
146
|
-
}
|
|
147
|
-
let vectorStore;
|
|
148
|
-
try {
|
|
149
|
-
const { getVectorStore } = await import('../embeddings/vectorStore.js');
|
|
150
|
-
vectorStore = await getVectorStore(config);
|
|
151
|
-
} catch {
|
|
152
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
153
|
-
}
|
|
154
|
-
if (!vectorStore) {
|
|
155
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
156
|
-
}
|
|
157
|
-
let semanticHits;
|
|
158
|
-
try {
|
|
159
|
-
semanticHits = await vectorStore.search(config, queryEmbedding, Math.max(limit * 2, 30));
|
|
160
|
-
} catch {
|
|
161
|
-
return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
|
|
162
|
-
}
|
|
163
|
-
const byChunk = new Map(ftsRows.map((r) => [r.chunkId, { ...r }]));
|
|
164
|
-
const ftsScores = ftsRows.map((r) => -Number(r._rank));
|
|
165
|
-
const minFts = Math.min(...ftsScores);
|
|
166
|
-
const maxFts = Math.max(...ftsScores);
|
|
167
|
-
const rangeFts = maxFts - minFts || 1;
|
|
168
|
-
for (const row of ftsRows) {
|
|
169
|
-
const normFts = (minFts === maxFts) ? 1 : ((-Number(row._rank)) - minFts) / rangeFts;
|
|
170
|
-
const sem = semanticHits.find((h) => h.chunk_id === row.chunkId);
|
|
171
|
-
const similarity = sem ? 1 - sem.distance : 0;
|
|
172
|
-
const combined = ftsWeight * normFts + semanticWeight * similarity;
|
|
173
|
-
byChunk.get(row.chunkId).score = combined;
|
|
174
|
-
}
|
|
175
|
-
for (const sem of semanticHits) {
|
|
176
|
-
if (!byChunk.has(sem.chunk_id)) {
|
|
177
|
-
const db = getDb(config);
|
|
178
|
-
const r = db.prepare(`
|
|
179
|
-
SELECT c.id AS chunkId, c.document_id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
|
|
180
|
-
FROM document_chunks c
|
|
181
|
-
JOIN documents d ON d.id = c.document_id
|
|
182
|
-
WHERE c.id = ?
|
|
183
|
-
`).get(sem.chunk_id);
|
|
184
|
-
if (r) {
|
|
185
|
-
const similarity = 1 - sem.distance;
|
|
186
|
-
byChunk.set(sem.chunk_id, {
|
|
187
|
-
documentId: r.documentId,
|
|
188
|
-
chunkId: r.chunkId,
|
|
189
|
-
title: r.title,
|
|
190
|
-
topicKey: r.topicKey,
|
|
191
|
-
type: r.type,
|
|
192
|
-
scope: r.scope,
|
|
193
|
-
project: r.project,
|
|
194
|
-
headingPath: r.headingPath,
|
|
195
|
-
snippet: (r.content || '').slice(0, 200).trim(),
|
|
196
|
-
score: semanticWeight * similarity
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
return Array.from(byChunk.values())
|
|
202
|
-
.sort((a, b) => b.score - a.score)
|
|
203
|
-
.slice(0, limit)
|
|
204
|
-
.map(({ _rank, ...r }) => r);
|
|
205
108
|
}
|
|
206
109
|
|
|
207
110
|
/**
|
|
@@ -211,9 +114,8 @@ function getRecentContext(config, options = {}) {
|
|
|
211
114
|
const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
|
|
212
115
|
const db = getDb(config);
|
|
213
116
|
let sql = `
|
|
214
|
-
SELECT
|
|
215
|
-
FROM
|
|
216
|
-
JOIN documents d ON d.id = c.document_id
|
|
117
|
+
SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.body
|
|
118
|
+
FROM documents d
|
|
217
119
|
WHERE d.deleted_at IS NULL
|
|
218
120
|
`;
|
|
219
121
|
const params = [];
|
|
@@ -229,14 +131,12 @@ function getRecentContext(config, options = {}) {
|
|
|
229
131
|
const rows = db.prepare(sql).all(...params);
|
|
230
132
|
return rows.map((r) => ({
|
|
231
133
|
documentId: r.documentId,
|
|
232
|
-
chunkId: r.chunkId,
|
|
233
134
|
title: r.title,
|
|
234
135
|
topicKey: r.topicKey,
|
|
235
136
|
type: r.type,
|
|
236
137
|
scope: r.scope,
|
|
237
138
|
project: r.project,
|
|
238
|
-
|
|
239
|
-
snippet: (r.content || '').slice(0, 300).trim()
|
|
139
|
+
snippet: (r.body || '').slice(0, 300).trim()
|
|
240
140
|
}));
|
|
241
141
|
}
|
|
242
142
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
-- documents
|
|
1
|
+
-- documents (one row per doc; body and section tree stored here)
|
|
2
2
|
CREATE TABLE IF NOT EXISTS documents (
|
|
3
3
|
id TEXT PRIMARY KEY,
|
|
4
4
|
path TEXT NOT NULL UNIQUE,
|
|
@@ -17,21 +17,10 @@ CREATE TABLE IF NOT EXISTS documents (
|
|
|
17
17
|
deleted_at TEXT,
|
|
18
18
|
revision_count INTEGER NOT NULL DEFAULT 1,
|
|
19
19
|
duplicate_count INTEGER NOT NULL DEFAULT 0,
|
|
20
|
-
embedding_status TEXT NOT NULL DEFAULT 'disabled'
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
id TEXT PRIMARY KEY,
|
|
25
|
-
document_id TEXT NOT NULL,
|
|
26
|
-
chunk_index INTEGER NOT NULL,
|
|
27
|
-
heading_path TEXT,
|
|
28
|
-
heading_level INTEGER,
|
|
29
|
-
content TEXT NOT NULL,
|
|
30
|
-
token_count INTEGER,
|
|
31
|
-
content_hash TEXT NOT NULL,
|
|
32
|
-
created_at TEXT NOT NULL,
|
|
33
|
-
updated_at TEXT NOT NULL,
|
|
34
|
-
FOREIGN KEY(document_id) REFERENCES documents(id)
|
|
20
|
+
embedding_status TEXT NOT NULL DEFAULT 'disabled',
|
|
21
|
+
body TEXT,
|
|
22
|
+
structure TEXT,
|
|
23
|
+
sections_index TEXT
|
|
35
24
|
);
|
|
36
25
|
|
|
37
26
|
CREATE TABLE IF NOT EXISTS document_tags (
|
|
@@ -67,20 +56,19 @@ CREATE TABLE IF NOT EXISTS embedding_jobs (
|
|
|
67
56
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
68
57
|
last_error TEXT,
|
|
69
58
|
created_at TEXT NOT NULL,
|
|
70
|
-
updated_at TEXT NOT NULL
|
|
71
|
-
FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
|
|
59
|
+
updated_at TEXT NOT NULL
|
|
72
60
|
);
|
|
73
61
|
|
|
74
|
-
-- FTS5:
|
|
75
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS
|
|
76
|
-
chunk_id UNINDEXED,
|
|
62
|
+
-- FTS5: one row per document (title, h1–h6, body)
|
|
63
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
|
|
77
64
|
document_id UNINDEXED,
|
|
78
65
|
title,
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
66
|
+
h1,
|
|
67
|
+
h2,
|
|
68
|
+
h3,
|
|
69
|
+
h4,
|
|
70
|
+
h5,
|
|
71
|
+
h6,
|
|
84
72
|
body,
|
|
85
73
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
86
74
|
);
|
package/src/utils/markdown.js
CHANGED
|
@@ -6,6 +6,29 @@ export const FRONTMATTER_KEYS = [
|
|
|
6
6
|
'embedding_status', 'indexed_at', 'deleted_at'
|
|
7
7
|
];
|
|
8
8
|
|
|
9
|
+
/** Keys written to .md frontmatter (needed for re-index, scope/project filtering, timeline). */
|
|
10
|
+
const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'scope', 'project', 'topic_key', 'session_id', 'created'];
|
|
11
|
+
|
|
12
|
+
function formatCreated(createdAt) {
|
|
13
|
+
if (!createdAt) return null;
|
|
14
|
+
const d = new Date(createdAt);
|
|
15
|
+
if (Number.isNaN(d.getTime())) return createdAt;
|
|
16
|
+
const y = d.getFullYear();
|
|
17
|
+
const m = String(d.getMonth() + 1).padStart(2, '0');
|
|
18
|
+
const day = String(d.getDate()).padStart(2, '0');
|
|
19
|
+
const h = String(d.getHours()).padStart(2, '0');
|
|
20
|
+
const min = String(d.getMinutes()).padStart(2, '0');
|
|
21
|
+
const s = String(d.getSeconds()).padStart(2, '0');
|
|
22
|
+
return `${y}-${m}-${day} ${h}:${min}:${s}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function parseCreated(value) {
|
|
26
|
+
if (!value) return null;
|
|
27
|
+
if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value)) return value;
|
|
28
|
+
const d = new Date(value);
|
|
29
|
+
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
|
30
|
+
}
|
|
31
|
+
|
|
9
32
|
function serializeFrontmatter(fm) {
|
|
10
33
|
const obj = {};
|
|
11
34
|
for (const k of FRONTMATTER_KEYS) {
|
|
@@ -16,9 +39,26 @@ function serializeFrontmatter(fm) {
|
|
|
16
39
|
return obj;
|
|
17
40
|
}
|
|
18
41
|
|
|
42
|
+
/** Minimal frontmatter for .md (id required for re-index; created in friendly format). */
|
|
43
|
+
function serializeDisplayFrontmatter(fm) {
|
|
44
|
+
const obj = {};
|
|
45
|
+
for (const k of DISPLAY_FRONTMATTER_KEYS) {
|
|
46
|
+
if (k === 'created') {
|
|
47
|
+
const v = fm.created_at != null ? formatCreated(fm.created_at) : (fm.created ?? null);
|
|
48
|
+
if (v) obj[k] = v;
|
|
49
|
+
} else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
|
|
50
|
+
obj[k] = fm[k];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
if (fm.deleted_at) obj.deleted_at = fm.deleted_at;
|
|
54
|
+
if (fm.deleted === true) obj.deleted = true;
|
|
55
|
+
return obj;
|
|
56
|
+
}
|
|
57
|
+
|
|
19
58
|
export function parse(content) {
|
|
20
59
|
const parsed = matter(content);
|
|
21
60
|
const data = parsed.data || {};
|
|
61
|
+
const createdAt = data.created_at ?? parseCreated(data.created);
|
|
22
62
|
return {
|
|
23
63
|
frontmatter: {
|
|
24
64
|
id: data.id,
|
|
@@ -29,7 +69,7 @@ export function parse(content) {
|
|
|
29
69
|
topic_key: data.topic_key,
|
|
30
70
|
session_id: data.session_id,
|
|
31
71
|
tool_name: data.tool_name,
|
|
32
|
-
created_at:
|
|
72
|
+
created_at: createdAt,
|
|
33
73
|
updated_at: data.updated_at,
|
|
34
74
|
revision_count: data.revision_count ?? 1,
|
|
35
75
|
duplicate_count: data.duplicate_count ?? 0,
|
|
@@ -44,6 +84,6 @@ export function parse(content) {
|
|
|
44
84
|
}
|
|
45
85
|
|
|
46
86
|
export function toMarkdown(frontmatter, body) {
|
|
47
|
-
const fm =
|
|
87
|
+
const fm = serializeDisplayFrontmatter(frontmatter);
|
|
48
88
|
return matter.stringify(body || '', fm, { lineWidth: -1 });
|
|
49
89
|
}
|