mixgram 1.1.6 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,16 @@
1
1
  import { getDb } from '../../db/sqlite.js';
2
2
  import { normalizeForFts } from '../indexing/indexer.js';
3
+
3
4
  function getBm25Weights(config) {
4
5
  const w = config.indexing?.ftsWeights || {};
5
6
  return [
6
7
  w.title ?? 10,
7
- w.topicKey ?? 8,
8
- 1, 1, 1,
9
- w.heading ?? 5,
8
+ w.h1 ?? 8,
9
+ w.h2 ?? 6,
10
+ w.h3 ?? 5,
11
+ w.h4 ?? 4,
12
+ w.h5 ?? 3,
13
+ w.h6 ?? 2,
10
14
  w.body ?? 1
11
15
  ];
12
16
  }
@@ -17,7 +21,7 @@ function getBm25Weights(config) {
17
21
  */
18
22
  function normalizeQuery(q) {
19
23
  if (q == null || typeof q !== 'string') return '';
20
- return q.trim();
24
+ return normalizeForFts(q.trim());
21
25
  }
22
26
 
23
27
  /**
@@ -27,7 +31,7 @@ function normalizeQuery(q) {
27
31
  * @param {string} [options.scopeMode] - 'project-only' | 'home-only' | 'merged'
28
32
  * @param {string} [options.project] - project name (required for project-only and merged)
29
33
  * @param {number} [options.limit] - max results
30
- * @returns {Array<{ documentId: string, chunkId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, headingPath: string, snippet: string, score: number }>}
34
+ * @returns {Array<{ documentId: string, title: string, topicKey: string, type: string, scope: string, project: string | null, snippet: string, score: number }>}
31
35
  */
32
36
  function search(config, options = {}) {
33
37
  const {
@@ -57,33 +61,28 @@ function search(config, options = {}) {
57
61
  scopeCondition = ' AND d.scope = ? AND d.deleted_at IS NULL';
58
62
  params.push('home');
59
63
  } else {
60
- // merged: project or home; when project given, prefer project
61
64
  scopeCondition = ' AND d.deleted_at IS NULL AND (d.scope = ? OR (d.scope = ? AND d.project = ?))';
62
65
  params.push('home', 'project', project || '');
63
66
  }
64
67
 
65
- params.push(project || ''); // ORDER BY: project first in merged, else no match
66
- const ftsLimit = config.embeddings?.enabled ? Math.max(limit * 3, 50) : limit;
67
- params.push(ftsLimit);
68
+ params.push(project || '');
69
+ params.push(limit);
68
70
 
69
71
  const sql = `
70
72
  SELECT
71
- f.chunk_id AS chunkId,
72
73
  f.document_id AS documentId,
73
74
  d.title AS title,
74
75
  d.topic_key AS topicKey,
75
76
  d.type AS type,
76
77
  d.scope AS scope,
77
78
  d.project AS project,
78
- c.heading_path AS headingPath,
79
- c.content AS content,
80
- snippet(document_chunks_fts, 0, '<b>', '</b>', '...', 24) AS snippetTitle,
81
- snippet(document_chunks_fts, 6, '<b>', '</b>', '...', 32) AS snippetBody,
82
- bm25(document_chunks_fts, ${bm25Args}) AS rank
83
- FROM document_chunks_fts f
79
+ d.body AS body,
80
+ snippet(document_fts, 1, '<b>', '</b>', '...', 24) AS snippetTitle,
81
+ snippet(document_fts, 8, '<b>', '</b>', '...', 64) AS snippetBody,
82
+ bm25(document_fts, ${bm25Args}) AS rank
83
+ FROM document_fts f
84
84
  JOIN documents d ON d.id = f.document_id
85
- JOIN document_chunks c ON c.id = f.chunk_id
86
- WHERE document_chunks_fts MATCH ? ${scopeCondition}
85
+ WHERE document_fts MATCH ? ${scopeCondition}
87
86
  ORDER BY (CASE WHEN d.scope = 'project' AND d.project = ? THEN 0 ELSE 1 END), rank
88
87
  LIMIT ?
89
88
  `;
@@ -91,117 +90,21 @@ function search(config, options = {}) {
91
90
  const stmt = db.prepare(sql);
92
91
  const rows = stmt.all(...params);
93
92
 
94
- const mapped = rows.map((r) => {
95
- const snippet = [r.snippetTitle, r.snippetBody].filter(Boolean).join(' … ') || r.content?.slice(0, 200) || '';
93
+ return rows.map((r) => {
94
+ const fromFts = [r.snippetTitle, r.snippetBody].filter((x) => x != null && x !== '').join(' … ');
95
+ const fallback = r.body != null ? String(r.body).slice(0, 200) : '';
96
+ const snippet = (fromFts && fromFts.trim()) || fallback || '';
96
97
  return {
97
98
  documentId: r.documentId,
98
- chunkId: r.chunkId,
99
99
  title: r.title,
100
100
  topicKey: r.topicKey,
101
101
  type: r.type,
102
102
  scope: r.scope,
103
103
  project: r.project,
104
- headingPath: r.headingPath,
105
- snippet: snippet.trim(),
106
- score: -Number(r.rank), // bm25 returns negative (lower = better), we expose positive
107
- _rank: r.rank
104
+ snippet: String(snippet).trim(),
105
+ score: -Number(r.rank)
108
106
  };
109
107
  });
110
-
111
- if (!config.embeddings?.enabled || mapped.length === 0) {
112
- return mapped.slice(0, limit).map(({ _rank, ...r }) => r);
113
- }
114
-
115
- return hybridSearch(config, { query, scopeMode, project, limit, ftsRows: mapped });
116
- }
117
-
118
- async function hybridSearch(config, { query, scopeMode, project, limit, ftsRows }) {
119
- const ftsWeight = config.search?.ftsWeight ?? 0.7;
120
- const semanticWeight = config.search?.semanticWeight ?? 0.3;
121
- let queryEmbedding;
122
- if (typeof config.getQueryEmbedding === 'function') {
123
- try {
124
- queryEmbedding = await config.getQueryEmbedding(query);
125
- } catch {
126
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
127
- }
128
- } else {
129
- let embedder;
130
- try {
131
- const [{ getEmbedder }, { getVectorStore }] = await Promise.all([
132
- import('../embeddings/embedder.js'),
133
- import('../embeddings/vectorStore.js')
134
- ]);
135
- embedder = await getEmbedder(config);
136
- if (!embedder) {
137
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
138
- }
139
- queryEmbedding = await embedder.embed(query);
140
- } catch {
141
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
142
- }
143
- }
144
- if (!queryEmbedding) {
145
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
146
- }
147
- let vectorStore;
148
- try {
149
- const { getVectorStore } = await import('../embeddings/vectorStore.js');
150
- vectorStore = await getVectorStore(config);
151
- } catch {
152
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
153
- }
154
- if (!vectorStore) {
155
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
156
- }
157
- let semanticHits;
158
- try {
159
- semanticHits = await vectorStore.search(config, queryEmbedding, Math.max(limit * 2, 30));
160
- } catch {
161
- return ftsRows.slice(0, limit).map(({ _rank, ...r }) => r);
162
- }
163
- const byChunk = new Map(ftsRows.map((r) => [r.chunkId, { ...r }]));
164
- const ftsScores = ftsRows.map((r) => -Number(r._rank));
165
- const minFts = Math.min(...ftsScores);
166
- const maxFts = Math.max(...ftsScores);
167
- const rangeFts = maxFts - minFts || 1;
168
- for (const row of ftsRows) {
169
- const normFts = (minFts === maxFts) ? 1 : ((-Number(row._rank)) - minFts) / rangeFts;
170
- const sem = semanticHits.find((h) => h.chunk_id === row.chunkId);
171
- const similarity = sem ? 1 - sem.distance : 0;
172
- const combined = ftsWeight * normFts + semanticWeight * similarity;
173
- byChunk.get(row.chunkId).score = combined;
174
- }
175
- for (const sem of semanticHits) {
176
- if (!byChunk.has(sem.chunk_id)) {
177
- const db = getDb(config);
178
- const r = db.prepare(`
179
- SELECT c.id AS chunkId, c.document_id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
180
- FROM document_chunks c
181
- JOIN documents d ON d.id = c.document_id
182
- WHERE c.id = ?
183
- `).get(sem.chunk_id);
184
- if (r) {
185
- const similarity = 1 - sem.distance;
186
- byChunk.set(sem.chunk_id, {
187
- documentId: r.documentId,
188
- chunkId: r.chunkId,
189
- title: r.title,
190
- topicKey: r.topicKey,
191
- type: r.type,
192
- scope: r.scope,
193
- project: r.project,
194
- headingPath: r.headingPath,
195
- snippet: (r.content || '').slice(0, 200).trim(),
196
- score: semanticWeight * similarity
197
- });
198
- }
199
- }
200
- }
201
- return Array.from(byChunk.values())
202
- .sort((a, b) => b.score - a.score)
203
- .slice(0, limit)
204
- .map(({ _rank, ...r }) => r);
205
108
  }
206
109
 
207
110
  /**
@@ -211,9 +114,8 @@ function getRecentContext(config, options = {}) {
211
114
  const { project = null, limit = config.search?.defaultLimit ?? 10 } = options;
212
115
  const db = getDb(config);
213
116
  let sql = `
214
- SELECT c.id AS chunkId, d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, c.heading_path AS headingPath, c.content
215
- FROM document_chunks c
216
- JOIN documents d ON d.id = c.document_id
117
+ SELECT d.id AS documentId, d.title, d.topic_key AS topicKey, d.type, d.scope, d.project, d.body
118
+ FROM documents d
217
119
  WHERE d.deleted_at IS NULL
218
120
  `;
219
121
  const params = [];
@@ -229,14 +131,12 @@ function getRecentContext(config, options = {}) {
229
131
  const rows = db.prepare(sql).all(...params);
230
132
  return rows.map((r) => ({
231
133
  documentId: r.documentId,
232
- chunkId: r.chunkId,
233
134
  title: r.title,
234
135
  topicKey: r.topicKey,
235
136
  type: r.type,
236
137
  scope: r.scope,
237
138
  project: r.project,
238
- headingPath: r.headingPath,
239
- snippet: (r.content || '').slice(0, 300).trim()
139
+ snippet: (r.body || '').slice(0, 300).trim()
240
140
  }));
241
141
  }
242
142
 
@@ -1,4 +1,4 @@
1
- -- documents
1
+ -- documents (one row per doc; body and section tree stored here)
2
2
  CREATE TABLE IF NOT EXISTS documents (
3
3
  id TEXT PRIMARY KEY,
4
4
  path TEXT NOT NULL UNIQUE,
@@ -17,21 +17,10 @@ CREATE TABLE IF NOT EXISTS documents (
17
17
  deleted_at TEXT,
18
18
  revision_count INTEGER NOT NULL DEFAULT 1,
19
19
  duplicate_count INTEGER NOT NULL DEFAULT 0,
20
- embedding_status TEXT NOT NULL DEFAULT 'disabled'
21
- );
22
-
23
- CREATE TABLE IF NOT EXISTS document_chunks (
24
- id TEXT PRIMARY KEY,
25
- document_id TEXT NOT NULL,
26
- chunk_index INTEGER NOT NULL,
27
- heading_path TEXT,
28
- heading_level INTEGER,
29
- content TEXT NOT NULL,
30
- token_count INTEGER,
31
- content_hash TEXT NOT NULL,
32
- created_at TEXT NOT NULL,
33
- updated_at TEXT NOT NULL,
34
- FOREIGN KEY(document_id) REFERENCES documents(id)
20
+ embedding_status TEXT NOT NULL DEFAULT 'disabled',
21
+ body TEXT,
22
+ structure TEXT,
23
+ sections_index TEXT
35
24
  );
36
25
 
37
26
  CREATE TABLE IF NOT EXISTS document_tags (
@@ -67,20 +56,19 @@ CREATE TABLE IF NOT EXISTS embedding_jobs (
67
56
  attempts INTEGER NOT NULL DEFAULT 0,
68
57
  last_error TEXT,
69
58
  created_at TEXT NOT NULL,
70
- updated_at TEXT NOT NULL,
71
- FOREIGN KEY(chunk_id) REFERENCES document_chunks(id)
59
+ updated_at TEXT NOT NULL
72
60
  );
73
61
 
74
- -- FTS5: content must be not null for columns; use empty string for missing
75
- CREATE VIRTUAL TABLE IF NOT EXISTS document_chunks_fts USING fts5(
76
- chunk_id UNINDEXED,
62
+ -- FTS5: one row per document (title, h1–h6, body)
63
+ CREATE VIRTUAL TABLE IF NOT EXISTS document_fts USING fts5(
77
64
  document_id UNINDEXED,
78
65
  title,
79
- topic_key,
80
- type,
81
- scope,
82
- project,
83
- heading_path,
66
+ h1,
67
+ h2,
68
+ h3,
69
+ h4,
70
+ h5,
71
+ h6,
84
72
  body,
85
73
  tokenize = 'unicode61 remove_diacritics 2'
86
74
  );
@@ -6,6 +6,29 @@ export const FRONTMATTER_KEYS = [
6
6
  'embedding_status', 'indexed_at', 'deleted_at'
7
7
  ];
8
8
 
9
+ /** Keys written to .md frontmatter (needed for re-index, scope/project filtering, timeline). */
10
+ const DISPLAY_FRONTMATTER_KEYS = ['id', 'type', 'title', 'scope', 'project', 'topic_key', 'session_id', 'created'];
11
+
12
+ function formatCreated(createdAt) {
13
+ if (!createdAt) return null;
14
+ const d = new Date(createdAt);
15
+ if (Number.isNaN(d.getTime())) return createdAt;
16
+ const y = d.getFullYear();
17
+ const m = String(d.getMonth() + 1).padStart(2, '0');
18
+ const day = String(d.getDate()).padStart(2, '0');
19
+ const h = String(d.getHours()).padStart(2, '0');
20
+ const min = String(d.getMinutes()).padStart(2, '0');
21
+ const s = String(d.getSeconds()).padStart(2, '0');
22
+ return `${y}-${m}-${day} ${h}:${min}:${s}`;
23
+ }
24
+
25
+ function parseCreated(value) {
26
+ if (!value) return null;
27
+ if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(value)) return value;
28
+ const d = new Date(value);
29
+ return Number.isNaN(d.getTime()) ? null : d.toISOString();
30
+ }
31
+
9
32
  function serializeFrontmatter(fm) {
10
33
  const obj = {};
11
34
  for (const k of FRONTMATTER_KEYS) {
@@ -16,9 +39,26 @@ function serializeFrontmatter(fm) {
16
39
  return obj;
17
40
  }
18
41
 
42
+ /** Minimal frontmatter for .md (id required for re-index; created in friendly format). */
43
+ function serializeDisplayFrontmatter(fm) {
44
+ const obj = {};
45
+ for (const k of DISPLAY_FRONTMATTER_KEYS) {
46
+ if (k === 'created') {
47
+ const v = fm.created_at != null ? formatCreated(fm.created_at) : (fm.created ?? null);
48
+ if (v) obj[k] = v;
49
+ } else if (fm[k] !== undefined && fm[k] !== null && fm[k] !== '') {
50
+ obj[k] = fm[k];
51
+ }
52
+ }
53
+ if (fm.deleted_at) obj.deleted_at = fm.deleted_at;
54
+ if (fm.deleted === true) obj.deleted = true;
55
+ return obj;
56
+ }
57
+
19
58
  export function parse(content) {
20
59
  const parsed = matter(content);
21
60
  const data = parsed.data || {};
61
+ const createdAt = data.created_at ?? parseCreated(data.created);
22
62
  return {
23
63
  frontmatter: {
24
64
  id: data.id,
@@ -29,7 +69,7 @@ export function parse(content) {
29
69
  topic_key: data.topic_key,
30
70
  session_id: data.session_id,
31
71
  tool_name: data.tool_name,
32
- created_at: data.created_at,
72
+ created_at: createdAt,
33
73
  updated_at: data.updated_at,
34
74
  revision_count: data.revision_count ?? 1,
35
75
  duplicate_count: data.duplicate_count ?? 0,
@@ -44,6 +84,6 @@ export function parse(content) {
44
84
  }
45
85
 
46
86
  export function toMarkdown(frontmatter, body) {
47
- const fm = serializeFrontmatter(frontmatter);
87
+ const fm = serializeDisplayFrontmatter(frontmatter);
48
88
  return matter.stringify(body || '', fm, { lineWidth: -1 });
49
89
  }