noteplan-mcp 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +257 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/noteplan/embeddings.d.ts +170 -0
- package/dist/noteplan/embeddings.d.ts.map +1 -0
- package/dist/noteplan/embeddings.js +684 -0
- package/dist/noteplan/embeddings.js.map +1 -0
- package/dist/noteplan/file-reader.d.ts +77 -0
- package/dist/noteplan/file-reader.d.ts.map +1 -0
- package/dist/noteplan/file-reader.js +488 -0
- package/dist/noteplan/file-reader.js.map +1 -0
- package/dist/noteplan/file-writer.d.ts +108 -0
- package/dist/noteplan/file-writer.d.ts.map +1 -0
- package/dist/noteplan/file-writer.js +621 -0
- package/dist/noteplan/file-writer.js.map +1 -0
- package/dist/noteplan/filter-store.d.ts +28 -0
- package/dist/noteplan/filter-store.d.ts.map +1 -0
- package/dist/noteplan/filter-store.js +180 -0
- package/dist/noteplan/filter-store.js.map +1 -0
- package/dist/noteplan/frontmatter-parser.d.ts +45 -0
- package/dist/noteplan/frontmatter-parser.d.ts.map +1 -0
- package/dist/noteplan/frontmatter-parser.js +259 -0
- package/dist/noteplan/frontmatter-parser.js.map +1 -0
- package/dist/noteplan/fuzzy-search.d.ts +7 -0
- package/dist/noteplan/fuzzy-search.d.ts.map +1 -0
- package/dist/noteplan/fuzzy-search.js +66 -0
- package/dist/noteplan/fuzzy-search.js.map +1 -0
- package/dist/noteplan/markdown-parser.d.ts +87 -0
- package/dist/noteplan/markdown-parser.d.ts.map +1 -0
- package/dist/noteplan/markdown-parser.js +519 -0
- package/dist/noteplan/markdown-parser.js.map +1 -0
- package/dist/noteplan/preferences.d.ts +44 -0
- package/dist/noteplan/preferences.d.ts.map +1 -0
- package/dist/noteplan/preferences.js +156 -0
- package/dist/noteplan/preferences.js.map +1 -0
- package/dist/noteplan/ripgrep-search.d.ts +29 -0
- package/dist/noteplan/ripgrep-search.d.ts.map +1 -0
- package/dist/noteplan/ripgrep-search.js +110 -0
- package/dist/noteplan/ripgrep-search.js.map +1 -0
- package/dist/noteplan/sqlite-reader.d.ts +77 -0
- package/dist/noteplan/sqlite-reader.d.ts.map +1 -0
- package/dist/noteplan/sqlite-reader.js +605 -0
- package/dist/noteplan/sqlite-reader.js.map +1 -0
- package/dist/noteplan/sqlite-writer.d.ts +63 -0
- package/dist/noteplan/sqlite-writer.d.ts.map +1 -0
- package/dist/noteplan/sqlite-writer.js +574 -0
- package/dist/noteplan/sqlite-writer.js.map +1 -0
- package/dist/noteplan/types.d.ts +97 -0
- package/dist/noteplan/types.d.ts.map +1 -0
- package/dist/noteplan/types.js +33 -0
- package/dist/noteplan/types.js.map +1 -0
- package/dist/noteplan/unified-store.d.ts +289 -0
- package/dist/noteplan/unified-store.d.ts.map +1 -0
- package/dist/noteplan/unified-store.js +1308 -0
- package/dist/noteplan/unified-store.js.map +1 -0
- package/dist/server.d.ts +4 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +2468 -0
- package/dist/server.js.map +1 -0
- package/dist/tools/calendar.d.ts +311 -0
- package/dist/tools/calendar.d.ts.map +1 -0
- package/dist/tools/calendar.js +504 -0
- package/dist/tools/calendar.js.map +1 -0
- package/dist/tools/embeddings.d.ts +244 -0
- package/dist/tools/embeddings.d.ts.map +1 -0
- package/dist/tools/embeddings.js +226 -0
- package/dist/tools/embeddings.js.map +1 -0
- package/dist/tools/events.d.ts +176 -0
- package/dist/tools/events.d.ts.map +1 -0
- package/dist/tools/events.js +326 -0
- package/dist/tools/events.js.map +1 -0
- package/dist/tools/filters.d.ts +205 -0
- package/dist/tools/filters.d.ts.map +1 -0
- package/dist/tools/filters.js +347 -0
- package/dist/tools/filters.js.map +1 -0
- package/dist/tools/memory.d.ts +6 -0
- package/dist/tools/memory.d.ts.map +1 -0
- package/dist/tools/memory.js +161 -0
- package/dist/tools/memory.js.map +1 -0
- package/dist/tools/notes.d.ts +1221 -0
- package/dist/tools/notes.d.ts.map +1 -0
- package/dist/tools/notes.js +1868 -0
- package/dist/tools/notes.js.map +1 -0
- package/dist/tools/plugins.d.ts +140 -0
- package/dist/tools/plugins.d.ts.map +1 -0
- package/dist/tools/plugins.js +782 -0
- package/dist/tools/plugins.js.map +1 -0
- package/dist/tools/reminders.d.ts +207 -0
- package/dist/tools/reminders.d.ts.map +1 -0
- package/dist/tools/reminders.js +323 -0
- package/dist/tools/reminders.js.map +1 -0
- package/dist/tools/search.d.ts +58 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +373 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/tools/spaces.d.ts +484 -0
- package/dist/tools/spaces.d.ts.map +1 -0
- package/dist/tools/spaces.js +870 -0
- package/dist/tools/spaces.js.map +1 -0
- package/dist/tools/tasks.d.ts +313 -0
- package/dist/tools/tasks.d.ts.map +1 -0
- package/dist/tools/tasks.js +690 -0
- package/dist/tools/tasks.js.map +1 -0
- package/dist/tools/themes.d.ts +91 -0
- package/dist/tools/themes.d.ts.map +1 -0
- package/dist/tools/themes.js +294 -0
- package/dist/tools/themes.js.map +1 -0
- package/dist/tools/ui.d.ts +89 -0
- package/dist/tools/ui.d.ts.map +1 -0
- package/dist/tools/ui.js +137 -0
- package/dist/tools/ui.js.map +1 -0
- package/dist/utils/applescript.d.ts +5 -0
- package/dist/utils/applescript.d.ts.map +1 -0
- package/dist/utils/applescript.js +27 -0
- package/dist/utils/applescript.js.map +1 -0
- package/dist/utils/confirmation-tokens.d.ts +19 -0
- package/dist/utils/confirmation-tokens.d.ts.map +1 -0
- package/dist/utils/confirmation-tokens.js +58 -0
- package/dist/utils/confirmation-tokens.js.map +1 -0
- package/dist/utils/date-filters.d.ts +15 -0
- package/dist/utils/date-filters.d.ts.map +1 -0
- package/dist/utils/date-filters.js +129 -0
- package/dist/utils/date-filters.js.map +1 -0
- package/dist/utils/date-utils.d.ts +113 -0
- package/dist/utils/date-utils.d.ts.map +1 -0
- package/dist/utils/date-utils.js +341 -0
- package/dist/utils/date-utils.js.map +1 -0
- package/dist/utils/folder-matcher.d.ts +14 -0
- package/dist/utils/folder-matcher.d.ts.map +1 -0
- package/dist/utils/folder-matcher.js +191 -0
- package/dist/utils/folder-matcher.js.map +1 -0
- package/dist/utils/version.d.ts +10 -0
- package/dist/utils/version.d.ts.map +1 -0
- package/dist/utils/version.js +88 -0
- package/dist/utils/version.js.map +1 -0
- package/docs/plugin-api/Calendar.md +448 -0
- package/docs/plugin-api/CalendarItem.md +198 -0
- package/docs/plugin-api/Clipboard.md +101 -0
- package/docs/plugin-api/CommandBar.md +251 -0
- package/docs/plugin-api/DataStore.md +700 -0
- package/docs/plugin-api/Editor.md +982 -0
- package/docs/plugin-api/HTMLView.md +337 -0
- package/docs/plugin-api/NoteObject.md +588 -0
- package/docs/plugin-api/NotePlan.md +398 -0
- package/docs/plugin-api/ParagraphObject.md +242 -0
- package/docs/plugin-api/RangeObject.md +56 -0
- package/docs/plugin-api/getting-started.md +545 -0
- package/docs/plugin-api/plugin-api-condensed.md +526 -0
- package/docs/plugin-api/plugin.json +26 -0
- package/docs/plugin-api/script.js +542 -0
- package/package.json +60 -0
- package/scripts/calendar-helper +0 -0
- package/scripts/reminders-helper +0 -0
|
@@ -0,0 +1,684 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import * as fs from 'fs';
|
|
3
|
+
import * as os from 'os';
|
|
4
|
+
import * as path from 'path';
|
|
5
|
+
import { createHash } from 'crypto';
|
|
6
|
+
import * as store from './unified-store.js';
|
|
7
|
+
function parseBoolean(value, defaultValue) {
|
|
8
|
+
if (!value)
|
|
9
|
+
return defaultValue;
|
|
10
|
+
const normalized = value.trim().toLowerCase();
|
|
11
|
+
if (normalized === 'true' || normalized === '1' || normalized === 'yes')
|
|
12
|
+
return true;
|
|
13
|
+
if (normalized === 'false' || normalized === '0' || normalized === 'no')
|
|
14
|
+
return false;
|
|
15
|
+
return defaultValue;
|
|
16
|
+
}
|
|
17
|
+
function parseBoundedInt(value, defaultValue, min, max) {
|
|
18
|
+
const numeric = Number(value);
|
|
19
|
+
if (!Number.isFinite(numeric))
|
|
20
|
+
return defaultValue;
|
|
21
|
+
return Math.min(max, Math.max(min, Math.floor(numeric)));
|
|
22
|
+
}
|
|
23
|
+
let cachedConfig = null;
|
|
24
|
+
let db = null;
|
|
25
|
+
let dbPathForConnection = null;
|
|
26
|
+
function getDefaultModel(provider) {
|
|
27
|
+
if (provider === 'mistral')
|
|
28
|
+
return 'mistral-embed';
|
|
29
|
+
return 'text-embedding-3-small';
|
|
30
|
+
}
|
|
31
|
+
function getDefaultBaseUrl(provider) {
|
|
32
|
+
if (provider === 'mistral')
|
|
33
|
+
return 'https://api.mistral.ai';
|
|
34
|
+
if (provider === 'custom')
|
|
35
|
+
return 'http://localhost:11434';
|
|
36
|
+
return 'https://api.openai.com';
|
|
37
|
+
}
|
|
38
|
+
function normalizeBaseUrl(value) {
|
|
39
|
+
return value.trim().replace(/\/+$/, '');
|
|
40
|
+
}
|
|
41
|
+
function resolveEmbeddingsDbPath() {
|
|
42
|
+
const customPath = process.env.NOTEPLAN_EMBEDDINGS_DB_PATH?.trim();
|
|
43
|
+
if (customPath) {
|
|
44
|
+
return path.resolve(customPath);
|
|
45
|
+
}
|
|
46
|
+
return path.join(os.homedir(), '.noteplan-mcp', 'embeddings.db');
|
|
47
|
+
}
|
|
48
|
+
export function getEmbeddingsConfig() {
|
|
49
|
+
if (cachedConfig)
|
|
50
|
+
return cachedConfig;
|
|
51
|
+
const enabled = parseBoolean(process.env.NOTEPLAN_EMBEDDINGS_ENABLED, false);
|
|
52
|
+
const providerRaw = (process.env.NOTEPLAN_EMBEDDINGS_PROVIDER || 'openai').trim().toLowerCase();
|
|
53
|
+
const provider = providerRaw === 'mistral'
|
|
54
|
+
? 'mistral'
|
|
55
|
+
: providerRaw === 'custom'
|
|
56
|
+
? 'custom'
|
|
57
|
+
: 'openai';
|
|
58
|
+
const model = (process.env.NOTEPLAN_EMBEDDINGS_MODEL || getDefaultModel(provider)).trim();
|
|
59
|
+
const baseUrl = normalizeBaseUrl(process.env.NOTEPLAN_EMBEDDINGS_BASE_URL || getDefaultBaseUrl(provider));
|
|
60
|
+
cachedConfig = {
|
|
61
|
+
enabled,
|
|
62
|
+
provider,
|
|
63
|
+
apiKey: (process.env.NOTEPLAN_EMBEDDINGS_API_KEY || '').trim(),
|
|
64
|
+
model,
|
|
65
|
+
baseUrl,
|
|
66
|
+
dbPath: resolveEmbeddingsDbPath(),
|
|
67
|
+
chunkChars: parseBoundedInt(process.env.NOTEPLAN_EMBEDDINGS_CHUNK_CHARS, 1200, 300, 4000),
|
|
68
|
+
chunkOverlap: parseBoundedInt(process.env.NOTEPLAN_EMBEDDINGS_CHUNK_OVERLAP, 200, 0, 1000),
|
|
69
|
+
previewChars: parseBoundedInt(process.env.NOTEPLAN_EMBEDDINGS_PREVIEW_CHARS, 220, 60, 1000),
|
|
70
|
+
defaultBatchSize: parseBoundedInt(process.env.NOTEPLAN_EMBEDDINGS_BATCH_SIZE, 16, 1, 64),
|
|
71
|
+
defaultMaxChunksPerNote: parseBoundedInt(process.env.NOTEPLAN_EMBEDDINGS_MAX_CHUNKS_PER_NOTE, 60, 1, 400),
|
|
72
|
+
};
|
|
73
|
+
return cachedConfig;
|
|
74
|
+
}
|
|
75
|
+
export function areEmbeddingsEnabled() {
|
|
76
|
+
return getEmbeddingsConfig().enabled;
|
|
77
|
+
}
|
|
78
|
+
function buildEmbeddingsEndpoint(baseUrl) {
|
|
79
|
+
if (baseUrl.endsWith('/v1/embeddings'))
|
|
80
|
+
return baseUrl;
|
|
81
|
+
if (baseUrl.endsWith('/v1'))
|
|
82
|
+
return `${baseUrl}/embeddings`;
|
|
83
|
+
return `${baseUrl}/v1/embeddings`;
|
|
84
|
+
}
|
|
85
|
+
function ensureEmbeddingsApiConfigured() {
|
|
86
|
+
const config = getEmbeddingsConfig();
|
|
87
|
+
if (!config.enabled) {
|
|
88
|
+
return {
|
|
89
|
+
ok: false,
|
|
90
|
+
error: 'Embeddings are disabled. Set NOTEPLAN_EMBEDDINGS_ENABLED=true to enable embeddings tools.',
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if ((config.provider === 'openai' || config.provider === 'mistral') && !config.apiKey) {
|
|
94
|
+
return {
|
|
95
|
+
ok: false,
|
|
96
|
+
error: 'Embeddings API key is missing. Set NOTEPLAN_EMBEDDINGS_API_KEY in your MCP server environment.',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return { ok: true };
|
|
100
|
+
}
|
|
101
|
+
function openEmbeddingsDb() {
|
|
102
|
+
const config = getEmbeddingsConfig();
|
|
103
|
+
if (db && dbPathForConnection === config.dbPath) {
|
|
104
|
+
return db;
|
|
105
|
+
}
|
|
106
|
+
if (db) {
|
|
107
|
+
db.close();
|
|
108
|
+
db = null;
|
|
109
|
+
dbPathForConnection = null;
|
|
110
|
+
}
|
|
111
|
+
fs.mkdirSync(path.dirname(config.dbPath), { recursive: true });
|
|
112
|
+
db = new Database(config.dbPath);
|
|
113
|
+
dbPathForConnection = config.dbPath;
|
|
114
|
+
db.pragma('journal_mode = WAL');
|
|
115
|
+
db.pragma('synchronous = NORMAL');
|
|
116
|
+
db.pragma('foreign_keys = ON');
|
|
117
|
+
db.exec(`
|
|
118
|
+
CREATE TABLE IF NOT EXISTS metadata (
|
|
119
|
+
key TEXT PRIMARY KEY,
|
|
120
|
+
value TEXT NOT NULL
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
CREATE TABLE IF NOT EXISTS notes (
|
|
124
|
+
note_key TEXT PRIMARY KEY,
|
|
125
|
+
note_id TEXT NOT NULL,
|
|
126
|
+
filename TEXT NOT NULL,
|
|
127
|
+
title TEXT NOT NULL,
|
|
128
|
+
source TEXT NOT NULL,
|
|
129
|
+
space_id TEXT,
|
|
130
|
+
folder TEXT,
|
|
131
|
+
type TEXT NOT NULL,
|
|
132
|
+
modified_at TEXT,
|
|
133
|
+
content_hash TEXT NOT NULL,
|
|
134
|
+
chunk_count INTEGER NOT NULL,
|
|
135
|
+
updated_at TEXT NOT NULL
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
139
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
140
|
+
note_key TEXT NOT NULL,
|
|
141
|
+
chunk_index INTEGER NOT NULL,
|
|
142
|
+
chunk_text TEXT NOT NULL,
|
|
143
|
+
chunk_preview TEXT NOT NULL,
|
|
144
|
+
chunk_hash TEXT NOT NULL,
|
|
145
|
+
embedding_json TEXT NOT NULL,
|
|
146
|
+
dim INTEGER NOT NULL,
|
|
147
|
+
created_at TEXT NOT NULL,
|
|
148
|
+
UNIQUE(note_key, chunk_index),
|
|
149
|
+
FOREIGN KEY(note_key) REFERENCES notes(note_key) ON DELETE CASCADE
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
CREATE INDEX IF NOT EXISTS idx_chunks_note_key ON chunks(note_key);
|
|
153
|
+
CREATE INDEX IF NOT EXISTS idx_notes_space_id ON notes(space_id);
|
|
154
|
+
CREATE INDEX IF NOT EXISTS idx_notes_source ON notes(source);
|
|
155
|
+
CREATE INDEX IF NOT EXISTS idx_notes_type ON notes(type);
|
|
156
|
+
CREATE INDEX IF NOT EXISTS idx_notes_updated_at ON notes(updated_at);
|
|
157
|
+
`);
|
|
158
|
+
return db;
|
|
159
|
+
}
|
|
160
|
+
function setMetadata(key, value) {
|
|
161
|
+
const database = openEmbeddingsDb();
|
|
162
|
+
database
|
|
163
|
+
.prepare(`
|
|
164
|
+
INSERT INTO metadata (key, value)
|
|
165
|
+
VALUES (?, ?)
|
|
166
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
|
167
|
+
`)
|
|
168
|
+
.run(key, value);
|
|
169
|
+
}
|
|
170
|
+
function getMetadata(key) {
|
|
171
|
+
const database = openEmbeddingsDb();
|
|
172
|
+
const row = database.prepare('SELECT value FROM metadata WHERE key = ?').get(key);
|
|
173
|
+
return row?.value ?? null;
|
|
174
|
+
}
|
|
175
|
+
function nowIso() {
|
|
176
|
+
return new Date().toISOString();
|
|
177
|
+
}
|
|
178
|
+
function computeHash(input) {
|
|
179
|
+
return createHash('sha256').update(input).digest('hex');
|
|
180
|
+
}
|
|
181
|
+
function toNoteKey(note) {
|
|
182
|
+
if (note.source === 'space') {
|
|
183
|
+
const idOrFilename = note.id?.trim().length ? note.id.trim() : note.filename;
|
|
184
|
+
return `space:${idOrFilename}`;
|
|
185
|
+
}
|
|
186
|
+
return `local:${note.filename}`;
|
|
187
|
+
}
|
|
188
|
+
function buildPreview(text, maxChars) {
|
|
189
|
+
const collapsed = text.replace(/\s+/g, ' ').trim();
|
|
190
|
+
if (collapsed.length <= maxChars)
|
|
191
|
+
return collapsed;
|
|
192
|
+
return `${collapsed.slice(0, Math.max(0, maxChars - 1))}…`;
|
|
193
|
+
}
|
|
194
|
+
function chunkContent(content, chunkChars, chunkOverlap, maxChunks) {
|
|
195
|
+
const normalized = content.replace(/\r\n/g, '\n').trim();
|
|
196
|
+
if (!normalized)
|
|
197
|
+
return [];
|
|
198
|
+
const effectiveOverlap = Math.min(Math.max(0, chunkOverlap), Math.max(0, chunkChars - 1));
|
|
199
|
+
const step = Math.max(1, chunkChars - effectiveOverlap);
|
|
200
|
+
const chunks = [];
|
|
201
|
+
let start = 0;
|
|
202
|
+
while (start < normalized.length && chunks.length < maxChunks) {
|
|
203
|
+
let end = Math.min(normalized.length, start + chunkChars);
|
|
204
|
+
if (end < normalized.length) {
|
|
205
|
+
const breakAt = normalized.lastIndexOf('\n', end);
|
|
206
|
+
if (breakAt > start + Math.floor(chunkChars * 0.6)) {
|
|
207
|
+
end = breakAt;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const chunk = normalized.slice(start, end).trim();
|
|
211
|
+
if (chunk.length > 0) {
|
|
212
|
+
chunks.push(chunk);
|
|
213
|
+
}
|
|
214
|
+
if (end >= normalized.length)
|
|
215
|
+
break;
|
|
216
|
+
start = Math.max(start + 1, end - effectiveOverlap);
|
|
217
|
+
}
|
|
218
|
+
return chunks;
|
|
219
|
+
}
|
|
220
|
+
async function fetchEmbeddings(texts) {
|
|
221
|
+
if (texts.length === 0)
|
|
222
|
+
return [];
|
|
223
|
+
const config = getEmbeddingsConfig();
|
|
224
|
+
const endpoint = buildEmbeddingsEndpoint(config.baseUrl);
|
|
225
|
+
const headers = {
|
|
226
|
+
'Content-Type': 'application/json',
|
|
227
|
+
};
|
|
228
|
+
if (config.apiKey) {
|
|
229
|
+
headers.Authorization = `Bearer ${config.apiKey}`;
|
|
230
|
+
}
|
|
231
|
+
const response = await fetch(endpoint, {
|
|
232
|
+
method: 'POST',
|
|
233
|
+
headers,
|
|
234
|
+
body: JSON.stringify({
|
|
235
|
+
model: config.model,
|
|
236
|
+
input: texts,
|
|
237
|
+
}),
|
|
238
|
+
});
|
|
239
|
+
if (!response.ok) {
|
|
240
|
+
const body = await response.text();
|
|
241
|
+
const excerpt = body.slice(0, 320);
|
|
242
|
+
throw new Error(`Embeddings request failed (${response.status}): ${excerpt}`);
|
|
243
|
+
}
|
|
244
|
+
const json = (await response.json());
|
|
245
|
+
const rows = Array.isArray(json.data) ? json.data : [];
|
|
246
|
+
if (rows.length !== texts.length) {
|
|
247
|
+
throw new Error(`Embeddings response mismatch: expected ${texts.length} vectors, got ${rows.length}`);
|
|
248
|
+
}
|
|
249
|
+
const sorted = rows
|
|
250
|
+
.slice()
|
|
251
|
+
.sort((a, b) => (Number.isFinite(a.index) ? a.index : 0) - (Number.isFinite(b.index) ? b.index : 0));
|
|
252
|
+
return sorted.map((item) => item.embedding);
|
|
253
|
+
}
|
|
254
|
+
async function embedInBatches(texts, batchSize) {
|
|
255
|
+
const vectors = [];
|
|
256
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
257
|
+
const batch = texts.slice(i, i + batchSize);
|
|
258
|
+
const batchVectors = await fetchEmbeddings(batch);
|
|
259
|
+
vectors.push(...batchVectors);
|
|
260
|
+
}
|
|
261
|
+
return vectors;
|
|
262
|
+
}
|
|
263
|
+
function cosineSimilarity(a, b) {
|
|
264
|
+
if (a.length === 0 || b.length === 0 || a.length !== b.length)
|
|
265
|
+
return 0;
|
|
266
|
+
let dot = 0;
|
|
267
|
+
let normA = 0;
|
|
268
|
+
let normB = 0;
|
|
269
|
+
for (let i = 0; i < a.length; i += 1) {
|
|
270
|
+
const av = a[i];
|
|
271
|
+
const bv = b[i];
|
|
272
|
+
dot += av * bv;
|
|
273
|
+
normA += av * av;
|
|
274
|
+
normB += bv * bv;
|
|
275
|
+
}
|
|
276
|
+
if (normA <= 0 || normB <= 0)
|
|
277
|
+
return 0;
|
|
278
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
279
|
+
}
|
|
280
|
+
function filterNotesByQuery(notes, noteQuery) {
|
|
281
|
+
const query = (noteQuery || '').trim().toLowerCase();
|
|
282
|
+
if (!query)
|
|
283
|
+
return notes;
|
|
284
|
+
return notes.filter((note) => {
|
|
285
|
+
const haystack = `${note.title} ${note.filename} ${note.folder || ''}`.toLowerCase();
|
|
286
|
+
return haystack.includes(query);
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
function clampNumber(value, fallback, min, max) {
|
|
290
|
+
if (!Number.isFinite(value))
|
|
291
|
+
return fallback;
|
|
292
|
+
return Math.min(max, Math.max(min, Math.floor(value)));
|
|
293
|
+
}
|
|
294
|
+
function getScopeWhere(scope) {
|
|
295
|
+
if (scope.space && scope.space.trim().length > 0) {
|
|
296
|
+
return {
|
|
297
|
+
whereSql: 'WHERE space_id = ?',
|
|
298
|
+
params: [scope.space.trim()],
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
return {
|
|
302
|
+
whereSql: '',
|
|
303
|
+
params: [],
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
export function getEmbeddingsStatus(scope = {}) {
|
|
307
|
+
const config = getEmbeddingsConfig();
|
|
308
|
+
const apiCheck = ensureEmbeddingsApiConfigured();
|
|
309
|
+
if (!config.enabled) {
|
|
310
|
+
return {
|
|
311
|
+
success: true,
|
|
312
|
+
enabled: false,
|
|
313
|
+
configured: false,
|
|
314
|
+
provider: config.provider,
|
|
315
|
+
model: config.model,
|
|
316
|
+
baseUrl: config.baseUrl,
|
|
317
|
+
dbPath: config.dbPath,
|
|
318
|
+
noteCount: 0,
|
|
319
|
+
chunkCount: 0,
|
|
320
|
+
lastSyncAt: null,
|
|
321
|
+
warning: 'Embeddings are disabled. Set NOTEPLAN_EMBEDDINGS_ENABLED=true to enable embeddings tools.',
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
const database = openEmbeddingsDb();
|
|
325
|
+
const scopeWhere = getScopeWhere(scope);
|
|
326
|
+
const noteCountRow = database
|
|
327
|
+
.prepare(`SELECT COUNT(*) as count FROM notes ${scopeWhere.whereSql}`)
|
|
328
|
+
.get(...scopeWhere.params);
|
|
329
|
+
const chunkCountRow = database
|
|
330
|
+
.prepare(`
|
|
331
|
+
SELECT COUNT(*) as count
|
|
332
|
+
FROM chunks c
|
|
333
|
+
JOIN notes n ON n.note_key = c.note_key
|
|
334
|
+
${scopeWhere.whereSql ? scopeWhere.whereSql.replace(/space_id/g, 'n.space_id') : ''}
|
|
335
|
+
`)
|
|
336
|
+
.get(...scopeWhere.params);
|
|
337
|
+
const lastUpdatedRow = database
|
|
338
|
+
.prepare(`SELECT MAX(updated_at) as lastUpdatedAt FROM notes ${scopeWhere.whereSql}`)
|
|
339
|
+
.get(...scopeWhere.params);
|
|
340
|
+
return {
|
|
341
|
+
success: true,
|
|
342
|
+
enabled: true,
|
|
343
|
+
configured: apiCheck.ok,
|
|
344
|
+
provider: config.provider,
|
|
345
|
+
model: config.model,
|
|
346
|
+
baseUrl: config.baseUrl,
|
|
347
|
+
dbPath: config.dbPath,
|
|
348
|
+
hasApiKey: config.apiKey.length > 0,
|
|
349
|
+
chunkChars: config.chunkChars,
|
|
350
|
+
chunkOverlap: config.chunkOverlap,
|
|
351
|
+
previewChars: config.previewChars,
|
|
352
|
+
noteCount: noteCountRow.count,
|
|
353
|
+
chunkCount: chunkCountRow.count,
|
|
354
|
+
lastSyncAt: getMetadata('lastSyncAt'),
|
|
355
|
+
lastIndexedUpdateAt: lastUpdatedRow.lastUpdatedAt,
|
|
356
|
+
...(apiCheck.ok
|
|
357
|
+
? {}
|
|
358
|
+
: {
|
|
359
|
+
warning: apiCheck.error,
|
|
360
|
+
}),
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
export function previewResetEmbeddings(scope = {}) {
|
|
364
|
+
const database = openEmbeddingsDb();
|
|
365
|
+
const scopeWhere = getScopeWhere(scope);
|
|
366
|
+
const noteCountRow = database
|
|
367
|
+
.prepare(`SELECT COUNT(*) as count FROM notes ${scopeWhere.whereSql}`)
|
|
368
|
+
.get(...scopeWhere.params);
|
|
369
|
+
const chunkCountRow = database
|
|
370
|
+
.prepare(`
|
|
371
|
+
SELECT COUNT(*) as count
|
|
372
|
+
FROM chunks c
|
|
373
|
+
JOIN notes n ON n.note_key = c.note_key
|
|
374
|
+
${scopeWhere.whereSql ? scopeWhere.whereSql.replace(/space_id/g, 'n.space_id') : ''}
|
|
375
|
+
`)
|
|
376
|
+
.get(...scopeWhere.params);
|
|
377
|
+
return {
|
|
378
|
+
noteCount: noteCountRow.count,
|
|
379
|
+
chunkCount: chunkCountRow.count,
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
export function resetEmbeddings(scope = {}) {
|
|
383
|
+
const database = openEmbeddingsDb();
|
|
384
|
+
const preview = previewResetEmbeddings(scope);
|
|
385
|
+
const scopeWhere = getScopeWhere(scope);
|
|
386
|
+
const tx = database.transaction(() => {
|
|
387
|
+
if (!scopeWhere.whereSql) {
|
|
388
|
+
database.prepare('DELETE FROM chunks').run();
|
|
389
|
+
database.prepare('DELETE FROM notes').run();
|
|
390
|
+
}
|
|
391
|
+
else {
|
|
392
|
+
database
|
|
393
|
+
.prepare(`
|
|
394
|
+
DELETE FROM chunks
|
|
395
|
+
WHERE note_key IN (
|
|
396
|
+
SELECT note_key FROM notes ${scopeWhere.whereSql}
|
|
397
|
+
)
|
|
398
|
+
`)
|
|
399
|
+
.run(...scopeWhere.params);
|
|
400
|
+
database.prepare(`DELETE FROM notes ${scopeWhere.whereSql}`).run(...scopeWhere.params);
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
tx();
|
|
404
|
+
if (!scope.space) {
|
|
405
|
+
setMetadata('lastSyncAt', '');
|
|
406
|
+
}
|
|
407
|
+
return {
|
|
408
|
+
removedNotes: preview.noteCount,
|
|
409
|
+
removedChunks: preview.chunkCount,
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
export async function syncEmbeddings(params = {}) {
|
|
413
|
+
const apiCheck = ensureEmbeddingsApiConfigured();
|
|
414
|
+
if (!apiCheck.ok) {
|
|
415
|
+
return {
|
|
416
|
+
success: false,
|
|
417
|
+
error: apiCheck.error,
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
const config = getEmbeddingsConfig();
|
|
421
|
+
const database = openEmbeddingsDb();
|
|
422
|
+
const requestedLimit = clampNumber(params.limit, 500, 1, 5000);
|
|
423
|
+
const requestedOffset = clampNumber(params.offset, 0, 0, Number.MAX_SAFE_INTEGER);
|
|
424
|
+
const batchSize = clampNumber(params.batchSize, config.defaultBatchSize, 1, 64);
|
|
425
|
+
const maxChunksPerNote = clampNumber(params.maxChunksPerNote, config.defaultMaxChunksPerNote, 1, 400);
|
|
426
|
+
const typeFilter = params.types && params.types.length > 0 ? new Set(params.types) : null;
|
|
427
|
+
const allNotes = store.listNotes({
|
|
428
|
+
space: params.space,
|
|
429
|
+
});
|
|
430
|
+
const filteredByType = typeFilter
|
|
431
|
+
? allNotes.filter((note) => typeFilter.has(note.type))
|
|
432
|
+
: allNotes.filter((note) => note.type !== 'trash');
|
|
433
|
+
const queryFiltered = filterNotesByQuery(filteredByType, params.noteQuery);
|
|
434
|
+
const pagedNotes = queryFiltered.slice(requestedOffset, requestedOffset + requestedLimit);
|
|
435
|
+
const existingRows = database.prepare('SELECT note_key, content_hash FROM notes').all();
|
|
436
|
+
const existingByKey = new Map(existingRows.map((row) => [row.note_key, row.content_hash]));
|
|
437
|
+
let indexedNotes = 0;
|
|
438
|
+
let unchangedNotes = 0;
|
|
439
|
+
let addedNotes = 0;
|
|
440
|
+
let updatedNotes = 0;
|
|
441
|
+
let indexedChunks = 0;
|
|
442
|
+
const warnings = [];
|
|
443
|
+
const upsertNote = database.prepare(`
|
|
444
|
+
INSERT INTO notes (
|
|
445
|
+
note_key, note_id, filename, title, source, space_id, folder, type,
|
|
446
|
+
modified_at, content_hash, chunk_count, updated_at
|
|
447
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
448
|
+
ON CONFLICT(note_key) DO UPDATE SET
|
|
449
|
+
note_id = excluded.note_id,
|
|
450
|
+
filename = excluded.filename,
|
|
451
|
+
title = excluded.title,
|
|
452
|
+
source = excluded.source,
|
|
453
|
+
space_id = excluded.space_id,
|
|
454
|
+
folder = excluded.folder,
|
|
455
|
+
type = excluded.type,
|
|
456
|
+
modified_at = excluded.modified_at,
|
|
457
|
+
content_hash = excluded.content_hash,
|
|
458
|
+
chunk_count = excluded.chunk_count,
|
|
459
|
+
updated_at = excluded.updated_at
|
|
460
|
+
`);
|
|
461
|
+
const deleteChunksByNoteKey = database.prepare('DELETE FROM chunks WHERE note_key = ?');
|
|
462
|
+
const insertChunk = database.prepare(`
|
|
463
|
+
INSERT INTO chunks (
|
|
464
|
+
note_key, chunk_index, chunk_text, chunk_preview, chunk_hash, embedding_json, dim, created_at
|
|
465
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
466
|
+
`);
|
|
467
|
+
const writeNoteTransaction = database.transaction((payload) => {
|
|
468
|
+
deleteChunksByNoteKey.run(payload.noteKey);
|
|
469
|
+
const timestamp = nowIso();
|
|
470
|
+
upsertNote.run(payload.noteKey, payload.note.id, payload.note.filename, payload.note.title, payload.note.source, payload.note.spaceId ?? null, payload.note.folder ?? null, payload.note.type, payload.note.modifiedAt ? payload.note.modifiedAt.toISOString() : null, payload.contentHash, payload.chunks.length, timestamp);
|
|
471
|
+
for (let i = 0; i < payload.chunks.length; i += 1) {
|
|
472
|
+
const chunkText = payload.chunks[i];
|
|
473
|
+
const vector = payload.embeddings[i] || [];
|
|
474
|
+
insertChunk.run(payload.noteKey, i, chunkText, buildPreview(chunkText, config.previewChars), computeHash(chunkText), JSON.stringify(vector), vector.length, timestamp);
|
|
475
|
+
}
|
|
476
|
+
});
|
|
477
|
+
for (const note of pagedNotes) {
|
|
478
|
+
const noteKey = toNoteKey(note);
|
|
479
|
+
const contentHash = computeHash(note.content);
|
|
480
|
+
const existingHash = existingByKey.get(noteKey);
|
|
481
|
+
if (!params.forceReembed && existingHash === contentHash) {
|
|
482
|
+
unchangedNotes += 1;
|
|
483
|
+
continue;
|
|
484
|
+
}
|
|
485
|
+
const chunks = chunkContent(note.content, config.chunkChars, config.chunkOverlap, maxChunksPerNote);
|
|
486
|
+
if (chunks.length === 0) {
|
|
487
|
+
writeNoteTransaction({
|
|
488
|
+
note,
|
|
489
|
+
noteKey,
|
|
490
|
+
contentHash,
|
|
491
|
+
chunks: [],
|
|
492
|
+
embeddings: [],
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
else {
|
|
496
|
+
const vectors = await embedInBatches(chunks, batchSize);
|
|
497
|
+
if (vectors.length !== chunks.length) {
|
|
498
|
+
throw new Error(`Embedding mismatch for ${note.filename}: ${chunks.length} chunks but ${vectors.length} vectors`);
|
|
499
|
+
}
|
|
500
|
+
writeNoteTransaction({
|
|
501
|
+
note,
|
|
502
|
+
noteKey,
|
|
503
|
+
contentHash,
|
|
504
|
+
chunks,
|
|
505
|
+
embeddings: vectors,
|
|
506
|
+
});
|
|
507
|
+
indexedChunks += chunks.length;
|
|
508
|
+
}
|
|
509
|
+
indexedNotes += 1;
|
|
510
|
+
if (existingHash) {
|
|
511
|
+
updatedNotes += 1;
|
|
512
|
+
}
|
|
513
|
+
else {
|
|
514
|
+
addedNotes += 1;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
let prunedNotes = 0;
|
|
518
|
+
let prunedChunks = 0;
|
|
519
|
+
const canPruneAllScope = params.pruneMissing === true &&
|
|
520
|
+
requestedOffset === 0 &&
|
|
521
|
+
pagedNotes.length === queryFiltered.length &&
|
|
522
|
+
!typeFilter &&
|
|
523
|
+
!(params.noteQuery && params.noteQuery.trim().length > 0);
|
|
524
|
+
if (params.pruneMissing === true && !canPruneAllScope) {
|
|
525
|
+
warnings.push('pruneMissing=true was ignored because prune is only safe for full-scope sync (offset=0, full result set, no type/query filters).');
|
|
526
|
+
}
|
|
527
|
+
if (canPruneAllScope) {
|
|
528
|
+
const validKeys = new Set(pagedNotes.map((note) => toNoteKey(note)));
|
|
529
|
+
const scopeWhere = getScopeWhere({ space: params.space });
|
|
530
|
+
const scopedRows = database
|
|
531
|
+
.prepare(`SELECT note_key FROM notes ${scopeWhere.whereSql}`)
|
|
532
|
+
.all(...scopeWhere.params);
|
|
533
|
+
const staleKeys = scopedRows
|
|
534
|
+
.map((row) => row.note_key)
|
|
535
|
+
.filter((noteKey) => !validKeys.has(noteKey));
|
|
536
|
+
if (staleKeys.length > 0) {
|
|
537
|
+
const deleteStale = database.transaction((keys) => {
|
|
538
|
+
const selectChunkCount = database.prepare('SELECT COUNT(*) as count FROM chunks WHERE note_key = ?');
|
|
539
|
+
const deleteChunks = database.prepare('DELETE FROM chunks WHERE note_key = ?');
|
|
540
|
+
const deleteNote = database.prepare('DELETE FROM notes WHERE note_key = ?');
|
|
541
|
+
for (const noteKey of keys) {
|
|
542
|
+
const chunkRow = selectChunkCount.get(noteKey);
|
|
543
|
+
prunedChunks += chunkRow.count;
|
|
544
|
+
prunedNotes += 1;
|
|
545
|
+
deleteChunks.run(noteKey);
|
|
546
|
+
deleteNote.run(noteKey);
|
|
547
|
+
}
|
|
548
|
+
});
|
|
549
|
+
deleteStale(staleKeys);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
setMetadata('lastSyncAt', nowIso());
|
|
553
|
+
setMetadata('lastSyncProvider', config.provider);
|
|
554
|
+
setMetadata('lastSyncModel', config.model);
|
|
555
|
+
return {
|
|
556
|
+
success: true,
|
|
557
|
+
provider: config.provider,
|
|
558
|
+
model: config.model,
|
|
559
|
+
scope: params.space ? { space: params.space } : { scope: 'all' },
|
|
560
|
+
totalCandidates: queryFiltered.length,
|
|
561
|
+
scannedNotes: pagedNotes.length,
|
|
562
|
+
indexedNotes,
|
|
563
|
+
unchangedNotes,
|
|
564
|
+
addedNotes,
|
|
565
|
+
updatedNotes,
|
|
566
|
+
indexedChunks,
|
|
567
|
+
prunedNotes,
|
|
568
|
+
prunedChunks,
|
|
569
|
+
offset: requestedOffset,
|
|
570
|
+
limit: requestedLimit,
|
|
571
|
+
hasMore: requestedOffset + pagedNotes.length < queryFiltered.length,
|
|
572
|
+
nextCursor: requestedOffset + pagedNotes.length < queryFiltered.length
|
|
573
|
+
? String(requestedOffset + pagedNotes.length)
|
|
574
|
+
: null,
|
|
575
|
+
warnings,
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
export async function searchEmbeddings(params) {
|
|
579
|
+
const apiCheck = ensureEmbeddingsApiConfigured();
|
|
580
|
+
if (!apiCheck.ok) {
|
|
581
|
+
return {
|
|
582
|
+
success: false,
|
|
583
|
+
error: apiCheck.error,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
const config = getEmbeddingsConfig();
|
|
587
|
+
const database = openEmbeddingsDb();
|
|
588
|
+
const query = params.query.trim();
|
|
589
|
+
if (!query) {
|
|
590
|
+
return {
|
|
591
|
+
success: false,
|
|
592
|
+
error: 'query is required',
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
const includeText = params.includeText === true;
|
|
596
|
+
const previewChars = clampNumber(params.previewChars, config.previewChars, 60, 1000);
|
|
597
|
+
const limit = clampNumber(params.limit, 10, 1, 100);
|
|
598
|
+
const minScore = Math.min(1, Math.max(0, Number(params.minScore ?? 0.2)));
|
|
599
|
+
const maxChunks = clampNumber(params.maxChunks, 8000, 1, 50000);
|
|
600
|
+
const queryVector = (await fetchEmbeddings([query]))[0];
|
|
601
|
+
const whereParts = [];
|
|
602
|
+
const whereParams = [];
|
|
603
|
+
if (params.space && params.space.trim().length > 0) {
|
|
604
|
+
whereParts.push('n.space_id = ?');
|
|
605
|
+
whereParams.push(params.space.trim());
|
|
606
|
+
}
|
|
607
|
+
if (params.source) {
|
|
608
|
+
whereParts.push('n.source = ?');
|
|
609
|
+
whereParams.push(params.source);
|
|
610
|
+
}
|
|
611
|
+
if (params.types && params.types.length > 0) {
|
|
612
|
+
const placeholders = params.types.map(() => '?').join(',');
|
|
613
|
+
whereParts.push(`n.type IN (${placeholders})`);
|
|
614
|
+
whereParams.push(...params.types);
|
|
615
|
+
}
|
|
616
|
+
const whereSql = whereParts.length > 0 ? `WHERE ${whereParts.join(' AND ')}` : '';
|
|
617
|
+
const rows = database
|
|
618
|
+
.prepare(`
|
|
619
|
+
SELECT
|
|
620
|
+
c.note_key,
|
|
621
|
+
c.chunk_index,
|
|
622
|
+
c.chunk_text,
|
|
623
|
+
c.chunk_preview,
|
|
624
|
+
c.embedding_json,
|
|
625
|
+
n.note_id,
|
|
626
|
+
n.filename,
|
|
627
|
+
n.title,
|
|
628
|
+
n.source,
|
|
629
|
+
n.space_id,
|
|
630
|
+
n.folder,
|
|
631
|
+
n.type,
|
|
632
|
+
n.modified_at
|
|
633
|
+
FROM chunks c
|
|
634
|
+
JOIN notes n ON n.note_key = c.note_key
|
|
635
|
+
${whereSql}
|
|
636
|
+
ORDER BY n.updated_at DESC, c.note_key ASC, c.chunk_index ASC
|
|
637
|
+
LIMIT ?
|
|
638
|
+
`)
|
|
639
|
+
.all(...whereParams, maxChunks);
|
|
640
|
+
const scored = rows
|
|
641
|
+
.map((row) => {
|
|
642
|
+
let chunkVector;
|
|
643
|
+
try {
|
|
644
|
+
chunkVector = JSON.parse(row.embedding_json);
|
|
645
|
+
}
|
|
646
|
+
catch {
|
|
647
|
+
chunkVector = [];
|
|
648
|
+
}
|
|
649
|
+
const score = cosineSimilarity(queryVector, chunkVector);
|
|
650
|
+
return { row, score };
|
|
651
|
+
})
|
|
652
|
+
.filter((entry) => Number.isFinite(entry.score) && entry.score >= minScore)
|
|
653
|
+
.sort((a, b) => b.score - a.score)
|
|
654
|
+
.slice(0, limit);
|
|
655
|
+
return {
|
|
656
|
+
success: true,
|
|
657
|
+
query,
|
|
658
|
+
provider: config.provider,
|
|
659
|
+
model: config.model,
|
|
660
|
+
includeText,
|
|
661
|
+
minScore,
|
|
662
|
+
scannedChunks: rows.length,
|
|
663
|
+
count: scored.length,
|
|
664
|
+
matches: scored.map((entry) => ({
|
|
665
|
+
score: Number(entry.score.toFixed(4)),
|
|
666
|
+
note: {
|
|
667
|
+
id: entry.row.note_id,
|
|
668
|
+
filename: entry.row.filename,
|
|
669
|
+
title: entry.row.title,
|
|
670
|
+
source: entry.row.source,
|
|
671
|
+
spaceId: entry.row.space_id,
|
|
672
|
+
folder: entry.row.folder,
|
|
673
|
+
type: entry.row.type,
|
|
674
|
+
modifiedAt: entry.row.modified_at,
|
|
675
|
+
},
|
|
676
|
+
chunk: {
|
|
677
|
+
index: entry.row.chunk_index,
|
|
678
|
+
preview: buildPreview(entry.row.chunk_preview, previewChars),
|
|
679
|
+
...(includeText ? { text: entry.row.chunk_text } : {}),
|
|
680
|
+
},
|
|
681
|
+
})),
|
|
682
|
+
};
|
|
683
|
+
}
|
|
684
|
+
//# sourceMappingURL=embeddings.js.map
|