@andespindola/brainlink 1.0.6 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +38 -0
- package/dist/application/find-similar-notes.js +75 -0
- package/dist/application/index-vault-phases.js +5 -4
- package/dist/application/index-vault.js +2 -2
- package/dist/application/memory-suggestions.js +4 -1
- package/dist/application/search-knowledge.js +19 -3
- package/dist/domain/context.js +27 -14
- package/dist/domain/diversity.js +64 -0
- package/dist/domain/embeddings.js +117 -1
- package/dist/domain/markdown.js +10 -1
- package/dist/domain/scoring.js +42 -0
- package/dist/domain/tokens.js +17 -1
- package/dist/infrastructure/config.js +27 -1
- package/dist/infrastructure/file-index.js +217 -59
- package/dist/mcp/server.js +16 -1
- package/dist/mcp/tools/read-tools.js +33 -0
- package/dist/mcp/tools/write-tools.js +106 -0
- package/dist/mcp/tools.js +2 -2
- package/docs/AGENT_USAGE.md +4 -1
- package/docs/ARCHITECTURE.md +4 -3
- package/package.json +1 -1
|
@@ -18,6 +18,7 @@ export const defaultBrainlinkConfig = {
|
|
|
18
18
|
defaultContextStrategy: 'auto',
|
|
19
19
|
defaultContextCacheTtlMs: 120_000,
|
|
20
20
|
embeddingProvider: 'local',
|
|
21
|
+
embedding: {},
|
|
21
22
|
defaultSearchMode: 'hybrid',
|
|
22
23
|
chunkSize: 1200,
|
|
23
24
|
searchPack: {
|
|
@@ -43,10 +44,34 @@ const safeCwd = () => {
|
|
|
43
44
|
}
|
|
44
45
|
};
|
|
45
46
|
const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
46
|
-
const embeddingProviders = new Set(['none', 'local']);
|
|
47
|
+
const embeddingProviders = new Set(['none', 'local', 'ollama', 'openai']);
|
|
47
48
|
const searchModes = new Set(['fts', 'semantic', 'hybrid']);
|
|
48
49
|
const contextStrategies = new Set(['rag', 'cag', 'auto']);
|
|
49
50
|
const sanitizeEmbeddingProvider = (value) => typeof value === 'string' && embeddingProviders.has(value) ? value : defaultBrainlinkConfig.embeddingProvider;
|
|
51
|
+
const sanitizeOptionalString = (value) => typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
|
52
|
+
const sanitizeEmbeddingConfig = (value) => {
|
|
53
|
+
if (!isRecord(value)) {
|
|
54
|
+
return {};
|
|
55
|
+
}
|
|
56
|
+
const url = sanitizeOptionalString(value.url);
|
|
57
|
+
const model = sanitizeOptionalString(value.model);
|
|
58
|
+
const apiKeyEnv = sanitizeOptionalString(value.apiKeyEnv);
|
|
59
|
+
const timeoutMs = sanitizePositiveNumber(value.timeoutMs);
|
|
60
|
+
const batchSize = sanitizePositiveNumber(value.batchSize);
|
|
61
|
+
return {
|
|
62
|
+
...(url ? { url } : {}),
|
|
63
|
+
...(model ? { model } : {}),
|
|
64
|
+
...(apiKeyEnv ? { apiKeyEnv } : {}),
|
|
65
|
+
...(timeoutMs ? { timeoutMs: Math.floor(timeoutMs) } : {}),
|
|
66
|
+
...(batchSize ? { batchSize: Math.floor(batchSize) } : {})
|
|
67
|
+
};
|
|
68
|
+
};
|
|
69
|
+
// Stable identity of the embedding space, used to decide when stored vectors
|
|
70
|
+
// must be rebuilt. A model swap on an external provider changes the space even
|
|
71
|
+
// though the provider name is unchanged, so the model is folded in.
|
|
72
|
+
export const embeddingSignature = (config) => config.embeddingProvider === 'ollama' || config.embeddingProvider === 'openai'
|
|
73
|
+
? `${config.embeddingProvider}:${config.embedding.model?.trim() || 'default'}`
|
|
74
|
+
: config.embeddingProvider;
|
|
50
75
|
export const sanitizeSearchMode = (value, fallback = defaultBrainlinkConfig.defaultSearchMode) => typeof value === 'string' && searchModes.has(value) ? value : fallback;
|
|
51
76
|
export const sanitizeContextStrategy = (value, fallback = defaultBrainlinkConfig.defaultContextStrategy) => typeof value === 'string' && contextStrategies.has(value) ? value : fallback;
|
|
52
77
|
const sanitizeAllowedVaults = (value) => Array.isArray(value) ? value.filter((item) => typeof item === 'string' && item.trim().length > 0) : [];
|
|
@@ -197,6 +222,7 @@ const sanitizeConfig = (value) => ({
|
|
|
197
222
|
chunkSize: typeof value.chunkSize === 'number' && value.chunkSize > 0 ? value.chunkSize : defaultBrainlinkConfig.chunkSize,
|
|
198
223
|
searchPack: sanitizeSearchPackConfig(value.searchPack),
|
|
199
224
|
embeddingProvider: sanitizeEmbeddingProvider(value.embeddingProvider),
|
|
225
|
+
embedding: sanitizeEmbeddingConfig(value.embedding),
|
|
200
226
|
defaultSearchMode: sanitizeSearchMode(value.defaultSearchMode),
|
|
201
227
|
agentProfiles: sanitizeAgentProfiles(value.agentProfiles)
|
|
202
228
|
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { createEmbeddingBuckets, dotProduct } from '../domain/embeddings.js';
|
|
4
|
+
import { bm25TermScore, minMaxNormalize } from '../domain/scoring.js';
|
|
4
5
|
import { selectSemanticCandidates } from './semantic-prefilter.js';
|
|
5
6
|
const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
|
|
6
7
|
const indexCacheMaxEntries = 16;
|
|
@@ -12,8 +13,48 @@ const emptyIndex = () => ({
|
|
|
12
13
|
chunks: [],
|
|
13
14
|
links: []
|
|
14
15
|
});
|
|
16
|
+
// Raised when index.json exists but cannot be parsed. Search treats it as a
|
|
17
|
+
// signal to fall back to the independent compressed search packs instead of
|
|
18
|
+
// silently returning an empty result set.
|
|
19
|
+
export class IndexCorruptionError extends Error {
|
|
20
|
+
indexPath;
|
|
21
|
+
constructor(indexPath, cause) {
|
|
22
|
+
super(`Corrupted index at ${indexPath}: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
23
|
+
this.indexPath = indexPath;
|
|
24
|
+
this.name = 'IndexCorruptionError';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
15
27
|
export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
|
|
16
|
-
const
|
|
28
|
+
const isMissingFileError = (error) => error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
|
29
|
+
// Paths already reported as corrupt, so a non-removable corrupt index (read-only
|
|
30
|
+
// vault, locked file) does not spam stderr on every read. Cleared when a fresh
|
|
31
|
+
// index is written to the path.
|
|
32
|
+
const warnedCorruptIndexPaths = new Set();
|
|
33
|
+
// Move a corrupt index aside so it is not read again and the next index run
|
|
34
|
+
// rebuilds from Markdown source, and surface a warning instead of failing
|
|
35
|
+
// silently. Quarantine is best-effort: a read-only vault simply keeps the file.
|
|
36
|
+
const quarantineCorruptIndex = async (path, cause) => {
|
|
37
|
+
indexCache.delete(path);
|
|
38
|
+
if (!warnedCorruptIndexPaths.has(path)) {
|
|
39
|
+
warnedCorruptIndexPaths.add(path);
|
|
40
|
+
process.stderr.write(`brainlink: index at ${path} is unreadable and was quarantined; rebuild with 'index --full' (${cause instanceof Error ? cause.message : String(cause)})\n`);
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
await rename(path, `${path}.corrupt`);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// The primary name can already exist (a prior corruption on a filesystem
|
|
47
|
+
// where rename does not overwrite, e.g. Windows). Fall back to a unique name
|
|
48
|
+
// so the damaged file is still moved out of the read path.
|
|
49
|
+
try {
|
|
50
|
+
await rename(path, `${path}.corrupt-${Date.now()}`);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
// Leave the file in place when it cannot be renamed; the warning still fired.
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
const readIndex = async (vaultPath, strict = false) => {
|
|
17
58
|
const path = indexStoragePath(vaultPath);
|
|
18
59
|
let stats = null;
|
|
19
60
|
try {
|
|
@@ -21,9 +62,8 @@ const readIndex = async (vaultPath) => {
|
|
|
21
62
|
stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
|
|
22
63
|
}
|
|
23
64
|
catch (error) {
|
|
24
|
-
if (error
|
|
65
|
+
if (isMissingFileError(error)) {
|
|
25
66
|
indexCache.delete(path);
|
|
26
|
-
return emptyIndex();
|
|
27
67
|
}
|
|
28
68
|
return emptyIndex();
|
|
29
69
|
}
|
|
@@ -31,31 +71,46 @@ const readIndex = async (vaultPath) => {
|
|
|
31
71
|
if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
|
|
32
72
|
return cached.index;
|
|
33
73
|
}
|
|
74
|
+
let raw;
|
|
34
75
|
try {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (indexCache.size > indexCacheMaxEntries) {
|
|
45
|
-
const oldest = indexCache.keys().next().value;
|
|
46
|
-
if (typeof oldest === 'string') {
|
|
47
|
-
indexCache.delete(oldest);
|
|
48
|
-
}
|
|
76
|
+
raw = await readFile(path, 'utf8');
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
indexCache.delete(path);
|
|
80
|
+
// The file existed at stat but vanished before the read — a concurrent
|
|
81
|
+
// rebuild or quarantine. Strict (search) callers throw so they fall back to
|
|
82
|
+
// packs rather than racing to an empty result; tolerant callers get empty.
|
|
83
|
+
if (strict) {
|
|
84
|
+
throw new IndexCorruptionError(path, error);
|
|
49
85
|
}
|
|
50
|
-
return
|
|
86
|
+
return emptyIndex();
|
|
87
|
+
}
|
|
88
|
+
let parsed;
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(raw);
|
|
51
91
|
}
|
|
52
92
|
catch (error) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
93
|
+
await quarantineCorruptIndex(path, error);
|
|
94
|
+
if (strict) {
|
|
95
|
+
throw new IndexCorruptionError(path, error);
|
|
56
96
|
}
|
|
57
97
|
return emptyIndex();
|
|
58
98
|
}
|
|
99
|
+
const loaded = {
|
|
100
|
+
version: 1,
|
|
101
|
+
updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
|
|
102
|
+
documents: Array.isArray(parsed.documents) ? parsed.documents : [],
|
|
103
|
+
chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
|
|
104
|
+
links: Array.isArray(parsed.links) ? parsed.links : []
|
|
105
|
+
};
|
|
106
|
+
indexCache.set(path, { ...stats, index: loaded });
|
|
107
|
+
if (indexCache.size > indexCacheMaxEntries) {
|
|
108
|
+
const oldest = indexCache.keys().next().value;
|
|
109
|
+
if (typeof oldest === 'string') {
|
|
110
|
+
indexCache.delete(oldest);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return loaded;
|
|
59
114
|
};
|
|
60
115
|
const writeIndex = async (vaultPath, index) => {
|
|
61
116
|
const target = indexStoragePath(vaultPath);
|
|
@@ -63,6 +118,7 @@ const writeIndex = async (vaultPath, index) => {
|
|
|
63
118
|
await mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
|
64
119
|
await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
65
120
|
await rename(temp, target);
|
|
121
|
+
warnedCorruptIndexPaths.delete(target);
|
|
66
122
|
const fileStats = await stat(target);
|
|
67
123
|
indexCache.set(target, {
|
|
68
124
|
mtimeMs: fileStats.mtimeMs,
|
|
@@ -91,40 +147,89 @@ const countOccurrences = (text, token) => {
|
|
|
91
147
|
}
|
|
92
148
|
return hits;
|
|
93
149
|
};
|
|
94
|
-
const
|
|
150
|
+
const titleFieldWeight = 5;
|
|
151
|
+
const tagFieldWeight = 4;
|
|
152
|
+
const pathFieldWeight = 2;
|
|
153
|
+
const contentHitCap = 6;
|
|
154
|
+
const hybridTextWeight = 0.6;
|
|
155
|
+
const hybridSemanticWeight = 0.4;
|
|
156
|
+
const normalizeFields = (row) => ({
|
|
157
|
+
title: normalizeToken(row.title),
|
|
158
|
+
path: normalizeToken(row.path),
|
|
159
|
+
content: normalizeToken(row.content),
|
|
160
|
+
tags: normalizeToken(row.tags.join(' '))
|
|
161
|
+
});
|
|
162
|
+
// Field-weighted term frequency: title/tag/path matches count more than body
|
|
163
|
+
// matches, and body hits saturate at a cap so a term repeated many times in one
|
|
164
|
+
// chunk cannot dominate. This weighted frequency feeds BM25 below.
|
|
165
|
+
const weightedTermFrequency = (fields, token) => {
|
|
166
|
+
const titleHits = countOccurrences(fields.title, token);
|
|
167
|
+
const tagHits = countOccurrences(fields.tags, token);
|
|
168
|
+
const pathHits = countOccurrences(fields.path, token);
|
|
169
|
+
const contentHits = countOccurrences(fields.content, token);
|
|
170
|
+
return (titleHits * titleFieldWeight +
|
|
171
|
+
tagHits * tagFieldWeight +
|
|
172
|
+
pathHits * pathFieldWeight +
|
|
173
|
+
Math.min(contentHits, contentHitCap));
|
|
174
|
+
};
|
|
175
|
+
// Naive additive field score retained for graph-node title/path lookup, where
|
|
176
|
+
// the candidate set is documents (not chunks) and corpus-wide BM25 statistics
|
|
177
|
+
// add no signal over a direct weighted hit count.
|
|
178
|
+
const lexicalFieldScore = (row, tokens) => {
|
|
95
179
|
if (tokens.length === 0) {
|
|
96
180
|
return 0;
|
|
97
181
|
}
|
|
98
|
-
const
|
|
99
|
-
|
|
100
|
-
const content = normalizeToken(row.content);
|
|
101
|
-
const tags = normalizeToken(row.tags.join(' '));
|
|
102
|
-
return tokens.reduce((score, token) => {
|
|
103
|
-
const titleHits = countOccurrences(title, token);
|
|
104
|
-
const tagHits = countOccurrences(tags, token);
|
|
105
|
-
const pathHits = countOccurrences(path, token);
|
|
106
|
-
const contentHits = countOccurrences(content, token);
|
|
107
|
-
return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 6);
|
|
108
|
-
}, 0);
|
|
182
|
+
const fields = normalizeFields(row);
|
|
183
|
+
return tokens.reduce((score, token) => score + weightedTermFrequency(fields, token), 0);
|
|
109
184
|
};
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
185
|
+
// A single chunk with a non-finite cached tokenCount (legacy index, partial
|
|
186
|
+
// write, manual edit) would otherwise poison the shared averageLength and zero
|
|
187
|
+
// out lexical scoring for the whole corpus, so coerce to a safe positive length.
|
|
188
|
+
const safeDocumentLength = (tokenCount) => Number.isFinite(tokenCount) && tokenCount > 0 ? tokenCount : 1;
|
|
189
|
+
const buildCorpusStatistics = (prepared, tokens) => {
|
|
190
|
+
const documentCount = prepared.length;
|
|
191
|
+
const totalLength = prepared.reduce((sum, entry) => sum + safeDocumentLength(entry.row.tokenCount), 0);
|
|
192
|
+
const documentFrequencyByTerm = new Map();
|
|
193
|
+
for (const term of tokens) {
|
|
194
|
+
let documentFrequency = 0;
|
|
195
|
+
for (const entry of prepared) {
|
|
196
|
+
if ((entry.weightedTermFrequencies.get(term) ?? 0) > 0) {
|
|
197
|
+
documentFrequency += 1;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
documentFrequencyByTerm.set(term, documentFrequency);
|
|
201
|
+
}
|
|
113
202
|
return {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
path: row.path,
|
|
118
|
-
chunkId: row.chunkId,
|
|
119
|
-
chunkOrdinal: row.chunkOrdinal,
|
|
120
|
-
content: row.content,
|
|
121
|
-
score,
|
|
122
|
-
textScore: text,
|
|
123
|
-
semanticScore: semantic,
|
|
124
|
-
searchMode: mode,
|
|
125
|
-
tags: row.tags
|
|
203
|
+
documentCount,
|
|
204
|
+
averageLength: documentCount > 0 ? totalLength / documentCount : 0,
|
|
205
|
+
documentFrequencyByTerm
|
|
126
206
|
};
|
|
127
207
|
};
|
|
208
|
+
const bm25TextScore = (entry, tokens, stats) => tokens.reduce((score, term) => score +
|
|
209
|
+
bm25TermScore({
|
|
210
|
+
termFrequency: entry.weightedTermFrequencies.get(term) ?? 0,
|
|
211
|
+
documentLength: safeDocumentLength(entry.row.tokenCount),
|
|
212
|
+
documentCount: stats.documentCount,
|
|
213
|
+
documentFrequency: stats.documentFrequencyByTerm.get(term) ?? 0,
|
|
214
|
+
averageLength: stats.averageLength
|
|
215
|
+
}), 0);
|
|
216
|
+
const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? dotProduct(queryEmbedding, row.embedding) : 0;
|
|
217
|
+
const rangeOf = (values) => values.reduce((range, value) => ({ min: Math.min(range.min, value), max: Math.max(range.max, value) }), { min: Infinity, max: -Infinity });
|
|
218
|
+
const toResult = (row, mode, score, text, semantic) => ({
|
|
219
|
+
documentId: row.documentId,
|
|
220
|
+
agentId: row.agentId,
|
|
221
|
+
title: row.title,
|
|
222
|
+
path: row.path,
|
|
223
|
+
chunkId: row.chunkId,
|
|
224
|
+
chunkOrdinal: row.chunkOrdinal,
|
|
225
|
+
content: row.content,
|
|
226
|
+
score,
|
|
227
|
+
textScore: text,
|
|
228
|
+
semanticScore: semantic,
|
|
229
|
+
tokenCount: row.tokenCount,
|
|
230
|
+
searchMode: mode,
|
|
231
|
+
tags: row.tags
|
|
232
|
+
});
|
|
128
233
|
const toGraphLink = (link, documentsById) => {
|
|
129
234
|
const source = documentsById.get(link.fromDocumentId);
|
|
130
235
|
const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
|
|
@@ -187,7 +292,9 @@ export const openFileIndex = (vaultPath) => {
|
|
|
187
292
|
.sort((left, right) => left.document.path.localeCompare(right.document.path));
|
|
188
293
|
},
|
|
189
294
|
search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
|
|
190
|
-
|
|
295
|
+
// Strict load: a corrupt index throws so the caller can fall back to the
|
|
296
|
+
// independent search packs rather than reporting "no results".
|
|
297
|
+
const index = await readIndex(vaultPath, true);
|
|
191
298
|
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
192
299
|
const rows = index.chunks.flatMap((chunk) => {
|
|
193
300
|
const document = documentsById.get(chunk.documentId);
|
|
@@ -206,6 +313,7 @@ export const openFileIndex = (vaultPath) => {
|
|
|
206
313
|
chunkId: chunk.id,
|
|
207
314
|
chunkOrdinal: chunk.ordinal,
|
|
208
315
|
content: chunk.content,
|
|
316
|
+
tokenCount: chunk.tokenCount,
|
|
209
317
|
tags: document.tags,
|
|
210
318
|
embedding: chunk.embedding,
|
|
211
319
|
buckets: chunk.buckets
|
|
@@ -217,19 +325,68 @@ export const openFileIndex = (vaultPath) => {
|
|
|
217
325
|
// embedding bucket with the query. Hybrid/fts keep every row so lexical
|
|
218
326
|
// matches are never dropped; the prefilter also falls back to a full scan
|
|
219
327
|
// on small or partially-indexed vaults.
|
|
220
|
-
const
|
|
328
|
+
const candidates = mode === 'semantic' && queryEmbedding.length > 0
|
|
221
329
|
? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
|
|
222
330
|
: rows;
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
331
|
+
// Pass 1: field-weighted term frequencies per candidate, then corpus-wide
|
|
332
|
+
// statistics (document frequency, average length) needed for BM25 IDF and
|
|
333
|
+
// length normalization.
|
|
334
|
+
const prepared = candidates.map((row) => {
|
|
335
|
+
const fields = normalizeFields(row);
|
|
336
|
+
const weightedTermFrequencies = new Map();
|
|
337
|
+
for (const term of tokens) {
|
|
338
|
+
const frequency = weightedTermFrequency(fields, term);
|
|
339
|
+
if (frequency > 0) {
|
|
340
|
+
weightedTermFrequencies.set(term, frequency);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
return { row, weightedTermFrequencies };
|
|
344
|
+
});
|
|
345
|
+
const stats = buildCorpusStatistics(prepared, tokens);
|
|
346
|
+
// Pass 2: raw lexical (BM25) and semantic (cosine) component scores.
|
|
347
|
+
const componentScored = prepared.map((entry) => ({
|
|
348
|
+
row: entry.row,
|
|
349
|
+
text: tokens.length > 0 ? bm25TextScore(entry, tokens, stats) : 0,
|
|
350
|
+
semantic: semanticScore(entry.row, queryEmbedding)
|
|
351
|
+
}));
|
|
352
|
+
// Filter on raw relevance before normalization so a legitimate match that
|
|
353
|
+
// happens to be the weakest in its mode is never min-max scaled to zero
|
|
354
|
+
// and dropped. fts stays lexical-only and semantic stays vector-only.
|
|
355
|
+
const relevant = componentScored.filter((entry) => {
|
|
356
|
+
if (tokens.length === 0) {
|
|
357
|
+
return true;
|
|
358
|
+
}
|
|
359
|
+
if (mode === 'fts') {
|
|
360
|
+
return entry.text > 0;
|
|
361
|
+
}
|
|
362
|
+
if (mode === 'semantic') {
|
|
363
|
+
return entry.semantic > 0;
|
|
364
|
+
}
|
|
365
|
+
return entry.text > 0 || entry.semantic > 0;
|
|
366
|
+
});
|
|
367
|
+
if (relevant.length === 0) {
|
|
368
|
+
return [];
|
|
369
|
+
}
|
|
370
|
+
const textRange = rangeOf(relevant.map((entry) => entry.text));
|
|
371
|
+
const semanticRange = rangeOf(relevant.map((entry) => entry.semantic));
|
|
372
|
+
const finalScore = (entry) => {
|
|
373
|
+
if (mode === 'fts') {
|
|
374
|
+
return entry.text;
|
|
375
|
+
}
|
|
376
|
+
if (mode === 'semantic') {
|
|
377
|
+
return entry.semantic;
|
|
378
|
+
}
|
|
379
|
+
// Hybrid mixes the two components on a shared [0, 1] scale so neither the
|
|
380
|
+
// unbounded BM25 magnitude nor the bounded cosine value can swamp the
|
|
381
|
+
// other, replacing the previous fixed `text + semantic * 8` blend.
|
|
382
|
+
const textNorm = minMaxNormalize(entry.text, textRange.min, textRange.max);
|
|
383
|
+
const semanticNorm = minMaxNormalize(entry.semantic, semanticRange.min, semanticRange.max);
|
|
384
|
+
return textNorm * hybridTextWeight + semanticNorm * hybridSemanticWeight;
|
|
385
|
+
};
|
|
386
|
+
return relevant
|
|
387
|
+
.map((entry) => toResult(entry.row, mode, finalScore(entry), entry.text, entry.semantic))
|
|
230
388
|
.sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
|
|
231
389
|
.slice(0, Math.max(0, limit));
|
|
232
|
-
return results;
|
|
233
390
|
},
|
|
234
391
|
listLinks: async (agentId) => {
|
|
235
392
|
const index = await load();
|
|
@@ -335,7 +492,7 @@ export const openFileIndex = (vaultPath) => {
|
|
|
335
492
|
const scored = index.documents
|
|
336
493
|
.filter((document) => (!agentId || document.agentId === agentId))
|
|
337
494
|
.map((document) => {
|
|
338
|
-
const score =
|
|
495
|
+
const score = lexicalFieldScore({
|
|
339
496
|
documentId: document.id,
|
|
340
497
|
agentId: document.agentId,
|
|
341
498
|
title: document.title,
|
|
@@ -343,6 +500,7 @@ export const openFileIndex = (vaultPath) => {
|
|
|
343
500
|
chunkId: document.id,
|
|
344
501
|
chunkOrdinal: 0,
|
|
345
502
|
content: document.content,
|
|
503
|
+
tokenCount: 0,
|
|
346
504
|
tags: document.tags,
|
|
347
505
|
embedding: []
|
|
348
506
|
}, tokens);
|
package/dist/mcp/server.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
-
import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, deleteNoteInputSchema, deleteNoteTool, volatileAddInputSchema, volatileAddTool, volatileClearInputSchema, volatileClearTool, dedupeInputSchema, dedupeResolveInputSchema, dedupeResolveTool, dedupeTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, canonicalizeContextLinksInputSchema, canonicalizeContextLinksTool, contextInputSchema, contextPacksInputSchema, contextPacksTool, contextTool, doctorActionsInputSchema, doctorActionsTool, graphContextsInputSchema, graphContextsTool, graphInputSchema, graphTool, explainInputSchema, explainTool, indexInputSchema, indexTool, inboxAddInputSchema, inboxAddTool, inboxListInputSchema, inboxListTool, inboxProcessInputSchema, inboxProcessTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, projectInitInputSchema, projectInitTool, recommendationsInputSchema, recommendationsTool, rememberInputSchema, rememberTool, repairLinksInputSchema, repairLinksTool, searchInputSchema, searchTool, sessionCloseInputSchema, sessionCloseTool, statsInputSchema, statsTool, suggestLinksInputSchema, suggestLinksTool, syncInputSchema, syncTool, validateInputSchema, validateTool, versionInputSchema, versionTool } from './tools.js';
|
|
2
|
+
import { addNoteInputSchema, addFileInputSchema, addFileTool, addNoteTool, addNotesInputSchema, addNotesTool, deleteNoteInputSchema, deleteNoteTool, deleteNotesInputSchema, deleteNotesTool, volatileAddInputSchema, volatileAddTool, volatileClearInputSchema, volatileClearTool, dedupeInputSchema, dedupeResolveInputSchema, dedupeResolveTool, dedupeTool, brokenLinksInputSchema, brokenLinksTool, bootstrapInputSchema, bootstrapTool, canonicalizeContextLinksInputSchema, canonicalizeContextLinksTool, contextInputSchema, contextPacksInputSchema, contextPacksTool, contextTool, doctorActionsInputSchema, doctorActionsTool, graphContextsInputSchema, graphContextsTool, graphInputSchema, graphTool, explainInputSchema, explainTool, indexInputSchema, indexTool, inboxAddInputSchema, inboxAddTool, inboxListInputSchema, inboxListTool, inboxProcessInputSchema, inboxProcessTool, orphansInputSchema, orphansTool, policyInputSchema, policyTool, projectInitInputSchema, projectInitTool, recommendationsInputSchema, recommendationsTool, rememberInputSchema, rememberTool, repairLinksInputSchema, repairLinksTool, searchInputSchema, searchTool, similarNotesInputSchema, similarNotesTool, sessionCloseInputSchema, sessionCloseTool, statsInputSchema, statsTool, suggestLinksInputSchema, suggestLinksTool, syncInputSchema, syncTool, validateInputSchema, validateTool, versionInputSchema, versionTool } from './tools.js';
|
|
3
3
|
import { getRuntimeVersion } from './runtime.js';
|
|
4
4
|
import { guardToolHandler } from './tool-guard.js';
|
|
5
5
|
export const createBrainlinkMcpServer = () => {
|
|
@@ -50,6 +50,11 @@ export const createBrainlinkMcpServer = () => {
|
|
|
50
50
|
description: 'Search indexed Brainlink notes with FTS, semantic or hybrid retrieval.',
|
|
51
51
|
inputSchema: searchInputSchema
|
|
52
52
|
}, searchTool);
|
|
53
|
+
server.registerTool('brainlink_similar_notes', {
|
|
54
|
+
title: 'Find Similar Brainlink Notes',
|
|
55
|
+
description: 'Find the notes most similar to a given note (selected by title or path), using the note content as the retrieval query and excluding the note itself.',
|
|
56
|
+
inputSchema: similarNotesInputSchema
|
|
57
|
+
}, similarNotesTool);
|
|
53
58
|
server.registerTool('brainlink_explain', {
|
|
54
59
|
title: 'Explain Brainlink Search Results',
|
|
55
60
|
description: 'Explain why indexed notes matched a query, including score components and match reasons.',
|
|
@@ -70,6 +75,11 @@ export const createBrainlinkMcpServer = () => {
|
|
|
70
75
|
description: 'Write durable Markdown memory, then reindex the vault. Include explicit [[wiki links]] for connected graph memory. Add priority markers near links, such as priority: high, #important or #critical, when a relationship should be weighted higher.',
|
|
71
76
|
inputSchema: addNoteInputSchema
|
|
72
77
|
}, addNoteTool);
|
|
78
|
+
server.registerTool('brainlink_add_notes', {
|
|
79
|
+
title: 'Add Brainlink Notes (batch)',
|
|
80
|
+
description: 'Write several durable Markdown notes in one call, reindexing the vault once after all are written. Use for bulk capture instead of repeated brainlink_add_note calls.',
|
|
81
|
+
inputSchema: addNotesInputSchema
|
|
82
|
+
}, addNotesTool);
|
|
73
83
|
server.registerTool('brainlink_remember', {
|
|
74
84
|
title: 'Capture Assisted Brainlink Memory',
|
|
75
85
|
description: 'Capture durable memory with inferred title, tags and Context Links. Supports dry-run preview before writing.',
|
|
@@ -95,6 +105,11 @@ export const createBrainlinkMcpServer = () => {
|
|
|
95
105
|
description: 'Delete a durable Markdown note from the vault after explicit confirmation. Select by title or path; reindexes by default.',
|
|
96
106
|
inputSchema: deleteNoteInputSchema
|
|
97
107
|
}, deleteNoteTool);
|
|
108
|
+
server.registerTool('brainlink_delete_notes', {
|
|
109
|
+
title: 'Delete Brainlink Notes (batch)',
|
|
110
|
+
description: 'Delete several notes in one confirmed call, reindexing once after the batch. Each note is selected by exactly one of title or path; per-note failures are reported without aborting the batch.',
|
|
111
|
+
inputSchema: deleteNotesInputSchema
|
|
112
|
+
}, deleteNotesTool);
|
|
98
113
|
server.registerTool('brainlink_volatile_add', {
|
|
99
114
|
title: 'Add Volatile Brainlink Memory',
|
|
100
115
|
description: 'Write temporary agent-decided memory with TTL. Use for transient task state without polluting durable Markdown memory.',
|
|
@@ -6,6 +6,7 @@ import { getGraphContexts } from '../../application/get-graph-contexts.js';
|
|
|
6
6
|
import { explainSearchResults, suggestBrokenLinkFixes, suggestContextLinks } from '../../application/memory-suggestions.js';
|
|
7
7
|
import { buildActionableDoctor } from '../../application/operational-workflows.js';
|
|
8
8
|
import { searchKnowledge } from '../../application/search-knowledge.js';
|
|
9
|
+
import { findSimilarNotes } from '../../application/find-similar-notes.js';
|
|
9
10
|
import { sanitizeContextStrategy, sanitizeSearchMode } from '../../infrastructure/config.js';
|
|
10
11
|
import { clearContextPacks, listContextPacks } from '../../infrastructure/context-packs.js';
|
|
11
12
|
import { getBootstrapPolicy, getBootstrapSessionStatus, getContextSessionStatus, touchContextSession } from '../../infrastructure/session-state.js';
|
|
@@ -40,6 +41,14 @@ export const explainInputSchema = {
|
|
|
40
41
|
query: z.string().min(1).describe('Search query to explain.'),
|
|
41
42
|
limit: optionalPositiveInteger().describe('Maximum result count.')
|
|
42
43
|
};
|
|
44
|
+
export const similarNotesInputSchema = {
|
|
45
|
+
...vaultInput,
|
|
46
|
+
...agentInput,
|
|
47
|
+
...searchModeInput,
|
|
48
|
+
title: z.string().min(1).optional().describe('Title of the note to find neighbors for. Use agent to disambiguate.'),
|
|
49
|
+
path: z.string().min(1).optional().describe('Vault-relative or absolute path of the note to find neighbors for.'),
|
|
50
|
+
limit: optionalPositiveInteger().describe('Maximum number of similar notes to return. Defaults to the configured search limit.')
|
|
51
|
+
};
|
|
43
52
|
export const validateInputSchema = {
|
|
44
53
|
...vaultInput,
|
|
45
54
|
...agentInput
|
|
@@ -162,6 +171,30 @@ export const searchTool = async (input) => {
|
|
|
162
171
|
results
|
|
163
172
|
});
|
|
164
173
|
};
|
|
174
|
+
export const similarNotesTool = async (input) => {
|
|
175
|
+
const context = await resolveExecutionContext(input);
|
|
176
|
+
const readiness = await ensureBootstrapReady(context, input, 'brainlink_similar_notes');
|
|
177
|
+
if (readiness.preflight) {
|
|
178
|
+
return readiness.preflight;
|
|
179
|
+
}
|
|
180
|
+
const mode = sanitizeSearchMode(input.mode, context.defaults.defaultSearchMode);
|
|
181
|
+
const limit = input.limit ?? context.defaults.defaultSearchLimit;
|
|
182
|
+
const result = await findSimilarNotes(context.vault, {
|
|
183
|
+
title: input.title,
|
|
184
|
+
path: input.path,
|
|
185
|
+
agentId: context.agent,
|
|
186
|
+
limit,
|
|
187
|
+
mode
|
|
188
|
+
});
|
|
189
|
+
return jsonResult({
|
|
190
|
+
vault: context.vault,
|
|
191
|
+
agent: context.agent,
|
|
192
|
+
mode,
|
|
193
|
+
limit,
|
|
194
|
+
...(readiness.bootstrap ? { bootstrap: readiness.bootstrap } : {}),
|
|
195
|
+
...result
|
|
196
|
+
});
|
|
197
|
+
};
|
|
165
198
|
export const explainTool = async (input) => {
|
|
166
199
|
const context = await resolveExecutionContext(input);
|
|
167
200
|
const readiness = await ensureBootstrapReady(context, input, 'brainlink_explain');
|
|
@@ -59,6 +59,38 @@ export const deleteNoteInputSchema = {
|
|
|
59
59
|
confirm: z.boolean().describe('Must be true to confirm deletion.'),
|
|
60
60
|
autoIndex: z.boolean().optional().default(true).describe('Reindex vault after deletion. Defaults to true.')
|
|
61
61
|
};
|
|
62
|
+
export const addNotesInputSchema = {
|
|
63
|
+
...vaultInput,
|
|
64
|
+
...agentInput,
|
|
65
|
+
notes: z
|
|
66
|
+
.array(z.object({
|
|
67
|
+
title: z.string().min(1).describe('Markdown note title.'),
|
|
68
|
+
content: z.string().min(1).describe('Durable Markdown memory. Include [[wiki links]] and #tags to connect it.')
|
|
69
|
+
}))
|
|
70
|
+
.min(1)
|
|
71
|
+
.max(100)
|
|
72
|
+
.describe('Notes to add in a single batch. The vault is reindexed once after all notes are written.'),
|
|
73
|
+
allowSensitive: z.boolean().optional().default(false).describe('Allow content that looks like a secret.'),
|
|
74
|
+
autoIndex: z.boolean().optional().default(true).describe('Reindex vault once after writing all notes.'),
|
|
75
|
+
autoContextLinks: z
|
|
76
|
+
.boolean()
|
|
77
|
+
.optional()
|
|
78
|
+
.describe('Automatically add canonical Context Links to the inferred context hub. Defaults to Brainlink config.')
|
|
79
|
+
};
|
|
80
|
+
export const deleteNotesInputSchema = {
|
|
81
|
+
...vaultInput,
|
|
82
|
+
...agentInput,
|
|
83
|
+
notes: z
|
|
84
|
+
.array(z.object({
|
|
85
|
+
title: z.string().min(1).optional().describe('Note title to delete.'),
|
|
86
|
+
path: z.string().min(1).optional().describe('Vault-relative or absolute Markdown note path to delete.')
|
|
87
|
+
}))
|
|
88
|
+
.min(1)
|
|
89
|
+
.max(100)
|
|
90
|
+
.describe('Notes to delete in a single batch, each selected by exactly one of title or path.'),
|
|
91
|
+
confirm: z.boolean().describe('Must be true to confirm deletion of every listed note.'),
|
|
92
|
+
autoIndex: z.boolean().optional().default(true).describe('Reindex vault once after all deletions.')
|
|
93
|
+
};
|
|
62
94
|
export const volatileAddInputSchema = {
|
|
63
95
|
...vaultInput,
|
|
64
96
|
...agentInput,
|
|
@@ -196,6 +228,80 @@ export const deleteNoteTool = async (input) => {
|
|
|
196
228
|
...result
|
|
197
229
|
});
|
|
198
230
|
};
|
|
231
|
+
export const addNotesTool = async (input) => {
|
|
232
|
+
const context = await resolveExecutionContext(input);
|
|
233
|
+
const autoContextLinks = input.autoContextLinks ?? context.config.autoCanonicalContextLinks;
|
|
234
|
+
// Per-note isolation mirrors deleteNotesTool: one failing note (secret,
|
|
235
|
+
// validation, FS error) is reported without discarding the notes that did
|
|
236
|
+
// write or skipping the single end-of-batch reindex.
|
|
237
|
+
const notes = [];
|
|
238
|
+
for (const note of input.notes) {
|
|
239
|
+
try {
|
|
240
|
+
const added = await addNoteWithMetadata(context.vault, note.title, note.content, context.agent, {
|
|
241
|
+
allowSensitive: input.allowSensitive,
|
|
242
|
+
autoContextLinks
|
|
243
|
+
});
|
|
244
|
+
notes.push({
|
|
245
|
+
ok: true,
|
|
246
|
+
title: note.title,
|
|
247
|
+
path: added.path,
|
|
248
|
+
writeConnectivity: {
|
|
249
|
+
autoLinked: added.autoLinked,
|
|
250
|
+
linkTarget: added.linkTarget,
|
|
251
|
+
context: added.context,
|
|
252
|
+
hubCreated: added.hubCreated,
|
|
253
|
+
guaranteedEdge: added.autoLinked
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
catch (error) {
|
|
258
|
+
notes.push({ ok: false, title: note.title, error: error instanceof Error ? error.message : String(error) });
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const added = notes.filter((entry) => entry.ok === true).length;
|
|
262
|
+
const index = isTruthy(input.autoIndex) && added > 0 ? await indexVault(context.vault) : undefined;
|
|
263
|
+
return jsonResult({
|
|
264
|
+
vault: context.vault,
|
|
265
|
+
agent: context.agent,
|
|
266
|
+
added,
|
|
267
|
+
failed: notes.length - added,
|
|
268
|
+
notes,
|
|
269
|
+
...(index ? { index } : {})
|
|
270
|
+
});
|
|
271
|
+
};
|
|
272
|
+
export const deleteNotesTool = async (input) => {
|
|
273
|
+
const context = await resolveExecutionContext(input);
|
|
274
|
+
if (!input.confirm) {
|
|
275
|
+
throw new Error('Refusing to delete notes without explicit confirmation.');
|
|
276
|
+
}
|
|
277
|
+
const results = [];
|
|
278
|
+
for (const note of input.notes) {
|
|
279
|
+
try {
|
|
280
|
+
// Defer indexing: reindex once after the whole batch instead of per note.
|
|
281
|
+
const result = await deleteNote(context.vault, {
|
|
282
|
+
title: note.title,
|
|
283
|
+
path: note.path,
|
|
284
|
+
agentId: context.agent,
|
|
285
|
+
confirm: true,
|
|
286
|
+
autoIndex: false
|
|
287
|
+
});
|
|
288
|
+
results.push({ ok: true, title: result.title, path: result.relativePath });
|
|
289
|
+
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
results.push({ ok: false, selector: note, error: error instanceof Error ? error.message : String(error) });
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const deleted = results.filter((entry) => entry.ok === true).length;
|
|
295
|
+
const index = isTruthy(input.autoIndex) && deleted > 0 ? await indexVault(context.vault) : undefined;
|
|
296
|
+
return jsonResult({
|
|
297
|
+
vault: context.vault,
|
|
298
|
+
agent: context.agent,
|
|
299
|
+
deleted,
|
|
300
|
+
failed: results.length - deleted,
|
|
301
|
+
results,
|
|
302
|
+
...(index ? { index } : {})
|
|
303
|
+
});
|
|
304
|
+
};
|
|
199
305
|
export const volatileAddTool = async (input) => {
|
|
200
306
|
const context = await resolveExecutionContext(input);
|
|
201
307
|
const entry = await addVolatileMemory(context.vault, input.content, context.agent ?? 'shared', input.ttlMinutes ?? 240, input.tags);
|
package/dist/mcp/tools.js
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
export { brokenLinksInputSchema, brokenLinksTool, contextInputSchema, contextPacksInputSchema, contextPacksTool, contextTool, doctorActionsInputSchema, doctorActionsTool, explainInputSchema, explainTool, graphContextsInputSchema, graphContextsTool, graphInputSchema, graphTool, orphansInputSchema, orphansTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, statsInputSchema, statsTool, suggestLinksInputSchema, suggestLinksTool, validateInputSchema, validateTool, versionInputSchema, versionTool } from './tools/read-tools.js';
|
|
2
|
-
export { addFileInputSchema, addFileTool, addNoteInputSchema, addNoteTool, deleteNoteInputSchema, deleteNoteTool, inboxAddInputSchema, inboxAddTool, inboxListInputSchema, inboxListTool, inboxProcessInputSchema, inboxProcessTool, rememberInputSchema, rememberTool, volatileAddInputSchema, volatileAddTool, volatileClearInputSchema, volatileClearTool } from './tools/write-tools.js';
|
|
1
|
+
export { brokenLinksInputSchema, brokenLinksTool, contextInputSchema, contextPacksInputSchema, contextPacksTool, contextTool, doctorActionsInputSchema, doctorActionsTool, explainInputSchema, explainTool, graphContextsInputSchema, graphContextsTool, graphInputSchema, graphTool, orphansInputSchema, orphansTool, recommendationsInputSchema, recommendationsTool, searchInputSchema, searchTool, similarNotesInputSchema, similarNotesTool, statsInputSchema, statsTool, suggestLinksInputSchema, suggestLinksTool, validateInputSchema, validateTool, versionInputSchema, versionTool } from './tools/read-tools.js';
|
|
2
|
+
export { addFileInputSchema, addFileTool, addNoteInputSchema, addNoteTool, addNotesInputSchema, addNotesTool, deleteNoteInputSchema, deleteNoteTool, deleteNotesInputSchema, deleteNotesTool, inboxAddInputSchema, inboxAddTool, inboxListInputSchema, inboxListTool, inboxProcessInputSchema, inboxProcessTool, rememberInputSchema, rememberTool, volatileAddInputSchema, volatileAddTool, volatileClearInputSchema, volatileClearTool } from './tools/write-tools.js';
|
|
3
3
|
export { bootstrapInputSchema, bootstrapTool, canonicalizeContextLinksInputSchema, canonicalizeContextLinksTool, dedupeInputSchema, dedupeResolveInputSchema, dedupeResolveTool, dedupeTool, indexInputSchema, indexTool, policyInputSchema, policyTool, projectInitInputSchema, projectInitTool, repairLinksInputSchema, repairLinksTool, sessionCloseInputSchema, sessionCloseTool, syncInputSchema, syncTool } from './tools/maintenance-tools.js';
|