@andespindola/brainlink 0.1.0-beta.7 → 0.1.0-beta.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -5
- package/CHANGELOG.md +58 -2
- package/CONTRIBUTING.md +2 -2
- package/COPYRIGHT.md +5 -0
- package/README.md +266 -20
- package/SECURITY.md +1 -1
- package/dist/application/add-note.js +62 -13
- package/dist/application/analyze-vault.js +95 -8
- package/dist/application/build-context.js +56 -1
- package/dist/application/dedupe-notes.js +226 -0
- package/dist/application/frontend/client-css.js +138 -103
- package/dist/application/frontend/client-html.js +47 -41
- package/dist/application/frontend/client-js.js +2111 -128
- package/dist/application/frontend/client-worker-js.js +66 -0
- package/dist/application/get-graph-layout.js +18 -6
- package/dist/application/get-graph-node.js +12 -0
- package/dist/application/get-graph-summary.js +12 -0
- package/dist/application/get-graph.js +3 -3
- package/dist/application/import-legacy-sqlite.js +296 -0
- package/dist/application/index-vault.js +252 -19
- package/dist/application/list-agents.js +3 -3
- package/dist/application/list-links.js +5 -5
- package/dist/application/migrate-vault.js +91 -0
- package/dist/application/offline-pack-backup.js +44 -0
- package/dist/application/search-graph-node-ids.js +12 -0
- package/dist/application/search-knowledge.js +75 -5
- package/dist/application/server/routes.js +102 -1
- package/dist/application/start-server.js +75 -4
- package/dist/application/watch-vault.js +23 -2
- package/dist/benchmarks/large-vault.js +1 -1
- package/dist/cli/commands/agent-commands.js +419 -0
- package/dist/cli/commands/config-commands.js +167 -0
- package/dist/cli/commands/read-commands.js +25 -8
- package/dist/cli/commands/write-commands.js +989 -10
- package/dist/cli/main.js +4 -0
- package/dist/cli/runtime.js +5 -2
- package/dist/domain/context.js +53 -11
- package/dist/domain/embeddings.js +2 -1
- package/dist/domain/graph-layout.js +62 -15
- package/dist/domain/markdown.js +36 -4
- package/dist/domain/middle-out.js +18 -0
- package/dist/infrastructure/config.js +132 -8
- package/dist/infrastructure/file-index.js +358 -0
- package/dist/infrastructure/file-system-vault.js +30 -0
- package/dist/infrastructure/index-state.js +56 -0
- package/dist/infrastructure/paths.js +9 -1
- package/dist/infrastructure/private-pack-codec.js +134 -0
- package/dist/infrastructure/search-packs.js +452 -0
- package/dist/infrastructure/session-state.js +172 -0
- package/dist/mcp/main.js +11 -3
- package/dist/mcp/server.js +27 -2
- package/dist/mcp/startup.js +35 -0
- package/dist/mcp/tools.js +633 -19
- package/docs/AGENT_USAGE.md +178 -16
- package/docs/ARCHITECTURE.md +37 -26
- package/docs/QUICKSTART.md +111 -0
- package/package.json +6 -4
- package/dist/infrastructure/sqlite/document-writer.js +0 -51
- package/dist/infrastructure/sqlite/graph-reader.js +0 -120
- package/dist/infrastructure/sqlite/schema.js +0 -111
- package/dist/infrastructure/sqlite/search-reader.js +0 -156
- package/dist/infrastructure/sqlite/types.js +0 -1
- package/dist/infrastructure/sqlite-index.js +0 -25
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
import { gunzipSync } from 'node:zlib';
|
|
2
|
+
import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { middleOutIndices } from '../domain/middle-out.js';
|
|
5
|
+
import { decodePrivatePack, encodePrivatePack, isPrivatePackPayload } from './private-pack-codec.js';
|
|
6
|
+
const packsDirectoryName = 'search-packs';
|
|
7
|
+
const manifestFileName = 'manifest.json';
|
|
8
|
+
const defaultBuildOptions = {
|
|
9
|
+
rowChunkSize: 5_000,
|
|
10
|
+
compressionLevel: 5,
|
|
11
|
+
useDictionary: true
|
|
12
|
+
};
|
|
13
|
+
const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
|
|
14
|
+
const bloomBytes = 256;
|
|
15
|
+
const bloomBitSize = bloomBytes * 8;
|
|
16
|
+
const bloomSeeds = [0x9e3779b1, 0x85ebca6b, 0xc2b2ae35];
|
|
17
|
+
const toPackDirectory = (vaultPath) => join(vaultPath, '.brainlink', packsDirectoryName);
|
|
18
|
+
const toManifestPath = (vaultPath) => join(toPackDirectory(vaultPath), manifestFileName);
|
|
19
|
+
const parseRowsFromPack = async (vaultPath, content) => {
|
|
20
|
+
const raw = isPrivatePackPayload(content) ? await decodePrivatePack(vaultPath, content) : gunzipSync(content);
|
|
21
|
+
return raw
|
|
22
|
+
.toString('utf8')
|
|
23
|
+
.split('\n')
|
|
24
|
+
.map((line) => line.trim())
|
|
25
|
+
.filter((line) => line.length > 0)
|
|
26
|
+
.map((line) => JSON.parse(line))
|
|
27
|
+
.flatMap((row) => {
|
|
28
|
+
if (typeof row.documentId !== 'string' ||
|
|
29
|
+
typeof row.agentId !== 'string' ||
|
|
30
|
+
typeof row.title !== 'string' ||
|
|
31
|
+
typeof row.path !== 'string' ||
|
|
32
|
+
typeof row.chunkId !== 'string' ||
|
|
33
|
+
typeof row.content !== 'string') {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return [
|
|
37
|
+
{
|
|
38
|
+
documentId: row.documentId,
|
|
39
|
+
agentId: row.agentId,
|
|
40
|
+
title: row.title,
|
|
41
|
+
path: row.path,
|
|
42
|
+
chunkId: row.chunkId,
|
|
43
|
+
chunkOrdinal: typeof row.chunkOrdinal === 'number' ? row.chunkOrdinal : 0,
|
|
44
|
+
content: row.content,
|
|
45
|
+
tags: Array.isArray(row.tags) ? row.tags.filter((item) => typeof item === 'string') : []
|
|
46
|
+
}
|
|
47
|
+
];
|
|
48
|
+
});
|
|
49
|
+
};
|
|
50
|
+
const toRows = (documents) => documents.flatMap((document) => document.chunks.map((chunk) => ({
|
|
51
|
+
documentId: document.document.id,
|
|
52
|
+
agentId: document.document.agentId,
|
|
53
|
+
title: document.document.title,
|
|
54
|
+
path: document.document.path,
|
|
55
|
+
chunkId: chunk.id,
|
|
56
|
+
chunkOrdinal: chunk.ordinal,
|
|
57
|
+
content: chunk.content,
|
|
58
|
+
tags: document.document.tags
|
|
59
|
+
})));
|
|
60
|
+
const writeManifest = async (vaultPath, manifest) => {
|
|
61
|
+
await writeFile(toManifestPath(vaultPath), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');
|
|
62
|
+
};
|
|
63
|
+
const readManifest = async (vaultPath) => {
|
|
64
|
+
try {
|
|
65
|
+
const parsed = JSON.parse(await readFile(toManifestPath(vaultPath), 'utf8'));
|
|
66
|
+
if (parsed.version === 2 && parsed.format === 'private-v2') {
|
|
67
|
+
return {
|
|
68
|
+
version: 2,
|
|
69
|
+
createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
|
|
70
|
+
packCount: typeof parsed.packCount === 'number' ? parsed.packCount : 0,
|
|
71
|
+
recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
|
|
72
|
+
format: 'private-v2'
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
if (parsed.version === 3 && parsed.format === 'private-v2') {
|
|
76
|
+
const packIndex = Array.isArray(parsed.packIndex)
|
|
77
|
+
? parsed.packIndex.flatMap((entry) => {
|
|
78
|
+
if (!entry || typeof entry !== 'object') {
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
81
|
+
const candidate = entry;
|
|
82
|
+
if (typeof candidate.fileName !== 'string' || typeof candidate.tokenBloomB64 !== 'string') {
|
|
83
|
+
return [];
|
|
84
|
+
}
|
|
85
|
+
return [
|
|
86
|
+
{
|
|
87
|
+
fileName: candidate.fileName,
|
|
88
|
+
recordCount: typeof candidate.recordCount === 'number' ? candidate.recordCount : 0,
|
|
89
|
+
agents: Array.isArray(candidate.agents) ? candidate.agents.filter((item) => typeof item === 'string') : [],
|
|
90
|
+
tokenBloomB64: candidate.tokenBloomB64
|
|
91
|
+
}
|
|
92
|
+
];
|
|
93
|
+
})
|
|
94
|
+
: [];
|
|
95
|
+
return {
|
|
96
|
+
version: 3,
|
|
97
|
+
createdAt: typeof parsed.createdAt === 'string' ? parsed.createdAt : new Date().toISOString(),
|
|
98
|
+
packCount: typeof parsed.packCount === 'number' ? parsed.packCount : packIndex.length,
|
|
99
|
+
recordCount: typeof parsed.recordCount === 'number' ? parsed.recordCount : 0,
|
|
100
|
+
format: 'private-v2',
|
|
101
|
+
packIndex,
|
|
102
|
+
...(parsed.packConfig && typeof parsed.packConfig === 'object'
|
|
103
|
+
? {
|
|
104
|
+
packConfig: {
|
|
105
|
+
rowChunkSize: typeof parsed.packConfig.rowChunkSize === 'number'
|
|
106
|
+
? parsed.packConfig.rowChunkSize
|
|
107
|
+
: defaultBuildOptions.rowChunkSize,
|
|
108
|
+
compressionLevel: typeof parsed.packConfig.compressionLevel === 'number'
|
|
109
|
+
? parsed.packConfig.compressionLevel
|
|
110
|
+
: defaultBuildOptions.compressionLevel,
|
|
111
|
+
useDictionary: typeof parsed.packConfig.useDictionary === 'boolean'
|
|
112
|
+
? parsed.packConfig.useDictionary
|
|
113
|
+
: defaultBuildOptions.useDictionary
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
: {}),
|
|
117
|
+
...(parsed.compression &&
|
|
118
|
+
typeof parsed.compression === 'object' &&
|
|
119
|
+
typeof parsed.compression.inputBytes === 'number' &&
|
|
120
|
+
typeof parsed.compression.outputBytes === 'number' &&
|
|
121
|
+
typeof parsed.compression.ratio === 'number' &&
|
|
122
|
+
typeof parsed.compression.savedBytes === 'number'
|
|
123
|
+
? {
|
|
124
|
+
compression: {
|
|
125
|
+
inputBytes: parsed.compression.inputBytes,
|
|
126
|
+
outputBytes: parsed.compression.outputBytes,
|
|
127
|
+
ratio: parsed.compression.ratio,
|
|
128
|
+
savedBytes: parsed.compression.savedBytes
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
: {})
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return null;
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
export const ensureSearchPackManifest = async (vaultPath) => {
|
|
141
|
+
const manifest = await readManifest(vaultPath);
|
|
142
|
+
if (manifest) {
|
|
143
|
+
return {
|
|
144
|
+
repaired: false,
|
|
145
|
+
source: 'not-needed',
|
|
146
|
+
packCount: manifest.packCount
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const files = await sortedPackFiles(vaultPath);
|
|
150
|
+
const packFiles = files.filter((file) => file.endsWith('.blpk'));
|
|
151
|
+
if (packFiles.length === 0) {
|
|
152
|
+
return {
|
|
153
|
+
repaired: false,
|
|
154
|
+
source: 'no-packs',
|
|
155
|
+
packCount: 0
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
await writeManifest(vaultPath, {
|
|
159
|
+
version: 2,
|
|
160
|
+
createdAt: new Date().toISOString(),
|
|
161
|
+
packCount: packFiles.length,
|
|
162
|
+
recordCount: 0,
|
|
163
|
+
format: 'private-v2'
|
|
164
|
+
});
|
|
165
|
+
return {
|
|
166
|
+
repaired: true,
|
|
167
|
+
source: 'existing-packs',
|
|
168
|
+
packCount: packFiles.length
|
|
169
|
+
};
|
|
170
|
+
};
|
|
171
|
+
const chunkRows = (rows, size) => {
|
|
172
|
+
const chunks = [];
|
|
173
|
+
for (let index = 0; index < rows.length; index += size) {
|
|
174
|
+
chunks.push(rows.slice(index, index + size));
|
|
175
|
+
}
|
|
176
|
+
return chunks;
|
|
177
|
+
};
|
|
178
|
+
const normalizeToken = (value) => value
|
|
179
|
+
.normalize('NFKD')
|
|
180
|
+
.replace(/\p{Diacritic}/gu, '')
|
|
181
|
+
.toLowerCase();
|
|
182
|
+
const tokenize = (query) => query
|
|
183
|
+
.match(queryTokenPattern)
|
|
184
|
+
?.map(normalizeToken)
|
|
185
|
+
.filter((token) => token.length > 1) ?? [];
|
|
186
|
+
const countOccurrences = (text, token) => {
|
|
187
|
+
let hits = 0;
|
|
188
|
+
let start = 0;
|
|
189
|
+
while (start < text.length) {
|
|
190
|
+
const index = text.indexOf(token, start);
|
|
191
|
+
if (index < 0) {
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
hits += 1;
|
|
195
|
+
start = index + token.length;
|
|
196
|
+
}
|
|
197
|
+
return hits;
|
|
198
|
+
};
|
|
199
|
+
const hashToken = (token, seed) => {
|
|
200
|
+
let hash = seed >>> 0;
|
|
201
|
+
for (let index = 0; index < token.length; index += 1) {
|
|
202
|
+
hash ^= token.charCodeAt(index);
|
|
203
|
+
hash = Math.imul(hash, 16777619) >>> 0;
|
|
204
|
+
}
|
|
205
|
+
return hash >>> 0;
|
|
206
|
+
};
|
|
207
|
+
const createBloom = () => new Uint8Array(bloomBytes);
|
|
208
|
+
const bloomAdd = (bloom, token) => {
|
|
209
|
+
bloomSeeds.forEach((seed) => {
|
|
210
|
+
const bit = hashToken(token, seed) % bloomBitSize;
|
|
211
|
+
bloom[Math.floor(bit / 8)] |= 1 << (bit % 8);
|
|
212
|
+
});
|
|
213
|
+
};
|
|
214
|
+
const bloomMayContain = (bloom, token) => bloomSeeds.every((seed) => {
|
|
215
|
+
const bit = hashToken(token, seed) % bloomBitSize;
|
|
216
|
+
return (bloom[Math.floor(bit / 8)] & (1 << (bit % 8))) !== 0;
|
|
217
|
+
});
|
|
218
|
+
const bloomFromRows = (rows) => {
|
|
219
|
+
const bloom = createBloom();
|
|
220
|
+
rows.forEach((row) => {
|
|
221
|
+
tokenize([row.title, row.path, row.tags.join(' '), row.content].join(' ')).forEach((token) => bloomAdd(bloom, token));
|
|
222
|
+
});
|
|
223
|
+
return bloom;
|
|
224
|
+
};
|
|
225
|
+
const bloomToBase64 = (bloom) => Buffer.from(bloom).toString('base64url');
|
|
226
|
+
const bloomFromBase64 = (value) => {
|
|
227
|
+
try {
|
|
228
|
+
const decoded = Buffer.from(value, 'base64url');
|
|
229
|
+
if (decoded.byteLength === bloomBytes) {
|
|
230
|
+
return {
|
|
231
|
+
bloom: new Uint8Array(decoded),
|
|
232
|
+
valid: true
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
// fallback below
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
bloom: createBloom(),
|
|
241
|
+
valid: false
|
|
242
|
+
};
|
|
243
|
+
};
|
|
244
|
+
const computeTextScore = (row, tokens) => {
|
|
245
|
+
if (tokens.length === 0) {
|
|
246
|
+
return 0;
|
|
247
|
+
}
|
|
248
|
+
const title = normalizeToken(row.title);
|
|
249
|
+
const path = normalizeToken(row.path);
|
|
250
|
+
const content = normalizeToken(row.content);
|
|
251
|
+
const tags = normalizeToken(row.tags.join(' '));
|
|
252
|
+
return tokens.reduce((score, token) => {
|
|
253
|
+
const titleHits = countOccurrences(title, token);
|
|
254
|
+
const tagHits = countOccurrences(tags, token);
|
|
255
|
+
const pathHits = countOccurrences(path, token);
|
|
256
|
+
const contentHits = countOccurrences(content, token);
|
|
257
|
+
return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 5);
|
|
258
|
+
}, 0);
|
|
259
|
+
};
|
|
260
|
+
const toSearchResult = (row, score) => ({
|
|
261
|
+
documentId: row.documentId,
|
|
262
|
+
agentId: row.agentId,
|
|
263
|
+
title: row.title,
|
|
264
|
+
path: row.path,
|
|
265
|
+
chunkId: row.chunkId,
|
|
266
|
+
chunkOrdinal: row.chunkOrdinal,
|
|
267
|
+
content: row.content,
|
|
268
|
+
score,
|
|
269
|
+
textScore: score,
|
|
270
|
+
semanticScore: 0,
|
|
271
|
+
searchMode: 'fts',
|
|
272
|
+
tags: row.tags
|
|
273
|
+
});
|
|
274
|
+
const sortedPackFiles = async (vaultPath) => {
|
|
275
|
+
try {
|
|
276
|
+
const files = await readdir(toPackDirectory(vaultPath));
|
|
277
|
+
return files
|
|
278
|
+
.filter((file) => file.endsWith('.blpk') || file.endsWith('.jsonl.gz'))
|
|
279
|
+
.sort((left, right) => left.localeCompare(right));
|
|
280
|
+
}
|
|
281
|
+
catch (error) {
|
|
282
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
283
|
+
return [];
|
|
284
|
+
}
|
|
285
|
+
throw error;
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
const writeRowsAsPrivatePacks = async (vaultPath, rows, clearExisting, options) => {
|
|
289
|
+
const startedAt = process.hrtime.bigint();
|
|
290
|
+
const directory = toPackDirectory(vaultPath);
|
|
291
|
+
await mkdir(directory, { recursive: true });
|
|
292
|
+
if (clearExisting) {
|
|
293
|
+
const current = await readdir(directory);
|
|
294
|
+
await Promise.all(current
|
|
295
|
+
.filter((name) => name.endsWith('.blpk') || name.endsWith('.jsonl.gz') || name === manifestFileName)
|
|
296
|
+
.map((name) => rm(join(directory, name), { force: true })));
|
|
297
|
+
}
|
|
298
|
+
const chunks = chunkRows(rows, options.rowChunkSize);
|
|
299
|
+
const packIndex = [];
|
|
300
|
+
let inputBytes = 0;
|
|
301
|
+
let outputBytes = 0;
|
|
302
|
+
for (let index = 0; index < chunks.length; index += 1) {
|
|
303
|
+
const chunk = chunks[index];
|
|
304
|
+
const fileName = `pack-${String(index + 1).padStart(4, '0')}.blpk`;
|
|
305
|
+
const serialized = `${chunk.map((row) => JSON.stringify(row)).join('\n')}\n`;
|
|
306
|
+
const compressed = await encodePrivatePack(vaultPath, Buffer.from(serialized, 'utf8'), {
|
|
307
|
+
compressionLevel: options.compressionLevel,
|
|
308
|
+
useDictionary: options.useDictionary
|
|
309
|
+
});
|
|
310
|
+
const tokenBloomB64 = bloomToBase64(bloomFromRows(chunk));
|
|
311
|
+
await writeFile(join(directory, fileName), compressed);
|
|
312
|
+
inputBytes += Buffer.byteLength(serialized, 'utf8');
|
|
313
|
+
outputBytes += compressed.byteLength;
|
|
314
|
+
packIndex.push({
|
|
315
|
+
fileName,
|
|
316
|
+
recordCount: chunk.length,
|
|
317
|
+
agents: Array.from(new Set(chunk.map((row) => row.agentId))).sort((left, right) => left.localeCompare(right)),
|
|
318
|
+
tokenBloomB64
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
await writeManifest(vaultPath, {
|
|
322
|
+
version: 3,
|
|
323
|
+
createdAt: new Date().toISOString(),
|
|
324
|
+
packCount: chunks.length,
|
|
325
|
+
recordCount: rows.length,
|
|
326
|
+
format: 'private-v2',
|
|
327
|
+
packIndex,
|
|
328
|
+
packConfig: {
|
|
329
|
+
rowChunkSize: options.rowChunkSize,
|
|
330
|
+
compressionLevel: options.compressionLevel,
|
|
331
|
+
useDictionary: options.useDictionary
|
|
332
|
+
},
|
|
333
|
+
compression: {
|
|
334
|
+
inputBytes,
|
|
335
|
+
outputBytes,
|
|
336
|
+
ratio: outputBytes / Math.max(inputBytes, 1),
|
|
337
|
+
savedBytes: Math.max(inputBytes - outputBytes, 0)
|
|
338
|
+
}
|
|
339
|
+
});
|
|
340
|
+
const durationMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
|
|
341
|
+
const safeInput = Math.max(inputBytes, 1);
|
|
342
|
+
const savedBytes = Math.max(inputBytes - outputBytes, 0);
|
|
343
|
+
return {
|
|
344
|
+
packCount: chunks.length,
|
|
345
|
+
recordCount: rows.length,
|
|
346
|
+
compression: {
|
|
347
|
+
inputBytes,
|
|
348
|
+
outputBytes,
|
|
349
|
+
ratio: outputBytes / safeInput,
|
|
350
|
+
savedBytes
|
|
351
|
+
},
|
|
352
|
+
durationMs
|
|
353
|
+
};
|
|
354
|
+
};
|
|
355
|
+
const selectCandidatePackFiles = async (vaultPath, tokens, agentId) => {
|
|
356
|
+
const allFiles = await sortedPackFiles(vaultPath);
|
|
357
|
+
if (allFiles.length === 0) {
|
|
358
|
+
return [];
|
|
359
|
+
}
|
|
360
|
+
const manifest = await readManifest(vaultPath);
|
|
361
|
+
if (!manifest || manifest.version !== 3 || !Array.isArray(manifest.packIndex)) {
|
|
362
|
+
return allFiles;
|
|
363
|
+
}
|
|
364
|
+
const normalizedAgent = agentId?.trim();
|
|
365
|
+
const byAgent = manifest.packIndex.filter((entry) => normalizedAgent ? entry.agents.includes(normalizedAgent) : true);
|
|
366
|
+
if (tokens.length === 0) {
|
|
367
|
+
return byAgent.map((entry) => entry.fileName);
|
|
368
|
+
}
|
|
369
|
+
let hasInvalidBloomIndex = false;
|
|
370
|
+
const byToken = byAgent.filter((entry) => {
|
|
371
|
+
const decoded = bloomFromBase64(entry.tokenBloomB64);
|
|
372
|
+
if (!decoded.valid) {
|
|
373
|
+
hasInvalidBloomIndex = true;
|
|
374
|
+
return true;
|
|
375
|
+
}
|
|
376
|
+
return tokens.some((token) => bloomMayContain(decoded.bloom, token));
|
|
377
|
+
});
|
|
378
|
+
// Lossless guarantee: if compressed metadata is partially invalid, do not prune packs.
|
|
379
|
+
if (hasInvalidBloomIndex) {
|
|
380
|
+
return byAgent.map((entry) => entry.fileName);
|
|
381
|
+
}
|
|
382
|
+
if (byToken.length > 0) {
|
|
383
|
+
return byToken.map((entry) => entry.fileName);
|
|
384
|
+
}
|
|
385
|
+
return byAgent.length > 0 ? byAgent.map((entry) => entry.fileName) : allFiles;
|
|
386
|
+
};
|
|
387
|
+
export const buildSearchPacks = async (vaultPath, documents, options) => {
|
|
388
|
+
const resolvedOptions = {
|
|
389
|
+
rowChunkSize: options?.rowChunkSize ?? defaultBuildOptions.rowChunkSize,
|
|
390
|
+
compressionLevel: options?.compressionLevel ?? defaultBuildOptions.compressionLevel,
|
|
391
|
+
useDictionary: options?.useDictionary ?? defaultBuildOptions.useDictionary
|
|
392
|
+
};
|
|
393
|
+
return writeRowsAsPrivatePacks(vaultPath, toRows(documents), true, resolvedOptions);
|
|
394
|
+
};
|
|
395
|
+
export const ensurePrivatePacksFromLegacyIndex = async (vaultPath) => {
|
|
396
|
+
const files = await sortedPackFiles(vaultPath);
|
|
397
|
+
if (files.some((file) => file.endsWith('.blpk'))) {
|
|
398
|
+
return { imported: false };
|
|
399
|
+
}
|
|
400
|
+
const legacyPackFiles = files.filter((file) => file.endsWith('.jsonl.gz'));
|
|
401
|
+
if (legacyPackFiles.length > 0) {
|
|
402
|
+
const rows = [];
|
|
403
|
+
for (const file of legacyPackFiles) {
|
|
404
|
+
const parsed = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
|
|
405
|
+
rows.push(...parsed);
|
|
406
|
+
}
|
|
407
|
+
const report = await writeRowsAsPrivatePacks(vaultPath, rows, true, defaultBuildOptions);
|
|
408
|
+
return {
|
|
409
|
+
imported: true,
|
|
410
|
+
source: 'legacy-packs',
|
|
411
|
+
...report
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
return { imported: false };
|
|
415
|
+
};
|
|
416
|
+
export const toSearchPackBuildOptions = (config) => ({
|
|
417
|
+
rowChunkSize: config.searchPack.rowChunkSize,
|
|
418
|
+
compressionLevel: config.searchPack.compressionLevel,
|
|
419
|
+
useDictionary: config.searchPack.useDictionary
|
|
420
|
+
});
|
|
421
|
+
export const searchInPacks = async (vaultPath, query, limit, agentId) => {
|
|
422
|
+
const normalizedAgent = agentId?.trim();
|
|
423
|
+
const tokens = tokenize(query);
|
|
424
|
+
if (limit <= 0 || tokens.length === 0) {
|
|
425
|
+
return [];
|
|
426
|
+
}
|
|
427
|
+
const files = await selectCandidatePackFiles(vaultPath, tokens, normalizedAgent);
|
|
428
|
+
if (files.length === 0) {
|
|
429
|
+
return [];
|
|
430
|
+
}
|
|
431
|
+
const scored = [];
|
|
432
|
+
for (const file of files) {
|
|
433
|
+
const rows = await parseRowsFromPack(vaultPath, await readFile(join(toPackDirectory(vaultPath), file)));
|
|
434
|
+
const traversal = middleOutIndices(rows.length, Math.floor(rows.length / 2));
|
|
435
|
+
traversal.forEach((rowIndex) => {
|
|
436
|
+
const row = rows[rowIndex];
|
|
437
|
+
if (!row) {
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
if (normalizedAgent && row.agentId !== normalizedAgent) {
|
|
441
|
+
return;
|
|
442
|
+
}
|
|
443
|
+
const score = computeTextScore(row, tokens);
|
|
444
|
+
if (score > 0) {
|
|
445
|
+
scored.push(toSearchResult(row, score));
|
|
446
|
+
}
|
|
447
|
+
});
|
|
448
|
+
}
|
|
449
|
+
return scored
|
|
450
|
+
.sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
|
|
451
|
+
.slice(0, limit);
|
|
452
|
+
};
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { getBrainlinkHomePath } from './paths.js';
|
|
4
|
+
const defaultPolicy = {
|
|
5
|
+
enforceBootstrap: true,
|
|
6
|
+
enforceContextFirst: true,
|
|
7
|
+
autoBootstrapOnRead: true,
|
|
8
|
+
autoBootstrapOnStartup: true,
|
|
9
|
+
staleAfterMinutes: 120
|
|
10
|
+
};
|
|
11
|
+
const defaultState = {
|
|
12
|
+
policy: defaultPolicy,
|
|
13
|
+
bootstraps: [],
|
|
14
|
+
contexts: []
|
|
15
|
+
};
|
|
16
|
+
const sessionStatePath = () => join(getBrainlinkHomePath(), 'session-state.json');
|
|
17
|
+
const normalizeAgent = (agent) => agent?.trim() ? agent.trim() : '*';
|
|
18
|
+
const safePositive = (value, fallback) => typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : fallback;
|
|
19
|
+
const sanitizeState = (value) => {
|
|
20
|
+
if (typeof value !== 'object' || value === null) {
|
|
21
|
+
return defaultState;
|
|
22
|
+
}
|
|
23
|
+
const record = value;
|
|
24
|
+
const policyRecord = typeof record.policy === 'object' && record.policy !== null ? record.policy : {};
|
|
25
|
+
const rawBootstraps = Array.isArray(record.bootstraps) ? record.bootstraps : [];
|
|
26
|
+
const rawContexts = Array.isArray(record.contexts) ? record.contexts : [];
|
|
27
|
+
const bootstraps = rawBootstraps.flatMap((entry) => {
|
|
28
|
+
if (typeof entry !== 'object' || entry === null) {
|
|
29
|
+
return [];
|
|
30
|
+
}
|
|
31
|
+
const row = entry;
|
|
32
|
+
const vault = typeof row.vault === 'string' && row.vault.trim().length > 0 ? row.vault.trim() : undefined;
|
|
33
|
+
const agent = typeof row.agent === 'string' && row.agent.trim().length > 0 ? row.agent.trim() : undefined;
|
|
34
|
+
const lastBootstrappedAt = typeof row.lastBootstrappedAt === 'string' && row.lastBootstrappedAt.trim().length > 0 ? row.lastBootstrappedAt.trim() : undefined;
|
|
35
|
+
return vault && agent && lastBootstrappedAt ? [{ vault, agent, lastBootstrappedAt }] : [];
|
|
36
|
+
});
|
|
37
|
+
const contexts = rawContexts.flatMap((entry) => {
|
|
38
|
+
if (typeof entry !== 'object' || entry === null) {
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
41
|
+
const row = entry;
|
|
42
|
+
const vault = typeof row.vault === 'string' && row.vault.trim().length > 0 ? row.vault.trim() : undefined;
|
|
43
|
+
const agent = typeof row.agent === 'string' && row.agent.trim().length > 0 ? row.agent.trim() : undefined;
|
|
44
|
+
const lastContextAt = typeof row.lastContextAt === 'string' && row.lastContextAt.trim().length > 0 ? row.lastContextAt.trim() : undefined;
|
|
45
|
+
return vault && agent && lastContextAt ? [{ vault, agent, lastContextAt }] : [];
|
|
46
|
+
});
|
|
47
|
+
return {
|
|
48
|
+
policy: {
|
|
49
|
+
enforceBootstrap: typeof policyRecord.enforceBootstrap === 'boolean' ? policyRecord.enforceBootstrap : defaultPolicy.enforceBootstrap,
|
|
50
|
+
enforceContextFirst: typeof policyRecord.enforceContextFirst === 'boolean'
|
|
51
|
+
? policyRecord.enforceContextFirst
|
|
52
|
+
: defaultPolicy.enforceContextFirst,
|
|
53
|
+
autoBootstrapOnRead: typeof policyRecord.autoBootstrapOnRead === 'boolean'
|
|
54
|
+
? policyRecord.autoBootstrapOnRead
|
|
55
|
+
: defaultPolicy.autoBootstrapOnRead,
|
|
56
|
+
autoBootstrapOnStartup: typeof policyRecord.autoBootstrapOnStartup === 'boolean'
|
|
57
|
+
? policyRecord.autoBootstrapOnStartup
|
|
58
|
+
: defaultPolicy.autoBootstrapOnStartup,
|
|
59
|
+
staleAfterMinutes: safePositive(policyRecord.staleAfterMinutes, defaultPolicy.staleAfterMinutes)
|
|
60
|
+
},
|
|
61
|
+
bootstraps,
|
|
62
|
+
contexts
|
|
63
|
+
};
|
|
64
|
+
};
|
|
65
|
+
const readState = async () => {
|
|
66
|
+
try {
|
|
67
|
+
const content = await readFile(sessionStatePath(), 'utf8');
|
|
68
|
+
return sanitizeState(JSON.parse(content));
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
|
|
72
|
+
return defaultState;
|
|
73
|
+
}
|
|
74
|
+
throw error;
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
const writeState = async (state) => {
|
|
78
|
+
const path = sessionStatePath();
|
|
79
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
80
|
+
await writeFile(path, `${JSON.stringify(state, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
81
|
+
};
|
|
82
|
+
export const getSessionStatePath = () => sessionStatePath();
|
|
83
|
+
export const getBootstrapPolicy = async () => (await readState()).policy;
|
|
84
|
+
export const setBootstrapPolicy = async (patch) => {
|
|
85
|
+
const state = await readState();
|
|
86
|
+
const next = {
|
|
87
|
+
enforceBootstrap: typeof patch.enforceBootstrap === 'boolean' ? patch.enforceBootstrap : state.policy.enforceBootstrap,
|
|
88
|
+
enforceContextFirst: typeof patch.enforceContextFirst === 'boolean' ? patch.enforceContextFirst : state.policy.enforceContextFirst,
|
|
89
|
+
autoBootstrapOnRead: typeof patch.autoBootstrapOnRead === 'boolean' ? patch.autoBootstrapOnRead : state.policy.autoBootstrapOnRead,
|
|
90
|
+
autoBootstrapOnStartup: typeof patch.autoBootstrapOnStartup === 'boolean' ? patch.autoBootstrapOnStartup : state.policy.autoBootstrapOnStartup,
|
|
91
|
+
staleAfterMinutes: safePositive(patch.staleAfterMinutes, state.policy.staleAfterMinutes)
|
|
92
|
+
};
|
|
93
|
+
await writeState({
|
|
94
|
+
...state,
|
|
95
|
+
policy: next
|
|
96
|
+
});
|
|
97
|
+
return next;
|
|
98
|
+
};
|
|
99
|
+
export const touchBootstrapSession = async (vault, agent) => {
|
|
100
|
+
const state = await readState();
|
|
101
|
+
const normalizedAgent = normalizeAgent(agent);
|
|
102
|
+
const entry = {
|
|
103
|
+
vault,
|
|
104
|
+
agent: normalizedAgent,
|
|
105
|
+
lastBootstrappedAt: new Date().toISOString()
|
|
106
|
+
};
|
|
107
|
+
const bootstraps = [
|
|
108
|
+
entry,
|
|
109
|
+
...state.bootstraps.filter((item) => !(item.vault === entry.vault && item.agent === entry.agent))
|
|
110
|
+
].slice(0, 500);
|
|
111
|
+
await writeState({
|
|
112
|
+
...state,
|
|
113
|
+
bootstraps
|
|
114
|
+
});
|
|
115
|
+
return entry;
|
|
116
|
+
};
|
|
117
|
+
export const touchContextSession = async (vault, agent) => {
|
|
118
|
+
const state = await readState();
|
|
119
|
+
const normalizedAgent = normalizeAgent(agent);
|
|
120
|
+
const entry = {
|
|
121
|
+
vault,
|
|
122
|
+
agent: normalizedAgent,
|
|
123
|
+
lastContextAt: new Date().toISOString()
|
|
124
|
+
};
|
|
125
|
+
const contexts = [
|
|
126
|
+
entry,
|
|
127
|
+
...state.contexts.filter((item) => !(item.vault === entry.vault && item.agent === entry.agent))
|
|
128
|
+
].slice(0, 500);
|
|
129
|
+
await writeState({
|
|
130
|
+
...state,
|
|
131
|
+
contexts
|
|
132
|
+
});
|
|
133
|
+
return entry;
|
|
134
|
+
};
|
|
135
|
+
export const getBootstrapSessionStatus = async (vault, agent) => {
|
|
136
|
+
const state = await readState();
|
|
137
|
+
const normalizedAgent = normalizeAgent(agent);
|
|
138
|
+
const match = state.bootstraps.find((entry) => entry.vault === vault && entry.agent === normalizedAgent);
|
|
139
|
+
if (!match) {
|
|
140
|
+
return {
|
|
141
|
+
ready: false,
|
|
142
|
+
stale: true
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
const ageMinutes = Math.max(0, (Date.now() - new Date(match.lastBootstrappedAt).getTime()) / 60000);
|
|
146
|
+
const stale = ageMinutes > state.policy.staleAfterMinutes;
|
|
147
|
+
return {
|
|
148
|
+
ready: !stale,
|
|
149
|
+
stale,
|
|
150
|
+
lastBootstrappedAt: match.lastBootstrappedAt,
|
|
151
|
+
ageMinutes
|
|
152
|
+
};
|
|
153
|
+
};
|
|
154
|
+
export const getContextSessionStatus = async (vault, agent) => {
|
|
155
|
+
const state = await readState();
|
|
156
|
+
const normalizedAgent = normalizeAgent(agent);
|
|
157
|
+
const match = state.contexts.find((entry) => entry.vault === vault && entry.agent === normalizedAgent);
|
|
158
|
+
if (!match) {
|
|
159
|
+
return {
|
|
160
|
+
ready: false,
|
|
161
|
+
stale: true
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
const ageMinutes = Math.max(0, (Date.now() - new Date(match.lastContextAt).getTime()) / 60000);
|
|
165
|
+
const stale = ageMinutes > state.policy.staleAfterMinutes;
|
|
166
|
+
return {
|
|
167
|
+
ready: !stale,
|
|
168
|
+
stale,
|
|
169
|
+
lastContextAt: match.lastContextAt,
|
|
170
|
+
ageMinutes
|
|
171
|
+
};
|
|
172
|
+
};
|
package/dist/mcp/main.js
CHANGED
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
3
3
|
import { createBrainlinkMcpServer } from './server.js';
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
|
|
4
|
+
import { runStartupBootstrap } from './startup.js';
|
|
5
|
+
const start = async () => {
|
|
6
|
+
const startup = await runStartupBootstrap();
|
|
7
|
+
if (startup.error) {
|
|
8
|
+
console.error(`Brainlink MCP startup bootstrap warning: ${startup.error}`);
|
|
9
|
+
}
|
|
10
|
+
const server = createBrainlinkMcpServer();
|
|
11
|
+
const transport = new StdioServerTransport();
|
|
12
|
+
await server.connect(transport);
|
|
13
|
+
};
|
|
14
|
+
start().catch((error) => {
|
|
7
15
|
const message = error instanceof Error ? error.message : String(error);
|
|
8
16
|
console.error(message);
|
|
9
17
|
process.exitCode = 1;
|