@andespindola/brainlink 0.1.0-beta.6 → 0.1.0-beta.61

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/AGENTS.md +8 -5
  2. package/CHANGELOG.md +58 -2
  3. package/CONTRIBUTING.md +2 -2
  4. package/COPYRIGHT.md +5 -0
  5. package/README.md +266 -20
  6. package/SECURITY.md +1 -1
  7. package/dist/application/add-note.js +62 -13
  8. package/dist/application/analyze-vault.js +95 -8
  9. package/dist/application/build-context.js +56 -1
  10. package/dist/application/dedupe-notes.js +226 -0
  11. package/dist/application/frontend/client-css.js +214 -100
  12. package/dist/application/frontend/client-html.js +60 -45
  13. package/dist/application/frontend/client-js.js +1853 -139
  14. package/dist/application/frontend/client-worker-js.js +66 -0
  15. package/dist/application/get-graph-layout.js +18 -6
  16. package/dist/application/get-graph-node.js +12 -0
  17. package/dist/application/get-graph-summary.js +12 -0
  18. package/dist/application/get-graph.js +3 -3
  19. package/dist/application/import-legacy-sqlite.js +296 -0
  20. package/dist/application/index-vault.js +252 -19
  21. package/dist/application/list-agents.js +3 -3
  22. package/dist/application/list-links.js +5 -5
  23. package/dist/application/migrate-vault.js +91 -0
  24. package/dist/application/offline-pack-backup.js +44 -0
  25. package/dist/application/search-graph-node-ids.js +12 -0
  26. package/dist/application/search-knowledge.js +75 -5
  27. package/dist/application/server/routes.js +102 -1
  28. package/dist/application/start-server.js +75 -4
  29. package/dist/application/watch-vault.js +23 -2
  30. package/dist/benchmarks/large-vault.js +1 -1
  31. package/dist/cli/commands/agent-commands.js +419 -0
  32. package/dist/cli/commands/config-commands.js +167 -0
  33. package/dist/cli/commands/read-commands.js +25 -8
  34. package/dist/cli/commands/write-commands.js +989 -10
  35. package/dist/cli/main.js +4 -0
  36. package/dist/cli/runtime.js +5 -2
  37. package/dist/domain/context.js +53 -11
  38. package/dist/domain/embeddings.js +2 -1
  39. package/dist/domain/graph-layout.js +62 -15
  40. package/dist/domain/markdown.js +36 -4
  41. package/dist/domain/middle-out.js +18 -0
  42. package/dist/infrastructure/config.js +132 -8
  43. package/dist/infrastructure/file-index.js +358 -0
  44. package/dist/infrastructure/file-system-vault.js +30 -0
  45. package/dist/infrastructure/index-state.js +56 -0
  46. package/dist/infrastructure/paths.js +9 -1
  47. package/dist/infrastructure/private-pack-codec.js +134 -0
  48. package/dist/infrastructure/search-packs.js +452 -0
  49. package/dist/infrastructure/session-state.js +172 -0
  50. package/dist/mcp/main.js +11 -3
  51. package/dist/mcp/server.js +27 -2
  52. package/dist/mcp/startup.js +35 -0
  53. package/dist/mcp/tools.js +633 -19
  54. package/docs/AGENT_USAGE.md +178 -16
  55. package/docs/ARCHITECTURE.md +37 -26
  56. package/docs/QUICKSTART.md +111 -0
  57. package/package.json +6 -4
  58. package/dist/infrastructure/sqlite/document-writer.js +0 -51
  59. package/dist/infrastructure/sqlite/graph-reader.js +0 -120
  60. package/dist/infrastructure/sqlite/schema.js +0 -111
  61. package/dist/infrastructure/sqlite/search-reader.js +0 -156
  62. package/dist/infrastructure/sqlite/types.js +0 -1
  63. package/dist/infrastructure/sqlite-index.js +0 -25
@@ -0,0 +1,358 @@
1
+ import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
2
+ import { dirname, join } from 'node:path';
3
+ import { cosineSimilarity } from '../domain/embeddings.js';
4
+ const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
5
+ const indexCacheMaxEntries = 16;
6
+ const indexCache = new Map();
7
+ const emptyIndex = () => ({
8
+ version: 1,
9
+ updatedAt: new Date().toISOString(),
10
+ documents: [],
11
+ chunks: [],
12
+ links: []
13
+ });
14
+ export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
15
+ const readIndex = async (vaultPath) => {
16
+ const path = indexStoragePath(vaultPath);
17
+ let stats = null;
18
+ try {
19
+ const fileStats = await stat(path);
20
+ stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
21
+ }
22
+ catch (error) {
23
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
24
+ indexCache.delete(path);
25
+ return emptyIndex();
26
+ }
27
+ return emptyIndex();
28
+ }
29
+ const cached = indexCache.get(path);
30
+ if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
31
+ return cached.index;
32
+ }
33
+ try {
34
+ const parsed = JSON.parse(await readFile(path, 'utf8'));
35
+ const loaded = {
36
+ version: 1,
37
+ updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
38
+ documents: Array.isArray(parsed.documents) ? parsed.documents : [],
39
+ chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
40
+ links: Array.isArray(parsed.links) ? parsed.links : []
41
+ };
42
+ indexCache.set(path, { ...stats, index: loaded });
43
+ if (indexCache.size > indexCacheMaxEntries) {
44
+ const oldest = indexCache.keys().next().value;
45
+ if (typeof oldest === 'string') {
46
+ indexCache.delete(oldest);
47
+ }
48
+ }
49
+ return loaded;
50
+ }
51
+ catch (error) {
52
+ if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
53
+ indexCache.delete(path);
54
+ return emptyIndex();
55
+ }
56
+ return emptyIndex();
57
+ }
58
+ };
59
+ const writeIndex = async (vaultPath, index) => {
60
+ const target = indexStoragePath(vaultPath);
61
+ const temp = `${target}.tmp`;
62
+ await mkdir(dirname(target), { recursive: true, mode: 0o700 });
63
+ await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
64
+ await rename(temp, target);
65
+ const fileStats = await stat(target);
66
+ indexCache.set(target, {
67
+ mtimeMs: fileStats.mtimeMs,
68
+ size: fileStats.size,
69
+ index
70
+ });
71
+ };
72
+ const normalizeToken = (value) => value
73
+ .normalize('NFKD')
74
+ .replace(/\p{Diacritic}/gu, '')
75
+ .toLowerCase();
76
+ const tokenize = (query) => query
77
+ .match(queryTokenPattern)
78
+ ?.map(normalizeToken)
79
+ .filter((token) => token.length > 1) ?? [];
80
+ const countOccurrences = (text, token) => {
81
+ let hits = 0;
82
+ let cursor = 0;
83
+ while (cursor < text.length) {
84
+ const index = text.indexOf(token, cursor);
85
+ if (index < 0) {
86
+ break;
87
+ }
88
+ hits += 1;
89
+ cursor = index + token.length;
90
+ }
91
+ return hits;
92
+ };
93
+ const textScore = (row, tokens) => {
94
+ if (tokens.length === 0) {
95
+ return 0;
96
+ }
97
+ const title = normalizeToken(row.title);
98
+ const path = normalizeToken(row.path);
99
+ const content = normalizeToken(row.content);
100
+ const tags = normalizeToken(row.tags.join(' '));
101
+ return tokens.reduce((score, token) => {
102
+ const titleHits = countOccurrences(title, token);
103
+ const tagHits = countOccurrences(tags, token);
104
+ const pathHits = countOccurrences(path, token);
105
+ const contentHits = countOccurrences(content, token);
106
+ return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 6);
107
+ }, 0);
108
+ };
109
+ const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? cosineSimilarity(queryEmbedding, row.embedding) : 0;
110
+ const toResult = (row, mode, text, semantic) => {
111
+ const score = mode === 'fts' ? text : mode === 'semantic' ? semantic : text + semantic * 8;
112
+ return {
113
+ documentId: row.documentId,
114
+ agentId: row.agentId,
115
+ title: row.title,
116
+ path: row.path,
117
+ chunkId: row.chunkId,
118
+ chunkOrdinal: row.chunkOrdinal,
119
+ content: row.content,
120
+ score,
121
+ textScore: text,
122
+ semanticScore: semantic,
123
+ searchMode: mode,
124
+ tags: row.tags
125
+ };
126
+ };
127
+ const toGraphLink = (link, documentsById) => {
128
+ const source = documentsById.get(link.fromDocumentId);
129
+ const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
130
+ return {
131
+ agentId: source?.agentId ?? 'shared',
132
+ fromTitle: source?.title ?? 'Unknown',
133
+ fromPath: source?.path ?? 'Unknown',
134
+ toTitle: target?.title ?? link.toTitle,
135
+ toPath: target?.path ?? null,
136
+ weight: link.weight,
137
+ priority: link.priority
138
+ };
139
+ };
140
+ export const openFileIndex = (vaultPath) => {
141
+ const load = async () => readIndex(vaultPath);
142
+ const persist = async (index) => writeIndex(vaultPath, index);
143
+ return {
144
+ reset: async () => {
145
+ await persist(emptyIndex());
146
+ },
147
+ saveDocuments: async (documents) => {
148
+ const chunks = documents.flatMap((document) => document.chunks);
149
+ const links = documents.flatMap((document) => document.links);
150
+ await persist({
151
+ version: 1,
152
+ updatedAt: new Date().toISOString(),
153
+ documents: documents.map((document) => document.document),
154
+ chunks,
155
+ links
156
+ });
157
+ },
158
+ getIndexedDocuments: async (agentId) => {
159
+ const index = await load();
160
+ const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
161
+ const selectedDocumentIds = new Set(documents.map((document) => document.id));
162
+ const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
163
+ if (!selectedDocumentIds.has(chunk.documentId)) {
164
+ return state;
165
+ }
166
+ const current = state.get(chunk.documentId) ?? [];
167
+ current.push(chunk);
168
+ state.set(chunk.documentId, current);
169
+ return state;
170
+ }, new Map());
171
+ const linksByDocumentId = index.links.reduce((state, link) => {
172
+ if (!selectedDocumentIds.has(link.fromDocumentId)) {
173
+ return state;
174
+ }
175
+ const current = state.get(link.fromDocumentId) ?? [];
176
+ current.push(link);
177
+ state.set(link.fromDocumentId, current);
178
+ return state;
179
+ }, new Map());
180
+ return documents
181
+ .map((document) => ({
182
+ document,
183
+ chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
184
+ links: linksByDocumentId.get(document.id) ?? []
185
+ }))
186
+ .sort((left, right) => left.document.path.localeCompare(right.document.path));
187
+ },
188
+ search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
189
+ const index = await load();
190
+ const documentsById = new Map(index.documents.map((document) => [document.id, document]));
191
+ const rows = index.chunks.flatMap((chunk) => {
192
+ const document = documentsById.get(chunk.documentId);
193
+ if (!document) {
194
+ return [];
195
+ }
196
+ if (agentId && document.agentId !== agentId) {
197
+ return [];
198
+ }
199
+ return [
200
+ {
201
+ documentId: document.id,
202
+ agentId: document.agentId,
203
+ title: document.title,
204
+ path: document.path,
205
+ chunkId: chunk.id,
206
+ chunkOrdinal: chunk.ordinal,
207
+ content: chunk.content,
208
+ tags: document.tags,
209
+ embedding: chunk.embedding
210
+ }
211
+ ];
212
+ });
213
+ const tokens = tokenize(query);
214
+ const results = rows
215
+ .map((row) => {
216
+ const text = textScore(row, tokens);
217
+ const semantic = semanticScore(row, queryEmbedding);
218
+ return toResult(row, mode, text, semantic);
219
+ })
220
+ .filter((row) => row.score > 0 || tokens.length === 0)
221
+ .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
222
+ .slice(0, Math.max(0, limit));
223
+ return results;
224
+ },
225
+ listLinks: async (agentId) => {
226
+ const index = await load();
227
+ const documentsById = new Map(index.documents.map((document) => [document.id, document]));
228
+ return index.links
229
+ .filter((link) => {
230
+ const source = documentsById.get(link.fromDocumentId);
231
+ return agentId ? source?.agentId === agentId : true;
232
+ })
233
+ .map((link) => toGraphLink(link, documentsById))
234
+ .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
235
+ },
236
+ listBacklinks: async (title, agentId) => {
237
+ const index = await load();
238
+ const titleKey = title.toLowerCase();
239
+ const documentsById = new Map(index.documents.map((document) => [document.id, document]));
240
+ return index.links
241
+ .filter((link) => link.toTitle.toLowerCase() === titleKey)
242
+ .filter((link) => {
243
+ const source = documentsById.get(link.fromDocumentId);
244
+ return agentId ? source?.agentId === agentId : true;
245
+ })
246
+ .map((link) => toGraphLink(link, documentsById))
247
+ .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
248
+ },
249
+ getGraph: async (agentId) => {
250
+ const index = await load();
251
+ const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
252
+ const documentIds = new Set(documents.map((document) => document.id));
253
+ const edges = index.links
254
+ .filter((link) => documentIds.has(link.fromDocumentId))
255
+ .map((link) => ({
256
+ source: link.fromDocumentId,
257
+ target: link.toDocumentId,
258
+ targetTitle: link.toTitle,
259
+ weight: link.weight,
260
+ priority: link.priority
261
+ }));
262
+ return {
263
+ nodes: documents.map((document) => ({
264
+ id: document.id,
265
+ agentId: document.agentId,
266
+ title: document.title,
267
+ path: document.path,
268
+ content: document.content,
269
+ tags: document.tags
270
+ })),
271
+ edges
272
+ };
273
+ },
274
+ getGraphSummary: async (agentId) => {
275
+ const graph = await (async () => {
276
+ const index = await load();
277
+ const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
278
+ const documentIds = new Set(documents.map((document) => document.id));
279
+ const edges = index.links
280
+ .filter((link) => documentIds.has(link.fromDocumentId))
281
+ .map((link) => ({
282
+ source: link.fromDocumentId,
283
+ target: link.toDocumentId,
284
+ targetTitle: link.toTitle,
285
+ weight: link.weight,
286
+ priority: link.priority
287
+ }));
288
+ return {
289
+ nodes: documents.map((document) => ({
290
+ id: document.id,
291
+ agentId: document.agentId,
292
+ title: document.title,
293
+ path: document.path,
294
+ content: '',
295
+ tags: document.tags
296
+ })),
297
+ edges
298
+ };
299
+ })();
300
+ return graph;
301
+ },
302
+ getGraphNode: async (id, agentId) => {
303
+ const index = await load();
304
+ const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
305
+ return document
306
+ ? {
307
+ id: document.id,
308
+ agentId: document.agentId,
309
+ title: document.title,
310
+ path: document.path,
311
+ content: document.content,
312
+ tags: document.tags
313
+ }
314
+ : undefined;
315
+ },
316
+ searchGraphNodeIds: async (query, limit, agentId) => {
317
+ const index = await load();
318
+ const normalized = normalizeToken(query);
319
+ if (normalized.length === 0 || limit <= 0) {
320
+ return [];
321
+ }
322
+ const tokens = tokenize(query);
323
+ const scored = index.documents
324
+ .filter((document) => (!agentId || document.agentId === agentId))
325
+ .map((document) => {
326
+ const score = textScore({
327
+ documentId: document.id,
328
+ agentId: document.agentId,
329
+ title: document.title,
330
+ path: document.path,
331
+ chunkId: document.id,
332
+ chunkOrdinal: 0,
333
+ content: document.content,
334
+ tags: document.tags,
335
+ embedding: []
336
+ }, tokens);
337
+ return { id: document.id, score };
338
+ })
339
+ .filter((row) => row.score > 0)
340
+ .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
341
+ .slice(0, limit);
342
+ return scored.map((row) => row.id);
343
+ },
344
+ listAgents: async () => {
345
+ const index = await load();
346
+ const counts = index.documents.reduce((state, document) => {
347
+ state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
348
+ return state;
349
+ }, new Map());
350
+ return Array.from(counts.entries())
351
+ .sort((left, right) => left[0].localeCompare(right[0]))
352
+ .map(([id, documentCount]) => ({ id, documentCount }));
353
+ },
354
+ close: () => {
355
+ // File-based index has no persistent connection.
356
+ }
357
+ };
358
+ };
@@ -16,6 +16,17 @@ const walkMarkdownFiles = async (directory) => {
16
16
  }));
17
17
  return nested.flat();
18
18
  };
19
+ const walkVaultFiles = async (directory) => {
20
+ const entries = await readdir(directory, { withFileTypes: true });
21
+ const nested = await Promise.all(entries.map(async (entry) => {
22
+ const absolutePath = join(directory, entry.name);
23
+ if (entry.isDirectory()) {
24
+ return excludedDirectories.has(entry.name) ? [] : walkVaultFiles(absolutePath);
25
+ }
26
+ return entry.isFile() ? [absolutePath] : [];
27
+ }));
28
+ return nested.flat();
29
+ };
19
30
  export const resolveVaultPath = (vaultPath) => isBucketVaultUri(vaultPath) ? getBucketVaultCachePath(vaultPath) : resolvePath(vaultPath);
20
31
  export const isBucketVaultPath = (vaultPath) => isBucketVaultUri(vaultPath);
21
32
  const isPathInside = (parent, child) => {
@@ -65,6 +76,25 @@ export const readMarkdownFiles = async (vaultPath) => {
65
76
  };
66
77
  }));
67
78
  };
79
+ export const readMarkdownFileSummaries = async (vaultPath) => {
80
+ const absoluteVaultPath = await ensureVault(vaultPath);
81
+ const paths = await walkMarkdownFiles(absoluteVaultPath);
82
+ const summaries = await Promise.all(paths.map(async (absolutePath) => {
83
+ const fileStats = await stat(absolutePath);
84
+ return {
85
+ absolutePath,
86
+ relativePath: relative(absoluteVaultPath, absolutePath),
87
+ createdAt: fileStats.birthtime,
88
+ updatedAt: fileStats.mtime,
89
+ size: fileStats.size
90
+ };
91
+ }));
92
+ return summaries.sort((left, right) => left.relativePath.localeCompare(right.relativePath));
93
+ };
94
+ export const listVaultFiles = async (vaultPath) => {
95
+ const absoluteVaultPath = await ensureVault(vaultPath);
96
+ return walkVaultFiles(absoluteVaultPath);
97
+ };
68
98
  export const writeMarkdownFile = async (vaultPath, filename, content) => {
69
99
  if (isBucketVaultUri(vaultPath)) {
70
100
  return writeBucketMarkdownFile(vaultPath, filename, content);
@@ -0,0 +1,56 @@
1
+ import { readFile, writeFile } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ const indexStateFileName = 'index-state.json';
4
+ const toIndexStatePath = (vaultPath) => join(vaultPath, '.brainlink', indexStateFileName);
5
+ export const readIndexState = async (vaultPath) => {
6
+ try {
7
+ const parsed = JSON.parse(await readFile(toIndexStatePath(vaultPath), 'utf8'));
8
+ if (parsed.version !== 1 || !Array.isArray(parsed.files)) {
9
+ return null;
10
+ }
11
+ const files = parsed.files.flatMap((entry) => {
12
+ if (!entry || typeof entry !== 'object') {
13
+ return [];
14
+ }
15
+ const row = entry;
16
+ if (typeof row.path !== 'string' || typeof row.mtimeMs !== 'number' || typeof row.size !== 'number') {
17
+ return [];
18
+ }
19
+ return [
20
+ {
21
+ path: row.path,
22
+ mtimeMs: row.mtimeMs,
23
+ size: row.size
24
+ }
25
+ ];
26
+ });
27
+ return {
28
+ version: 1,
29
+ updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
30
+ chunkSize: typeof parsed.chunkSize === 'number' ? parsed.chunkSize : 1200,
31
+ embeddingProvider: typeof parsed.embeddingProvider === 'string' ? parsed.embeddingProvider : 'none',
32
+ searchPackRowChunkSize: typeof parsed.searchPackRowChunkSize === 'number' ? parsed.searchPackRowChunkSize : 5_000,
33
+ searchPackCompressionLevel: typeof parsed.searchPackCompressionLevel === 'number' ? parsed.searchPackCompressionLevel : 5,
34
+ searchPackUseDictionary: typeof parsed.searchPackUseDictionary === 'boolean' ? parsed.searchPackUseDictionary : true,
35
+ files,
36
+ pendingPackChanges: typeof parsed.pendingPackChanges === 'number' && parsed.pendingPackChanges >= 0 ? parsed.pendingPackChanges : 0
37
+ };
38
+ }
39
+ catch {
40
+ return null;
41
+ }
42
+ };
43
+ export const writeIndexState = async (vaultPath, state) => {
44
+ const payload = {
45
+ version: 1,
46
+ updatedAt: new Date().toISOString(),
47
+ chunkSize: state.chunkSize,
48
+ embeddingProvider: state.embeddingProvider,
49
+ searchPackRowChunkSize: state.searchPackRowChunkSize,
50
+ searchPackCompressionLevel: state.searchPackCompressionLevel,
51
+ searchPackUseDictionary: state.searchPackUseDictionary,
52
+ files: [...state.files].sort((left, right) => left.path.localeCompare(right.path)),
53
+ pendingPackChanges: Math.max(0, Math.floor(state.pendingPackChanges))
54
+ };
55
+ await writeFile(toIndexStatePath(vaultPath), `${JSON.stringify(payload)}\n`, 'utf8');
56
+ };
@@ -2,8 +2,16 @@ import { homedir } from 'node:os';
2
2
  import { isAbsolute, join, resolve } from 'node:path';
3
3
  const defaultHomeDirectoryName = '.brainlink';
4
4
  const defaultVaultDirectoryName = 'vault';
5
+ const resolveSafeCwd = () => {
6
+ try {
7
+ return process.cwd();
8
+ }
9
+ catch {
10
+ return homedir();
11
+ }
12
+ };
5
13
  export const expandHomePath = (path) => path === '~' || path.startsWith('~/') ? join(homedir(), path.slice(2)) : path;
6
- export const resolvePath = (path, cwd = process.cwd()) => {
14
+ export const resolvePath = (path, cwd = resolveSafeCwd()) => {
7
15
  const expandedPath = expandHomePath(path);
8
16
  return isAbsolute(expandedPath) ? expandedPath : resolve(cwd, expandedPath);
9
17
  };
@@ -0,0 +1,134 @@
1
+ import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto';
2
+ import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
3
+ import { mkdir, readFile, writeFile } from 'node:fs/promises';
4
+ import { dirname, join } from 'node:path';
5
+ import { getBrainlinkHomePath } from './paths.js';
6
+ const magic = Buffer.from('BLPK2', 'ascii');
7
+ const legacyVersion = 1;
8
+ const currentVersion = 2;
9
+ const nonceLength = 12;
10
+ const authTagLength = 16;
11
+ const algorithm = 'aes-256-gcm';
12
+ const compressionLevelMask = 0x0f;
13
+ const compressionDictionaryMask = 0x10;
14
+ const defaultCompressionLevel = 5;
15
+ const builtinDictionary = Buffer.from([
16
+ '"documentId","agentId","title","path","chunkId","chunkOrdinal","content","tags"',
17
+ '"searchMode","textScore","semanticScore","weight","priority","shared"',
18
+ 'agents/shared memory-hub architecture context index search graph markdown tags links',
19
+ '#memory #architecture #context #graph #search #index [[Memory Hub]] [[Architecture]]',
20
+ 'The quick brown fox jumps over the lazy dog. Brainlink context package metadata.',
21
+ 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-:/.#[]{}(), '
22
+ ].join('\n'), 'utf8');
23
+ const keyFilePath = (vaultPath) => {
24
+ const vaultHash = createHash('sha256').update(vaultPath).digest('hex').slice(0, 24);
25
+ return join(getBrainlinkHomePath(), 'keys', `search-pack-${vaultHash}.key`);
26
+ };
27
+ const deriveKeyFromSecret = (secret) => createHash('sha256').update(secret, 'utf8').digest();
28
+ const readOrCreateKey = async (vaultPath) => {
29
+ const envSecret = process.env.BRAINLINK_SEARCH_PACK_KEY?.trim();
30
+ if (envSecret && envSecret.length > 0) {
31
+ return deriveKeyFromSecret(envSecret);
32
+ }
33
+ const path = keyFilePath(vaultPath);
34
+ try {
35
+ const existing = (await readFile(path, 'utf8')).trim();
36
+ if (existing.length > 0) {
37
+ return deriveKeyFromSecret(existing);
38
+ }
39
+ }
40
+ catch (error) {
41
+ if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
42
+ throw error;
43
+ }
44
+ }
45
+ const secret = randomBytes(48).toString('base64url');
46
+ await mkdir(dirname(path), { recursive: true, mode: 0o700 });
47
+ await writeFile(path, `${secret}\n`, { encoding: 'utf8', mode: 0o600 });
48
+ return deriveKeyFromSecret(secret);
49
+ };
50
+ const parseHeader = (payload) => {
51
+ if (payload.length < magic.length + 1 + nonceLength + authTagLength) {
52
+ throw new Error('Invalid private pack payload: too short.');
53
+ }
54
+ const payloadMagic = payload.subarray(0, magic.length);
55
+ const payloadVersion = payload[magic.length] ?? 0;
56
+ if (!payloadMagic.equals(magic) || (payloadVersion !== legacyVersion && payloadVersion !== currentVersion)) {
57
+ throw new Error('Invalid private pack payload: unsupported format.');
58
+ }
59
+ const hasCompressionSettings = payloadVersion >= 2;
60
+ const settingsByte = hasCompressionSettings ? payload[magic.length + 1] ?? 0 : null;
61
+ const nonceStart = magic.length + 1 + (hasCompressionSettings ? 1 : 0);
62
+ const authTagStart = nonceStart + nonceLength;
63
+ const dataStart = authTagStart + authTagLength;
64
+ return {
65
+ compression: settingsByte != null
66
+ ? {
67
+ compressionLevel: settingsByte & compressionLevelMask,
68
+ useDictionary: (settingsByte & compressionDictionaryMask) !== 0
69
+ }
70
+ : {
71
+ compressionLevel: defaultCompressionLevel,
72
+ useDictionary: false
73
+ },
74
+ nonce: payload.subarray(nonceStart, authTagStart),
75
+ authTag: payload.subarray(authTagStart, dataStart),
76
+ ciphertext: payload.subarray(dataStart)
77
+ };
78
+ };
79
+ const toCompressionLevel = (value) => {
80
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
81
+ return defaultCompressionLevel;
82
+ }
83
+ const normalized = Math.round(value);
84
+ if (normalized < 0) {
85
+ return 0;
86
+ }
87
+ if (normalized > 11) {
88
+ return 11;
89
+ }
90
+ return normalized;
91
+ };
92
+ const encodeCompressionSettings = (settings) => (settings.compressionLevel & compressionLevelMask) | (settings.useDictionary ? compressionDictionaryMask : 0);
93
+ const brotliEncode = (content, settings) => {
94
+ const options = {
95
+ params: {
96
+ [zlibConstants.BROTLI_PARAM_MODE]: zlibConstants.BROTLI_MODE_TEXT,
97
+ [zlibConstants.BROTLI_PARAM_QUALITY]: settings.compressionLevel
98
+ }
99
+ };
100
+ if (settings.useDictionary) {
101
+ options.dictionary = builtinDictionary;
102
+ }
103
+ return brotliCompressSync(content, options);
104
+ };
105
+ const brotliDecode = (content, settings) => {
106
+ const options = {};
107
+ if (settings.useDictionary) {
108
+ options.dictionary = builtinDictionary;
109
+ }
110
+ return brotliDecompressSync(content, options);
111
+ };
112
+ export const encodePrivatePack = async (vaultPath, content, settings) => {
113
+ const key = await readOrCreateKey(vaultPath);
114
+ const nonce = randomBytes(nonceLength);
115
+ const normalizedSettings = {
116
+ compressionLevel: toCompressionLevel(settings?.compressionLevel),
117
+ useDictionary: settings?.useDictionary ?? true
118
+ };
119
+ const compressed = brotliEncode(content, normalizedSettings);
120
+ const cipher = createCipheriv(algorithm, key, nonce);
121
+ const ciphertext = Buffer.concat([cipher.update(compressed), cipher.final()]);
122
+ const authTag = cipher.getAuthTag();
123
+ const settingsByte = Buffer.from([encodeCompressionSettings(normalizedSettings)]);
124
+ return Buffer.concat([magic, Buffer.from([currentVersion]), settingsByte, nonce, authTag, ciphertext]);
125
+ };
126
+ export const decodePrivatePack = async (vaultPath, payload) => {
127
+ const key = await readOrCreateKey(vaultPath);
128
+ const { nonce, authTag, ciphertext, compression } = parseHeader(payload);
129
+ const decipher = createDecipheriv(algorithm, key, nonce);
130
+ decipher.setAuthTag(authTag);
131
+ const compressed = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
132
+ return brotliDecode(compressed, compression);
133
+ };
134
+ export const isPrivatePackPayload = (payload) => payload.length >= magic.length + 1 && payload.subarray(0, magic.length).equals(magic);