@andespindola/brainlink 0.1.0-beta.4 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -5
- package/CHANGELOG.md +43 -2
- package/CONTRIBUTING.md +2 -2
- package/COPYRIGHT.md +5 -0
- package/README.md +213 -20
- package/SECURITY.md +1 -1
- package/dist/application/add-note.js +62 -13
- package/dist/application/analyze-vault.js +95 -8
- package/dist/application/build-context.js +56 -1
- package/dist/application/dedupe-notes.js +226 -0
- package/dist/application/frontend/client-css.js +214 -100
- package/dist/application/frontend/client-html.js +60 -45
- package/dist/application/frontend/client-js.js +656 -94
- package/dist/application/get-graph-layout.js +22 -7
- package/dist/application/get-graph-node.js +12 -0
- package/dist/application/get-graph-summary.js +12 -0
- package/dist/application/get-graph.js +3 -3
- package/dist/application/import-legacy-sqlite.js +296 -0
- package/dist/application/index-vault.js +11 -4
- package/dist/application/list-agents.js +3 -3
- package/dist/application/list-links.js +5 -5
- package/dist/application/migrate-vault.js +91 -0
- package/dist/application/search-graph-node-ids.js +12 -0
- package/dist/application/search-knowledge.js +75 -5
- package/dist/application/server/routes.js +27 -1
- package/dist/benchmarks/large-vault.js +1 -1
- package/dist/cli/commands/agent-commands.js +412 -0
- package/dist/cli/commands/config-commands.js +167 -0
- package/dist/cli/commands/read-commands.js +25 -8
- package/dist/cli/commands/write-commands.js +669 -9
- package/dist/cli/main.js +4 -0
- package/dist/cli/runtime.js +5 -2
- package/dist/domain/context.js +53 -11
- package/dist/domain/embeddings.js +2 -1
- package/dist/domain/graph-layout.js +20 -14
- package/dist/domain/markdown.js +36 -4
- package/dist/domain/middle-out.js +18 -0
- package/dist/infrastructure/config.js +94 -8
- package/dist/infrastructure/file-index.js +328 -0
- package/dist/infrastructure/file-system-vault.js +15 -0
- package/dist/infrastructure/paths.js +9 -1
- package/dist/infrastructure/private-pack-codec.js +73 -0
- package/dist/infrastructure/search-packs.js +348 -0
- package/dist/infrastructure/session-state.js +172 -0
- package/dist/mcp/main.js +11 -3
- package/dist/mcp/server.js +27 -2
- package/dist/mcp/startup.js +35 -0
- package/dist/mcp/tools.js +633 -19
- package/docs/AGENT_USAGE.md +144 -16
- package/docs/ARCHITECTURE.md +37 -26
- package/docs/QUICKSTART.md +111 -0
- package/package.json +6 -4
- package/dist/infrastructure/sqlite/document-writer.js +0 -51
- package/dist/infrastructure/sqlite/graph-reader.js +0 -120
- package/dist/infrastructure/sqlite/schema.js +0 -111
- package/dist/infrastructure/sqlite/search-reader.js +0 -156
- package/dist/infrastructure/sqlite/types.js +0 -1
- package/dist/infrastructure/sqlite-index.js +0 -25
|
@@ -1,26 +1,41 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { stat } from 'node:fs/promises';
|
|
1
3
|
import { createCauliflowerGraphLayout } from '../domain/graph-layout.js';
|
|
2
|
-
import {
|
|
4
|
+
import { indexStoragePath } from '../infrastructure/file-index.js';
|
|
5
|
+
import { getGraphSummary } from './get-graph-summary.js';
|
|
3
6
|
const graphLayoutCache = new Map();
|
|
7
|
+
const readDatabaseSignature = async (vaultPath) => {
|
|
8
|
+
try {
|
|
9
|
+
const info = await stat(indexStoragePath(vaultPath));
|
|
10
|
+
return `${Math.floor(info.mtimeMs)}:${info.size}`;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return '0:0';
|
|
14
|
+
}
|
|
15
|
+
};
|
|
4
16
|
const createGraphSignature = (graph) => {
|
|
5
17
|
const nodesSignature = graph.nodes.map((node) => `${node.id}|${node.agentId}|${node.title}|${node.path}`).join('\n');
|
|
6
18
|
const edgesSignature = graph.edges
|
|
7
19
|
.map((edge) => `${edge.source}|${edge.target ?? ''}|${edge.targetTitle}|${edge.weight}|${edge.priority}`)
|
|
8
20
|
.join('\n');
|
|
9
|
-
return
|
|
21
|
+
return createHash('sha256')
|
|
22
|
+
.update(`${graph.nodes.length}:${nodesSignature}|${graph.edges.length}:${edgesSignature}`)
|
|
23
|
+
.digest('hex');
|
|
10
24
|
};
|
|
11
25
|
export const getGraphLayout = async (vaultPath, agentId) => {
|
|
12
|
-
const
|
|
13
|
-
const signature = createGraphSignature(graph);
|
|
26
|
+
const databaseSignature = await readDatabaseSignature(vaultPath);
|
|
14
27
|
const cacheKey = `${vaultPath}:${agentId ?? ''}`;
|
|
15
28
|
const cached = graphLayoutCache.get(cacheKey);
|
|
16
|
-
if (cached?.
|
|
29
|
+
if (cached?.databaseSignature === databaseSignature) {
|
|
17
30
|
return {
|
|
18
|
-
signature,
|
|
31
|
+
signature: cached.signature,
|
|
19
32
|
layout: cached.layout
|
|
20
33
|
};
|
|
21
34
|
}
|
|
35
|
+
const graph = await getGraphSummary(vaultPath, agentId);
|
|
36
|
+
const signature = createGraphSignature(graph);
|
|
22
37
|
const layout = createCauliflowerGraphLayout(graph);
|
|
23
|
-
graphLayoutCache.set(cacheKey, { signature, layout });
|
|
38
|
+
graphLayoutCache.set(cacheKey, { databaseSignature, signature, layout });
|
|
24
39
|
return {
|
|
25
40
|
signature,
|
|
26
41
|
layout
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
|
+
export const getGraphNode = async (vaultPath, id, agentId) => {
|
|
4
|
+
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
|
+
try {
|
|
7
|
+
return await index.getGraphNode(id, agentId);
|
|
8
|
+
}
|
|
9
|
+
finally {
|
|
10
|
+
index.close();
|
|
11
|
+
}
|
|
12
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
|
+
export const getGraphSummary = async (vaultPath, agentId) => {
|
|
4
|
+
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
|
+
try {
|
|
7
|
+
return await index.getGraphSummary(agentId);
|
|
8
|
+
}
|
|
9
|
+
finally {
|
|
10
|
+
index.close();
|
|
11
|
+
}
|
|
12
|
+
};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
-
import {
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
3
|
export const getGraph = async (vaultPath, agentId) => {
|
|
4
4
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
-
const index =
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
6
|
try {
|
|
7
|
-
return index.getGraph(agentId);
|
|
7
|
+
return await index.getGraph(agentId);
|
|
8
8
|
}
|
|
9
9
|
finally {
|
|
10
10
|
index.close();
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { access } from 'node:fs/promises';
|
|
3
|
+
import { basename, extname, join, relative, resolve } from 'node:path';
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { promisify } from 'node:util';
|
|
6
|
+
import { extractTags, extractWikiLinks } from '../domain/markdown.js';
|
|
7
|
+
import { sanitizeAgentId, sharedAgentId } from '../domain/agents.js';
|
|
8
|
+
import { ensureVault, listVaultFiles, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
|
|
9
|
+
import { getBrainlinkHomePath } from '../infrastructure/paths.js';
|
|
10
|
+
const execFileAsync = promisify(execFile);
|
|
11
|
+
const fieldSeparator = '\u001f';
|
|
12
|
+
const rowSeparator = '\u001e';
|
|
13
|
+
const contentColumnCandidates = ['content', 'markdown', 'body', 'text', 'note'];
|
|
14
|
+
const titleColumnCandidates = ['title', 'note_title', 'name', 'headline'];
|
|
15
|
+
const pathColumnCandidates = ['path', 'file_path', 'filepath', 'source_path', 'source'];
|
|
16
|
+
const agentColumnCandidates = ['agent', 'agent_id', 'namespace', 'scope'];
|
|
17
|
+
const tagColumnCandidates = ['tags', 'tag_list', 'keywords'];
|
|
18
|
+
const createdColumnCandidates = ['created_at', 'createdat', 'created', 'ctime'];
|
|
19
|
+
const updatedColumnCandidates = ['updated_at', 'updatedat', 'updated', 'mtime'];
|
|
20
|
+
const systemHubTitle = 'Memory Hub';
|
|
21
|
+
const systemRootTitle = 'Knowledge Root';
|
|
22
|
+
const normalizeTitle = (title) => title.trim().replace(/\.md$/i, '').toLowerCase();
|
|
23
|
+
const slugify = (title) => title
|
|
24
|
+
.normalize('NFKD')
|
|
25
|
+
.replace(/[\u0300-\u036f]/g, '')
|
|
26
|
+
.toLowerCase()
|
|
27
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
28
|
+
.replace(/^-+|-+$/g, '');
|
|
29
|
+
const quoteIdentifier = (value) => `"${value.replaceAll('"', '""')}"`;
|
|
30
|
+
const pickColumn = (columns, candidates) => {
|
|
31
|
+
const byLower = new Map(columns.map((column) => [column.toLowerCase(), column]));
|
|
32
|
+
return candidates.map((candidate) => byLower.get(candidate)).find((column) => Boolean(column)) ?? null;
|
|
33
|
+
};
|
|
34
|
+
const parseDelimitedRows = (rawOutput) => {
|
|
35
|
+
const normalized = rawOutput.trim();
|
|
36
|
+
if (normalized.length === 0) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
return normalized
|
|
40
|
+
.split(rowSeparator)
|
|
41
|
+
.map((row) => row.trim())
|
|
42
|
+
.filter(Boolean)
|
|
43
|
+
.map((row) => row.split(fieldSeparator));
|
|
44
|
+
};
|
|
45
|
+
const runSqliteQuery = async (databasePath, sql) => {
|
|
46
|
+
const baseArgs = ['-noheader', '-separator', fieldSeparator, '-newline', rowSeparator, '-cmd', '.timeout 5000'];
|
|
47
|
+
const runQuery = async (args) => {
|
|
48
|
+
const { stdout } = await execFileAsync('sqlite3', [...args, sql], { maxBuffer: 1024 * 1024 * 64 });
|
|
49
|
+
return parseDelimitedRows(stdout);
|
|
50
|
+
};
|
|
51
|
+
try {
|
|
52
|
+
return await runQuery(['--readonly', ...baseArgs, databasePath]);
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
56
|
+
const lower = message.toLowerCase();
|
|
57
|
+
if (lower.includes('enoent') || lower.includes('not found')) {
|
|
58
|
+
throw new Error('sqlite3 CLI was not found. Install sqlite3 to use db-import.');
|
|
59
|
+
}
|
|
60
|
+
if (lower.includes('database is locked') || lower.includes('(5)')) {
|
|
61
|
+
try {
|
|
62
|
+
const uri = pathToFileURL(databasePath);
|
|
63
|
+
uri.search = 'mode=ro&immutable=1';
|
|
64
|
+
return await runQuery(['-uri', ...baseArgs, uri.toString()]);
|
|
65
|
+
}
|
|
66
|
+
catch (fallbackError) {
|
|
67
|
+
const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
|
|
68
|
+
throw new Error(`Unable to read SQLite database (locked). Close writers (server/watch/mcp) or rerun with DB idle. Details: ${fallbackMessage}`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
throw new Error(`Unable to read SQLite database: ${message}`);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
const detectLegacyDbPath = async (vaultPath, explicitPath) => {
|
|
75
|
+
if (explicitPath) {
|
|
76
|
+
return resolve(explicitPath);
|
|
77
|
+
}
|
|
78
|
+
const vaultRoot = await ensureVault(vaultPath);
|
|
79
|
+
const candidates = [
|
|
80
|
+
join(vaultRoot, '.brainlink', 'brainlink.db'),
|
|
81
|
+
join(vaultRoot, '.brainlink', 'index.db'),
|
|
82
|
+
join(getBrainlinkHomePath(), 'brainlink.db'),
|
|
83
|
+
join(getBrainlinkHomePath(), 'vault', '.brainlink', 'brainlink.db')
|
|
84
|
+
];
|
|
85
|
+
for (const candidate of candidates) {
|
|
86
|
+
try {
|
|
87
|
+
await access(candidate);
|
|
88
|
+
return candidate;
|
|
89
|
+
}
|
|
90
|
+
catch { }
|
|
91
|
+
}
|
|
92
|
+
throw new Error(`No legacy SQLite database found. Checked: ${candidates.join(', ')}. Use --db <path-to-db> to import explicitly.`);
|
|
93
|
+
};
|
|
94
|
+
const listTables = async (dbPath) => {
|
|
95
|
+
const rows = await runSqliteQuery(dbPath, `SELECT name
|
|
96
|
+
FROM sqlite_master
|
|
97
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
98
|
+
ORDER BY name`);
|
|
99
|
+
return rows.map((columns) => columns[0]).filter(Boolean);
|
|
100
|
+
};
|
|
101
|
+
const listColumns = async (dbPath, table) => {
|
|
102
|
+
const rows = await runSqliteQuery(dbPath, `PRAGMA table_info(${quoteIdentifier(table)})`);
|
|
103
|
+
return rows.map((columns) => columns[1]).filter(Boolean);
|
|
104
|
+
};
|
|
105
|
+
const tableScore = (columns) => {
|
|
106
|
+
const contentColumn = pickColumn(columns, contentColumnCandidates);
|
|
107
|
+
const titleColumn = pickColumn(columns, titleColumnCandidates);
|
|
108
|
+
const pathColumn = pickColumn(columns, pathColumnCandidates);
|
|
109
|
+
const agentColumn = pickColumn(columns, agentColumnCandidates);
|
|
110
|
+
return (contentColumn ? 6 : 0) + (titleColumn ? 4 : 0) + (pathColumn ? 2 : 0) + (agentColumn ? 1 : 0);
|
|
111
|
+
};
|
|
112
|
+
const detectTableMapping = async (dbPath, tableOverride) => {
|
|
113
|
+
const tables = await listTables(dbPath);
|
|
114
|
+
if (tables.length === 0) {
|
|
115
|
+
throw new Error('Legacy SQLite database has no readable tables.');
|
|
116
|
+
}
|
|
117
|
+
const mappings = await Promise.all(tables.map(async (table) => {
|
|
118
|
+
const columns = await listColumns(dbPath, table);
|
|
119
|
+
return {
|
|
120
|
+
table,
|
|
121
|
+
columns,
|
|
122
|
+
titleColumn: pickColumn(columns, titleColumnCandidates),
|
|
123
|
+
contentColumn: pickColumn(columns, contentColumnCandidates),
|
|
124
|
+
pathColumn: pickColumn(columns, pathColumnCandidates),
|
|
125
|
+
agentColumn: pickColumn(columns, agentColumnCandidates),
|
|
126
|
+
tagsColumn: pickColumn(columns, tagColumnCandidates),
|
|
127
|
+
createdColumn: pickColumn(columns, createdColumnCandidates),
|
|
128
|
+
updatedColumn: pickColumn(columns, updatedColumnCandidates),
|
|
129
|
+
score: tableScore(columns)
|
|
130
|
+
};
|
|
131
|
+
}));
|
|
132
|
+
if (tableOverride) {
|
|
133
|
+
const overridden = mappings.find((mapping) => mapping.table === tableOverride);
|
|
134
|
+
if (!overridden) {
|
|
135
|
+
throw new Error(`Table not found in SQLite database: ${tableOverride}`);
|
|
136
|
+
}
|
|
137
|
+
if (!overridden.contentColumn) {
|
|
138
|
+
throw new Error(`Table ${tableOverride} does not expose a readable content column.`);
|
|
139
|
+
}
|
|
140
|
+
return { mapping: overridden, detectedTables: tables };
|
|
141
|
+
}
|
|
142
|
+
const selected = [...mappings]
|
|
143
|
+
.filter((mapping) => mapping.contentColumn)
|
|
144
|
+
.sort((left, right) => right.score - left.score)[0];
|
|
145
|
+
if (!selected) {
|
|
146
|
+
throw new Error('Could not detect a legacy table with content column in SQLite database.');
|
|
147
|
+
}
|
|
148
|
+
return { mapping: selected, detectedTables: tables };
|
|
149
|
+
};
|
|
150
|
+
const hexExpression = (column) => column ? `hex(COALESCE(CAST(${quoteIdentifier(column)} AS BLOB), X''))` : `hex(X'')`;
|
|
151
|
+
const decodeHexUtf8 = (value) => value ? Buffer.from(value, 'hex').toString('utf8') : '';
|
|
152
|
+
const parseLegacyTags = (value) => Array.from(new Set(value
|
|
153
|
+
.split(/[\s,;|]+/)
|
|
154
|
+
.map((item) => item.trim().replace(/^#/, '').toLowerCase())
|
|
155
|
+
.filter((item) => /^[a-z0-9][a-z0-9_-]*$/i.test(item))));
|
|
156
|
+
const titleFromPath = (pathValue) => basename(pathValue).replace(extname(pathValue), '').replace(/[-_]+/g, ' ').trim();
|
|
157
|
+
const appendMissingTags = (content, tags) => {
|
|
158
|
+
if (tags.length === 0) {
|
|
159
|
+
return content;
|
|
160
|
+
}
|
|
161
|
+
const existingTags = new Set(extractTags(content).map((tag) => tag.toLowerCase()));
|
|
162
|
+
const missing = tags.filter((tag) => !existingTags.has(tag.toLowerCase()));
|
|
163
|
+
if (missing.length === 0) {
|
|
164
|
+
return content;
|
|
165
|
+
}
|
|
166
|
+
return `${content.trim()}\n\nTags: ${missing.map((tag) => `#${tag}`).join(' ')}`;
|
|
167
|
+
};
|
|
168
|
+
const buildNote = (title, content, agentId) => [
|
|
169
|
+
'---',
|
|
170
|
+
`title: "${title.replaceAll('"', '\\"')}"`,
|
|
171
|
+
`agent: "${agentId}"`,
|
|
172
|
+
'---',
|
|
173
|
+
'',
|
|
174
|
+
`# ${title}`,
|
|
175
|
+
'',
|
|
176
|
+
content.trim(),
|
|
177
|
+
''
|
|
178
|
+
].join('\n');
|
|
179
|
+
const parseLegacyRow = (columns, rowIndex) => {
|
|
180
|
+
const [titleHex, contentHex, pathHex, agentHex, tagsHex] = columns;
|
|
181
|
+
const content = decodeHexUtf8(contentHex).trim();
|
|
182
|
+
const path = decodeHexUtf8(pathHex).trim();
|
|
183
|
+
const titleCandidate = decodeHexUtf8(titleHex).trim();
|
|
184
|
+
const fallbackTitleFromPath = path ? titleFromPath(path) : '';
|
|
185
|
+
const title = titleCandidate || fallbackTitleFromPath || `Imported Memory ${rowIndex + 1}`;
|
|
186
|
+
return {
|
|
187
|
+
title,
|
|
188
|
+
content,
|
|
189
|
+
path,
|
|
190
|
+
agent: decodeHexUtf8(agentHex).trim(),
|
|
191
|
+
tags: parseLegacyTags(decodeHexUtf8(tagsHex))
|
|
192
|
+
};
|
|
193
|
+
};
|
|
194
|
+
const noteRelativePath = (agentId, slug, suffix = 0) => `agents/${agentId}/${suffix > 0 ? `${slug}-${suffix + 1}` : slug || 'untitled'}.md`;
|
|
195
|
+
const reserveUniquePath = (agentId, title, reserved) => {
|
|
196
|
+
const slug = slugify(title);
|
|
197
|
+
for (let suffix = 0; suffix < 10_000; suffix += 1) {
|
|
198
|
+
const relativePath = noteRelativePath(agentId, slug, suffix);
|
|
199
|
+
if (!reserved.has(relativePath)) {
|
|
200
|
+
reserved.add(relativePath);
|
|
201
|
+
return relativePath;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
throw new Error(`Could not allocate unique path for imported note: ${title}`);
|
|
205
|
+
};
|
|
206
|
+
const ensureSystemNote = async (vaultPath, reserved, created, agentId, title, content, dryRun) => {
|
|
207
|
+
const filename = noteRelativePath(agentId, slugify(title));
|
|
208
|
+
if (reserved.has(filename)) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
reserved.add(filename);
|
|
212
|
+
created.add(filename);
|
|
213
|
+
if (dryRun) {
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
await writeMarkdownFile(vaultPath, filename, buildNote(title, content, agentId));
|
|
217
|
+
};
|
|
218
|
+
const applyConnectivityRule = async (vaultPath, reserved, created, title, content, agentId, dryRun) => {
|
|
219
|
+
const links = extractWikiLinks(content).filter((link) => normalizeTitle(link) !== normalizeTitle(title));
|
|
220
|
+
if (links.length > 0) {
|
|
221
|
+
return content.trim();
|
|
222
|
+
}
|
|
223
|
+
const normalized = normalizeTitle(title);
|
|
224
|
+
if (normalized === normalizeTitle(systemHubTitle)) {
|
|
225
|
+
await ensureSystemNote(vaultPath, reserved, created, agentId, systemRootTitle, `Entry point for agent memory. [[${systemHubTitle}]] #memory #root`, dryRun);
|
|
226
|
+
return `${content.trim()}\n\nRelated: [[${systemRootTitle}]]`;
|
|
227
|
+
}
|
|
228
|
+
await ensureSystemNote(vaultPath, reserved, created, agentId, systemHubTitle, 'Central memory index for this agent namespace. #memory #hub', dryRun);
|
|
229
|
+
return `${content.trim()}\n\nRelated: [[${systemHubTitle}]]`;
|
|
230
|
+
};
|
|
231
|
+
const importRowsFromMapping = async (vaultPath, dbPath, mapping, options, reserved) => {
|
|
232
|
+
const limit = Number.isFinite(options.limit) && (options.limit ?? 0) > 0 ? Math.floor(options.limit ?? 0) : undefined;
|
|
233
|
+
const sql = [
|
|
234
|
+
'SELECT',
|
|
235
|
+
`${hexExpression(mapping.titleColumn)} AS title_hex,`,
|
|
236
|
+
`${hexExpression(mapping.contentColumn)} AS content_hex,`,
|
|
237
|
+
`${hexExpression(mapping.pathColumn)} AS path_hex,`,
|
|
238
|
+
`${hexExpression(mapping.agentColumn)} AS agent_hex,`,
|
|
239
|
+
`${hexExpression(mapping.tagsColumn)} AS tags_hex,`,
|
|
240
|
+
`${hexExpression(mapping.createdColumn)} AS created_hex,`,
|
|
241
|
+
`${hexExpression(mapping.updatedColumn)} AS updated_hex`,
|
|
242
|
+
`FROM ${quoteIdentifier(mapping.table)}`,
|
|
243
|
+
...(limit ? [`LIMIT ${limit}`] : [])
|
|
244
|
+
].join(' ');
|
|
245
|
+
const rows = await runSqliteQuery(dbPath, sql);
|
|
246
|
+
const createdSystemNotes = new Set();
|
|
247
|
+
const importedFiles = [];
|
|
248
|
+
let imported = 0;
|
|
249
|
+
let skipped = 0;
|
|
250
|
+
for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
|
|
251
|
+
const row = parseLegacyRow(rows[rowIndex], rowIndex);
|
|
252
|
+
if (!row.content) {
|
|
253
|
+
skipped += 1;
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
const agentId = sanitizeAgentId(options.agentOverride || row.agent || sharedAgentId);
|
|
257
|
+
const filename = reserveUniquePath(agentId, row.title, reserved);
|
|
258
|
+
const mergedContent = appendMissingTags(row.content, row.tags);
|
|
259
|
+
const connectedContent = await applyConnectivityRule(vaultPath, reserved, createdSystemNotes, row.title, mergedContent, agentId, options.dryRun === true);
|
|
260
|
+
const note = buildNote(row.title, connectedContent, agentId);
|
|
261
|
+
if (options.dryRun !== true) {
|
|
262
|
+
await writeMarkdownFile(vaultPath, filename, note);
|
|
263
|
+
}
|
|
264
|
+
importedFiles.push(filename);
|
|
265
|
+
imported += 1;
|
|
266
|
+
}
|
|
267
|
+
return {
|
|
268
|
+
rowsRead: rows.length,
|
|
269
|
+
imported,
|
|
270
|
+
skipped,
|
|
271
|
+
createdSystemNotes: createdSystemNotes.size,
|
|
272
|
+
importedFiles
|
|
273
|
+
};
|
|
274
|
+
};
|
|
275
|
+
export const importLegacySqliteDatabase = async (vaultPath, options = {}) => {
|
|
276
|
+
const vault = await ensureVault(vaultPath);
|
|
277
|
+
const dbPath = await detectLegacyDbPath(vaultPath, options.dbPath);
|
|
278
|
+
const { mapping, detectedTables } = await detectTableMapping(dbPath, options.table);
|
|
279
|
+
const existingFiles = (await listVaultFiles(vaultPath))
|
|
280
|
+
.filter((path) => extname(path).toLowerCase() === '.md')
|
|
281
|
+
.map((path) => relative(vault, path));
|
|
282
|
+
const reserved = new Set(existingFiles);
|
|
283
|
+
const imported = await importRowsFromMapping(vaultPath, dbPath, mapping, options, reserved);
|
|
284
|
+
return {
|
|
285
|
+
vault,
|
|
286
|
+
dbPath,
|
|
287
|
+
table: mapping.table,
|
|
288
|
+
detectedTables,
|
|
289
|
+
rowsRead: imported.rowsRead,
|
|
290
|
+
imported: imported.imported,
|
|
291
|
+
skipped: imported.skipped,
|
|
292
|
+
createdSystemNotes: imported.createdSystemNotes,
|
|
293
|
+
dryRun: options.dryRun === true,
|
|
294
|
+
importedFiles: imported.importedFiles
|
|
295
|
+
};
|
|
296
|
+
};
|
|
@@ -3,7 +3,8 @@ import { sharedAgentId } from '../domain/agents.js';
|
|
|
3
3
|
import { createEmbeddingProvider } from '../domain/embeddings.js';
|
|
4
4
|
import { loadBrainlinkConfig } from '../infrastructure/config.js';
|
|
5
5
|
import { ensureVault, readMarkdownFiles } from '../infrastructure/file-system-vault.js';
|
|
6
|
-
import {
|
|
6
|
+
import { buildSearchPacks } from '../infrastructure/search-packs.js';
|
|
7
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
7
8
|
const toTitleKey = (title) => title.toLowerCase();
|
|
8
9
|
const appendTitleEntry = (map, document) => {
|
|
9
10
|
const key = toTitleKey(document.title);
|
|
@@ -59,10 +60,16 @@ export const indexVault = async (vaultPath) => {
|
|
|
59
60
|
}));
|
|
60
61
|
const titleMaps = createTitleMaps(documents);
|
|
61
62
|
const indexedDocuments = await embedIndexedDocuments(documents.map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider);
|
|
62
|
-
const index =
|
|
63
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
63
64
|
try {
|
|
64
|
-
index.reset();
|
|
65
|
-
index.saveDocuments(indexedDocuments);
|
|
65
|
+
await index.reset();
|
|
66
|
+
await index.saveDocuments(indexedDocuments);
|
|
67
|
+
try {
|
|
68
|
+
await buildSearchPacks(absoluteVaultPath, indexedDocuments);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Pack generation is best-effort. The JSON index remains the primary path.
|
|
72
|
+
}
|
|
66
73
|
return {
|
|
67
74
|
documentCount: indexedDocuments.length,
|
|
68
75
|
chunkCount: indexedDocuments.reduce((total, document) => total + document.chunks.length, 0),
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
-
import {
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
3
|
export const listAgents = async (vaultPath) => {
|
|
4
4
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
-
const index =
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
6
|
try {
|
|
7
|
-
return index.listAgents();
|
|
7
|
+
return await index.listAgents();
|
|
8
8
|
}
|
|
9
9
|
finally {
|
|
10
10
|
index.close();
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
-
import {
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
3
|
export const listLinks = async (vaultPath, agentId) => {
|
|
4
4
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
-
const index =
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
6
|
try {
|
|
7
|
-
return index.listLinks(agentId);
|
|
7
|
+
return await index.listLinks(agentId);
|
|
8
8
|
}
|
|
9
9
|
finally {
|
|
10
10
|
index.close();
|
|
@@ -12,9 +12,9 @@ export const listLinks = async (vaultPath, agentId) => {
|
|
|
12
12
|
};
|
|
13
13
|
export const listBacklinks = async (vaultPath, title, agentId) => {
|
|
14
14
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
15
|
-
const index =
|
|
15
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
16
16
|
try {
|
|
17
|
-
return index.listBacklinks(title, agentId);
|
|
17
|
+
return await index.listBacklinks(title, agentId);
|
|
18
18
|
}
|
|
19
19
|
finally {
|
|
20
20
|
index.close();
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, extname, isAbsolute, join, relative } from 'node:path';
|
|
3
|
+
import { ensureVault, isBucketVaultPath, listVaultFiles, resolveVaultPath, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
|
|
4
|
+
const directoryMode = 0o700;
|
|
5
|
+
const fileMode = 0o600;
|
|
6
|
+
const isMarkdownPath = (path) => extname(path).toLowerCase() === '.md';
|
|
7
|
+
const timestamp = () => new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, 'Z');
|
|
8
|
+
const isPathInside = (parent, child) => {
|
|
9
|
+
const path = relative(parent, child);
|
|
10
|
+
return path === '' || (!path.startsWith('..') && !isAbsolute(path));
|
|
11
|
+
};
|
|
12
|
+
const conflictPath = (targetPath) => {
|
|
13
|
+
const extension = extname(targetPath);
|
|
14
|
+
const base = extension ? targetPath.slice(0, -extension.length) : targetPath;
|
|
15
|
+
return `${base}.conflict-${timestamp()}${extension}`;
|
|
16
|
+
};
|
|
17
|
+
const writePreservedFile = async (absolutePath, content) => {
|
|
18
|
+
await mkdir(dirname(absolutePath), { recursive: true, mode: directoryMode });
|
|
19
|
+
await writeFile(absolutePath, content, { mode: fileMode });
|
|
20
|
+
await chmod(absolutePath, fileMode);
|
|
21
|
+
};
|
|
22
|
+
const writeMigratedFile = async (targetVault, targetRoot, absolutePath, content) => {
|
|
23
|
+
if (isBucketVaultPath(targetVault)) {
|
|
24
|
+
await writeMarkdownFile(targetVault, relative(targetRoot, absolutePath), content.toString('utf8'));
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
await writePreservedFile(absolutePath, content);
|
|
28
|
+
};
|
|
29
|
+
export const planVaultMigration = async (source, target) => {
|
|
30
|
+
const sourceFiles = (await listVaultFiles(source)).filter(isMarkdownPath);
|
|
31
|
+
return sourceFiles.reduce(async (statePromise, sourceFile) => {
|
|
32
|
+
const state = await statePromise;
|
|
33
|
+
const targetFile = join(target, relative(source, sourceFile));
|
|
34
|
+
if (!isPathInside(target, targetFile)) {
|
|
35
|
+
return state;
|
|
36
|
+
}
|
|
37
|
+
const sourceContent = await readFile(sourceFile);
|
|
38
|
+
try {
|
|
39
|
+
const targetContent = await readFile(targetFile);
|
|
40
|
+
if (sourceContent.equals(targetContent)) {
|
|
41
|
+
return [...state, { kind: 'unchanged', sourcePath: sourceFile, targetPath: targetFile, sourceContent }];
|
|
42
|
+
}
|
|
43
|
+
return [...state, { kind: 'conflict', sourcePath: sourceFile, targetPath: conflictPath(targetFile), sourceContent }];
|
|
44
|
+
}
|
|
45
|
+
catch (error) {
|
|
46
|
+
if (!(error instanceof Error) || !('code' in error) || error.code !== 'ENOENT') {
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
return [...state, { kind: 'copy', sourcePath: sourceFile, targetPath: targetFile, sourceContent }];
|
|
50
|
+
}
|
|
51
|
+
}, Promise.resolve([]));
|
|
52
|
+
};
|
|
53
|
+
export const previewVaultMigration = async (sourceVault, targetVault) => {
|
|
54
|
+
const source = await ensureVault(sourceVault);
|
|
55
|
+
const target = await ensureVault(targetVault);
|
|
56
|
+
if (source === target) {
|
|
57
|
+
return { source, target, copied: 0, unchanged: 0, conflicted: 0 };
|
|
58
|
+
}
|
|
59
|
+
const actions = await planVaultMigration(source, target);
|
|
60
|
+
const copied = actions.filter((action) => action.kind === 'copy').length;
|
|
61
|
+
const unchanged = actions.filter((action) => action.kind === 'unchanged').length;
|
|
62
|
+
const conflicted = actions.filter((action) => action.kind === 'conflict').length;
|
|
63
|
+
return { source, target, copied, unchanged, conflicted };
|
|
64
|
+
};
|
|
65
|
+
export const migrateVaultContent = async (sourceVault, targetVault) => {
|
|
66
|
+
const source = await ensureVault(sourceVault);
|
|
67
|
+
const target = await ensureVault(targetVault);
|
|
68
|
+
if (source === target) {
|
|
69
|
+
return { source, target, copied: 0, unchanged: 0, conflicted: 0 };
|
|
70
|
+
}
|
|
71
|
+
const actions = await planVaultMigration(source, target);
|
|
72
|
+
for (const action of actions) {
|
|
73
|
+
if (action.kind === 'unchanged') {
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
await writeMigratedFile(targetVault, target, action.targetPath, action.sourceContent);
|
|
77
|
+
}
|
|
78
|
+
const copied = actions.filter((action) => action.kind === 'copy').length;
|
|
79
|
+
const unchanged = actions.filter((action) => action.kind === 'unchanged').length;
|
|
80
|
+
const conflicted = actions.filter((action) => action.kind === 'conflict').length;
|
|
81
|
+
return { source, target, copied, unchanged, conflicted };
|
|
82
|
+
};
|
|
83
|
+
export const shouldMigrateDefaultVault = async (sourceVault, targetVault) => {
|
|
84
|
+
const source = resolveVaultPath(sourceVault);
|
|
85
|
+
const target = resolveVaultPath(targetVault);
|
|
86
|
+
if (source === target) {
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
const [sourceFiles, targetFiles] = await Promise.all([listVaultFiles(source), listVaultFiles(target)]);
|
|
90
|
+
return sourceFiles.filter(isMarkdownPath).length > 0 && targetFiles.filter(isMarkdownPath).length === 0;
|
|
91
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
+
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
3
|
+
export const searchGraphNodeIds = async (vaultPath, query, limit, agentId) => {
|
|
4
|
+
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
5
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
6
|
+
try {
|
|
7
|
+
return await index.searchGraphNodeIds(query, limit, agentId);
|
|
8
|
+
}
|
|
9
|
+
finally {
|
|
10
|
+
index.close();
|
|
11
|
+
}
|
|
12
|
+
};
|
|
@@ -1,19 +1,89 @@
|
|
|
1
|
+
import { stat } from 'node:fs/promises';
|
|
1
2
|
import { ensureVault } from '../infrastructure/file-system-vault.js';
|
|
2
|
-
import {
|
|
3
|
+
import { ensurePrivatePacksFromLegacyIndex, searchInPacks } from '../infrastructure/search-packs.js';
|
|
4
|
+
import { indexStoragePath, openFileIndex } from '../infrastructure/file-index.js';
|
|
3
5
|
import { createEmbeddingProvider } from '../domain/embeddings.js';
|
|
4
6
|
import { loadBrainlinkConfig, sanitizeSearchMode } from '../infrastructure/config.js';
|
|
7
|
+
const hybridCacheTtlMs = 30_000;
|
|
8
|
+
const hybridCacheMaxEntries = 200;
|
|
9
|
+
const hybridSearchCache = new Map();
|
|
10
|
+
const readIndexMtimeMs = async (vaultPath) => {
|
|
11
|
+
try {
|
|
12
|
+
return (await stat(indexStoragePath(vaultPath))).mtimeMs;
|
|
13
|
+
}
|
|
14
|
+
catch {
|
|
15
|
+
return 0;
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
const toCacheKey = (vaultPath, query, limit, agentId) => JSON.stringify({
|
|
19
|
+
vaultPath,
|
|
20
|
+
query: query.trim().toLowerCase(),
|
|
21
|
+
limit,
|
|
22
|
+
agentId: agentId?.trim().toLowerCase() ?? '*'
|
|
23
|
+
});
|
|
24
|
+
const cacheGet = (key, indexMtimeMs) => {
|
|
25
|
+
const entry = hybridSearchCache.get(key);
|
|
26
|
+
if (!entry) {
|
|
27
|
+
return undefined;
|
|
28
|
+
}
|
|
29
|
+
const fresh = Date.now() - entry.createdAt <= hybridCacheTtlMs && entry.indexMtimeMs === indexMtimeMs;
|
|
30
|
+
if (!fresh) {
|
|
31
|
+
hybridSearchCache.delete(key);
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
return entry.results;
|
|
35
|
+
};
|
|
36
|
+
const cacheSet = (entry) => {
|
|
37
|
+
hybridSearchCache.set(entry.key, entry);
|
|
38
|
+
if (hybridSearchCache.size <= hybridCacheMaxEntries) {
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const overflow = hybridSearchCache.size - hybridCacheMaxEntries;
|
|
42
|
+
const keys = Array.from(hybridSearchCache.keys()).slice(0, overflow);
|
|
43
|
+
keys.forEach((key) => hybridSearchCache.delete(key));
|
|
44
|
+
};
|
|
5
45
|
export const searchKnowledge = async (vaultPath, query, limit, agentId, mode) => {
|
|
6
46
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
7
47
|
const config = await loadBrainlinkConfig();
|
|
8
48
|
const searchMode = sanitizeSearchMode(mode, config.defaultSearchMode);
|
|
49
|
+
await ensurePrivatePacksFromLegacyIndex(absoluteVaultPath);
|
|
50
|
+
const cacheKey = searchMode === 'hybrid' ? toCacheKey(absoluteVaultPath, query, limit, agentId) : undefined;
|
|
51
|
+
const indexMtimeMs = cacheKey ? await readIndexMtimeMs(absoluteVaultPath) : 0;
|
|
52
|
+
const cached = cacheKey ? cacheGet(cacheKey, indexMtimeMs) : undefined;
|
|
53
|
+
if (cached) {
|
|
54
|
+
return cached;
|
|
55
|
+
}
|
|
9
56
|
const provider = createEmbeddingProvider(config.embeddingProvider);
|
|
10
57
|
const shouldEmbedQuery = searchMode !== 'fts' && provider.name !== 'none';
|
|
11
58
|
const queryEmbedding = shouldEmbedQuery ? (await provider.embed([query]))[0] ?? [] : [];
|
|
12
|
-
const index = openSqliteIndex(absoluteVaultPath);
|
|
13
59
|
try {
|
|
14
|
-
|
|
60
|
+
const index = openFileIndex(absoluteVaultPath);
|
|
61
|
+
try {
|
|
62
|
+
const results = await index.search(query, limit, agentId, searchMode, queryEmbedding);
|
|
63
|
+
if (cacheKey) {
|
|
64
|
+
cacheSet({
|
|
65
|
+
key: cacheKey,
|
|
66
|
+
createdAt: Date.now(),
|
|
67
|
+
indexMtimeMs,
|
|
68
|
+
results
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return results;
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
index.close();
|
|
75
|
+
}
|
|
15
76
|
}
|
|
16
|
-
|
|
17
|
-
|
|
77
|
+
catch {
|
|
78
|
+
const fallbackResults = await searchInPacks(absoluteVaultPath, query, limit, agentId);
|
|
79
|
+
if (cacheKey) {
|
|
80
|
+
cacheSet({
|
|
81
|
+
key: cacheKey,
|
|
82
|
+
createdAt: Date.now(),
|
|
83
|
+
indexMtimeMs,
|
|
84
|
+
results: fallbackResults
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
return fallbackResults;
|
|
18
88
|
}
|
|
19
89
|
};
|