@andespindola/brainlink 0.1.0-beta.16 → 0.1.0-beta.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +9 -6
- package/CHANGELOG.md +27 -0
- package/COPYRIGHT.md +5 -0
- package/README.md +177 -20
- package/dist/application/add-note.js +13 -44
- package/dist/application/auto-migrate-configured-vault.js +37 -0
- package/dist/application/build-context.js +64 -3
- package/dist/application/canonical-context-links.js +209 -0
- package/dist/application/dedupe-notes.js +226 -0
- package/dist/application/frontend/client-css.js +258 -51
- package/dist/application/frontend/client-html.js +50 -27
- package/dist/application/frontend/client-js.js +1369 -605
- package/dist/application/frontend/client-render-worker-js.js +645 -0
- package/dist/application/frontend/client-worker-js.js +66 -0
- package/dist/application/get-graph-contexts.js +33 -0
- package/dist/application/get-graph-layout.js +62 -8
- package/dist/application/get-graph-stream-chunk.js +326 -0
- package/dist/application/get-graph-view.js +246 -0
- package/dist/application/graph-view-state.js +66 -0
- package/dist/application/import-legacy-sqlite.js +266 -0
- package/dist/application/index-vault.js +262 -23
- package/dist/application/migrate-context-links.js +79 -0
- package/dist/application/offline-pack-backup.js +44 -0
- package/dist/application/search-graph-node-ids.js +63 -3
- package/dist/application/server/routes.js +247 -7
- package/dist/application/start-server.js +75 -4
- package/dist/application/watch-vault.js +23 -2
- package/dist/cli/commands/agent-commands.js +7 -0
- package/dist/cli/commands/write-commands.js +924 -14
- package/dist/cli/runtime.js +10 -2
- package/dist/domain/context.js +54 -11
- package/dist/domain/graph-contexts.js +180 -0
- package/dist/domain/graph-layout.js +389 -18
- package/dist/domain/markdown.js +53 -9
- package/dist/domain/middle-out.js +18 -0
- package/dist/infrastructure/config.js +121 -4
- package/dist/infrastructure/file-index.js +76 -6
- package/dist/infrastructure/file-system-vault.js +15 -0
- package/dist/infrastructure/index-state.js +58 -0
- package/dist/infrastructure/private-pack-codec.js +71 -10
- package/dist/infrastructure/search-packs.js +286 -15
- package/dist/infrastructure/vault-migration-state.js +69 -0
- package/dist/infrastructure/volatile-memory.js +100 -0
- package/dist/mcp/runtime.js +20 -0
- package/dist/mcp/server.js +39 -11
- package/dist/mcp/tools.js +183 -7
- package/docs/AGENT_USAGE.md +96 -5
- package/docs/ARCHITECTURE.md +8 -0
- package/docs/QUICKSTART.md +7 -0
- package/package.json +7 -2
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { getGraphLayout } from './get-graph-layout.js';
|
|
2
|
+
const macroScale = 0.24;
|
|
3
|
+
const microCoverage = 0.72;
|
|
4
|
+
const nodeLimit = 1000;
|
|
5
|
+
const edgeLimit = 1400;
|
|
6
|
+
const groupEdgeLimit = 900;
|
|
7
|
+
const inViewport = (item, input) => {
|
|
8
|
+
const radius = item.radius ?? 48;
|
|
9
|
+
return (item.x + radius >= input.x &&
|
|
10
|
+
item.x - radius <= input.x + input.width &&
|
|
11
|
+
item.y + radius >= input.y &&
|
|
12
|
+
item.y - radius <= input.y + input.height);
|
|
13
|
+
};
|
|
14
|
+
const groupCoverage = (group, input) => {
|
|
15
|
+
const viewportRadius = Math.max(input.width, input.height) / 2;
|
|
16
|
+
const centerX = input.x + input.width / 2;
|
|
17
|
+
const centerY = input.y + input.height / 2;
|
|
18
|
+
const centerDistance = Math.hypot(group.x - centerX, group.y - centerY);
|
|
19
|
+
const fitCoverage = Math.min(1, childGraphRenderRadius(group) / Math.max(viewportRadius, 1));
|
|
20
|
+
const centerCoverage = 1 - Math.min(1, centerDistance / Math.max(viewportRadius, 1));
|
|
21
|
+
return fitCoverage * 0.72 + centerCoverage * 0.28;
|
|
22
|
+
};
|
|
23
|
+
const childGraphRenderRadius = (group) => {
|
|
24
|
+
const childCount = Math.max(group.nodeIds.length, group.childGroupIds.length, 1);
|
|
25
|
+
return Math.max(420, Math.min(1800, Math.sqrt(childCount) * 24));
|
|
26
|
+
};
|
|
27
|
+
const groupRenderRadius = (group) => {
|
|
28
|
+
const childCount = Math.max(group.nodeIds.length, group.childGroupIds.length, 1);
|
|
29
|
+
return 10 + Math.min(Math.log2(childCount + 1) * 4.2, 22);
|
|
30
|
+
};
|
|
31
|
+
const arrangeGraphLevelGroups = (groups) => {
|
|
32
|
+
if (groups.length <= 1) {
|
|
33
|
+
return groups.map((group) => ({ ...group, radius: groupRenderRadius(group) }));
|
|
34
|
+
}
|
|
35
|
+
const centerGroup = groups
|
|
36
|
+
.map((group) => ({
|
|
37
|
+
group,
|
|
38
|
+
score: Math.max(group.nodeIds.length, group.childGroupIds.length, 1) + group.externalEdges.length
|
|
39
|
+
}))
|
|
40
|
+
.sort((left, right) => right.score - left.score || left.group.title.localeCompare(right.group.title))[0]?.group;
|
|
41
|
+
const outerGroups = groups
|
|
42
|
+
.filter((group) => group.id !== centerGroup?.id)
|
|
43
|
+
.sort((left, right) => left.segment.localeCompare(right.segment) || left.title.localeCompare(right.title));
|
|
44
|
+
const baseRadius = Math.max(520, Math.min(2200, Math.sqrt(groups.length) * 135));
|
|
45
|
+
const goldenAngle = Math.PI * (3 - Math.sqrt(5));
|
|
46
|
+
const arranged = centerGroup
|
|
47
|
+
? [{ ...centerGroup, x: 0, y: 0, radius: groupRenderRadius(centerGroup) }]
|
|
48
|
+
: [];
|
|
49
|
+
outerGroups.forEach((group, index) => {
|
|
50
|
+
const ringRadius = baseRadius * Math.sqrt((index + 1) / Math.max(outerGroups.length, 1));
|
|
51
|
+
const angle = index * goldenAngle;
|
|
52
|
+
arranged.push({
|
|
53
|
+
...group,
|
|
54
|
+
x: Math.cos(angle) * ringRadius,
|
|
55
|
+
y: Math.sin(angle) * ringRadius,
|
|
56
|
+
radius: groupRenderRadius(group)
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
return arranged;
|
|
60
|
+
};
|
|
61
|
+
const distanceToViewportCenter = (item, input) => {
|
|
62
|
+
const centerX = input.x + input.width / 2;
|
|
63
|
+
const centerY = input.y + input.height / 2;
|
|
64
|
+
return Math.hypot(item.x - centerX, item.y - centerY);
|
|
65
|
+
};
|
|
66
|
+
const selectViewportItemsWithFill = (items, input, limit = nodeLimit) => {
|
|
67
|
+
const visible = items.filter((item) => inViewport(item, input));
|
|
68
|
+
if (visible.length >= limit) {
|
|
69
|
+
return visible.slice(0, limit);
|
|
70
|
+
}
|
|
71
|
+
const selectedIds = new Set(visible.map((item) => item.id));
|
|
72
|
+
const fill = items
|
|
73
|
+
.filter((item) => !selectedIds.has(item.id))
|
|
74
|
+
.sort((left, right) => distanceToViewportCenter(left, input) - distanceToViewportCenter(right, input) || left.id.localeCompare(right.id))
|
|
75
|
+
.slice(0, Math.max(0, limit - visible.length));
|
|
76
|
+
return visible.concat(fill);
|
|
77
|
+
};
|
|
78
|
+
const groupNode = (group) => [
|
|
79
|
+
`group:${group.id}`,
|
|
80
|
+
group.title,
|
|
81
|
+
group.x,
|
|
82
|
+
group.y,
|
|
83
|
+
group.group,
|
|
84
|
+
group.segment,
|
|
85
|
+
'group'
|
|
86
|
+
];
|
|
87
|
+
const realNode = (node) => [
|
|
88
|
+
node.id,
|
|
89
|
+
node.title,
|
|
90
|
+
node.x,
|
|
91
|
+
node.y,
|
|
92
|
+
node.group,
|
|
93
|
+
node.segment,
|
|
94
|
+
'node'
|
|
95
|
+
];
|
|
96
|
+
const descendants = (group, groupById) => group.nodeIds.length > 0
|
|
97
|
+
? group.nodeIds
|
|
98
|
+
: group.childGroupIds.flatMap((childId) => {
|
|
99
|
+
const child = groupById.get(childId);
|
|
100
|
+
return child ? descendants(child, groupById) : [];
|
|
101
|
+
});
|
|
102
|
+
const aggregateGroupEdges = (groups, edges, groupById) => {
|
|
103
|
+
const groupNodeByNodeId = new Map();
|
|
104
|
+
groups.forEach((group) => {
|
|
105
|
+
descendants(group, groupById).forEach((nodeId) => groupNodeByNodeId.set(nodeId, `group:${group.id}`));
|
|
106
|
+
});
|
|
107
|
+
const selected = new Map();
|
|
108
|
+
edges.forEach((edge) => {
|
|
109
|
+
if (!edge.target)
|
|
110
|
+
return;
|
|
111
|
+
const source = groupNodeByNodeId.get(edge.source);
|
|
112
|
+
const target = groupNodeByNodeId.get(edge.target);
|
|
113
|
+
if (!source || !target || source === target)
|
|
114
|
+
return;
|
|
115
|
+
const key = source < target ? `${source}|${target}` : `${target}|${source}`;
|
|
116
|
+
const current = selected.get(key);
|
|
117
|
+
if (current && current[2] >= edge.weight)
|
|
118
|
+
return;
|
|
119
|
+
selected.set(key, [source, target, edge.weight, edge.priority]);
|
|
120
|
+
});
|
|
121
|
+
const degreeCounts = new Map();
|
|
122
|
+
return Array.from(selected.values())
|
|
123
|
+
.sort((left, right) => right[2] - left[2] || left[0].localeCompare(right[0]) || left[1].localeCompare(right[1]))
|
|
124
|
+
.filter((edge) => {
|
|
125
|
+
const sourceCount = degreeCounts.get(edge[0]) ?? 0;
|
|
126
|
+
const targetCount = degreeCounts.get(edge[1]) ?? 0;
|
|
127
|
+
if (sourceCount >= 3 || targetCount >= 3) {
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
degreeCounts.set(edge[0], sourceCount + 1);
|
|
131
|
+
degreeCounts.set(edge[1], targetCount + 1);
|
|
132
|
+
return true;
|
|
133
|
+
})
|
|
134
|
+
.slice(0, groupEdgeLimit);
|
|
135
|
+
};
|
|
136
|
+
const realEdges = (edges, nodeIds) => edges
|
|
137
|
+
.filter((edge) => Boolean(edge.target && nodeIds.has(edge.source) && nodeIds.has(edge.target)))
|
|
138
|
+
.slice(0, edgeLimit)
|
|
139
|
+
.map((edge) => [edge.source, edge.target, edge.weight, edge.priority]);
|
|
140
|
+
const degreeMap = (edges) => edges.reduce((degrees, edge) => {
|
|
141
|
+
degrees.set(edge.source, (degrees.get(edge.source) ?? 0) + edge.weight);
|
|
142
|
+
if (edge.target) {
|
|
143
|
+
degrees.set(edge.target, (degrees.get(edge.target) ?? 0) + edge.weight);
|
|
144
|
+
}
|
|
145
|
+
return degrees;
|
|
146
|
+
}, new Map());
|
|
147
|
+
const arrangeChildGraphNodes = (nodes, group, degrees) => {
|
|
148
|
+
if (nodes.length <= 1) {
|
|
149
|
+
return nodes.map((node) => ({ ...node, x: group.x, y: group.y }));
|
|
150
|
+
}
|
|
151
|
+
const centerNode = nodes
|
|
152
|
+
.map((node) => ({
|
|
153
|
+
node,
|
|
154
|
+
score: (degrees.get(node.id) ?? 0) + node.tags.length
|
|
155
|
+
}))
|
|
156
|
+
.sort((left, right) => right.score - left.score || left.node.title.localeCompare(right.node.title))[0]?.node;
|
|
157
|
+
const outerNodes = nodes
|
|
158
|
+
.filter((node) => node.id !== centerNode?.id)
|
|
159
|
+
.sort((left, right) => {
|
|
160
|
+
const degreeDelta = (degrees.get(right.id) ?? 0) - (degrees.get(left.id) ?? 0);
|
|
161
|
+
if (degreeDelta !== 0)
|
|
162
|
+
return degreeDelta;
|
|
163
|
+
return left.title.localeCompare(right.title);
|
|
164
|
+
});
|
|
165
|
+
const targetRadius = childGraphRenderRadius(group);
|
|
166
|
+
const goldenAngle = Math.PI * (3 - Math.sqrt(5));
|
|
167
|
+
const arranged = centerNode
|
|
168
|
+
? [{ ...centerNode, x: group.x, y: group.y }]
|
|
169
|
+
: [];
|
|
170
|
+
outerNodes.forEach((node, index) => {
|
|
171
|
+
const ringRadius = targetRadius * Math.sqrt((index + 1) / Math.max(outerNodes.length, 1));
|
|
172
|
+
const angle = index * goldenAngle;
|
|
173
|
+
arranged.push({
|
|
174
|
+
...node,
|
|
175
|
+
x: group.x + Math.cos(angle) * ringRadius,
|
|
176
|
+
y: group.y + Math.sin(angle) * ringRadius
|
|
177
|
+
});
|
|
178
|
+
});
|
|
179
|
+
return arranged;
|
|
180
|
+
};
|
|
181
|
+
const limitEdges = (edges) => edges.slice(0, edgeLimit);
|
|
182
|
+
export const getGraphView = async (vaultPath, input) => {
|
|
183
|
+
const { signature, layout } = await getGraphLayout(vaultPath, {
|
|
184
|
+
agentId: input.agentId,
|
|
185
|
+
context: input.context
|
|
186
|
+
});
|
|
187
|
+
const groups = layout.groups ?? [];
|
|
188
|
+
const degrees = degreeMap(layout.edges);
|
|
189
|
+
const groupById = new Map(groups.map((group) => [group.id, group]));
|
|
190
|
+
if (groups.length === 0) {
|
|
191
|
+
const nodes = layout.nodes.filter((node) => inViewport(node, input)).slice(0, nodeLimit);
|
|
192
|
+
const nodeIds = new Set(nodes.map((node) => node.id));
|
|
193
|
+
const viewNodes = nodes.map(realNode);
|
|
194
|
+
return {
|
|
195
|
+
signature,
|
|
196
|
+
mode: 'flat',
|
|
197
|
+
nodes: viewNodes,
|
|
198
|
+
edges: limitEdges(realEdges(layout.edges, nodeIds)),
|
|
199
|
+
totals: {
|
|
200
|
+
nodes: layout.nodes.length,
|
|
201
|
+
edges: layout.edges.length
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
const rootGroups = arrangeGraphLevelGroups(groups.filter((group) => group.parentId === null));
|
|
206
|
+
const leafGroups = arrangeGraphLevelGroups(groups.filter((group) => group.nodeIds.length > 0));
|
|
207
|
+
const visibleGroups = selectViewportItemsWithFill(rootGroups, input);
|
|
208
|
+
const focused = leafGroups
|
|
209
|
+
.filter((group) => group.nodeIds.length > 0 && inViewport(group, input))
|
|
210
|
+
.map((group) => ({ group, coverage: groupCoverage(group, input) }))
|
|
211
|
+
.sort((left, right) => right.coverage - left.coverage)[0];
|
|
212
|
+
if (focused && input.scale >= macroScale && focused.coverage >= microCoverage) {
|
|
213
|
+
const nodeIds = new Set(focused.group.nodeIds);
|
|
214
|
+
const arrangedNodes = arrangeChildGraphNodes(layout.nodes.filter((node) => nodeIds.has(node.id)), focused.group, degrees);
|
|
215
|
+
const nodesInViewport = arrangedNodes
|
|
216
|
+
.filter((node) => inViewport(node, input))
|
|
217
|
+
.slice(0, nodeLimit);
|
|
218
|
+
const nodes = nodesInViewport.length > 0
|
|
219
|
+
? nodesInViewport
|
|
220
|
+
: arrangedNodes.slice(0, nodeLimit);
|
|
221
|
+
const visibleNodeIds = new Set(nodes.map((node) => node.id));
|
|
222
|
+
const viewNodes = nodes.map(realNode);
|
|
223
|
+
return {
|
|
224
|
+
signature,
|
|
225
|
+
mode: 'micro',
|
|
226
|
+
nodes: viewNodes,
|
|
227
|
+
edges: limitEdges(realEdges(layout.edges, visibleNodeIds)),
|
|
228
|
+
totals: {
|
|
229
|
+
nodes: layout.nodes.length,
|
|
230
|
+
edges: layout.edges.length
|
|
231
|
+
}
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
const groupsToRender = visibleGroups.slice(0, nodeLimit);
|
|
235
|
+
const viewNodes = groupsToRender.map(groupNode);
|
|
236
|
+
return {
|
|
237
|
+
signature,
|
|
238
|
+
mode: 'macro',
|
|
239
|
+
nodes: viewNodes,
|
|
240
|
+
edges: limitEdges(aggregateGroupEdges(groupsToRender, layout.edges, groupById)),
|
|
241
|
+
totals: {
|
|
242
|
+
nodes: layout.nodes.length,
|
|
243
|
+
edges: layout.edges.length
|
|
244
|
+
}
|
|
245
|
+
};
|
|
246
|
+
};
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
const stateVersion = 1;
|
|
4
|
+
const graphViewStatePath = (vaultPath) => join(vaultPath, '.brainlink', 'graph-view-state.json');
|
|
5
|
+
const stateKey = (input) => [input.signature, input.agentId ?? 'all-agents', input.context ?? 'all-contexts'].join(':');
|
|
6
|
+
const emptyPersistedState = () => ({
|
|
7
|
+
version: stateVersion,
|
|
8
|
+
states: {}
|
|
9
|
+
});
|
|
10
|
+
const readPersistedState = async (vaultPath) => {
|
|
11
|
+
try {
|
|
12
|
+
const parsed = JSON.parse(await readFile(graphViewStatePath(vaultPath), 'utf8'));
|
|
13
|
+
return parsed.version === stateVersion && parsed.states && typeof parsed.states === 'object' ? parsed : emptyPersistedState();
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return emptyPersistedState();
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
const writePersistedState = async (vaultPath, state) => {
|
|
20
|
+
const target = graphViewStatePath(vaultPath);
|
|
21
|
+
const temp = `${target}.tmp`;
|
|
22
|
+
await mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
|
23
|
+
await writeFile(temp, `${JSON.stringify(state)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
24
|
+
await rename(temp, target);
|
|
25
|
+
};
|
|
26
|
+
const normalizePositions = (positions) => positions.flatMap((position) => {
|
|
27
|
+
const id = typeof position.id === 'string' ? position.id.trim() : '';
|
|
28
|
+
const x = Number(position.x);
|
|
29
|
+
const y = Number(position.y);
|
|
30
|
+
return id && Number.isFinite(x) && Number.isFinite(y) ? [{ id, x, y }] : [];
|
|
31
|
+
});
|
|
32
|
+
export const getGraphViewState = async (vaultPath, input) => {
|
|
33
|
+
const persisted = await readPersistedState(vaultPath);
|
|
34
|
+
const state = persisted.states[stateKey(input)];
|
|
35
|
+
return state ?? {
|
|
36
|
+
...input,
|
|
37
|
+
positions: []
|
|
38
|
+
};
|
|
39
|
+
};
|
|
40
|
+
export const saveGraphViewState = async (vaultPath, input) => {
|
|
41
|
+
const persisted = await readPersistedState(vaultPath);
|
|
42
|
+
const nextState = {
|
|
43
|
+
...input,
|
|
44
|
+
positions: normalizePositions(input.positions)
|
|
45
|
+
};
|
|
46
|
+
await writePersistedState(vaultPath, {
|
|
47
|
+
version: stateVersion,
|
|
48
|
+
states: {
|
|
49
|
+
...persisted.states,
|
|
50
|
+
[stateKey(input)]: nextState
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
return nextState;
|
|
54
|
+
};
|
|
55
|
+
export const deleteGraphViewState = async (vaultPath, input) => {
|
|
56
|
+
const persisted = await readPersistedState(vaultPath);
|
|
57
|
+
const { [stateKey(input)]: _removed, ...states } = persisted.states;
|
|
58
|
+
await writePersistedState(vaultPath, {
|
|
59
|
+
version: stateVersion,
|
|
60
|
+
states
|
|
61
|
+
});
|
|
62
|
+
return {
|
|
63
|
+
...input,
|
|
64
|
+
positions: []
|
|
65
|
+
};
|
|
66
|
+
};
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { access } from 'node:fs/promises';
|
|
3
|
+
import { basename, extname, join, relative, resolve } from 'node:path';
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { promisify } from 'node:util';
|
|
6
|
+
import { extractTags } from '../domain/markdown.js';
|
|
7
|
+
import { sanitizeAgentId, sharedAgentId } from '../domain/agents.js';
|
|
8
|
+
import { ensureVault, listVaultFiles, writeMarkdownFile } from '../infrastructure/file-system-vault.js';
|
|
9
|
+
import { getBrainlinkHomePath } from '../infrastructure/paths.js';
|
|
10
|
+
const execFileAsync = promisify(execFile);
|
|
11
|
+
const fieldSeparator = '\u001f';
|
|
12
|
+
const rowSeparator = '\u001e';
|
|
13
|
+
const contentColumnCandidates = ['content', 'markdown', 'body', 'text', 'note'];
|
|
14
|
+
const titleColumnCandidates = ['title', 'note_title', 'name', 'headline'];
|
|
15
|
+
const pathColumnCandidates = ['path', 'file_path', 'filepath', 'source_path', 'source'];
|
|
16
|
+
const agentColumnCandidates = ['agent', 'agent_id', 'namespace', 'scope'];
|
|
17
|
+
const tagColumnCandidates = ['tags', 'tag_list', 'keywords'];
|
|
18
|
+
const createdColumnCandidates = ['created_at', 'createdat', 'created', 'ctime'];
|
|
19
|
+
const updatedColumnCandidates = ['updated_at', 'updatedat', 'updated', 'mtime'];
|
|
20
|
+
const slugify = (title) => title
|
|
21
|
+
.normalize('NFKD')
|
|
22
|
+
.replace(/[\u0300-\u036f]/g, '')
|
|
23
|
+
.toLowerCase()
|
|
24
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
25
|
+
.replace(/^-+|-+$/g, '');
|
|
26
|
+
const quoteIdentifier = (value) => `"${value.replaceAll('"', '""')}"`;
|
|
27
|
+
const pickColumn = (columns, candidates) => {
|
|
28
|
+
const byLower = new Map(columns.map((column) => [column.toLowerCase(), column]));
|
|
29
|
+
return candidates.map((candidate) => byLower.get(candidate)).find((column) => Boolean(column)) ?? null;
|
|
30
|
+
};
|
|
31
|
+
const parseDelimitedRows = (rawOutput) => {
|
|
32
|
+
const normalized = rawOutput.trim();
|
|
33
|
+
if (normalized.length === 0) {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return normalized
|
|
37
|
+
.split(rowSeparator)
|
|
38
|
+
.map((row) => row.trim())
|
|
39
|
+
.filter(Boolean)
|
|
40
|
+
.map((row) => row.split(fieldSeparator));
|
|
41
|
+
};
|
|
42
|
+
const runSqliteQuery = async (databasePath, sql) => {
|
|
43
|
+
const baseArgs = ['-noheader', '-separator', fieldSeparator, '-newline', rowSeparator, '-cmd', '.timeout 5000'];
|
|
44
|
+
const runQuery = async (args) => {
|
|
45
|
+
const { stdout } = await execFileAsync('sqlite3', [...args, sql], { maxBuffer: 1024 * 1024 * 64 });
|
|
46
|
+
return parseDelimitedRows(stdout);
|
|
47
|
+
};
|
|
48
|
+
try {
|
|
49
|
+
return await runQuery(['--readonly', ...baseArgs, databasePath]);
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
53
|
+
const lower = message.toLowerCase();
|
|
54
|
+
if (lower.includes('enoent') || lower.includes('not found')) {
|
|
55
|
+
throw new Error('sqlite3 CLI was not found. Install sqlite3 to use db-import.');
|
|
56
|
+
}
|
|
57
|
+
if (lower.includes('database is locked') || lower.includes('(5)')) {
|
|
58
|
+
try {
|
|
59
|
+
const uri = pathToFileURL(databasePath);
|
|
60
|
+
uri.search = 'mode=ro&immutable=1';
|
|
61
|
+
return await runQuery(['-uri', ...baseArgs, uri.toString()]);
|
|
62
|
+
}
|
|
63
|
+
catch (fallbackError) {
|
|
64
|
+
const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
|
|
65
|
+
throw new Error(`Unable to read SQLite database (locked). Close writers (server/watch/mcp) or rerun with DB idle. Details: ${fallbackMessage}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
throw new Error(`Unable to read SQLite database: ${message}`);
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
const detectLegacyDbPath = async (vaultPath, explicitPath) => {
|
|
72
|
+
if (explicitPath) {
|
|
73
|
+
return resolve(explicitPath);
|
|
74
|
+
}
|
|
75
|
+
const vaultRoot = await ensureVault(vaultPath);
|
|
76
|
+
const candidates = [
|
|
77
|
+
join(vaultRoot, '.brainlink', 'brainlink.db'),
|
|
78
|
+
join(vaultRoot, '.brainlink', 'index.db'),
|
|
79
|
+
join(getBrainlinkHomePath(), 'brainlink.db'),
|
|
80
|
+
join(getBrainlinkHomePath(), 'vault', '.brainlink', 'brainlink.db')
|
|
81
|
+
];
|
|
82
|
+
for (const candidate of candidates) {
|
|
83
|
+
try {
|
|
84
|
+
await access(candidate);
|
|
85
|
+
return candidate;
|
|
86
|
+
}
|
|
87
|
+
catch { }
|
|
88
|
+
}
|
|
89
|
+
throw new Error(`No legacy SQLite database found. Checked: ${candidates.join(', ')}. Use --db <path-to-db> to import explicitly.`);
|
|
90
|
+
};
|
|
91
|
+
const listTables = async (dbPath) => {
|
|
92
|
+
const rows = await runSqliteQuery(dbPath, `SELECT name
|
|
93
|
+
FROM sqlite_master
|
|
94
|
+
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
|
95
|
+
ORDER BY name`);
|
|
96
|
+
return rows.map((columns) => columns[0]).filter(Boolean);
|
|
97
|
+
};
|
|
98
|
+
const listColumns = async (dbPath, table) => {
|
|
99
|
+
const rows = await runSqliteQuery(dbPath, `PRAGMA table_info(${quoteIdentifier(table)})`);
|
|
100
|
+
return rows.map((columns) => columns[1]).filter(Boolean);
|
|
101
|
+
};
|
|
102
|
+
const tableScore = (columns) => {
|
|
103
|
+
const contentColumn = pickColumn(columns, contentColumnCandidates);
|
|
104
|
+
const titleColumn = pickColumn(columns, titleColumnCandidates);
|
|
105
|
+
const pathColumn = pickColumn(columns, pathColumnCandidates);
|
|
106
|
+
const agentColumn = pickColumn(columns, agentColumnCandidates);
|
|
107
|
+
return (contentColumn ? 6 : 0) + (titleColumn ? 4 : 0) + (pathColumn ? 2 : 0) + (agentColumn ? 1 : 0);
|
|
108
|
+
};
|
|
109
|
+
const detectTableMapping = async (dbPath, tableOverride) => {
|
|
110
|
+
const tables = await listTables(dbPath);
|
|
111
|
+
if (tables.length === 0) {
|
|
112
|
+
throw new Error('Legacy SQLite database has no readable tables.');
|
|
113
|
+
}
|
|
114
|
+
const mappings = await Promise.all(tables.map(async (table) => {
|
|
115
|
+
const columns = await listColumns(dbPath, table);
|
|
116
|
+
return {
|
|
117
|
+
table,
|
|
118
|
+
columns,
|
|
119
|
+
titleColumn: pickColumn(columns, titleColumnCandidates),
|
|
120
|
+
contentColumn: pickColumn(columns, contentColumnCandidates),
|
|
121
|
+
pathColumn: pickColumn(columns, pathColumnCandidates),
|
|
122
|
+
agentColumn: pickColumn(columns, agentColumnCandidates),
|
|
123
|
+
tagsColumn: pickColumn(columns, tagColumnCandidates),
|
|
124
|
+
createdColumn: pickColumn(columns, createdColumnCandidates),
|
|
125
|
+
updatedColumn: pickColumn(columns, updatedColumnCandidates),
|
|
126
|
+
score: tableScore(columns)
|
|
127
|
+
};
|
|
128
|
+
}));
|
|
129
|
+
if (tableOverride) {
|
|
130
|
+
const overridden = mappings.find((mapping) => mapping.table === tableOverride);
|
|
131
|
+
if (!overridden) {
|
|
132
|
+
throw new Error(`Table not found in SQLite database: ${tableOverride}`);
|
|
133
|
+
}
|
|
134
|
+
if (!overridden.contentColumn) {
|
|
135
|
+
throw new Error(`Table ${tableOverride} does not expose a readable content column.`);
|
|
136
|
+
}
|
|
137
|
+
return { mapping: overridden, detectedTables: tables };
|
|
138
|
+
}
|
|
139
|
+
const selected = [...mappings]
|
|
140
|
+
.filter((mapping) => mapping.contentColumn)
|
|
141
|
+
.sort((left, right) => right.score - left.score)[0];
|
|
142
|
+
if (!selected) {
|
|
143
|
+
throw new Error('Could not detect a legacy table with content column in SQLite database.');
|
|
144
|
+
}
|
|
145
|
+
return { mapping: selected, detectedTables: tables };
|
|
146
|
+
};
|
|
147
|
+
const hexExpression = (column) => column ? `hex(COALESCE(CAST(${quoteIdentifier(column)} AS BLOB), X''))` : `hex(X'')`;
|
|
148
|
+
const decodeHexUtf8 = (value) => value ? Buffer.from(value, 'hex').toString('utf8') : '';
|
|
149
|
+
const parseLegacyTags = (value) => Array.from(new Set(value
|
|
150
|
+
.split(/[\s,;|]+/)
|
|
151
|
+
.map((item) => item.trim().replace(/^#/, '').toLowerCase())
|
|
152
|
+
.filter((item) => /^[a-z0-9][a-z0-9_-]*$/i.test(item))));
|
|
153
|
+
const titleFromPath = (pathValue) => basename(pathValue).replace(extname(pathValue), '').replace(/[-_]+/g, ' ').trim();
|
|
154
|
+
const appendMissingTags = (content, tags) => {
|
|
155
|
+
if (tags.length === 0) {
|
|
156
|
+
return content;
|
|
157
|
+
}
|
|
158
|
+
const existingTags = new Set(extractTags(content).map((tag) => tag.toLowerCase()));
|
|
159
|
+
const missing = tags.filter((tag) => !existingTags.has(tag.toLowerCase()));
|
|
160
|
+
if (missing.length === 0) {
|
|
161
|
+
return content;
|
|
162
|
+
}
|
|
163
|
+
return `${content.trim()}\n\nTags: ${missing.map((tag) => `#${tag}`).join(' ')}`;
|
|
164
|
+
};
|
|
165
|
+
const buildNote = (title, content, agentId) => [
|
|
166
|
+
'---',
|
|
167
|
+
`title: "${title.replaceAll('"', '\\"')}"`,
|
|
168
|
+
`agent: "${agentId}"`,
|
|
169
|
+
'---',
|
|
170
|
+
'',
|
|
171
|
+
`# ${title}`,
|
|
172
|
+
'',
|
|
173
|
+
content.trim(),
|
|
174
|
+
''
|
|
175
|
+
].join('\n');
|
|
176
|
+
const parseLegacyRow = (columns, rowIndex) => {
|
|
177
|
+
const [titleHex, contentHex, pathHex, agentHex, tagsHex] = columns;
|
|
178
|
+
const content = decodeHexUtf8(contentHex).trim();
|
|
179
|
+
const path = decodeHexUtf8(pathHex).trim();
|
|
180
|
+
const titleCandidate = decodeHexUtf8(titleHex).trim();
|
|
181
|
+
const fallbackTitleFromPath = path ? titleFromPath(path) : '';
|
|
182
|
+
const title = titleCandidate || fallbackTitleFromPath || `Imported Memory ${rowIndex + 1}`;
|
|
183
|
+
return {
|
|
184
|
+
title,
|
|
185
|
+
content,
|
|
186
|
+
path,
|
|
187
|
+
agent: decodeHexUtf8(agentHex).trim(),
|
|
188
|
+
tags: parseLegacyTags(decodeHexUtf8(tagsHex))
|
|
189
|
+
};
|
|
190
|
+
};
|
|
191
|
+
const noteRelativePath = (agentId, slug, suffix = 0) => `agents/${agentId}/${suffix > 0 ? `${slug}-${suffix + 1}` : slug || 'untitled'}.md`;
|
|
192
|
+
const reserveUniquePath = (agentId, title, reserved) => {
|
|
193
|
+
const slug = slugify(title);
|
|
194
|
+
for (let suffix = 0; suffix < 10_000; suffix += 1) {
|
|
195
|
+
const relativePath = noteRelativePath(agentId, slug, suffix);
|
|
196
|
+
if (!reserved.has(relativePath)) {
|
|
197
|
+
reserved.add(relativePath);
|
|
198
|
+
return relativePath;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
throw new Error(`Could not allocate unique path for imported note: ${title}`);
|
|
202
|
+
};
|
|
203
|
+
const importRowsFromMapping = async (vaultPath, dbPath, mapping, options, reserved) => {
|
|
204
|
+
const limit = Number.isFinite(options.limit) && (options.limit ?? 0) > 0 ? Math.floor(options.limit ?? 0) : undefined;
|
|
205
|
+
const sql = [
|
|
206
|
+
'SELECT',
|
|
207
|
+
`${hexExpression(mapping.titleColumn)} AS title_hex,`,
|
|
208
|
+
`${hexExpression(mapping.contentColumn)} AS content_hex,`,
|
|
209
|
+
`${hexExpression(mapping.pathColumn)} AS path_hex,`,
|
|
210
|
+
`${hexExpression(mapping.agentColumn)} AS agent_hex,`,
|
|
211
|
+
`${hexExpression(mapping.tagsColumn)} AS tags_hex,`,
|
|
212
|
+
`${hexExpression(mapping.createdColumn)} AS created_hex,`,
|
|
213
|
+
`${hexExpression(mapping.updatedColumn)} AS updated_hex`,
|
|
214
|
+
`FROM ${quoteIdentifier(mapping.table)}`,
|
|
215
|
+
...(limit ? [`LIMIT ${limit}`] : [])
|
|
216
|
+
].join(' ');
|
|
217
|
+
const rows = await runSqliteQuery(dbPath, sql);
|
|
218
|
+
const importedFiles = [];
|
|
219
|
+
let imported = 0;
|
|
220
|
+
let skipped = 0;
|
|
221
|
+
for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) {
|
|
222
|
+
const row = parseLegacyRow(rows[rowIndex], rowIndex);
|
|
223
|
+
if (!row.content) {
|
|
224
|
+
skipped += 1;
|
|
225
|
+
continue;
|
|
226
|
+
}
|
|
227
|
+
const agentId = sanitizeAgentId(options.agentOverride || row.agent || sharedAgentId);
|
|
228
|
+
const filename = reserveUniquePath(agentId, row.title, reserved);
|
|
229
|
+
const mergedContent = appendMissingTags(row.content, row.tags);
|
|
230
|
+
const note = buildNote(row.title, mergedContent.trim(), agentId);
|
|
231
|
+
if (options.dryRun !== true) {
|
|
232
|
+
await writeMarkdownFile(vaultPath, filename, note);
|
|
233
|
+
}
|
|
234
|
+
importedFiles.push(filename);
|
|
235
|
+
imported += 1;
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
rowsRead: rows.length,
|
|
239
|
+
imported,
|
|
240
|
+
skipped,
|
|
241
|
+
createdSystemNotes: 0,
|
|
242
|
+
importedFiles
|
|
243
|
+
};
|
|
244
|
+
};
|
|
245
|
+
export const importLegacySqliteDatabase = async (vaultPath, options = {}) => {
|
|
246
|
+
const vault = await ensureVault(vaultPath);
|
|
247
|
+
const dbPath = await detectLegacyDbPath(vaultPath, options.dbPath);
|
|
248
|
+
const { mapping, detectedTables } = await detectTableMapping(dbPath, options.table);
|
|
249
|
+
const existingFiles = (await listVaultFiles(vaultPath))
|
|
250
|
+
.filter((path) => extname(path).toLowerCase() === '.md')
|
|
251
|
+
.map((path) => relative(vault, path));
|
|
252
|
+
const reserved = new Set(existingFiles);
|
|
253
|
+
const imported = await importRowsFromMapping(vaultPath, dbPath, mapping, options, reserved);
|
|
254
|
+
return {
|
|
255
|
+
vault,
|
|
256
|
+
dbPath,
|
|
257
|
+
table: mapping.table,
|
|
258
|
+
detectedTables,
|
|
259
|
+
rowsRead: imported.rowsRead,
|
|
260
|
+
imported: imported.imported,
|
|
261
|
+
skipped: imported.skipped,
|
|
262
|
+
createdSystemNotes: imported.createdSystemNotes,
|
|
263
|
+
dryRun: options.dryRun === true,
|
|
264
|
+
importedFiles: imported.importedFiles
|
|
265
|
+
};
|
|
266
|
+
};
|