@andespindola/brainlink 0.1.0-beta.41 → 0.1.0-beta.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/CHANGELOG.md +6 -0
- package/README.md +26 -0
- package/dist/application/frontend/client-js.js +484 -47
- package/dist/application/frontend/client-worker-js.js +66 -0
- package/dist/application/index-vault.js +110 -4
- package/dist/application/server/routes.js +36 -1
- package/dist/application/start-server.js +75 -4
- package/dist/application/watch-vault.js +23 -2
- package/dist/cli/commands/write-commands.js +140 -13
- package/dist/infrastructure/search-packs.js +16 -1
- package/docs/AGENT_USAGE.md +18 -0
- package/package.json +1 -1
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export const createClientWorkerJs = () => `const normalize = value => String(value || '')
|
|
2
|
+
.normalize('NFKD')
|
|
3
|
+
.replace(/\\p{Diacritic}/gu, '')
|
|
4
|
+
.toLowerCase()
|
|
5
|
+
|
|
6
|
+
let nodeIndex = []
|
|
7
|
+
|
|
8
|
+
const toNodeIndex = nodes =>
|
|
9
|
+
(Array.isArray(nodes) ? nodes : [])
|
|
10
|
+
.map(node => {
|
|
11
|
+
const id = typeof node.id === 'string' ? node.id : ''
|
|
12
|
+
if (!id) {
|
|
13
|
+
return null
|
|
14
|
+
}
|
|
15
|
+
const title = normalize(node.title)
|
|
16
|
+
const path = normalize(node.path)
|
|
17
|
+
const tags = Array.isArray(node.tags) ? node.tags.map(tag => normalize(tag)) : []
|
|
18
|
+
return {
|
|
19
|
+
id,
|
|
20
|
+
text: [title, path, ...tags].join(' ')
|
|
21
|
+
}
|
|
22
|
+
})
|
|
23
|
+
.filter(Boolean)
|
|
24
|
+
|
|
25
|
+
const scoreText = (text, query) => {
|
|
26
|
+
if (!query) return 0
|
|
27
|
+
if (!text.includes(query)) return 0
|
|
28
|
+
if (text.startsWith(query)) return 4
|
|
29
|
+
return 1
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const filterIds = (query, limit) => {
|
|
33
|
+
const normalizedQuery = normalize(query).trim()
|
|
34
|
+
if (!normalizedQuery) {
|
|
35
|
+
return []
|
|
36
|
+
}
|
|
37
|
+
const rows = []
|
|
38
|
+
for (let index = 0; index < nodeIndex.length; index += 1) {
|
|
39
|
+
const row = nodeIndex[index]
|
|
40
|
+
const score = scoreText(row.text, normalizedQuery)
|
|
41
|
+
if (score > 0) {
|
|
42
|
+
rows.push({ id: row.id, score })
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
rows.sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
|
|
46
|
+
return rows.slice(0, Math.max(1, Number.isFinite(limit) ? limit : rows.length)).map(row => row.id)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
self.onmessage = event => {
|
|
50
|
+
const payload = event.data
|
|
51
|
+
if (!payload || typeof payload !== 'object') {
|
|
52
|
+
return
|
|
53
|
+
}
|
|
54
|
+
if (payload.type === 'load-nodes') {
|
|
55
|
+
nodeIndex = toNodeIndex(payload.nodes)
|
|
56
|
+
return
|
|
57
|
+
}
|
|
58
|
+
if (payload.type === 'filter') {
|
|
59
|
+
const token = payload.token
|
|
60
|
+
const ids = filterIds(payload.query, payload.limit)
|
|
61
|
+
self.postMessage({ type: 'filter-result', token, ids })
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
self.postMessage({ type: 'ready' })
|
|
66
|
+
`;
|
|
@@ -97,12 +97,33 @@ const readChangedDocuments = async (absoluteVaultPath, changedSummaries) => {
|
|
|
97
97
|
return new Map(parsed.map((document) => [document.path, document]));
|
|
98
98
|
};
|
|
99
99
|
export const indexVault = async (vaultPath) => {
|
|
100
|
+
return indexVaultWithOptions(vaultPath, {});
|
|
101
|
+
};
|
|
102
|
+
export const indexVaultWithOptions = async (vaultPath, options) => {
|
|
103
|
+
const startedAt = process.hrtime.bigint();
|
|
104
|
+
const elapsedMs = () => Number(process.hrtime.bigint() - startedAt) / 1_000_000;
|
|
105
|
+
const emit = (phase, status, message, details) => {
|
|
106
|
+
options.onProgress?.({
|
|
107
|
+
phase,
|
|
108
|
+
status,
|
|
109
|
+
message,
|
|
110
|
+
elapsedMs: elapsedMs(),
|
|
111
|
+
timestamp: new Date().toISOString(),
|
|
112
|
+
details
|
|
113
|
+
});
|
|
114
|
+
};
|
|
115
|
+
emit('start', 'start', 'Indexing started');
|
|
100
116
|
const absoluteVaultPath = await ensureVault(vaultPath);
|
|
101
117
|
const config = await loadBrainlinkConfig();
|
|
118
|
+
emit('scan', 'start', 'Scanning markdown files');
|
|
102
119
|
const [summaries, previousState] = await Promise.all([
|
|
103
120
|
readMarkdownFileSummaries(absoluteVaultPath),
|
|
104
121
|
readIndexState(absoluteVaultPath)
|
|
105
122
|
]);
|
|
123
|
+
emit('scan', 'finish', 'Scan complete', {
|
|
124
|
+
markdownFiles: summaries.length,
|
|
125
|
+
hasPreviousState: previousState != null
|
|
126
|
+
});
|
|
106
127
|
const index = openFileIndex(absoluteVaultPath);
|
|
107
128
|
try {
|
|
108
129
|
const existingIndexedDocuments = await index.getIndexedDocuments();
|
|
@@ -133,10 +154,28 @@ export const indexVault = async (vaultPath) => {
|
|
|
133
154
|
!hasDeletes &&
|
|
134
155
|
existingIndexedDocuments.length === summaries.length &&
|
|
135
156
|
previousState != null) {
|
|
136
|
-
|
|
157
|
+
const result = {
|
|
158
|
+
...toIndexResult(existingIndexedDocuments),
|
|
159
|
+
elapsedMs: elapsedMs(),
|
|
160
|
+
changedDocumentCount: 0,
|
|
161
|
+
packs: {
|
|
162
|
+
rebuilt: false,
|
|
163
|
+
reason: 'No changes detected'
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
emit('complete', 'skip', 'Index skipped: no changes detected', {
|
|
167
|
+
elapsedMs: result.elapsedMs
|
|
168
|
+
});
|
|
169
|
+
return result;
|
|
137
170
|
}
|
|
138
171
|
const changedSummaries = summaries.filter((summary) => changedPaths.has(summary.relativePath));
|
|
172
|
+
emit('parse', 'start', 'Parsing changed markdown files', {
|
|
173
|
+
changedFiles: changedSummaries.length
|
|
174
|
+
});
|
|
139
175
|
const changedDocumentsByPath = await readChangedDocuments(absoluteVaultPath, changedSummaries);
|
|
176
|
+
emit('parse', 'finish', 'Parse complete', {
|
|
177
|
+
changedDocuments: changedDocumentsByPath.size
|
|
178
|
+
});
|
|
140
179
|
const documents = summaries.flatMap((summary) => {
|
|
141
180
|
const changed = changedDocumentsByPath.get(summary.relativePath);
|
|
142
181
|
if (changed) {
|
|
@@ -146,9 +185,15 @@ export const indexVault = async (vaultPath) => {
|
|
|
146
185
|
return existing ? [existing.document] : [];
|
|
147
186
|
});
|
|
148
187
|
const titleMaps = createTitleMaps(documents);
|
|
188
|
+
emit('embed', 'start', 'Embedding changed chunks', {
|
|
189
|
+
changedDocuments: changedDocumentsByPath.size
|
|
190
|
+
});
|
|
149
191
|
const changedIndexedDocuments = changedDocumentsByPath.size > 0
|
|
150
192
|
? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
|
|
151
193
|
: [];
|
|
194
|
+
emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
|
|
195
|
+
changedIndexedDocuments: changedIndexedDocuments.length
|
|
196
|
+
});
|
|
152
197
|
const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
|
|
153
198
|
const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
|
|
154
199
|
const indexedDocuments = documents.map((document) => {
|
|
@@ -162,8 +207,12 @@ export const indexVault = async (vaultPath) => {
|
|
|
162
207
|
}
|
|
163
208
|
return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
|
|
164
209
|
});
|
|
210
|
+
emit('persist', 'start', 'Persisting index');
|
|
165
211
|
await index.reset();
|
|
166
212
|
await index.saveDocuments(indexedDocuments);
|
|
213
|
+
emit('persist', 'finish', 'Index persisted', {
|
|
214
|
+
indexedDocuments: indexedDocuments.length
|
|
215
|
+
});
|
|
167
216
|
const existingPackManifest = await hasPackManifest(absoluteVaultPath);
|
|
168
217
|
const changedCount = changedPaths.size;
|
|
169
218
|
const documentCount = Math.max(indexedDocuments.length, 1);
|
|
@@ -176,21 +225,78 @@ export const indexVault = async (vaultPath) => {
|
|
|
176
225
|
changedCount >= 400 ||
|
|
177
226
|
changeRatio >= 0.04 ||
|
|
178
227
|
pendingPackChanges >= 1200;
|
|
228
|
+
let packResult;
|
|
229
|
+
const packReason = !existingPackManifest
|
|
230
|
+
? 'Missing pack manifest'
|
|
231
|
+
: settingsChanged
|
|
232
|
+
? 'Index settings changed'
|
|
233
|
+
: hasDeletes
|
|
234
|
+
? 'Document deletions detected'
|
|
235
|
+
: changedCount >= 400
|
|
236
|
+
? 'Changed file count threshold reached'
|
|
237
|
+
: changeRatio >= 0.04
|
|
238
|
+
? 'Change ratio threshold reached'
|
|
239
|
+
: pendingPackChanges >= 1200
|
|
240
|
+
? 'Pending pack changes threshold reached'
|
|
241
|
+
: 'Pack rebuild skipped';
|
|
179
242
|
if (shouldRebuildPacks) {
|
|
243
|
+
emit('packs', 'start', 'Rebuilding compressed search packs', {
|
|
244
|
+
reason: packReason
|
|
245
|
+
});
|
|
180
246
|
try {
|
|
181
|
-
await buildSearchPacks(absoluteVaultPath, indexedDocuments);
|
|
247
|
+
packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments);
|
|
248
|
+
emit('packs', 'finish', 'Compressed packs rebuilt', {
|
|
249
|
+
reason: packReason,
|
|
250
|
+
packCount: packResult.packCount,
|
|
251
|
+
recordCount: packResult.recordCount,
|
|
252
|
+
durationMs: packResult.durationMs,
|
|
253
|
+
compressionRatio: packResult.compression.ratio
|
|
254
|
+
});
|
|
182
255
|
}
|
|
183
256
|
catch {
|
|
184
257
|
// Pack generation is best-effort. The JSON index remains the primary path.
|
|
258
|
+
emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
|
|
259
|
+
reason: packReason
|
|
260
|
+
});
|
|
185
261
|
}
|
|
186
262
|
}
|
|
263
|
+
else {
|
|
264
|
+
emit('packs', 'skip', 'Pack rebuild not required', {
|
|
265
|
+
reason: packReason
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
const packsRebuilt = packResult != null;
|
|
269
|
+
const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
|
|
187
270
|
await writeIndexState(absoluteVaultPath, {
|
|
188
271
|
chunkSize: config.chunkSize,
|
|
189
272
|
embeddingProvider: config.embeddingProvider,
|
|
190
273
|
files: currentSnapshot,
|
|
191
|
-
pendingPackChanges:
|
|
274
|
+
pendingPackChanges: packsRebuilt ? 0 : pendingPackChanges
|
|
275
|
+
});
|
|
276
|
+
const result = {
|
|
277
|
+
...toIndexResult(indexedDocuments),
|
|
278
|
+
elapsedMs: elapsedMs(),
|
|
279
|
+
changedDocumentCount: changedDocumentsByPath.size,
|
|
280
|
+
packs: {
|
|
281
|
+
rebuilt: packsRebuilt,
|
|
282
|
+
reason: packResultReason,
|
|
283
|
+
...(packResult
|
|
284
|
+
? {
|
|
285
|
+
packCount: packResult.packCount,
|
|
286
|
+
recordCount: packResult.recordCount,
|
|
287
|
+
durationMs: packResult.durationMs,
|
|
288
|
+
compression: packResult.compression
|
|
289
|
+
}
|
|
290
|
+
: {})
|
|
291
|
+
}
|
|
292
|
+
};
|
|
293
|
+
emit('complete', 'finish', 'Indexing complete', {
|
|
294
|
+
documentCount: result.documentCount,
|
|
295
|
+
chunkCount: result.chunkCount,
|
|
296
|
+
linkCount: result.linkCount,
|
|
297
|
+
elapsedMs: result.elapsedMs
|
|
192
298
|
});
|
|
193
|
-
return
|
|
299
|
+
return result;
|
|
194
300
|
}
|
|
195
301
|
finally {
|
|
196
302
|
index.close();
|
|
@@ -11,6 +11,7 @@ import { loadBrainlinkConfig, sanitizeSearchMode } from '../../infrastructure/co
|
|
|
11
11
|
import { createClientCss } from '../frontend/client-css.js';
|
|
12
12
|
import { createClientHtml } from '../frontend/client-html.js';
|
|
13
13
|
import { createClientJs } from '../frontend/client-js.js';
|
|
14
|
+
import { createClientWorkerJs } from '../frontend/client-worker-js.js';
|
|
14
15
|
import { contentTypes, createJsonResponse, isReadMethod, parsePositiveInteger } from './http.js';
|
|
15
16
|
const readSearchMode = async (url) => {
|
|
16
17
|
const config = await loadBrainlinkConfig();
|
|
@@ -51,10 +52,41 @@ const sameEntityTag = (candidate, signature) => {
|
|
|
51
52
|
return decodeEntityTag(candidate) === signature;
|
|
52
53
|
};
|
|
53
54
|
const readAgentQuery = (url) => url.searchParams.get('agent') ?? undefined;
|
|
55
|
+
const compactGraphLayoutThreshold = 12_000;
|
|
56
|
+
const compactGraphLayoutEdgeLimit = 60_000;
|
|
57
|
+
const compactGraphLayoutEdgeLimitFor = (nodeCount) => {
|
|
58
|
+
if (nodeCount > 100_000)
|
|
59
|
+
return 15_000;
|
|
60
|
+
if (nodeCount > 50_000)
|
|
61
|
+
return 22_000;
|
|
62
|
+
if (nodeCount > 25_000)
|
|
63
|
+
return 30_000;
|
|
64
|
+
return compactGraphLayoutEdgeLimit;
|
|
65
|
+
};
|
|
54
66
|
const stripLayoutContent = (layout) => ({
|
|
55
67
|
...layout,
|
|
56
68
|
nodes: layout.nodes.map(({ content, ...node }) => node)
|
|
57
69
|
});
|
|
70
|
+
const compactLayoutPayload = (layout) => {
|
|
71
|
+
const edgeLimit = compactGraphLayoutEdgeLimitFor(layout.nodes.length);
|
|
72
|
+
const compactNodes = layout.nodes.map((node) => [node.id, node.title, node.x, node.y, node.group, node.segment]);
|
|
73
|
+
const compactEdges = [...layout.edges]
|
|
74
|
+
.sort((left, right) => (right.weight ?? 1) - (left.weight ?? 1))
|
|
75
|
+
.slice(0, edgeLimit)
|
|
76
|
+
.map((edge) => [edge.source, edge.target, edge.weight, edge.priority]);
|
|
77
|
+
return {
|
|
78
|
+
compact: true,
|
|
79
|
+
layout: {
|
|
80
|
+
nodes: compactNodes,
|
|
81
|
+
edges: compactEdges
|
|
82
|
+
},
|
|
83
|
+
totals: {
|
|
84
|
+
nodes: layout.nodes.length,
|
|
85
|
+
edges: layout.edges.length
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
};
|
|
89
|
+
const encodeLayoutPayload = (layout) => layout.nodes.length > compactGraphLayoutThreshold ? compactLayoutPayload(layout) : { compact: false, layout: stripLayoutContent(layout) };
|
|
58
90
|
export const route = async (request, url, vaultPath) => {
|
|
59
91
|
if (isReadMethod(request) && (url.pathname === '/' || url.pathname === '/index.html')) {
|
|
60
92
|
return createResponse(createClientHtml(), 200, contentTypes['.html']);
|
|
@@ -65,6 +97,9 @@ export const route = async (request, url, vaultPath) => {
|
|
|
65
97
|
if (isReadMethod(request) && url.pathname === '/app.js') {
|
|
66
98
|
return createResponse(createClientJs(), 200, contentTypes['.js']);
|
|
67
99
|
}
|
|
100
|
+
if (isReadMethod(request) && url.pathname === '/app-worker.js') {
|
|
101
|
+
return createResponse(createClientWorkerJs(), 200, contentTypes['.js']);
|
|
102
|
+
}
|
|
68
103
|
if (isReadMethod(request) && url.pathname === '/api/graph') {
|
|
69
104
|
return createResponse(createJsonResponse(await getGraph(vaultPath, readAgentQuery(url))), 200, contentTypes['.json']);
|
|
70
105
|
}
|
|
@@ -73,7 +108,7 @@ export const route = async (request, url, vaultPath) => {
|
|
|
73
108
|
const requestEtags = request.headers['if-none-match'];
|
|
74
109
|
const notModified = sameEntityTag(requestEtags, signature);
|
|
75
110
|
const etag = encodeEntityTag(signature);
|
|
76
|
-
const body = createJsonResponse({ signature,
|
|
111
|
+
const body = createJsonResponse({ signature, ...encodeLayoutPayload(layout) });
|
|
77
112
|
const jsonResponse = createResponse(body, 200, contentTypes['.json']);
|
|
78
113
|
const notModifiedResponse = createResponse('', 304, contentTypes['.json']);
|
|
79
114
|
if (notModified) {
|
|
@@ -1,9 +1,78 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
|
+
import { brotliCompressSync, constants, gzipSync } from 'node:zlib';
|
|
2
3
|
import { indexVault } from './index-vault.js';
|
|
3
4
|
import { startVaultWatcher } from './watch-vault.js';
|
|
4
5
|
import { assertLoopbackHost } from './server/host-security.js';
|
|
5
6
|
import { contentTypes, createJsonResponse, isHttpError } from './server/http.js';
|
|
6
7
|
import { route } from './server/routes.js';
|
|
8
|
+
const compressionThresholdBytes = 1024;
|
|
9
|
+
const normalizeEncodingToken = (value) => value.trim().toLowerCase();
|
|
10
|
+
const supportsEncoding = (acceptEncoding, target) => {
|
|
11
|
+
if (!acceptEncoding) {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
return acceptEncoding
|
|
15
|
+
.split(',')
|
|
16
|
+
.map((entry) => entry.split(';')[0] ?? '')
|
|
17
|
+
.map(normalizeEncodingToken)
|
|
18
|
+
.includes(target);
|
|
19
|
+
};
|
|
20
|
+
const isCompressibleContentType = (contentType) => {
|
|
21
|
+
const normalized = contentType?.toLowerCase() ?? '';
|
|
22
|
+
return (normalized.includes('application/json') ||
|
|
23
|
+
normalized.includes('text/javascript') ||
|
|
24
|
+
normalized.includes('text/css') ||
|
|
25
|
+
normalized.includes('text/html') ||
|
|
26
|
+
normalized.startsWith('text/'));
|
|
27
|
+
};
|
|
28
|
+
const maybeCompressResponse = (requestHeaders, statusCode, headers, body) => {
|
|
29
|
+
if (statusCode === 204 || statusCode === 304) {
|
|
30
|
+
return { headers, body: '' };
|
|
31
|
+
}
|
|
32
|
+
if (!isCompressibleContentType(headers['content-type'])) {
|
|
33
|
+
return { headers, body };
|
|
34
|
+
}
|
|
35
|
+
const bodyBuffer = Buffer.from(body, 'utf8');
|
|
36
|
+
if (bodyBuffer.byteLength < compressionThresholdBytes) {
|
|
37
|
+
return { headers, body };
|
|
38
|
+
}
|
|
39
|
+
if (headers['content-encoding']) {
|
|
40
|
+
return { headers, body };
|
|
41
|
+
}
|
|
42
|
+
const acceptEncodingHeader = Array.isArray(requestHeaders['accept-encoding'])
|
|
43
|
+
? requestHeaders['accept-encoding'].join(',')
|
|
44
|
+
: requestHeaders['accept-encoding'];
|
|
45
|
+
const vary = headers.vary ? `${headers.vary}, Accept-Encoding` : 'Accept-Encoding';
|
|
46
|
+
const withVary = {
|
|
47
|
+
...headers,
|
|
48
|
+
vary
|
|
49
|
+
};
|
|
50
|
+
if (supportsEncoding(acceptEncodingHeader, 'br')) {
|
|
51
|
+
return {
|
|
52
|
+
headers: {
|
|
53
|
+
...withVary,
|
|
54
|
+
'content-encoding': 'br'
|
|
55
|
+
},
|
|
56
|
+
body: brotliCompressSync(bodyBuffer, {
|
|
57
|
+
params: {
|
|
58
|
+
[constants.BROTLI_PARAM_QUALITY]: 5
|
|
59
|
+
}
|
|
60
|
+
})
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (supportsEncoding(acceptEncodingHeader, 'gzip')) {
|
|
64
|
+
return {
|
|
65
|
+
headers: {
|
|
66
|
+
...withVary,
|
|
67
|
+
'content-encoding': 'gzip'
|
|
68
|
+
},
|
|
69
|
+
body: gzipSync(bodyBuffer, {
|
|
70
|
+
level: 6
|
|
71
|
+
})
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return { headers: withVary, body };
|
|
75
|
+
};
|
|
7
76
|
export const startServer = async (input) => {
|
|
8
77
|
assertLoopbackHost(input.host);
|
|
9
78
|
if (input.shouldIndex) {
|
|
@@ -19,14 +88,16 @@ export const startServer = async (input) => {
|
|
|
19
88
|
const url = new URL(request.url ?? '/', `http://${request.headers.host ?? input.host}`);
|
|
20
89
|
route(request, url, input.vaultPath)
|
|
21
90
|
.then((result) => {
|
|
22
|
-
|
|
23
|
-
response.
|
|
91
|
+
const encoded = maybeCompressResponse(request.headers, result.statusCode, result.headers, result.body);
|
|
92
|
+
response.writeHead(result.statusCode, encoded.headers);
|
|
93
|
+
response.end(encoded.body);
|
|
24
94
|
})
|
|
25
95
|
.catch((error) => {
|
|
26
96
|
const message = error instanceof Error ? error.message : String(error);
|
|
27
97
|
const statusCode = isHttpError(error) ? error.statusCode : 500;
|
|
28
|
-
|
|
29
|
-
response.
|
|
98
|
+
const fallback = maybeCompressResponse(request.headers, statusCode, { 'content-type': contentTypes['.json'] }, createJsonResponse({ error: message }));
|
|
99
|
+
response.writeHead(statusCode, fallback.headers);
|
|
100
|
+
response.end(fallback.body);
|
|
30
101
|
});
|
|
31
102
|
});
|
|
32
103
|
await new Promise((resolve, reject) => {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { watch } from 'node:fs';
|
|
2
|
-
import {
|
|
2
|
+
import { indexVaultWithOptions } from './index-vault.js';
|
|
3
3
|
import { isBucketVaultPath, resolveVaultPath } from '../infrastructure/file-system-vault.js';
|
|
4
4
|
const shouldIgnore = (filename) => {
|
|
5
5
|
if (!filename) {
|
|
@@ -14,6 +14,27 @@ export const startVaultWatcher = (input) => {
|
|
|
14
14
|
const absoluteVaultPath = resolveVaultPath(input.vaultPath);
|
|
15
15
|
const debounceMs = input.debounceMs ?? 350;
|
|
16
16
|
let timeout = null;
|
|
17
|
+
let running = false;
|
|
18
|
+
let pending = false;
|
|
19
|
+
const runIndex = () => {
|
|
20
|
+
if (running) {
|
|
21
|
+
pending = true;
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
running = true;
|
|
25
|
+
indexVaultWithOptions(absoluteVaultPath, {
|
|
26
|
+
onProgress: input.onProgress
|
|
27
|
+
})
|
|
28
|
+
.then(input.onIndex)
|
|
29
|
+
.catch(input.onError)
|
|
30
|
+
.finally(() => {
|
|
31
|
+
running = false;
|
|
32
|
+
if (pending) {
|
|
33
|
+
pending = false;
|
|
34
|
+
runIndex();
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
};
|
|
17
38
|
const schedule = (filename) => {
|
|
18
39
|
if (shouldIgnore(filename)) {
|
|
19
40
|
return;
|
|
@@ -22,7 +43,7 @@ export const startVaultWatcher = (input) => {
|
|
|
22
43
|
clearTimeout(timeout);
|
|
23
44
|
}
|
|
24
45
|
timeout = setTimeout(() => {
|
|
25
|
-
|
|
46
|
+
runIndex();
|
|
26
47
|
}, debounceMs);
|
|
27
48
|
};
|
|
28
49
|
const watcher = watch(absoluteVaultPath, { recursive: true }, (_eventType, filename) => {
|
|
@@ -7,7 +7,7 @@ import { addNoteWithMetadata } from '../../application/add-note.js';
|
|
|
7
7
|
import { buildContextPackage } from '../../application/build-context.js';
|
|
8
8
|
import { resolveDuplicateNotes, scanDuplicateNotes } from '../../application/dedupe-notes.js';
|
|
9
9
|
import { importLegacySqliteDatabase } from '../../application/import-legacy-sqlite.js';
|
|
10
|
-
import { indexVault } from '../../application/index-vault.js';
|
|
10
|
+
import { indexVault, indexVaultWithOptions } from '../../application/index-vault.js';
|
|
11
11
|
import { migrateVaultContent, planVaultMigration, previewVaultMigration, shouldMigrateDefaultVault } from '../../application/migrate-vault.js';
|
|
12
12
|
import { startServer } from '../../application/start-server.js';
|
|
13
13
|
import { startVaultWatcher } from '../../application/watch-vault.js';
|
|
@@ -37,9 +37,59 @@ const parseScore = (value, fallback) => {
|
|
|
37
37
|
}
|
|
38
38
|
return parsed;
|
|
39
39
|
};
|
|
40
|
-
const
|
|
40
|
+
const formatBytes = (bytes) => {
|
|
41
|
+
if (!Number.isFinite(bytes) || bytes == null) {
|
|
42
|
+
return 'n/a';
|
|
43
|
+
}
|
|
44
|
+
if (bytes < 1024)
|
|
45
|
+
return `${bytes} B`;
|
|
46
|
+
const units = ['KB', 'MB', 'GB', 'TB'];
|
|
47
|
+
let value = bytes / 1024;
|
|
48
|
+
let unitIndex = 0;
|
|
49
|
+
while (value >= 1024 && unitIndex < units.length - 1) {
|
|
50
|
+
value /= 1024;
|
|
51
|
+
unitIndex += 1;
|
|
52
|
+
}
|
|
53
|
+
return `${value.toFixed(value >= 10 ? 1 : 2)} ${units[unitIndex]}`;
|
|
54
|
+
};
|
|
55
|
+
const formatMs = (value) => Number.isFinite(value) && value != null ? `${value.toFixed(value >= 100 ? 0 : 1)}ms` : 'n/a';
|
|
56
|
+
const benchEventLabel = (event) => `${event.phase}:${event.status}`;
|
|
57
|
+
const printBenchRealtimeEvent = (json, event) => {
|
|
58
|
+
print(json, {
|
|
59
|
+
event: 'bench-progress',
|
|
60
|
+
...event
|
|
61
|
+
}, () => `[bench] ${benchEventLabel(event)} ${event.message} (${formatMs(event.elapsedMs)})`);
|
|
62
|
+
};
|
|
63
|
+
const printBenchSummary = (json, trigger, vault, result) => {
|
|
64
|
+
print(json, {
|
|
65
|
+
event: 'bench-result',
|
|
66
|
+
trigger,
|
|
67
|
+
vault,
|
|
68
|
+
result
|
|
69
|
+
}, () => {
|
|
70
|
+
const packs = result.packs;
|
|
71
|
+
const compression = packs?.compression;
|
|
72
|
+
const savedPercent = compression && compression.inputBytes > 0
|
|
73
|
+
? `${((1 - compression.ratio) * 100).toFixed(1)}%`
|
|
74
|
+
: 'n/a';
|
|
75
|
+
return [
|
|
76
|
+
`[bench] trigger=${trigger}`,
|
|
77
|
+
`documents=${result.documentCount} chunks=${result.chunkCount} links=${result.linkCount}`,
|
|
78
|
+
`changedDocuments=${result.changedDocumentCount ?? 0} totalElapsed=${formatMs(result.elapsedMs)}`,
|
|
79
|
+
`packsRebuilt=${packs?.rebuilt ? 'yes' : 'no'} reason=${packs?.reason ?? 'n/a'}`,
|
|
80
|
+
packs?.rebuilt
|
|
81
|
+
? `packCount=${packs.packCount ?? 0} packDuration=${formatMs(packs.durationMs)} input=${formatBytes(compression?.inputBytes)} output=${formatBytes(compression?.outputBytes)} saved=${savedPercent}`
|
|
82
|
+
: 'packCompression=n/a'
|
|
83
|
+
].join('\n');
|
|
84
|
+
});
|
|
85
|
+
};
|
|
86
|
+
const spawnDetached = (command, args, envOverrides) => {
|
|
41
87
|
try {
|
|
42
|
-
const child = spawn(command, args, {
|
|
88
|
+
const child = spawn(command, args, {
|
|
89
|
+
detached: true,
|
|
90
|
+
stdio: 'ignore',
|
|
91
|
+
env: envOverrides ? { ...process.env, ...envOverrides } : process.env
|
|
92
|
+
});
|
|
43
93
|
child.unref();
|
|
44
94
|
return true;
|
|
45
95
|
}
|
|
@@ -219,6 +269,7 @@ const commandExists = (command) => {
|
|
|
219
269
|
};
|
|
220
270
|
const envFlagEnabled = (name) => process.env[name] === '1' || process.env[name] === 'true';
|
|
221
271
|
const spawnAnyDetached = (candidates) => candidates.some(([command, args]) => spawnDetached(command, args));
|
|
272
|
+
const spawnAnyDetachedWithEnv = (candidates) => candidates.some(([command, args, env]) => spawnDetached(command, args, env));
|
|
222
273
|
const windowsStartCandidates = (program, args = []) => [
|
|
223
274
|
['cmd', ['/c', 'start', '', program, ...args]]
|
|
224
275
|
];
|
|
@@ -318,6 +369,16 @@ const openGraphInAppWindow = (url) => {
|
|
|
318
369
|
]);
|
|
319
370
|
}
|
|
320
371
|
const appArgument = `--app=${url}`;
|
|
372
|
+
const linuxChromiumStableFlags = [
|
|
373
|
+
'--ozone-platform=x11',
|
|
374
|
+
'--disable-gpu',
|
|
375
|
+
'--disable-features=Vulkan,VaapiVideoDecoder',
|
|
376
|
+
'--disable-background-networking'
|
|
377
|
+
];
|
|
378
|
+
const linuxChromiumEnv = {
|
|
379
|
+
GDK_BACKEND: 'x11',
|
|
380
|
+
OZONE_PLATFORM: 'x11'
|
|
381
|
+
};
|
|
321
382
|
const linuxAppWindowCandidates = [
|
|
322
383
|
'microsoft-edge',
|
|
323
384
|
'microsoft-edge-stable',
|
|
@@ -327,7 +388,11 @@ const openGraphInAppWindow = (url) => {
|
|
|
327
388
|
'chromium-browser',
|
|
328
389
|
'brave-browser'
|
|
329
390
|
].filter((candidate) => commandExists(candidate));
|
|
330
|
-
return
|
|
391
|
+
return spawnAnyDetachedWithEnv(linuxAppWindowCandidates.map((command) => [
|
|
392
|
+
command,
|
|
393
|
+
[...linuxChromiumStableFlags, appArgument, '--new-window'],
|
|
394
|
+
linuxChromiumEnv
|
|
395
|
+
]));
|
|
331
396
|
};
|
|
332
397
|
const openGraphInDetectedBrowser = (url) => {
|
|
333
398
|
if (platform() === 'win32') {
|
|
@@ -339,18 +404,28 @@ const openGraphInDetectedBrowser = (url) => {
|
|
|
339
404
|
...windowsStartCandidates('brave', [url])
|
|
340
405
|
]);
|
|
341
406
|
}
|
|
407
|
+
const linuxChromiumStableFlags = [
|
|
408
|
+
'--ozone-platform=x11',
|
|
409
|
+
'--disable-gpu',
|
|
410
|
+
'--disable-features=Vulkan,VaapiVideoDecoder',
|
|
411
|
+
'--disable-background-networking'
|
|
412
|
+
];
|
|
413
|
+
const linuxChromiumEnv = {
|
|
414
|
+
GDK_BACKEND: 'x11',
|
|
415
|
+
OZONE_PLATFORM: 'x11'
|
|
416
|
+
};
|
|
342
417
|
const linuxBrowserCandidates = [
|
|
343
|
-
['microsoft-edge', [url]],
|
|
344
|
-
['microsoft-edge-stable', [url]],
|
|
345
|
-
['google-chrome', [url]],
|
|
346
|
-
['google-chrome-stable', [url]],
|
|
347
|
-
['chromium', [url]],
|
|
348
|
-
['chromium-browser', [url]],
|
|
349
|
-
['brave-browser', [url]],
|
|
350
|
-
['firefox', ['-new-window', url]]
|
|
418
|
+
['microsoft-edge', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
419
|
+
['microsoft-edge-stable', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
420
|
+
['google-chrome', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
421
|
+
['google-chrome-stable', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
422
|
+
['chromium', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
423
|
+
['chromium-browser', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
424
|
+
['brave-browser', [...linuxChromiumStableFlags, url], linuxChromiumEnv],
|
|
425
|
+
['firefox', ['-new-window', url], undefined]
|
|
351
426
|
];
|
|
352
427
|
const available = linuxBrowserCandidates.filter(([command]) => commandExists(command));
|
|
353
|
-
return
|
|
428
|
+
return spawnAnyDetachedWithEnv(available);
|
|
354
429
|
};
|
|
355
430
|
const openUrlInUi = (url, parentPid) => {
|
|
356
431
|
const openDisabled = process.env.BRAINLINK_NO_BROWSER === '1' ||
|
|
@@ -605,6 +680,58 @@ export const registerWriteCommands = (program) => {
|
|
|
605
680
|
const result = await indexVault(resolved.vault);
|
|
606
681
|
print(options.json, result, () => `Indexed ${result.documentCount} documents, ${result.chunkCount} chunks and ${result.linkCount} links`);
|
|
607
682
|
});
|
|
683
|
+
program
|
|
684
|
+
.command('bench')
|
|
685
|
+
.option('-v, --vault <vault>', 'vault directory')
|
|
686
|
+
.option('-w, --watch', 'watch markdown changes and re-run benchmark in realtime')
|
|
687
|
+
.option('--debounce <ms>', 'watch debounce in milliseconds', '350')
|
|
688
|
+
.option('--json', 'print machine-readable JSON events')
|
|
689
|
+
.description('benchmark indexing in realtime, including compressed pack behavior')
|
|
690
|
+
.action(async (options) => {
|
|
691
|
+
const resolved = await resolveOptions(options);
|
|
692
|
+
const emitProgress = (event) => {
|
|
693
|
+
printBenchRealtimeEvent(options.json, event);
|
|
694
|
+
};
|
|
695
|
+
const printBenchError = (error) => {
|
|
696
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
697
|
+
print(options.json, { event: 'bench-error', message }, () => `[bench] error ${message}`);
|
|
698
|
+
};
|
|
699
|
+
const runAndPrint = async (trigger) => {
|
|
700
|
+
const result = await indexVaultWithOptions(resolved.vault, {
|
|
701
|
+
onProgress: emitProgress
|
|
702
|
+
});
|
|
703
|
+
printBenchSummary(options.json, trigger, resolved.vault, result);
|
|
704
|
+
return result;
|
|
705
|
+
};
|
|
706
|
+
if (!options.watch) {
|
|
707
|
+
await runAndPrint('manual');
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
const debounceMs = parsePositiveInteger(options.debounce ?? '350', 350);
|
|
711
|
+
await runAndPrint('manual');
|
|
712
|
+
print(options.json, {
|
|
713
|
+
event: 'bench-watching',
|
|
714
|
+
vault: resolved.vault,
|
|
715
|
+
debounceMs
|
|
716
|
+
}, () => `[bench] watching ${resolved.vault} (debounce=${debounceMs}ms)`);
|
|
717
|
+
const watcher = startVaultWatcher({
|
|
718
|
+
vaultPath: resolved.vault,
|
|
719
|
+
debounceMs,
|
|
720
|
+
onProgress: emitProgress,
|
|
721
|
+
onIndex: (result) => {
|
|
722
|
+
printBenchSummary(options.json, 'watch', resolved.vault, result);
|
|
723
|
+
},
|
|
724
|
+
onError: printBenchError
|
|
725
|
+
});
|
|
726
|
+
await new Promise((resolveSignal) => {
|
|
727
|
+
const shutdown = () => {
|
|
728
|
+
watcher.close();
|
|
729
|
+
resolveSignal();
|
|
730
|
+
};
|
|
731
|
+
process.once('SIGINT', shutdown);
|
|
732
|
+
process.once('SIGTERM', shutdown);
|
|
733
|
+
});
|
|
734
|
+
});
|
|
608
735
|
program
|
|
609
736
|
.command('doctor')
|
|
610
737
|
.option('-v, --vault <vault>', 'vault directory')
|