@andespindola/brainlink 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +46 -0
- package/dist/application/add-note.js +2 -2
- package/dist/application/build-context.js +16 -10
- package/dist/application/canonical-context-links.js +44 -5
- package/dist/application/check-package-update.js +105 -0
- package/dist/application/find-similar-notes.js +75 -0
- package/dist/application/frontend/client/chunk-fetch.js +236 -0
- package/dist/application/frontend/client/controls.js +178 -0
- package/dist/application/frontend/client/elements.js +122 -0
- package/dist/application/frontend/client/input.js +202 -0
- package/dist/application/frontend/client/node-details.js +191 -0
- package/dist/application/frontend/client/rendering.js +296 -0
- package/dist/application/frontend/client/scope-theme.js +114 -0
- package/dist/application/frontend/client/spatial.js +98 -0
- package/dist/application/frontend/client/storage.js +215 -0
- package/dist/application/frontend/client/upload.js +90 -0
- package/dist/application/frontend/client/worker-bootstrap.js +147 -0
- package/dist/application/frontend/client-js.js +24 -1837
- package/dist/application/frontend/client-render-worker-js.js +1 -1
- package/dist/application/index-vault-phases.js +190 -0
- package/dist/application/index-vault.js +46 -167
- package/dist/application/memory-suggestions.js +4 -1
- package/dist/application/search-knowledge.js +19 -3
- package/dist/cli/commands/write/dedupe-commands.js +59 -0
- package/dist/cli/commands/write/index-commands.js +205 -0
- package/dist/cli/commands/write/link-commands.js +68 -0
- package/dist/cli/commands/write/note-commands.js +146 -0
- package/dist/cli/commands/write/server-commands.js +553 -0
- package/dist/cli/commands/write/shared.js +35 -0
- package/dist/cli/commands/write/vault-lifecycle-commands.js +270 -0
- package/dist/cli/commands/write-commands.js +12 -1303
- package/dist/cli/main.js +39 -3
- package/dist/domain/context.js +60 -11
- package/dist/domain/diversity.js +64 -0
- package/dist/domain/embeddings.js +148 -6
- package/dist/domain/graph-contexts.js +62 -57
- package/dist/domain/graph-layout/cauliflower-layout.js +116 -0
- package/dist/domain/graph-layout/collisions.js +100 -0
- package/dist/domain/graph-layout/hierarchy.js +135 -0
- package/dist/domain/graph-layout/metrics.js +111 -0
- package/dist/domain/graph-layout/segments.js +76 -0
- package/dist/domain/graph-layout/star-layout.js +110 -0
- package/dist/domain/graph-layout.js +4 -625
- package/dist/domain/markdown.js +10 -1
- package/dist/domain/scoring.js +42 -0
- package/dist/domain/tokens.js +17 -1
- package/dist/infrastructure/config.js +33 -1
- package/dist/infrastructure/file-index.js +227 -60
- package/dist/infrastructure/semantic-prefilter.js +24 -0
- package/dist/mcp/server.js +23 -1
- package/dist/mcp/tool-guard.js +29 -0
- package/dist/mcp/tools/maintenance-tools.js +409 -0
- package/dist/mcp/tools/read-tools.js +537 -0
- package/dist/mcp/tools/shared.js +216 -0
- package/dist/mcp/tools/write-tools.js +353 -0
- package/dist/mcp/tools.js +3 -1357
- package/docs/AGENT_USAGE.md +4 -1
- package/docs/ARCHITECTURE.md +4 -3
- package/docs/QUICKSTART.md +4 -0
- package/package.json +2 -2
package/dist/cli/main.js
CHANGED
|
@@ -3,17 +3,52 @@ import { Command } from 'commander';
|
|
|
3
3
|
import { readFileSync } from 'node:fs';
|
|
4
4
|
import { basename, dirname, join } from 'node:path';
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { checkPackageUpdate } from '../application/check-package-update.js';
|
|
7
|
+
import { loadBrainlinkConfig } from '../infrastructure/config.js';
|
|
6
8
|
import { registerAgentCommands } from './commands/agent-commands.js';
|
|
7
9
|
import { registerConfigCommands } from './commands/config-commands.js';
|
|
8
10
|
import { registerPracticalCommands } from './commands/practical-commands.js';
|
|
9
11
|
import { registerReadCommands } from './commands/read-commands.js';
|
|
10
12
|
import { registerVaultCommands } from './commands/vault-commands.js';
|
|
11
13
|
import { registerWriteCommands } from './commands/write-commands.js';
|
|
12
|
-
const
|
|
14
|
+
const readPackageMetadata = () => {
|
|
13
15
|
const packagePath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
|
|
14
16
|
const metadata = JSON.parse(readFileSync(packagePath, 'utf8'));
|
|
15
|
-
return
|
|
17
|
+
return {
|
|
18
|
+
name: metadata.name ?? 'brainlink',
|
|
19
|
+
version: metadata.version ?? '0.0.0'
|
|
20
|
+
};
|
|
16
21
|
};
|
|
22
|
+
const shouldSkipUpdateCheck = () => process.env.BRAINLINK_NO_UPDATE_CHECK === '1' ||
|
|
23
|
+
process.env.BRAINLINK_NO_UPDATE_CHECK === 'true' ||
|
|
24
|
+
process.env.CI === 'true' ||
|
|
25
|
+
process.env.VITEST === 'true' ||
|
|
26
|
+
process.env.NODE_ENV === 'test' ||
|
|
27
|
+
process.argv.includes('--version') ||
|
|
28
|
+
process.argv.includes('-V') ||
|
|
29
|
+
process.argv.includes('--help') ||
|
|
30
|
+
process.argv.includes('-h');
|
|
31
|
+
const maybePrintUpdateNotice = async (metadata) => {
|
|
32
|
+
if (shouldSkipUpdateCheck()) {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
const config = await loadBrainlinkConfig();
|
|
37
|
+
const status = await checkPackageUpdate({
|
|
38
|
+
packageName: metadata.name,
|
|
39
|
+
currentVersion: metadata.version,
|
|
40
|
+
enabled: config.autoUpdateCheck,
|
|
41
|
+
intervalMs: config.updateCheckIntervalMs
|
|
42
|
+
});
|
|
43
|
+
if (!status.skipped && status.updateAvailable && status.latestVersion) {
|
|
44
|
+
console.error(`Brainlink ${status.latestVersion} is available. Current version: ${status.currentVersion}. Update with: ${status.installCommand}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Update checks must never block normal CLI execution.
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
const packageMetadata = readPackageMetadata();
|
|
17
52
|
const program = new Command();
|
|
18
53
|
const cliName = basename(process.argv[1] ?? 'brainlink');
|
|
19
54
|
const displayName = cliName === 'blink' ? 'blink' : 'brainlink';
|
|
@@ -22,13 +57,14 @@ program
|
|
|
22
57
|
.name(displayName)
|
|
23
58
|
.alias(aliasName)
|
|
24
59
|
.description('Local-first knowledge memory for agents')
|
|
25
|
-
.version(
|
|
60
|
+
.version(packageMetadata.version);
|
|
26
61
|
registerWriteCommands(program);
|
|
27
62
|
registerReadCommands(program);
|
|
28
63
|
registerPracticalCommands(program);
|
|
29
64
|
registerConfigCommands(program);
|
|
30
65
|
registerVaultCommands(program);
|
|
31
66
|
registerAgentCommands(program);
|
|
67
|
+
await maybePrintUpdateNotice(packageMetadata);
|
|
32
68
|
program.parseAsync().catch((error) => {
|
|
33
69
|
const message = error instanceof Error ? error.message : String(error);
|
|
34
70
|
console.error(message);
|
package/dist/domain/context.js
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
import { middleOutIndices } from './middle-out.js';
|
|
2
|
+
import { estimateTokenCount } from './tokens.js';
|
|
3
|
+
import { orderByMaximalMarginalRelevance } from './diversity.js';
|
|
4
|
+
const contentTokenPattern = /[\p{L}\p{N}_-]+/gu;
|
|
5
|
+
const tokenizeForDiversity = (text) => new Set((text.toLowerCase().match(contentTokenPattern) ?? []).filter((token) => token.length > 1));
|
|
2
6
|
const maxSectionsPerDocument = 3;
|
|
7
|
+
// Character budget used only for truncating an overflowing chunk back down to a
|
|
8
|
+
// token target; the conservative per-token character ratio keeps the rendered
|
|
9
|
+
// slice within budget even though estimateTokenCount is the source of truth for
|
|
10
|
+
// counting.
|
|
11
|
+
const truncationCharsPerToken = 4;
|
|
12
|
+
const sectionSeparatorTokens = 1;
|
|
13
|
+
const headerReserveTokens = 8;
|
|
14
|
+
const truncateToTokens = (text, maxTokens) => {
|
|
15
|
+
const maxChars = Math.max(0, maxTokens) * truncationCharsPerToken;
|
|
16
|
+
return text.length <= maxChars ? text : text.slice(0, maxChars);
|
|
17
|
+
};
|
|
18
|
+
// Prefer the token count cached on the chunk at index time; fall back to a live
|
|
19
|
+
// estimate when the result did not carry one (legacy indexes, pack fallback).
|
|
20
|
+
const contentTokens = (result) => result.tokenCount ?? estimateTokenCount(result.content);
|
|
21
|
+
// Mirror the per-section framing that formatContextPackage renders (heading,
|
|
22
|
+
// Source/Tags/Score/Mode lines and the trailing separator) so the budget
|
|
23
|
+
// reflects the package the agent actually receives, not just chunk content.
|
|
24
|
+
const estimateFramingTokens = (result) => {
|
|
25
|
+
const tagsLine = result.tags.length > 0 ? `Tags: ${result.tags.map((tag) => `#${tag}`).join(' ')}\n` : '';
|
|
26
|
+
const framing = `## . ${result.title}\nSource: ${result.path}\n${tagsLine}Score: 0.000\nMode: ${result.searchMode}\n\n`;
|
|
27
|
+
return estimateTokenCount(framing) + sectionSeparatorTokens;
|
|
28
|
+
};
|
|
29
|
+
const estimateSectionTokens = (result) => contentTokens(result) + estimateFramingTokens(result);
|
|
3
30
|
const byScore = (left, right) => right.score - left.score || left.title.localeCompare(right.title);
|
|
4
31
|
const byOrdinal = (left, right) => (left.chunkOrdinal ?? Number.MAX_SAFE_INTEGER) - (right.chunkOrdinal ?? Number.MAX_SAFE_INTEGER);
|
|
5
32
|
const middleOutDocumentResults = (results) => {
|
|
@@ -20,14 +47,16 @@ export const selectContextSections = (results, maxTokens) => {
|
|
|
20
47
|
state.set(result.documentId, [...current, result]);
|
|
21
48
|
return state;
|
|
22
49
|
}, new Map());
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
.
|
|
30
|
-
.map((
|
|
50
|
+
// Order documents by maximal marginal relevance so the budget is not spent on
|
|
51
|
+
// several near-duplicate notes: each pick trades its own relevance against how
|
|
52
|
+
// similar it is to documents already chosen. The strongest document still
|
|
53
|
+
// leads (no redundancy penalty on an empty selection).
|
|
54
|
+
const documentCandidates = Array.from(grouped.entries()).map(([documentId, group]) => ({
|
|
55
|
+
id: documentId,
|
|
56
|
+
relevance: group.reduce((max, result) => Math.max(max, result.score), Number.NEGATIVE_INFINITY),
|
|
57
|
+
tokens: tokenizeForDiversity(group.map((result) => result.content).join(' '))
|
|
58
|
+
}));
|
|
59
|
+
const documentOrder = orderByMaximalMarginalRelevance(documentCandidates);
|
|
31
60
|
const selected = documentOrder.reduce((state, documentId) => {
|
|
32
61
|
const ordered = middleOutDocumentResults(grouped.get(documentId) ?? []);
|
|
33
62
|
let usedTokens = state.usedTokens;
|
|
@@ -38,7 +67,7 @@ export const selectContextSections = (results, maxTokens) => {
|
|
|
38
67
|
if (seenChunks.has(result.chunkId)) {
|
|
39
68
|
continue;
|
|
40
69
|
}
|
|
41
|
-
const tokenCost =
|
|
70
|
+
const tokenCost = estimateSectionTokens(result);
|
|
42
71
|
if (usedTokens + tokenCost > maxTokens) {
|
|
43
72
|
break;
|
|
44
73
|
}
|
|
@@ -62,11 +91,31 @@ export const selectContextSections = (results, maxTokens) => {
|
|
|
62
91
|
seenChunks
|
|
63
92
|
};
|
|
64
93
|
}, {
|
|
65
|
-
usedTokens:
|
|
94
|
+
usedTokens: headerReserveTokens,
|
|
66
95
|
sections: [],
|
|
67
96
|
seenChunks: new Set()
|
|
68
97
|
});
|
|
69
|
-
|
|
98
|
+
if (selected.sections.length > 0 || results.length === 0) {
|
|
99
|
+
return selected.sections;
|
|
100
|
+
}
|
|
101
|
+
// Retrieval found matches but every chunk overflowed the token budget. Never
|
|
102
|
+
// surface an empty context in that case: include the highest-scored chunk,
|
|
103
|
+
// truncated so the rendered package still respects the budget. This keeps
|
|
104
|
+
// brainlink_context consistent with brainlink_search instead of reporting
|
|
105
|
+
// "No relevant context found." whenever the strongest chunk is large.
|
|
106
|
+
const topResult = [...results].sort(byScore)[0];
|
|
107
|
+
const framingTokens = estimateFramingTokens(topResult);
|
|
108
|
+
const contentTokenBudget = maxTokens - headerReserveTokens - framingTokens;
|
|
109
|
+
return [
|
|
110
|
+
{
|
|
111
|
+
title: topResult.title,
|
|
112
|
+
path: topResult.path,
|
|
113
|
+
content: truncateToTokens(topResult.content, contentTokenBudget),
|
|
114
|
+
score: topResult.score,
|
|
115
|
+
searchMode: topResult.searchMode,
|
|
116
|
+
tags: topResult.tags
|
|
117
|
+
}
|
|
118
|
+
];
|
|
70
119
|
};
|
|
71
120
|
export const formatContextPackage = (query, sections) => {
|
|
72
121
|
const body = sections
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Maximal Marginal Relevance ordering. Greedily orders candidates so each pick
|
|
2
|
+
// balances its own relevance against how redundant it is versus everything
|
|
3
|
+
// already chosen, preventing a context window from filling with near-duplicate
|
|
4
|
+
// material. Pure and deterministic: ties resolve by relevance then id.
|
|
5
|
+
const jaccardSimilarity = (left, right) => {
|
|
6
|
+
if (left.size === 0 || right.size === 0) {
|
|
7
|
+
return 0;
|
|
8
|
+
}
|
|
9
|
+
let intersection = 0;
|
|
10
|
+
const [small, large] = left.size <= right.size ? [left, right] : [right, left];
|
|
11
|
+
for (const token of small) {
|
|
12
|
+
if (large.has(token)) {
|
|
13
|
+
intersection += 1;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
const union = left.size + right.size - intersection;
|
|
17
|
+
return union === 0 ? 0 : intersection / union;
|
|
18
|
+
};
|
|
19
|
+
const normalizeRelevance = (candidates) => {
|
|
20
|
+
// Reduce instead of Math.min(...values): the candidate list can be as large as
|
|
21
|
+
// the caller's result limit, and spreading a very large array overflows the
|
|
22
|
+
// call stack.
|
|
23
|
+
const bounds = candidates.reduce((range, candidate) => ({
|
|
24
|
+
min: Math.min(range.min, candidate.relevance),
|
|
25
|
+
max: Math.max(range.max, candidate.relevance)
|
|
26
|
+
}), { min: Infinity, max: -Infinity });
|
|
27
|
+
const span = bounds.max - bounds.min;
|
|
28
|
+
return new Map(candidates.map((candidate) => [candidate.id, span === 0 ? 1 : (candidate.relevance - bounds.min) / span]));
|
|
29
|
+
};
|
|
30
|
+
// lambda in [0,1]: 1 ignores redundancy (pure relevance order), lower values
|
|
31
|
+
// push diverse picks higher. The default balances relevance and diversity; the
|
|
32
|
+
// strongest candidate still always leads (no redundancy penalty on an empty
|
|
33
|
+
// selection).
|
|
34
|
+
export const orderByMaximalMarginalRelevance = (candidates, lambda = 0.5) => {
|
|
35
|
+
if (candidates.length <= 1) {
|
|
36
|
+
return candidates.map((candidate) => candidate.id);
|
|
37
|
+
}
|
|
38
|
+
const relevanceById = normalizeRelevance(candidates);
|
|
39
|
+
const remaining = new Map(candidates.map((candidate) => [candidate.id, candidate]));
|
|
40
|
+
const selected = [];
|
|
41
|
+
const order = [];
|
|
42
|
+
while (remaining.size > 0) {
|
|
43
|
+
let best = null;
|
|
44
|
+
for (const candidate of remaining.values()) {
|
|
45
|
+
const relevance = relevanceById.get(candidate.id) ?? 0;
|
|
46
|
+
const redundancy = selected.reduce((max, picked) => Math.max(max, jaccardSimilarity(candidate.tokens, picked.tokens)), 0);
|
|
47
|
+
const score = lambda * relevance - (1 - lambda) * redundancy;
|
|
48
|
+
if (best === null ||
|
|
49
|
+
score > best.score ||
|
|
50
|
+
(score === best.score &&
|
|
51
|
+
(relevance > best.relevance ||
|
|
52
|
+
(relevance === best.relevance && candidate.id.localeCompare(best.candidate.id) < 0)))) {
|
|
53
|
+
best = { candidate, score, relevance };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (best === null) {
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
selected.push(best.candidate);
|
|
60
|
+
order.push(best.candidate.id);
|
|
61
|
+
remaining.delete(best.candidate.id);
|
|
62
|
+
}
|
|
63
|
+
return order;
|
|
64
|
+
};
|
|
@@ -67,8 +67,16 @@ const featureHash = (feature) => {
|
|
|
67
67
|
const sign = value & 1 ? 1 : -1;
|
|
68
68
|
return [index, sign];
|
|
69
69
|
};
|
|
70
|
+
const vectorMagnitude = (vector) => {
|
|
71
|
+
let sumSquares = 0;
|
|
72
|
+
for (let index = 0; index < vector.length; index += 1) {
|
|
73
|
+
const value = vector[index];
|
|
74
|
+
sumSquares += value * value;
|
|
75
|
+
}
|
|
76
|
+
return Math.sqrt(sumSquares);
|
|
77
|
+
};
|
|
70
78
|
const normalizeVector = (vector) => {
|
|
71
|
-
const magnitude =
|
|
79
|
+
const magnitude = vectorMagnitude(vector);
|
|
72
80
|
return magnitude === 0 ? vector : vector.map((value) => value / magnitude);
|
|
73
81
|
};
|
|
74
82
|
const applyFeature = (vector, feature, weight) => {
|
|
@@ -86,15 +94,33 @@ export const createLocalEmbedding = (input) => {
|
|
|
86
94
|
const weighted = tokenFeatures(tokens).reduce((vector, feature) => applyFeature(vector, feature, feature.startsWith('b:') ? 0.65 : 1), initial);
|
|
87
95
|
return normalizeVector(weighted);
|
|
88
96
|
};
|
|
97
|
+
// Dot product over the shared prefix. For L2-normalized vectors (every vector
|
|
98
|
+
// from createLocalEmbedding is normalized) this equals cosine similarity, so
|
|
99
|
+
// the hot retrieval loop can skip the per-comparison magnitude recompute.
|
|
100
|
+
export const dotProduct = (left, right) => {
|
|
101
|
+
const length = Math.min(left.length, right.length);
|
|
102
|
+
let total = 0;
|
|
103
|
+
for (let index = 0; index < length; index += 1) {
|
|
104
|
+
total += left[index] * right[index];
|
|
105
|
+
}
|
|
106
|
+
return total;
|
|
107
|
+
};
|
|
89
108
|
export const cosineSimilarity = (left, right) => {
|
|
90
109
|
const length = Math.min(left.length, right.length);
|
|
91
110
|
if (length === 0) {
|
|
92
111
|
return 0;
|
|
93
112
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
113
|
+
let dot = 0;
|
|
114
|
+
let leftSumSquares = 0;
|
|
115
|
+
let rightSumSquares = 0;
|
|
116
|
+
for (let index = 0; index < length; index += 1) {
|
|
117
|
+
const leftValue = left[index];
|
|
118
|
+
const rightValue = right[index];
|
|
119
|
+
dot += leftValue * rightValue;
|
|
120
|
+
leftSumSquares += leftValue * leftValue;
|
|
121
|
+
rightSumSquares += rightValue * rightValue;
|
|
122
|
+
}
|
|
123
|
+
return leftSumSquares === 0 || rightSumSquares === 0 ? 0 : dot / Math.sqrt(leftSumSquares * rightSumSquares);
|
|
98
124
|
};
|
|
99
125
|
const bucketKey = (index, value) => `${value >= 0 ? 'p' : 'n'}:${index}`;
|
|
100
126
|
export const createEmbeddingBuckets = (vector, bucketCount = defaultEmbeddingBucketCount) => vector
|
|
@@ -115,4 +141,120 @@ export const createLocalEmbeddingProvider = () => ({
|
|
|
115
141
|
name: 'local',
|
|
116
142
|
embed: async (input) => input.map(createLocalEmbedding)
|
|
117
143
|
});
|
|
118
|
-
|
|
144
|
+
const defaultExternalTimeoutMs = 10_000;
|
|
145
|
+
const defaultExternalBatchSize = 64;
|
|
146
|
+
const ollamaDefaultUrl = 'http://127.0.0.1:11434';
|
|
147
|
+
const ollamaDefaultModel = 'nomic-embed-text';
|
|
148
|
+
const openAiDefaultUrl = 'https://api.openai.com/v1';
|
|
149
|
+
const openAiDefaultModel = 'text-embedding-3-small';
|
|
150
|
+
const trimTrailingSlash = (value) => value.replace(/\/+$/, '');
|
|
151
|
+
const chunkBatches = (items, size) => {
|
|
152
|
+
const batchSize = Number.isFinite(size) && size > 0 ? Math.floor(size) : defaultExternalBatchSize;
|
|
153
|
+
const batches = [];
|
|
154
|
+
for (let index = 0; index < items.length; index += batchSize) {
|
|
155
|
+
batches.push(items.slice(index, index + batchSize));
|
|
156
|
+
}
|
|
157
|
+
return batches;
|
|
158
|
+
};
|
|
159
|
+
const isNumberArray = (value) => Array.isArray(value) && value.every((entry) => typeof entry === 'number' && Number.isFinite(entry));
|
|
160
|
+
const fetchJson = async (url, init, timeoutMs, providerName) => {
|
|
161
|
+
const controller = new AbortController();
|
|
162
|
+
const timeout = setTimeout(() => controller.abort(), Math.max(1, timeoutMs));
|
|
163
|
+
try {
|
|
164
|
+
const response = await fetch(url, { ...init, signal: controller.signal });
|
|
165
|
+
if (!response.ok) {
|
|
166
|
+
throw new Error(`${providerName} embedding request failed with HTTP ${response.status}`);
|
|
167
|
+
}
|
|
168
|
+
return (await response.json());
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
172
|
+
throw new Error(`${providerName} embedding request to ${url} failed: ${reason}`);
|
|
173
|
+
}
|
|
174
|
+
finally {
|
|
175
|
+
clearTimeout(timeout);
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
// External providers return vectors in their own space; L2-normalize so dot
|
|
179
|
+
// product equals cosine similarity, matching the contract local embeddings rely
|
|
180
|
+
// on across the retrieval and bucket-prefilter paths.
|
|
181
|
+
const createOllamaEmbeddingProvider = (config) => {
|
|
182
|
+
const baseUrl = trimTrailingSlash(config.url?.trim() || ollamaDefaultUrl);
|
|
183
|
+
const model = config.model?.trim() || ollamaDefaultModel;
|
|
184
|
+
const timeoutMs = config.timeoutMs ?? defaultExternalTimeoutMs;
|
|
185
|
+
return {
|
|
186
|
+
name: 'ollama',
|
|
187
|
+
embed: async (input) => {
|
|
188
|
+
const vectors = [];
|
|
189
|
+
for (const batch of chunkBatches(input, config.batchSize ?? defaultExternalBatchSize)) {
|
|
190
|
+
const payload = await fetchJson(`${baseUrl}/api/embed`, {
|
|
191
|
+
method: 'POST',
|
|
192
|
+
headers: { 'content-type': 'application/json' },
|
|
193
|
+
body: JSON.stringify({ model, input: batch })
|
|
194
|
+
}, timeoutMs, 'ollama');
|
|
195
|
+
const embeddings = payload.embeddings;
|
|
196
|
+
if (!Array.isArray(embeddings) || embeddings.length !== batch.length) {
|
|
197
|
+
throw new Error('ollama embedding response shape was invalid');
|
|
198
|
+
}
|
|
199
|
+
for (const embedding of embeddings) {
|
|
200
|
+
if (!isNumberArray(embedding) || embedding.length === 0) {
|
|
201
|
+
throw new Error('ollama embedding response contained an invalid vector');
|
|
202
|
+
}
|
|
203
|
+
vectors.push(normalizeVector(embedding));
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return vectors;
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
};
|
|
210
|
+
const createOpenAiEmbeddingProvider = (config) => {
|
|
211
|
+
const baseUrl = trimTrailingSlash(config.url?.trim() || openAiDefaultUrl);
|
|
212
|
+
const model = config.model?.trim() || openAiDefaultModel;
|
|
213
|
+
const timeoutMs = config.timeoutMs ?? defaultExternalTimeoutMs;
|
|
214
|
+
const apiKeyEnv = config.apiKeyEnv?.trim() || 'OPENAI_API_KEY';
|
|
215
|
+
return {
|
|
216
|
+
name: 'openai',
|
|
217
|
+
embed: async (input) => {
|
|
218
|
+
const apiKey = process.env[apiKeyEnv]?.trim();
|
|
219
|
+
if (!apiKey) {
|
|
220
|
+
throw new Error(`openai embedding requires the ${apiKeyEnv} environment variable to be set`);
|
|
221
|
+
}
|
|
222
|
+
const vectors = [];
|
|
223
|
+
for (const batch of chunkBatches(input, config.batchSize ?? defaultExternalBatchSize)) {
|
|
224
|
+
const payload = await fetchJson(`${baseUrl}/embeddings`, {
|
|
225
|
+
method: 'POST',
|
|
226
|
+
headers: {
|
|
227
|
+
'content-type': 'application/json',
|
|
228
|
+
authorization: `Bearer ${apiKey}`
|
|
229
|
+
},
|
|
230
|
+
body: JSON.stringify({ model, input: batch })
|
|
231
|
+
}, timeoutMs, 'openai');
|
|
232
|
+
const data = payload.data;
|
|
233
|
+
if (!Array.isArray(data) || data.length !== batch.length) {
|
|
234
|
+
throw new Error('openai embedding response shape was invalid');
|
|
235
|
+
}
|
|
236
|
+
const ordered = [...data].sort((left, right) => (left.index ?? 0) - (right.index ?? 0));
|
|
237
|
+
for (const entry of ordered) {
|
|
238
|
+
const embedding = entry.embedding;
|
|
239
|
+
if (!isNumberArray(embedding) || embedding.length === 0) {
|
|
240
|
+
throw new Error('openai embedding response contained an invalid vector');
|
|
241
|
+
}
|
|
242
|
+
vectors.push(normalizeVector(embedding));
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
return vectors;
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
};
|
|
249
|
+
export const createEmbeddingProvider = (name, config = {}) => {
|
|
250
|
+
switch (name) {
|
|
251
|
+
case 'local':
|
|
252
|
+
return createLocalEmbeddingProvider();
|
|
253
|
+
case 'ollama':
|
|
254
|
+
return createOllamaEmbeddingProvider(config);
|
|
255
|
+
case 'openai':
|
|
256
|
+
return createOpenAiEmbeddingProvider(config);
|
|
257
|
+
default:
|
|
258
|
+
return createDisabledEmbeddingProvider();
|
|
259
|
+
}
|
|
260
|
+
};
|
|
@@ -8,7 +8,59 @@ const context = (title) => ({
|
|
|
8
8
|
const byTitle = (left, right) => left.title.localeCompare(right.title);
|
|
9
9
|
const edgeKey = (source, target) => source < target ? `${source}|${target}` : `${target}|${source}`;
|
|
10
10
|
const nodeSearchText = (node) => normalize([node.title, node.path, ...node.tags].join(' '));
|
|
11
|
-
|
|
11
|
+
const escapeRegExp = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
12
|
+
// Hub notes name their context as "<Context> Hub". These reserved hubs are
|
|
13
|
+
// structural anchors of the graph, not visual contexts, so they never become
|
|
14
|
+
// keyword rules.
|
|
15
|
+
const reservedHubTitles = new Set(['memory hub', 'knowledge root']);
|
|
16
|
+
const hubContextPattern = /^(.+?)\s+hub$/i;
|
|
17
|
+
// Extra keywords can be declared in a hub note body via a "Keywords: a, b, c"
|
|
18
|
+
// line so a context can match terms beyond its own name.
|
|
19
|
+
const parseHubKeywords = (content) => {
|
|
20
|
+
const match = content.match(/^\s*keywords?\s*:\s*(.+)$/im);
|
|
21
|
+
if (!match) {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
return match[1]
|
|
25
|
+
.split(',')
|
|
26
|
+
.map((keyword) => normalize(keyword))
|
|
27
|
+
.filter((keyword) => keyword.length > 0);
|
|
28
|
+
};
|
|
29
|
+
// Derive content-classification rules from the hub notes present in the graph.
|
|
30
|
+
// A "<Context> Hub" note contributes the context, keyed by its normalized name
|
|
31
|
+
// plus any declared extra keywords. This keeps vault-specific taxonomy in the
|
|
32
|
+
// vault instead of hardcoding it in the engine.
|
|
33
|
+
export const deriveVisualContextRules = (nodes) => {
|
|
34
|
+
const rules = new Map();
|
|
35
|
+
nodes.forEach((node) => {
|
|
36
|
+
const match = node.title.trim().match(hubContextPattern);
|
|
37
|
+
if (!match) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (reservedHubTitles.has(normalize(node.title))) {
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
const contextTitle = match[1].trim();
|
|
44
|
+
if (contextTitle.length === 0) {
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const keywords = rules.get(contextTitle) ?? new Set();
|
|
48
|
+
keywords.add(normalize(contextTitle));
|
|
49
|
+
parseHubKeywords(node.content).forEach((keyword) => keywords.add(keyword));
|
|
50
|
+
rules.set(contextTitle, keywords);
|
|
51
|
+
});
|
|
52
|
+
return Array.from(rules.entries(), ([contextTitle, keywords]) => ({
|
|
53
|
+
context: contextTitle,
|
|
54
|
+
keywords: Array.from(keywords)
|
|
55
|
+
}));
|
|
56
|
+
};
|
|
57
|
+
const matchRule = (text, rules) => {
|
|
58
|
+
const matched = rules.find((rule) => rule.keywords.some((keyword) => new RegExp(`\\b${escapeRegExp(keyword)}\\b`).test(text)));
|
|
59
|
+
return matched ? context(matched.context) : null;
|
|
60
|
+
};
|
|
61
|
+
// Structural, vault-agnostic classification. Named-hub phrases and path layout
|
|
62
|
+
// are generic conventions; vault-specific contexts come from `rules`.
|
|
63
|
+
export const inferExplicitVisualGraphContext = (node, rules = []) => {
|
|
12
64
|
const text = nodeSearchText(node);
|
|
13
65
|
const path = normalize(node.path);
|
|
14
66
|
if (includesAny(text, [/\bgithub repositories hub\b/]))
|
|
@@ -25,80 +77,32 @@ export const inferExplicitVisualGraphContext = (node) => {
|
|
|
25
77
|
return context('Git Workflow');
|
|
26
78
|
if (includesAny(text, [/\bagent memory hub\b/]))
|
|
27
79
|
return context('Agent Memory');
|
|
28
|
-
if (includesAny(text, [/pingu_ai_codding_pair_programming/, /\bpingu\b/]))
|
|
29
|
-
return context('Pingu');
|
|
30
80
|
if (path.startsWith('github-repos/'))
|
|
31
81
|
return context('GitHub Repositories');
|
|
32
82
|
if (path.startsWith('github-org-repos/'))
|
|
33
83
|
return context('GitHub Organizations');
|
|
34
84
|
if (path.startsWith('machine-config/'))
|
|
35
85
|
return context('Machine Configuration');
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return context('AnonSpace');
|
|
40
|
-
if (includesAny(text, [/\bsubstructa\b/]))
|
|
41
|
-
return context('Substructa');
|
|
42
|
-
if (includesAny(text, [/\bnebula\b/]))
|
|
43
|
-
return context('Nebula');
|
|
44
|
-
if (includesAny(text, [/\bsnippets?\b/, /\bupgrader\b/, /\bversion-map\b/]))
|
|
45
|
-
return context('Snippets');
|
|
46
|
-
if (includesAny(text, [
|
|
47
|
-
/\bpreference\b/,
|
|
48
|
-
/\bpreferences\b/,
|
|
49
|
-
/\bpreferencia\b/,
|
|
50
|
-
/\bpreferencias\b/,
|
|
51
|
-
/\bpreferência\b/,
|
|
52
|
-
/\bpreferências\b/,
|
|
53
|
-
/\bplaybook\b/,
|
|
54
|
-
/\bdirective\b/,
|
|
55
|
-
/\bdirectives\b/,
|
|
56
|
-
/\bdiretiva\b/,
|
|
57
|
-
/\bdiretivas\b/,
|
|
58
|
-
/\bengineering-style\b/,
|
|
59
|
-
/\bglobal-engineering\b/,
|
|
60
|
-
/\bcoding-identity\b/,
|
|
61
|
-
/\bagents\.md\b/,
|
|
62
|
-
/\bagents-md\b/,
|
|
63
|
-
/\bordem direta\b/,
|
|
64
|
-
/\bordem-direta\b/,
|
|
65
|
-
/\bman-in-the-loop\b/,
|
|
66
|
-
/\bconfig geral\b/,
|
|
67
|
-
/\bconfig-geral\b/,
|
|
68
|
-
/\bsync config_files\b/,
|
|
69
|
-
/\bsync-config-files\b/,
|
|
70
|
-
/\bregra operacional\b/,
|
|
71
|
-
/\bregras operacionais\b/,
|
|
72
|
-
/\boperational rule\b/,
|
|
73
|
-
/\boperational rules\b/,
|
|
74
|
-
/\boperational policy\b/
|
|
75
|
-
])) {
|
|
76
|
-
return context('User Preferences');
|
|
86
|
+
const ruleMatch = matchRule(text, rules);
|
|
87
|
+
if (ruleMatch) {
|
|
88
|
+
return ruleMatch;
|
|
77
89
|
}
|
|
78
|
-
if (includesAny(text, [/\binkdrop\b/]))
|
|
79
|
-
return context('Inkdrop');
|
|
80
|
-
if (includesAny(text, [/\blazyvim\b/, /\bneovim\b/, /\bnvim\b/, /\bmason\b/, /\bwrapper\b/]))
|
|
81
|
-
return context('Neovim LazyVim');
|
|
82
|
-
if (includesAny(text, [/\bgit-flow\b/, /\borigin-sync\b/, /\bgit-identidade\b/, /\bcommit\b/, /\bpush\b/]))
|
|
83
|
-
return context('Git Workflow');
|
|
84
|
-
if (includesAny(text, [/\bdocker\b/, /\bkubernetes\b/, /\bdeploy\b/, /\bredeploy\b/]))
|
|
85
|
-
return context('Operations');
|
|
86
90
|
if (path.startsWith('agents/'))
|
|
87
91
|
return context('Agent Memory');
|
|
88
92
|
return null;
|
|
89
93
|
};
|
|
90
|
-
export const inferVisualGraphContext = (node) => {
|
|
91
|
-
const explicit = inferExplicitVisualGraphContext(node);
|
|
94
|
+
export const inferVisualGraphContext = (node, rules = []) => {
|
|
95
|
+
const explicit = inferExplicitVisualGraphContext(node, rules);
|
|
92
96
|
if (explicit) {
|
|
93
97
|
return explicit;
|
|
94
98
|
}
|
|
95
99
|
const [root] = node.path.split('/').filter(Boolean);
|
|
96
100
|
return context(root ? root.replace(/[-_]+/g, ' ') : 'Root');
|
|
97
101
|
};
|
|
98
|
-
export const groupNodesByVisualContext = (nodes) => {
|
|
102
|
+
export const groupNodesByVisualContext = (nodes, rules = deriveVisualContextRules(nodes)) => {
|
|
99
103
|
const groups = new Map();
|
|
100
104
|
nodes.forEach((node) => {
|
|
101
|
-
const visualContext = inferVisualGraphContext(node);
|
|
105
|
+
const visualContext = inferVisualGraphContext(node, rules);
|
|
102
106
|
const bucket = groups.get(visualContext.title);
|
|
103
107
|
if (bucket) {
|
|
104
108
|
bucket.push(node);
|
|
@@ -147,8 +151,9 @@ export const addVisualContextEdges = (graph) => {
|
|
|
147
151
|
.filter((edge) => Boolean(edge.target))
|
|
148
152
|
.map((edge) => edgeKey(edge.source, edge.target)));
|
|
149
153
|
const degrees = countDegrees(graph.edges);
|
|
154
|
+
const rules = deriveVisualContextRules(graph.nodes);
|
|
150
155
|
const derivedEdges = [];
|
|
151
|
-
for (const [contextTitle, nodes] of groupNodesByVisualContext(graph.nodes).entries()) {
|
|
156
|
+
for (const [contextTitle, nodes] of groupNodesByVisualContext(graph.nodes, rules).entries()) {
|
|
152
157
|
if (nodes.length <= 1) {
|
|
153
158
|
continue;
|
|
154
159
|
}
|