@andespindola/brainlink 1.0.6 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +73 -0
  3. package/dist/application/find-similar-notes.js +75 -0
  4. package/dist/application/get-graph-node.js +2 -2
  5. package/dist/application/get-graph-summary.js +2 -2
  6. package/dist/application/get-graph.js +2 -2
  7. package/dist/application/index-vault-phases.js +5 -4
  8. package/dist/application/index-vault.js +4 -4
  9. package/dist/application/list-agents.js +2 -2
  10. package/dist/application/list-links.js +3 -3
  11. package/dist/application/memory-suggestions.js +4 -1
  12. package/dist/application/ports/knowledge-store.js +1 -0
  13. package/dist/application/ranking/run-search.js +177 -0
  14. package/dist/application/search-graph-node-ids.js +3 -2
  15. package/dist/application/search-knowledge.js +22 -5
  16. package/dist/application/vault-git.js +180 -0
  17. package/dist/application/vault-snapshot.js +87 -0
  18. package/dist/cli/commands/vault-sync-commands.js +115 -0
  19. package/dist/cli/main.js +2 -0
  20. package/dist/domain/context.js +27 -14
  21. package/dist/domain/diversity.js +64 -0
  22. package/dist/domain/embeddings.js +117 -1
  23. package/dist/domain/markdown.js +10 -1
  24. package/dist/domain/scoring.js +42 -0
  25. package/dist/domain/tokens.js +17 -1
  26. package/dist/infrastructure/config.js +31 -1
  27. package/dist/infrastructure/file-index.js +99 -328
  28. package/dist/infrastructure/knowledge-store/binary-store.js +163 -0
  29. package/dist/infrastructure/knowledge-store/index.js +36 -0
  30. package/dist/infrastructure/knowledge-store/stored-index.js +216 -0
  31. package/dist/mcp/server.js +16 -1
  32. package/dist/mcp/tools/read-tools.js +33 -0
  33. package/dist/mcp/tools/write-tools.js +106 -0
  34. package/dist/mcp/tools.js +2 -2
  35. package/docs/AGENT_USAGE.md +4 -1
  36. package/docs/ARCHITECTURE.md +4 -3
  37. package/package.json +3 -2
@@ -188,7 +188,15 @@ const splitLongParagraph = (paragraph, maxCharacters) => {
188
188
  return [...state, sentence];
189
189
  }, []);
190
190
  };
191
+ const headingParagraphPattern = /^#{1,6}\s/;
192
+ const startsWithHeading = (paragraph) => headingParagraphPattern.test(paragraph);
191
193
  export const splitIntoChunks = (documentId, content, maxCharacters = 1200) => {
194
+ // A heading should begin a new chunk so a section's body is not glued onto the
195
+ // tail of the previous section — but only once the current chunk already holds
196
+ // substantial content. This keeps large notes split on topical (heading)
197
+ // boundaries while small notes (and trailing metadata sections like
198
+ // "## Context Links") stay in a single chunk instead of fragmenting.
199
+ const minChunkCharacters = Math.floor(maxCharacters / 2);
192
200
  const paragraphs = normalizeChunkContent(stripFrontmatter(content))
193
201
  .split(/\n{2,}/)
194
202
  .filter(Boolean)
@@ -198,8 +206,9 @@ export const splitIntoChunks = (documentId, content, maxCharacters = 1200) => {
198
206
  if (!lastChunk) {
199
207
  return [paragraph];
200
208
  }
209
+ const startsNewSection = startsWithHeading(paragraph) && lastChunk.length >= minChunkCharacters;
201
210
  const merged = `${lastChunk}\n\n${paragraph}`;
202
- if (merged.length <= maxCharacters) {
211
+ if (!startsNewSection && merged.length <= maxCharacters) {
203
212
  return [...state.slice(0, -1), merged];
204
213
  }
205
214
  return [...state, paragraph];
@@ -0,0 +1,42 @@
1
+ // Pure retrieval scoring primitives: BM25 lexical relevance and min-max
2
+ // normalization. Kept dependency-free and deterministic so the hot retrieval
3
+ // loop in the infrastructure layer stays testable and the domain owns the
4
+ // ranking math.
5
+ export const defaultBm25Parameters = {
6
+ k1: 1.2,
7
+ b: 0.75
8
+ };
9
+ // Robertson/Sparck-Jones probabilistic IDF with the +0.5 smoothing that keeps
10
+ // the value finite for terms present in every chunk. Floored at 0 so a term
11
+ // shared by the whole candidate set can never push a score negative.
12
+ export const inverseDocumentFrequency = (documentCount, documentFrequency) => {
13
+ if (documentCount <= 0) {
14
+ return 0;
15
+ }
16
+ const raw = Math.log(1 + (documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
17
+ return raw > 0 ? raw : 0;
18
+ };
19
+ // BM25 contribution for a single query term given its (field-weighted)
20
+ // frequency in a chunk. `documentLength` and `averageLength` are in the same
21
+ // weighted unit so length normalization stays consistent with how the caller
22
+ // counts terms.
23
+ export const bm25TermScore = (params) => {
24
+ if (params.termFrequency <= 0) {
25
+ return 0;
26
+ }
27
+ const { k1, b } = params.parameters ?? defaultBm25Parameters;
28
+ const idf = inverseDocumentFrequency(params.documentCount, params.documentFrequency);
29
+ const normalizedLength = params.averageLength > 0 ? params.documentLength / params.averageLength : 1;
30
+ const denominator = params.termFrequency + k1 * (1 - b + b * normalizedLength);
31
+ return denominator > 0 ? (idf * (params.termFrequency * (k1 + 1))) / denominator : 0;
32
+ };
33
+ // Scale a value into [0, 1] given the observed range. When every candidate
34
+ // shares the same score (range collapses) a positive score maps to 1 and a
35
+ // zero score to 0, so a single result is never silently nulled out.
36
+ export const minMaxNormalize = (value, minimum, maximum) => {
37
+ if (maximum <= minimum) {
38
+ return value > 0 ? 1 : 0;
39
+ }
40
+ const scaled = (value - minimum) / (maximum - minimum);
41
+ return scaled < 0 ? 0 : scaled > 1 ? 1 : scaled;
42
+ };
@@ -1 +1,17 @@
1
- export const estimateTokenCount = (content) => Math.ceil(content.trim().split(/\s+/).filter(Boolean).join(' ').length / 4);
1
+ // Subword tokenizers emit roughly 1.3 tokens per whitespace word for natural
2
+ // language. Pure prose tracks that ratio; punctuation- or symbol-dense content
3
+ // (code, URLs, long unbroken strings) tokenizes far denser, which the word
4
+ // count alone underestimates. Taking the max with the character heuristic keeps
5
+ // the estimate conservative — never below chars/4 — so callers that budget
6
+ // against it can rely on the rendered output staying within the token limit.
7
+ const tokensPerWord = 1.3;
8
+ const charactersPerToken = 4;
9
+ export const estimateTokenCount = (content) => {
10
+ const words = content.trim().split(/\s+/).filter(Boolean);
11
+ if (words.length === 0) {
12
+ return 0;
13
+ }
14
+ const characterEstimate = Math.ceil(words.join(' ').length / charactersPerToken);
15
+ const wordEstimate = Math.ceil(words.length * tokensPerWord);
16
+ return Math.max(characterEstimate, wordEstimate);
17
+ };
@@ -18,8 +18,10 @@ export const defaultBrainlinkConfig = {
18
18
  defaultContextStrategy: 'auto',
19
19
  defaultContextCacheTtlMs: 120_000,
20
20
  embeddingProvider: 'local',
21
+ embedding: {},
21
22
  defaultSearchMode: 'hybrid',
22
23
  chunkSize: 1200,
24
+ storageBackend: 'json',
23
25
  searchPack: {
24
26
  rowChunkSize: 5_000,
25
27
  compressionLevel: 5,
@@ -43,10 +45,36 @@ const safeCwd = () => {
43
45
  }
44
46
  };
45
47
  const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
46
- const embeddingProviders = new Set(['none', 'local']);
48
+ const embeddingProviders = new Set(['none', 'local', 'ollama', 'openai']);
47
49
  const searchModes = new Set(['fts', 'semantic', 'hybrid']);
48
50
  const contextStrategies = new Set(['rag', 'cag', 'auto']);
51
+ const storageBackends = new Set(['json', 'binary', 'lmdb']);
52
+ const sanitizeStorageBackend = (value) => typeof value === 'string' && storageBackends.has(value) ? value : defaultBrainlinkConfig.storageBackend;
49
53
  const sanitizeEmbeddingProvider = (value) => typeof value === 'string' && embeddingProviders.has(value) ? value : defaultBrainlinkConfig.embeddingProvider;
54
+ const sanitizeOptionalString = (value) => typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
55
+ const sanitizeEmbeddingConfig = (value) => {
56
+ if (!isRecord(value)) {
57
+ return {};
58
+ }
59
+ const url = sanitizeOptionalString(value.url);
60
+ const model = sanitizeOptionalString(value.model);
61
+ const apiKeyEnv = sanitizeOptionalString(value.apiKeyEnv);
62
+ const timeoutMs = sanitizePositiveNumber(value.timeoutMs);
63
+ const batchSize = sanitizePositiveNumber(value.batchSize);
64
+ return {
65
+ ...(url ? { url } : {}),
66
+ ...(model ? { model } : {}),
67
+ ...(apiKeyEnv ? { apiKeyEnv } : {}),
68
+ ...(timeoutMs ? { timeoutMs: Math.floor(timeoutMs) } : {}),
69
+ ...(batchSize ? { batchSize: Math.floor(batchSize) } : {})
70
+ };
71
+ };
72
+ // Stable identity of the embedding space, used to decide when stored vectors
73
+ // must be rebuilt. A model swap on an external provider changes the space even
74
+ // though the provider name is unchanged, so the model is folded in.
75
+ export const embeddingSignature = (config) => config.embeddingProvider === 'ollama' || config.embeddingProvider === 'openai'
76
+ ? `${config.embeddingProvider}:${config.embedding.model?.trim() || 'default'}`
77
+ : config.embeddingProvider;
50
78
  export const sanitizeSearchMode = (value, fallback = defaultBrainlinkConfig.defaultSearchMode) => typeof value === 'string' && searchModes.has(value) ? value : fallback;
51
79
  export const sanitizeContextStrategy = (value, fallback = defaultBrainlinkConfig.defaultContextStrategy) => typeof value === 'string' && contextStrategies.has(value) ? value : fallback;
52
80
  const sanitizeAllowedVaults = (value) => Array.isArray(value) ? value.filter((item) => typeof item === 'string' && item.trim().length > 0) : [];
@@ -195,8 +223,10 @@ const sanitizeConfig = (value) => ({
195
223
  : defaultBrainlinkConfig.defaultContextCacheTtlMs,
196
224
  allowedVaults: [...sanitizeAllowedVaults(value.allowedVaults), ...readAllowedVaultsFromEnv()],
197
225
  chunkSize: typeof value.chunkSize === 'number' && value.chunkSize > 0 ? value.chunkSize : defaultBrainlinkConfig.chunkSize,
226
+ storageBackend: sanitizeStorageBackend(value.storageBackend),
198
227
  searchPack: sanitizeSearchPackConfig(value.searchPack),
199
228
  embeddingProvider: sanitizeEmbeddingProvider(value.embeddingProvider),
229
+ embedding: sanitizeEmbeddingConfig(value.embedding),
200
230
  defaultSearchMode: sanitizeSearchMode(value.defaultSearchMode),
201
231
  agentProfiles: sanitizeAgentProfiles(value.agentProfiles)
202
232
  });
@@ -1,19 +1,55 @@
1
1
  import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
2
2
  import { dirname, join } from 'node:path';
3
- import { createEmbeddingBuckets, dotProduct } from '../domain/embeddings.js';
4
- import { selectSemanticCandidates } from './semantic-prefilter.js';
5
- const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
3
+ import { createStoredIndexQueries, emptyIndex, toStoredIndex } from './knowledge-store/stored-index.js';
6
4
  const indexCacheMaxEntries = 16;
7
5
  const indexCache = new Map();
8
- const emptyIndex = () => ({
9
- version: 1,
10
- updatedAt: new Date().toISOString(),
11
- documents: [],
12
- chunks: [],
13
- links: []
14
- });
6
+ // Raised when index.json exists but cannot be parsed. Search treats it as a
7
+ // signal to fall back to the independent compressed search packs instead of
8
+ // silently returning an empty result set.
9
+ export class IndexCorruptionError extends Error {
10
+ indexPath;
11
+ constructor(indexPath, cause) {
12
+ super(`Corrupted index at ${indexPath}: ${cause instanceof Error ? cause.message : String(cause)}`);
13
+ this.indexPath = indexPath;
14
+ this.name = 'IndexCorruptionError';
15
+ }
16
+ }
15
17
  export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
16
- const readIndex = async (vaultPath) => {
18
+ const isMissingFileError = (error) => error instanceof Error && 'code' in error && error.code === 'ENOENT';
19
+ // Paths already reported as corrupt, so a non-removable corrupt index (read-only
20
+ // vault, locked file) does not spam stderr on every read. Cleared when a fresh
21
+ // index is written to the path.
22
+ const warnedCorruptIndexPaths = new Set();
23
+ // Move a corrupt index aside so it is not read again and the next index run
24
+ // rebuilds from Markdown source, and surface a warning instead of failing
25
+ // silently. Quarantine is best-effort: a read-only vault simply keeps the file.
26
+ const quarantineCorruptIndex = async (path, cause) => {
27
+ indexCache.delete(path);
28
+ if (!warnedCorruptIndexPaths.has(path)) {
29
+ warnedCorruptIndexPaths.add(path);
30
+ process.stderr.write(`brainlink: index at ${path} is unreadable and was quarantined; rebuild with 'index --full' (${cause instanceof Error ? cause.message : String(cause)})\n`);
31
+ }
32
+ try {
33
+ await rename(path, `${path}.corrupt`);
34
+ }
35
+ catch {
36
+ // The primary name can already exist (a prior corruption on a filesystem
37
+ // where rename does not overwrite, e.g. Windows). Fall back to a unique name
38
+ // so the damaged file is still moved out of the read path.
39
+ try {
40
+ await rename(path, `${path}.corrupt-${Date.now()}`);
41
+ }
42
+ catch {
43
+ // Leave the file in place when it cannot be renamed; the warning still fired.
44
+ }
45
+ }
46
+ };
47
+ // Exposed so the binary backend can serve reads from a legacy index.json until
48
+ // the next index run materializes the binary store (seamless migration).
49
+ export function readJsonStoredIndex(vaultPath, strict = false) {
50
+ return readIndex(vaultPath, strict);
51
+ }
52
+ const readIndex = async (vaultPath, strict = false) => {
17
53
  const path = indexStoragePath(vaultPath);
18
54
  let stats = null;
19
55
  try {
@@ -21,9 +57,8 @@ const readIndex = async (vaultPath) => {
21
57
  stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
22
58
  }
23
59
  catch (error) {
24
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
60
+ if (isMissingFileError(error)) {
25
61
  indexCache.delete(path);
26
- return emptyIndex();
27
62
  }
28
63
  return emptyIndex();
29
64
  }
@@ -31,31 +66,46 @@ const readIndex = async (vaultPath) => {
31
66
  if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
32
67
  return cached.index;
33
68
  }
69
+ let raw;
34
70
  try {
35
- const parsed = JSON.parse(await readFile(path, 'utf8'));
36
- const loaded = {
37
- version: 1,
38
- updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
39
- documents: Array.isArray(parsed.documents) ? parsed.documents : [],
40
- chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
41
- links: Array.isArray(parsed.links) ? parsed.links : []
42
- };
43
- indexCache.set(path, { ...stats, index: loaded });
44
- if (indexCache.size > indexCacheMaxEntries) {
45
- const oldest = indexCache.keys().next().value;
46
- if (typeof oldest === 'string') {
47
- indexCache.delete(oldest);
48
- }
71
+ raw = await readFile(path, 'utf8');
72
+ }
73
+ catch (error) {
74
+ indexCache.delete(path);
75
+ // The file existed at stat but vanished before the read — a concurrent
76
+ // rebuild or quarantine. Strict (search) callers throw so they fall back to
77
+ // packs rather than racing to an empty result; tolerant callers get empty.
78
+ if (strict) {
79
+ throw new IndexCorruptionError(path, error);
49
80
  }
50
- return loaded;
81
+ return emptyIndex();
82
+ }
83
+ let parsed;
84
+ try {
85
+ parsed = JSON.parse(raw);
51
86
  }
52
87
  catch (error) {
53
- if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
54
- indexCache.delete(path);
55
- return emptyIndex();
88
+ await quarantineCorruptIndex(path, error);
89
+ if (strict) {
90
+ throw new IndexCorruptionError(path, error);
56
91
  }
57
92
  return emptyIndex();
58
93
  }
94
+ const loaded = {
95
+ version: 1,
96
+ updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
97
+ documents: Array.isArray(parsed.documents) ? parsed.documents : [],
98
+ chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
99
+ links: Array.isArray(parsed.links) ? parsed.links : []
100
+ };
101
+ indexCache.set(path, { ...stats, index: loaded });
102
+ if (indexCache.size > indexCacheMaxEntries) {
103
+ const oldest = indexCache.keys().next().value;
104
+ if (typeof oldest === 'string') {
105
+ indexCache.delete(oldest);
106
+ }
107
+ }
108
+ return loaded;
59
109
  };
60
110
  const writeIndex = async (vaultPath, index) => {
61
111
  const target = indexStoragePath(vaultPath);
@@ -63,6 +113,7 @@ const writeIndex = async (vaultPath, index) => {
63
113
  await mkdir(dirname(target), { recursive: true, mode: 0o700 });
64
114
  await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
65
115
  await rename(temp, target);
116
+ warnedCorruptIndexPaths.delete(target);
66
117
  const fileStats = await stat(target);
67
118
  indexCache.set(target, {
68
119
  mtimeMs: fileStats.mtimeMs,
@@ -70,301 +121,21 @@ const writeIndex = async (vaultPath, index) => {
70
121
  index
71
122
  });
72
123
  };
73
- const normalizeToken = (value) => value
74
- .normalize('NFKD')
75
- .replace(/\p{Diacritic}/gu, '')
76
- .toLowerCase();
77
- const tokenize = (query) => query
78
- .match(queryTokenPattern)
79
- ?.map(normalizeToken)
80
- .filter((token) => token.length > 1) ?? [];
81
- const countOccurrences = (text, token) => {
82
- let hits = 0;
83
- let cursor = 0;
84
- while (cursor < text.length) {
85
- const index = text.indexOf(token, cursor);
86
- if (index < 0) {
87
- break;
88
- }
89
- hits += 1;
90
- cursor = index + token.length;
91
- }
92
- return hits;
93
- };
94
- const textScore = (row, tokens) => {
95
- if (tokens.length === 0) {
96
- return 0;
124
+ // JSON file backend: the original single `index.json` store. Loading/persisting
125
+ // is its own concern; the graph/search/link reads come from the shared query
126
+ // core so this backend stays behavior-identical to the binary backend.
127
+ export const openFileIndex = (vaultPath) => ({
128
+ reset: async () => {
129
+ await writeIndex(vaultPath, emptyIndex());
130
+ },
131
+ saveDocuments: async (documents) => {
132
+ await writeIndex(vaultPath, toStoredIndex(documents));
133
+ },
134
+ ...createStoredIndexQueries({
135
+ load: () => readIndex(vaultPath),
136
+ loadStrict: () => readIndex(vaultPath, true)
137
+ }),
138
+ close: () => {
139
+ // File-based index has no persistent connection.
97
140
  }
98
- const title = normalizeToken(row.title);
99
- const path = normalizeToken(row.path);
100
- const content = normalizeToken(row.content);
101
- const tags = normalizeToken(row.tags.join(' '));
102
- return tokens.reduce((score, token) => {
103
- const titleHits = countOccurrences(title, token);
104
- const tagHits = countOccurrences(tags, token);
105
- const pathHits = countOccurrences(path, token);
106
- const contentHits = countOccurrences(content, token);
107
- return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 6);
108
- }, 0);
109
- };
110
- const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? dotProduct(queryEmbedding, row.embedding) : 0;
111
- const toResult = (row, mode, text, semantic) => {
112
- const score = mode === 'fts' ? text : mode === 'semantic' ? semantic : text + semantic * 8;
113
- return {
114
- documentId: row.documentId,
115
- agentId: row.agentId,
116
- title: row.title,
117
- path: row.path,
118
- chunkId: row.chunkId,
119
- chunkOrdinal: row.chunkOrdinal,
120
- content: row.content,
121
- score,
122
- textScore: text,
123
- semanticScore: semantic,
124
- searchMode: mode,
125
- tags: row.tags
126
- };
127
- };
128
- const toGraphLink = (link, documentsById) => {
129
- const source = documentsById.get(link.fromDocumentId);
130
- const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
131
- return {
132
- agentId: source?.agentId ?? 'shared',
133
- fromTitle: source?.title ?? 'Unknown',
134
- fromPath: source?.path ?? 'Unknown',
135
- toTitle: target?.title ?? link.toTitle,
136
- toPath: target?.path ?? null,
137
- weight: link.weight,
138
- priority: link.priority
139
- };
140
- };
141
- export const openFileIndex = (vaultPath) => {
142
- const load = async () => readIndex(vaultPath);
143
- const persist = async (index) => writeIndex(vaultPath, index);
144
- return {
145
- reset: async () => {
146
- await persist(emptyIndex());
147
- },
148
- saveDocuments: async (documents) => {
149
- const chunks = documents.flatMap((document) => document.chunks);
150
- const links = documents.flatMap((document) => document.links);
151
- await persist({
152
- version: 1,
153
- updatedAt: new Date().toISOString(),
154
- documents: documents.map((document) => document.document),
155
- chunks,
156
- links
157
- });
158
- },
159
- getIndexedDocuments: async (agentId) => {
160
- const index = await load();
161
- const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
162
- const selectedDocumentIds = new Set(documents.map((document) => document.id));
163
- const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
164
- if (!selectedDocumentIds.has(chunk.documentId)) {
165
- return state;
166
- }
167
- const current = state.get(chunk.documentId) ?? [];
168
- current.push(chunk);
169
- state.set(chunk.documentId, current);
170
- return state;
171
- }, new Map());
172
- const linksByDocumentId = index.links.reduce((state, link) => {
173
- if (!selectedDocumentIds.has(link.fromDocumentId)) {
174
- return state;
175
- }
176
- const current = state.get(link.fromDocumentId) ?? [];
177
- current.push(link);
178
- state.set(link.fromDocumentId, current);
179
- return state;
180
- }, new Map());
181
- return documents
182
- .map((document) => ({
183
- document,
184
- chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
185
- links: linksByDocumentId.get(document.id) ?? []
186
- }))
187
- .sort((left, right) => left.document.path.localeCompare(right.document.path));
188
- },
189
- search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
190
- const index = await load();
191
- const documentsById = new Map(index.documents.map((document) => [document.id, document]));
192
- const rows = index.chunks.flatMap((chunk) => {
193
- const document = documentsById.get(chunk.documentId);
194
- if (!document) {
195
- return [];
196
- }
197
- if (agentId && document.agentId !== agentId) {
198
- return [];
199
- }
200
- return [
201
- {
202
- documentId: document.id,
203
- agentId: document.agentId,
204
- title: document.title,
205
- path: document.path,
206
- chunkId: chunk.id,
207
- chunkOrdinal: chunk.ordinal,
208
- content: chunk.content,
209
- tags: document.tags,
210
- embedding: chunk.embedding,
211
- buckets: chunk.buckets
212
- }
213
- ];
214
- });
215
- const tokens = tokenize(query);
216
- // Pure-semantic scoring on a large vault can skip chunks that share no
217
- // embedding bucket with the query. Hybrid/fts keep every row so lexical
218
- // matches are never dropped; the prefilter also falls back to a full scan
219
- // on small or partially-indexed vaults.
220
- const scored = mode === 'semantic' && queryEmbedding.length > 0
221
- ? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
222
- : rows;
223
- const results = scored
224
- .map((row) => {
225
- const text = textScore(row, tokens);
226
- const semantic = semanticScore(row, queryEmbedding);
227
- return toResult(row, mode, text, semantic);
228
- })
229
- .filter((row) => row.score > 0 || tokens.length === 0)
230
- .sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
231
- .slice(0, Math.max(0, limit));
232
- return results;
233
- },
234
- listLinks: async (agentId) => {
235
- const index = await load();
236
- const documentsById = new Map(index.documents.map((document) => [document.id, document]));
237
- return index.links
238
- .filter((link) => {
239
- const source = documentsById.get(link.fromDocumentId);
240
- return agentId ? source?.agentId === agentId : true;
241
- })
242
- .map((link) => toGraphLink(link, documentsById))
243
- .sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
244
- },
245
- listBacklinks: async (title, agentId) => {
246
- const index = await load();
247
- const titleKey = title.toLowerCase();
248
- const documentsById = new Map(index.documents.map((document) => [document.id, document]));
249
- return index.links
250
- .filter((link) => link.toTitle.toLowerCase() === titleKey)
251
- .filter((link) => {
252
- const source = documentsById.get(link.fromDocumentId);
253
- return agentId ? source?.agentId === agentId : true;
254
- })
255
- .map((link) => toGraphLink(link, documentsById))
256
- .sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
257
- },
258
- getGraph: async (agentId) => {
259
- const index = await load();
260
- const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
261
- const documentIds = new Set(documents.map((document) => document.id));
262
- const edges = index.links
263
- .filter((link) => documentIds.has(link.fromDocumentId))
264
- .map((link) => ({
265
- source: link.fromDocumentId,
266
- target: link.toDocumentId,
267
- targetTitle: link.toTitle,
268
- weight: link.weight,
269
- priority: link.priority
270
- }));
271
- return {
272
- nodes: documents.map((document) => ({
273
- id: document.id,
274
- agentId: document.agentId,
275
- title: document.title,
276
- path: document.path,
277
- content: document.content,
278
- tags: document.tags,
279
- contextLinks: document.contextLinks ?? []
280
- })),
281
- edges
282
- };
283
- },
284
- getGraphSummary: async (agentId) => {
285
- const graph = await (async () => {
286
- const index = await load();
287
- const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
288
- const documentIds = new Set(documents.map((document) => document.id));
289
- const edges = index.links
290
- .filter((link) => documentIds.has(link.fromDocumentId))
291
- .map((link) => ({
292
- source: link.fromDocumentId,
293
- target: link.toDocumentId,
294
- targetTitle: link.toTitle,
295
- weight: link.weight,
296
- priority: link.priority
297
- }));
298
- return {
299
- nodes: documents.map((document) => ({
300
- id: document.id,
301
- agentId: document.agentId,
302
- title: document.title,
303
- path: document.path,
304
- content: '',
305
- tags: document.tags,
306
- contextLinks: document.contextLinks ?? []
307
- })),
308
- edges
309
- };
310
- })();
311
- return graph;
312
- },
313
- getGraphNode: async (id, agentId) => {
314
- const index = await load();
315
- const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
316
- return document
317
- ? {
318
- id: document.id,
319
- agentId: document.agentId,
320
- title: document.title,
321
- path: document.path,
322
- content: document.content,
323
- tags: document.tags,
324
- contextLinks: document.contextLinks ?? []
325
- }
326
- : undefined;
327
- },
328
- searchGraphNodeIds: async (query, limit, agentId) => {
329
- const index = await load();
330
- const normalized = normalizeToken(query);
331
- if (normalized.length === 0 || limit <= 0) {
332
- return [];
333
- }
334
- const tokens = tokenize(query);
335
- const scored = index.documents
336
- .filter((document) => (!agentId || document.agentId === agentId))
337
- .map((document) => {
338
- const score = textScore({
339
- documentId: document.id,
340
- agentId: document.agentId,
341
- title: document.title,
342
- path: document.path,
343
- chunkId: document.id,
344
- chunkOrdinal: 0,
345
- content: document.content,
346
- tags: document.tags,
347
- embedding: []
348
- }, tokens);
349
- return { id: document.id, score };
350
- })
351
- .filter((row) => row.score > 0)
352
- .sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
353
- .slice(0, limit);
354
- return scored.map((row) => row.id);
355
- },
356
- listAgents: async () => {
357
- const index = await load();
358
- const counts = index.documents.reduce((state, document) => {
359
- state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
360
- return state;
361
- }, new Map());
362
- return Array.from(counts.entries())
363
- .sort((left, right) => left[0].localeCompare(right[0]))
364
- .map(([id, documentCount]) => ({ id, documentCount }));
365
- },
366
- close: () => {
367
- // File-based index has no persistent connection.
368
- }
369
- };
370
- };
141
+ });