@andespindola/brainlink 1.0.7 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/README.md +35 -0
- package/dist/application/get-graph-node.js +2 -2
- package/dist/application/get-graph-summary.js +2 -2
- package/dist/application/get-graph.js +2 -2
- package/dist/application/index-vault.js +2 -2
- package/dist/application/list-agents.js +2 -2
- package/dist/application/list-links.js +3 -3
- package/dist/application/ports/knowledge-store.js +1 -0
- package/dist/application/ranking/run-search.js +177 -0
- package/dist/application/search-graph-node-ids.js +3 -2
- package/dist/application/search-knowledge.js +3 -2
- package/dist/application/vault-git.js +180 -0
- package/dist/application/vault-snapshot.js +87 -0
- package/dist/cli/commands/vault-sync-commands.js +115 -0
- package/dist/cli/main.js +2 -0
- package/dist/infrastructure/config.js +4 -0
- package/dist/infrastructure/file-index.js +22 -409
- package/dist/infrastructure/knowledge-store/binary-store.js +163 -0
- package/dist/infrastructure/knowledge-store/index.js +36 -0
- package/dist/infrastructure/knowledge-store/stored-index.js +216 -0
- package/package.json +3 -2
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { cloneVault, commitVault, initVaultGit, pullVault, pushVault, restoreVault, vaultGitStatus, vaultHistory } from '../../application/vault-git.js';
|
|
2
|
+
import { exportVaultSnapshot, importVaultSnapshot } from '../../application/vault-snapshot.js';
|
|
3
|
+
import { print, resolveOptions } from '../runtime.js';
|
|
4
|
+
export const registerVaultSyncCommands = (program) => {
|
|
5
|
+
program
|
|
6
|
+
.command('vault-init-git')
|
|
7
|
+
.option('--vault <vault>', 'vault directory')
|
|
8
|
+
.option('--remote <url>', 'git remote URL (SSH or credential-helper backed; inline tokens are rejected)')
|
|
9
|
+
.option('--json', 'print machine-readable JSON')
|
|
10
|
+
.description('initialize git versioning for the vault (ignores the derived .brainlink/)')
|
|
11
|
+
.action(async (options) => {
|
|
12
|
+
const { vault } = await resolveOptions(options);
|
|
13
|
+
const result = await initVaultGit(vault, { remote: options.remote });
|
|
14
|
+
print(options.json, result, () => `${result.initialized ? 'Initialized' : 'Reused'} git repo at ${result.root}${options.remote ? ` (origin: ${options.remote})` : ''}.`);
|
|
15
|
+
});
|
|
16
|
+
program
|
|
17
|
+
.command('vault-commit')
|
|
18
|
+
.requiredOption('-m, --message <message>', 'commit message')
|
|
19
|
+
.option('--vault <vault>', 'vault directory')
|
|
20
|
+
.option('--json', 'print machine-readable JSON')
|
|
21
|
+
.description('commit the vault Markdown')
|
|
22
|
+
.action(async (options) => {
|
|
23
|
+
const { vault } = await resolveOptions(options);
|
|
24
|
+
const hash = await commitVault(vault, options.message ?? 'Update vault');
|
|
25
|
+
print(options.json, { committed: hash != null, hash }, () => hash ? `Committed ${hash.slice(0, 10)}.` : 'Nothing to commit.');
|
|
26
|
+
});
|
|
27
|
+
program
|
|
28
|
+
.command('vault-status')
|
|
29
|
+
.option('--vault <vault>', 'vault directory')
|
|
30
|
+
.option('--json', 'print machine-readable JSON')
|
|
31
|
+
.description('show git status of the vault Markdown')
|
|
32
|
+
.action(async (options) => {
|
|
33
|
+
const { vault } = await resolveOptions(options);
|
|
34
|
+
const status = await vaultGitStatus(vault);
|
|
35
|
+
print(options.json, status, () => `Branch ${status.branch}; ${status.dirty ? `${status.changedMarkdown.length} changed markdown file(s)` : 'clean'}.`);
|
|
36
|
+
});
|
|
37
|
+
program
|
|
38
|
+
.command('vault-history')
|
|
39
|
+
.option('--vault <vault>', 'vault directory')
|
|
40
|
+
.option('--limit <n>', 'number of entries', '20')
|
|
41
|
+
.option('--json', 'print machine-readable JSON')
|
|
42
|
+
.description('show vault commit history')
|
|
43
|
+
.action(async (options) => {
|
|
44
|
+
const { vault } = await resolveOptions(options);
|
|
45
|
+
const entries = await vaultHistory(vault, Number.parseInt(options.limit ?? '20', 10));
|
|
46
|
+
print(options.json, entries, () => entries.map((entry) => `${entry.hash.slice(0, 10)} ${entry.date} ${entry.subject}`).join('\n') || 'No history.');
|
|
47
|
+
});
|
|
48
|
+
program
|
|
49
|
+
.command('vault-push')
|
|
50
|
+
.option('--vault <vault>', 'vault directory')
|
|
51
|
+
.option('--json', 'print machine-readable JSON')
|
|
52
|
+
.description('push the vault to its git remote')
|
|
53
|
+
.action(async (options) => {
|
|
54
|
+
const { vault } = await resolveOptions(options);
|
|
55
|
+
await pushVault(vault);
|
|
56
|
+
print(options.json, { pushed: true }, () => 'Pushed to origin.');
|
|
57
|
+
});
|
|
58
|
+
program
|
|
59
|
+
.command('vault-pull')
|
|
60
|
+
.option('--vault <vault>', 'vault directory')
|
|
61
|
+
.option('--no-index', 'skip reindexing after pull')
|
|
62
|
+
.option('--json', 'print machine-readable JSON')
|
|
63
|
+
.description('pull the vault from git and reindex changed notes')
|
|
64
|
+
.action(async (options) => {
|
|
65
|
+
const { vault } = await resolveOptions(options);
|
|
66
|
+
const result = await pullVault(vault, { skipIndex: options.index === false });
|
|
67
|
+
print(options.json, result, () => `Pulled. ${result.changedMarkdown.length} markdown file(s) changed${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
|
|
68
|
+
});
|
|
69
|
+
program
|
|
70
|
+
.command('vault-restore')
|
|
71
|
+
.argument('<ref>', 'git ref (commit, tag, branch) to restore Markdown from')
|
|
72
|
+
.option('--vault <vault>', 'vault directory')
|
|
73
|
+
.option('--no-index', 'skip reindexing after restore')
|
|
74
|
+
.option('--json', 'print machine-readable JSON')
|
|
75
|
+
.description('restore vault Markdown from a git ref and reindex')
|
|
76
|
+
.action(async (ref, options) => {
|
|
77
|
+
const { vault } = await resolveOptions(options);
|
|
78
|
+
const result = await restoreVault(vault, ref, { skipIndex: options.index === false });
|
|
79
|
+
print(options.json, result, () => `Restored from ${ref}. ${result.changedMarkdown.length} markdown file(s) changed${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
|
|
80
|
+
});
|
|
81
|
+
program
|
|
82
|
+
.command('vault-clone')
|
|
83
|
+
.argument('<remote>', 'git remote URL')
|
|
84
|
+
.argument('<dir>', 'target directory')
|
|
85
|
+
.option('--no-index', 'skip building the index after clone')
|
|
86
|
+
.option('--json', 'print machine-readable JSON')
|
|
87
|
+
.description('clone a remote vault and build its index')
|
|
88
|
+
.action(async (remote, dir, options) => {
|
|
89
|
+
const result = await cloneVault(remote, dir, { skipIndex: options.index === false });
|
|
90
|
+
print(options.json, result, () => `Cloned to ${result.root}${result.index ? `; indexed ${result.index.documentCount} docs` : ''}.`);
|
|
91
|
+
});
|
|
92
|
+
program
|
|
93
|
+
.command('vault-export')
|
|
94
|
+
.requiredOption('-o, --output <file>', 'output .blvault file')
|
|
95
|
+
.option('--vault <vault>', 'vault directory')
|
|
96
|
+
.option('--json', 'print machine-readable JSON')
|
|
97
|
+
.description('export the vault Markdown to a portable .blvault snapshot')
|
|
98
|
+
.action(async (options) => {
|
|
99
|
+
const { vault } = await resolveOptions(options);
|
|
100
|
+
const result = await exportVaultSnapshot(vault, options.output ?? 'vault.blvault');
|
|
101
|
+
print(options.json, result, () => `Exported ${result.fileCount} files to ${result.file} (${result.bytes} bytes).`);
|
|
102
|
+
});
|
|
103
|
+
program
|
|
104
|
+
.command('vault-import')
|
|
105
|
+
.argument('<file>', 'input .blvault file')
|
|
106
|
+
.option('--vault <vault>', 'vault directory')
|
|
107
|
+
.option('--no-index', 'skip reindexing after import')
|
|
108
|
+
.option('--json', 'print machine-readable JSON')
|
|
109
|
+
.description('import a .blvault snapshot into the vault and reindex')
|
|
110
|
+
.action(async (file, options) => {
|
|
111
|
+
const { vault } = await resolveOptions(options);
|
|
112
|
+
const result = await importVaultSnapshot(file, vault, { skipIndex: options.index === false });
|
|
113
|
+
print(options.json, result, () => `Imported ${result.imported} files (${result.conflicted} conflicts)${result.index ? `; reindexed ${result.index.documentCount} docs` : ''}.`);
|
|
114
|
+
});
|
|
115
|
+
};
|
package/dist/cli/main.js
CHANGED
|
@@ -10,6 +10,7 @@ import { registerConfigCommands } from './commands/config-commands.js';
|
|
|
10
10
|
import { registerPracticalCommands } from './commands/practical-commands.js';
|
|
11
11
|
import { registerReadCommands } from './commands/read-commands.js';
|
|
12
12
|
import { registerVaultCommands } from './commands/vault-commands.js';
|
|
13
|
+
import { registerVaultSyncCommands } from './commands/vault-sync-commands.js';
|
|
13
14
|
import { registerWriteCommands } from './commands/write-commands.js';
|
|
14
15
|
const readPackageMetadata = () => {
|
|
15
16
|
const packagePath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
|
|
@@ -63,6 +64,7 @@ registerReadCommands(program);
|
|
|
63
64
|
registerPracticalCommands(program);
|
|
64
65
|
registerConfigCommands(program);
|
|
65
66
|
registerVaultCommands(program);
|
|
67
|
+
registerVaultSyncCommands(program);
|
|
66
68
|
registerAgentCommands(program);
|
|
67
69
|
await maybePrintUpdateNotice(packageMetadata);
|
|
68
70
|
program.parseAsync().catch((error) => {
|
|
@@ -21,6 +21,7 @@ export const defaultBrainlinkConfig = {
|
|
|
21
21
|
embedding: {},
|
|
22
22
|
defaultSearchMode: 'hybrid',
|
|
23
23
|
chunkSize: 1200,
|
|
24
|
+
storageBackend: 'json',
|
|
24
25
|
searchPack: {
|
|
25
26
|
rowChunkSize: 5_000,
|
|
26
27
|
compressionLevel: 5,
|
|
@@ -47,6 +48,8 @@ const isRecord = (value) => typeof value === 'object' && value !== null && !Arra
|
|
|
47
48
|
const embeddingProviders = new Set(['none', 'local', 'ollama', 'openai']);
|
|
48
49
|
const searchModes = new Set(['fts', 'semantic', 'hybrid']);
|
|
49
50
|
const contextStrategies = new Set(['rag', 'cag', 'auto']);
|
|
51
|
+
const storageBackends = new Set(['json', 'binary', 'lmdb']);
|
|
52
|
+
const sanitizeStorageBackend = (value) => typeof value === 'string' && storageBackends.has(value) ? value : defaultBrainlinkConfig.storageBackend;
|
|
50
53
|
const sanitizeEmbeddingProvider = (value) => typeof value === 'string' && embeddingProviders.has(value) ? value : defaultBrainlinkConfig.embeddingProvider;
|
|
51
54
|
const sanitizeOptionalString = (value) => typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
|
52
55
|
const sanitizeEmbeddingConfig = (value) => {
|
|
@@ -220,6 +223,7 @@ const sanitizeConfig = (value) => ({
|
|
|
220
223
|
: defaultBrainlinkConfig.defaultContextCacheTtlMs,
|
|
221
224
|
allowedVaults: [...sanitizeAllowedVaults(value.allowedVaults), ...readAllowedVaultsFromEnv()],
|
|
222
225
|
chunkSize: typeof value.chunkSize === 'number' && value.chunkSize > 0 ? value.chunkSize : defaultBrainlinkConfig.chunkSize,
|
|
226
|
+
storageBackend: sanitizeStorageBackend(value.storageBackend),
|
|
223
227
|
searchPack: sanitizeSearchPackConfig(value.searchPack),
|
|
224
228
|
embeddingProvider: sanitizeEmbeddingProvider(value.embeddingProvider),
|
|
225
229
|
embedding: sanitizeEmbeddingConfig(value.embedding),
|
|
@@ -1,18 +1,8 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import { bm25TermScore, minMaxNormalize } from '../domain/scoring.js';
|
|
5
|
-
import { selectSemanticCandidates } from './semantic-prefilter.js';
|
|
6
|
-
const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
|
|
3
|
+
import { createStoredIndexQueries, emptyIndex, toStoredIndex } from './knowledge-store/stored-index.js';
|
|
7
4
|
const indexCacheMaxEntries = 16;
|
|
8
5
|
const indexCache = new Map();
|
|
9
|
-
const emptyIndex = () => ({
|
|
10
|
-
version: 1,
|
|
11
|
-
updatedAt: new Date().toISOString(),
|
|
12
|
-
documents: [],
|
|
13
|
-
chunks: [],
|
|
14
|
-
links: []
|
|
15
|
-
});
|
|
16
6
|
// Raised when index.json exists but cannot be parsed. Search treats it as a
|
|
17
7
|
// signal to fall back to the independent compressed search packs instead of
|
|
18
8
|
// silently returning an empty result set.
|
|
@@ -54,6 +44,11 @@ const quarantineCorruptIndex = async (path, cause) => {
|
|
|
54
44
|
}
|
|
55
45
|
}
|
|
56
46
|
};
|
|
47
|
+
// Exposed so the binary backend can serve reads from a legacy index.json until
|
|
48
|
+
// the next index run materializes the binary store (seamless migration).
|
|
49
|
+
export function readJsonStoredIndex(vaultPath, strict = false) {
|
|
50
|
+
return readIndex(vaultPath, strict);
|
|
51
|
+
}
|
|
57
52
|
const readIndex = async (vaultPath, strict = false) => {
|
|
58
53
|
const path = indexStoragePath(vaultPath);
|
|
59
54
|
let stats = null;
|
|
@@ -126,403 +121,21 @@ const writeIndex = async (vaultPath, index) => {
|
|
|
126
121
|
index
|
|
127
122
|
});
|
|
128
123
|
};
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
hits += 1;
|
|
146
|
-
cursor = index + token.length;
|
|
124
|
+
// JSON file backend: the original single `index.json` store. Loading/persisting
|
|
125
|
+
// is its own concern; the graph/search/link reads come from the shared query
|
|
126
|
+
// core so this backend stays behavior-identical to the binary backend.
|
|
127
|
+
export const openFileIndex = (vaultPath) => ({
|
|
128
|
+
reset: async () => {
|
|
129
|
+
await writeIndex(vaultPath, emptyIndex());
|
|
130
|
+
},
|
|
131
|
+
saveDocuments: async (documents) => {
|
|
132
|
+
await writeIndex(vaultPath, toStoredIndex(documents));
|
|
133
|
+
},
|
|
134
|
+
...createStoredIndexQueries({
|
|
135
|
+
load: () => readIndex(vaultPath),
|
|
136
|
+
loadStrict: () => readIndex(vaultPath, true)
|
|
137
|
+
}),
|
|
138
|
+
close: () => {
|
|
139
|
+
// File-based index has no persistent connection.
|
|
147
140
|
}
|
|
148
|
-
return hits;
|
|
149
|
-
};
|
|
150
|
-
const titleFieldWeight = 5;
|
|
151
|
-
const tagFieldWeight = 4;
|
|
152
|
-
const pathFieldWeight = 2;
|
|
153
|
-
const contentHitCap = 6;
|
|
154
|
-
const hybridTextWeight = 0.6;
|
|
155
|
-
const hybridSemanticWeight = 0.4;
|
|
156
|
-
const normalizeFields = (row) => ({
|
|
157
|
-
title: normalizeToken(row.title),
|
|
158
|
-
path: normalizeToken(row.path),
|
|
159
|
-
content: normalizeToken(row.content),
|
|
160
|
-
tags: normalizeToken(row.tags.join(' '))
|
|
161
141
|
});
|
|
162
|
-
// Field-weighted term frequency: title/tag/path matches count more than body
|
|
163
|
-
// matches, and body hits saturate at a cap so a term repeated many times in one
|
|
164
|
-
// chunk cannot dominate. This weighted frequency feeds BM25 below.
|
|
165
|
-
const weightedTermFrequency = (fields, token) => {
|
|
166
|
-
const titleHits = countOccurrences(fields.title, token);
|
|
167
|
-
const tagHits = countOccurrences(fields.tags, token);
|
|
168
|
-
const pathHits = countOccurrences(fields.path, token);
|
|
169
|
-
const contentHits = countOccurrences(fields.content, token);
|
|
170
|
-
return (titleHits * titleFieldWeight +
|
|
171
|
-
tagHits * tagFieldWeight +
|
|
172
|
-
pathHits * pathFieldWeight +
|
|
173
|
-
Math.min(contentHits, contentHitCap));
|
|
174
|
-
};
|
|
175
|
-
// Naive additive field score retained for graph-node title/path lookup, where
|
|
176
|
-
// the candidate set is documents (not chunks) and corpus-wide BM25 statistics
|
|
177
|
-
// add no signal over a direct weighted hit count.
|
|
178
|
-
const lexicalFieldScore = (row, tokens) => {
|
|
179
|
-
if (tokens.length === 0) {
|
|
180
|
-
return 0;
|
|
181
|
-
}
|
|
182
|
-
const fields = normalizeFields(row);
|
|
183
|
-
return tokens.reduce((score, token) => score + weightedTermFrequency(fields, token), 0);
|
|
184
|
-
};
|
|
185
|
-
// A single chunk with a non-finite cached tokenCount (legacy index, partial
|
|
186
|
-
// write, manual edit) would otherwise poison the shared averageLength and zero
|
|
187
|
-
// out lexical scoring for the whole corpus, so coerce to a safe positive length.
|
|
188
|
-
const safeDocumentLength = (tokenCount) => Number.isFinite(tokenCount) && tokenCount > 0 ? tokenCount : 1;
|
|
189
|
-
const buildCorpusStatistics = (prepared, tokens) => {
|
|
190
|
-
const documentCount = prepared.length;
|
|
191
|
-
const totalLength = prepared.reduce((sum, entry) => sum + safeDocumentLength(entry.row.tokenCount), 0);
|
|
192
|
-
const documentFrequencyByTerm = new Map();
|
|
193
|
-
for (const term of tokens) {
|
|
194
|
-
let documentFrequency = 0;
|
|
195
|
-
for (const entry of prepared) {
|
|
196
|
-
if ((entry.weightedTermFrequencies.get(term) ?? 0) > 0) {
|
|
197
|
-
documentFrequency += 1;
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
documentFrequencyByTerm.set(term, documentFrequency);
|
|
201
|
-
}
|
|
202
|
-
return {
|
|
203
|
-
documentCount,
|
|
204
|
-
averageLength: documentCount > 0 ? totalLength / documentCount : 0,
|
|
205
|
-
documentFrequencyByTerm
|
|
206
|
-
};
|
|
207
|
-
};
|
|
208
|
-
const bm25TextScore = (entry, tokens, stats) => tokens.reduce((score, term) => score +
|
|
209
|
-
bm25TermScore({
|
|
210
|
-
termFrequency: entry.weightedTermFrequencies.get(term) ?? 0,
|
|
211
|
-
documentLength: safeDocumentLength(entry.row.tokenCount),
|
|
212
|
-
documentCount: stats.documentCount,
|
|
213
|
-
documentFrequency: stats.documentFrequencyByTerm.get(term) ?? 0,
|
|
214
|
-
averageLength: stats.averageLength
|
|
215
|
-
}), 0);
|
|
216
|
-
const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? dotProduct(queryEmbedding, row.embedding) : 0;
|
|
217
|
-
const rangeOf = (values) => values.reduce((range, value) => ({ min: Math.min(range.min, value), max: Math.max(range.max, value) }), { min: Infinity, max: -Infinity });
|
|
218
|
-
const toResult = (row, mode, score, text, semantic) => ({
|
|
219
|
-
documentId: row.documentId,
|
|
220
|
-
agentId: row.agentId,
|
|
221
|
-
title: row.title,
|
|
222
|
-
path: row.path,
|
|
223
|
-
chunkId: row.chunkId,
|
|
224
|
-
chunkOrdinal: row.chunkOrdinal,
|
|
225
|
-
content: row.content,
|
|
226
|
-
score,
|
|
227
|
-
textScore: text,
|
|
228
|
-
semanticScore: semantic,
|
|
229
|
-
tokenCount: row.tokenCount,
|
|
230
|
-
searchMode: mode,
|
|
231
|
-
tags: row.tags
|
|
232
|
-
});
|
|
233
|
-
const toGraphLink = (link, documentsById) => {
|
|
234
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
235
|
-
const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
|
|
236
|
-
return {
|
|
237
|
-
agentId: source?.agentId ?? 'shared',
|
|
238
|
-
fromTitle: source?.title ?? 'Unknown',
|
|
239
|
-
fromPath: source?.path ?? 'Unknown',
|
|
240
|
-
toTitle: target?.title ?? link.toTitle,
|
|
241
|
-
toPath: target?.path ?? null,
|
|
242
|
-
weight: link.weight,
|
|
243
|
-
priority: link.priority
|
|
244
|
-
};
|
|
245
|
-
};
|
|
246
|
-
export const openFileIndex = (vaultPath) => {
|
|
247
|
-
const load = async () => readIndex(vaultPath);
|
|
248
|
-
const persist = async (index) => writeIndex(vaultPath, index);
|
|
249
|
-
return {
|
|
250
|
-
reset: async () => {
|
|
251
|
-
await persist(emptyIndex());
|
|
252
|
-
},
|
|
253
|
-
saveDocuments: async (documents) => {
|
|
254
|
-
const chunks = documents.flatMap((document) => document.chunks);
|
|
255
|
-
const links = documents.flatMap((document) => document.links);
|
|
256
|
-
await persist({
|
|
257
|
-
version: 1,
|
|
258
|
-
updatedAt: new Date().toISOString(),
|
|
259
|
-
documents: documents.map((document) => document.document),
|
|
260
|
-
chunks,
|
|
261
|
-
links
|
|
262
|
-
});
|
|
263
|
-
},
|
|
264
|
-
getIndexedDocuments: async (agentId) => {
|
|
265
|
-
const index = await load();
|
|
266
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
267
|
-
const selectedDocumentIds = new Set(documents.map((document) => document.id));
|
|
268
|
-
const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
|
|
269
|
-
if (!selectedDocumentIds.has(chunk.documentId)) {
|
|
270
|
-
return state;
|
|
271
|
-
}
|
|
272
|
-
const current = state.get(chunk.documentId) ?? [];
|
|
273
|
-
current.push(chunk);
|
|
274
|
-
state.set(chunk.documentId, current);
|
|
275
|
-
return state;
|
|
276
|
-
}, new Map());
|
|
277
|
-
const linksByDocumentId = index.links.reduce((state, link) => {
|
|
278
|
-
if (!selectedDocumentIds.has(link.fromDocumentId)) {
|
|
279
|
-
return state;
|
|
280
|
-
}
|
|
281
|
-
const current = state.get(link.fromDocumentId) ?? [];
|
|
282
|
-
current.push(link);
|
|
283
|
-
state.set(link.fromDocumentId, current);
|
|
284
|
-
return state;
|
|
285
|
-
}, new Map());
|
|
286
|
-
return documents
|
|
287
|
-
.map((document) => ({
|
|
288
|
-
document,
|
|
289
|
-
chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
|
|
290
|
-
links: linksByDocumentId.get(document.id) ?? []
|
|
291
|
-
}))
|
|
292
|
-
.sort((left, right) => left.document.path.localeCompare(right.document.path));
|
|
293
|
-
},
|
|
294
|
-
search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
|
|
295
|
-
// Strict load: a corrupt index throws so the caller can fall back to the
|
|
296
|
-
// independent search packs rather than reporting "no results".
|
|
297
|
-
const index = await readIndex(vaultPath, true);
|
|
298
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
299
|
-
const rows = index.chunks.flatMap((chunk) => {
|
|
300
|
-
const document = documentsById.get(chunk.documentId);
|
|
301
|
-
if (!document) {
|
|
302
|
-
return [];
|
|
303
|
-
}
|
|
304
|
-
if (agentId && document.agentId !== agentId) {
|
|
305
|
-
return [];
|
|
306
|
-
}
|
|
307
|
-
return [
|
|
308
|
-
{
|
|
309
|
-
documentId: document.id,
|
|
310
|
-
agentId: document.agentId,
|
|
311
|
-
title: document.title,
|
|
312
|
-
path: document.path,
|
|
313
|
-
chunkId: chunk.id,
|
|
314
|
-
chunkOrdinal: chunk.ordinal,
|
|
315
|
-
content: chunk.content,
|
|
316
|
-
tokenCount: chunk.tokenCount,
|
|
317
|
-
tags: document.tags,
|
|
318
|
-
embedding: chunk.embedding,
|
|
319
|
-
buckets: chunk.buckets
|
|
320
|
-
}
|
|
321
|
-
];
|
|
322
|
-
});
|
|
323
|
-
const tokens = tokenize(query);
|
|
324
|
-
// Pure-semantic scoring on a large vault can skip chunks that share no
|
|
325
|
-
// embedding bucket with the query. Hybrid/fts keep every row so lexical
|
|
326
|
-
// matches are never dropped; the prefilter also falls back to a full scan
|
|
327
|
-
// on small or partially-indexed vaults.
|
|
328
|
-
const candidates = mode === 'semantic' && queryEmbedding.length > 0
|
|
329
|
-
? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
|
|
330
|
-
: rows;
|
|
331
|
-
// Pass 1: field-weighted term frequencies per candidate, then corpus-wide
|
|
332
|
-
// statistics (document frequency, average length) needed for BM25 IDF and
|
|
333
|
-
// length normalization.
|
|
334
|
-
const prepared = candidates.map((row) => {
|
|
335
|
-
const fields = normalizeFields(row);
|
|
336
|
-
const weightedTermFrequencies = new Map();
|
|
337
|
-
for (const term of tokens) {
|
|
338
|
-
const frequency = weightedTermFrequency(fields, term);
|
|
339
|
-
if (frequency > 0) {
|
|
340
|
-
weightedTermFrequencies.set(term, frequency);
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
return { row, weightedTermFrequencies };
|
|
344
|
-
});
|
|
345
|
-
const stats = buildCorpusStatistics(prepared, tokens);
|
|
346
|
-
// Pass 2: raw lexical (BM25) and semantic (cosine) component scores.
|
|
347
|
-
const componentScored = prepared.map((entry) => ({
|
|
348
|
-
row: entry.row,
|
|
349
|
-
text: tokens.length > 0 ? bm25TextScore(entry, tokens, stats) : 0,
|
|
350
|
-
semantic: semanticScore(entry.row, queryEmbedding)
|
|
351
|
-
}));
|
|
352
|
-
// Filter on raw relevance before normalization so a legitimate match that
|
|
353
|
-
// happens to be the weakest in its mode is never min-max scaled to zero
|
|
354
|
-
// and dropped. fts stays lexical-only and semantic stays vector-only.
|
|
355
|
-
const relevant = componentScored.filter((entry) => {
|
|
356
|
-
if (tokens.length === 0) {
|
|
357
|
-
return true;
|
|
358
|
-
}
|
|
359
|
-
if (mode === 'fts') {
|
|
360
|
-
return entry.text > 0;
|
|
361
|
-
}
|
|
362
|
-
if (mode === 'semantic') {
|
|
363
|
-
return entry.semantic > 0;
|
|
364
|
-
}
|
|
365
|
-
return entry.text > 0 || entry.semantic > 0;
|
|
366
|
-
});
|
|
367
|
-
if (relevant.length === 0) {
|
|
368
|
-
return [];
|
|
369
|
-
}
|
|
370
|
-
const textRange = rangeOf(relevant.map((entry) => entry.text));
|
|
371
|
-
const semanticRange = rangeOf(relevant.map((entry) => entry.semantic));
|
|
372
|
-
const finalScore = (entry) => {
|
|
373
|
-
if (mode === 'fts') {
|
|
374
|
-
return entry.text;
|
|
375
|
-
}
|
|
376
|
-
if (mode === 'semantic') {
|
|
377
|
-
return entry.semantic;
|
|
378
|
-
}
|
|
379
|
-
// Hybrid mixes the two components on a shared [0, 1] scale so neither the
|
|
380
|
-
// unbounded BM25 magnitude nor the bounded cosine value can swamp the
|
|
381
|
-
// other, replacing the previous fixed `text + semantic * 8` blend.
|
|
382
|
-
const textNorm = minMaxNormalize(entry.text, textRange.min, textRange.max);
|
|
383
|
-
const semanticNorm = minMaxNormalize(entry.semantic, semanticRange.min, semanticRange.max);
|
|
384
|
-
return textNorm * hybridTextWeight + semanticNorm * hybridSemanticWeight;
|
|
385
|
-
};
|
|
386
|
-
return relevant
|
|
387
|
-
.map((entry) => toResult(entry.row, mode, finalScore(entry), entry.text, entry.semantic))
|
|
388
|
-
.sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
|
|
389
|
-
.slice(0, Math.max(0, limit));
|
|
390
|
-
},
|
|
391
|
-
listLinks: async (agentId) => {
|
|
392
|
-
const index = await load();
|
|
393
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
394
|
-
return index.links
|
|
395
|
-
.filter((link) => {
|
|
396
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
397
|
-
return agentId ? source?.agentId === agentId : true;
|
|
398
|
-
})
|
|
399
|
-
.map((link) => toGraphLink(link, documentsById))
|
|
400
|
-
.sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
|
|
401
|
-
},
|
|
402
|
-
listBacklinks: async (title, agentId) => {
|
|
403
|
-
const index = await load();
|
|
404
|
-
const titleKey = title.toLowerCase();
|
|
405
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
406
|
-
return index.links
|
|
407
|
-
.filter((link) => link.toTitle.toLowerCase() === titleKey)
|
|
408
|
-
.filter((link) => {
|
|
409
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
410
|
-
return agentId ? source?.agentId === agentId : true;
|
|
411
|
-
})
|
|
412
|
-
.map((link) => toGraphLink(link, documentsById))
|
|
413
|
-
.sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
|
|
414
|
-
},
|
|
415
|
-
getGraph: async (agentId) => {
|
|
416
|
-
const index = await load();
|
|
417
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
418
|
-
const documentIds = new Set(documents.map((document) => document.id));
|
|
419
|
-
const edges = index.links
|
|
420
|
-
.filter((link) => documentIds.has(link.fromDocumentId))
|
|
421
|
-
.map((link) => ({
|
|
422
|
-
source: link.fromDocumentId,
|
|
423
|
-
target: link.toDocumentId,
|
|
424
|
-
targetTitle: link.toTitle,
|
|
425
|
-
weight: link.weight,
|
|
426
|
-
priority: link.priority
|
|
427
|
-
}));
|
|
428
|
-
return {
|
|
429
|
-
nodes: documents.map((document) => ({
|
|
430
|
-
id: document.id,
|
|
431
|
-
agentId: document.agentId,
|
|
432
|
-
title: document.title,
|
|
433
|
-
path: document.path,
|
|
434
|
-
content: document.content,
|
|
435
|
-
tags: document.tags,
|
|
436
|
-
contextLinks: document.contextLinks ?? []
|
|
437
|
-
})),
|
|
438
|
-
edges
|
|
439
|
-
};
|
|
440
|
-
},
|
|
441
|
-
getGraphSummary: async (agentId) => {
|
|
442
|
-
const graph = await (async () => {
|
|
443
|
-
const index = await load();
|
|
444
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
445
|
-
const documentIds = new Set(documents.map((document) => document.id));
|
|
446
|
-
const edges = index.links
|
|
447
|
-
.filter((link) => documentIds.has(link.fromDocumentId))
|
|
448
|
-
.map((link) => ({
|
|
449
|
-
source: link.fromDocumentId,
|
|
450
|
-
target: link.toDocumentId,
|
|
451
|
-
targetTitle: link.toTitle,
|
|
452
|
-
weight: link.weight,
|
|
453
|
-
priority: link.priority
|
|
454
|
-
}));
|
|
455
|
-
return {
|
|
456
|
-
nodes: documents.map((document) => ({
|
|
457
|
-
id: document.id,
|
|
458
|
-
agentId: document.agentId,
|
|
459
|
-
title: document.title,
|
|
460
|
-
path: document.path,
|
|
461
|
-
content: '',
|
|
462
|
-
tags: document.tags,
|
|
463
|
-
contextLinks: document.contextLinks ?? []
|
|
464
|
-
})),
|
|
465
|
-
edges
|
|
466
|
-
};
|
|
467
|
-
})();
|
|
468
|
-
return graph;
|
|
469
|
-
},
|
|
470
|
-
getGraphNode: async (id, agentId) => {
|
|
471
|
-
const index = await load();
|
|
472
|
-
const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
|
|
473
|
-
return document
|
|
474
|
-
? {
|
|
475
|
-
id: document.id,
|
|
476
|
-
agentId: document.agentId,
|
|
477
|
-
title: document.title,
|
|
478
|
-
path: document.path,
|
|
479
|
-
content: document.content,
|
|
480
|
-
tags: document.tags,
|
|
481
|
-
contextLinks: document.contextLinks ?? []
|
|
482
|
-
}
|
|
483
|
-
: undefined;
|
|
484
|
-
},
|
|
485
|
-
searchGraphNodeIds: async (query, limit, agentId) => {
|
|
486
|
-
const index = await load();
|
|
487
|
-
const normalized = normalizeToken(query);
|
|
488
|
-
if (normalized.length === 0 || limit <= 0) {
|
|
489
|
-
return [];
|
|
490
|
-
}
|
|
491
|
-
const tokens = tokenize(query);
|
|
492
|
-
const scored = index.documents
|
|
493
|
-
.filter((document) => (!agentId || document.agentId === agentId))
|
|
494
|
-
.map((document) => {
|
|
495
|
-
const score = lexicalFieldScore({
|
|
496
|
-
documentId: document.id,
|
|
497
|
-
agentId: document.agentId,
|
|
498
|
-
title: document.title,
|
|
499
|
-
path: document.path,
|
|
500
|
-
chunkId: document.id,
|
|
501
|
-
chunkOrdinal: 0,
|
|
502
|
-
content: document.content,
|
|
503
|
-
tokenCount: 0,
|
|
504
|
-
tags: document.tags,
|
|
505
|
-
embedding: []
|
|
506
|
-
}, tokens);
|
|
507
|
-
return { id: document.id, score };
|
|
508
|
-
})
|
|
509
|
-
.filter((row) => row.score > 0)
|
|
510
|
-
.sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
|
|
511
|
-
.slice(0, limit);
|
|
512
|
-
return scored.map((row) => row.id);
|
|
513
|
-
},
|
|
514
|
-
listAgents: async () => {
|
|
515
|
-
const index = await load();
|
|
516
|
-
const counts = index.documents.reduce((state, document) => {
|
|
517
|
-
state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
|
|
518
|
-
return state;
|
|
519
|
-
}, new Map());
|
|
520
|
-
return Array.from(counts.entries())
|
|
521
|
-
.sort((left, right) => left[0].localeCompare(right[0]))
|
|
522
|
-
.map(([id, documentCount]) => ({ id, documentCount }));
|
|
523
|
-
},
|
|
524
|
-
close: () => {
|
|
525
|
-
// File-based index has no persistent connection.
|
|
526
|
-
}
|
|
527
|
-
};
|
|
528
|
-
};
|