@andespindola/brainlink 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +73 -0
- package/dist/application/find-similar-notes.js +75 -0
- package/dist/application/get-graph-node.js +2 -2
- package/dist/application/get-graph-summary.js +2 -2
- package/dist/application/get-graph.js +2 -2
- package/dist/application/index-vault-phases.js +5 -4
- package/dist/application/index-vault.js +4 -4
- package/dist/application/list-agents.js +2 -2
- package/dist/application/list-links.js +3 -3
- package/dist/application/memory-suggestions.js +4 -1
- package/dist/application/ports/knowledge-store.js +1 -0
- package/dist/application/ranking/run-search.js +177 -0
- package/dist/application/search-graph-node-ids.js +3 -2
- package/dist/application/search-knowledge.js +22 -5
- package/dist/application/vault-git.js +180 -0
- package/dist/application/vault-snapshot.js +87 -0
- package/dist/cli/commands/vault-sync-commands.js +115 -0
- package/dist/cli/main.js +2 -0
- package/dist/domain/context.js +27 -14
- package/dist/domain/diversity.js +64 -0
- package/dist/domain/embeddings.js +117 -1
- package/dist/domain/markdown.js +10 -1
- package/dist/domain/scoring.js +42 -0
- package/dist/domain/tokens.js +17 -1
- package/dist/infrastructure/config.js +31 -1
- package/dist/infrastructure/file-index.js +99 -328
- package/dist/infrastructure/knowledge-store/binary-store.js +163 -0
- package/dist/infrastructure/knowledge-store/index.js +36 -0
- package/dist/infrastructure/knowledge-store/stored-index.js +216 -0
- package/dist/mcp/server.js +16 -1
- package/dist/mcp/tools/read-tools.js +33 -0
- package/dist/mcp/tools/write-tools.js +106 -0
- package/dist/mcp/tools.js +2 -2
- package/docs/AGENT_USAGE.md +4 -1
- package/docs/ARCHITECTURE.md +4 -3
- package/package.json +3 -2
package/dist/domain/markdown.js
CHANGED
|
@@ -188,7 +188,15 @@ const splitLongParagraph = (paragraph, maxCharacters) => {
|
|
|
188
188
|
return [...state, sentence];
|
|
189
189
|
}, []);
|
|
190
190
|
};
|
|
191
|
+
const headingParagraphPattern = /^#{1,6}\s/;
|
|
192
|
+
const startsWithHeading = (paragraph) => headingParagraphPattern.test(paragraph);
|
|
191
193
|
export const splitIntoChunks = (documentId, content, maxCharacters = 1200) => {
|
|
194
|
+
// A heading should begin a new chunk so a section's body is not glued onto the
|
|
195
|
+
// tail of the previous section — but only once the current chunk already holds
|
|
196
|
+
// substantial content. This keeps large notes split on topical (heading)
|
|
197
|
+
// boundaries while small notes (and trailing metadata sections like
|
|
198
|
+
// "## Context Links") stay in a single chunk instead of fragmenting.
|
|
199
|
+
const minChunkCharacters = Math.floor(maxCharacters / 2);
|
|
192
200
|
const paragraphs = normalizeChunkContent(stripFrontmatter(content))
|
|
193
201
|
.split(/\n{2,}/)
|
|
194
202
|
.filter(Boolean)
|
|
@@ -198,8 +206,9 @@ export const splitIntoChunks = (documentId, content, maxCharacters = 1200) => {
|
|
|
198
206
|
if (!lastChunk) {
|
|
199
207
|
return [paragraph];
|
|
200
208
|
}
|
|
209
|
+
const startsNewSection = startsWithHeading(paragraph) && lastChunk.length >= minChunkCharacters;
|
|
201
210
|
const merged = `${lastChunk}\n\n${paragraph}`;
|
|
202
|
-
if (merged.length <= maxCharacters) {
|
|
211
|
+
if (!startsNewSection && merged.length <= maxCharacters) {
|
|
203
212
|
return [...state.slice(0, -1), merged];
|
|
204
213
|
}
|
|
205
214
|
return [...state, paragraph];
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Pure retrieval scoring primitives: BM25 lexical relevance and min-max
|
|
2
|
+
// normalization. Kept dependency-free and deterministic so the hot retrieval
|
|
3
|
+
// loop in the infrastructure layer stays testable and the domain owns the
|
|
4
|
+
// ranking math.
|
|
5
|
+
export const defaultBm25Parameters = {
|
|
6
|
+
k1: 1.2,
|
|
7
|
+
b: 0.75
|
|
8
|
+
};
|
|
9
|
+
// Robertson/Sparck-Jones probabilistic IDF with the +0.5 smoothing that keeps
|
|
10
|
+
// the value finite for terms present in every chunk. Floored at 0 so a term
|
|
11
|
+
// shared by the whole candidate set can never push a score negative.
|
|
12
|
+
export const inverseDocumentFrequency = (documentCount, documentFrequency) => {
|
|
13
|
+
if (documentCount <= 0) {
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
|
16
|
+
const raw = Math.log(1 + (documentCount - documentFrequency + 0.5) / (documentFrequency + 0.5));
|
|
17
|
+
return raw > 0 ? raw : 0;
|
|
18
|
+
};
|
|
19
|
+
// BM25 contribution for a single query term given its (field-weighted)
|
|
20
|
+
// frequency in a chunk. `documentLength` and `averageLength` are in the same
|
|
21
|
+
// weighted unit so length normalization stays consistent with how the caller
|
|
22
|
+
// counts terms.
|
|
23
|
+
export const bm25TermScore = (params) => {
|
|
24
|
+
if (params.termFrequency <= 0) {
|
|
25
|
+
return 0;
|
|
26
|
+
}
|
|
27
|
+
const { k1, b } = params.parameters ?? defaultBm25Parameters;
|
|
28
|
+
const idf = inverseDocumentFrequency(params.documentCount, params.documentFrequency);
|
|
29
|
+
const normalizedLength = params.averageLength > 0 ? params.documentLength / params.averageLength : 1;
|
|
30
|
+
const denominator = params.termFrequency + k1 * (1 - b + b * normalizedLength);
|
|
31
|
+
return denominator > 0 ? (idf * (params.termFrequency * (k1 + 1))) / denominator : 0;
|
|
32
|
+
};
|
|
33
|
+
// Scale a value into [0, 1] given the observed range. When every candidate
|
|
34
|
+
// shares the same score (range collapses) a positive score maps to 1 and a
|
|
35
|
+
// zero score to 0, so a single result is never silently nulled out.
|
|
36
|
+
export const minMaxNormalize = (value, minimum, maximum) => {
|
|
37
|
+
if (maximum <= minimum) {
|
|
38
|
+
return value > 0 ? 1 : 0;
|
|
39
|
+
}
|
|
40
|
+
const scaled = (value - minimum) / (maximum - minimum);
|
|
41
|
+
return scaled < 0 ? 0 : scaled > 1 ? 1 : scaled;
|
|
42
|
+
};
|
package/dist/domain/tokens.js
CHANGED
|
@@ -1 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
// Subword tokenizers emit roughly 1.3 tokens per whitespace word for natural
|
|
2
|
+
// language. Pure prose tracks that ratio; punctuation- or symbol-dense content
|
|
3
|
+
// (code, URLs, long unbroken strings) tokenizes far denser, which the word
|
|
4
|
+
// count alone underestimates. Taking the max with the character heuristic keeps
|
|
5
|
+
// the estimate conservative — never below chars/4 — so callers that budget
|
|
6
|
+
// against it can rely on the rendered output staying within the token limit.
|
|
7
|
+
const tokensPerWord = 1.3;
|
|
8
|
+
const charactersPerToken = 4;
|
|
9
|
+
export const estimateTokenCount = (content) => {
|
|
10
|
+
const words = content.trim().split(/\s+/).filter(Boolean);
|
|
11
|
+
if (words.length === 0) {
|
|
12
|
+
return 0;
|
|
13
|
+
}
|
|
14
|
+
const characterEstimate = Math.ceil(words.join(' ').length / charactersPerToken);
|
|
15
|
+
const wordEstimate = Math.ceil(words.length * tokensPerWord);
|
|
16
|
+
return Math.max(characterEstimate, wordEstimate);
|
|
17
|
+
};
|
|
@@ -18,8 +18,10 @@ export const defaultBrainlinkConfig = {
|
|
|
18
18
|
defaultContextStrategy: 'auto',
|
|
19
19
|
defaultContextCacheTtlMs: 120_000,
|
|
20
20
|
embeddingProvider: 'local',
|
|
21
|
+
embedding: {},
|
|
21
22
|
defaultSearchMode: 'hybrid',
|
|
22
23
|
chunkSize: 1200,
|
|
24
|
+
storageBackend: 'json',
|
|
23
25
|
searchPack: {
|
|
24
26
|
rowChunkSize: 5_000,
|
|
25
27
|
compressionLevel: 5,
|
|
@@ -43,10 +45,36 @@ const safeCwd = () => {
|
|
|
43
45
|
}
|
|
44
46
|
};
|
|
45
47
|
const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
46
|
-
const embeddingProviders = new Set(['none', 'local']);
|
|
48
|
+
const embeddingProviders = new Set(['none', 'local', 'ollama', 'openai']);
|
|
47
49
|
const searchModes = new Set(['fts', 'semantic', 'hybrid']);
|
|
48
50
|
const contextStrategies = new Set(['rag', 'cag', 'auto']);
|
|
51
|
+
const storageBackends = new Set(['json', 'binary', 'lmdb']);
|
|
52
|
+
const sanitizeStorageBackend = (value) => typeof value === 'string' && storageBackends.has(value) ? value : defaultBrainlinkConfig.storageBackend;
|
|
49
53
|
const sanitizeEmbeddingProvider = (value) => typeof value === 'string' && embeddingProviders.has(value) ? value : defaultBrainlinkConfig.embeddingProvider;
|
|
54
|
+
const sanitizeOptionalString = (value) => typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
|
55
|
+
const sanitizeEmbeddingConfig = (value) => {
|
|
56
|
+
if (!isRecord(value)) {
|
|
57
|
+
return {};
|
|
58
|
+
}
|
|
59
|
+
const url = sanitizeOptionalString(value.url);
|
|
60
|
+
const model = sanitizeOptionalString(value.model);
|
|
61
|
+
const apiKeyEnv = sanitizeOptionalString(value.apiKeyEnv);
|
|
62
|
+
const timeoutMs = sanitizePositiveNumber(value.timeoutMs);
|
|
63
|
+
const batchSize = sanitizePositiveNumber(value.batchSize);
|
|
64
|
+
return {
|
|
65
|
+
...(url ? { url } : {}),
|
|
66
|
+
...(model ? { model } : {}),
|
|
67
|
+
...(apiKeyEnv ? { apiKeyEnv } : {}),
|
|
68
|
+
...(timeoutMs ? { timeoutMs: Math.floor(timeoutMs) } : {}),
|
|
69
|
+
...(batchSize ? { batchSize: Math.floor(batchSize) } : {})
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
// Stable identity of the embedding space, used to decide when stored vectors
|
|
73
|
+
// must be rebuilt. A model swap on an external provider changes the space even
|
|
74
|
+
// though the provider name is unchanged, so the model is folded in.
|
|
75
|
+
export const embeddingSignature = (config) => config.embeddingProvider === 'ollama' || config.embeddingProvider === 'openai'
|
|
76
|
+
? `${config.embeddingProvider}:${config.embedding.model?.trim() || 'default'}`
|
|
77
|
+
: config.embeddingProvider;
|
|
50
78
|
export const sanitizeSearchMode = (value, fallback = defaultBrainlinkConfig.defaultSearchMode) => typeof value === 'string' && searchModes.has(value) ? value : fallback;
|
|
51
79
|
export const sanitizeContextStrategy = (value, fallback = defaultBrainlinkConfig.defaultContextStrategy) => typeof value === 'string' && contextStrategies.has(value) ? value : fallback;
|
|
52
80
|
const sanitizeAllowedVaults = (value) => Array.isArray(value) ? value.filter((item) => typeof item === 'string' && item.trim().length > 0) : [];
|
|
@@ -195,8 +223,10 @@ const sanitizeConfig = (value) => ({
|
|
|
195
223
|
: defaultBrainlinkConfig.defaultContextCacheTtlMs,
|
|
196
224
|
allowedVaults: [...sanitizeAllowedVaults(value.allowedVaults), ...readAllowedVaultsFromEnv()],
|
|
197
225
|
chunkSize: typeof value.chunkSize === 'number' && value.chunkSize > 0 ? value.chunkSize : defaultBrainlinkConfig.chunkSize,
|
|
226
|
+
storageBackend: sanitizeStorageBackend(value.storageBackend),
|
|
198
227
|
searchPack: sanitizeSearchPackConfig(value.searchPack),
|
|
199
228
|
embeddingProvider: sanitizeEmbeddingProvider(value.embeddingProvider),
|
|
229
|
+
embedding: sanitizeEmbeddingConfig(value.embedding),
|
|
200
230
|
defaultSearchMode: sanitizeSearchMode(value.defaultSearchMode),
|
|
201
231
|
agentProfiles: sanitizeAgentProfiles(value.agentProfiles)
|
|
202
232
|
});
|
|
@@ -1,19 +1,55 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import { selectSemanticCandidates } from './semantic-prefilter.js';
|
|
5
|
-
const queryTokenPattern = /[\p{L}\p{N}_-]+/gu;
|
|
3
|
+
import { createStoredIndexQueries, emptyIndex, toStoredIndex } from './knowledge-store/stored-index.js';
|
|
6
4
|
const indexCacheMaxEntries = 16;
|
|
7
5
|
const indexCache = new Map();
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
});
|
|
6
|
+
// Raised when index.json exists but cannot be parsed. Search treats it as a
|
|
7
|
+
// signal to fall back to the independent compressed search packs instead of
|
|
8
|
+
// silently returning an empty result set.
|
|
9
|
+
export class IndexCorruptionError extends Error {
|
|
10
|
+
indexPath;
|
|
11
|
+
constructor(indexPath, cause) {
|
|
12
|
+
super(`Corrupted index at ${indexPath}: ${cause instanceof Error ? cause.message : String(cause)}`);
|
|
13
|
+
this.indexPath = indexPath;
|
|
14
|
+
this.name = 'IndexCorruptionError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
15
17
|
export const indexStoragePath = (vaultPath) => join(vaultPath, '.brainlink', 'index.json');
|
|
16
|
-
const
|
|
18
|
+
const isMissingFileError = (error) => error instanceof Error && 'code' in error && error.code === 'ENOENT';
|
|
19
|
+
// Paths already reported as corrupt, so a non-removable corrupt index (read-only
|
|
20
|
+
// vault, locked file) does not spam stderr on every read. Cleared when a fresh
|
|
21
|
+
// index is written to the path.
|
|
22
|
+
const warnedCorruptIndexPaths = new Set();
|
|
23
|
+
// Move a corrupt index aside so it is not read again and the next index run
|
|
24
|
+
// rebuilds from Markdown source, and surface a warning instead of failing
|
|
25
|
+
// silently. Quarantine is best-effort: a read-only vault simply keeps the file.
|
|
26
|
+
const quarantineCorruptIndex = async (path, cause) => {
|
|
27
|
+
indexCache.delete(path);
|
|
28
|
+
if (!warnedCorruptIndexPaths.has(path)) {
|
|
29
|
+
warnedCorruptIndexPaths.add(path);
|
|
30
|
+
process.stderr.write(`brainlink: index at ${path} is unreadable and was quarantined; rebuild with 'index --full' (${cause instanceof Error ? cause.message : String(cause)})\n`);
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
await rename(path, `${path}.corrupt`);
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
// The primary name can already exist (a prior corruption on a filesystem
|
|
37
|
+
// where rename does not overwrite, e.g. Windows). Fall back to a unique name
|
|
38
|
+
// so the damaged file is still moved out of the read path.
|
|
39
|
+
try {
|
|
40
|
+
await rename(path, `${path}.corrupt-${Date.now()}`);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
// Leave the file in place when it cannot be renamed; the warning still fired.
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
// Exposed so the binary backend can serve reads from a legacy index.json until
|
|
48
|
+
// the next index run materializes the binary store (seamless migration).
|
|
49
|
+
export function readJsonStoredIndex(vaultPath, strict = false) {
|
|
50
|
+
return readIndex(vaultPath, strict);
|
|
51
|
+
}
|
|
52
|
+
const readIndex = async (vaultPath, strict = false) => {
|
|
17
53
|
const path = indexStoragePath(vaultPath);
|
|
18
54
|
let stats = null;
|
|
19
55
|
try {
|
|
@@ -21,9 +57,8 @@ const readIndex = async (vaultPath) => {
|
|
|
21
57
|
stats = { mtimeMs: fileStats.mtimeMs, size: fileStats.size };
|
|
22
58
|
}
|
|
23
59
|
catch (error) {
|
|
24
|
-
if (error
|
|
60
|
+
if (isMissingFileError(error)) {
|
|
25
61
|
indexCache.delete(path);
|
|
26
|
-
return emptyIndex();
|
|
27
62
|
}
|
|
28
63
|
return emptyIndex();
|
|
29
64
|
}
|
|
@@ -31,31 +66,46 @@ const readIndex = async (vaultPath) => {
|
|
|
31
66
|
if (cached && cached.mtimeMs === stats.mtimeMs && cached.size === stats.size) {
|
|
32
67
|
return cached.index;
|
|
33
68
|
}
|
|
69
|
+
let raw;
|
|
34
70
|
try {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
if (indexCache.size > indexCacheMaxEntries) {
|
|
45
|
-
const oldest = indexCache.keys().next().value;
|
|
46
|
-
if (typeof oldest === 'string') {
|
|
47
|
-
indexCache.delete(oldest);
|
|
48
|
-
}
|
|
71
|
+
raw = await readFile(path, 'utf8');
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
indexCache.delete(path);
|
|
75
|
+
// The file existed at stat but vanished before the read — a concurrent
|
|
76
|
+
// rebuild or quarantine. Strict (search) callers throw so they fall back to
|
|
77
|
+
// packs rather than racing to an empty result; tolerant callers get empty.
|
|
78
|
+
if (strict) {
|
|
79
|
+
throw new IndexCorruptionError(path, error);
|
|
49
80
|
}
|
|
50
|
-
return
|
|
81
|
+
return emptyIndex();
|
|
82
|
+
}
|
|
83
|
+
let parsed;
|
|
84
|
+
try {
|
|
85
|
+
parsed = JSON.parse(raw);
|
|
51
86
|
}
|
|
52
87
|
catch (error) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
await quarantineCorruptIndex(path, error);
|
|
89
|
+
if (strict) {
|
|
90
|
+
throw new IndexCorruptionError(path, error);
|
|
56
91
|
}
|
|
57
92
|
return emptyIndex();
|
|
58
93
|
}
|
|
94
|
+
const loaded = {
|
|
95
|
+
version: 1,
|
|
96
|
+
updatedAt: typeof parsed.updatedAt === 'string' ? parsed.updatedAt : new Date().toISOString(),
|
|
97
|
+
documents: Array.isArray(parsed.documents) ? parsed.documents : [],
|
|
98
|
+
chunks: Array.isArray(parsed.chunks) ? parsed.chunks : [],
|
|
99
|
+
links: Array.isArray(parsed.links) ? parsed.links : []
|
|
100
|
+
};
|
|
101
|
+
indexCache.set(path, { ...stats, index: loaded });
|
|
102
|
+
if (indexCache.size > indexCacheMaxEntries) {
|
|
103
|
+
const oldest = indexCache.keys().next().value;
|
|
104
|
+
if (typeof oldest === 'string') {
|
|
105
|
+
indexCache.delete(oldest);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return loaded;
|
|
59
109
|
};
|
|
60
110
|
const writeIndex = async (vaultPath, index) => {
|
|
61
111
|
const target = indexStoragePath(vaultPath);
|
|
@@ -63,6 +113,7 @@ const writeIndex = async (vaultPath, index) => {
|
|
|
63
113
|
await mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
|
64
114
|
await writeFile(temp, `${JSON.stringify(index)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
65
115
|
await rename(temp, target);
|
|
116
|
+
warnedCorruptIndexPaths.delete(target);
|
|
66
117
|
const fileStats = await stat(target);
|
|
67
118
|
indexCache.set(target, {
|
|
68
119
|
mtimeMs: fileStats.mtimeMs,
|
|
@@ -70,301 +121,21 @@ const writeIndex = async (vaultPath, index) => {
|
|
|
70
121
|
index
|
|
71
122
|
});
|
|
72
123
|
};
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
hits += 1;
|
|
90
|
-
cursor = index + token.length;
|
|
91
|
-
}
|
|
92
|
-
return hits;
|
|
93
|
-
};
|
|
94
|
-
const textScore = (row, tokens) => {
|
|
95
|
-
if (tokens.length === 0) {
|
|
96
|
-
return 0;
|
|
124
|
+
// JSON file backend: the original single `index.json` store. Loading/persisting
|
|
125
|
+
// is its own concern; the graph/search/link reads come from the shared query
|
|
126
|
+
// core so this backend stays behavior-identical to the binary backend.
|
|
127
|
+
export const openFileIndex = (vaultPath) => ({
|
|
128
|
+
reset: async () => {
|
|
129
|
+
await writeIndex(vaultPath, emptyIndex());
|
|
130
|
+
},
|
|
131
|
+
saveDocuments: async (documents) => {
|
|
132
|
+
await writeIndex(vaultPath, toStoredIndex(documents));
|
|
133
|
+
},
|
|
134
|
+
...createStoredIndexQueries({
|
|
135
|
+
load: () => readIndex(vaultPath),
|
|
136
|
+
loadStrict: () => readIndex(vaultPath, true)
|
|
137
|
+
}),
|
|
138
|
+
close: () => {
|
|
139
|
+
// File-based index has no persistent connection.
|
|
97
140
|
}
|
|
98
|
-
|
|
99
|
-
const path = normalizeToken(row.path);
|
|
100
|
-
const content = normalizeToken(row.content);
|
|
101
|
-
const tags = normalizeToken(row.tags.join(' '));
|
|
102
|
-
return tokens.reduce((score, token) => {
|
|
103
|
-
const titleHits = countOccurrences(title, token);
|
|
104
|
-
const tagHits = countOccurrences(tags, token);
|
|
105
|
-
const pathHits = countOccurrences(path, token);
|
|
106
|
-
const contentHits = countOccurrences(content, token);
|
|
107
|
-
return score + titleHits * 5 + tagHits * 4 + pathHits * 2 + Math.min(contentHits, 6);
|
|
108
|
-
}, 0);
|
|
109
|
-
};
|
|
110
|
-
const semanticScore = (row, queryEmbedding) => queryEmbedding.length > 0 && row.embedding.length > 0 ? dotProduct(queryEmbedding, row.embedding) : 0;
|
|
111
|
-
const toResult = (row, mode, text, semantic) => {
|
|
112
|
-
const score = mode === 'fts' ? text : mode === 'semantic' ? semantic : text + semantic * 8;
|
|
113
|
-
return {
|
|
114
|
-
documentId: row.documentId,
|
|
115
|
-
agentId: row.agentId,
|
|
116
|
-
title: row.title,
|
|
117
|
-
path: row.path,
|
|
118
|
-
chunkId: row.chunkId,
|
|
119
|
-
chunkOrdinal: row.chunkOrdinal,
|
|
120
|
-
content: row.content,
|
|
121
|
-
score,
|
|
122
|
-
textScore: text,
|
|
123
|
-
semanticScore: semantic,
|
|
124
|
-
searchMode: mode,
|
|
125
|
-
tags: row.tags
|
|
126
|
-
};
|
|
127
|
-
};
|
|
128
|
-
const toGraphLink = (link, documentsById) => {
|
|
129
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
130
|
-
const target = link.toDocumentId ? documentsById.get(link.toDocumentId) : undefined;
|
|
131
|
-
return {
|
|
132
|
-
agentId: source?.agentId ?? 'shared',
|
|
133
|
-
fromTitle: source?.title ?? 'Unknown',
|
|
134
|
-
fromPath: source?.path ?? 'Unknown',
|
|
135
|
-
toTitle: target?.title ?? link.toTitle,
|
|
136
|
-
toPath: target?.path ?? null,
|
|
137
|
-
weight: link.weight,
|
|
138
|
-
priority: link.priority
|
|
139
|
-
};
|
|
140
|
-
};
|
|
141
|
-
export const openFileIndex = (vaultPath) => {
|
|
142
|
-
const load = async () => readIndex(vaultPath);
|
|
143
|
-
const persist = async (index) => writeIndex(vaultPath, index);
|
|
144
|
-
return {
|
|
145
|
-
reset: async () => {
|
|
146
|
-
await persist(emptyIndex());
|
|
147
|
-
},
|
|
148
|
-
saveDocuments: async (documents) => {
|
|
149
|
-
const chunks = documents.flatMap((document) => document.chunks);
|
|
150
|
-
const links = documents.flatMap((document) => document.links);
|
|
151
|
-
await persist({
|
|
152
|
-
version: 1,
|
|
153
|
-
updatedAt: new Date().toISOString(),
|
|
154
|
-
documents: documents.map((document) => document.document),
|
|
155
|
-
chunks,
|
|
156
|
-
links
|
|
157
|
-
});
|
|
158
|
-
},
|
|
159
|
-
getIndexedDocuments: async (agentId) => {
|
|
160
|
-
const index = await load();
|
|
161
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
162
|
-
const selectedDocumentIds = new Set(documents.map((document) => document.id));
|
|
163
|
-
const chunksByDocumentId = index.chunks.reduce((state, chunk) => {
|
|
164
|
-
if (!selectedDocumentIds.has(chunk.documentId)) {
|
|
165
|
-
return state;
|
|
166
|
-
}
|
|
167
|
-
const current = state.get(chunk.documentId) ?? [];
|
|
168
|
-
current.push(chunk);
|
|
169
|
-
state.set(chunk.documentId, current);
|
|
170
|
-
return state;
|
|
171
|
-
}, new Map());
|
|
172
|
-
const linksByDocumentId = index.links.reduce((state, link) => {
|
|
173
|
-
if (!selectedDocumentIds.has(link.fromDocumentId)) {
|
|
174
|
-
return state;
|
|
175
|
-
}
|
|
176
|
-
const current = state.get(link.fromDocumentId) ?? [];
|
|
177
|
-
current.push(link);
|
|
178
|
-
state.set(link.fromDocumentId, current);
|
|
179
|
-
return state;
|
|
180
|
-
}, new Map());
|
|
181
|
-
return documents
|
|
182
|
-
.map((document) => ({
|
|
183
|
-
document,
|
|
184
|
-
chunks: [...(chunksByDocumentId.get(document.id) ?? [])].sort((left, right) => left.ordinal - right.ordinal),
|
|
185
|
-
links: linksByDocumentId.get(document.id) ?? []
|
|
186
|
-
}))
|
|
187
|
-
.sort((left, right) => left.document.path.localeCompare(right.document.path));
|
|
188
|
-
},
|
|
189
|
-
search: async (query, limit, agentId, mode = 'hybrid', queryEmbedding = []) => {
|
|
190
|
-
const index = await load();
|
|
191
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
192
|
-
const rows = index.chunks.flatMap((chunk) => {
|
|
193
|
-
const document = documentsById.get(chunk.documentId);
|
|
194
|
-
if (!document) {
|
|
195
|
-
return [];
|
|
196
|
-
}
|
|
197
|
-
if (agentId && document.agentId !== agentId) {
|
|
198
|
-
return [];
|
|
199
|
-
}
|
|
200
|
-
return [
|
|
201
|
-
{
|
|
202
|
-
documentId: document.id,
|
|
203
|
-
agentId: document.agentId,
|
|
204
|
-
title: document.title,
|
|
205
|
-
path: document.path,
|
|
206
|
-
chunkId: chunk.id,
|
|
207
|
-
chunkOrdinal: chunk.ordinal,
|
|
208
|
-
content: chunk.content,
|
|
209
|
-
tags: document.tags,
|
|
210
|
-
embedding: chunk.embedding,
|
|
211
|
-
buckets: chunk.buckets
|
|
212
|
-
}
|
|
213
|
-
];
|
|
214
|
-
});
|
|
215
|
-
const tokens = tokenize(query);
|
|
216
|
-
// Pure-semantic scoring on a large vault can skip chunks that share no
|
|
217
|
-
// embedding bucket with the query. Hybrid/fts keep every row so lexical
|
|
218
|
-
// matches are never dropped; the prefilter also falls back to a full scan
|
|
219
|
-
// on small or partially-indexed vaults.
|
|
220
|
-
const scored = mode === 'semantic' && queryEmbedding.length > 0
|
|
221
|
-
? selectSemanticCandidates(rows, createEmbeddingBuckets(queryEmbedding), limit)
|
|
222
|
-
: rows;
|
|
223
|
-
const results = scored
|
|
224
|
-
.map((row) => {
|
|
225
|
-
const text = textScore(row, tokens);
|
|
226
|
-
const semantic = semanticScore(row, queryEmbedding);
|
|
227
|
-
return toResult(row, mode, text, semantic);
|
|
228
|
-
})
|
|
229
|
-
.filter((row) => row.score > 0 || tokens.length === 0)
|
|
230
|
-
.sort((left, right) => right.score - left.score || left.title.localeCompare(right.title))
|
|
231
|
-
.slice(0, Math.max(0, limit));
|
|
232
|
-
return results;
|
|
233
|
-
},
|
|
234
|
-
listLinks: async (agentId) => {
|
|
235
|
-
const index = await load();
|
|
236
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
237
|
-
return index.links
|
|
238
|
-
.filter((link) => {
|
|
239
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
240
|
-
return agentId ? source?.agentId === agentId : true;
|
|
241
|
-
})
|
|
242
|
-
.map((link) => toGraphLink(link, documentsById))
|
|
243
|
-
.sort((left, right) => left.fromTitle.localeCompare(right.fromTitle));
|
|
244
|
-
},
|
|
245
|
-
listBacklinks: async (title, agentId) => {
|
|
246
|
-
const index = await load();
|
|
247
|
-
const titleKey = title.toLowerCase();
|
|
248
|
-
const documentsById = new Map(index.documents.map((document) => [document.id, document]));
|
|
249
|
-
return index.links
|
|
250
|
-
.filter((link) => link.toTitle.toLowerCase() === titleKey)
|
|
251
|
-
.filter((link) => {
|
|
252
|
-
const source = documentsById.get(link.fromDocumentId);
|
|
253
|
-
return agentId ? source?.agentId === agentId : true;
|
|
254
|
-
})
|
|
255
|
-
.map((link) => toGraphLink(link, documentsById))
|
|
256
|
-
.sort((left, right) => right.weight - left.weight || left.fromTitle.localeCompare(right.fromTitle));
|
|
257
|
-
},
|
|
258
|
-
getGraph: async (agentId) => {
|
|
259
|
-
const index = await load();
|
|
260
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
261
|
-
const documentIds = new Set(documents.map((document) => document.id));
|
|
262
|
-
const edges = index.links
|
|
263
|
-
.filter((link) => documentIds.has(link.fromDocumentId))
|
|
264
|
-
.map((link) => ({
|
|
265
|
-
source: link.fromDocumentId,
|
|
266
|
-
target: link.toDocumentId,
|
|
267
|
-
targetTitle: link.toTitle,
|
|
268
|
-
weight: link.weight,
|
|
269
|
-
priority: link.priority
|
|
270
|
-
}));
|
|
271
|
-
return {
|
|
272
|
-
nodes: documents.map((document) => ({
|
|
273
|
-
id: document.id,
|
|
274
|
-
agentId: document.agentId,
|
|
275
|
-
title: document.title,
|
|
276
|
-
path: document.path,
|
|
277
|
-
content: document.content,
|
|
278
|
-
tags: document.tags,
|
|
279
|
-
contextLinks: document.contextLinks ?? []
|
|
280
|
-
})),
|
|
281
|
-
edges
|
|
282
|
-
};
|
|
283
|
-
},
|
|
284
|
-
getGraphSummary: async (agentId) => {
|
|
285
|
-
const graph = await (async () => {
|
|
286
|
-
const index = await load();
|
|
287
|
-
const documents = agentId ? index.documents.filter((document) => document.agentId === agentId) : index.documents;
|
|
288
|
-
const documentIds = new Set(documents.map((document) => document.id));
|
|
289
|
-
const edges = index.links
|
|
290
|
-
.filter((link) => documentIds.has(link.fromDocumentId))
|
|
291
|
-
.map((link) => ({
|
|
292
|
-
source: link.fromDocumentId,
|
|
293
|
-
target: link.toDocumentId,
|
|
294
|
-
targetTitle: link.toTitle,
|
|
295
|
-
weight: link.weight,
|
|
296
|
-
priority: link.priority
|
|
297
|
-
}));
|
|
298
|
-
return {
|
|
299
|
-
nodes: documents.map((document) => ({
|
|
300
|
-
id: document.id,
|
|
301
|
-
agentId: document.agentId,
|
|
302
|
-
title: document.title,
|
|
303
|
-
path: document.path,
|
|
304
|
-
content: '',
|
|
305
|
-
tags: document.tags,
|
|
306
|
-
contextLinks: document.contextLinks ?? []
|
|
307
|
-
})),
|
|
308
|
-
edges
|
|
309
|
-
};
|
|
310
|
-
})();
|
|
311
|
-
return graph;
|
|
312
|
-
},
|
|
313
|
-
getGraphNode: async (id, agentId) => {
|
|
314
|
-
const index = await load();
|
|
315
|
-
const document = index.documents.find((row) => row.id === id && (!agentId || row.agentId === agentId));
|
|
316
|
-
return document
|
|
317
|
-
? {
|
|
318
|
-
id: document.id,
|
|
319
|
-
agentId: document.agentId,
|
|
320
|
-
title: document.title,
|
|
321
|
-
path: document.path,
|
|
322
|
-
content: document.content,
|
|
323
|
-
tags: document.tags,
|
|
324
|
-
contextLinks: document.contextLinks ?? []
|
|
325
|
-
}
|
|
326
|
-
: undefined;
|
|
327
|
-
},
|
|
328
|
-
searchGraphNodeIds: async (query, limit, agentId) => {
|
|
329
|
-
const index = await load();
|
|
330
|
-
const normalized = normalizeToken(query);
|
|
331
|
-
if (normalized.length === 0 || limit <= 0) {
|
|
332
|
-
return [];
|
|
333
|
-
}
|
|
334
|
-
const tokens = tokenize(query);
|
|
335
|
-
const scored = index.documents
|
|
336
|
-
.filter((document) => (!agentId || document.agentId === agentId))
|
|
337
|
-
.map((document) => {
|
|
338
|
-
const score = textScore({
|
|
339
|
-
documentId: document.id,
|
|
340
|
-
agentId: document.agentId,
|
|
341
|
-
title: document.title,
|
|
342
|
-
path: document.path,
|
|
343
|
-
chunkId: document.id,
|
|
344
|
-
chunkOrdinal: 0,
|
|
345
|
-
content: document.content,
|
|
346
|
-
tags: document.tags,
|
|
347
|
-
embedding: []
|
|
348
|
-
}, tokens);
|
|
349
|
-
return { id: document.id, score };
|
|
350
|
-
})
|
|
351
|
-
.filter((row) => row.score > 0)
|
|
352
|
-
.sort((left, right) => right.score - left.score || left.id.localeCompare(right.id))
|
|
353
|
-
.slice(0, limit);
|
|
354
|
-
return scored.map((row) => row.id);
|
|
355
|
-
},
|
|
356
|
-
listAgents: async () => {
|
|
357
|
-
const index = await load();
|
|
358
|
-
const counts = index.documents.reduce((state, document) => {
|
|
359
|
-
state.set(document.agentId, (state.get(document.agentId) ?? 0) + 1);
|
|
360
|
-
return state;
|
|
361
|
-
}, new Map());
|
|
362
|
-
return Array.from(counts.entries())
|
|
363
|
-
.sort((left, right) => left[0].localeCompare(right[0]))
|
|
364
|
-
.map(([id, documentCount]) => ({ id, documentCount }));
|
|
365
|
-
},
|
|
366
|
-
close: () => {
|
|
367
|
-
// File-based index has no persistent connection.
|
|
368
|
-
}
|
|
369
|
-
};
|
|
370
|
-
};
|
|
141
|
+
});
|