@andespindola/brainlink 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -9
- package/dist/application/add-note.js +2 -2
- package/dist/application/build-context.js +16 -10
- package/dist/application/canonical-context-links.js +44 -5
- package/dist/application/check-package-update.js +105 -0
- package/dist/application/frontend/client/chunk-fetch.js +236 -0
- package/dist/application/frontend/client/controls.js +178 -0
- package/dist/application/frontend/client/elements.js +122 -0
- package/dist/application/frontend/client/input.js +202 -0
- package/dist/application/frontend/client/node-details.js +191 -0
- package/dist/application/frontend/client/rendering.js +296 -0
- package/dist/application/frontend/client/scope-theme.js +114 -0
- package/dist/application/frontend/client/spatial.js +98 -0
- package/dist/application/frontend/client/storage.js +215 -0
- package/dist/application/frontend/client/upload.js +90 -0
- package/dist/application/frontend/client/worker-bootstrap.js +147 -0
- package/dist/application/frontend/client-js.js +24 -1837
- package/dist/application/frontend/client-render-worker-js.js +1 -1
- package/dist/application/index-vault-phases.js +189 -0
- package/dist/application/index-vault.js +44 -165
- package/dist/application/server/routes.js +12 -9
- package/dist/cli/commands/write/dedupe-commands.js +59 -0
- package/dist/cli/commands/write/index-commands.js +205 -0
- package/dist/cli/commands/write/link-commands.js +68 -0
- package/dist/cli/commands/write/note-commands.js +146 -0
- package/dist/cli/commands/write/server-commands.js +553 -0
- package/dist/cli/commands/write/shared.js +35 -0
- package/dist/cli/commands/write/vault-lifecycle-commands.js +270 -0
- package/dist/cli/commands/write-commands.js +12 -1303
- package/dist/cli/main.js +39 -3
- package/dist/domain/context.js +39 -3
- package/dist/domain/embeddings.js +31 -5
- package/dist/domain/graph-contexts.js +62 -57
- package/dist/domain/graph-layout/cauliflower-layout.js +116 -0
- package/dist/domain/graph-layout/collisions.js +100 -0
- package/dist/domain/graph-layout/hierarchy.js +135 -0
- package/dist/domain/graph-layout/metrics.js +111 -0
- package/dist/domain/graph-layout/segments.js +76 -0
- package/dist/domain/graph-layout/star-layout.js +110 -0
- package/dist/domain/graph-layout.js +4 -625
- package/dist/infrastructure/config.js +10 -4
- package/dist/infrastructure/file-index.js +13 -4
- package/dist/infrastructure/semantic-prefilter.js +24 -0
- package/dist/mcp/server.js +7 -0
- package/dist/mcp/tool-guard.js +29 -0
- package/dist/mcp/tools/maintenance-tools.js +409 -0
- package/dist/mcp/tools/read-tools.js +504 -0
- package/dist/mcp/tools/shared.js +216 -0
- package/dist/mcp/tools/write-tools.js +247 -0
- package/dist/mcp/tools.js +3 -1357
- package/docs/AGENT_USAGE.md +4 -4
- package/docs/QUICKSTART.md +5 -1
- package/package.json +2 -2
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { createIndexedDocument, graphLinkModelVersion } from '../domain/markdown.js';
|
|
2
|
+
import { sharedAgentId } from '../domain/agents.js';
|
|
3
|
+
import { createEmbeddingBuckets, createEmbeddingProvider } from '../domain/embeddings.js';
|
|
4
|
+
import { buildSearchPacks, toSearchPackBuildOptions } from '../infrastructure/search-packs.js';
|
|
5
|
+
const toTitleKey = (title) => title.toLowerCase();
|
|
6
|
+
const appendTitleEntry = (map, document) => {
|
|
7
|
+
const key = toTitleKey(document.title);
|
|
8
|
+
if (!map.has(key)) {
|
|
9
|
+
map.set(key, {
|
|
10
|
+
id: document.id,
|
|
11
|
+
path: document.path
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
return map;
|
|
15
|
+
};
|
|
16
|
+
export const createTitleMaps = (documents) => [...documents]
|
|
17
|
+
.sort((left, right) => left.path.localeCompare(right.path))
|
|
18
|
+
.reduce((state, document) => {
|
|
19
|
+
const agentMap = state.byAgent.get(document.agentId) ?? new Map();
|
|
20
|
+
appendTitleEntry(agentMap, document);
|
|
21
|
+
state.byAgent.set(document.agentId, agentMap);
|
|
22
|
+
if (document.agentId === sharedAgentId) {
|
|
23
|
+
appendTitleEntry(state.shared, document);
|
|
24
|
+
}
|
|
25
|
+
return state;
|
|
26
|
+
}, {
|
|
27
|
+
shared: new Map(),
|
|
28
|
+
byAgent: new Map()
|
|
29
|
+
});
|
|
30
|
+
const createScopedTitleResolver = (document, titleMaps) => ({
|
|
31
|
+
get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
|
|
32
|
+
});
|
|
33
|
+
const embedIndexedDocuments = async (documents, providerName) => {
|
|
34
|
+
if (documents.length === 0) {
|
|
35
|
+
return documents;
|
|
36
|
+
}
|
|
37
|
+
const provider = createEmbeddingProvider(providerName);
|
|
38
|
+
const chunks = documents.flatMap((document) => document.chunks);
|
|
39
|
+
const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
|
|
40
|
+
const embeddingByChunkId = new Map(chunks.map((chunk, index) => [chunk.id, embeddings[index] ?? []]));
|
|
41
|
+
return documents.map((indexedDocument) => ({
|
|
42
|
+
...indexedDocument,
|
|
43
|
+
chunks: indexedDocument.chunks.map((chunk) => {
|
|
44
|
+
const embedding = embeddingByChunkId.get(chunk.id) ?? [];
|
|
45
|
+
return {
|
|
46
|
+
...chunk,
|
|
47
|
+
embeddingProvider: provider.name,
|
|
48
|
+
embedding,
|
|
49
|
+
buckets: embedding.length > 0 ? createEmbeddingBuckets(embedding) : []
|
|
50
|
+
};
|
|
51
|
+
})
|
|
52
|
+
}));
|
|
53
|
+
};
|
|
54
|
+
const relinkIndexedDocument = (indexedDocument, titleMaps) => {
|
|
55
|
+
const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
|
|
56
|
+
return {
|
|
57
|
+
...indexedDocument,
|
|
58
|
+
links: indexedDocument.links
|
|
59
|
+
.map((link) => ({
|
|
60
|
+
...link,
|
|
61
|
+
toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
|
|
62
|
+
}))
|
|
63
|
+
.filter((link) => link.toDocumentId !== indexedDocument.document.id)
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
export const detectChanges = (params) => {
|
|
67
|
+
const { summaries, previousState, config, fullReindex, existingByPath, previousSnapshotMap, currentSnapshotMap } = params;
|
|
68
|
+
const graphLinkModelChanged = previousState != null &&
|
|
69
|
+
previousState.graphLinkModelVersion !== graphLinkModelVersion;
|
|
70
|
+
const fullSourceReindex = fullReindex || graphLinkModelChanged;
|
|
71
|
+
const settingsChanged = previousState == null ||
|
|
72
|
+
previousState.chunkSize !== config.chunkSize ||
|
|
73
|
+
previousState.embeddingProvider !== config.embeddingProvider ||
|
|
74
|
+
graphLinkModelChanged;
|
|
75
|
+
const packSettingsChanged = previousState == null ||
|
|
76
|
+
previousState.searchPackRowChunkSize !== config.searchPack.rowChunkSize ||
|
|
77
|
+
previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
|
|
78
|
+
previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
|
|
79
|
+
const changedPaths = new Set();
|
|
80
|
+
for (let index = 0; index < summaries.length; index += 1) {
|
|
81
|
+
const summary = summaries[index];
|
|
82
|
+
const previous = previousSnapshotMap.get(summary.relativePath);
|
|
83
|
+
const changed = fullSourceReindex ||
|
|
84
|
+
settingsChanged ||
|
|
85
|
+
previous == null ||
|
|
86
|
+
previous.mtimeMs !== summary.updatedAt.getTime() ||
|
|
87
|
+
previous.size !== summary.size ||
|
|
88
|
+
!existingByPath.has(summary.relativePath);
|
|
89
|
+
if (changed) {
|
|
90
|
+
changedPaths.add(summary.relativePath);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const hasDeletes = previousState
|
|
94
|
+
? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
|
|
95
|
+
: false;
|
|
96
|
+
return { graphLinkModelChanged, settingsChanged, packSettingsChanged, changedPaths, hasDeletes };
|
|
97
|
+
};
|
|
98
|
+
export const decidePackRebuild = (params) => {
|
|
99
|
+
const { existingPackManifest, fullReindex, graphLinkModelChanged, manifestRepaired, settingsChanged, packSettingsChanged, hasDeletes, changedCount, changeRatio, pendingPackChanges } = params;
|
|
100
|
+
const shouldRebuild = !existingPackManifest ||
|
|
101
|
+
fullReindex ||
|
|
102
|
+
graphLinkModelChanged ||
|
|
103
|
+
settingsChanged ||
|
|
104
|
+
packSettingsChanged ||
|
|
105
|
+
hasDeletes ||
|
|
106
|
+
changedCount >= 400 ||
|
|
107
|
+
changeRatio >= 0.04 ||
|
|
108
|
+
pendingPackChanges >= 1200;
|
|
109
|
+
const reason = !existingPackManifest
|
|
110
|
+
? 'Missing pack manifest'
|
|
111
|
+
: fullReindex
|
|
112
|
+
? 'Full reindex requested'
|
|
113
|
+
: graphLinkModelChanged
|
|
114
|
+
? 'Graph link model changed'
|
|
115
|
+
: manifestRepaired
|
|
116
|
+
? 'Pack manifest repaired from existing packs'
|
|
117
|
+
: settingsChanged
|
|
118
|
+
? 'Index settings changed'
|
|
119
|
+
: packSettingsChanged
|
|
120
|
+
? 'Search pack settings changed'
|
|
121
|
+
: hasDeletes
|
|
122
|
+
? 'Document deletions detected'
|
|
123
|
+
: changedCount >= 400
|
|
124
|
+
? 'Changed file count threshold reached'
|
|
125
|
+
: changeRatio >= 0.04
|
|
126
|
+
? 'Change ratio threshold reached'
|
|
127
|
+
: pendingPackChanges >= 1200
|
|
128
|
+
? 'Pending pack changes threshold reached'
|
|
129
|
+
: 'Pack rebuild skipped';
|
|
130
|
+
return { shouldRebuild, reason };
|
|
131
|
+
};
|
|
132
|
+
export const embedChangedDocuments = async (params) => {
|
|
133
|
+
const { changedDocumentsByPath, titleMaps, config, emit } = params;
|
|
134
|
+
emit('embed', 'start', 'Embedding changed chunks', {
|
|
135
|
+
changedDocuments: changedDocumentsByPath.size
|
|
136
|
+
});
|
|
137
|
+
const changedIndexedDocuments = changedDocumentsByPath.size > 0
|
|
138
|
+
? await embedIndexedDocuments(Array.from(changedDocumentsByPath.values()).map((document) => createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize)), config.embeddingProvider)
|
|
139
|
+
: [];
|
|
140
|
+
emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
|
|
141
|
+
changedIndexedDocuments: changedIndexedDocuments.length
|
|
142
|
+
});
|
|
143
|
+
return changedIndexedDocuments;
|
|
144
|
+
};
|
|
145
|
+
export const assembleIndexedDocuments = (params) => {
|
|
146
|
+
const { documents, changedIndexedDocuments, existingByPath, titleMaps, config, needsRelink } = params;
|
|
147
|
+
const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
|
|
148
|
+
return documents.map((document) => {
|
|
149
|
+
const changed = changedIndexedByPath.get(document.path);
|
|
150
|
+
if (changed) {
|
|
151
|
+
return changed;
|
|
152
|
+
}
|
|
153
|
+
const existing = existingByPath.get(document.path);
|
|
154
|
+
if (!existing) {
|
|
155
|
+
return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
|
|
156
|
+
}
|
|
157
|
+
return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
|
|
158
|
+
});
|
|
159
|
+
};
|
|
160
|
+
export const rebuildSearchPacksIfNeeded = async (params) => {
|
|
161
|
+
const { shouldRebuild, reason, absoluteVaultPath, indexedDocuments, config, emit } = params;
|
|
162
|
+
if (!shouldRebuild) {
|
|
163
|
+
emit('packs', 'skip', 'Pack rebuild not required', {
|
|
164
|
+
reason
|
|
165
|
+
});
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
emit('packs', 'start', 'Rebuilding compressed search packs', {
|
|
169
|
+
reason
|
|
170
|
+
});
|
|
171
|
+
try {
|
|
172
|
+
const packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
|
|
173
|
+
emit('packs', 'finish', 'Compressed packs rebuilt', {
|
|
174
|
+
reason,
|
|
175
|
+
packCount: packResult.packCount,
|
|
176
|
+
recordCount: packResult.recordCount,
|
|
177
|
+
durationMs: packResult.durationMs,
|
|
178
|
+
compressionRatio: packResult.compression.ratio
|
|
179
|
+
});
|
|
180
|
+
return packResult;
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// Pack generation is best-effort. The JSON index remains the primary path.
|
|
184
|
+
emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
|
|
185
|
+
reason
|
|
186
|
+
});
|
|
187
|
+
return undefined;
|
|
188
|
+
}
|
|
189
|
+
};
|
|
@@ -1,69 +1,11 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
|
-
import {
|
|
3
|
-
import { sharedAgentId } from '../domain/agents.js';
|
|
4
|
-
import { createEmbeddingProvider } from '../domain/embeddings.js';
|
|
2
|
+
import { graphLinkModelVersion, parseMarkdownDocument } from '../domain/markdown.js';
|
|
5
3
|
import { loadBrainlinkConfig } from '../infrastructure/config.js';
|
|
6
4
|
import { ensureVault, readMarkdownFileSummaries } from '../infrastructure/file-system-vault.js';
|
|
7
5
|
import { readIndexState, writeIndexState } from '../infrastructure/index-state.js';
|
|
8
|
-
import {
|
|
6
|
+
import { ensureSearchPackManifest } from '../infrastructure/search-packs.js';
|
|
9
7
|
import { openFileIndex } from '../infrastructure/file-index.js';
|
|
10
|
-
|
|
11
|
-
const appendTitleEntry = (map, document) => {
|
|
12
|
-
const key = toTitleKey(document.title);
|
|
13
|
-
if (!map.has(key)) {
|
|
14
|
-
map.set(key, {
|
|
15
|
-
id: document.id,
|
|
16
|
-
path: document.path
|
|
17
|
-
});
|
|
18
|
-
}
|
|
19
|
-
return map;
|
|
20
|
-
};
|
|
21
|
-
const createTitleMaps = (documents) => [...documents]
|
|
22
|
-
.sort((left, right) => left.path.localeCompare(right.path))
|
|
23
|
-
.reduce((state, document) => {
|
|
24
|
-
const agentMap = state.byAgent.get(document.agentId) ?? new Map();
|
|
25
|
-
appendTitleEntry(agentMap, document);
|
|
26
|
-
state.byAgent.set(document.agentId, agentMap);
|
|
27
|
-
if (document.agentId === sharedAgentId) {
|
|
28
|
-
appendTitleEntry(state.shared, document);
|
|
29
|
-
}
|
|
30
|
-
return state;
|
|
31
|
-
}, {
|
|
32
|
-
shared: new Map(),
|
|
33
|
-
byAgent: new Map()
|
|
34
|
-
});
|
|
35
|
-
const createScopedTitleResolver = (document, titleMaps) => ({
|
|
36
|
-
get: (title) => titleMaps.byAgent.get(document.agentId)?.get(title)?.id ?? titleMaps.shared.get(title)?.id
|
|
37
|
-
});
|
|
38
|
-
const embedIndexedDocuments = async (documents, providerName) => {
|
|
39
|
-
if (documents.length === 0) {
|
|
40
|
-
return documents;
|
|
41
|
-
}
|
|
42
|
-
const provider = createEmbeddingProvider(providerName);
|
|
43
|
-
const chunks = documents.flatMap((document) => document.chunks);
|
|
44
|
-
const embeddings = await provider.embed(chunks.map((chunk) => chunk.content));
|
|
45
|
-
const embeddingByChunkId = new Map(chunks.map((chunk, index) => [chunk.id, embeddings[index] ?? []]));
|
|
46
|
-
return documents.map((indexedDocument) => ({
|
|
47
|
-
...indexedDocument,
|
|
48
|
-
chunks: indexedDocument.chunks.map((chunk) => ({
|
|
49
|
-
...chunk,
|
|
50
|
-
embeddingProvider: provider.name,
|
|
51
|
-
embedding: embeddingByChunkId.get(chunk.id) ?? []
|
|
52
|
-
}))
|
|
53
|
-
}));
|
|
54
|
-
};
|
|
55
|
-
const relinkIndexedDocument = (indexedDocument, titleMaps) => {
|
|
56
|
-
const resolver = createScopedTitleResolver(indexedDocument.document, titleMaps);
|
|
57
|
-
return {
|
|
58
|
-
...indexedDocument,
|
|
59
|
-
links: indexedDocument.links
|
|
60
|
-
.map((link) => ({
|
|
61
|
-
...link,
|
|
62
|
-
toDocumentId: resolver.get(link.toTitle.toLowerCase()) ?? null
|
|
63
|
-
}))
|
|
64
|
-
.filter((link) => link.toDocumentId !== indexedDocument.document.id)
|
|
65
|
-
};
|
|
66
|
-
};
|
|
8
|
+
import { assembleIndexedDocuments, createTitleMaps, decidePackRebuild, detectChanges, embedChangedDocuments, rebuildSearchPacksIfNeeded } from './index-vault-phases.js';
|
|
67
9
|
const toIndexResult = (documents) => ({
|
|
68
10
|
documentCount: documents.length,
|
|
69
11
|
chunkCount: documents.reduce((total, document) => total + document.chunks.length, 0),
|
|
@@ -121,34 +63,15 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
|
|
|
121
63
|
const currentSnapshot = toSnapshot(summaries);
|
|
122
64
|
const currentSnapshotMap = createSnapshotMap(currentSnapshot);
|
|
123
65
|
const previousSnapshotMap = createSnapshotMap(previousState?.files ?? []);
|
|
124
|
-
const graphLinkModelChanged
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
previousState.searchPackCompressionLevel !== config.searchPack.compressionLevel ||
|
|
134
|
-
previousState.searchPackUseDictionary !== config.searchPack.useDictionary;
|
|
135
|
-
const changedPaths = new Set();
|
|
136
|
-
for (let index = 0; index < summaries.length; index += 1) {
|
|
137
|
-
const summary = summaries[index];
|
|
138
|
-
const previous = previousSnapshotMap.get(summary.relativePath);
|
|
139
|
-
const changed = fullSourceReindex ||
|
|
140
|
-
settingsChanged ||
|
|
141
|
-
previous == null ||
|
|
142
|
-
previous.mtimeMs !== summary.updatedAt.getTime() ||
|
|
143
|
-
previous.size !== summary.size ||
|
|
144
|
-
!existingByPath.has(summary.relativePath);
|
|
145
|
-
if (changed) {
|
|
146
|
-
changedPaths.add(summary.relativePath);
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
const hasDeletes = previousState
|
|
150
|
-
? previousState.files.some((entry) => !currentSnapshotMap.has(entry.path))
|
|
151
|
-
: false;
|
|
66
|
+
const { graphLinkModelChanged, settingsChanged, packSettingsChanged, changedPaths, hasDeletes } = detectChanges({
|
|
67
|
+
summaries,
|
|
68
|
+
previousState,
|
|
69
|
+
config,
|
|
70
|
+
fullReindex,
|
|
71
|
+
existingByPath,
|
|
72
|
+
previousSnapshotMap,
|
|
73
|
+
currentSnapshotMap
|
|
74
|
+
});
|
|
152
75
|
const manifestRecovery = await ensureSearchPackManifest(absoluteVaultPath);
|
|
153
76
|
if (changedPaths.size === 0 &&
|
|
154
77
|
!hasDeletes &&
|
|
@@ -188,27 +111,20 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
|
|
|
188
111
|
return existing ? [existing.document] : [];
|
|
189
112
|
});
|
|
190
113
|
const titleMaps = createTitleMaps(documents);
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
: [];
|
|
197
|
-
emit('embed', changedDocumentsByPath.size > 0 ? 'finish' : 'skip', changedDocumentsByPath.size > 0 ? 'Embedding complete' : 'Embedding skipped', {
|
|
198
|
-
changedIndexedDocuments: changedIndexedDocuments.length
|
|
114
|
+
const changedIndexedDocuments = await embedChangedDocuments({
|
|
115
|
+
changedDocumentsByPath,
|
|
116
|
+
titleMaps,
|
|
117
|
+
config,
|
|
118
|
+
emit
|
|
199
119
|
});
|
|
200
|
-
const changedIndexedByPath = new Map(changedIndexedDocuments.map((document) => [document.document.path, document]));
|
|
201
120
|
const needsRelink = settingsChanged || hasDeletes || changedPaths.size > 0;
|
|
202
|
-
const indexedDocuments =
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
return createIndexedDocument(document, createScopedTitleResolver(document, titleMaps), config.chunkSize);
|
|
210
|
-
}
|
|
211
|
-
return needsRelink ? relinkIndexedDocument(existing, titleMaps) : existing;
|
|
121
|
+
const indexedDocuments = assembleIndexedDocuments({
|
|
122
|
+
documents,
|
|
123
|
+
changedIndexedDocuments,
|
|
124
|
+
existingByPath,
|
|
125
|
+
titleMaps,
|
|
126
|
+
config,
|
|
127
|
+
needsRelink
|
|
212
128
|
});
|
|
213
129
|
emit('persist', 'start', 'Persisting index');
|
|
214
130
|
await index.saveDocuments(indexedDocuments);
|
|
@@ -221,63 +137,26 @@ export const indexVaultWithOptions = async (vaultPath, options) => {
|
|
|
221
137
|
const changeRatio = changedCount / documentCount;
|
|
222
138
|
const previousPendingPackChanges = previousState?.pendingPackChanges ?? 0;
|
|
223
139
|
const pendingPackChanges = previousPendingPackChanges + changedCount;
|
|
224
|
-
const shouldRebuildPacks =
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
: packSettingsChanged
|
|
245
|
-
? 'Search pack settings changed'
|
|
246
|
-
: hasDeletes
|
|
247
|
-
? 'Document deletions detected'
|
|
248
|
-
: changedCount >= 400
|
|
249
|
-
? 'Changed file count threshold reached'
|
|
250
|
-
: changeRatio >= 0.04
|
|
251
|
-
? 'Change ratio threshold reached'
|
|
252
|
-
: pendingPackChanges >= 1200
|
|
253
|
-
? 'Pending pack changes threshold reached'
|
|
254
|
-
: 'Pack rebuild skipped';
|
|
255
|
-
if (shouldRebuildPacks) {
|
|
256
|
-
emit('packs', 'start', 'Rebuilding compressed search packs', {
|
|
257
|
-
reason: packReason
|
|
258
|
-
});
|
|
259
|
-
try {
|
|
260
|
-
packResult = await buildSearchPacks(absoluteVaultPath, indexedDocuments, toSearchPackBuildOptions(config));
|
|
261
|
-
emit('packs', 'finish', 'Compressed packs rebuilt', {
|
|
262
|
-
reason: packReason,
|
|
263
|
-
packCount: packResult.packCount,
|
|
264
|
-
recordCount: packResult.recordCount,
|
|
265
|
-
durationMs: packResult.durationMs,
|
|
266
|
-
compressionRatio: packResult.compression.ratio
|
|
267
|
-
});
|
|
268
|
-
}
|
|
269
|
-
catch {
|
|
270
|
-
// Pack generation is best-effort. The JSON index remains the primary path.
|
|
271
|
-
emit('packs', 'skip', 'Pack rebuild failed; continuing with JSON index', {
|
|
272
|
-
reason: packReason
|
|
273
|
-
});
|
|
274
|
-
}
|
|
275
|
-
}
|
|
276
|
-
else {
|
|
277
|
-
emit('packs', 'skip', 'Pack rebuild not required', {
|
|
278
|
-
reason: packReason
|
|
279
|
-
});
|
|
280
|
-
}
|
|
140
|
+
const { shouldRebuild: shouldRebuildPacks, reason: packReason } = decidePackRebuild({
|
|
141
|
+
existingPackManifest,
|
|
142
|
+
fullReindex,
|
|
143
|
+
graphLinkModelChanged,
|
|
144
|
+
manifestRepaired: manifestRecovery.repaired,
|
|
145
|
+
settingsChanged,
|
|
146
|
+
packSettingsChanged,
|
|
147
|
+
hasDeletes,
|
|
148
|
+
changedCount,
|
|
149
|
+
changeRatio,
|
|
150
|
+
pendingPackChanges
|
|
151
|
+
});
|
|
152
|
+
const packResult = await rebuildSearchPacksIfNeeded({
|
|
153
|
+
shouldRebuild: shouldRebuildPacks,
|
|
154
|
+
reason: packReason,
|
|
155
|
+
absoluteVaultPath,
|
|
156
|
+
indexedDocuments,
|
|
157
|
+
config,
|
|
158
|
+
emit
|
|
159
|
+
});
|
|
281
160
|
const packsRebuilt = packResult != null;
|
|
282
161
|
const packResultReason = shouldRebuildPacks && !packsRebuilt ? `${packReason} (failed)` : packReason;
|
|
283
162
|
await writeIndexState(absoluteVaultPath, {
|
|
@@ -24,19 +24,20 @@ import { createClientWorkerJs } from '../frontend/client-worker-js.js';
|
|
|
24
24
|
import { createClientRenderWorkerJs } from '../frontend/client-render-worker-js.js';
|
|
25
25
|
import { contentTypes, createJsonResponse, isReadMethod, parsePositiveInteger } from './http.js';
|
|
26
26
|
import { parseMultipartForm } from './multipart.js';
|
|
27
|
-
const
|
|
27
|
+
const readRuntimeDefaults = async (url) => {
|
|
28
28
|
const config = await loadBrainlinkConfig();
|
|
29
|
-
|
|
29
|
+
return resolveAgentRuntimeDefaults(config, readAgentQuery(url));
|
|
30
|
+
};
|
|
31
|
+
const readSearchMode = async (url) => {
|
|
32
|
+
const defaults = await readRuntimeDefaults(url);
|
|
30
33
|
return sanitizeSearchMode(url.searchParams.get('mode'), defaults.defaultSearchMode);
|
|
31
34
|
};
|
|
32
35
|
const readContextStrategy = async (url) => {
|
|
33
|
-
const
|
|
34
|
-
const defaults = resolveAgentRuntimeDefaults(config, readAgentQuery(url));
|
|
36
|
+
const defaults = await readRuntimeDefaults(url);
|
|
35
37
|
return sanitizeContextStrategy(url.searchParams.get('strategy'), defaults.defaultContextStrategy);
|
|
36
38
|
};
|
|
37
39
|
const readContextCacheTtlMs = async (url) => {
|
|
38
|
-
const
|
|
39
|
-
const defaults = resolveAgentRuntimeDefaults(config, readAgentQuery(url));
|
|
40
|
+
const defaults = await readRuntimeDefaults(url);
|
|
40
41
|
return defaults.defaultContextCacheTtlMs;
|
|
41
42
|
};
|
|
42
43
|
const hasInvalidSearchMode = (url) => {
|
|
@@ -428,7 +429,8 @@ export const route = async (request, url, vaultPath) => {
|
|
|
428
429
|
}
|
|
429
430
|
if (isReadMethod(request) && url.pathname === '/api/search') {
|
|
430
431
|
const query = url.searchParams.get('q') ?? '';
|
|
431
|
-
const
|
|
432
|
+
const defaults = await readRuntimeDefaults(url);
|
|
433
|
+
const limit = parsePositiveInteger(url.searchParams.get('limit'), defaults.defaultSearchLimit);
|
|
432
434
|
const mode = await readSearchMode(url);
|
|
433
435
|
if (hasInvalidSearchMode(url)) {
|
|
434
436
|
return createResponse(createJsonResponse({ error: 'Invalid mode. Use fts, semantic or hybrid.' }), 400, contentTypes['.json']);
|
|
@@ -437,8 +439,9 @@ export const route = async (request, url, vaultPath) => {
|
|
|
437
439
|
}
|
|
438
440
|
if (isReadMethod(request) && url.pathname === '/api/context') {
|
|
439
441
|
const query = url.searchParams.get('q') ?? '';
|
|
440
|
-
const
|
|
441
|
-
const
|
|
442
|
+
const defaults = await readRuntimeDefaults(url);
|
|
443
|
+
const limit = parsePositiveInteger(url.searchParams.get('limit'), defaults.defaultSearchLimit);
|
|
444
|
+
const tokens = parsePositiveInteger(url.searchParams.get('tokens'), defaults.defaultContextTokens);
|
|
442
445
|
const mode = await readSearchMode(url);
|
|
443
446
|
const strategy = await readContextStrategy(url);
|
|
444
447
|
const contextCacheTtlMs = await readContextCacheTtlMs(url);
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { resolveDuplicateNotes, scanDuplicateNotes } from '../../../application/dedupe-notes.js';
|
|
2
|
+
import { parsePositiveInteger, print, resolveOptions } from '../../runtime.js';
|
|
3
|
+
import { parseScore } from './shared.js';
|
|
4
|
+
export const registerDedupeCommands = (program) => {
|
|
5
|
+
program
|
|
6
|
+
.command('dedupe')
|
|
7
|
+
.option('-v, --vault <vault>', 'vault directory')
|
|
8
|
+
.option('-a, --agent <agent>', 'agent memory namespace')
|
|
9
|
+
.option('-l, --limit <limit>', 'maximum duplicate candidate pairs')
|
|
10
|
+
.option('--min-score <score>', 'minimum semantic similarity score between 0 and 1', '0.92')
|
|
11
|
+
.option('--no-semantic', 'disable semantic duplicate detection and keep exact-content matching only')
|
|
12
|
+
.option('--json', 'print machine-readable JSON')
|
|
13
|
+
.description('detect possible duplicate notes with exact hash and semantic similarity scores')
|
|
14
|
+
.action(async (options) => {
|
|
15
|
+
const resolved = await resolveOptions(options);
|
|
16
|
+
const duplicates = await scanDuplicateNotes(resolved.vault, {
|
|
17
|
+
agentId: resolved.agent,
|
|
18
|
+
limit: parsePositiveInteger(options.limit ?? '25', 25),
|
|
19
|
+
minSemanticScore: parseScore(options.minScore, 0.92),
|
|
20
|
+
includeSemantic: options.semantic !== false
|
|
21
|
+
});
|
|
22
|
+
print(options.json, { vault: resolved.vault, agent: resolved.agent, duplicates }, () => {
|
|
23
|
+
if (duplicates.length === 0) {
|
|
24
|
+
return 'No possible duplicates found.';
|
|
25
|
+
}
|
|
26
|
+
return duplicates
|
|
27
|
+
.map((item, index) => `${index + 1}. [${item.kind}] score=${item.score.toFixed(4)} ${item.left.path} <-> ${item.right.path} (${item.reason})`)
|
|
28
|
+
.join('\n');
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
program
|
|
32
|
+
.command('dedupe-resolve')
|
|
33
|
+
.option('-v, --vault <vault>', 'vault directory')
|
|
34
|
+
.option('--left <path>', 'left note relative path from dedupe result')
|
|
35
|
+
.option('--right <path>', 'right note relative path from dedupe result')
|
|
36
|
+
.option('--action <action>', 'resolution action: merge, link or ignore')
|
|
37
|
+
.option('--no-auto-index', 'skip reindex after duplicate resolution')
|
|
38
|
+
.option('--json', 'print machine-readable JSON')
|
|
39
|
+
.description('resolve a duplicate candidate with merge, link or ignore')
|
|
40
|
+
.action(async (options) => {
|
|
41
|
+
const resolved = await resolveOptions(options);
|
|
42
|
+
if (!options.left || !options.right) {
|
|
43
|
+
throw new Error('Use --left <path> and --right <path> to resolve a duplicate pair.');
|
|
44
|
+
}
|
|
45
|
+
if (options.action !== 'merge' && options.action !== 'link' && options.action !== 'ignore') {
|
|
46
|
+
throw new Error('Use --action merge|link|ignore.');
|
|
47
|
+
}
|
|
48
|
+
const result = await resolveDuplicateNotes(resolved.vault, {
|
|
49
|
+
leftPath: options.left,
|
|
50
|
+
rightPath: options.right,
|
|
51
|
+
action: options.action,
|
|
52
|
+
autoIndex: options.autoIndex !== false
|
|
53
|
+
});
|
|
54
|
+
print(options.json, {
|
|
55
|
+
vault: resolved.vault,
|
|
56
|
+
...result
|
|
57
|
+
}, () => `Resolved duplicate (${result.action}) for ${result.leftPath} <-> ${result.rightPath}`);
|
|
58
|
+
});
|
|
59
|
+
};
|