byterover-cli 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -81
- package/dist/agent/core/domain/llm/index.d.ts +1 -1
- package/dist/agent/core/domain/llm/index.js +1 -1
- package/dist/agent/core/domain/llm/registry.d.ts +8 -0
- package/dist/agent/core/domain/llm/registry.js +34 -0
- package/dist/agent/core/domain/sandbox/types.d.ts +2 -0
- package/dist/agent/core/domain/tools/constants.d.ts +3 -0
- package/dist/agent/core/domain/tools/constants.js +3 -0
- package/dist/agent/core/interfaces/cipher-services.d.ts +2 -4
- package/dist/agent/core/interfaces/i-cipher-agent.d.ts +9 -1
- package/dist/agent/core/interfaces/i-sandbox-service.d.ts +8 -0
- package/dist/agent/core/interfaces/i-tool-provider.d.ts +10 -0
- package/dist/agent/core/interfaces/i-tool-scheduler.d.ts +9 -0
- package/dist/agent/infra/agent/agent-schemas.d.ts +0 -9
- package/dist/agent/infra/agent/agent-schemas.js +0 -3
- package/dist/agent/infra/agent/cipher-agent.d.ts +25 -1
- package/dist/agent/infra/agent/cipher-agent.js +138 -11
- package/dist/agent/infra/agent/provider-update-config.d.ts +0 -2
- package/dist/agent/infra/agent/service-initializer.d.ts +2 -6
- package/dist/agent/infra/agent/service-initializer.js +45 -38
- package/dist/agent/infra/blob/blob-storage-factory.d.ts +2 -2
- package/dist/agent/infra/blob/blob-storage-factory.js +4 -4
- package/dist/agent/infra/blob/file-blob-storage.d.ts +96 -0
- package/dist/agent/infra/blob/file-blob-storage.js +454 -0
- package/dist/agent/infra/blob/index.d.ts +2 -3
- package/dist/agent/infra/blob/index.js +4 -6
- package/dist/agent/infra/llm/agent-llm-service.d.ts +3 -0
- package/dist/agent/infra/llm/agent-llm-service.js +34 -52
- package/dist/agent/infra/llm/context/compression/compression-helpers.d.ts +35 -0
- package/dist/agent/infra/llm/context/compression/compression-helpers.js +124 -0
- package/dist/agent/infra/llm/context/compression/escalated-compression.d.ts +62 -0
- package/dist/agent/infra/llm/context/compression/escalated-compression.js +144 -0
- package/dist/agent/infra/llm/context/compression/index.d.ts +3 -0
- package/dist/agent/infra/llm/context/compression/index.js +3 -0
- package/dist/agent/infra/llm/context/compression/reactive-overflow.d.ts +0 -27
- package/dist/agent/infra/llm/context/compression/reactive-overflow.js +5 -122
- package/dist/agent/infra/llm/context/context-manager.d.ts +20 -1
- package/dist/agent/infra/llm/context/context-manager.js +37 -7
- package/dist/agent/infra/llm/providers/index.js +0 -2
- package/dist/agent/infra/llm/providers/types.d.ts +1 -5
- package/dist/agent/infra/map/agentic-map-service.d.ts +97 -0
- package/dist/agent/infra/map/agentic-map-service.js +309 -0
- package/dist/agent/infra/map/context-tree-store.d.ts +94 -0
- package/dist/agent/infra/map/context-tree-store.js +278 -0
- package/dist/agent/infra/map/index.d.ts +4 -0
- package/dist/agent/infra/map/index.js +4 -0
- package/dist/agent/infra/map/llm-map-memory.d.ts +59 -0
- package/dist/agent/infra/map/llm-map-memory.js +187 -0
- package/dist/agent/infra/map/llm-map-service.d.ts +36 -0
- package/dist/agent/infra/map/llm-map-service.js +118 -0
- package/dist/agent/infra/map/map-shared.d.ts +140 -0
- package/dist/agent/infra/map/map-shared.js +325 -0
- package/dist/agent/infra/map/worker-pool.d.ts +45 -0
- package/dist/agent/infra/map/worker-pool.js +73 -0
- package/dist/agent/infra/sandbox/curation-helpers.d.ts +62 -0
- package/dist/agent/infra/sandbox/curation-helpers.js +219 -0
- package/dist/agent/infra/sandbox/sandbox-service.d.ts +12 -0
- package/dist/agent/infra/sandbox/sandbox-service.js +39 -7
- package/dist/agent/infra/sandbox/tools-sdk.d.ts +48 -1
- package/dist/agent/infra/sandbox/tools-sdk.js +52 -1
- package/dist/agent/infra/session/session-manager.d.ts +8 -1
- package/dist/agent/infra/session/session-manager.js +24 -4
- package/dist/agent/infra/storage/file-key-storage.d.ts +142 -0
- package/dist/agent/infra/storage/file-key-storage.js +572 -0
- package/dist/agent/infra/storage/granular-history-storage.d.ts +1 -1
- package/dist/agent/infra/storage/granular-history-storage.js +1 -1
- package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.d.ts +4 -0
- package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.js +42 -14
- package/dist/agent/infra/system-prompt/contributors/map-selection-contributor.d.ts +16 -0
- package/dist/agent/infra/system-prompt/contributors/map-selection-contributor.js +47 -0
- package/dist/agent/infra/tools/core-tool-scheduler.js +3 -1
- package/dist/agent/infra/tools/implementations/agentic-map-tool.d.ts +35 -0
- package/dist/agent/infra/tools/implementations/agentic-map-tool.js +156 -0
- package/dist/agent/infra/tools/implementations/code-exec-tool.js +1 -0
- package/dist/agent/infra/tools/implementations/curate-tool.d.ts +9 -9
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.d.ts +18 -0
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.js +43 -0
- package/dist/agent/infra/tools/implementations/llm-map-tool.d.ts +24 -0
- package/dist/agent/infra/tools/implementations/llm-map-tool.js +87 -0
- package/dist/agent/infra/tools/implementations/memory-symbol-tree.d.ts +28 -1
- package/dist/agent/infra/tools/implementations/memory-symbol-tree.js +27 -3
- package/dist/agent/infra/tools/implementations/search-knowledge-service.d.ts +1 -0
- package/dist/agent/infra/tools/implementations/search-knowledge-service.js +83 -12
- package/dist/agent/infra/tools/implementations/search-knowledge-tool.js +2 -2
- package/dist/agent/infra/tools/tool-manager.js +6 -0
- package/dist/agent/infra/tools/tool-provider.d.ts +12 -0
- package/dist/agent/infra/tools/tool-provider.js +78 -0
- package/dist/agent/infra/tools/tool-registry.d.ts +14 -0
- package/dist/agent/infra/tools/tool-registry.js +32 -0
- package/dist/agent/resources/prompts/system-prompt.yml +48 -74
- package/dist/agent/resources/tools/expand_knowledge.txt +20 -0
- package/dist/oclif/commands/curate/index.js +1 -2
- package/dist/oclif/commands/main.js +1 -0
- package/dist/oclif/commands/providers/connect.d.ts +1 -3
- package/dist/oclif/commands/providers/connect.js +7 -29
- package/dist/oclif/commands/query.js +1 -2
- package/dist/server/constants.d.ts +7 -0
- package/dist/server/constants.js +8 -0
- package/dist/server/core/domain/entities/provider-registry.js +1 -15
- package/dist/server/core/domain/knowledge/memory-scoring.js +1 -1
- package/dist/server/core/domain/knowledge/summary-types.d.ts +126 -0
- package/dist/server/core/domain/knowledge/summary-types.js +7 -0
- package/dist/server/core/domain/transport/schemas.d.ts +0 -4
- package/dist/server/core/interfaces/context-tree/i-context-tree-archive-service.d.ts +30 -0
- package/dist/server/core/interfaces/context-tree/i-context-tree-archive-service.js +1 -0
- package/dist/server/core/interfaces/context-tree/i-context-tree-manifest-service.d.ts +30 -0
- package/dist/server/core/interfaces/context-tree/i-context-tree-manifest-service.js +1 -0
- package/dist/server/core/interfaces/context-tree/i-context-tree-summary-service.d.ts +29 -0
- package/dist/server/core/interfaces/context-tree/i-context-tree-summary-service.js +1 -0
- package/dist/server/infra/cogit/context-tree-to-push-context-mapper.js +10 -3
- package/dist/server/infra/connectors/skill/skill-connector.d.ts +4 -0
- package/dist/server/infra/connectors/skill/skill-connector.js +4 -0
- package/dist/server/infra/context-tree/children-hash.d.ts +20 -0
- package/dist/server/infra/context-tree/children-hash.js +22 -0
- package/dist/server/infra/context-tree/derived-artifact.d.ts +28 -0
- package/dist/server/infra/context-tree/derived-artifact.js +48 -0
- package/dist/server/infra/context-tree/file-context-tree-archive-service.d.ts +37 -0
- package/dist/server/infra/context-tree/file-context-tree-archive-service.js +219 -0
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.d.ts +50 -0
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.js +278 -0
- package/dist/server/infra/context-tree/file-context-tree-merger.js +4 -0
- package/dist/server/infra/context-tree/file-context-tree-snapshot-service.js +12 -4
- package/dist/server/infra/context-tree/file-context-tree-summary-service.d.ts +44 -0
- package/dist/server/infra/context-tree/file-context-tree-summary-service.js +313 -0
- package/dist/server/infra/context-tree/file-context-tree-writer-service.js +5 -0
- package/dist/server/infra/context-tree/prompts/summary-generation.d.ts +22 -0
- package/dist/server/infra/context-tree/prompts/summary-generation.js +45 -0
- package/dist/server/infra/context-tree/snapshot-diff.d.ts +19 -0
- package/dist/server/infra/context-tree/snapshot-diff.js +39 -0
- package/dist/server/infra/context-tree/summary-frontmatter.d.ts +24 -0
- package/dist/server/infra/context-tree/summary-frontmatter.js +111 -0
- package/dist/server/infra/daemon/agent-process.js +2 -14
- package/dist/server/infra/executor/curate-executor.d.ts +1 -0
- package/dist/server/infra/executor/curate-executor.js +82 -34
- package/dist/server/infra/executor/folder-pack-executor.js +1 -1
- package/dist/server/infra/executor/pre-compaction/compaction-escalation.d.ts +6 -0
- package/dist/server/infra/executor/pre-compaction/compaction-escalation.js +6 -0
- package/dist/server/infra/executor/pre-compaction/index.d.ts +3 -0
- package/dist/server/infra/executor/pre-compaction/index.js +1 -0
- package/dist/server/infra/executor/pre-compaction/pre-compaction-service.d.ts +59 -0
- package/dist/server/infra/executor/pre-compaction/pre-compaction-service.js +124 -0
- package/dist/server/infra/executor/pre-compaction/prompts.d.ts +24 -0
- package/dist/server/infra/executor/pre-compaction/prompts.js +47 -0
- package/dist/server/infra/executor/query-executor.d.ts +3 -0
- package/dist/server/infra/executor/query-executor.js +39 -4
- package/dist/server/infra/http/authenticated-http-client.js +4 -0
- package/dist/server/infra/http/provider-model-fetcher-registry.js +1 -5
- package/dist/server/infra/http/provider-model-fetchers.d.ts +0 -14
- package/dist/server/infra/http/provider-model-fetchers.js +0 -132
- package/dist/server/infra/provider/provider-config-resolver.js +0 -55
- package/dist/server/utils/curate-result-parser.d.ts +4 -4
- package/dist/shared/constants/curation.d.ts +6 -0
- package/dist/shared/constants/curation.js +6 -0
- package/dist/shared/utils/escalation-utils.d.ts +59 -0
- package/dist/shared/utils/escalation-utils.js +141 -0
- package/dist/tui/components/command-input.js +1 -1
- package/dist/tui/components/inline-prompts/inline-confirm.js +6 -1
- package/dist/tui/features/commands/definitions/exit.d.ts +2 -0
- package/dist/tui/features/commands/definitions/exit.js +9 -0
- package/dist/tui/features/commands/definitions/index.js +3 -0
- package/dist/tui/features/exit/components/exit-flow.d.ts +10 -0
- package/dist/tui/features/exit/components/exit-flow.js +19 -0
- package/dist/tui/features/provider/components/provider-flow.js +1 -21
- package/oclif.manifest.json +100 -109
- package/package.json +11 -4
- package/dist/agent/infra/blob/migrations.d.ts +0 -63
- package/dist/agent/infra/blob/migrations.js +0 -148
- package/dist/agent/infra/blob/sqlite-blob-storage.d.ts +0 -82
- package/dist/agent/infra/blob/sqlite-blob-storage.js +0 -307
- package/dist/agent/infra/llm/providers/google-vertex.d.ts +0 -15
- package/dist/agent/infra/llm/providers/google-vertex.js +0 -36
- package/dist/agent/infra/storage/blob-history-storage.d.ts +0 -81
- package/dist/agent/infra/storage/blob-history-storage.js +0 -193
- package/dist/agent/infra/storage/dual-format-history-storage.d.ts +0 -83
- package/dist/agent/infra/storage/dual-format-history-storage.js +0 -165
- package/dist/agent/infra/storage/sqlite-key-storage.d.ts +0 -113
- package/dist/agent/infra/storage/sqlite-key-storage.js +0 -438
- package/dist/server/infra/provider/vertex-ai-utils.d.ts +0 -10
- package/dist/server/infra/provider/vertex-ai-utils.js +0 -28
- package/dist/tui/features/provider/components/credential-path-dialog.d.ts +0 -30
- package/dist/tui/features/provider/components/credential-path-dialog.js +0 -85
|
@@ -9,12 +9,14 @@ export var MemorySymbolKind;
|
|
|
9
9
|
MemorySymbolKind[MemorySymbolKind["Context"] = 4] = "Context";
|
|
10
10
|
MemorySymbolKind[MemorySymbolKind["Domain"] = 1] = "Domain";
|
|
11
11
|
MemorySymbolKind[MemorySymbolKind["Subtopic"] = 3] = "Subtopic";
|
|
12
|
+
MemorySymbolKind[MemorySymbolKind["Summary"] = 5] = "Summary";
|
|
12
13
|
MemorySymbolKind[MemorySymbolKind["Topic"] = 2] = "Topic";
|
|
13
14
|
})(MemorySymbolKind || (MemorySymbolKind = {}));
|
|
14
15
|
const SYMBOL_KIND_LABELS = {
|
|
15
16
|
[MemorySymbolKind.Context]: 'context',
|
|
16
17
|
[MemorySymbolKind.Domain]: 'domain',
|
|
17
18
|
[MemorySymbolKind.Subtopic]: 'subtopic',
|
|
19
|
+
[MemorySymbolKind.Summary]: 'summary',
|
|
18
20
|
[MemorySymbolKind.Topic]: 'topic',
|
|
19
21
|
};
|
|
20
22
|
const DEFAULT_METADATA = {
|
|
@@ -93,8 +95,11 @@ function getOrCreateFolderNode(symbolMap, root, folderPath, folderSegments) {
|
|
|
93
95
|
*
|
|
94
96
|
* context.md files are absorbed into their parent folder node (enriching its metadata)
|
|
95
97
|
* rather than being treated as leaf Context nodes.
|
|
98
|
+
*
|
|
99
|
+
* @param documentMap - Indexed documents (excludes _index.md, includes stubs)
|
|
100
|
+
* @param summaryMap - Optional map of _index.md summary documents for folder annotation
|
|
96
101
|
*/
|
|
97
|
-
export function buildSymbolTree(documentMap) {
|
|
102
|
+
export function buildSymbolTree(documentMap, summaryMap) {
|
|
98
103
|
const root = [];
|
|
99
104
|
const symbolMap = new Map();
|
|
100
105
|
// First pass: collect all documents, create folder nodes, identify context.md files
|
|
@@ -155,6 +160,20 @@ export function buildSymbolTree(documentMap) {
|
|
|
155
160
|
parentNode.children.push(contextNode);
|
|
156
161
|
}
|
|
157
162
|
}
|
|
163
|
+
// Fifth pass: attach summary info from _index.md files to their parent folder nodes
|
|
164
|
+
if (summaryMap) {
|
|
165
|
+
for (const summary of summaryMap.values()) {
|
|
166
|
+
const segments = summary.path.split('/');
|
|
167
|
+
const folderPath = segments.slice(0, -1).join('/');
|
|
168
|
+
const folderNode = symbolMap.get(folderPath);
|
|
169
|
+
if (folderNode) {
|
|
170
|
+
folderNode.summaryInfo = {
|
|
171
|
+
condensationOrder: summary.condensationOrder,
|
|
172
|
+
tokenCount: summary.tokenCount,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
158
177
|
// Sort root domains and all children alphabetically
|
|
159
178
|
sortSymbolChildren(root);
|
|
160
179
|
return { root, symbolMap };
|
|
@@ -189,14 +208,19 @@ export function getSymbolOverview(tree, path, depth = 2) {
|
|
|
189
208
|
}
|
|
190
209
|
function traverse(symbols, currentDepth) {
|
|
191
210
|
for (const symbol of symbols) {
|
|
192
|
-
|
|
211
|
+
const entry = {
|
|
193
212
|
childCount: symbol.children.length,
|
|
194
213
|
importance: symbol.metadata.importance,
|
|
195
214
|
kind: SYMBOL_KIND_LABELS[symbol.kind] ?? 'unknown',
|
|
196
215
|
maturity: symbol.metadata.maturity,
|
|
197
216
|
name: symbol.name,
|
|
198
217
|
path: symbol.path,
|
|
199
|
-
}
|
|
218
|
+
};
|
|
219
|
+
if (symbol.summaryInfo) {
|
|
220
|
+
entry.condensationOrder = symbol.summaryInfo.condensationOrder;
|
|
221
|
+
entry.tokenCount = symbol.summaryInfo.tokenCount;
|
|
222
|
+
}
|
|
223
|
+
entries.push(entry);
|
|
200
224
|
if (currentDepth < depth && symbol.children.length > 0) {
|
|
201
225
|
traverse(symbol.children, currentDepth + 1);
|
|
202
226
|
}
|
|
@@ -61,6 +61,7 @@ export declare class SearchKnowledgeService implements ISearchKnowledgeService {
|
|
|
61
61
|
private buildOverviewResult;
|
|
62
62
|
/**
|
|
63
63
|
* Enrich a search result with symbolic metadata and backlink info.
|
|
64
|
+
* For archive stubs, extracts points_to path into archiveFullPath.
|
|
64
65
|
*/
|
|
65
66
|
private enrichResult;
|
|
66
67
|
/**
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import MiniSearch from 'minisearch';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { removeStopwords } from 'stopword';
|
|
4
|
-
import { BRV_DIR, CONTEXT_FILE_EXTENSION, CONTEXT_TREE_DIR } from '../../../../server/constants.js';
|
|
4
|
+
import { BRV_DIR, CONTEXT_FILE_EXTENSION, CONTEXT_TREE_DIR, SUMMARY_INDEX_FILE } from '../../../../server/constants.js';
|
|
5
5
|
import { parseFrontmatterScoring, updateScoringInContent, } from '../../../../server/core/domain/knowledge/markdown-writer.js';
|
|
6
6
|
import { applyDecay, applyDefaultScoring, compoundScore, determineTier, recordAccessHits, } from '../../../../server/core/domain/knowledge/memory-scoring.js';
|
|
7
|
+
import { isArchiveStub, isDerivedArtifact } from '../../../../server/infra/context-tree/derived-artifact.js';
|
|
8
|
+
import { parseArchiveStubFrontmatter, parseSummaryFrontmatter } from '../../../../server/infra/context-tree/summary-frontmatter.js';
|
|
7
9
|
import { isPathLikeQuery, matchMemoryPath, parseSymbolicQuery } from './memory-path-matcher.js';
|
|
8
10
|
import { buildReferenceIndex, buildSymbolTree, getSubtreeDocumentIds, getSymbolKindLabel, getSymbolOverview, MemorySymbolKind, } from './memory-symbol-tree.js';
|
|
9
11
|
const MAX_CONTEXT_TREE_FILES = 10_000;
|
|
@@ -207,10 +209,26 @@ async function buildFreshIndex(fileSystem, contextTreePath, filesWithMtime) {
|
|
|
207
209
|
lastValidatedAt: now,
|
|
208
210
|
referenceIndex: { backlinks: new Map(), forwardLinks: new Map() },
|
|
209
211
|
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
212
|
+
summaryMap: new Map(),
|
|
210
213
|
symbolTree: { root: [], symbolMap: new Map() },
|
|
211
214
|
};
|
|
212
215
|
}
|
|
213
|
-
|
|
216
|
+
// Partition files: _index.md → summaryFiles, derived artifacts → skip, rest → indexable
|
|
217
|
+
const summaryFiles = [];
|
|
218
|
+
const indexableFiles = [];
|
|
219
|
+
for (const file of filesWithMtime) {
|
|
220
|
+
const fileName = file.path.split('/').at(-1) ?? '';
|
|
221
|
+
if (fileName === SUMMARY_INDEX_FILE) {
|
|
222
|
+
summaryFiles.push(file);
|
|
223
|
+
}
|
|
224
|
+
else if (!isDerivedArtifact(file.path)) {
|
|
225
|
+
// Includes regular .md files AND .stub.md files (stubs are searchable)
|
|
226
|
+
indexableFiles.push(file);
|
|
227
|
+
}
|
|
228
|
+
// .full.md and _manifest.json are skipped (isDerivedArtifact returns true)
|
|
229
|
+
}
|
|
230
|
+
// Read indexable documents for BM25 index
|
|
231
|
+
const documentPromises = indexableFiles.map(async ({ mtime, path: filePath }) => {
|
|
214
232
|
try {
|
|
215
233
|
const fullPath = join(contextTreePath, filePath);
|
|
216
234
|
const { content } = await fileSystem.readFile(fullPath);
|
|
@@ -229,18 +247,49 @@ async function buildFreshIndex(fileSystem, contextTreePath, filesWithMtime) {
|
|
|
229
247
|
return null;
|
|
230
248
|
}
|
|
231
249
|
});
|
|
232
|
-
|
|
233
|
-
const
|
|
250
|
+
// Read _index.md files separately for summaryMap (not indexed in BM25)
|
|
251
|
+
const summaryPromises = summaryFiles.map(async ({ path: filePath }) => {
|
|
252
|
+
try {
|
|
253
|
+
const fullPath = join(contextTreePath, filePath);
|
|
254
|
+
const { content } = await fileSystem.readFile(fullPath);
|
|
255
|
+
const fm = parseSummaryFrontmatter(content);
|
|
256
|
+
if (!fm)
|
|
257
|
+
return null;
|
|
258
|
+
return {
|
|
259
|
+
condensationOrder: fm.condensation_order,
|
|
260
|
+
path: filePath,
|
|
261
|
+
tokenCount: fm.token_count,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
catch {
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
267
|
+
});
|
|
268
|
+
const [docResults, summaryResults] = await Promise.all([
|
|
269
|
+
Promise.all(documentPromises),
|
|
270
|
+
Promise.all(summaryPromises),
|
|
271
|
+
]);
|
|
272
|
+
const documents = docResults.filter((doc) => doc !== null);
|
|
234
273
|
const documentMap = new Map();
|
|
235
274
|
const fileMtimes = new Map();
|
|
236
275
|
for (const doc of documents) {
|
|
237
276
|
documentMap.set(doc.id, doc);
|
|
238
277
|
fileMtimes.set(doc.path, doc.mtime);
|
|
239
278
|
}
|
|
279
|
+
// Also track summary file mtimes for cache invalidation
|
|
280
|
+
for (const sf of summaryFiles) {
|
|
281
|
+
fileMtimes.set(sf.path, sf.mtime);
|
|
282
|
+
}
|
|
283
|
+
const summaryMap = new Map();
|
|
284
|
+
for (const summary of summaryResults) {
|
|
285
|
+
if (summary) {
|
|
286
|
+
summaryMap.set(summary.path, summary);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
240
289
|
const index = new MiniSearch(MINISEARCH_OPTIONS);
|
|
241
290
|
index.addAll(documents);
|
|
242
|
-
// Build symbolic structures from the document map
|
|
243
|
-
const symbolTree = buildSymbolTree(documentMap);
|
|
291
|
+
// Build symbolic structures from the document map, with summary annotations
|
|
292
|
+
const symbolTree = buildSymbolTree(documentMap, summaryMap);
|
|
244
293
|
const referenceIndex = buildReferenceIndex(documentMap);
|
|
245
294
|
return {
|
|
246
295
|
contextTreePath,
|
|
@@ -250,6 +299,7 @@ async function buildFreshIndex(fileSystem, contextTreePath, filesWithMtime) {
|
|
|
250
299
|
lastValidatedAt: now,
|
|
251
300
|
referenceIndex,
|
|
252
301
|
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
302
|
+
summaryMap,
|
|
253
303
|
symbolTree,
|
|
254
304
|
};
|
|
255
305
|
}
|
|
@@ -290,11 +340,17 @@ async function acquireIndex(state, fileSystem, contextTreePath, ttlMs, onBeforeB
|
|
|
290
340
|
lastValidatedAt: 0,
|
|
291
341
|
referenceIndex: { backlinks: new Map(), forwardLinks: new Map() },
|
|
292
342
|
schemaVersion: INDEX_SCHEMA_VERSION,
|
|
343
|
+
summaryMap: new Map(),
|
|
293
344
|
symbolTree: { root: [], symbolMap: new Map() },
|
|
294
345
|
};
|
|
295
346
|
}
|
|
296
347
|
}
|
|
297
|
-
const
|
|
348
|
+
const allFiles = await findMarkdownFilesWithMtime(fileSystem, contextTreePath);
|
|
349
|
+
// Exclude non-indexable derived artifacts (.full.md) so that currentFiles
|
|
350
|
+
// matches what buildFreshIndex tracks in fileMtimes. Without this filter,
|
|
351
|
+
// isCacheValid() sees a size mismatch once archives exist, causing cache thrash.
|
|
352
|
+
// _index.md is kept (tracked for summary staleness), .stub.md is kept (BM25 indexed).
|
|
353
|
+
const currentFiles = allFiles.filter((f) => !isDerivedArtifact(f.path) || f.path.split('/').at(-1) === SUMMARY_INDEX_FILE);
|
|
298
354
|
// Re-check cache validity after getting file list (another call may have finished)
|
|
299
355
|
if (state.cachedIndex &&
|
|
300
356
|
state.cachedIndex.contextTreePath === contextTreePath &&
|
|
@@ -462,15 +518,30 @@ export class SearchKnowledgeService {
|
|
|
462
518
|
}
|
|
463
519
|
/**
|
|
464
520
|
* Enrich a search result with symbolic metadata and backlink info.
|
|
521
|
+
* For archive stubs, extracts points_to path into archiveFullPath.
|
|
465
522
|
*/
|
|
466
|
-
enrichResult(result, symbolTree, referenceIndex) {
|
|
523
|
+
enrichResult(result, symbolTree, referenceIndex, documentMap) {
|
|
467
524
|
const symbol = symbolTree.symbolMap.get(result.path);
|
|
468
525
|
const backlinks = referenceIndex.backlinks.get(result.path);
|
|
526
|
+
// Detect archive stubs and extract points_to for drill-down
|
|
527
|
+
let archiveFullPath;
|
|
528
|
+
let symbolKind = symbol ? getSymbolKindLabel(symbol.kind) : undefined;
|
|
529
|
+
if (isArchiveStub(result.path)) {
|
|
530
|
+
symbolKind = 'archive_stub';
|
|
531
|
+
const doc = documentMap.get(result.path);
|
|
532
|
+
if (doc) {
|
|
533
|
+
const stubFm = parseArchiveStubFrontmatter(doc.content);
|
|
534
|
+
if (stubFm) {
|
|
535
|
+
archiveFullPath = stubFm.points_to;
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
469
539
|
return {
|
|
470
540
|
...result,
|
|
541
|
+
...(archiveFullPath && { archiveFullPath }),
|
|
471
542
|
backlinkCount: backlinks?.length ?? 0,
|
|
472
543
|
relatedPaths: backlinks?.slice(0, 3),
|
|
473
|
-
symbolKind
|
|
544
|
+
symbolKind,
|
|
474
545
|
symbolPath: symbol?.path,
|
|
475
546
|
};
|
|
476
547
|
}
|
|
@@ -558,7 +629,7 @@ export class SearchKnowledgeService {
|
|
|
558
629
|
path: document.path,
|
|
559
630
|
score: Math.round(result.score * 100) / 100,
|
|
560
631
|
title: document.title,
|
|
561
|
-
}, symbolTree, referenceIndex);
|
|
632
|
+
}, symbolTree, referenceIndex, documentMap);
|
|
562
633
|
// Apply kind/maturity filters if specified
|
|
563
634
|
if (options?.includeKinds && enriched.symbolKind && !options.includeKinds.includes(enriched.symbolKind)) {
|
|
564
635
|
continue;
|
|
@@ -606,7 +677,7 @@ export class SearchKnowledgeService {
|
|
|
606
677
|
if (!doc) {
|
|
607
678
|
return null;
|
|
608
679
|
}
|
|
609
|
-
const result = this.enrichResult({ excerpt: extractExcerpt(doc.content, query), path: doc.path, score: 1, title: doc.title }, symbolTree, referenceIndex);
|
|
680
|
+
const result = this.enrichResult({ excerpt: extractExcerpt(doc.content, query), path: doc.path, score: 1, title: doc.title }, symbolTree, referenceIndex, documentMap);
|
|
610
681
|
this.accumulateAccessHits([doc.path]);
|
|
611
682
|
return {
|
|
612
683
|
message: `Found exact match: ${topMatch.path}`,
|
|
@@ -631,7 +702,7 @@ export class SearchKnowledgeService {
|
|
|
631
702
|
const doc = documentMap.get(docId);
|
|
632
703
|
if (!doc)
|
|
633
704
|
continue;
|
|
634
|
-
results.push(this.enrichResult({ excerpt: extractExcerpt(doc.content, query), path: doc.path, score: 0.9, title: doc.title }, symbolTree, referenceIndex));
|
|
705
|
+
results.push(this.enrichResult({ excerpt: extractExcerpt(doc.content, query), path: doc.path, score: 0.9, title: doc.title }, symbolTree, referenceIndex, documentMap));
|
|
635
706
|
}
|
|
636
707
|
if (results.length > 0) {
|
|
637
708
|
this.accumulateAccessHits(results.map((r) => r.path));
|
|
@@ -4,11 +4,11 @@ import { SearchKnowledgeService } from './search-knowledge-service.js';
|
|
|
4
4
|
const SearchKnowledgeInputSchema = z
|
|
5
5
|
.object({
|
|
6
6
|
excludeKinds: z
|
|
7
|
-
.array(z.enum(['context', 'domain', 'subtopic', 'topic']))
|
|
7
|
+
.array(z.enum(['archive_stub', 'context', 'domain', 'subtopic', 'topic']))
|
|
8
8
|
.optional()
|
|
9
9
|
.describe('Symbol kinds to exclude from results'),
|
|
10
10
|
includeKinds: z
|
|
11
|
-
.array(z.enum(['context', 'domain', 'subtopic', 'topic']))
|
|
11
|
+
.array(z.enum(['archive_stub', 'context', 'domain', 'subtopic', 'topic']))
|
|
12
12
|
.optional()
|
|
13
13
|
.describe('Symbol kinds to include in results (filters out others)'),
|
|
14
14
|
limit: z
|
|
@@ -29,7 +29,10 @@ export class ToolManager {
|
|
|
29
29
|
* Uses code_exec only - curate operations available via tools.curate() in sandbox.
|
|
30
30
|
*/
|
|
31
31
|
static CURATE_TOOL_NAMES = [
|
|
32
|
+
'agentic_map',
|
|
32
33
|
'code_exec',
|
|
34
|
+
'expand_knowledge',
|
|
35
|
+
'llm_map',
|
|
33
36
|
];
|
|
34
37
|
/**
|
|
35
38
|
* Tools allowed for query operations - only code_exec for programmatic search
|
|
@@ -38,6 +41,7 @@ export class ToolManager {
|
|
|
38
41
|
*/
|
|
39
42
|
static QUERY_TOOL_NAMES = [
|
|
40
43
|
'code_exec',
|
|
44
|
+
'expand_knowledge',
|
|
41
45
|
];
|
|
42
46
|
cacheValid = false;
|
|
43
47
|
callIdCounter = 0;
|
|
@@ -113,6 +117,8 @@ export class ToolManager {
|
|
|
113
117
|
// Execute tool via scheduler (with policy check) or directly via provider
|
|
114
118
|
const result = this.scheduler
|
|
115
119
|
? await this.scheduler.execute(toolName, effectiveArgs, {
|
|
120
|
+
commandType: effectiveContext.commandType,
|
|
121
|
+
metadata: effectiveContext.metadata,
|
|
116
122
|
sessionId: sessionId ?? 'default',
|
|
117
123
|
taskId: effectiveContext.taskId,
|
|
118
124
|
})
|
|
@@ -13,6 +13,12 @@ import { ToolMarker } from './tool-markers.js';
|
|
|
13
13
|
* 2. Execution phase (via invocation)
|
|
14
14
|
*/
|
|
15
15
|
export declare class ToolProvider implements IToolProvider {
|
|
16
|
+
/**
|
|
17
|
+
* Known keys of ToolServices — used for runtime validation of Record<string, unknown> input.
|
|
18
|
+
* Derived from ToolServices type to prevent drift.
|
|
19
|
+
* TypeScript ensures this object matches ToolServices keys at compile time.
|
|
20
|
+
*/
|
|
21
|
+
private static readonly VALID_SERVICE_KEYS;
|
|
16
22
|
private readonly descriptionLoader?;
|
|
17
23
|
private initialized;
|
|
18
24
|
private invocationBuilder?;
|
|
@@ -74,6 +80,12 @@ export declare class ToolProvider implements IToolProvider {
|
|
|
74
80
|
* If a description loader is provided, tool descriptions are loaded from external files.
|
|
75
81
|
*/
|
|
76
82
|
initialize(): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* Atomically replace specific tools with new service dependencies.
|
|
85
|
+
* Build-then-swap pattern: builds new tool instances first, only swaps if all succeed.
|
|
86
|
+
* Throws if any requested tool cannot be rebuilt (no partial swap).
|
|
87
|
+
*/
|
|
88
|
+
replaceTools(toolNames: string[], newServices: Record<string, unknown>): void;
|
|
77
89
|
/**
|
|
78
90
|
* Update services and re-register tools that depend on newly available services.
|
|
79
91
|
* This is used to inject services that are created after ToolProvider initialization
|
|
@@ -12,6 +12,25 @@ import { convertZodToJsonSchema } from './utils/schema-converter.js';
|
|
|
12
12
|
* 2. Execution phase (via invocation)
|
|
13
13
|
*/
|
|
14
14
|
export class ToolProvider {
|
|
15
|
+
/**
|
|
16
|
+
* Known keys of ToolServices — used for runtime validation of Record<string, unknown> input.
|
|
17
|
+
* Derived from ToolServices type to prevent drift.
|
|
18
|
+
* TypeScript ensures this object matches ToolServices keys at compile time.
|
|
19
|
+
*/
|
|
20
|
+
static VALID_SERVICE_KEYS = new Set(Object.keys({
|
|
21
|
+
agentInstance: 0,
|
|
22
|
+
contentGenerator: 0,
|
|
23
|
+
environmentContext: 0,
|
|
24
|
+
fileSystemService: 0,
|
|
25
|
+
getToolProvider: 0,
|
|
26
|
+
logger: 0,
|
|
27
|
+
maxContextTokens: 0,
|
|
28
|
+
memoryManager: 0,
|
|
29
|
+
processService: 0,
|
|
30
|
+
sandboxService: 0,
|
|
31
|
+
todoStorage: 0,
|
|
32
|
+
tokenizer: 0,
|
|
33
|
+
}));
|
|
15
34
|
descriptionLoader;
|
|
16
35
|
initialized = false;
|
|
17
36
|
invocationBuilder;
|
|
@@ -178,6 +197,65 @@ export class ToolProvider {
|
|
|
178
197
|
this.invocationBuilder = new ToolInvocationBuilder(this.tools);
|
|
179
198
|
this.initialized = true;
|
|
180
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Atomically replace specific tools with new service dependencies.
|
|
202
|
+
* Build-then-swap pattern: builds new tool instances first, only swaps if all succeed.
|
|
203
|
+
* Throws if any requested tool cannot be rebuilt (no partial swap).
|
|
204
|
+
*/
|
|
205
|
+
replaceTools(toolNames, newServices) {
|
|
206
|
+
// 0. Runtime key validation — catch typos in Record<string, unknown> early
|
|
207
|
+
for (const key of Object.keys(newServices)) {
|
|
208
|
+
if (!ToolProvider.VALID_SERVICE_KEYS.has(key)) {
|
|
209
|
+
throw new Error(`replaceTools: unknown service key "${key}" — valid keys: ${[...ToolProvider.VALID_SERVICE_KEYS].join(', ')}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// 1. Merge new services into existing (cast from Record to ToolServices)
|
|
213
|
+
const mergedServices = {
|
|
214
|
+
...this.services,
|
|
215
|
+
...newServices,
|
|
216
|
+
};
|
|
217
|
+
// 2. Deduplicate input tool names
|
|
218
|
+
const uniqueToolNames = [...new Set(toolNames)];
|
|
219
|
+
// 3. Stage new tool instances — fail hard if any tool cannot be built
|
|
220
|
+
const stagedTools = new Map();
|
|
221
|
+
for (const toolName of uniqueToolNames) {
|
|
222
|
+
const entry = TOOL_REGISTRY[toolName];
|
|
223
|
+
if (!entry) {
|
|
224
|
+
throw new Error(`replaceTools: unknown tool "${toolName}" not in TOOL_REGISTRY`);
|
|
225
|
+
}
|
|
226
|
+
// Check required services
|
|
227
|
+
const allServicesAvailable = entry.requiredServices.every((serviceName) => mergedServices[serviceName] !== undefined);
|
|
228
|
+
if (!allServicesAvailable) {
|
|
229
|
+
throw new Error(`replaceTools: missing required services for "${toolName}": ${entry.requiredServices.join(', ')}`);
|
|
230
|
+
}
|
|
231
|
+
// Build tool (throws on factory failure — old tools remain intact)
|
|
232
|
+
const tool = entry.factory(mergedServices);
|
|
233
|
+
// Apply description loader overrides
|
|
234
|
+
const fileDescription = this.loadExternalDescription(entry.descriptionFile);
|
|
235
|
+
if (fileDescription) {
|
|
236
|
+
tool.description = fileDescription;
|
|
237
|
+
}
|
|
238
|
+
stagedTools.set(toolName, tool);
|
|
239
|
+
}
|
|
240
|
+
// 4. Swap atomically — only reached if ALL builds succeeded
|
|
241
|
+
this.services = mergedServices;
|
|
242
|
+
for (const [name, tool] of stagedTools) {
|
|
243
|
+
this.tools.set(name, tool);
|
|
244
|
+
}
|
|
245
|
+
// 5. Recompute markers from all registered tools
|
|
246
|
+
this.toolMarkers.clear();
|
|
247
|
+
for (const [toolName, entry] of Object.entries(TOOL_REGISTRY)) {
|
|
248
|
+
if (this.tools.has(toolName)) {
|
|
249
|
+
for (const marker of entry.markers) {
|
|
250
|
+
this.toolMarkers.add(marker);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
// 6. Rebuild invocationBuilder
|
|
255
|
+
if (this.initialized && this.invocationBuilder) {
|
|
256
|
+
this.invocationBuilder = new ToolInvocationBuilder(this.tools);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
181
259
|
/**
|
|
182
260
|
* Update services and re-register tools that depend on newly available services.
|
|
183
261
|
* This is used to inject services that are created after ToolProvider initialization
|
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { EnvironmentContext } from '../../core/domain/environment/types.js';
|
|
2
2
|
import type { KnownTool } from '../../core/domain/tools/constants.js';
|
|
3
3
|
import type { Tool } from '../../core/domain/tools/types.js';
|
|
4
|
+
import type { ICipherAgent } from '../../core/interfaces/i-cipher-agent.js';
|
|
5
|
+
import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
4
6
|
import type { IFileSystem } from '../../core/interfaces/i-file-system.js';
|
|
7
|
+
import type { ILogger } from '../../core/interfaces/i-logger.js';
|
|
5
8
|
import type { IProcessService } from '../../core/interfaces/i-process-service.js';
|
|
6
9
|
import type { ISandboxService } from '../../core/interfaces/i-sandbox-service.js';
|
|
7
10
|
import type { ITodoStorage } from '../../core/interfaces/i-todo-storage.js';
|
|
11
|
+
import type { ITokenizer } from '../../core/interfaces/i-tokenizer.js';
|
|
8
12
|
import type { MemoryManager } from '../memory/memory-manager.js';
|
|
9
13
|
import type { ToolProviderGetter } from './tool-provider-getter.js';
|
|
10
14
|
import { ToolMarker } from './tool-markers.js';
|
|
@@ -13,6 +17,10 @@ import { ToolMarker } from './tool-markers.js';
|
|
|
13
17
|
* Tools declare which services they need via requiredServices.
|
|
14
18
|
*/
|
|
15
19
|
export interface ToolServices {
|
|
20
|
+
/** Agent instance for creating sub-sessions (used by agentic_map) */
|
|
21
|
+
agentInstance?: ICipherAgent;
|
|
22
|
+
/** Content generator for stateless LLM calls (used by llm_map) */
|
|
23
|
+
contentGenerator?: IContentGenerator;
|
|
16
24
|
/** Environment context for sandbox injection */
|
|
17
25
|
environmentContext?: EnvironmentContext;
|
|
18
26
|
/** File system service for file operations */
|
|
@@ -22,6 +30,10 @@ export interface ToolServices {
|
|
|
22
30
|
* Used by batch tool to execute other tools.
|
|
23
31
|
*/
|
|
24
32
|
getToolProvider?: ToolProviderGetter;
|
|
33
|
+
/** Logger for fail-open warnings in map tools */
|
|
34
|
+
logger?: ILogger;
|
|
35
|
+
/** Max context tokens for ContextTreeStore τ_hard computation */
|
|
36
|
+
maxContextTokens?: number;
|
|
25
37
|
/** Memory manager for agent memory operations */
|
|
26
38
|
memoryManager?: MemoryManager;
|
|
27
39
|
/** Process service for command execution */
|
|
@@ -30,6 +42,8 @@ export interface ToolServices {
|
|
|
30
42
|
sandboxService?: ISandboxService;
|
|
31
43
|
/** Todo storage service for session-based todo persistence */
|
|
32
44
|
todoStorage?: ITodoStorage;
|
|
45
|
+
/** Tokenizer for ContextTreeStore token counting */
|
|
46
|
+
tokenizer?: ITokenizer;
|
|
33
47
|
}
|
|
34
48
|
/**
|
|
35
49
|
* Tool factory function type.
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
import { ToolName } from '../../core/domain/tools/constants.js';
|
|
2
2
|
import { createCurateService } from '../sandbox/curate-service.js';
|
|
3
|
+
import { createAgenticMapTool } from './implementations/agentic-map-tool.js';
|
|
3
4
|
import { createCodeExecTool } from './implementations/code-exec-tool.js';
|
|
4
5
|
import { createCurateTool } from './implementations/curate-tool.js';
|
|
6
|
+
import { createExpandKnowledgeTool } from './implementations/expand-knowledge-tool.js';
|
|
5
7
|
import { createGlobFilesTool } from './implementations/glob-files-tool.js';
|
|
6
8
|
import { createGrepContentTool } from './implementations/grep-content-tool.js';
|
|
7
9
|
import { createListDirectoryTool } from './implementations/list-directory-tool.js';
|
|
10
|
+
import { createLlmMapTool } from './implementations/llm-map-tool.js';
|
|
8
11
|
import { createReadFileTool } from './implementations/read-file-tool.js';
|
|
9
12
|
import { createSearchKnowledgeService } from './implementations/search-knowledge-service.js';
|
|
10
13
|
import { createSearchKnowledgeTool } from './implementations/search-knowledge-tool.js';
|
|
@@ -35,6 +38,20 @@ function getRequiredService(service, serviceName) {
|
|
|
35
38
|
* 3. Add entry to this registry
|
|
36
39
|
*/
|
|
37
40
|
export const TOOL_REGISTRY = {
|
|
41
|
+
[ToolName.AGENTIC_MAP]: {
|
|
42
|
+
factory({ agentInstance, contentGenerator, environmentContext, logger, maxContextTokens, tokenizer }) {
|
|
43
|
+
const agent = getRequiredService(agentInstance, 'agentInstance');
|
|
44
|
+
const workingDirectory = environmentContext?.workingDirectory ?? process.cwd();
|
|
45
|
+
return createAgenticMapTool(agent, workingDirectory, {
|
|
46
|
+
generator: contentGenerator,
|
|
47
|
+
logger,
|
|
48
|
+
maxContextTokens,
|
|
49
|
+
tokenizer,
|
|
50
|
+
});
|
|
51
|
+
},
|
|
52
|
+
markers: [ToolMarker.Execution],
|
|
53
|
+
requiredServices: ['agentInstance'],
|
|
54
|
+
},
|
|
38
55
|
[ToolName.CODE_EXEC]: {
|
|
39
56
|
descriptionFile: 'code_exec',
|
|
40
57
|
factory({ environmentContext, fileSystemService, sandboxService }) {
|
|
@@ -69,6 +86,12 @@ export const TOOL_REGISTRY = {
|
|
|
69
86
|
outputGuidance: 'curate',
|
|
70
87
|
requiredServices: [], // Uses DirectoryManager and MarkdownWriter for file operations
|
|
71
88
|
},
|
|
89
|
+
[ToolName.EXPAND_KNOWLEDGE]: {
|
|
90
|
+
descriptionFile: 'expand_knowledge',
|
|
91
|
+
factory: ({ environmentContext }) => createExpandKnowledgeTool({ baseDirectory: environmentContext?.workingDirectory }),
|
|
92
|
+
markers: [ToolMarker.Discovery],
|
|
93
|
+
requiredServices: [],
|
|
94
|
+
},
|
|
72
95
|
[ToolName.GLOB_FILES]: {
|
|
73
96
|
descriptionFile: 'glob_files',
|
|
74
97
|
factory: (services) => createGlobFilesTool(getRequiredService(services.fileSystemService, 'fileSystemService')),
|
|
@@ -87,6 +110,15 @@ export const TOOL_REGISTRY = {
|
|
|
87
110
|
markers: [ToolMarker.Discovery],
|
|
88
111
|
requiredServices: ['fileSystemService'],
|
|
89
112
|
},
|
|
113
|
+
[ToolName.LLM_MAP]: {
|
|
114
|
+
factory({ contentGenerator, environmentContext, logger, maxContextTokens, tokenizer }) {
|
|
115
|
+
const generator = getRequiredService(contentGenerator, 'contentGenerator');
|
|
116
|
+
const workingDirectory = environmentContext?.workingDirectory ?? process.cwd();
|
|
117
|
+
return createLlmMapTool(generator, workingDirectory, { logger, maxContextTokens, tokenizer });
|
|
118
|
+
},
|
|
119
|
+
markers: [ToolMarker.Execution],
|
|
120
|
+
requiredServices: ['contentGenerator'],
|
|
121
|
+
},
|
|
90
122
|
[ToolName.READ_FILE]: {
|
|
91
123
|
descriptionFile: 'read_file',
|
|
92
124
|
factory: (services) => createReadFileTool(getRequiredService(services.fileSystemService, 'fileSystemService')),
|