@customize-agent/knowledge 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/text-chunker.d.ts +24 -0
- package/dist/chunking/text-chunker.js +90 -0
- package/dist/classification/classifier.d.ts +12 -0
- package/dist/classification/classifier.js +95 -0
- package/dist/constants.d.ts +8 -0
- package/dist/constants.js +40 -0
- package/dist/core/change-tracker.d.ts +10 -0
- package/dist/core/change-tracker.js +66 -0
- package/dist/core/file-scanner.d.ts +8 -0
- package/dist/core/file-scanner.js +32 -0
- package/dist/core/index-state-store.d.ts +103 -0
- package/dist/core/index-state-store.js +439 -0
- package/dist/core/knowledge-base-manager.d.ts +76 -0
- package/dist/core/knowledge-base-manager.js +300 -0
- package/dist/core/multi-project-manager.d.ts +30 -0
- package/dist/core/multi-project-manager.js +163 -0
- package/dist/core/project-config.d.ts +9 -0
- package/dist/core/project-config.js +73 -0
- package/dist/core/project-id.d.ts +1 -0
- package/dist/core/project-id.js +8 -0
- package/dist/core/project-registry.d.ts +10 -0
- package/dist/core/project-registry.js +70 -0
- package/dist/dedup/dedup-engine.d.ts +20 -0
- package/dist/dedup/dedup-engine.js +78 -0
- package/dist/dedup/relationship-detector.d.ts +10 -0
- package/dist/dedup/relationship-detector.js +84 -0
- package/dist/embedding/embedding-provider.d.ts +15 -0
- package/dist/embedding/embedding-provider.js +31 -0
- package/dist/extraction/content-extractor.d.ts +36 -0
- package/dist/extraction/content-extractor.js +655 -0
- package/dist/extraction/external-extractor.d.ts +63 -0
- package/dist/extraction/external-extractor.js +139 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +22 -0
- package/dist/search/federation-search.d.ts +41 -0
- package/dist/search/federation-search.js +102 -0
- package/dist/server/dashboard-client.d.ts +2 -0
- package/dist/server/dashboard-client.js +396 -0
- package/dist/server/dashboard-i18n.d.ts +112 -0
- package/dist/server/dashboard-i18n.js +220 -0
- package/dist/server/dashboard-page.d.ts +6 -0
- package/dist/server/dashboard-page.js +138 -0
- package/dist/server/dashboard-server.d.ts +13 -0
- package/dist/server/dashboard-server.js +225 -0
- package/dist/server/dashboard-styles.d.ts +1 -0
- package/dist/server/dashboard-styles.js +152 -0
- package/dist/types.d.ts +82 -0
- package/dist/types.js +1 -0
- package/dist/vector/chroma-store.d.ts +39 -0
- package/dist/vector/chroma-store.js +131 -0
- package/dist/vector/collection-manager.d.ts +16 -0
- package/dist/vector/collection-manager.js +77 -0
- package/dist/vector/types.d.ts +33 -0
- package/dist/vector/types.js +1 -0
- package/dist/vector/vector-indexer.d.ts +18 -0
- package/dist/vector/vector-indexer.js +61 -0
- package/package.json +45 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { TextChunker } from '../chunking/text-chunker.js';
|
|
5
|
+
import { FileClassifier } from '../classification/classifier.js';
|
|
6
|
+
import { ALL_CATEGORIES, DEFAULT_CATEGORY_DIRS, GLOBAL_KNOWLEDGE_DIR, USER_DATA_DIR } from '../constants.js';
|
|
7
|
+
import { DedupEngine } from '../dedup/dedup-engine.js';
|
|
8
|
+
import { RelationshipDetector } from '../dedup/relationship-detector.js';
|
|
9
|
+
import { HashEmbeddingProvider } from '../embedding/embedding-provider.js';
|
|
10
|
+
import { ContentExtractor } from '../extraction/content-extractor.js';
|
|
11
|
+
import { FederationSearch } from '../search/federation-search.js';
|
|
12
|
+
import { CollectionManager } from '../vector/collection-manager.js';
|
|
13
|
+
import { VectorIndexer } from '../vector/vector-indexer.js';
|
|
14
|
+
import { ChangeTracker } from './change-tracker.js';
|
|
15
|
+
import { KnowledgeFileScanner } from './file-scanner.js';
|
|
16
|
+
import { IndexStateStore } from './index-state-store.js';
|
|
17
|
+
import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
|
|
18
|
+
export class KnowledgeBaseManager {
|
|
19
|
+
scope;
|
|
20
|
+
projectRoot;
|
|
21
|
+
projectId;
|
|
22
|
+
kbPath;
|
|
23
|
+
store;
|
|
24
|
+
classifier = new FileClassifier();
|
|
25
|
+
scanner = new KnowledgeFileScanner();
|
|
26
|
+
collections = new CollectionManager();
|
|
27
|
+
extractor;
|
|
28
|
+
chunker = new TextChunker();
|
|
29
|
+
dedup = new DedupEngine();
|
|
30
|
+
relationshipDetector = new RelationshipDetector();
|
|
31
|
+
embeddingProvider;
|
|
32
|
+
vectorStores;
|
|
33
|
+
configManager = new ProjectConfigManager();
|
|
34
|
+
projectConfig;
|
|
35
|
+
lastSkippedFiles = [];
|
|
36
|
+
constructor(options) {
|
|
37
|
+
this.scope = options.scope;
|
|
38
|
+
this.projectRoot = options.projectRoot;
|
|
39
|
+
this.projectId = options.projectId;
|
|
40
|
+
this.embeddingProvider = options.embeddingProvider ?? new HashEmbeddingProvider();
|
|
41
|
+
this.vectorStores = options.vectorStores ?? new Map();
|
|
42
|
+
this.extractor = new ContentExtractor(options.externalExtractors);
|
|
43
|
+
const storageRoot = options.storageRoot ?? path.join(os.homedir(), USER_DATA_DIR);
|
|
44
|
+
if (this.scope === 'global') {
|
|
45
|
+
this.kbPath = options.kbPath ?? path.join(storageRoot, GLOBAL_KNOWLEDGE_DIR);
|
|
46
|
+
this.store = new IndexStateStore(path.join(storageRoot, 'global-knowledge.db'));
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
if (!options.projectRoot || !options.projectId) {
|
|
50
|
+
throw new Error('project knowledge base requires projectRoot and projectId');
|
|
51
|
+
}
|
|
52
|
+
this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot);
|
|
53
|
+
this.store = new IndexStateStore(path.join(storageRoot, 'projects', options.projectId, 'kb.db'));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
initialize() {
|
|
57
|
+
if (this.scope === 'project' && this.projectRoot) {
|
|
58
|
+
this.projectConfig = this.configManager.loadOrCreate(this.projectRoot);
|
|
59
|
+
fs.mkdirSync(this.kbPath, { recursive: true });
|
|
60
|
+
const dirs = this.projectConfig.categoryDirs;
|
|
61
|
+
for (const category of ALL_CATEGORIES) {
|
|
62
|
+
fs.mkdirSync(path.join(this.kbPath, dirs[category] ?? DEFAULT_CATEGORY_DIRS[category]), { recursive: true });
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (this.scope === 'global') {
|
|
66
|
+
fs.mkdirSync(this.kbPath, { recursive: true });
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async incrementalIndex() {
|
|
70
|
+
this.initialize();
|
|
71
|
+
const kbIgnore = this.scanner.loadKbIgnore(this.kbPath);
|
|
72
|
+
const configIgnore = this.projectConfig?.kbignore ?? [];
|
|
73
|
+
const diskFiles = await this.scanner.scan(this.kbPath, [...kbIgnore, ...configIgnore]);
|
|
74
|
+
const tracker = new ChangeTracker(this.store);
|
|
75
|
+
const diff = await tracker.computeDiff(diskFiles, this.classifier, this.kbPath);
|
|
76
|
+
for (const deleted of diff.deletedFiles) {
|
|
77
|
+
this.store.deleteRecord(deleted.relativePath);
|
|
78
|
+
}
|
|
79
|
+
const now = Date.now();
|
|
80
|
+
const indexedBefore = [...this.store.loadActiveRecords().values()];
|
|
81
|
+
for (const file of [...diff.newFiles, ...diff.modifiedFiles]) {
|
|
82
|
+
const hash = tracker.hashFile(file.absolutePath);
|
|
83
|
+
const duplicate = this.store.findExactDuplicate(hash, file.relativePath);
|
|
84
|
+
const extraction = await this.extractor.extract(file);
|
|
85
|
+
if (!this.hasUsableContent(extraction.text, extraction.metadata)) {
|
|
86
|
+
diff.skippedFiles.push({ file, reason: extraction.warnings[0] ?? '未解析出可用于模型的正文内容,已跳过向量化' });
|
|
87
|
+
this.store.deleteRecord(file.relativePath);
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const normalizedHash = this.dedup.normalizedHash(extraction.text);
|
|
91
|
+
const normalizedDuplicate = !duplicate && normalizedHash
|
|
92
|
+
? this.store.findNormalizedDuplicate(normalizedHash, file.relativePath)
|
|
93
|
+
: undefined;
|
|
94
|
+
const chunks = duplicate ? [] : this.chunker.chunk(extraction.text, file, extraction.metadata);
|
|
95
|
+
const collectionName = this.scope === 'global'
|
|
96
|
+
? this.collections.getCollectionName('global', file.category)
|
|
97
|
+
: this.collections.getCollectionName('project', file.category, this.projectId);
|
|
98
|
+
this.store.upsertFileHash({
|
|
99
|
+
contentHash: hash,
|
|
100
|
+
filePath: file.relativePath,
|
|
101
|
+
fileSize: file.fileSize,
|
|
102
|
+
category: file.category,
|
|
103
|
+
normalizedHash,
|
|
104
|
+
});
|
|
105
|
+
if (duplicate) {
|
|
106
|
+
this.store.addRelationship({
|
|
107
|
+
sourceFile: file.relativePath,
|
|
108
|
+
targetFile: duplicate.filePath,
|
|
109
|
+
relationshipType: 'exact_duplicate',
|
|
110
|
+
confidence: 1,
|
|
111
|
+
detail: `SHA-256 完全相同: ${hash}`,
|
|
112
|
+
userConfirmed: 0,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
else if (normalizedDuplicate && normalizedHash) {
|
|
116
|
+
this.store.addRelationship({
|
|
117
|
+
sourceFile: file.relativePath,
|
|
118
|
+
targetFile: normalizedDuplicate.filePath,
|
|
119
|
+
relationshipType: this.dedup.relationshipForFormats(file.format, normalizedDuplicate.category),
|
|
120
|
+
confidence: 0.95,
|
|
121
|
+
detail: `归一化内容哈希相同: ${normalizedHash}`,
|
|
122
|
+
userConfirmed: 0,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
if (!duplicate && extraction.text.length > 1000) {
|
|
126
|
+
const minHash = this.dedup.computeMinHash(extraction.text);
|
|
127
|
+
if (minHash) {
|
|
128
|
+
for (const existing of this.store.listMinHashes(file.relativePath)) {
|
|
129
|
+
const similarity = this.dedup.estimateSimilarity(minHash.signature, existing.signature);
|
|
130
|
+
const relationshipType = this.dedup.relationshipForSimilarity(similarity);
|
|
131
|
+
if (relationshipType) {
|
|
132
|
+
this.store.addRelationship({
|
|
133
|
+
sourceFile: file.relativePath,
|
|
134
|
+
targetFile: existing.filePath,
|
|
135
|
+
relationshipType,
|
|
136
|
+
confidence: similarity,
|
|
137
|
+
detail: `MinHash 相似度: ${similarity.toFixed(3)}`,
|
|
138
|
+
userConfirmed: 0,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
this.store.upsertMinHash({
|
|
143
|
+
filePath: file.relativePath,
|
|
144
|
+
signature: minHash.signature,
|
|
145
|
+
shingleCount: minHash.shingleCount,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
for (const relationship of this.relationshipDetector.detect(file, indexedBefore)) {
|
|
150
|
+
this.store.addRelationship(relationship);
|
|
151
|
+
}
|
|
152
|
+
this.store.upsertRecord({
|
|
153
|
+
relativePath: file.relativePath,
|
|
154
|
+
category: file.category,
|
|
155
|
+
format: file.format,
|
|
156
|
+
contentHash: hash,
|
|
157
|
+
fileSize: file.fileSize,
|
|
158
|
+
mtime: file.mtime,
|
|
159
|
+
chunkCount: chunks.length,
|
|
160
|
+
collectionName,
|
|
161
|
+
indexedAt: now,
|
|
162
|
+
lastVerifiedAt: now,
|
|
163
|
+
status: 'active',
|
|
164
|
+
metadataJson: JSON.stringify({
|
|
165
|
+
mimeType: file.mimeType,
|
|
166
|
+
extraction: extraction.metadata,
|
|
167
|
+
warnings: extraction.warnings,
|
|
168
|
+
extractionTimeMs: extraction.extractionTimeMs,
|
|
169
|
+
}),
|
|
170
|
+
});
|
|
171
|
+
this.store.replaceChunks(file.relativePath, chunks, {
|
|
172
|
+
category: file.category,
|
|
173
|
+
format: file.format,
|
|
174
|
+
collectionName,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
const stats = this.getStats();
|
|
178
|
+
this.store.setMetadata('last_incremental_index_at', String(now));
|
|
179
|
+
this.store.setMetadata('total_chunks', String(stats.chunkCount));
|
|
180
|
+
this.store.setMetadata('total_files_indexed', String(stats.fileCount));
|
|
181
|
+
this.lastSkippedFiles = diff.skippedFiles;
|
|
182
|
+
return diff;
|
|
183
|
+
}
|
|
184
|
+
search(query, limit = 10) {
|
|
185
|
+
return this.store.searchChunks(query, limit);
|
|
186
|
+
}
|
|
187
|
+
async semanticSearch(query, options = {}) {
|
|
188
|
+
const queryEmbedding = await this.embeddingProvider.embedQuery(query);
|
|
189
|
+
const search = new FederationSearch(this.vectorStores);
|
|
190
|
+
return search.search({
|
|
191
|
+
query,
|
|
192
|
+
queryEmbedding,
|
|
193
|
+
topK: options.limit ?? 10,
|
|
194
|
+
scope: this.scope,
|
|
195
|
+
projectId: this.projectId,
|
|
196
|
+
collections: options.collections,
|
|
197
|
+
filters: options.filters,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
listRelationships(filePath) {
|
|
201
|
+
return this.store.listRelationships(filePath);
|
|
202
|
+
}
|
|
203
|
+
listFiles() {
|
|
204
|
+
return this.store.listRecords();
|
|
205
|
+
}
|
|
206
|
+
async addFile(sourcePath, targetRelativePath) {
|
|
207
|
+
this.initialize();
|
|
208
|
+
const resolvedSource = path.resolve(sourcePath);
|
|
209
|
+
const relativePath = targetRelativePath ?? path.basename(resolvedSource);
|
|
210
|
+
const targetPath = this.resolveKbRelativePath(relativePath);
|
|
211
|
+
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
|
212
|
+
fs.copyFileSync(resolvedSource, targetPath);
|
|
213
|
+
return this.incrementalIndex();
|
|
214
|
+
}
|
|
215
|
+
async uploadFile(fileName, content, targetRelativePath) {
|
|
216
|
+
this.initialize();
|
|
217
|
+
const relativePath = targetRelativePath ?? this.defaultUploadRelativePath(fileName);
|
|
218
|
+
const targetPath = this.resolveKbRelativePath(relativePath);
|
|
219
|
+
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
|
220
|
+
fs.writeFileSync(targetPath, content);
|
|
221
|
+
return this.incrementalIndex();
|
|
222
|
+
}
|
|
223
|
+
listFailedFiles() {
|
|
224
|
+
return this.lastSkippedFiles;
|
|
225
|
+
}
|
|
226
|
+
async removeFile(relativePath) {
|
|
227
|
+
const targetPath = this.resolveKbRelativePath(relativePath);
|
|
228
|
+
if (fs.existsSync(targetPath))
|
|
229
|
+
fs.unlinkSync(targetPath);
|
|
230
|
+
this.store.deleteRecord(this.normalizeRelativePath(relativePath));
|
|
231
|
+
}
|
|
232
|
+
tagFile(relativePath, tags) {
|
|
233
|
+
this.store.setTags(this.normalizeRelativePath(relativePath), tags);
|
|
234
|
+
}
|
|
235
|
+
listTags(relativePath) {
|
|
236
|
+
return this.store.listTags(relativePath ? this.normalizeRelativePath(relativePath) : undefined);
|
|
237
|
+
}
|
|
238
|
+
addIgnoreRule(pattern) {
|
|
239
|
+
this.store.addIgnoreRule(pattern);
|
|
240
|
+
if (this.scope === 'project' && this.projectRoot) {
|
|
241
|
+
const config = this.projectConfig ?? this.configManager.loadOrCreate(this.projectRoot);
|
|
242
|
+
if (!config.kbignore.includes(pattern)) {
|
|
243
|
+
this.configManager.save(this.projectRoot, { ...config, kbignore: [...config.kbignore, pattern] });
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
listIgnoreRules() {
|
|
248
|
+
return this.store.listIgnoreRules();
|
|
249
|
+
}
|
|
250
|
+
async indexVectors(options = {}) {
|
|
251
|
+
const chunks = this.store.listChunks(options);
|
|
252
|
+
const indexer = new VectorIndexer(this.embeddingProvider, this.vectorStores);
|
|
253
|
+
const results = await indexer.indexChunks(chunks);
|
|
254
|
+
this.store.setMetadata('embedding_model', this.embeddingProvider.model);
|
|
255
|
+
this.store.setMetadata('embedding_dimension', String(this.embeddingProvider.dimensions));
|
|
256
|
+
this.store.setMetadata('last_vector_index_at', String(Date.now()));
|
|
257
|
+
return results;
|
|
258
|
+
}
|
|
259
|
+
getProjectConfig() {
|
|
260
|
+
return this.projectConfig;
|
|
261
|
+
}
|
|
262
|
+
getStats() {
|
|
263
|
+
const stats = this.store.getStats();
|
|
264
|
+
return {
|
|
265
|
+
scope: this.scope,
|
|
266
|
+
projectId: this.projectId,
|
|
267
|
+
fileCount: stats.fileCount,
|
|
268
|
+
chunkCount: stats.chunkCount,
|
|
269
|
+
totalSizeBytes: stats.totalSizeBytes,
|
|
270
|
+
lastIndexedAt: stats.lastIndexedAt,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
close() {
|
|
274
|
+
this.store.close();
|
|
275
|
+
}
|
|
276
|
+
hasUsableContent(text, metadata) {
|
|
277
|
+
const coverage = String(metadata.contentCoverage ?? '');
|
|
278
|
+
if (coverage === 'metadata' || coverage === 'metadata_filename')
|
|
279
|
+
return false;
|
|
280
|
+
return text.trim().length > 0;
|
|
281
|
+
}
|
|
282
|
+
defaultUploadRelativePath(fileName) {
|
|
283
|
+
const classification = this.classifier.classifyVirtual(fileName);
|
|
284
|
+
const configDirs = this.projectConfig?.categoryDirs ?? DEFAULT_CATEGORY_DIRS;
|
|
285
|
+
const dir = configDirs[classification.category] ?? DEFAULT_CATEGORY_DIRS[classification.category];
|
|
286
|
+
return `${dir}/${path.basename(fileName)}`;
|
|
287
|
+
}
|
|
288
|
+
resolveKbRelativePath(relativePath) {
|
|
289
|
+
const normalized = this.normalizeRelativePath(relativePath);
|
|
290
|
+
const targetPath = path.resolve(this.kbPath, normalized);
|
|
291
|
+
const root = path.resolve(this.kbPath);
|
|
292
|
+
if (targetPath !== root && !targetPath.startsWith(`${root}${path.sep}`)) {
|
|
293
|
+
throw new Error('relativePath escapes knowledge base root');
|
|
294
|
+
}
|
|
295
|
+
return targetPath;
|
|
296
|
+
}
|
|
297
|
+
normalizeRelativePath(relativePath) {
|
|
298
|
+
return relativePath.split(path.sep).join('/').replace(/^\/+/, '');
|
|
299
|
+
}
|
|
300
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { FederatedResult, SearchFilters, SearchScope } from '../search/federation-search.js';
|
|
2
|
+
import { FederationSearch } from '../search/federation-search.js';
|
|
3
|
+
import type { CrossProjectDuplicate, ProjectInfo } from '../types.js';
|
|
4
|
+
import { KnowledgeBaseManager } from './knowledge-base-manager.js';
|
|
5
|
+
export declare class MultiProjectManager {
|
|
6
|
+
private readonly storageRoot;
|
|
7
|
+
private readonly registry;
|
|
8
|
+
private readonly configManager;
|
|
9
|
+
private readonly projects;
|
|
10
|
+
private globalKB?;
|
|
11
|
+
constructor(storageRoot?: string);
|
|
12
|
+
getProject(projectRoot: string): Promise<KnowledgeBaseManager>;
|
|
13
|
+
getGlobalKB(): Promise<KnowledgeBaseManager>;
|
|
14
|
+
listProjects(): Promise<ProjectInfo[]>;
|
|
15
|
+
search(projectRoot: string, query: string, options?: {
|
|
16
|
+
limit?: number;
|
|
17
|
+
scope?: SearchScope;
|
|
18
|
+
}): Promise<ReturnType<FederationSearch['merge']>>;
|
|
19
|
+
semanticSearch(projectRoot: string, query: string, options?: {
|
|
20
|
+
limit?: number;
|
|
21
|
+
scope?: SearchScope;
|
|
22
|
+
filters?: SearchFilters;
|
|
23
|
+
collections?: string[];
|
|
24
|
+
}): Promise<FederatedResult>;
|
|
25
|
+
findCrossProjectDuplicates(): Promise<CrossProjectDuplicate[]>;
|
|
26
|
+
forgetProject(projectId: string): Promise<void>;
|
|
27
|
+
closeProject(projectId: string): Promise<void>;
|
|
28
|
+
shutdown(): Promise<void>;
|
|
29
|
+
private updateRegistry;
|
|
30
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import * as os from 'node:os';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { USER_DATA_DIR } from '../constants.js';
|
|
4
|
+
import { FederationSearch } from '../search/federation-search.js';
|
|
5
|
+
import { KnowledgeBaseManager } from './knowledge-base-manager.js';
|
|
6
|
+
import { computeProjectId } from './project-id.js';
|
|
7
|
+
import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
|
|
8
|
+
import { ProjectRegistry } from './project-registry.js';
|
|
9
|
+
export class MultiProjectManager {
|
|
10
|
+
storageRoot;
|
|
11
|
+
registry;
|
|
12
|
+
configManager = new ProjectConfigManager();
|
|
13
|
+
projects = new Map();
|
|
14
|
+
globalKB;
|
|
15
|
+
constructor(storageRoot = path.join(os.homedir(), USER_DATA_DIR)) {
|
|
16
|
+
this.storageRoot = storageRoot;
|
|
17
|
+
this.registry = new ProjectRegistry(path.join(storageRoot, 'projects', 'registry.db'));
|
|
18
|
+
}
|
|
19
|
+
async getProject(projectRoot) {
|
|
20
|
+
const resolvedRoot = path.resolve(projectRoot);
|
|
21
|
+
const projectId = computeProjectId(resolvedRoot);
|
|
22
|
+
const existing = this.projects.get(projectId);
|
|
23
|
+
if (existing)
|
|
24
|
+
return existing;
|
|
25
|
+
const config = this.configManager.loadOrCreate(resolvedRoot);
|
|
26
|
+
const manager = new KnowledgeBaseManager({
|
|
27
|
+
scope: 'project',
|
|
28
|
+
projectRoot: resolvedRoot,
|
|
29
|
+
projectId,
|
|
30
|
+
kbPath: getProjectKbPath(resolvedRoot),
|
|
31
|
+
storageRoot: this.storageRoot,
|
|
32
|
+
});
|
|
33
|
+
manager.initialize();
|
|
34
|
+
this.projects.set(projectId, manager);
|
|
35
|
+
this.updateRegistry(manager, resolvedRoot, config.projectName, config.lastOpenedAt);
|
|
36
|
+
return manager;
|
|
37
|
+
}
|
|
38
|
+
async getGlobalKB() {
|
|
39
|
+
if (this.globalKB)
|
|
40
|
+
return this.globalKB;
|
|
41
|
+
const manager = new KnowledgeBaseManager({ scope: 'global', storageRoot: this.storageRoot });
|
|
42
|
+
manager.initialize();
|
|
43
|
+
await manager.incrementalIndex();
|
|
44
|
+
this.globalKB = manager;
|
|
45
|
+
return manager;
|
|
46
|
+
}
|
|
47
|
+
async listProjects() {
|
|
48
|
+
return this.registry.list();
|
|
49
|
+
}
|
|
50
|
+
async search(projectRoot, query, options = {}) {
|
|
51
|
+
const limit = options.limit ?? 10;
|
|
52
|
+
const scope = options.scope ?? 'all';
|
|
53
|
+
const project = await this.getProject(projectRoot);
|
|
54
|
+
await project.incrementalIndex();
|
|
55
|
+
const items = [];
|
|
56
|
+
if (scope === 'project' || scope === 'all') {
|
|
57
|
+
items.push(...project.search(query, limit).map(result => ({
|
|
58
|
+
id: result.id,
|
|
59
|
+
content: result.content,
|
|
60
|
+
filePath: result.relativePath,
|
|
61
|
+
scope: 'project',
|
|
62
|
+
collection: result.collectionName,
|
|
63
|
+
score: result.score,
|
|
64
|
+
})));
|
|
65
|
+
}
|
|
66
|
+
if (scope === 'global' || scope === 'all') {
|
|
67
|
+
const global = await this.getGlobalKB();
|
|
68
|
+
items.push(...global.search(query, limit).map(result => ({
|
|
69
|
+
id: result.id,
|
|
70
|
+
content: result.content,
|
|
71
|
+
filePath: result.relativePath,
|
|
72
|
+
scope: 'global',
|
|
73
|
+
collection: result.collectionName,
|
|
74
|
+
score: result.score,
|
|
75
|
+
})));
|
|
76
|
+
}
|
|
77
|
+
return new FederationSearch().merge(items, limit, scope);
|
|
78
|
+
}
|
|
79
|
+
async semanticSearch(projectRoot, query, options = {}) {
|
|
80
|
+
const scope = options.scope ?? 'all';
|
|
81
|
+
const project = await this.getProject(projectRoot);
|
|
82
|
+
await project.incrementalIndex();
|
|
83
|
+
if (scope === 'project') {
|
|
84
|
+
return project.semanticSearch(query, options);
|
|
85
|
+
}
|
|
86
|
+
const projectResults = scope === 'all'
|
|
87
|
+
? await project.semanticSearch(query, options)
|
|
88
|
+
: { results: [], scopesSearched: [], queryTimeMs: 0 };
|
|
89
|
+
if (scope === 'global') {
|
|
90
|
+
const global = await this.getGlobalKB();
|
|
91
|
+
return global.semanticSearch(query, options);
|
|
92
|
+
}
|
|
93
|
+
const global = await this.getGlobalKB();
|
|
94
|
+
const globalResults = await global.semanticSearch(query, options);
|
|
95
|
+
return new FederationSearch().merge([...projectResults.results, ...globalResults.results], options.limit ?? 10, 'all');
|
|
96
|
+
}
|
|
97
|
+
async findCrossProjectDuplicates() {
|
|
98
|
+
const projects = this.registry.list();
|
|
99
|
+
const byHash = new Map();
|
|
100
|
+
for (const project of projects) {
|
|
101
|
+
const manager = this.projects.get(project.projectId) ?? new KnowledgeBaseManager({
|
|
102
|
+
scope: 'project',
|
|
103
|
+
projectRoot: project.projectRoot,
|
|
104
|
+
projectId: project.projectId,
|
|
105
|
+
kbPath: project.kbPath,
|
|
106
|
+
storageRoot: this.storageRoot,
|
|
107
|
+
});
|
|
108
|
+
for (const item of manager.store.listContentHashes()) {
|
|
109
|
+
const duplicate = byHash.get(item.contentHash) ?? { contentHash: item.contentHash, files: [] };
|
|
110
|
+
duplicate.files.push({
|
|
111
|
+
projectId: project.projectId,
|
|
112
|
+
projectRoot: project.projectRoot,
|
|
113
|
+
relativePath: item.relativePath,
|
|
114
|
+
});
|
|
115
|
+
byHash.set(item.contentHash, duplicate);
|
|
116
|
+
}
|
|
117
|
+
if (!this.projects.has(project.projectId))
|
|
118
|
+
manager.close();
|
|
119
|
+
}
|
|
120
|
+
return [...byHash.values()].filter(item => item.files.length > 1);
|
|
121
|
+
}
|
|
122
|
+
async forgetProject(projectId) {
|
|
123
|
+
const manager = this.projects.get(projectId);
|
|
124
|
+
if (manager) {
|
|
125
|
+
manager.close();
|
|
126
|
+
this.projects.delete(projectId);
|
|
127
|
+
}
|
|
128
|
+
this.registry.forget(projectId);
|
|
129
|
+
}
|
|
130
|
+
async closeProject(projectId) {
|
|
131
|
+
const manager = this.projects.get(projectId);
|
|
132
|
+
if (!manager)
|
|
133
|
+
return;
|
|
134
|
+
manager.close();
|
|
135
|
+
this.projects.delete(projectId);
|
|
136
|
+
}
|
|
137
|
+
async shutdown() {
|
|
138
|
+
for (const manager of this.projects.values()) {
|
|
139
|
+
manager.close();
|
|
140
|
+
}
|
|
141
|
+
this.projects.clear();
|
|
142
|
+
this.globalKB?.close();
|
|
143
|
+
this.globalKB = undefined;
|
|
144
|
+
this.registry.close();
|
|
145
|
+
}
|
|
146
|
+
updateRegistry(manager, projectRoot, projectName, lastOpenedAt) {
|
|
147
|
+
const stats = manager.getStats();
|
|
148
|
+
if (!manager.projectId)
|
|
149
|
+
throw new Error('project manager missing projectId');
|
|
150
|
+
this.registry.upsert({
|
|
151
|
+
projectId: manager.projectId,
|
|
152
|
+
projectRoot,
|
|
153
|
+
projectName,
|
|
154
|
+
kbPath: manager.kbPath,
|
|
155
|
+
fileCount: stats.fileCount,
|
|
156
|
+
chunkCount: stats.chunkCount,
|
|
157
|
+
totalSizeBytes: stats.totalSizeBytes,
|
|
158
|
+
lastIndexedAt: stats.lastIndexedAt,
|
|
159
|
+
lastOpenedAt,
|
|
160
|
+
status: 'active',
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ProjectConfig } from '../types.js';
|
|
2
|
+
export declare function getProjectConfigPath(projectRoot: string): string;
|
|
3
|
+
export declare function getProjectKbPath(projectRoot: string): string;
|
|
4
|
+
export declare function ensureProjectCustomizeFile(projectRoot: string): void;
|
|
5
|
+
export declare class ProjectConfigManager {
|
|
6
|
+
loadOrCreate(projectRoot: string): ProjectConfig;
|
|
7
|
+
save(projectRoot: string, config: ProjectConfig): void;
|
|
8
|
+
private withDefaults;
|
|
9
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import { DEFAULT_CATEGORY_DIRS, KNOWLEDGE_BASE_DIR, PROJECT_CONFIG_PATH, USER_DATA_DIR } from '../constants.js';
|
|
5
|
+
import { computeProjectId } from './project-id.js';
|
|
6
|
+
export function getProjectConfigPath(projectRoot) {
|
|
7
|
+
const projectId = computeProjectId(projectRoot);
|
|
8
|
+
return path.join(os.homedir(), USER_DATA_DIR, 'projects', projectId, ...PROJECT_CONFIG_PATH.slice(1));
|
|
9
|
+
}
|
|
10
|
+
export function getProjectKbPath(projectRoot) {
|
|
11
|
+
return path.join(projectRoot, KNOWLEDGE_BASE_DIR);
|
|
12
|
+
}
|
|
13
|
+
const DEFAULT_CUSTOMIZE_MD = `# Customize Agent 配置示例
|
|
14
|
+
|
|
15
|
+
你可以在这个文件里描述本项目希望 Agent 遵守的角色、规则和工作方式。
|
|
16
|
+
|
|
17
|
+
## Agent 角色
|
|
18
|
+
你是本项目的工程助手,请优先理解现有代码结构,再进行修改。
|
|
19
|
+
|
|
20
|
+
## 工作规则
|
|
21
|
+
- 修改代码前先阅读相关文件。
|
|
22
|
+
- 保持改动简单、直接、可验证。
|
|
23
|
+
- 不要覆盖用户已有文件或未确认的业务逻辑。
|
|
24
|
+
- 涉及知识库资料时,通过知识库检索使用解析后的内容,不直接读取 knowledgeBase 原始文件。
|
|
25
|
+
|
|
26
|
+
## 项目偏好
|
|
27
|
+
- 使用中文回复。
|
|
28
|
+
- 重要改动完成后运行必要的类型检查或构建检查。
|
|
29
|
+
`;
|
|
30
|
+
export function ensureProjectCustomizeFile(projectRoot) {
|
|
31
|
+
const filePath = path.join(projectRoot, 'CUSTOMIZE.md');
|
|
32
|
+
if (!fs.existsSync(filePath)) {
|
|
33
|
+
fs.mkdirSync(projectRoot, { recursive: true });
|
|
34
|
+
fs.writeFileSync(filePath, DEFAULT_CUSTOMIZE_MD, 'utf8');
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
export class ProjectConfigManager {
|
|
38
|
+
loadOrCreate(projectRoot) {
|
|
39
|
+
ensureProjectCustomizeFile(projectRoot);
|
|
40
|
+
const configPath = getProjectConfigPath(projectRoot);
|
|
41
|
+
const now = Date.now();
|
|
42
|
+
if (fs.existsSync(configPath)) {
|
|
43
|
+
const raw = JSON.parse(fs.readFileSync(configPath, 'utf8'));
|
|
44
|
+
const config = this.withDefaults(projectRoot, raw, now);
|
|
45
|
+
this.save(projectRoot, { ...config, lastOpenedAt: now });
|
|
46
|
+
return { ...config, lastOpenedAt: now };
|
|
47
|
+
}
|
|
48
|
+
const config = this.withDefaults(projectRoot, {}, now);
|
|
49
|
+
this.save(projectRoot, config);
|
|
50
|
+
return config;
|
|
51
|
+
}
|
|
52
|
+
save(projectRoot, config) {
|
|
53
|
+
const configPath = getProjectConfigPath(projectRoot);
|
|
54
|
+
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
|
55
|
+
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
|
|
56
|
+
}
|
|
57
|
+
withDefaults(projectRoot, raw, now) {
|
|
58
|
+
return {
|
|
59
|
+
projectId: raw.projectId ?? computeProjectId(projectRoot),
|
|
60
|
+
projectName: raw.projectName ?? path.basename(path.resolve(projectRoot)),
|
|
61
|
+
enabled: raw.enabled ?? true,
|
|
62
|
+
includeGlobal: raw.includeGlobal ?? true,
|
|
63
|
+
priorityOverGlobal: raw.priorityOverGlobal ?? true,
|
|
64
|
+
watch: raw.watch ?? true,
|
|
65
|
+
autoIndex: raw.autoIndex ?? true,
|
|
66
|
+
kbignore: raw.kbignore ?? [],
|
|
67
|
+
projectTags: raw.projectTags ?? [],
|
|
68
|
+
categoryDirs: { ...DEFAULT_CATEGORY_DIRS, ...(raw.categoryDirs ?? {}) },
|
|
69
|
+
createdAt: raw.createdAt ?? now,
|
|
70
|
+
lastOpenedAt: raw.lastOpenedAt ?? now,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function computeProjectId(projectRoot: string): string;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { ProjectInfo } from '../types.js';
|
|
2
|
+
export declare class ProjectRegistry {
|
|
3
|
+
private readonly db;
|
|
4
|
+
constructor(dbPath: string);
|
|
5
|
+
upsert(project: ProjectInfo): void;
|
|
6
|
+
list(): ProjectInfo[];
|
|
7
|
+
forget(projectId: string): void;
|
|
8
|
+
close(): void;
|
|
9
|
+
private initTables;
|
|
10
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
export class ProjectRegistry {
|
|
5
|
+
db;
|
|
6
|
+
constructor(dbPath) {
|
|
7
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
8
|
+
this.db = new Database(dbPath);
|
|
9
|
+
this.db.pragma('journal_mode = WAL');
|
|
10
|
+
this.initTables();
|
|
11
|
+
}
|
|
12
|
+
upsert(project) {
|
|
13
|
+
this.db.prepare(`
|
|
14
|
+
INSERT INTO project_registry (
|
|
15
|
+
project_id, project_root, project_name, kb_path, file_count,
|
|
16
|
+
chunk_count, total_size_bytes, last_indexed_at, created_at,
|
|
17
|
+
last_opened_at, status
|
|
18
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
19
|
+
ON CONFLICT(project_id) DO UPDATE SET
|
|
20
|
+
project_root = excluded.project_root,
|
|
21
|
+
project_name = excluded.project_name,
|
|
22
|
+
kb_path = excluded.kb_path,
|
|
23
|
+
file_count = excluded.file_count,
|
|
24
|
+
chunk_count = excluded.chunk_count,
|
|
25
|
+
total_size_bytes = excluded.total_size_bytes,
|
|
26
|
+
last_indexed_at = excluded.last_indexed_at,
|
|
27
|
+
last_opened_at = excluded.last_opened_at,
|
|
28
|
+
status = excluded.status
|
|
29
|
+
`).run(project.projectId, project.projectRoot, project.projectName ?? null, project.kbPath, project.fileCount, project.chunkCount, project.totalSizeBytes, project.lastIndexedAt, Date.now(), project.lastOpenedAt, project.status);
|
|
30
|
+
}
|
|
31
|
+
list() {
|
|
32
|
+
const rows = this.db.prepare('SELECT * FROM project_registry ORDER BY last_opened_at DESC').all();
|
|
33
|
+
return rows.map(row => ({
|
|
34
|
+
projectId: String(row.project_id),
|
|
35
|
+
projectRoot: String(row.project_root),
|
|
36
|
+
projectName: row.project_name == null ? undefined : String(row.project_name),
|
|
37
|
+
kbPath: String(row.kb_path),
|
|
38
|
+
fileCount: Number(row.file_count ?? 0),
|
|
39
|
+
chunkCount: Number(row.chunk_count ?? 0),
|
|
40
|
+
totalSizeBytes: Number(row.total_size_bytes ?? 0),
|
|
41
|
+
lastIndexedAt: Number(row.last_indexed_at ?? 0),
|
|
42
|
+
lastOpenedAt: Number(row.last_opened_at ?? 0),
|
|
43
|
+
status: String(row.status ?? 'idle'),
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
forget(projectId) {
|
|
47
|
+
this.db.prepare('DELETE FROM project_registry WHERE project_id = ?').run(projectId);
|
|
48
|
+
}
|
|
49
|
+
close() {
|
|
50
|
+
this.db.close();
|
|
51
|
+
}
|
|
52
|
+
initTables() {
|
|
53
|
+
this.db.exec(`
|
|
54
|
+
CREATE TABLE IF NOT EXISTS project_registry (
|
|
55
|
+
project_id TEXT PRIMARY KEY,
|
|
56
|
+
project_root TEXT NOT NULL UNIQUE,
|
|
57
|
+
project_name TEXT,
|
|
58
|
+
kb_path TEXT NOT NULL,
|
|
59
|
+
file_count INTEGER NOT NULL DEFAULT 0,
|
|
60
|
+
chunk_count INTEGER NOT NULL DEFAULT 0,
|
|
61
|
+
total_size_bytes INTEGER NOT NULL DEFAULT 0,
|
|
62
|
+
last_indexed_at INTEGER NOT NULL DEFAULT 0,
|
|
63
|
+
created_at INTEGER NOT NULL,
|
|
64
|
+
last_opened_at INTEGER NOT NULL,
|
|
65
|
+
status TEXT NOT NULL DEFAULT 'active'
|
|
66
|
+
);
|
|
67
|
+
CREATE INDEX IF NOT EXISTS idx_registry_status ON project_registry(status);
|
|
68
|
+
`);
|
|
69
|
+
}
|
|
70
|
+
}
|