@customize-agent/knowledge 4.0.41 → 4.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/index-runner.d.ts +20 -0
- package/dist/core/index-runner.js +45 -0
- package/dist/core/index-state-store.d.ts +3 -0
- package/dist/core/index-state-store.js +26 -7
- package/dist/core/knowledge-base-manager.d.ts +12 -3
- package/dist/core/knowledge-base-manager.js +96 -20
- package/dist/core/multi-project-manager.js +2 -2
- package/dist/core/project-config.d.ts +2 -2
- package/dist/core/project-config.js +4 -3
- package/dist/core/project-registry.js +2 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/llm/llm-search-provider.d.ts +5 -0
- package/package.json +1 -1
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { DiffResult } from '../types.js';
|
|
2
|
+
import type { KnowledgeBaseManager, KnowledgeIndexProgress } from './knowledge-base-manager.js';
|
|
3
|
+
export interface IndexRunJob {
|
|
4
|
+
relativePath?: string;
|
|
5
|
+
relativePaths?: string[];
|
|
6
|
+
forceReindexAll?: boolean;
|
|
7
|
+
vectorMode?: 'sync' | 'defer';
|
|
8
|
+
uploadOperationId?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface IndexRunOutcome {
|
|
11
|
+
diff: DiffResult;
|
|
12
|
+
vectorStatus: ReturnType<KnowledgeBaseManager['getVectorStatus']>;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* 知识库索引主循环:扫描变更 → 解析分块 → 等待上传批次 → 向量化收尾。
|
|
16
|
+
* 主进程直跑(CUSTOMIZE_AGENT_DISABLE_KB_CHILD_PROCESS=1)与后台子进程(kb-index-worker.cjs)
|
|
17
|
+
* 共用同一实现,避免两处逐行重复导致索引行为漂移。
|
|
18
|
+
* 进度上报通过 onProgress 回调交给调用方(主进程写操作日志、子进程经 IPC 转发),本函数不落盘。
|
|
19
|
+
*/
|
|
20
|
+
export declare function runIndexLoop(project: KnowledgeBaseManager, job: IndexRunJob, onProgress: (progress: KnowledgeIndexProgress) => void): Promise<IndexRunOutcome>;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 知识库索引主循环:扫描变更 → 解析分块 → 等待上传批次 → 向量化收尾。
|
|
3
|
+
* 主进程直跑(CUSTOMIZE_AGENT_DISABLE_KB_CHILD_PROCESS=1)与后台子进程(kb-index-worker.cjs)
|
|
4
|
+
* 共用同一实现,避免两处逐行重复导致索引行为漂移。
|
|
5
|
+
* 进度上报通过 onProgress 回调交给调用方(主进程写操作日志、子进程经 IPC 转发),本函数不落盘。
|
|
6
|
+
*/
|
|
7
|
+
export async function runIndexLoop(project, job, onProgress) {
|
|
8
|
+
let diff = job.relativePaths?.length
|
|
9
|
+
? await project.incrementalIndex({ vectorMode: job.vectorMode, onProgress, onlyRelativePaths: job.relativePaths })
|
|
10
|
+
: job.relativePath
|
|
11
|
+
? await project.reindexFile(job.relativePath, { vectorMode: job.vectorMode, onProgress })
|
|
12
|
+
: job.forceReindexAll
|
|
13
|
+
? await project.forceReindexAll({ vectorMode: job.vectorMode, onProgress })
|
|
14
|
+
: await project.consumePendingIndexJobs({ vectorMode: job.vectorMode, onProgress, waitForUploadId: job.uploadOperationId });
|
|
15
|
+
let idleChecks = 0;
|
|
16
|
+
// 上传 session 空闲上限:批次间无新文件到达超过该时长即退出等待,避免前端中断后 worker 永久挂起
|
|
17
|
+
const sessionIdleLimitMs = Math.max(60_000, Number(process.env.CUSTOMIZE_KB_UPLOAD_SESSION_IDLE_MS || 600_000));
|
|
18
|
+
const maxIdleChecks = Math.max(10, Math.ceil(sessionIdleLimitMs / 1000));
|
|
19
|
+
// 等待任何未关闭的上传 session(不限于本 operationId),避免重叠上传的后续批次文件无人消费
|
|
20
|
+
while (!job.relativePath && !job.relativePaths?.length && (project.countPendingIndexJobs() > 0 || (project.hasOpenUploadSessions() && idleChecks < maxIdleChecks))) {
|
|
21
|
+
if (project.countPendingIndexJobs() === 0) {
|
|
22
|
+
idleChecks += 1;
|
|
23
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
idleChecks = 0;
|
|
27
|
+
const nextDiff = await project.consumePendingIndexJobs({ vectorMode: job.vectorMode, onProgress, waitForUploadId: job.uploadOperationId });
|
|
28
|
+
diff = {
|
|
29
|
+
newFiles: [...diff.newFiles, ...nextDiff.newFiles],
|
|
30
|
+
modifiedFiles: [...diff.modifiedFiles, ...nextDiff.modifiedFiles],
|
|
31
|
+
deletedFiles: [...diff.deletedFiles, ...nextDiff.deletedFiles],
|
|
32
|
+
unchangedCount: diff.unchangedCount + nextDiff.unchangedCount,
|
|
33
|
+
mtimeOnlyCount: diff.mtimeOnlyCount + nextDiff.mtimeOnlyCount,
|
|
34
|
+
skippedFiles: [...diff.skippedFiles, ...nextDiff.skippedFiles],
|
|
35
|
+
hasChanges: diff.hasChanges || nextDiff.hasChanges,
|
|
36
|
+
diffTimeMs: diff.diffTimeMs + nextDiff.diffTimeMs,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
let vectorStatus = project.getVectorStatus();
|
|
40
|
+
if (job.vectorMode === 'defer' && vectorStatus.status === 'pending') {
|
|
41
|
+
await project.indexVectors();
|
|
42
|
+
vectorStatus = project.getVectorStatus();
|
|
43
|
+
}
|
|
44
|
+
return { diff, vectorStatus };
|
|
45
|
+
}
|
|
@@ -151,6 +151,8 @@ export declare class IndexStateStore {
|
|
|
151
151
|
listParentChunks(relativePath: string): StoredParentChunk[];
|
|
152
152
|
getParentChunk(relativePath: string, parentId: string): StoredParentChunk | undefined;
|
|
153
153
|
getDocumentChunk(relativePath: string): StoredDocumentChunk | undefined;
|
|
154
|
+
/** 聚合统计切片数:单条 SUM 查询,避免加载全部索引记录后再求和 */
|
|
155
|
+
countIndexedChunks(filePaths?: string[]): number;
|
|
154
156
|
getChunksByParent(relativePath: string, parentId: string, limit?: number): StoredChunk[];
|
|
155
157
|
/**
|
|
156
158
|
* 使用关键词搜索切片(支持 FTS5 全文搜索和 LIKE 模糊匹配)
|
|
@@ -193,6 +195,7 @@ export declare class IndexStateStore {
|
|
|
193
195
|
deleteRecord(relativePath: string): void;
|
|
194
196
|
setMetadata(key: string, value: string): void;
|
|
195
197
|
getMetadata(key: string): string | undefined;
|
|
198
|
+
listMetadataKeys(prefix?: string): string[];
|
|
196
199
|
getStats(): {
|
|
197
200
|
fileCount: number;
|
|
198
201
|
chunkCount: number;
|
|
@@ -9,6 +9,9 @@ export class IndexStateStore {
|
|
|
9
9
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
10
10
|
this.db = new Database(dbPath);
|
|
11
11
|
this.db.pragma('journal_mode = WAL');
|
|
12
|
+
// P1-8 SQLite 并发读加固:显式 busy_timeout(kb.db 大库并发读曾出现 SQLITE_READONLY 瞬态,
|
|
13
|
+
// WAL 模式下写锁升级竞争由 busy_timeout 兜底等待,避免立即抛锁错误)
|
|
14
|
+
this.db.pragma('busy_timeout = 10000');
|
|
12
15
|
this.initTables();
|
|
13
16
|
}
|
|
14
17
|
/** 加载所有活跃的索引记录 */
|
|
@@ -279,13 +282,23 @@ export class IndexStateStore {
|
|
|
279
282
|
`).get(relativePath);
|
|
280
283
|
return row ? this.rowToDocumentChunk(row) : undefined;
|
|
281
284
|
}
|
|
285
|
+
/** 聚合统计切片数:单条 SUM 查询,避免加载全部索引记录后再求和 */
|
|
286
|
+
countIndexedChunks(filePaths) {
|
|
287
|
+
const paths = filePaths?.filter(Boolean) ?? [];
|
|
288
|
+
const clause = paths.length > 0 ? ` WHERE relative_path IN (${paths.map(() => '?').join(', ')})` : '';
|
|
289
|
+
const row = this.db.prepare(`SELECT COALESCE(SUM(chunk_count), 0) AS total FROM kb_index_state${clause}`)
|
|
290
|
+
.get(...paths);
|
|
291
|
+
return Math.max(0, Math.ceil(Number(row?.total) || 0));
|
|
292
|
+
}
|
|
282
293
|
getChunksByParent(relativePath, parentId, limit = 6) {
|
|
294
|
+
// 使用 parent_id 列 + idx_kb_chunks_parent 索引;
|
|
295
|
+
// 原先的 metadata_json LIKE '%"parentId":"..."%' 无法使用索引,是每次调用的全表扫描
|
|
283
296
|
const rows = this.db.prepare(`
|
|
284
297
|
SELECT rowid, * FROM kb_chunks
|
|
285
|
-
WHERE relative_path = ? AND
|
|
298
|
+
WHERE relative_path = ? AND parent_id = ?
|
|
286
299
|
ORDER BY chunk_index
|
|
287
300
|
LIMIT ?
|
|
288
|
-
`).all(relativePath,
|
|
301
|
+
`).all(relativePath, parentId, limit);
|
|
289
302
|
return rows.map(row => this.rowToChunk(row, 0));
|
|
290
303
|
}
|
|
291
304
|
/**
|
|
@@ -300,11 +313,11 @@ export class IndexStateStore {
|
|
|
300
313
|
return [];
|
|
301
314
|
const filePaths = [...new Set((filters.filePaths ?? []).filter(Boolean))];
|
|
302
315
|
const effectiveLimit = this.resolveChunkSearchLimit(limit, filePaths);
|
|
303
|
-
const
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
];
|
|
307
|
-
return this.mergeKeywordResults(
|
|
316
|
+
const ftsResults = this.ftsEnabled ? this.searchChunksFts(terms, effectiveLimit, filePaths) : [];
|
|
317
|
+
// FTS 返回充分结果时跳过 LIKE 全表扫描(LOWER(x) LIKE '%term%' 无法使用索引,最多 40 term × 7 列);
|
|
318
|
+
// 中文子串匹配 FTS 命中通常不足,此时 LIKE 仍会执行补齐召回,行为与原先一致
|
|
319
|
+
const likeResults = ftsResults.length >= effectiveLimit ? [] : this.searchChunksLike(terms, effectiveLimit, filePaths);
|
|
320
|
+
return this.mergeKeywordResults([...ftsResults, ...likeResults], effectiveLimit);
|
|
308
321
|
}
|
|
309
322
|
filePathFilterClause(filePaths, column = 'relative_path') {
|
|
310
323
|
return filePaths.length > 0 ? ` AND ${column} IN (${filePaths.map(() => '?').join(', ')})` : '';
|
|
@@ -506,6 +519,12 @@ export class IndexStateStore {
|
|
|
506
519
|
const row = this.db.prepare('SELECT value FROM kb_metadata WHERE key = ?').get(key);
|
|
507
520
|
return row?.value;
|
|
508
521
|
}
|
|
522
|
+
listMetadataKeys(prefix) {
|
|
523
|
+
const rows = prefix
|
|
524
|
+
? this.db.prepare('SELECT key FROM kb_metadata WHERE key LIKE ?').all(`${prefix}%`)
|
|
525
|
+
: this.db.prepare('SELECT key FROM kb_metadata').all();
|
|
526
|
+
return rows.map(row => row.key);
|
|
527
|
+
}
|
|
509
528
|
getStats() {
|
|
510
529
|
const stats = this.db.prepare(`
|
|
511
530
|
SELECT
|
|
@@ -4,7 +4,7 @@ import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights,
|
|
|
4
4
|
import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
|
|
5
5
|
import type { VectorStoreInterface } from '../vector/types.js';
|
|
6
6
|
import { type VectorIndexResult } from '../vector/vector-indexer.js';
|
|
7
|
-
import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
|
|
7
|
+
import { IndexStateStore, type ChunkSearchResult, type FileRelationship, type StoredChunk } from './index-state-store.js';
|
|
8
8
|
export type KnowledgeIndexStage = 'scanning' | 'parsing' | 'chunking' | 'vectorizing' | 'done' | 'error';
|
|
9
9
|
export interface KnowledgeIndexProgress {
|
|
10
10
|
stage: KnowledgeIndexStage;
|
|
@@ -46,6 +46,10 @@ export declare class KnowledgeBaseManager {
|
|
|
46
46
|
private projectConfig?;
|
|
47
47
|
private lastSkippedFiles;
|
|
48
48
|
private readonly llmProvider?;
|
|
49
|
+
private readonly queryExpansionCache;
|
|
50
|
+
private readonly queryExpansionInFlight;
|
|
51
|
+
private queryExpansionActive;
|
|
52
|
+
private readonly queryExpansionWaiters;
|
|
49
53
|
private onProgress?;
|
|
50
54
|
constructor(options: KnowledgeBaseManagerOptions);
|
|
51
55
|
initialize(): void;
|
|
@@ -87,14 +91,14 @@ export declare class KnowledgeBaseManager {
|
|
|
87
91
|
listChunks(options?: {
|
|
88
92
|
relativePath?: string;
|
|
89
93
|
limit?: number;
|
|
90
|
-
}):
|
|
94
|
+
}): StoredChunk[];
|
|
91
95
|
getFileDetail(relativePath: string, options?: {
|
|
92
96
|
maxChunkContentChars?: number;
|
|
93
97
|
}): {
|
|
94
98
|
file: IndexStateRecord;
|
|
95
99
|
absolutePath: string;
|
|
96
100
|
directory: string;
|
|
97
|
-
chunks:
|
|
101
|
+
chunks: StoredChunk[];
|
|
98
102
|
totalChunkCount: number;
|
|
99
103
|
parents: import("./index-state-store.js").StoredParentChunk[];
|
|
100
104
|
relationships: FileRelationship[];
|
|
@@ -167,6 +171,9 @@ export declare class KnowledgeBaseManager {
|
|
|
167
171
|
};
|
|
168
172
|
private rewriteQueries;
|
|
169
173
|
private llmExpandQueries;
|
|
174
|
+
/** 查询扩展 LLM 调用的简易信号量:与文档生成端的全局信号量解耦,避免扩展请求无界并发击穿模型端点 */
|
|
175
|
+
private acquireQueryExpansionSlot;
|
|
176
|
+
private releaseQueryExpansionSlot;
|
|
170
177
|
private retrievalWeights;
|
|
171
178
|
private heuristicRerank;
|
|
172
179
|
private normalizeSearchText;
|
|
@@ -192,6 +199,8 @@ export declare class KnowledgeBaseManager {
|
|
|
192
199
|
private consumePendingVectorRelativePaths;
|
|
193
200
|
private ensureVectorIndexFresh;
|
|
194
201
|
uploadSessionIsOpen(operationId: string): boolean;
|
|
202
|
+
listOpenUploadSessions(): string[];
|
|
203
|
+
hasOpenUploadSessions(): boolean;
|
|
195
204
|
private emptyDiff;
|
|
196
205
|
private statRelativePaths;
|
|
197
206
|
private moveUploadedFile;
|
|
@@ -17,6 +17,11 @@ import { ChangeTracker } from './change-tracker.js';
|
|
|
17
17
|
import { KnowledgeFileScanner } from './file-scanner.js';
|
|
18
18
|
import { IndexStateStore } from './index-state-store.js';
|
|
19
19
|
import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
|
|
20
|
+
// 查询扩展(LLM)调用的约束:生成流程会高频触发查询扩展,
|
|
21
|
+
// TTL 缓存吸收重复扩展、信号量避免扩展请求无界并发击穿模型端点
|
|
22
|
+
const QUERY_EXPANSION_TTL_MS = 10 * 60 * 1000;
|
|
23
|
+
const QUERY_EXPANSION_CACHE_MAX = 256;
|
|
24
|
+
const QUERY_EXPANSION_MAX_CONCURRENCY = 2;
|
|
20
25
|
export class KnowledgeBaseManager {
|
|
21
26
|
scope;
|
|
22
27
|
projectRoot;
|
|
@@ -37,6 +42,10 @@ export class KnowledgeBaseManager {
|
|
|
37
42
|
projectConfig;
|
|
38
43
|
lastSkippedFiles = [];
|
|
39
44
|
llmProvider;
|
|
45
|
+
queryExpansionCache = new Map();
|
|
46
|
+
queryExpansionInFlight = new Map();
|
|
47
|
+
queryExpansionActive = 0;
|
|
48
|
+
queryExpansionWaiters = [];
|
|
40
49
|
onProgress;
|
|
41
50
|
constructor(options) {
|
|
42
51
|
this.scope = options.scope;
|
|
@@ -58,7 +67,7 @@ export class KnowledgeBaseManager {
|
|
|
58
67
|
if (!options.projectRoot || !options.projectId) {
|
|
59
68
|
throw new Error('project knowledge base requires projectRoot and projectId');
|
|
60
69
|
}
|
|
61
|
-
this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot);
|
|
70
|
+
this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot, storageRoot);
|
|
62
71
|
dbPath = path.join(storageRoot, 'projects', options.projectId, 'kb.db');
|
|
63
72
|
}
|
|
64
73
|
this.store = new IndexStateStore(dbPath);
|
|
@@ -339,11 +348,9 @@ export class KnowledgeBaseManager {
|
|
|
339
348
|
}
|
|
340
349
|
searchCorpusSize(filters) {
|
|
341
350
|
const paths = filters?.filePaths ?? (filters?.filePath ? [filters.filePath] : undefined);
|
|
342
|
-
const pathSet =
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
.reduce((sum, record) => sum + Math.max(0, Math.ceil(Number(record.chunkCount) || 0)), 0);
|
|
346
|
-
return Math.max(1, total);
|
|
351
|
+
const pathSet = paths?.filter(Boolean) ?? [];
|
|
352
|
+
// 单条 SUM 聚合,避免 listRecords 全量加载每条索引记录
|
|
353
|
+
return Math.max(1, this.store.countIndexedChunks(pathSet.length > 0 ? pathSet : undefined));
|
|
347
354
|
}
|
|
348
355
|
expandContext(item) {
|
|
349
356
|
const chunkIndex = item.chunkIndex ?? this.parseChunkIndex(item.id);
|
|
@@ -362,7 +369,14 @@ export class KnowledgeBaseManager {
|
|
|
362
369
|
const chunks = parentChunks.length > 0 ? parentChunks : this.store.getContextChunks(item.filePath, chunkIndex, 1);
|
|
363
370
|
if (chunks.length === 0) {
|
|
364
371
|
const document = this.store.getDocumentChunk(item.filePath);
|
|
365
|
-
|
|
372
|
+
if (!document)
|
|
373
|
+
return item;
|
|
374
|
+
// 文档级兜底必须截断:整份文档全文(可能 10 万字级)会撑爆 LLM 上下文预算并稀释命中片段信噪比
|
|
375
|
+
const MAX_DOCUMENT_CONTENT_CHARS = 6000;
|
|
376
|
+
const content = document.content.length > MAX_DOCUMENT_CONTENT_CHARS
|
|
377
|
+
? `${document.content.slice(0, MAX_DOCUMENT_CONTENT_CHARS)}\n……(文档过长,已截断,完整内容请查看原文件)`
|
|
378
|
+
: document.content;
|
|
379
|
+
return { ...item, content, sectionTitle: item.sectionTitle ?? 'Document Parent' };
|
|
366
380
|
}
|
|
367
381
|
return {
|
|
368
382
|
...item,
|
|
@@ -375,10 +389,13 @@ export class KnowledgeBaseManager {
|
|
|
375
389
|
}
|
|
376
390
|
async hybridSearch(query, options = {}) {
|
|
377
391
|
const requestedLimit = Number.isFinite(options.limit) && options.limit > 0 ? Math.ceil(options.limit) : undefined;
|
|
378
|
-
|
|
379
|
-
const effectiveLimit = requestedLimit ??
|
|
392
|
+
// 调用方已显式指定 limit 时无需计算语料总量,避免每次搜索都做一次全量聚合
|
|
393
|
+
const effectiveLimit = requestedLimit ?? this.searchCorpusSize(options.filters);
|
|
380
394
|
const start = Date.now();
|
|
381
395
|
const weights = this.retrievalWeights(options.weights);
|
|
396
|
+
// P1-10 检索语义澄清:generationMode=true 表示"生成场景检索"(文档正文生成链路调用),
|
|
397
|
+
// 该场景刻意跳过 LLM 查询重写——正文生成已占满 LLM 全局信号量,检索侧再触发 LLM 扩展会互相拖慢;
|
|
398
|
+
// 命名按"调用场景"而非"是否启用 LLM 重写",因此 true 反而跳过重写。
|
|
382
399
|
const rewrittenQueries = options.generationMode ? [query.trim()].filter(Boolean) : await this.rewriteQueries(query);
|
|
383
400
|
const rankedLists = [];
|
|
384
401
|
const keywordMultiplier = options.generationMode ? 2 : 3;
|
|
@@ -725,23 +742,65 @@ export class KnowledgeBaseManager {
|
|
|
725
742
|
async llmExpandQueries(query) {
|
|
726
743
|
if (!this.llmProvider)
|
|
727
744
|
return [];
|
|
728
|
-
|
|
745
|
+
// TTL 缓存 + 在途去重:生成流程会在短时间内对相近查询重复扩展,
|
|
746
|
+
// 缓存避免重复 LLM 调用;同一查询并发进入时复用同一 Promise
|
|
747
|
+
const cached = this.queryExpansionCache.get(query);
|
|
748
|
+
if (cached && cached.expiresAt > Date.now())
|
|
749
|
+
return cached.queries;
|
|
750
|
+
const inFlight = this.queryExpansionInFlight.get(query);
|
|
751
|
+
if (inFlight)
|
|
752
|
+
return inFlight;
|
|
753
|
+
const run = (async () => {
|
|
754
|
+
const release = await this.acquireQueryExpansionSlot();
|
|
755
|
+
try {
|
|
756
|
+
const prompt = `你是一个搜索查询优化器。用户输入了一个搜索查询,请生成 3-5 个不同的查询变体,用不同的措辞和同义词来表达相同的信息需求,以便在知识库中检索到更全面的结果。
|
|
729
757
|
|
|
730
758
|
如果查询是中文,请同时生成英文变体;如果查询是英文,请同时生成中文变体。
|
|
731
759
|
|
|
732
760
|
直接输出查询列表,每行一个,不要编号或其他文字。
|
|
733
761
|
|
|
734
762
|
原始查询:${query}`;
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
763
|
+
const response = await this.llmProvider.chat([
|
|
764
|
+
{ role: 'system', content: '你是一个精确的搜索查询扩展引擎。只输出查询列表。' },
|
|
765
|
+
{ role: 'user', content: prompt },
|
|
766
|
+
], { temperature: 0.3, maxTokens: 500 });
|
|
767
|
+
const queries = response.content
|
|
768
|
+
.split('\n')
|
|
769
|
+
.map(line => line.replace(/^[-*\d.]+\s*/, '').trim())
|
|
770
|
+
.filter(line => line.length > 0 && line !== query)
|
|
771
|
+
.slice(0, 5);
|
|
772
|
+
if (this.queryExpansionCache.size >= QUERY_EXPANSION_CACHE_MAX)
|
|
773
|
+
this.queryExpansionCache.clear();
|
|
774
|
+
this.queryExpansionCache.set(query, { queries, expiresAt: Date.now() + QUERY_EXPANSION_TTL_MS });
|
|
775
|
+
return queries;
|
|
776
|
+
}
|
|
777
|
+
finally {
|
|
778
|
+
release();
|
|
779
|
+
this.queryExpansionInFlight.delete(query);
|
|
780
|
+
}
|
|
781
|
+
})();
|
|
782
|
+
this.queryExpansionInFlight.set(query, run);
|
|
783
|
+
return run;
|
|
784
|
+
}
|
|
785
|
+
/** 查询扩展 LLM 调用的简易信号量:与文档生成端的全局信号量解耦,避免扩展请求无界并发击穿模型端点 */
|
|
786
|
+
acquireQueryExpansionSlot() {
|
|
787
|
+
return new Promise(resolve => {
|
|
788
|
+
if (this.queryExpansionActive < QUERY_EXPANSION_MAX_CONCURRENCY) {
|
|
789
|
+
this.queryExpansionActive += 1;
|
|
790
|
+
resolve(this.releaseQueryExpansionSlot);
|
|
791
|
+
}
|
|
792
|
+
else {
|
|
793
|
+
this.queryExpansionWaiters.push(() => {
|
|
794
|
+
this.queryExpansionActive += 1;
|
|
795
|
+
resolve(this.releaseQueryExpansionSlot);
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
});
|
|
744
799
|
}
|
|
800
|
+
releaseQueryExpansionSlot = () => {
|
|
801
|
+
this.queryExpansionActive = Math.max(0, this.queryExpansionActive - 1);
|
|
802
|
+
this.queryExpansionWaiters.shift()?.();
|
|
803
|
+
};
|
|
745
804
|
retrievalWeights(overrides = {}) {
|
|
746
805
|
return {
|
|
747
806
|
keyword: overrides.keyword ?? Number(process.env.KB_RETRIEVAL_KEYWORD_WEIGHT ?? 1),
|
|
@@ -864,10 +923,15 @@ export class KnowledgeBaseManager {
|
|
|
864
923
|
return [...new Set(labels)];
|
|
865
924
|
}
|
|
866
925
|
hydrateVectorResultsFromSqlite(items) {
|
|
926
|
+
// 批量按 rowid 取切片,避免对每条向量结果单发一次 SQL(N+1)
|
|
927
|
+
const rowids = [...new Set(items.filter(item => item.rowid).map(item => item.rowid))];
|
|
928
|
+
const chunksByRowid = new Map();
|
|
929
|
+
for (const chunk of this.store.getChunksByRowids(rowids))
|
|
930
|
+
chunksByRowid.set(chunk.rowid, chunk);
|
|
867
931
|
return items.map(item => {
|
|
868
932
|
if (!item.rowid)
|
|
869
933
|
return item;
|
|
870
|
-
const chunk =
|
|
934
|
+
const chunk = chunksByRowid.get(item.rowid);
|
|
871
935
|
if (!chunk)
|
|
872
936
|
return item;
|
|
873
937
|
const hydrated = this.toFederatedItem({ ...chunk, score: item.score, scoreDetails: item.scoreDetails }, 'vector');
|
|
@@ -1093,6 +1157,18 @@ export class KnowledgeBaseManager {
|
|
|
1093
1157
|
uploadSessionIsOpen(operationId) {
|
|
1094
1158
|
return this.store.getMetadata(`upload_session:${operationId}`) === 'open';
|
|
1095
1159
|
}
|
|
1160
|
+
listOpenUploadSessions() {
|
|
1161
|
+
this.initialize();
|
|
1162
|
+
const prefix = 'upload_session:';
|
|
1163
|
+
return this.store.listMetadataKeys(prefix)
|
|
1164
|
+
.filter(key => this.store.getMetadata(key) === 'open')
|
|
1165
|
+
.map(key => key.slice(prefix.length));
|
|
1166
|
+
}
|
|
1167
|
+
hasOpenUploadSessions() {
|
|
1168
|
+
this.initialize();
|
|
1169
|
+
const prefix = 'upload_session:';
|
|
1170
|
+
return this.store.listMetadataKeys(prefix).some(key => this.store.getMetadata(key) === 'open');
|
|
1171
|
+
}
|
|
1096
1172
|
emptyDiff() {
|
|
1097
1173
|
return { newFiles: [], modifiedFiles: [], deletedFiles: [], unchangedCount: 0, mtimeOnlyCount: 0, skippedFiles: [], hasChanges: false, diffTimeMs: 0 };
|
|
1098
1174
|
}
|
|
@@ -35,7 +35,7 @@ export class MultiProjectManager {
|
|
|
35
35
|
scope: 'project',
|
|
36
36
|
projectRoot: resolvedRoot,
|
|
37
37
|
projectId,
|
|
38
|
-
kbPath: getProjectKbPath(resolvedRoot),
|
|
38
|
+
kbPath: getProjectKbPath(resolvedRoot, this.storageRoot),
|
|
39
39
|
storageRoot: this.storageRoot,
|
|
40
40
|
llmProvider: this.llmProvider,
|
|
41
41
|
});
|
|
@@ -106,7 +106,7 @@ export class MultiProjectManager {
|
|
|
106
106
|
scope: 'project',
|
|
107
107
|
projectRoot: project.projectRoot,
|
|
108
108
|
projectId: project.projectId,
|
|
109
|
-
kbPath: project.
|
|
109
|
+
kbPath: getProjectKbPath(project.projectRoot, this.storageRoot),
|
|
110
110
|
storageRoot: this.storageRoot,
|
|
111
111
|
llmProvider: this.llmProvider,
|
|
112
112
|
});
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { ProjectConfig } from '../types.js';
|
|
2
2
|
/** 获取项目配置文件路径 */
|
|
3
3
|
export declare function getProjectConfigPath(projectRoot: string, storageRoot?: string): string;
|
|
4
|
-
/**
|
|
5
|
-
export declare function getProjectKbPath(projectRoot: string): string;
|
|
4
|
+
/** 获取项目知识库落盘目录路径(用户数据目录下,与 kb.db 索引同生命周期,不再落在项目根目录) */
|
|
5
|
+
export declare function getProjectKbPath(projectRoot: string, storageRoot?: string): string;
|
|
6
6
|
/** 确保项目存在 CUSTOMIZE.md 文件(如不存在则创建默认模板) */
|
|
7
7
|
export declare function ensureProjectCustomizeFile(projectRoot: string): void;
|
|
8
8
|
/** 项目配置管理器,负责加载和保存项目配置 */
|
|
@@ -8,9 +8,10 @@ export function getProjectConfigPath(projectRoot, storageRoot = path.join(os.hom
|
|
|
8
8
|
const projectId = computeProjectId(projectRoot);
|
|
9
9
|
return path.join(storageRoot, 'projects', projectId, 'project.json');
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
12
|
-
export function getProjectKbPath(projectRoot) {
|
|
13
|
-
|
|
11
|
+
/** 获取项目知识库落盘目录路径(用户数据目录下,与 kb.db 索引同生命周期,不再落在项目根目录) */
|
|
12
|
+
export function getProjectKbPath(projectRoot, storageRoot = path.join(os.homedir(), USER_DATA_DIR)) {
|
|
13
|
+
const projectId = computeProjectId(projectRoot);
|
|
14
|
+
return path.join(storageRoot, 'projects', projectId, KNOWLEDGE_BASE_DIR);
|
|
14
15
|
}
|
|
15
16
|
const DEFAULT_CUSTOMIZE_MD = `# Customize Agent 配置示例
|
|
16
17
|
|
|
@@ -8,6 +8,8 @@ export class ProjectRegistry {
|
|
|
8
8
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
9
9
|
this.db = new Database(dbPath);
|
|
10
10
|
this.db.pragma('journal_mode = WAL');
|
|
11
|
+
// P1-8 SQLite 并发读加固:显式 busy_timeout,registry.db 并发读写竞争由等待兜底而非立即抛锁
|
|
12
|
+
this.db.pragma('busy_timeout = 10000');
|
|
11
13
|
this.initTables();
|
|
12
14
|
}
|
|
13
15
|
upsert(project) {
|
package/dist/index.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ export { ChangeTracker } from './core/change-tracker.js';
|
|
|
9
9
|
export { KnowledgeFileScanner, type DiskFileStat } from './core/file-scanner.js';
|
|
10
10
|
export { IndexStateStore, type ChunkSearchResult, type FileHashRecord, type FileRelationship, type StoredChunk } from './core/index-state-store.js';
|
|
11
11
|
export { KnowledgeBaseManager, type KnowledgeBaseManagerOptions, type KnowledgeIndexProgress } from './core/knowledge-base-manager.js';
|
|
12
|
+
export { runIndexLoop, type IndexRunJob, type IndexRunOutcome } from './core/index-runner.js';
|
|
12
13
|
export { MultiProjectManager } from './core/multi-project-manager.js';
|
|
13
14
|
export { computeProjectId } from './core/project-id.js';
|
|
14
15
|
export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
|
package/dist/index.js
CHANGED
|
@@ -11,6 +11,7 @@ export { ChangeTracker } from './core/change-tracker.js';
|
|
|
11
11
|
export { KnowledgeFileScanner } from './core/file-scanner.js';
|
|
12
12
|
export { IndexStateStore } from './core/index-state-store.js';
|
|
13
13
|
export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
|
|
14
|
+
export { runIndexLoop } from './core/index-runner.js';
|
|
14
15
|
export { MultiProjectManager } from './core/multi-project-manager.js';
|
|
15
16
|
export { computeProjectId } from './core/project-id.js';
|
|
16
17
|
export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
|
|
@@ -3,6 +3,11 @@
|
|
|
3
3
|
*
|
|
4
4
|
* 定义在 knowledge 包内部,避免直接依赖 @customize-agent/llm。
|
|
5
5
|
* CLI 层的 ILLMProvider 在结构上兼容此接口,可直接传入。
|
|
6
|
+
*
|
|
7
|
+
* 接入状态(P1-10 检索语义澄清):当前未接入。
|
|
8
|
+
* apps/server 的 getMultiProjectManager() 刻意不注入 llmProvider(见 kbService.ts),
|
|
9
|
+
* 检索查询重写因此退化为本地规则扩展;若未来启用,需配独立低并发 LLM 通道,
|
|
10
|
+
* 避免与文档正文生成争抢全局 LLM 信号量。
|
|
6
11
|
*/
|
|
7
12
|
export interface LLMChatMessage {
|
|
8
13
|
role: 'system' | 'user' | 'assistant';
|