@customize-agent/knowledge 4.0.42 → 4.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ import type { DiffResult } from '../types.js';
2
+ import type { KnowledgeBaseManager, KnowledgeIndexProgress } from './knowledge-base-manager.js';
3
+ export interface IndexRunJob {
4
+ relativePath?: string;
5
+ relativePaths?: string[];
6
+ forceReindexAll?: boolean;
7
+ vectorMode?: 'sync' | 'defer';
8
+ uploadOperationId?: string;
9
+ }
10
+ export interface IndexRunOutcome {
11
+ diff: DiffResult;
12
+ vectorStatus: ReturnType<KnowledgeBaseManager['getVectorStatus']>;
13
+ }
14
+ /**
15
+ * 知识库索引主循环:扫描变更 → 解析分块 → 等待上传批次 → 向量化收尾。
16
+ * 主进程直跑(CUSTOMIZE_AGENT_DISABLE_KB_CHILD_PROCESS=1)与后台子进程(kb-index-worker.cjs)
17
+ * 共用同一实现,避免两处逐行重复导致索引行为漂移。
18
+ * 进度上报通过 onProgress 回调交给调用方(主进程写操作日志、子进程经 IPC 转发),本函数不落盘。
19
+ */
20
+ export declare function runIndexLoop(project: KnowledgeBaseManager, job: IndexRunJob, onProgress: (progress: KnowledgeIndexProgress) => void): Promise<IndexRunOutcome>;
@@ -0,0 +1,45 @@
1
+ /**
2
+ * 知识库索引主循环:扫描变更 → 解析分块 → 等待上传批次 → 向量化收尾。
3
+ * 主进程直跑(CUSTOMIZE_AGENT_DISABLE_KB_CHILD_PROCESS=1)与后台子进程(kb-index-worker.cjs)
4
+ * 共用同一实现,避免两处逐行重复导致索引行为漂移。
5
+ * 进度上报通过 onProgress 回调交给调用方(主进程写操作日志、子进程经 IPC 转发),本函数不落盘。
6
+ */
7
+ export async function runIndexLoop(project, job, onProgress) {
8
+ let diff = job.relativePaths?.length
9
+ ? await project.incrementalIndex({ vectorMode: job.vectorMode, onProgress, onlyRelativePaths: job.relativePaths })
10
+ : job.relativePath
11
+ ? await project.reindexFile(job.relativePath, { vectorMode: job.vectorMode, onProgress })
12
+ : job.forceReindexAll
13
+ ? await project.forceReindexAll({ vectorMode: job.vectorMode, onProgress })
14
+ : await project.consumePendingIndexJobs({ vectorMode: job.vectorMode, onProgress, waitForUploadId: job.uploadOperationId });
15
+ let idleChecks = 0;
16
+ // 上传 session 空闲上限:批次间无新文件到达超过该时长即退出等待,避免前端中断后 worker 永久挂起
17
+ const sessionIdleLimitMs = Math.max(60_000, Number(process.env.CUSTOMIZE_KB_UPLOAD_SESSION_IDLE_MS || 600_000));
18
+ const maxIdleChecks = Math.max(10, Math.ceil(sessionIdleLimitMs / 1000));
19
+ // 等待任何未关闭的上传 session(不限于本 operationId),避免重叠上传的后续批次文件无人消费
20
+ while (!job.relativePath && !job.relativePaths?.length && (project.countPendingIndexJobs() > 0 || (project.hasOpenUploadSessions() && idleChecks < maxIdleChecks))) {
21
+ if (project.countPendingIndexJobs() === 0) {
22
+ idleChecks += 1;
23
+ await new Promise(resolve => setTimeout(resolve, 1000));
24
+ continue;
25
+ }
26
+ idleChecks = 0;
27
+ const nextDiff = await project.consumePendingIndexJobs({ vectorMode: job.vectorMode, onProgress, waitForUploadId: job.uploadOperationId });
28
+ diff = {
29
+ newFiles: [...diff.newFiles, ...nextDiff.newFiles],
30
+ modifiedFiles: [...diff.modifiedFiles, ...nextDiff.modifiedFiles],
31
+ deletedFiles: [...diff.deletedFiles, ...nextDiff.deletedFiles],
32
+ unchangedCount: diff.unchangedCount + nextDiff.unchangedCount,
33
+ mtimeOnlyCount: diff.mtimeOnlyCount + nextDiff.mtimeOnlyCount,
34
+ skippedFiles: [...diff.skippedFiles, ...nextDiff.skippedFiles],
35
+ hasChanges: diff.hasChanges || nextDiff.hasChanges,
36
+ diffTimeMs: diff.diffTimeMs + nextDiff.diffTimeMs,
37
+ };
38
+ }
39
+ let vectorStatus = project.getVectorStatus();
40
+ if (job.vectorMode === 'defer' && vectorStatus.status === 'pending') {
41
+ await project.indexVectors();
42
+ vectorStatus = project.getVectorStatus();
43
+ }
44
+ return { diff, vectorStatus };
45
+ }
@@ -151,6 +151,8 @@ export declare class IndexStateStore {
151
151
  listParentChunks(relativePath: string): StoredParentChunk[];
152
152
  getParentChunk(relativePath: string, parentId: string): StoredParentChunk | undefined;
153
153
  getDocumentChunk(relativePath: string): StoredDocumentChunk | undefined;
154
+ /** 聚合统计切片数:单条 SUM 查询,避免加载全部索引记录后再求和 */
155
+ countIndexedChunks(filePaths?: string[]): number;
154
156
  getChunksByParent(relativePath: string, parentId: string, limit?: number): StoredChunk[];
155
157
  /**
156
158
  * 使用关键词搜索切片(支持 FTS5 全文搜索和 LIKE 模糊匹配)
@@ -193,6 +195,7 @@ export declare class IndexStateStore {
193
195
  deleteRecord(relativePath: string): void;
194
196
  setMetadata(key: string, value: string): void;
195
197
  getMetadata(key: string): string | undefined;
198
+ listMetadataKeys(prefix?: string): string[];
196
199
  getStats(): {
197
200
  fileCount: number;
198
201
  chunkCount: number;
@@ -9,6 +9,9 @@ export class IndexStateStore {
9
9
  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
10
10
  this.db = new Database(dbPath);
11
11
  this.db.pragma('journal_mode = WAL');
12
+ // P1-8 SQLite 并发读加固:显式 busy_timeout(kb.db 大库并发读曾出现 SQLITE_READONLY 瞬态,
13
+ // WAL 模式下写锁升级竞争由 busy_timeout 兜底等待,避免立即抛锁错误)
14
+ this.db.pragma('busy_timeout = 10000');
12
15
  this.initTables();
13
16
  }
14
17
  /** 加载所有活跃的索引记录 */
@@ -279,13 +282,23 @@ export class IndexStateStore {
279
282
  `).get(relativePath);
280
283
  return row ? this.rowToDocumentChunk(row) : undefined;
281
284
  }
285
+ /** 聚合统计切片数:单条 SUM 查询,避免加载全部索引记录后再求和 */
286
+ countIndexedChunks(filePaths) {
287
+ const paths = filePaths?.filter(Boolean) ?? [];
288
+ const clause = paths.length > 0 ? ` WHERE relative_path IN (${paths.map(() => '?').join(', ')})` : '';
289
+ const row = this.db.prepare(`SELECT COALESCE(SUM(chunk_count), 0) AS total FROM kb_index_state${clause}`)
290
+ .get(...paths);
291
+ return Math.max(0, Math.ceil(Number(row?.total) || 0));
292
+ }
282
293
  getChunksByParent(relativePath, parentId, limit = 6) {
294
+ // 使用 parent_id 列 + idx_kb_chunks_parent 索引;
295
+ // 原先的 metadata_json LIKE '%"parentId":"..."%' 无法使用索引,是每次调用的全表扫描
283
296
  const rows = this.db.prepare(`
284
297
  SELECT rowid, * FROM kb_chunks
285
- WHERE relative_path = ? AND metadata_json LIKE ?
298
+ WHERE relative_path = ? AND parent_id = ?
286
299
  ORDER BY chunk_index
287
300
  LIMIT ?
288
- `).all(relativePath, `%"parentId":"${parentId.replace(/[%_]/gu, '')}"%`, limit);
301
+ `).all(relativePath, parentId, limit);
289
302
  return rows.map(row => this.rowToChunk(row, 0));
290
303
  }
291
304
  /**
@@ -300,11 +313,11 @@ export class IndexStateStore {
300
313
  return [];
301
314
  const filePaths = [...new Set((filters.filePaths ?? []).filter(Boolean))];
302
315
  const effectiveLimit = this.resolveChunkSearchLimit(limit, filePaths);
303
- const results = [
304
- ...(this.ftsEnabled ? this.searchChunksFts(terms, effectiveLimit, filePaths) : []),
305
- ...this.searchChunksLike(terms, effectiveLimit, filePaths),
306
- ];
307
- return this.mergeKeywordResults(results, effectiveLimit);
316
+ const ftsResults = this.ftsEnabled ? this.searchChunksFts(terms, effectiveLimit, filePaths) : [];
317
+ // FTS 返回充分结果时跳过 LIKE 全表扫描(LOWER(x) LIKE '%term%' 无法使用索引,最多 40 term × 7 列);
318
+ // 中文子串匹配 FTS 命中通常不足,此时 LIKE 仍会执行补齐召回,行为与原先一致
319
+ const likeResults = ftsResults.length >= effectiveLimit ? [] : this.searchChunksLike(terms, effectiveLimit, filePaths);
320
+ return this.mergeKeywordResults([...ftsResults, ...likeResults], effectiveLimit);
308
321
  }
309
322
  filePathFilterClause(filePaths, column = 'relative_path') {
310
323
  return filePaths.length > 0 ? ` AND ${column} IN (${filePaths.map(() => '?').join(', ')})` : '';
@@ -506,6 +519,12 @@ export class IndexStateStore {
506
519
  const row = this.db.prepare('SELECT value FROM kb_metadata WHERE key = ?').get(key);
507
520
  return row?.value;
508
521
  }
522
+ listMetadataKeys(prefix) {
523
+ const rows = prefix
524
+ ? this.db.prepare('SELECT key FROM kb_metadata WHERE key LIKE ?').all(`${prefix}%`)
525
+ : this.db.prepare('SELECT key FROM kb_metadata').all();
526
+ return rows.map(row => row.key);
527
+ }
509
528
  getStats() {
510
529
  const stats = this.db.prepare(`
511
530
  SELECT
@@ -4,7 +4,7 @@ import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights,
4
4
  import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
5
5
  import type { VectorStoreInterface } from '../vector/types.js';
6
6
  import { type VectorIndexResult } from '../vector/vector-indexer.js';
7
- import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
7
+ import { IndexStateStore, type ChunkSearchResult, type FileRelationship, type StoredChunk } from './index-state-store.js';
8
8
  export type KnowledgeIndexStage = 'scanning' | 'parsing' | 'chunking' | 'vectorizing' | 'done' | 'error';
9
9
  export interface KnowledgeIndexProgress {
10
10
  stage: KnowledgeIndexStage;
@@ -46,6 +46,10 @@ export declare class KnowledgeBaseManager {
46
46
  private projectConfig?;
47
47
  private lastSkippedFiles;
48
48
  private readonly llmProvider?;
49
+ private readonly queryExpansionCache;
50
+ private readonly queryExpansionInFlight;
51
+ private queryExpansionActive;
52
+ private readonly queryExpansionWaiters;
49
53
  private onProgress?;
50
54
  constructor(options: KnowledgeBaseManagerOptions);
51
55
  initialize(): void;
@@ -87,14 +91,14 @@ export declare class KnowledgeBaseManager {
87
91
  listChunks(options?: {
88
92
  relativePath?: string;
89
93
  limit?: number;
90
- }): import("./index-state-store.js").StoredChunk[];
94
+ }): StoredChunk[];
91
95
  getFileDetail(relativePath: string, options?: {
92
96
  maxChunkContentChars?: number;
93
97
  }): {
94
98
  file: IndexStateRecord;
95
99
  absolutePath: string;
96
100
  directory: string;
97
- chunks: import("./index-state-store.js").StoredChunk[];
101
+ chunks: StoredChunk[];
98
102
  totalChunkCount: number;
99
103
  parents: import("./index-state-store.js").StoredParentChunk[];
100
104
  relationships: FileRelationship[];
@@ -167,6 +171,9 @@ export declare class KnowledgeBaseManager {
167
171
  };
168
172
  private rewriteQueries;
169
173
  private llmExpandQueries;
174
+ /** 查询扩展 LLM 调用的简易信号量:与文档生成端的全局信号量解耦,避免扩展请求无界并发击穿模型端点 */
175
+ private acquireQueryExpansionSlot;
176
+ private releaseQueryExpansionSlot;
170
177
  private retrievalWeights;
171
178
  private heuristicRerank;
172
179
  private normalizeSearchText;
@@ -192,6 +199,8 @@ export declare class KnowledgeBaseManager {
192
199
  private consumePendingVectorRelativePaths;
193
200
  private ensureVectorIndexFresh;
194
201
  uploadSessionIsOpen(operationId: string): boolean;
202
+ listOpenUploadSessions(): string[];
203
+ hasOpenUploadSessions(): boolean;
195
204
  private emptyDiff;
196
205
  private statRelativePaths;
197
206
  private moveUploadedFile;
@@ -17,6 +17,11 @@ import { ChangeTracker } from './change-tracker.js';
17
17
  import { KnowledgeFileScanner } from './file-scanner.js';
18
18
  import { IndexStateStore } from './index-state-store.js';
19
19
  import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
20
+ // 查询扩展(LLM)调用的约束:生成流程会高频触发查询扩展,
21
+ // TTL 缓存吸收重复扩展、信号量避免扩展请求无界并发击穿模型端点
22
+ const QUERY_EXPANSION_TTL_MS = 10 * 60 * 1000;
23
+ const QUERY_EXPANSION_CACHE_MAX = 256;
24
+ const QUERY_EXPANSION_MAX_CONCURRENCY = 2;
20
25
  export class KnowledgeBaseManager {
21
26
  scope;
22
27
  projectRoot;
@@ -37,6 +42,10 @@ export class KnowledgeBaseManager {
37
42
  projectConfig;
38
43
  lastSkippedFiles = [];
39
44
  llmProvider;
45
+ queryExpansionCache = new Map();
46
+ queryExpansionInFlight = new Map();
47
+ queryExpansionActive = 0;
48
+ queryExpansionWaiters = [];
40
49
  onProgress;
41
50
  constructor(options) {
42
51
  this.scope = options.scope;
@@ -58,7 +67,7 @@ export class KnowledgeBaseManager {
58
67
  if (!options.projectRoot || !options.projectId) {
59
68
  throw new Error('project knowledge base requires projectRoot and projectId');
60
69
  }
61
- this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot);
70
+ this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot, storageRoot);
62
71
  dbPath = path.join(storageRoot, 'projects', options.projectId, 'kb.db');
63
72
  }
64
73
  this.store = new IndexStateStore(dbPath);
@@ -339,11 +348,9 @@ export class KnowledgeBaseManager {
339
348
  }
340
349
  searchCorpusSize(filters) {
341
350
  const paths = filters?.filePaths ?? (filters?.filePath ? [filters.filePath] : undefined);
342
- const pathSet = new Set(paths?.filter(Boolean));
343
- const total = this.store.listRecords()
344
- .filter(record => pathSet.size === 0 || pathSet.has(record.relativePath))
345
- .reduce((sum, record) => sum + Math.max(0, Math.ceil(Number(record.chunkCount) || 0)), 0);
346
- return Math.max(1, total);
351
+ const pathSet = paths?.filter(Boolean) ?? [];
352
+ // 单条 SUM 聚合,避免 listRecords 全量加载每条索引记录
353
+ return Math.max(1, this.store.countIndexedChunks(pathSet.length > 0 ? pathSet : undefined));
347
354
  }
348
355
  expandContext(item) {
349
356
  const chunkIndex = item.chunkIndex ?? this.parseChunkIndex(item.id);
@@ -362,7 +369,14 @@ export class KnowledgeBaseManager {
362
369
  const chunks = parentChunks.length > 0 ? parentChunks : this.store.getContextChunks(item.filePath, chunkIndex, 1);
363
370
  if (chunks.length === 0) {
364
371
  const document = this.store.getDocumentChunk(item.filePath);
365
- return document ? { ...item, content: document.content, sectionTitle: item.sectionTitle ?? 'Document Parent' } : item;
372
+ if (!document)
373
+ return item;
374
+ // 文档级兜底必须截断:整份文档全文(可能 10 万字级)会撑爆 LLM 上下文预算并稀释命中片段信噪比
375
+ const MAX_DOCUMENT_CONTENT_CHARS = 6000;
376
+ const content = document.content.length > MAX_DOCUMENT_CONTENT_CHARS
377
+ ? `${document.content.slice(0, MAX_DOCUMENT_CONTENT_CHARS)}\n……(文档过长,已截断,完整内容请查看原文件)`
378
+ : document.content;
379
+ return { ...item, content, sectionTitle: item.sectionTitle ?? 'Document Parent' };
366
380
  }
367
381
  return {
368
382
  ...item,
@@ -375,10 +389,13 @@ export class KnowledgeBaseManager {
375
389
  }
376
390
  async hybridSearch(query, options = {}) {
377
391
  const requestedLimit = Number.isFinite(options.limit) && options.limit > 0 ? Math.ceil(options.limit) : undefined;
378
- const corpusSize = this.searchCorpusSize(options.filters);
379
- const effectiveLimit = requestedLimit ?? corpusSize;
392
+ // 调用方已显式指定 limit 时无需计算语料总量,避免每次搜索都做一次全量聚合
393
+ const effectiveLimit = requestedLimit ?? this.searchCorpusSize(options.filters);
380
394
  const start = Date.now();
381
395
  const weights = this.retrievalWeights(options.weights);
396
+ // P1-10 检索语义澄清:generationMode=true 表示"生成场景检索"(文档正文生成链路调用),
397
+ // 该场景刻意跳过 LLM 查询重写——正文生成已占满 LLM 全局信号量,检索侧再触发 LLM 扩展会互相拖慢;
398
+ // 命名按"调用场景"而非"是否启用 LLM 重写",因此 true 反而跳过重写。
382
399
  const rewrittenQueries = options.generationMode ? [query.trim()].filter(Boolean) : await this.rewriteQueries(query);
383
400
  const rankedLists = [];
384
401
  const keywordMultiplier = options.generationMode ? 2 : 3;
@@ -725,23 +742,65 @@ export class KnowledgeBaseManager {
725
742
  async llmExpandQueries(query) {
726
743
  if (!this.llmProvider)
727
744
  return [];
728
- const prompt = `你是一个搜索查询优化器。用户输入了一个搜索查询,请生成 3-5 个不同的查询变体,用不同的措辞和同义词来表达相同的信息需求,以便在知识库中检索到更全面的结果。
745
+ // TTL 缓存 + 在途去重:生成流程会在短时间内对相近查询重复扩展,
746
+ // 缓存避免重复 LLM 调用;同一查询并发进入时复用同一 Promise
747
+ const cached = this.queryExpansionCache.get(query);
748
+ if (cached && cached.expiresAt > Date.now())
749
+ return cached.queries;
750
+ const inFlight = this.queryExpansionInFlight.get(query);
751
+ if (inFlight)
752
+ return inFlight;
753
+ const run = (async () => {
754
+ const release = await this.acquireQueryExpansionSlot();
755
+ try {
756
+ const prompt = `你是一个搜索查询优化器。用户输入了一个搜索查询,请生成 3-5 个不同的查询变体,用不同的措辞和同义词来表达相同的信息需求,以便在知识库中检索到更全面的结果。
729
757
 
730
758
  如果查询是中文,请同时生成英文变体;如果查询是英文,请同时生成中文变体。
731
759
 
732
760
  直接输出查询列表,每行一个,不要编号或其他文字。
733
761
 
734
762
  原始查询:${query}`;
735
- const response = await this.llmProvider.chat([
736
- { role: 'system', content: '你是一个精确的搜索查询扩展引擎。只输出查询列表。' },
737
- { role: 'user', content: prompt },
738
- ], { temperature: 0.3, maxTokens: 500 });
739
- return response.content
740
- .split('\n')
741
- .map(line => line.replace(/^[-*\d.]+\s*/, '').trim())
742
- .filter(line => line.length > 0 && line !== query)
743
- .slice(0, 5);
763
+ const response = await this.llmProvider.chat([
764
+ { role: 'system', content: '你是一个精确的搜索查询扩展引擎。只输出查询列表。' },
765
+ { role: 'user', content: prompt },
766
+ ], { temperature: 0.3, maxTokens: 500 });
767
+ const queries = response.content
768
+ .split('\n')
769
+ .map(line => line.replace(/^[-*\d.]+\s*/, '').trim())
770
+ .filter(line => line.length > 0 && line !== query)
771
+ .slice(0, 5);
772
+ if (this.queryExpansionCache.size >= QUERY_EXPANSION_CACHE_MAX)
773
+ this.queryExpansionCache.clear();
774
+ this.queryExpansionCache.set(query, { queries, expiresAt: Date.now() + QUERY_EXPANSION_TTL_MS });
775
+ return queries;
776
+ }
777
+ finally {
778
+ release();
779
+ this.queryExpansionInFlight.delete(query);
780
+ }
781
+ })();
782
+ this.queryExpansionInFlight.set(query, run);
783
+ return run;
784
+ }
785
+ /** 查询扩展 LLM 调用的简易信号量:与文档生成端的全局信号量解耦,避免扩展请求无界并发击穿模型端点 */
786
+ acquireQueryExpansionSlot() {
787
+ return new Promise(resolve => {
788
+ if (this.queryExpansionActive < QUERY_EXPANSION_MAX_CONCURRENCY) {
789
+ this.queryExpansionActive += 1;
790
+ resolve(this.releaseQueryExpansionSlot);
791
+ }
792
+ else {
793
+ this.queryExpansionWaiters.push(() => {
794
+ this.queryExpansionActive += 1;
795
+ resolve(this.releaseQueryExpansionSlot);
796
+ });
797
+ }
798
+ });
744
799
  }
800
+ releaseQueryExpansionSlot = () => {
801
+ this.queryExpansionActive = Math.max(0, this.queryExpansionActive - 1);
802
+ this.queryExpansionWaiters.shift()?.();
803
+ };
745
804
  retrievalWeights(overrides = {}) {
746
805
  return {
747
806
  keyword: overrides.keyword ?? Number(process.env.KB_RETRIEVAL_KEYWORD_WEIGHT ?? 1),
@@ -864,10 +923,15 @@ export class KnowledgeBaseManager {
864
923
  return [...new Set(labels)];
865
924
  }
866
925
  hydrateVectorResultsFromSqlite(items) {
926
+ // 批量按 rowid 取切片,避免对每条向量结果单发一次 SQL(N+1)
927
+ const rowids = [...new Set(items.filter(item => item.rowid).map(item => item.rowid))];
928
+ const chunksByRowid = new Map();
929
+ for (const chunk of this.store.getChunksByRowids(rowids))
930
+ chunksByRowid.set(chunk.rowid, chunk);
867
931
  return items.map(item => {
868
932
  if (!item.rowid)
869
933
  return item;
870
- const chunk = this.store.getChunkByRowid(item.rowid);
934
+ const chunk = chunksByRowid.get(item.rowid);
871
935
  if (!chunk)
872
936
  return item;
873
937
  const hydrated = this.toFederatedItem({ ...chunk, score: item.score, scoreDetails: item.scoreDetails }, 'vector');
@@ -1093,6 +1157,18 @@ export class KnowledgeBaseManager {
1093
1157
  uploadSessionIsOpen(operationId) {
1094
1158
  return this.store.getMetadata(`upload_session:${operationId}`) === 'open';
1095
1159
  }
1160
+ listOpenUploadSessions() {
1161
+ this.initialize();
1162
+ const prefix = 'upload_session:';
1163
+ return this.store.listMetadataKeys(prefix)
1164
+ .filter(key => this.store.getMetadata(key) === 'open')
1165
+ .map(key => key.slice(prefix.length));
1166
+ }
1167
+ hasOpenUploadSessions() {
1168
+ this.initialize();
1169
+ const prefix = 'upload_session:';
1170
+ return this.store.listMetadataKeys(prefix).some(key => this.store.getMetadata(key) === 'open');
1171
+ }
1096
1172
  emptyDiff() {
1097
1173
  return { newFiles: [], modifiedFiles: [], deletedFiles: [], unchangedCount: 0, mtimeOnlyCount: 0, skippedFiles: [], hasChanges: false, diffTimeMs: 0 };
1098
1174
  }
@@ -1125,13 +1201,14 @@ export class KnowledgeBaseManager {
1125
1201
  }
1126
1202
  hasUsableContent(text, metadata) {
1127
1203
  const coverage = String(metadata.contentCoverage ?? '');
1128
- if (['metadata', 'metadata_filename', 'pdf_metadata_only', 'office_zip_empty_text', 'office_zip_failed'].includes(coverage))
1204
+ if (['metadata', 'metadata_filename', 'pdf_metadata_only', 'office_zip_empty_text', 'office_zip_failed', 'cad_no_extractable_text'].includes(coverage))
1129
1205
  return false;
1130
1206
  return text.trim().length > 0;
1131
1207
  }
1132
1208
  isMetadataOnlyNonBlocking(file, metadata) {
1133
1209
  const coverage = String(metadata.contentCoverage ?? '');
1134
- return file.category === 'image' && ['image_too_small_for_ocr', 'ocr_no_text'].includes(coverage);
1210
+ return (file.category === 'image' && ['image_too_small_for_ocr', 'ocr_no_text'].includes(coverage))
1211
+ || (file.category === 'cad' && coverage === 'cad_no_extractable_text');
1135
1212
  }
1136
1213
  defaultUploadRelativePath(fileName) {
1137
1214
  const classification = this.classifier.classifyVirtual(fileName);
@@ -35,7 +35,7 @@ export class MultiProjectManager {
35
35
  scope: 'project',
36
36
  projectRoot: resolvedRoot,
37
37
  projectId,
38
- kbPath: getProjectKbPath(resolvedRoot),
38
+ kbPath: getProjectKbPath(resolvedRoot, this.storageRoot),
39
39
  storageRoot: this.storageRoot,
40
40
  llmProvider: this.llmProvider,
41
41
  });
@@ -106,7 +106,7 @@ export class MultiProjectManager {
106
106
  scope: 'project',
107
107
  projectRoot: project.projectRoot,
108
108
  projectId: project.projectId,
109
- kbPath: project.kbPath,
109
+ kbPath: getProjectKbPath(project.projectRoot, this.storageRoot),
110
110
  storageRoot: this.storageRoot,
111
111
  llmProvider: this.llmProvider,
112
112
  });
@@ -1,8 +1,8 @@
1
1
  import type { ProjectConfig } from '../types.js';
2
2
  /** 获取项目配置文件路径 */
3
3
  export declare function getProjectConfigPath(projectRoot: string, storageRoot?: string): string;
4
- /** 获取项目知识库目录路径 */
5
- export declare function getProjectKbPath(projectRoot: string): string;
4
+ /** 获取项目知识库落盘目录路径(用户数据目录下,与 kb.db 索引同生命周期,不再落在项目根目录) */
5
+ export declare function getProjectKbPath(projectRoot: string, storageRoot?: string): string;
6
6
  /** 确保项目存在 CUSTOMIZE.md 文件(如不存在则创建默认模板) */
7
7
  export declare function ensureProjectCustomizeFile(projectRoot: string): void;
8
8
  /** 项目配置管理器,负责加载和保存项目配置 */
@@ -8,9 +8,10 @@ export function getProjectConfigPath(projectRoot, storageRoot = path.join(os.hom
8
8
  const projectId = computeProjectId(projectRoot);
9
9
  return path.join(storageRoot, 'projects', projectId, 'project.json');
10
10
  }
11
- /** 获取项目知识库目录路径 */
12
- export function getProjectKbPath(projectRoot) {
13
- return path.join(projectRoot, KNOWLEDGE_BASE_DIR);
11
+ /** 获取项目知识库落盘目录路径(用户数据目录下,与 kb.db 索引同生命周期,不再落在项目根目录) */
12
+ export function getProjectKbPath(projectRoot, storageRoot = path.join(os.homedir(), USER_DATA_DIR)) {
13
+ const projectId = computeProjectId(projectRoot);
14
+ return path.join(storageRoot, 'projects', projectId, KNOWLEDGE_BASE_DIR);
14
15
  }
15
16
  const DEFAULT_CUSTOMIZE_MD = `# Customize Agent 配置示例
16
17
 
@@ -8,6 +8,8 @@ export class ProjectRegistry {
8
8
  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
9
9
  this.db = new Database(dbPath);
10
10
  this.db.pragma('journal_mode = WAL');
11
+ // P1-8 SQLite 并发读加固:显式 busy_timeout,registry.db 并发读写竞争由等待兜底而非立即抛锁
12
+ this.db.pragma('busy_timeout = 10000');
11
13
  this.initTables();
12
14
  }
13
15
  upsert(project) {
@@ -17,6 +17,8 @@ export declare class ContentExtractor {
17
17
  private isReadableCadValue;
18
18
  private cleanExtractedText;
19
19
  private textScore;
20
+ /** 统计图纸提取片段中的实际字符数据量(汉字/字母/数字),用于判断"无字符数据不入库" */
21
+ private countCadCharacterData;
20
22
  private extractCad;
21
23
  private extractDxf;
22
24
  private buildCadSemanticNodes;
@@ -5,9 +5,15 @@ import { tmpdir } from 'node:os';
5
5
  import { fileURLToPath, pathToFileURL } from 'node:url';
6
6
  import { resolveAndImport, resolvePackage } from './module-resolver.js';
7
7
  import { createOcrProvider } from './ocr-providers.js';
8
- const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
9
- const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
10
- const CAD_DOMAIN_SIGNAL_RE = /工程|项目|施工|建筑|结构|装饰|电气|给排水|消防|暖通|平面|立面|剖面|节点|详图|材料|尺寸|标高|轴线|图层|门窗|墙|地面|顶面|照明|配电|弱电|空调|卫生间|楼梯|屋面|基础|柱|梁|板|图号|设计|说明/u;
8
+ const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb[\w:]+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*|MLEADERSTYLE|AppInfoHistory|AppInfoDataList|ObjectDBX|Classes|DICTIONARYVARP|ObjFreeSpaceP|AuxHeaderT|\$AUDIT_BAD_\w+)\b/giu;
9
+ const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb[\w:]+|\$AUDIT_BAD_\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous|MLEADERSTYLE|ObjectDBX)$/iu;
10
+ /** DWG 二进制误读产生的 C1 控制字符(U+0080-U+009F,如 GBK 半字节),正常图纸标注不会出现 */
11
+ const CAD_C1_CONTROL_RE = /[\u0080-\u009F]/gu;
12
+ /** AutoCAD 控制码:%%c=Φ、%%d=°、%%p=±、%%132 等数字码=Φ,统一解码为可读符号 */
13
+ const CAD_CONTROL_CODE_RE = /%%(?:c|d|p|\d{2,3})/giu;
14
+ const CAD_DOMAIN_SIGNAL_RE = /工程|项目|施工|建筑|结构|装饰|电气|给排水|消防|暖通|平面|立面|剖面|节点|详图|材料|尺寸|标高|轴线|图层|门窗|墙|地面|顶面|照明|配电|弱电|空调|卫生间|楼梯|屋面|基础|柱|梁|板|图号|设计|说明|轴|电|井|土|夯/u;
15
+ /** 图纸兜底解析可读字符(汉字/字母/数字)的最低总量,低于该值视为无字符数据,不入库 */
16
+ const MIN_CAD_CHARACTER_DATA = 32;
11
17
  const OCR_NATIVE_NOISE_PATTERNS = [/^Image too small to scale!!/u, /^Line cannot be recognized!!$/u];
12
18
  /** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
13
19
  export class ContentExtractor {
@@ -142,6 +148,8 @@ export class ContentExtractor {
142
148
  }
143
149
  cleanCadReadableText(value) {
144
150
  return value
151
+ .replace(CAD_C1_CONTROL_RE, ' ')
152
+ .replace(CAD_CONTROL_CODE_RE, code => (code.toLowerCase() === '%%d' ? '°' : code.toLowerCase() === '%%p' ? '±' : 'Φ'))
145
153
  .replace(CAD_INTERNAL_TOKEN_RE, '')
146
154
  .replace(/\b(?:LINE|LWPOLYLINE|POLYLINE|INSERT|HATCH|CIRCLE|ARC|DIMENSION|TEXT|MTEXT)\b/giu, '')
147
155
  .replace(/\s+/gu, ' ')
@@ -162,6 +170,13 @@ export class ContentExtractor {
162
170
  const hasDomainSignal = CAD_DOMAIN_SIGNAL_RE.test(compact);
163
171
  const hasCommonTextShape = /[,。;:、,.\-/()()]|\d+(?:\.\d+)?\s*(?:mm|cm|m|㎡|%|°)?/iu.test(compact);
164
172
  const latinVowelCount = chars.filter(char => /[aAeEiIoOuU]/u.test(char)).length;
173
+ // 替换符(U+FFFD):解码失败的标志,正常标注不会出现
174
+ if (chars.includes('\uFFFD'))
175
+ return true;
176
+ // 罕见符号(箭头补充、CJK 部首、杂项数学/圈符/地图符号):二进制误读产物,
177
+ // 正常图纸标注只用常用标点与工程符号
178
+ if (/[\u2046-\u205F\u2070-\u209F\u2100-\u2102\u2104-\u214F\u21B0-\u21FF\u2270-\u22FF\u2400-\u243F\u249C-\u24FF\u2640-\u26FF\u27C0-\u27EF\u2900-\u297F\u2A00-\u2AFF\u2B00-\u2BFF\u2E00-\u2FFF\u3200-\u33FF]/u.test(compact))
179
+ return true;
165
180
  if (readableRatio < 0.6)
166
181
  return true;
167
182
  if (symbolRatio > 0.35 && !hasDomainSignal)
@@ -170,6 +185,96 @@ export class ContentExtractor {
170
185
  return true;
171
186
  if (cjk >= 8 && digits === 0 && !hasDomainSignal && !hasCommonTextShape)
172
187
  return true;
188
+ // 短行内汉字-Latin-汉字交叉混排(考堂f肀、渱潑喲W晀耀):二进制误读的典型形态;
189
+ // 正常标注的字母编号在汉字前或后(JD 电井、AB轴),不会夹在汉字中间
190
+ if (chars.length <= 8 && cjk >= 2 && latin >= 1 && digits === 0 && !hasDomainSignal && /[\p{Script=Han}][\p{Script=Latin}][\p{Script=Han}]/u.test(compact))
191
+ return true;
192
+ // 纯 Latin 行含 Latin-1 扩展字符(VdA«UdA«UdANÒg、dAç dA+ dA«):正常英文标注不用扩展字符
193
+ if (cjk === 0 && digits === 0 && latin >= 8 && /[\u00A0-\u02AF\u1E00-\u1EFF]/u.test(compact))
194
+ return true;
195
+ // 超长无句读行:二进制误读产生的长连续乱码(常混入个别汉字/数字触发域信号豁免,
196
+ // 因此只认中文句读,数字/单位不再豁免)
197
+ if (chars.length > 400 && !/[\u3002\uFF0C\uFF1B\uFF1A\u3001、,。;:]/.test(compact))
198
+ return true;
199
+ // 行内短片段周期性重复(Ml+Ml+Ml、AM~AM~BM~BM、M|¶M|¶、2dA+2dA+2dA):
200
+ // 正常标注不会让同一 2-4 字符片段在一行内重复出现,二进制误读则产生大量循环模式
201
+ const shortTokens = compact.split(/[^\p{L}\p{N}]+/u).filter(token => token.length >= 1 && token.length <= 4);
202
+ if (shortTokens.length >= 3 && chars.length >= 8 && !hasDomainSignal) {
203
+ const tokenFreq = new Map();
204
+ for (const token of shortTokens)
205
+ tokenFreq.set(token, (tokenFreq.get(token) ?? 0) + 1);
206
+ let repeatedChars = 0;
207
+ for (const [token, count] of tokenFreq) {
208
+ if (count >= 2)
209
+ repeatedChars += token.length * count;
210
+ }
211
+ if (repeatedChars / Math.max(1, shortTokens.join('').length) >= 0.4)
212
+ return true;
213
+ }
214
+ // 短行内同一汉字高频重复(摁䭚摁譚摁譚摁、耀U耀W耀Y耀):正常标注不会在
215
+ // 14 字符以内的行里让重复汉字占比超过 60%
216
+ const hanChars = chars.filter(char => /[\p{Script=Han}]/u.test(char));
217
+ if (hanChars.length >= 4 && chars.length <= 14) {
218
+ const hanFreq = new Map();
219
+ for (const char of hanChars)
220
+ hanFreq.set(char, (hanFreq.get(char) ?? 0) + 1);
221
+ const repeatedHan = hanChars.filter(char => (hanFreq.get(char) ?? 0) >= 2).length;
222
+ if (repeatedHan / hanChars.length >= 0.6)
223
+ return true;
224
+ }
225
+ // Latin-1 扩展字符(¡-ÿ 区)密集行:中文图纸标注几乎不用这些字符,
226
+ // 二进制误读(GBK/CP1252 混读)会批量产生
227
+ const latinExtended = chars.filter(char => /[\u00A0-\u02AF\u1E00-\u1EFF]/u.test(char)).length;
228
+ if (latinExtended >= 3 && (latinExtended / chars.length >= 0.3 || latin / Math.max(1, chars.length) >= 0.6))
229
+ return true;
230
+ // 纯 Latin-1 扩展字母短行(无 ASCII 字母/数字):GBK 中文标注被 Latin-1 误读的
231
+ // 典型产物("上"→ÉÏ、"柜"→¹ñ),正常标注的英文是 ASCII、中文是汉字,
232
+ // 不会出现整行全由扩展拉丁字母构成的短行
233
+ if (cjk === 0 && digits === 0 && latin >= 2 && latinExtended === latin && chars.length <= 12)
234
+ return true;
235
+ // GBK 误读标点/字母混入 ASCII 标注("1APz:P4~P6"→1APz£ºP4~P6、"消防控制"→Ïû·À¿ØÖÆ):
236
+ // 正常工程标注只使用 °±×÷·µ²³Ø 等少数 Latin-1 字符,
237
+ // 无汉字行中出现 2 个以上其他 Latin-1 字符即判定为 GBK 编码误读
238
+ const gbkMisreadChars = chars.filter(char => /[\u00A0-\u00FF]/u.test(char) && !'°±×÷·µ²³Ø'.includes(char)).length;
239
+ if (gbkMisreadChars >= 2 && cjk === 0)
240
+ return true;
241
+ // 分数符号/上标数字(¼½¾¹):中文工程图纸标注几乎不用(仅 ²³ 常见保留),
242
+ // 出现在标注中基本是 GBK 误读(门宽 FM×1524 误读为 FM¼×1524)
243
+ if (/[¹¼½¾]/u.test(compact) && cjk === 0)
244
+ return true;
245
+ // 无空格的超长 Latin 字母串(≥16 字符):正常标注是词/编号,不会出现连续长字母串
246
+ if (/[\p{Script=Latin}\u00C0-\u024F]{16,}/u.test(compact))
247
+ return true;
248
+ // GUID:DWG 内部结构标识,不是图纸字符数据
249
+ if (/[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}/u.test(compact))
250
+ return true;
251
+ // 单汉字高频重复:正常标注/文本不会让同一汉字占比超过 20%(如乱码行中
252
+ // 生僻字“罍”重复 400+ 次仍被 readableRatio 计为可读字符而漏网)
253
+ const charCounts = new Map();
254
+ for (const char of chars)
255
+ charCounts.set(char, (charCounts.get(char) ?? 0) + 1);
256
+ let mostCommon = '';
257
+ let mostCommonCount = 0;
258
+ for (const [char, count] of charCounts) {
259
+ if (count > mostCommonCount) {
260
+ mostCommon = char;
261
+ mostCommonCount = count;
262
+ }
263
+ }
264
+ if (mostCommonCount >= 5 && mostCommonCount / chars.length > 0.2 && /[\p{Script=Han}]/u.test(mostCommon))
265
+ return true;
266
+ // CJK 扩展区生僻字(Ext A/B 等)混排:正常图纸标注几乎只用常用字(基本区
267
+ // U+4E00-U+9FFF),二进制误读会批量产生 U+3400-U+4DBF 等扩展区字符
268
+ const rareCjk = chars.filter(char => /[\u3400-\u4DBF\u{20000}-\u{2FA1F}]/u.test(char)).length;
269
+ if (rareCjk >= 2 && rareCjk / chars.length >= 0.1)
270
+ return true;
271
+ if (rareCjk >= 1 && rareCjk / chars.length >= 0.2)
272
+ return true;
273
+ // 非中/英/希字母脚本混排(韩文、藏文、彝文、泰文等):中文标注行内混入
274
+ // 其他文字系统字母是二进制误读的典型产物(希腊字母 ΦφΩ 在图纸中常见,豁免)
275
+ const foreignLetter = chars.filter(char => /\p{Letter}/u.test(char) && !/[\p{Script=Han}\p{Script=Latin}\p{Script=Greek}]/u.test(char)).length;
276
+ if (foreignLetter >= 1 && cjk > 0)
277
+ return true;
173
278
  return false;
174
279
  }
175
280
  isReadableCadValue(value) {
@@ -180,7 +285,8 @@ export class ContentExtractor {
180
285
  const normalized = [...value]
181
286
  .filter(char => {
182
287
  const code = char.charCodeAt(0);
183
- return code === 9 || code === 10 || code === 13 || code >= 32;
288
+ // U+FFFF 非字符:PDF 表单空白下划线/占位符的提取产物,无检索意义,统一剔除
289
+ return code !== 0xFFFF && (code === 9 || code === 10 || code === 13 || code >= 32);
184
290
  })
185
291
  .join('');
186
292
  if (file.category !== 'cad')
@@ -197,6 +303,10 @@ export class ContentExtractor {
197
303
  const alnum = (value.match(/[\p{L}\p{N}]/gu) ?? []).length;
198
304
  return cjk * 4 + alnum + Math.min(value.length, 200) / 20;
199
305
  }
306
+ /** 统计图纸提取片段中的实际字符数据量(汉字/字母/数字),用于判断"无字符数据不入库" */
307
+ countCadCharacterData(fragments) {
308
+ return (fragments.join('').match(/[\p{Script=Han}\p{L}\p{N}]/gu) ?? []).length;
309
+ }
200
310
  async extractCad(file) {
201
311
  const metadata = { extractionMode: 'builtin_cad_structural', vectorizable: true, preferredExtractionMode: 'dwg_to_dxf_semantic' };
202
312
  const warnings = [];
@@ -228,7 +338,16 @@ export class ContentExtractor {
228
338
  metadata.blockNames = uniqueBlocks.slice(0, 80);
229
339
  metadata.entityTypeCount = uniqueEntityTypes.length;
230
340
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
341
+ const characterDataCount = this.countCadCharacterData([...textEntities.map(annotation => annotation.text), ...uniqueLayers, ...uniqueBlocks]);
342
+ if (characterDataCount < MIN_CAD_CHARACTER_DATA) {
343
+ // 图纸无字符数据(无文字标注、图层/块名均为内部默认值),不入库
344
+ metadata.contentCoverage = 'cad_no_extractable_text';
345
+ metadata.characterDataCount = characterDataCount;
346
+ warnings.push(`${file.format} DXF 未提取到字符数据(仅 ${characterDataCount} 个可读字符),图纸内容未入库`);
347
+ return { text: this.metadataOnlyText(file), metadata, warnings };
348
+ }
231
349
  metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
350
+ metadata.characterDataCount = characterDataCount;
232
351
  const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
233
352
  return {
234
353
  text: [
@@ -291,21 +410,59 @@ export class ContentExtractor {
291
410
  return result;
292
411
  }
293
412
  const binaryFragments = this.extractBinaryReadableFragments(file.absolutePath);
294
- const readable = binaryFragments.filter(value => this.isReadableCadValue(value)).slice(0, 5000);
413
+ // OLE 属性集(<prop_set ...>)是 DWG 文件内部的二进制属性结构,不含图纸标注字符,
414
+ // 属于解析噪声而非字符数据,直接排除(不锚定行首:误读字符可能混在片段开头)
415
+ const withoutPropSets = binaryFragments.filter(value => !/<prop_set\b/u.test(value));
416
+ let readable = withoutPropSets.filter(value => this.isReadableCadValue(value)).slice(0, 5000);
417
+ // 文档级字符词频过滤:真实标注文字在图纸中会重复出现(标题/图层/图名等),
418
+ // 随机二进制噪声片段中每个字符几乎只出现一次(孤立字符);短片段若全部由
419
+ // 孤立字符构成即为随机噪声,丢弃
420
+ const charFreq = new Map();
421
+ for (const value of readable) {
422
+ for (const char of value)
423
+ charFreq.set(char, (charFreq.get(char) ?? 0) + 1);
424
+ }
425
+ const isolatedNoise = (value) => {
426
+ if (CAD_DOMAIN_SIGNAL_RE.test(value))
427
+ return false;
428
+ const letters = [...value].filter(char => /[\p{L}\p{N}]/u.test(char));
429
+ if (letters.length > 8)
430
+ return false;
431
+ const freqs = letters.map(char => charFreq.get(char) ?? 1);
432
+ // 短片段中 60% 以上字符在全文只出现一次 → 随机噪声(真实标注文字会重复出现)
433
+ const isolatedCount = freqs.filter(freq => freq <= 1).length;
434
+ return isolatedCount / Math.max(1, letters.length) >= 0.6;
435
+ };
436
+ readable = readable.filter(value => !isolatedNoise(value));
437
+ // 可信片段过滤:真实图纸标注至少含 3 个汉字(图名/说明/材料文字)或含域信号词,
438
+ // 且全部字符落在图纸常用字符白名单内(汉字/ASCII 字母数字/常用标点/工程符号);
439
+ // 二进制误读碎片常含 Latin 扩展字母、罕见符号或纯字母数字串,白名单直接排除
440
+ const trustedCadFragment = (value) => {
441
+ const hanCount = (value.match(/\p{Script=Han}/gu) ?? []).length;
442
+ if (hanCount < 3 && !CAD_DOMAIN_SIGNAL_RE.test(value))
443
+ return false;
444
+ return !/[^\p{Script=Han}A-Za-z0-9\s\u3000-\u303F\uFF01-\uFF5E\u2460-\u2473\u2160-\u2179°Φφ×±≤≥∠√℃‰※′″〇●○◆■□▲△★☆◇→←↑↓ΩΔαβγθλμω]/u.test(value);
445
+ };
446
+ readable = readable.filter(trustedCadFragment);
295
447
  const filteredCount = Math.max(0, binaryFragments.length - readable.length);
448
+ // 图纸"字符数据"= 提取到的标注/标题块中实际可读的文字(汉字/字母/数字)总量;
449
+ // 低于最低阈值视为无字符数据,不入库
450
+ const characterDataCount = this.countCadCharacterData(readable);
451
+ const hasCharacterData = characterDataCount >= MIN_CAD_CHARACTER_DATA;
296
452
  metadata.extractionMode = 'builtin_cad_readable_fragments';
297
453
  metadata.professionalConversionUsed = false;
298
- metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments_filtered' : 'metadata';
299
- metadata.contentConfidence = readable.length > 0 ? 'low_fallback_filtered' : 'metadata_only';
454
+ metadata.contentCoverage = hasCharacterData ? 'cad_readable_text_fragments_filtered' : 'cad_no_extractable_text';
455
+ metadata.contentConfidence = hasCharacterData ? 'low_fallback_filtered' : 'metadata_only';
300
456
  metadata.stringCandidateCount = binaryFragments.length;
301
457
  metadata.stringCount = readable.length;
458
+ metadata.characterDataCount = characterDataCount;
302
459
  metadata.filteredGarbledStringCount = filteredCount;
303
- if (readable.length === 0)
304
- warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本,仅记录文件元数据,未生成可检索正文切片`);
460
+ if (!hasCharacterData)
461
+ warnings.push(`${file.format} 内置 CAD 解析器未提取到字符数据(仅 ${characterDataCount} 个可读字符),图纸内容未入库`);
305
462
  else
306
463
  warnings.push(`${file.format} 内置 DWG→DXF 转换未成功,已使用低置信度可读标注/标题块兜底抽取并过滤疑似乱码 ${filteredCount} 条;该结果仅作为兜底证据,不应等同于完整图层、块、标注和尺寸语义解析`);
307
464
  return {
308
- text: readable.length > 0 ? [this.metadataOnlyText(file), `CAD 图纸可读标注/标题块/属性:\n${readable.join('\n')}`].join('\n') : this.metadataOnlyText(file),
465
+ text: hasCharacterData ? [this.metadataOnlyText(file), `CAD 图纸可读标注/标题块/属性:\n${readable.join('\n')}`].join('\n') : this.metadataOnlyText(file),
309
466
  metadata,
310
467
  warnings,
311
468
  };
@@ -335,8 +492,17 @@ export class ContentExtractor {
335
492
  metadata.blockNames = uniqueBlocks.slice(0, 80);
336
493
  metadata.entityTypeCount = uniqueEntityTypes.length;
337
494
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
338
- metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
339
495
  metadata.parsedByDxfParser = Boolean(parsed);
496
+ const characterDataCount = this.countCadCharacterData([...textEntities.map(annotation => annotation.text), ...uniqueLayers, ...uniqueBlocks]);
497
+ if (characterDataCount < MIN_CAD_CHARACTER_DATA) {
498
+ // 图纸无字符数据(无文字标注、图层/块名均为内部默认值),不入库
499
+ metadata.contentCoverage = 'cad_no_extractable_text';
500
+ metadata.characterDataCount = characterDataCount;
501
+ warnings.push(`${file.format} DXF 未提取到字符数据(仅 ${characterDataCount} 个可读字符),图纸内容未入库`);
502
+ return { text: this.metadataOnlyText(file), metadata, warnings };
503
+ }
504
+ metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
505
+ metadata.characterDataCount = characterDataCount;
340
506
  const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
341
507
  return {
342
508
  text: [
@@ -373,10 +539,14 @@ export class ContentExtractor {
373
539
  const text = this.cleanCadReadableText(/(?:^|\r?\n)\s*(?:1|3)\s*\r?\n([^\r\n]+)/u.exec(section)?.[1] ?? '');
374
540
  if (!text || !this.isReadableCadValue(text))
375
541
  return [];
542
+ // 图层/块名同样要过可读性过滤:DXF 里 GBK 误读(Ïä¹ñ)或纯数字/内部
543
+ // 标识(11、AcDb...)会直接混进节点文本,不合格时置空由语义节点回退
544
+ const layer = /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
545
+ const block = /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim() ?? '';
376
546
  return [{
377
547
  text,
378
- layer: /(?:^|\r?\n)\s*8\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
379
- block: /(?:^|\r?\n)\s*2\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]?.trim(),
548
+ layer: layer && this.isReadableCadValue(layer) ? layer : undefined,
549
+ block: block && this.isReadableCadValue(block) ? block : undefined,
380
550
  entityType: section.trim().split(/\s+/u)[0],
381
551
  x: Number(/(?:^|\r?\n)\s*10\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
382
552
  y: Number(/(?:^|\r?\n)\s*20\s*\r?\n([^\r\n]+)/u.exec(section)?.[1]),
@@ -406,25 +576,42 @@ export class ContentExtractor {
406
576
  if (!mod.convertDwgToDxf)
407
577
  return { tool: 'dwgdxf_wasm', warnings: ['内置 dwgdxf WASM 转换器未导出 convertDwgToDxf'] };
408
578
  const dwgBytes = fs.readFileSync(filePath);
409
- const attempts = [
410
- { label: 'package-default' },
411
- { label: 'package-dist-wasm', options: { wasmBase: pathToFileURL(path.join(path.dirname(resolvePackage('dwgdxf')), 'wasm')).href } },
412
- { label: 'package-root-dist-wasm', options: { wasmBase: pathToFileURL(path.join(path.dirname(path.dirname(resolvePackage('dwgdxf'))), 'wasm')).href } },
413
- ];
414
- const failures = [];
415
- for (const attempt of attempts) {
416
- try {
417
- const dxfBytes = await mod.convertDwgToDxf(dwgBytes, attempt.options);
418
- const dxfText = Buffer.from(dxfBytes).toString('utf8');
419
- if (dxfText.trim())
420
- return { dxfText, tool: `dwgdxf_wasm:${attempt.label}`, warnings: [] };
421
- failures.push(`${attempt.label}: 未输出 DXF 文本`);
579
+ // 注意:dwgdxf 内部对 WASM 基础 URL 有模块级缓存,只有首次调用传入的
580
+ // wasmBase 才会生效。若首次调用不带 wasmBase,会走到包内硬编码的构建机
581
+ // 路径(file:///home/runner/work/...)并永久缓存失败结果,后续重试全部
582
+ // 复用同一失败 Promise。因此第一次调用就必须传本地包内 wasm 目录的
583
+ // 正确 file:// URL,不允许再回退重试(重试无效且会污染缓存)。
584
+ let wasmBase;
585
+ try {
586
+ const localWasmDir = path.join(path.dirname(resolvePackage('dwgdxf')), 'wasm');
587
+ if (fs.existsSync(path.join(localWasmDir, 'dwgdxf_bg.wasm'))) {
588
+ wasmBase = pathToFileURL(localWasmDir).href;
422
589
  }
423
- catch (error) {
424
- failures.push(`${attempt.label}: ${error instanceof Error ? error.message : String(error)}`);
590
+ }
591
+ catch { /* 包解析失败时回退到包内默认路径(大概率也会失败,但由下方 catch 统一记录) */ }
592
+ try {
593
+ const dxfBytes = await mod.convertDwgToDxf(dwgBytes, wasmBase ? { wasmBase } : undefined);
594
+ const dxfText = Buffer.from(dxfBytes).toString('utf8');
595
+ if (dxfText.trim())
596
+ return { dxfText, tool: wasmBase ? 'dwgdxf_wasm:local_wasm' : 'dwgdxf_wasm:package_default', warnings: [] };
597
+ return { tool: 'dwgdxf_wasm', warnings: ['内置 dwgdxf WASM 未输出 DXF 文本'] };
598
+ }
599
+ catch (error) {
600
+ const detail = error instanceof Error ? error.message : String(error);
601
+ // unreachable 多为 WASM 内存压力下的瞬时失败(批量索引时主进程同时持有
602
+ // 大型 SQLite 缓存会挤压可用内存),换一个内存状态重试一次;格式类错误
603
+ // (Invalid file format 等)重试无意义,直接失败降级。
604
+ if (detail.includes('unreachable')) {
605
+ try {
606
+ const retryBytes = await mod.convertDwgToDxf(dwgBytes, wasmBase ? { wasmBase } : undefined);
607
+ const retryText = Buffer.from(retryBytes).toString('utf8');
608
+ if (retryText.trim())
609
+ return { dxfText: retryText, tool: wasmBase ? 'dwgdxf_wasm:local_wasm' : 'dwgdxf_wasm:package_default', warnings: ['内置 dwgdxf WASM 首次转换失败(unreachable),重试成功'] };
610
+ }
611
+ catch { /* 重试仍失败,按原错误降级 */ }
425
612
  }
613
+ return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 转换失败${wasmBase ? '' : '(未找到本地 WASM 文件,使用了包内默认路径)'}: ${detail}`] };
426
614
  }
427
- return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 转换失败: ${failures.join(';')}`] };
428
615
  }
429
616
  catch (error) {
430
617
  return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 加载失败: ${error instanceof Error ? error.message : String(error)}`] };
@@ -776,12 +963,20 @@ export class ContentExtractor {
776
963
  return fallback;
777
964
  }
778
965
  extractLegacyOfficeBinary(file) {
779
- const strings = this.extractBinaryStrings(file.absolutePath).slice(0, 1_000);
966
+ const candidates = this.extractBinaryStrings(file.absolutePath).slice(0, 1_000);
967
+ // 二进制兜底提取出的字符串中混有大量误读乱码(如 OLE 复合文档中 UTF-16LE
968
+ // 中文被 latin1/utf8 误读产生的长串生僻字),复用 CAD 乱码判定规则过滤,
969
+ // 避免乱码直接入库污染检索与预览。
970
+ const strings = candidates.filter(value => !this.isLikelyGarbledCadText(value));
971
+ const filteredCount = candidates.length - strings.length;
780
972
  const text = strings.join('\n').trim();
781
973
  return {
782
974
  text,
783
- metadata: { extractionMode: 'builtin_legacy_office_binary_strings', vectorizable: true, contentCoverage: 'legacy_office_binary_strings', stringCount: strings.length },
784
- warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库'],
975
+ metadata: { extractionMode: 'builtin_legacy_office_binary_strings', vectorizable: true, contentCoverage: 'legacy_office_binary_strings_filtered', stringCandidateCount: candidates.length, stringCount: strings.length, filteredGarbledStringCount: filteredCount },
976
+ warnings: [
977
+ ...(filteredCount > 0 ? [`已过滤疑似乱码字符串 ${filteredCount} 条`] : []),
978
+ ...(text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库']),
979
+ ],
785
980
  };
786
981
  }
787
982
  extractLegacyOfficeBinaryWithWarnings(file, warnings) {
package/dist/index.d.ts CHANGED
@@ -9,6 +9,7 @@ export { ChangeTracker } from './core/change-tracker.js';
9
9
  export { KnowledgeFileScanner, type DiskFileStat } from './core/file-scanner.js';
10
10
  export { IndexStateStore, type ChunkSearchResult, type FileHashRecord, type FileRelationship, type StoredChunk } from './core/index-state-store.js';
11
11
  export { KnowledgeBaseManager, type KnowledgeBaseManagerOptions, type KnowledgeIndexProgress } from './core/knowledge-base-manager.js';
12
+ export { runIndexLoop, type IndexRunJob, type IndexRunOutcome } from './core/index-runner.js';
12
13
  export { MultiProjectManager } from './core/multi-project-manager.js';
13
14
  export { computeProjectId } from './core/project-id.js';
14
15
  export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
package/dist/index.js CHANGED
@@ -11,6 +11,7 @@ export { ChangeTracker } from './core/change-tracker.js';
11
11
  export { KnowledgeFileScanner } from './core/file-scanner.js';
12
12
  export { IndexStateStore } from './core/index-state-store.js';
13
13
  export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
14
+ export { runIndexLoop } from './core/index-runner.js';
14
15
  export { MultiProjectManager } from './core/multi-project-manager.js';
15
16
  export { computeProjectId } from './core/project-id.js';
16
17
  export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
@@ -3,6 +3,11 @@
3
3
  *
4
4
  * 定义在 knowledge 包内部,避免直接依赖 @customize-agent/llm。
5
5
  * CLI 层的 ILLMProvider 在结构上兼容此接口,可直接传入。
6
+ *
7
+ * 接入状态(P1-10 检索语义澄清):当前未接入。
8
+ * apps/server 的 getMultiProjectManager() 刻意不注入 llmProvider(见 kbService.ts),
9
+ * 检索查询重写因此退化为本地规则扩展;若未来启用,需配独立低并发 LLM 通道,
10
+ * 避免与文档正文生成争抢全局 LLM 信号量。
6
11
  */
7
12
  export interface LLMChatMessage {
8
13
  role: 'system' | 'user' | 'assistant';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@customize-agent/knowledge",
3
- "version": "4.0.42",
3
+ "version": "4.0.44",
4
4
  "description": "Local knowledge base infrastructure for customize-agent",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -12,14 +12,6 @@
12
12
  "default": "./dist/index.js"
13
13
  }
14
14
  },
15
- "scripts": {
16
- "prebuild": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",
17
- "build": "tsc",
18
- "typecheck": "tsc --noEmit",
19
- "lint": "eslint src/",
20
- "postinstall": "node scripts/install-hnsw.cjs",
21
- "doctor:hnsw": "node scripts/install-hnsw.cjs"
22
- },
23
15
  "license": "MIT",
24
16
  "engines": {
25
17
  "node": ">=20.19.0"
@@ -70,5 +62,13 @@
70
62
  "devDependencies": {
71
63
  "@types/better-sqlite3": "^7.6.13",
72
64
  "@types/node": "^25.9.3"
65
+ },
66
+ "scripts": {
67
+ "prebuild": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",
68
+ "build": "tsc",
69
+ "typecheck": "tsc --noEmit",
70
+ "lint": "eslint src/",
71
+ "postinstall": "node scripts/install-hnsw.cjs",
72
+ "doctor:hnsw": "node scripts/install-hnsw.cjs"
73
73
  }
74
- }
74
+ }