@customize-agent/knowledge 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/chunking/text-chunker.d.ts +12 -1
  2. package/dist/chunking/text-chunker.js +219 -41
  3. package/dist/core/change-tracker.d.ts +1 -0
  4. package/dist/core/change-tracker.js +21 -0
  5. package/dist/core/index-state-store.d.ts +35 -1
  6. package/dist/core/index-state-store.js +249 -16
  7. package/dist/core/knowledge-base-manager.d.ts +69 -3
  8. package/dist/core/knowledge-base-manager.js +535 -132
  9. package/dist/core/multi-project-manager.d.ts +7 -4
  10. package/dist/core/multi-project-manager.js +35 -23
  11. package/dist/embedding/embedding-provider.d.ts +1 -0
  12. package/dist/embedding/embedding-provider.js +16 -1
  13. package/dist/extraction/content-extractor.d.ts +2 -0
  14. package/dist/extraction/content-extractor.js +209 -42
  15. package/dist/extraction/module-resolver.d.ts +17 -0
  16. package/dist/extraction/module-resolver.js +113 -0
  17. package/dist/index.d.ts +2 -3
  18. package/dist/index.js +2 -2
  19. package/dist/llm/llm-search-provider.d.ts +23 -0
  20. package/dist/llm/llm-search-provider.js +1 -0
  21. package/dist/search/federation-search.d.ts +29 -0
  22. package/dist/search/federation-search.js +8 -1
  23. package/dist/vector/chroma-store.d.ts +2 -0
  24. package/dist/vector/chroma-store.js +53 -22
  25. package/dist/vector/vector-indexer.d.ts +3 -0
  26. package/dist/vector/vector-indexer.js +23 -0
  27. package/package.json +11 -3
  28. package/dist/server/dashboard-client.d.ts +0 -2
  29. package/dist/server/dashboard-client.js +0 -396
  30. package/dist/server/dashboard-i18n.d.ts +0 -112
  31. package/dist/server/dashboard-i18n.js +0 -220
  32. package/dist/server/dashboard-page.d.ts +0 -6
  33. package/dist/server/dashboard-page.js +0 -138
  34. package/dist/server/dashboard-server.d.ts +0 -13
  35. package/dist/server/dashboard-server.js +0 -225
  36. package/dist/server/dashboard-styles.d.ts +0 -1
  37. package/dist/server/dashboard-styles.js +0 -152
@@ -1,21 +1,23 @@
1
- import type { FederatedResult, SearchFilters, SearchScope } from '../search/federation-search.js';
2
- import { FederationSearch } from '../search/federation-search.js';
1
+ import type { FederatedResult, RetrievalWeights, SearchFilters, SearchScope } from '../search/federation-search.js';
3
2
  import type { CrossProjectDuplicate, ProjectInfo } from '../types.js';
4
3
  import { KnowledgeBaseManager } from './knowledge-base-manager.js';
4
+ import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
5
5
  export declare class MultiProjectManager {
6
6
  private readonly storageRoot;
7
+ private readonly llmProvider?;
7
8
  private readonly registry;
8
9
  private readonly configManager;
9
10
  private readonly projects;
10
11
  private globalKB?;
11
- constructor(storageRoot?: string);
12
+ constructor(storageRoot?: string, llmProvider?: LLMSearchProvider);
12
13
  getProject(projectRoot: string): Promise<KnowledgeBaseManager>;
13
14
  getGlobalKB(): Promise<KnowledgeBaseManager>;
14
15
  listProjects(): Promise<ProjectInfo[]>;
15
16
  search(projectRoot: string, query: string, options?: {
16
17
  limit?: number;
17
18
  scope?: SearchScope;
18
- }): Promise<ReturnType<FederationSearch['merge']>>;
19
+ weights?: RetrievalWeights;
20
+ }): Promise<FederatedResult>;
19
21
  semanticSearch(projectRoot: string, query: string, options?: {
20
22
  limit?: number;
21
23
  scope?: SearchScope;
@@ -26,5 +28,6 @@ export declare class MultiProjectManager {
26
28
  forgetProject(projectId: string): Promise<void>;
27
29
  closeProject(projectId: string): Promise<void>;
28
30
  shutdown(): Promise<void>;
31
+ private mergeDebug;
29
32
  private updateRegistry;
30
33
  }
@@ -8,12 +8,14 @@ import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
8
8
  import { ProjectRegistry } from './project-registry.js';
9
9
  export class MultiProjectManager {
10
10
  storageRoot;
11
+ llmProvider;
11
12
  registry;
12
13
  configManager = new ProjectConfigManager();
13
14
  projects = new Map();
14
15
  globalKB;
15
- constructor(storageRoot = path.join(os.homedir(), USER_DATA_DIR)) {
16
+ constructor(storageRoot = path.join(os.homedir(), USER_DATA_DIR), llmProvider) {
16
17
  this.storageRoot = storageRoot;
18
+ this.llmProvider = llmProvider;
17
19
  this.registry = new ProjectRegistry(path.join(storageRoot, 'projects', 'registry.db'));
18
20
  }
19
21
  async getProject(projectRoot) {
@@ -29,6 +31,7 @@ export class MultiProjectManager {
29
31
  projectId,
30
32
  kbPath: getProjectKbPath(resolvedRoot),
31
33
  storageRoot: this.storageRoot,
34
+ llmProvider: this.llmProvider,
32
35
  });
33
36
  manager.initialize();
34
37
  this.projects.set(projectId, manager);
@@ -38,7 +41,7 @@ export class MultiProjectManager {
38
41
  async getGlobalKB() {
39
42
  if (this.globalKB)
40
43
  return this.globalKB;
41
- const manager = new KnowledgeBaseManager({ scope: 'global', storageRoot: this.storageRoot });
44
+ const manager = new KnowledgeBaseManager({ scope: 'global', storageRoot: this.storageRoot, llmProvider: this.llmProvider });
42
45
  manager.initialize();
43
46
  await manager.incrementalIndex();
44
47
  this.globalKB = manager;
@@ -52,29 +55,22 @@ export class MultiProjectManager {
52
55
  const scope = options.scope ?? 'all';
53
56
  const project = await this.getProject(projectRoot);
54
57
  await project.incrementalIndex();
55
- const items = [];
56
- if (scope === 'project' || scope === 'all') {
57
- items.push(...project.search(query, limit).map(result => ({
58
- id: result.id,
59
- content: result.content,
60
- filePath: result.relativePath,
61
- scope: 'project',
62
- collection: result.collectionName,
63
- score: result.score,
64
- })));
65
- }
66
- if (scope === 'global' || scope === 'all') {
58
+ if (scope === 'project')
59
+ return project.hybridSearch(query, { limit, weights: options.weights });
60
+ const projectResults = scope === 'all'
61
+ ? await project.hybridSearch(query, { limit, weights: options.weights })
62
+ : { results: [], scopesSearched: [], queryTimeMs: 0 };
63
+ if (scope === 'global') {
67
64
  const global = await this.getGlobalKB();
68
- items.push(...global.search(query, limit).map(result => ({
69
- id: result.id,
70
- content: result.content,
71
- filePath: result.relativePath,
72
- scope: 'global',
73
- collection: result.collectionName,
74
- score: result.score,
75
- })));
65
+ return global.hybridSearch(query, { limit, weights: options.weights });
76
66
  }
77
- return new FederationSearch().merge(items, limit, scope);
67
+ const global = await this.getGlobalKB();
68
+ const globalResults = await global.hybridSearch(query, { limit, weights: options.weights });
69
+ const merged = new FederationSearch().merge([...projectResults.results, ...globalResults.results], limit, 'all');
70
+ return {
71
+ ...merged,
72
+ debug: this.mergeDebug(projectResults.debug, globalResults.debug),
73
+ };
78
74
  }
79
75
  async semanticSearch(projectRoot, query, options = {}) {
80
76
  const scope = options.scope ?? 'all';
@@ -104,6 +100,7 @@ export class MultiProjectManager {
104
100
  projectId: project.projectId,
105
101
  kbPath: project.kbPath,
106
102
  storageRoot: this.storageRoot,
103
+ llmProvider: this.llmProvider,
107
104
  });
108
105
  for (const item of manager.store.listContentHashes()) {
109
106
  const duplicate = byHash.get(item.contentHash) ?? { contentHash: item.contentHash, files: [] };
@@ -143,6 +140,21 @@ export class MultiProjectManager {
143
140
  this.globalKB = undefined;
144
141
  this.registry.close();
145
142
  }
143
+ mergeDebug(projectDebug, globalDebug) {
144
+ if (!projectDebug && !globalDebug)
145
+ return undefined;
146
+ return {
147
+ originalQuery: projectDebug?.originalQuery ?? globalDebug?.originalQuery,
148
+ rewrittenQueries: [...new Set([...(projectDebug?.rewrittenQueries ?? []), ...(globalDebug?.rewrittenQueries ?? [])])],
149
+ weights: projectDebug?.weights ?? globalDebug?.weights,
150
+ recallCounts: {
151
+ keyword: (projectDebug?.recallCounts?.keyword ?? 0) + (globalDebug?.recallCounts?.keyword ?? 0),
152
+ vector: (projectDebug?.recallCounts?.vector ?? 0) + (globalDebug?.recallCounts?.vector ?? 0),
153
+ merged: (projectDebug?.recallCounts?.merged ?? 0) + (globalDebug?.recallCounts?.merged ?? 0),
154
+ },
155
+ reranker: projectDebug?.reranker ?? globalDebug?.reranker,
156
+ };
157
+ }
146
158
  updateRegistry(manager, projectRoot, projectName, lastOpenedAt) {
147
159
  const stats = manager.getStats();
148
160
  if (!manager.projectId)
@@ -11,5 +11,6 @@ export declare class HashEmbeddingProvider implements EmbeddingProvider {
11
11
  embedDocuments(texts: string[]): Promise<number[][]>;
12
12
  embedQuery(text: string): Promise<number[]>;
13
13
  private embed;
14
+ private tokenize;
14
15
  private normalize;
15
16
  }
@@ -13,7 +13,7 @@ export class HashEmbeddingProvider {
13
13
  }
14
14
  embed(text) {
15
15
  const vector = Array.from({ length: this.dimensions }, () => 0);
16
- const tokens = text.toLowerCase().normalize('NFKC').split(/\s+/u).filter(Boolean);
16
+ const tokens = this.tokenize(text);
17
17
  for (const token of tokens) {
18
18
  const hash = crypto.createHash('sha256').update(token).digest();
19
19
  const index = hash.readUInt32BE(0) % this.dimensions;
@@ -22,6 +22,21 @@ export class HashEmbeddingProvider {
22
22
  }
23
23
  return this.normalize(vector);
24
24
  }
25
+ tokenize(text) {
26
+ const normalized = text.toLowerCase().normalize('NFKC');
27
+ const tokens = normalized.match(/[\p{Script=Han}]{1,4}|[\p{Letter}\p{Number}_-]+/gu) ?? [];
28
+ const grams = [];
29
+ for (const token of tokens) {
30
+ grams.push(token);
31
+ if (/^[\p{Script=Han}]+$/u.test(token) && token.length > 1) {
32
+ for (let i = 0; i < token.length - 1; i++)
33
+ grams.push(token.slice(i, i + 2));
34
+ for (let i = 0; i < token.length - 2; i++)
35
+ grams.push(token.slice(i, i + 3));
36
+ }
37
+ }
38
+ return grams.filter(Boolean);
39
+ }
25
40
  normalize(vector) {
26
41
  const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
27
42
  if (norm === 0)
@@ -25,6 +25,8 @@ export declare class ContentExtractor {
25
25
  private extractRasterImage;
26
26
  private validateRasterImage;
27
27
  private extractPdf;
28
+ private pdfOcrPageLimit;
29
+ private extractScannedPdfOcr;
28
30
  private extractPdfText;
29
31
  private extractSvg;
30
32
  private isTextReadable;
@@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process';
2
2
  import * as fs from 'node:fs';
3
3
  import * as path from 'node:path';
4
4
  import { ExternalExtractorRegistry } from './external-extractor.js';
5
+ import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
5
6
  export class ContentExtractor {
6
7
  externalExtractors;
7
8
  constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
@@ -142,17 +143,23 @@ export class ContentExtractor {
142
143
  const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 500);
143
144
  const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
144
145
  const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
145
- metadata.layerCount = new Set(layers).size;
146
+ const uniqueLayers = Array.from(new Set(layers));
147
+ const uniqueBlocks = Array.from(new Set(blocks));
148
+ const uniqueEntityTypes = Array.from(new Set(entityTypes));
149
+ metadata.layerCount = uniqueLayers.length;
150
+ metadata.layerNames = uniqueLayers.slice(0, 80);
146
151
  metadata.textEntityCount = textEntities.length;
147
- metadata.blockCount = new Set(blocks).size;
148
- metadata.entityTypeCount = new Set(entityTypes).size;
152
+ metadata.blockCount = uniqueBlocks.length;
153
+ metadata.blockNames = uniqueBlocks.slice(0, 80);
154
+ metadata.entityTypeCount = uniqueEntityTypes.length;
155
+ metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
149
156
  metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
150
157
  return {
151
158
  text: [
152
159
  this.metadataOnlyText(file),
153
- `CAD DXF 图层: ${Array.from(new Set(layers)).join(', ')}`,
154
- `CAD DXF 块/符号: ${Array.from(new Set(blocks)).join(', ')}`,
155
- `CAD DXF 实体类型: ${Array.from(new Set(entityTypes)).join(', ')}`,
160
+ `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
161
+ `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
162
+ `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
156
163
  `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
157
164
  ].join('\n'),
158
165
  metadata,
@@ -165,16 +172,20 @@ export class ContentExtractor {
165
172
  const materials = this.matchAll(raw, /MATERIAL[^']*'([^']+)'/giu).slice(0, 120);
166
173
  const entities = this.matchAll(raw, /#\d+\s*=\s*([A-Z0-9_]+)/gu).slice(0, 1000);
167
174
  const names = this.matchAll(raw, /'([^']{2,120})'/gu).slice(0, 500);
175
+ const uniqueStepEntities = Array.from(new Set(entities));
168
176
  metadata.productCount = products.length;
177
+ metadata.productNames = products.slice(0, 80);
169
178
  metadata.materialCount = materials.length;
170
- metadata.entityTypeCount = new Set(entities).size;
179
+ metadata.materialNames = materials.slice(0, 80);
180
+ metadata.entityTypeCount = uniqueStepEntities.length;
181
+ metadata.entityTypes = uniqueStepEntities.slice(0, 80);
171
182
  metadata.contentCoverage = 'step_products_materials_entities_names';
172
183
  return {
173
184
  text: [
174
185
  this.metadataOnlyText(file),
175
186
  `STEP 产品/零件:\n${products.join('\n')}`,
176
187
  `STEP 材料: ${materials.join(', ')}`,
177
- `STEP 实体类型: ${Array.from(new Set(entities)).join(', ')}`,
188
+ `STEP 实体类型: ${uniqueStepEntities.join(', ')}`,
178
189
  `STEP 名称/属性:\n${names.join('\n')}`,
179
190
  ].join('\n'),
180
191
  metadata,
@@ -185,11 +196,14 @@ export class ContentExtractor {
185
196
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
186
197
  const names = this.matchAll(raw, /'([^']{2,120})'/gu).slice(0, 500);
187
198
  const entityTypes = this.matchAll(raw, /^\s*(\d{3,4})\s*,/gmu).slice(0, 1000);
199
+ const uniqueIgesTypes = Array.from(new Set(entityTypes));
188
200
  metadata.entityNameCount = names.length;
189
- metadata.entityTypeCount = new Set(entityTypes).size;
201
+ metadata.entityNames = names.slice(0, 80);
202
+ metadata.entityTypeCount = uniqueIgesTypes.length;
203
+ metadata.entityTypes = uniqueIgesTypes.slice(0, 80);
190
204
  metadata.contentCoverage = 'iges_entity_names_types';
191
205
  return {
192
- text: [this.metadataOnlyText(file), `IGES 实体类型: ${Array.from(new Set(entityTypes)).join(', ')}`, `IGES 实体/名称:\n${names.join('\n')}`].join('\n'),
206
+ text: [this.metadataOnlyText(file), `IGES 实体类型: ${uniqueIgesTypes.join(', ')}`, `IGES 实体/名称:\n${names.join('\n')}`].join('\n'),
193
207
  metadata,
194
208
  warnings,
195
209
  };
@@ -256,20 +270,31 @@ export class ContentExtractor {
256
270
  const metadata = { extractionMode: 'structured_data', vectorizable: true };
257
271
  try {
258
272
  if (file.format === 'json') {
259
- const lines = path.extname(file.absolutePath).toLowerCase() === '.jsonl'
260
- ? raw.split(/\r?\n/u).filter(Boolean).slice(0, 200).flatMap((line, index) => this.flattenJson(JSON.parse(line), `line${index + 1}`))
273
+ const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
274
+ const lines = isJsonl
275
+ ? raw.split(/\r?\n/u).filter(Boolean).slice(0, 500).flatMap((line, index) => this.flattenJson(JSON.parse(line), `line${index + 1}`))
261
276
  : this.flattenJson(JSON.parse(raw));
262
277
  metadata.fieldCount = lines.length;
263
- metadata.contentCoverage = 'json_paths_values';
264
- return { text: [this.metadataOnlyText(file), ...lines.slice(0, 1000)].join('\n'), metadata, warnings: [] };
278
+ metadata.recordCount = isJsonl ? raw.split(/\r?\n/u).filter(Boolean).length : 1;
279
+ metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
280
+ metadata.contentCoverage = isJsonl ? 'jsonl_records_paths_values' : 'json_paths_values';
281
+ return { text: [this.metadataOnlyText(file), ...lines.slice(0, 1500)].join('\n'), metadata, warnings: [] };
265
282
  }
266
283
  }
267
284
  catch {
268
285
  metadata.parseError = true;
269
286
  }
287
+ if (file.format === 'yaml') {
288
+ const pairs = this.matchAll(raw, /^\s*([\w.-]+)\s*:\s*(.{1,300})$/gmu).slice(0, 1500);
289
+ metadata.fieldCount = pairs.length;
290
+ metadata.dataPaths = pairs.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
291
+ metadata.contentCoverage = 'yaml_key_values';
292
+ return { text: [this.metadataOnlyText(file), ...pairs].join('\n'), metadata, warnings: [] };
293
+ }
270
294
  if (file.format === 'xml') {
271
295
  const elements = this.matchAll(raw, /<([A-Za-z_][\w:.-]*)\b[^>]*>([^<]{1,200})<\/\1>/gu).slice(0, 1000);
272
296
  metadata.elementTextCount = elements.length;
297
+ metadata.dataPaths = elements.map(line => line.match(/^<([A-Za-z_][\w:.-]*)/u)?.[1]).filter(Boolean).slice(0, 200);
273
298
  metadata.contentCoverage = 'xml_element_text';
274
299
  return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
275
300
  }
@@ -346,6 +371,7 @@ export class ContentExtractor {
346
371
  delimiter: file.format === 'tsv' ? 'tab' : 'comma',
347
372
  rowCount: rows.length,
348
373
  columnCount: header.length,
374
+ columnNames: header.slice(0, 120),
349
375
  contentCoverage: 'table_headers_cells_text',
350
376
  },
351
377
  warnings: [],
@@ -354,14 +380,14 @@ export class ContentExtractor {
354
380
  async extractOfficeDocument(file) {
355
381
  if (path.extname(file.absolutePath).toLowerCase() === '.docx') {
356
382
  try {
357
- const mammoth = await import('mammoth');
383
+ const mammoth = await resolveAndImport('mammoth');
358
384
  const result = await mammoth.extractRawText({ path: file.absolutePath });
359
385
  const text = result.value.trim();
360
386
  if (text) {
361
387
  return {
362
388
  text,
363
389
  metadata: { extractionMode: 'builtin_mammoth', vectorizable: true, contentCoverage: 'office_full_text' },
364
- warnings: result.messages.map(message => message.message),
390
+ warnings: result.messages.map(m => m.message),
365
391
  };
366
392
  }
367
393
  }
@@ -373,7 +399,7 @@ export class ContentExtractor {
373
399
  }
374
400
  async extractSpreadsheet(file) {
375
401
  try {
376
- const XLSX = await import('xlsx');
402
+ const XLSX = await resolveAndImport('xlsx');
377
403
  const workbook = XLSX.readFile(file.absolutePath, { cellDates: true, cellFormula: true, cellNF: true, cellStyles: true });
378
404
  const sheetTexts = [];
379
405
  let cellCount = 0;
@@ -410,7 +436,7 @@ export class ContentExtractor {
410
436
  if (sheetTexts.length > 0) {
411
437
  return {
412
438
  text: sheetTexts.join('\n\n'),
413
- metadata: { extractionMode: 'builtin_xlsx_structured_cells', vectorizable: true, sheetCount: sheetTexts.length, cellCount, formulaCount, mergeCount, contentCoverage: 'spreadsheet_cells_formulas_merges' },
439
+ metadata: { extractionMode: 'builtin_xlsx_structured_cells', vectorizable: true, sheetCount: sheetTexts.length, sheetNames: workbook.SheetNames.slice(0, 120), cellCount, formulaCount, mergeCount, contentCoverage: 'spreadsheet_cells_formulas_merges' },
414
440
  warnings: [],
415
441
  };
416
442
  }
@@ -423,7 +449,8 @@ export class ContentExtractor {
423
449
  async extractOfficeZip(file) {
424
450
  const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
425
451
  try {
426
- const JSZip = (await import('jszip')).default;
452
+ const jszipMod = await resolveAndImport('jszip');
453
+ const JSZip = jszipMod.default ?? jszipMod;
427
454
  const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
428
455
  const texts = [];
429
456
  const xmlEntries = Object.values(zip.files).filter(entry => !entry.dir && /\.(xml|rels)$/iu.test(entry.name)).slice(0, 80);
@@ -450,7 +477,8 @@ export class ContentExtractor {
450
477
  return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
451
478
  }
452
479
  try {
453
- const JSZip = (await import('jszip')).default;
480
+ const jszipMod = await resolveAndImport('jszip');
481
+ const JSZip = jszipMod.default ?? jszipMod;
454
482
  const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
455
483
  const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
456
484
  metadata.entryCount = Object.keys(zip.files).length;
@@ -470,10 +498,27 @@ export class ContentExtractor {
470
498
  metadata.parseError = validationError;
471
499
  return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
472
500
  }
501
+ let tesseractPath;
502
+ try {
503
+ tesseractPath = resolvePackage('tesseract.js');
504
+ }
505
+ catch (e) {
506
+ metadata.contentCoverage = 'ocr_unavailable';
507
+ metadata.parseError = e.message;
508
+ return { text: '', metadata, warnings: [`内置 OCR 不可用:${e.message}`] };
509
+ }
510
+ // 子进程用 createRequire 加载 tesseract.js(兼容打包 Server 的 vendor 目录)
473
511
  const result = spawnSync(process.execPath, [
474
512
  '--input-type=module',
475
513
  '-e',
476
- `const { createWorker } = await import('tesseract.js');
514
+ `import { createRequire } from 'node:module';
515
+ import { fileURLToPath } from 'node:url';
516
+ import path from 'node:path';
517
+
518
+ const localRequire = createRequire(import.meta.url);
519
+ const tesseractMod = localRequire(${JSON.stringify(tesseractPath)});
520
+ const { createWorker } = tesseractMod;
521
+
477
522
  const worker = await createWorker('chi_sim+eng');
478
523
  try {
479
524
  const result = await worker.recognize(process.argv[1]);
@@ -482,7 +527,7 @@ try {
482
527
  await worker.terminate();
483
528
  }`,
484
529
  file.absolutePath,
485
- ], { encoding: 'utf8', timeout: 60_000, maxBuffer: 10 * 1024 * 1024 });
530
+ ], { encoding: 'utf8', timeout: 120_000, maxBuffer: 20 * 1024 * 1024 });
486
531
  if (result.status !== 0 || result.error) {
487
532
  const message = result.error?.message || result.stderr.trim() || `OCR 子进程退出码 ${result.status ?? 'unknown'}`;
488
533
  metadata.contentCoverage = 'ocr_failed';
@@ -491,8 +536,11 @@ try {
491
536
  }
492
537
  const text = result.stdout.trim();
493
538
  metadata.contentCoverage = text ? 'ocr_text' : 'metadata_filename';
539
+ metadata.ocrProvider = 'tesseract.js';
540
+ metadata.ocrLanguages = 'chi_sim+eng';
541
+ metadata.ocrTextLength = text.length;
494
542
  return {
495
- text,
543
+ text: text ? [this.metadataOnlyText(file), `OCR 识别文本:\n${text}`].join('\n') : '',
496
544
  metadata,
497
545
  warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
498
546
  };
@@ -523,46 +571,164 @@ try {
523
571
  }
524
572
  async extractPdf(file) {
525
573
  const metadata = { extractionMode: 'pdf_text', vectorizable: true };
574
+ const warnings = [];
575
+ // Tier 1: pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
526
576
  try {
527
577
  const raw = fs.readFileSync(file.absolutePath);
528
578
  const text = await this.extractPdfText(raw);
529
579
  if (text.trim()) {
530
580
  metadata.contentCoverage = 'pdf_text_streams';
531
- return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings: [] };
581
+ metadata.pdfExtractor = 'pdfjs-dist';
582
+ return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
532
583
  }
533
584
  }
534
585
  catch (error) {
586
+ warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
535
587
  metadata.parseError = error instanceof Error ? error.message : String(error);
536
588
  }
589
+ // Tier 2: OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
590
+ const ocr = await this.extractScannedPdfOcr(file);
591
+ if (ocr.text.trim())
592
+ return ocr;
593
+ // Tier 3: 仅索引元数据(兜底)
537
594
  metadata.extractionMode = 'pdf_metadata_only';
538
595
  metadata.contentCoverage = 'metadata_filename';
596
+ metadata.ocrRecommended = true;
597
+ metadata.ocrReason = ocr.metadata.ocrReason ?? 'pdf_text_stream_empty_or_unavailable';
598
+ metadata.pdfPageOcrSupported = true;
539
599
  return {
540
- text: '',
600
+ text: this.metadataOnlyText(file),
541
601
  metadata,
542
- warnings: ['PDF 正文暂未提取到文本,内置解析器未提取到正文,未入库'],
602
+ warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
543
603
  };
544
604
  }
605
+ pdfOcrPageLimit() {
606
+ const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
607
+ return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
608
+ }
609
+ async extractScannedPdfOcr(file) {
610
+ const metadata = {
611
+ extractionMode: 'pdf_page_ocr_embedded',
612
+ vectorizable: true,
613
+ pdfPageOcrSupported: true,
614
+ ocrProvider: 'tesseract.js',
615
+ ocrLanguages: 'chi_sim+eng',
616
+ pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
617
+ pdfOcrPageLimit: this.pdfOcrPageLimit(),
618
+ };
619
+ const pageLimitConfig = this.pdfOcrPageLimit();
620
+ // 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
621
+ let canvasPath;
622
+ let pdfjsPath;
623
+ let tesseractPath;
624
+ try {
625
+ canvasPath = resolvePackage('@napi-rs/canvas');
626
+ pdfjsPath = resolvePackage('pdfjs-dist/legacy/build/pdf.mjs');
627
+ tesseractPath = resolvePackage('tesseract.js');
628
+ }
629
+ catch (e) {
630
+ metadata.ocrRecommended = true;
631
+ metadata.ocrReason = `OCR 依赖解析失败: ${e.message}`;
632
+ return { text: '', metadata, warnings: [`内置扫描 PDF OCR 不可用:${metadata.ocrReason}`] };
633
+ }
634
+ // NODE_PATH 确保子进程能解析 tesseract.js 的依赖
635
+ const childEnv = { ...process.env };
636
+ const nmRoot = getNodeModulesRoot();
637
+ if (nmRoot)
638
+ childEnv.NODE_PATH = nmRoot;
639
+ const result = spawnSync(process.execPath, [
640
+ '--input-type=module',
641
+ '-e',
642
+ `import fs from 'node:fs';
643
+ import os from 'node:os';
644
+ import path from 'node:path';
645
+ import { createCanvas } from ${JSON.stringify(canvasPath)};
646
+ import * as pdfjs from ${JSON.stringify(pdfjsPath)};
647
+ import { createWorker } from ${JSON.stringify(tesseractPath)};
648
+ const filePath = process.argv[1];
649
+ const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
650
+ const bytes = new Uint8Array(fs.readFileSync(filePath));
651
+ const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
652
+ const pageLimit = Math.min(doc.numPages, configuredLimit);
653
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
654
+ const worker = await createWorker('chi_sim+eng');
655
+ const pages = [];
656
+ try {
657
+ for (let i = 1; i <= pageLimit; i += 1) {
658
+ const page = await doc.getPage(i);
659
+ const viewport = page.getViewport({ scale: 2 });
660
+ const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
661
+ const context = canvas.getContext('2d');
662
+ await page.render({ canvasContext: context, viewport }).promise;
663
+ const imagePath = path.join(tmpDir, 'page-' + i + '.png');
664
+ fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
665
+ const recognized = await worker.recognize(imagePath);
666
+ const text = (recognized.data.text || '').trim();
667
+ if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + text);
668
+ }
669
+ } finally {
670
+ await worker.terminate();
671
+ fs.rmSync(tmpDir, { recursive: true, force: true });
672
+ }
673
+ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
674
+ file.absolutePath,
675
+ String(pageLimitConfig),
676
+ ], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
677
+ if (result.status !== 0 || result.error) {
678
+ metadata.ocrRecommended = true;
679
+ metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
680
+ return { text: '', metadata, warnings: [`内置扫描 PDF OCR 失败:${metadata.ocrReason}`] };
681
+ }
682
+ try {
683
+ const parsed = JSON.parse(result.stdout);
684
+ const text = parsed.text?.trim() ?? '';
685
+ metadata.ocrPageCount = parsed.pageCount ?? 0;
686
+ metadata.pdfOcrPageLimit = parsed.pageLimit ?? 5;
687
+ metadata.ocrTextLength = text.length;
688
+ metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
689
+ return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
690
+ }
691
+ catch {
692
+ metadata.ocrRecommended = true;
693
+ metadata.ocrReason = 'pdf_page_ocr_output_parse_failed';
694
+ return { text: '', metadata, warnings: ['内置扫描 PDF OCR 输出解析失败'] };
695
+ }
696
+ }
545
697
  async extractPdfText(buffer) {
698
+ // Tier 1: pdfjs-dist 文本提取(处理压缩内容流、CJK 字体、现代 PDF)
546
699
  try {
547
- const mod = await import('pdf-parse');
548
- const PDFParse = mod.PDFParse;
549
- if (PDFParse) {
550
- const parser = new PDFParse({ data: new Uint8Array(buffer) });
551
- try {
552
- const result = await parser.getText();
553
- if (result.text.trim())
554
- return result.text.slice(0, 250_000);
555
- }
556
- finally {
557
- await parser.destroy();
558
- }
700
+ const mod = await resolveAndImport('pdfjs-dist/legacy/build/pdf.mjs');
701
+ const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
702
+ const doc = await loadingTask.promise;
703
+ const pages = [];
704
+ const pageLimit = Math.min(doc.numPages, 200);
705
+ for (let i = 1; i <= pageLimit; i++) {
706
+ const page = await doc.getPage(i);
707
+ const content = await page.getTextContent();
708
+ const pageText = content.items
709
+ .map((item) => {
710
+ const hasStr = item != null && typeof item === 'object' && 'str' in item;
711
+ return hasStr ? String(item.str) : '';
712
+ })
713
+ .filter((s) => s.trim().length > 0)
714
+ .join(' ');
715
+ if (pageText.trim())
716
+ pages.push(pageText.trim());
717
+ }
718
+ await doc.destroy();
719
+ if (pages.length > 0) {
720
+ const combined = pages.join('\n\n');
721
+ if (combined.trim())
722
+ return combined.slice(0, 250_000);
559
723
  }
560
724
  }
561
- catch {
562
- // fallback to legacy API/raw stream extraction below
725
+ catch (e) {
726
+ if (process.env.KB_DEBUG === '1')
727
+ console.warn('[kb] pdfjs-dist extraction failed:', e.message);
563
728
  }
729
+ // Tier 2: pdf-parse(兼容旧版 PDF)
564
730
  try {
565
- const mod = await import('pdf-parse');
731
+ const mod = await resolveAndImport('pdf-parse');
566
732
  const pdfParse = mod.default;
567
733
  if (pdfParse) {
568
734
  const result = await pdfParse(buffer);
@@ -571,8 +737,9 @@ try {
571
737
  }
572
738
  }
573
739
  catch {
574
- // fallback below
740
+ // fallback to raw regex below
575
741
  }
742
+ // Tier 3: raw regex 回退(未压缩的古老 PDF)
576
743
  const raw = buffer.toString('latin1');
577
744
  const matches = Array.from(raw.matchAll(/\(([^()]{2,500})\)\s*T[jJ]/gu), match => match[1] ?? '')
578
745
  .concat(Array.from(raw.matchAll(/\[([^\]]{2,2000})\]\s*TJ/gu), match => match[1] ?? ''));
@@ -0,0 +1,17 @@
1
+ /**
2
+ * 使用 CJS require 解析 npm 包为绝对路径。
3
+ * 多层回退:CJS require.resolve → .pnpm 遍历。
4
+ */
5
+ export declare function resolvePackage(specifier: string): string;
6
+ /**
7
+ * 解析并加载一个 npm 包。
8
+ * 使用 CJS require() 而非 ESM import(),确保在 Next.js standalone /
9
+ * 打包 Server 上下文中也能正确加载(CJS 走 Module._resolveFilename
10
+ * monkey-patch,不受 ESM 自定义 loader 影响)。
11
+ */
12
+ export declare function resolveAndImport<T = unknown>(specifier: string): Promise<T>;
13
+ /**
14
+ * 获取 node_modules 根目录路径(包含 .pnpm 的那个)。
15
+ * 用于设置子进程 NODE_PATH,确保 OCR 等子进程能解析依赖。
16
+ */
17
+ export declare function getNodeModulesRoot(): string | null;