@customize-agent/knowledge 4.0.12 → 4.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,4 @@
1
1
  import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
2
- import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
3
2
  import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
4
3
  import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters } from '../search/federation-search.js';
5
4
  import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
@@ -23,7 +22,6 @@ export interface KnowledgeBaseManagerOptions {
23
22
  storageRoot?: string;
24
23
  embeddingProvider?: EmbeddingProvider;
25
24
  vectorStores?: Map<string, VectorStoreInterface>;
26
- externalExtractors?: ExternalExtractorRegistry;
27
25
  onProgress?: (progress: KnowledgeIndexProgress) => void;
28
26
  /** 可选的 LLM Provider,用于查询扩展和语义重排序 */
29
27
  llmProvider?: LLMSearchProvider;
@@ -43,7 +43,7 @@ export class KnowledgeBaseManager {
43
43
  this.projectId = options.projectId;
44
44
  this.embeddingProvider = options.embeddingProvider ?? createEmbeddingProviderFromEnvironment();
45
45
  this.vectorStores = options.vectorStores ?? new Map();
46
- this.extractor = new ContentExtractor(options.externalExtractors);
46
+ this.extractor = new ContentExtractor();
47
47
  this.llmProvider = options.llmProvider;
48
48
  this.onProgress = options.onProgress;
49
49
  const storageRoot = options.storageRoot ?? path.join(os.homedir(), USER_DATA_DIR);
@@ -1,4 +1,3 @@
1
- import { ExternalExtractorRegistry } from './external-extractor.js';
2
1
  import type { ClassifiedFile } from '../types.js';
3
2
  /** 文件内容提取结果 */
4
3
  export interface ExtractionResult {
@@ -9,10 +8,7 @@ export interface ExtractionResult {
9
8
  }
10
9
  /** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
11
10
  export declare class ContentExtractor {
12
- private readonly externalExtractors;
13
- constructor(externalExtractors?: ExternalExtractorRegistry);
14
11
  extract(file: ClassifiedFile): Promise<ExtractionResult>;
15
- private tryExternalExtractor;
16
12
  private extractTextClipping;
17
13
  private swapUtf16Bytes;
18
14
  private extractReadableFragments;
@@ -58,8 +54,7 @@ export declare class ContentExtractor {
58
54
  private formatBoundingBox;
59
55
  private validateRasterImage;
60
56
  private extractPdf;
61
- private shouldAugmentPdfWithOcr;
62
- private extractScannedPdfOcr;
57
+ private extractPdfHybridPages;
63
58
  private extractPdfText;
64
59
  private normalizedTextLength;
65
60
  private toPdfTextItem;
@@ -3,16 +3,11 @@ import * as fs from 'node:fs';
3
3
  import * as path from 'node:path';
4
4
  import { tmpdir } from 'node:os';
5
5
  import { pathToFileURL } from 'node:url';
6
- import { ExternalExtractorRegistry } from './external-extractor.js';
7
6
  import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
8
7
  const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
9
8
  const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
10
9
  /** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
11
10
  export class ContentExtractor {
12
- externalExtractors;
13
- constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
14
- this.externalExtractors = externalExtractors;
15
- }
16
11
  async extract(file) {
17
12
  const start = Date.now();
18
13
  const warnings = [];
@@ -22,86 +17,77 @@ export class ContentExtractor {
22
17
  category: file.category,
23
18
  format: file.format,
24
19
  };
25
- const external = this.tryExternalExtractor(file);
26
- if (external?.text.trim()) {
27
- text = external.text;
28
- Object.assign(metadata, external.metadata);
29
- warnings.push(...external.warnings);
20
+ if (file.category === 'cad') {
21
+ const result = await this.extractCad(file);
22
+ text = result.text;
23
+ Object.assign(metadata, result.metadata);
24
+ warnings.push(...result.warnings);
25
+ }
26
+ else if (file.category === 'data') {
27
+ const result = this.extractData(file);
28
+ text = result.text;
29
+ Object.assign(metadata, result.metadata);
30
+ warnings.push(...result.warnings);
31
+ }
32
+ else if (file.category === 'diagram') {
33
+ const result = this.extractDiagram(file);
34
+ text = result.text;
35
+ Object.assign(metadata, result.metadata);
36
+ warnings.push(...result.warnings);
37
+ }
38
+ else if (file.category === 'document' && file.format === 'pdf') {
39
+ const result = await this.extractPdf(file);
40
+ text = result.text;
41
+ Object.assign(metadata, result.metadata);
42
+ warnings.push(...result.warnings);
43
+ }
44
+ else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
45
+ const result = await this.extractOfficeDocument(file);
46
+ text = result.text;
47
+ Object.assign(metadata, result.metadata);
48
+ warnings.push(...result.warnings);
49
+ }
50
+ else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
51
+ const result = this.extractDelimitedText(file);
52
+ text = result.text;
53
+ Object.assign(metadata, result.metadata);
54
+ warnings.push(...result.warnings);
55
+ }
56
+ else if (file.category === 'spreadsheet') {
57
+ const result = await this.extractSpreadsheet(file);
58
+ text = result.text;
59
+ Object.assign(metadata, result.metadata);
60
+ warnings.push(...result.warnings);
61
+ }
62
+ else if (file.category === 'image' && file.format !== 'vector') {
63
+ const result = await this.extractRasterImage(file);
64
+ text = result.text;
65
+ Object.assign(metadata, result.metadata);
66
+ warnings.push(...result.warnings);
67
+ }
68
+ else if (file.category === 'image' && file.format === 'vector') {
69
+ const result = this.extractSvg(file);
70
+ text = result.text;
71
+ Object.assign(metadata, result.metadata);
72
+ warnings.push(...result.warnings);
73
+ }
74
+ else if (file.format === 'text_clipping') {
75
+ const result = this.extractTextClipping(file);
76
+ text = result.text;
77
+ Object.assign(metadata, result.metadata);
78
+ warnings.push(...result.warnings);
79
+ }
80
+ else if (this.isTextReadable(file)) {
81
+ text = fs.readFileSync(file.absolutePath, 'utf8');
82
+ metadata.extractionMode = 'plain_text';
83
+ metadata.vectorizable = true;
30
84
  }
31
85
  else {
32
- warnings.push(...(external?.warnings ?? []));
33
- if (file.category === 'cad') {
34
- const result = await this.extractCad(file);
35
- text = result.text;
36
- Object.assign(metadata, result.metadata);
37
- warnings.push(...result.warnings);
38
- }
39
- else if (file.category === 'data') {
40
- const result = this.extractData(file);
41
- text = result.text;
42
- Object.assign(metadata, result.metadata);
43
- warnings.push(...result.warnings);
44
- }
45
- else if (file.category === 'diagram') {
46
- const result = this.extractDiagram(file);
47
- text = result.text;
48
- Object.assign(metadata, result.metadata);
49
- warnings.push(...result.warnings);
50
- }
51
- else if (file.category === 'document' && file.format === 'pdf') {
52
- const result = await this.extractPdf(file);
53
- text = result.text;
54
- Object.assign(metadata, result.metadata);
55
- warnings.push(...result.warnings);
56
- }
57
- else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
58
- const result = await this.extractOfficeDocument(file);
59
- text = result.text;
60
- Object.assign(metadata, result.metadata);
61
- warnings.push(...result.warnings);
62
- }
63
- else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
64
- const result = this.extractDelimitedText(file);
65
- text = result.text;
66
- Object.assign(metadata, result.metadata);
67
- warnings.push(...result.warnings);
68
- }
69
- else if (file.category === 'spreadsheet') {
70
- const result = await this.extractSpreadsheet(file);
71
- text = result.text;
72
- Object.assign(metadata, result.metadata);
73
- warnings.push(...result.warnings);
74
- }
75
- else if (file.category === 'image' && file.format !== 'vector') {
76
- const result = await this.extractRasterImage(file);
77
- text = result.text;
78
- Object.assign(metadata, result.metadata);
79
- warnings.push(...result.warnings);
80
- }
81
- else if (file.category === 'image' && file.format === 'vector') {
82
- const result = this.extractSvg(file);
83
- text = result.text;
84
- Object.assign(metadata, result.metadata);
85
- warnings.push(...result.warnings);
86
- }
87
- else if (file.format === 'text_clipping') {
88
- const result = this.extractTextClipping(file);
89
- text = result.text;
90
- Object.assign(metadata, result.metadata);
91
- warnings.push(...result.warnings);
92
- }
93
- else if (this.isTextReadable(file)) {
94
- text = fs.readFileSync(file.absolutePath, 'utf8');
95
- metadata.extractionMode = 'plain_text';
96
- metadata.vectorizable = true;
97
- }
98
- else {
99
- text = this.metadataOnlyText(file);
100
- metadata.extractionMode = 'metadata_only';
101
- metadata.vectorizable = true;
102
- metadata.contentCoverage = 'metadata';
103
- warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取,未解析出正文,未入库`);
104
- }
86
+ text = this.metadataOnlyText(file);
87
+ metadata.extractionMode = 'metadata_only';
88
+ metadata.vectorizable = true;
89
+ metadata.contentCoverage = 'metadata';
90
+ warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取,未解析出正文,未入库`);
105
91
  }
106
92
  return {
107
93
  text: this.cleanExtractedText(text, file).trim(),
@@ -110,37 +96,6 @@ export class ContentExtractor {
110
96
  extractionTimeMs: Date.now() - start,
111
97
  };
112
98
  }
113
- tryExternalExtractor(file) {
114
- const extractors = this.externalExtractors.findAll(file);
115
- if (extractors.length === 0)
116
- return undefined;
117
- const warnings = [];
118
- for (const extractor of extractors) {
119
- try {
120
- const result = extractor.extract(file);
121
- const text = result.text.trim();
122
- if (!text) {
123
- warnings.push(`外部解析器 ${extractor.name} 未提取到正文`);
124
- continue;
125
- }
126
- return {
127
- text,
128
- metadata: {
129
- extractionMode: 'external_advanced_plugin',
130
- vectorizable: true,
131
- contentCoverage: 'external_full_text',
132
- externalExtractor: extractor.id,
133
- ...(result.metadata ?? {}),
134
- },
135
- warnings: [...warnings, ...(result.warnings ?? [])],
136
- };
137
- }
138
- catch (error) {
139
- warnings.push(`外部解析器 ${extractor.name} 失败: ${error instanceof Error ? error.message : String(error)}`);
140
- }
141
- }
142
- return warnings.length > 0 ? { text: '', metadata: {}, warnings } : undefined;
143
- }
144
99
  extractTextClipping(file) {
145
100
  const buffer = fs.readFileSync(file.absolutePath);
146
101
  const candidates = [
@@ -1054,83 +1009,58 @@ try {
1054
1009
  return undefined;
1055
1010
  }
1056
1011
  async extractPdf(file) {
1012
+ const hybrid = await this.extractPdfHybridPages(file);
1013
+ if (hybrid.text.trim())
1014
+ return hybrid;
1057
1015
  const metadata = { extractionMode: 'pdf_text', vectorizable: true };
1058
- const warnings = [];
1059
- // 第一层:pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
1016
+ const warnings = [...hybrid.warnings];
1060
1017
  try {
1061
1018
  const raw = fs.readFileSync(file.absolutePath);
1062
1019
  const text = await this.extractPdfText(raw);
1063
1020
  if (text.trim()) {
1064
1021
  metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
1065
1022
  metadata.pdfExtractor = 'pdfjs-dist';
1066
- const markdownText = this.toMarkdownDocument(text);
1067
- if (!this.shouldAugmentPdfWithOcr(markdownText))
1068
- return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata, warnings };
1069
- const ocr = await this.extractScannedPdfOcr(file);
1070
- if (ocr.text.trim()) {
1071
- metadata.contentCoverage = 'pdf_text_streams_plus_ocr';
1072
- metadata.ocrAugmented = true;
1073
- return { text: [this.metadataOnlyText(file), markdownText, '## PDF OCR 备用识别文本', ocr.text].join('\n\n'), metadata: { ...metadata, ocr: ocr.metadata }, warnings: [...warnings, ...ocr.warnings] };
1074
- }
1075
- return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata: { ...metadata, ocrRecommended: true, ocrReason: ocr.metadata.ocrReason ?? 'pdf_text_low_quality' }, warnings: [...warnings, ...ocr.warnings] };
1023
+ return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
1076
1024
  }
1077
1025
  }
1078
1026
  catch (error) {
1079
1027
  warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
1080
1028
  metadata.parseError = error instanceof Error ? error.message : String(error);
1081
1029
  }
1082
- // 第二层:OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
1083
- const ocr = await this.extractScannedPdfOcr(file);
1084
- if (ocr.text.trim())
1085
- return ocr;
1086
- // 第三层:仅索引元数据(兜底)
1087
1030
  metadata.extractionMode = 'pdf_metadata_only';
1088
1031
  metadata.contentCoverage = 'metadata_filename';
1089
1032
  metadata.ocrRecommended = true;
1090
- metadata.ocrReason = ocr.metadata.ocrReason ?? 'pdf_text_stream_empty_or_unavailable';
1033
+ metadata.ocrReason = hybrid.metadata.ocrReason ?? 'pdf_text_stream_empty_or_unavailable';
1091
1034
  metadata.pdfPageOcrSupported = true;
1092
1035
  return {
1093
1036
  text: this.metadataOnlyText(file),
1094
1037
  metadata,
1095
- warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
1038
+ warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据'],
1096
1039
  };
1097
1040
  }
1098
- shouldAugmentPdfWithOcr(text) {
1099
- const normalizedLength = this.normalizedTextLength(text);
1100
- if (normalizedLength < 1200)
1101
- return true;
1102
- const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
1103
- if (lines.length === 0)
1104
- return true;
1105
- const shortLineRatio = lines.filter(line => line.length <= 12).length / lines.length;
1106
- const cjkCount = (text.match(/[\p{Script=Han}]/gu) ?? []).length;
1107
- return shortLineRatio > 0.65 && cjkCount < 1200;
1108
- }
1109
- async extractScannedPdfOcr(file) {
1041
+ async extractPdfHybridPages(file) {
1110
1042
  const metadata = {
1111
- extractionMode: 'pdf_page_ocr_embedded',
1043
+ extractionMode: 'pdf_hybrid_pages',
1112
1044
  vectorizable: true,
1113
- pdfPageOcrSupported: true,
1114
- ocrProvider: 'tesseract.js',
1115
- ocrLanguages: 'chi_sim+eng',
1116
- pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
1117
- pdfOcrPageLimit: 'all',
1045
+ contentCoverage: 'pdf_page_text_plus_selective_ocr',
1046
+ pdfExtractor: 'pdfjs-dist',
1118
1047
  };
1119
- // 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
1048
+ const warnings = [];
1120
1049
  let canvasPath;
1121
1050
  let pdfjsPath;
1122
1051
  let tesseractPath;
1052
+ let sharpPath;
1123
1053
  try {
1124
1054
  canvasPath = resolvePackage('@napi-rs/canvas');
1125
1055
  pdfjsPath = resolvePackage('pdfjs-dist/legacy/build/pdf.mjs');
1126
1056
  tesseractPath = resolvePackage('tesseract.js');
1057
+ sharpPath = resolvePackage('sharp');
1127
1058
  }
1128
1059
  catch (e) {
1129
1060
  metadata.ocrRecommended = true;
1130
- metadata.ocrReason = `OCR 依赖解析失败: ${e.message}`;
1131
- return { text: '', metadata, warnings: [`内置扫描 PDF OCR 不可用:${metadata.ocrReason}`] };
1061
+ metadata.ocrReason = `PDF 混合解析依赖解析失败: ${e.message}`;
1062
+ return { text: '', metadata, warnings: [`内置 PDF 混合解析不可用:${metadata.ocrReason}`] };
1132
1063
  }
1133
- // NODE_PATH 确保子进程能解析 tesseract.js 的依赖
1134
1064
  const childEnv = { ...process.env };
1135
1065
  const nmRoot = getNodeModulesRoot();
1136
1066
  if (nmRoot)
@@ -1144,52 +1074,133 @@ import path from 'node:path';
1144
1074
  import { createCanvas } from ${JSON.stringify(canvasPath)};
1145
1075
  import * as pdfjs from ${JSON.stringify(pdfjsPath)};
1146
1076
  import { createWorker } from ${JSON.stringify(tesseractPath)};
1077
+ import sharp from ${JSON.stringify(sharpPath)};
1147
1078
  const filePath = process.argv[1];
1079
+ const minTextLength = Number(process.argv[2]);
1148
1080
  const bytes = new Uint8Array(fs.readFileSync(filePath));
1149
1081
  const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
1150
- const pageLimit = doc.numPages;
1151
- const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
1152
- const worker = await createWorker('chi_sim+eng');
1153
- const pages = [];
1082
+ const pageCount = doc.numPages;
1083
+ const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-hybrid-'));
1084
+ const pageTexts = [];
1085
+ const ocrPages = [];
1086
+ const failedPages = [];
1087
+ let worker;
1088
+ function normalizeTextLength(value) { return String(value || '').replace(/\\s+/gu, '').length; }
1089
+ function isLowQualityText(value) {
1090
+ const text = String(value || '');
1091
+ const normalizedLength = normalizeTextLength(text);
1092
+ if (normalizedLength < minTextLength) return true;
1093
+ const lines = text.split(/\\r?\\n/u).map(line => line.trim()).filter(Boolean);
1094
+ if (lines.length === 0) return true;
1095
+ const shortLineRatio = lines.filter(line => line.length <= 12).length / lines.length;
1096
+ const cjkCount = (text.match(/[\\p{Script=Han}]/gu) || []).length;
1097
+ const replacementCount = (text.match(/[�□]/gu) || []).length;
1098
+ return replacementCount > 5 || (shortLineRatio > 0.65 && cjkCount < 1200);
1099
+ }
1100
+ function itemsToMarkdown(items) {
1101
+ const rows = [];
1102
+ for (const item of items) {
1103
+ if (!item || typeof item.str !== 'string' || !item.str.trim()) continue;
1104
+ const transform = item.transform || [1, 0, 0, 1, 0, 0];
1105
+ rows.push({ text: item.str.trim(), x: transform[4] || 0, y: transform[5] || 0 });
1106
+ }
1107
+ rows.sort((a, b) => Math.abs(b.y - a.y) > 3 ? b.y - a.y : a.x - b.x);
1108
+ const lines = [];
1109
+ let currentY = null;
1110
+ let current = [];
1111
+ for (const row of rows) {
1112
+ if (currentY === null || Math.abs(row.y - currentY) <= 3) {
1113
+ current.push(row);
1114
+ currentY = currentY === null ? row.y : currentY;
1115
+ } else {
1116
+ lines.push(current.sort((a, b) => a.x - b.x).map(x => x.text).join(' '));
1117
+ current = [row];
1118
+ currentY = row.y;
1119
+ }
1120
+ }
1121
+ if (current.length > 0) lines.push(current.sort((a, b) => a.x - b.x).map(x => x.text).join(' '));
1122
+ return lines.join('\\n').trim();
1123
+ }
1124
+ async function recognizePage(page, pageNumber) {
1125
+ if (!worker) worker = await createWorker('chi_sim+eng');
1126
+ const viewport = page.getViewport({ scale: 2.5 });
1127
+ const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
1128
+ const context = canvas.getContext('2d');
1129
+ await page.render({ canvasContext: context, viewport }).promise;
1130
+ const rawPath = path.join(tmpDir, 'page-' + pageNumber + '-raw.png');
1131
+ const processedPath = path.join(tmpDir, 'page-' + pageNumber + '-processed.png');
1132
+ fs.writeFileSync(rawPath, canvas.toBuffer('image/png'));
1133
+ await sharp(rawPath)
1134
+ .resize({ width: Math.min(canvas.width, 4200), withoutEnlargement: true })
1135
+ .grayscale()
1136
+ .normalize()
1137
+ .sharpen()
1138
+ .threshold(170)
1139
+ .png()
1140
+ .toFile(processedPath);
1141
+ const recognized = await worker.recognize(processedPath);
1142
+ return (recognized.data.text || '').trim();
1143
+ }
1154
1144
  try {
1155
- for (let i = 1; i <= pageLimit; i += 1) {
1156
- const page = await doc.getPage(i);
1157
- const viewport = page.getViewport({ scale: 2 });
1158
- const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
1159
- const context = canvas.getContext('2d');
1160
- await page.render({ canvasContext: context, viewport }).promise;
1161
- const imagePath = path.join(tmpDir, 'page-' + i + '.png');
1162
- fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
1163
- const recognized = await worker.recognize(imagePath);
1164
- const text = (recognized.data.text || '').trim();
1165
- const lines = (recognized.data.lines || []).map((line, index) => ({ index: index + 1, text: line.text || '', bbox: line.bbox || null })).filter(line => line.text.trim());
1166
- if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + lines.map(line => '区域 ' + line.index + ' ' + JSON.stringify(line.bbox || {}) + ': ' + line.text).join('\\n') + '\\n\\n完整文本:\\n' + text);
1145
+ for (let i = 1; i <= pageCount; i += 1) {
1146
+ try {
1147
+ const page = await doc.getPage(i);
1148
+ const content = await page.getTextContent();
1149
+ const text = itemsToMarkdown(content.items || []);
1150
+ if (!isLowQualityText(text)) {
1151
+ pageTexts.push('## PDF ' + i + ' 页\\n\\n' + text);
1152
+ continue;
1153
+ }
1154
+ const ocrText = await recognizePage(page, i);
1155
+ if (ocrText) {
1156
+ ocrPages.push(i);
1157
+ pageTexts.push('## PDF 第 ' + i + ' 页(OCR)\\n\\n' + ocrText);
1158
+ } else if (text) {
1159
+ pageTexts.push('## PDF 第 ' + i + ' 页(低质量文本流)\\n\\n' + text);
1160
+ } else {
1161
+ failedPages.push({ page: i, reason: 'empty_text_and_ocr' });
1162
+ }
1163
+ } catch (error) {
1164
+ failedPages.push({ page: i, reason: error instanceof Error ? error.message : String(error) });
1165
+ }
1167
1166
  }
1168
1167
  } finally {
1169
- await worker.terminate();
1168
+ if (worker) await worker.terminate();
1169
+ await doc.destroy();
1170
1170
  fs.rmSync(tmpDir, { recursive: true, force: true });
1171
1171
  }
1172
- process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
1172
+ process.stdout.write(JSON.stringify({ pageCount, text: pageTexts.join('\\n\\n'), ocrPages, failedPages }));`,
1173
1173
  file.absolutePath,
1174
- ], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, env: childEnv });
1174
+ '80',
1175
+ ], { encoding: 'utf8', timeout: 0, maxBuffer: 80 * 1024 * 1024, env: childEnv });
1175
1176
  if (result.status !== 0 || result.error) {
1176
1177
  metadata.ocrRecommended = true;
1177
- metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
1178
- return { text: '', metadata, warnings: [`内置扫描 PDF OCR 失败:${metadata.ocrReason}`] };
1178
+ metadata.ocrReason = result.error ? result.error.message : (result.stderr || 'PDF 混合解析子进程失败').trim();
1179
+ return { text: '', metadata, warnings: [`内置 PDF 混合解析失败: ${metadata.ocrReason}`] };
1179
1180
  }
1180
1181
  try {
1181
- const parsed = JSON.parse(result.stdout);
1182
- const text = parsed.text?.trim() ?? '';
1183
- metadata.ocrPageCount = parsed.pageCount ?? 0;
1184
- metadata.pdfOcrPageLimit = parsed.pageLimit ?? 'all';
1185
- metadata.ocrTextLength = text.length;
1186
- metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
1187
- return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
1182
+ const parsed = JSON.parse(result.stdout || '{}');
1183
+ const ocrPages = parsed.ocrPages ?? [];
1184
+ const failedPages = parsed.failedPages ?? [];
1185
+ metadata.pdfPageCount = parsed.pageCount ?? 0;
1186
+ metadata.ocrAugmented = ocrPages.length > 0;
1187
+ metadata.ocrPages = ocrPages;
1188
+ metadata.textPages = Math.max(metadata.pdfPageCount - ocrPages.length - failedPages.length, 0);
1189
+ metadata.failedPages = failedPages;
1190
+ if (ocrPages.length > 0) {
1191
+ metadata.pdfRenderer = 'pdfjs-dist + @napi-rs/canvas';
1192
+ metadata.ocrProvider = 'tesseract.js';
1193
+ metadata.ocrLanguages = 'chi_sim+eng';
1194
+ metadata.imagePreprocessor = 'sharp';
1195
+ }
1196
+ if (failedPages.length > 0)
1197
+ warnings.push(`PDF 部分页解析失败: ${failedPages.map(p => `${p.page}:${p.reason}`).join('; ')}`);
1198
+ return { text: parsed.text?.trim() ? [this.metadataOnlyText(file), parsed.text.trim()].join('\n\n') : '', metadata, warnings };
1188
1199
  }
1189
- catch {
1200
+ catch (error) {
1190
1201
  metadata.ocrRecommended = true;
1191
- metadata.ocrReason = 'pdf_page_ocr_output_parse_failed';
1192
- return { text: '', metadata, warnings: ['内置扫描 PDF OCR 输出解析失败'] };
1202
+ metadata.ocrReason = `PDF 混合解析结果解析失败: ${error instanceof Error ? error.message : String(error)}`;
1203
+ return { text: '', metadata, warnings: [`内置 PDF 混合解析输出异常:${metadata.ocrReason}`] };
1193
1204
  }
1194
1205
  }
1195
1206
  async extractPdfText(buffer) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@customize-agent/knowledge",
3
- "version": "4.0.12",
3
+ "version": "4.0.14",
4
4
  "description": "Local knowledge base infrastructure for customize-agent",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -43,12 +43,13 @@
43
43
  "dwgdxf": "^2.0.1",
44
44
  "dxf-parser": "^1.1.2",
45
45
  "fast-glob": "^3.3.3",
46
+ "hnswlib-node": "^3.0.0",
46
47
  "jszip": "^3.10.1",
47
48
  "mammoth": "^1.12.0",
49
+ "node-gyp": "^12.1.0",
48
50
  "pdf-parse": "^2.4.5",
49
51
  "pdfjs-dist": "^5.4.394",
50
- "hnswlib-node": "^3.0.0",
51
- "node-gyp": "^12.1.0",
52
+ "sharp": "0.34.5",
52
53
  "tesseract.js": "^7.0.0",
53
54
  "word-extractor": "^1.0.4",
54
55
  "xlsx": "^0.18.5"