@customize-agent/knowledge 4.0.12 → 4.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
|
|
2
|
-
import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
|
|
3
2
|
import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
|
|
4
3
|
import { type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters } from '../search/federation-search.js';
|
|
5
4
|
import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
|
|
@@ -23,7 +22,6 @@ export interface KnowledgeBaseManagerOptions {
|
|
|
23
22
|
storageRoot?: string;
|
|
24
23
|
embeddingProvider?: EmbeddingProvider;
|
|
25
24
|
vectorStores?: Map<string, VectorStoreInterface>;
|
|
26
|
-
externalExtractors?: ExternalExtractorRegistry;
|
|
27
25
|
onProgress?: (progress: KnowledgeIndexProgress) => void;
|
|
28
26
|
/** 可选的 LLM Provider,用于查询扩展和语义重排序 */
|
|
29
27
|
llmProvider?: LLMSearchProvider;
|
|
@@ -43,7 +43,7 @@ export class KnowledgeBaseManager {
|
|
|
43
43
|
this.projectId = options.projectId;
|
|
44
44
|
this.embeddingProvider = options.embeddingProvider ?? createEmbeddingProviderFromEnvironment();
|
|
45
45
|
this.vectorStores = options.vectorStores ?? new Map();
|
|
46
|
-
this.extractor = new ContentExtractor(
|
|
46
|
+
this.extractor = new ContentExtractor();
|
|
47
47
|
this.llmProvider = options.llmProvider;
|
|
48
48
|
this.onProgress = options.onProgress;
|
|
49
49
|
const storageRoot = options.storageRoot ?? path.join(os.homedir(), USER_DATA_DIR);
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { ExternalExtractorRegistry } from './external-extractor.js';
|
|
2
1
|
import type { ClassifiedFile } from '../types.js';
|
|
3
2
|
/** 文件内容提取结果 */
|
|
4
3
|
export interface ExtractionResult {
|
|
@@ -9,10 +8,7 @@ export interface ExtractionResult {
|
|
|
9
8
|
}
|
|
10
9
|
/** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
|
|
11
10
|
export declare class ContentExtractor {
|
|
12
|
-
private readonly externalExtractors;
|
|
13
|
-
constructor(externalExtractors?: ExternalExtractorRegistry);
|
|
14
11
|
extract(file: ClassifiedFile): Promise<ExtractionResult>;
|
|
15
|
-
private tryExternalExtractor;
|
|
16
12
|
private extractTextClipping;
|
|
17
13
|
private swapUtf16Bytes;
|
|
18
14
|
private extractReadableFragments;
|
|
@@ -58,8 +54,7 @@ export declare class ContentExtractor {
|
|
|
58
54
|
private formatBoundingBox;
|
|
59
55
|
private validateRasterImage;
|
|
60
56
|
private extractPdf;
|
|
61
|
-
private
|
|
62
|
-
private extractScannedPdfOcr;
|
|
57
|
+
private extractPdfHybridPages;
|
|
63
58
|
private extractPdfText;
|
|
64
59
|
private normalizedTextLength;
|
|
65
60
|
private toPdfTextItem;
|
|
@@ -3,16 +3,11 @@ import * as fs from 'node:fs';
|
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { pathToFileURL } from 'node:url';
|
|
6
|
-
import { ExternalExtractorRegistry } from './external-extractor.js';
|
|
7
6
|
import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
|
|
8
7
|
const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
|
|
9
8
|
const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
|
|
10
9
|
/** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
|
|
11
10
|
export class ContentExtractor {
|
|
12
|
-
externalExtractors;
|
|
13
|
-
constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
|
|
14
|
-
this.externalExtractors = externalExtractors;
|
|
15
|
-
}
|
|
16
11
|
async extract(file) {
|
|
17
12
|
const start = Date.now();
|
|
18
13
|
const warnings = [];
|
|
@@ -22,86 +17,77 @@ export class ContentExtractor {
|
|
|
22
17
|
category: file.category,
|
|
23
18
|
format: file.format,
|
|
24
19
|
};
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
text =
|
|
28
|
-
Object.assign(metadata,
|
|
29
|
-
warnings.push(...
|
|
20
|
+
if (file.category === 'cad') {
|
|
21
|
+
const result = await this.extractCad(file);
|
|
22
|
+
text = result.text;
|
|
23
|
+
Object.assign(metadata, result.metadata);
|
|
24
|
+
warnings.push(...result.warnings);
|
|
25
|
+
}
|
|
26
|
+
else if (file.category === 'data') {
|
|
27
|
+
const result = this.extractData(file);
|
|
28
|
+
text = result.text;
|
|
29
|
+
Object.assign(metadata, result.metadata);
|
|
30
|
+
warnings.push(...result.warnings);
|
|
31
|
+
}
|
|
32
|
+
else if (file.category === 'diagram') {
|
|
33
|
+
const result = this.extractDiagram(file);
|
|
34
|
+
text = result.text;
|
|
35
|
+
Object.assign(metadata, result.metadata);
|
|
36
|
+
warnings.push(...result.warnings);
|
|
37
|
+
}
|
|
38
|
+
else if (file.category === 'document' && file.format === 'pdf') {
|
|
39
|
+
const result = await this.extractPdf(file);
|
|
40
|
+
text = result.text;
|
|
41
|
+
Object.assign(metadata, result.metadata);
|
|
42
|
+
warnings.push(...result.warnings);
|
|
43
|
+
}
|
|
44
|
+
else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
|
|
45
|
+
const result = await this.extractOfficeDocument(file);
|
|
46
|
+
text = result.text;
|
|
47
|
+
Object.assign(metadata, result.metadata);
|
|
48
|
+
warnings.push(...result.warnings);
|
|
49
|
+
}
|
|
50
|
+
else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
|
|
51
|
+
const result = this.extractDelimitedText(file);
|
|
52
|
+
text = result.text;
|
|
53
|
+
Object.assign(metadata, result.metadata);
|
|
54
|
+
warnings.push(...result.warnings);
|
|
55
|
+
}
|
|
56
|
+
else if (file.category === 'spreadsheet') {
|
|
57
|
+
const result = await this.extractSpreadsheet(file);
|
|
58
|
+
text = result.text;
|
|
59
|
+
Object.assign(metadata, result.metadata);
|
|
60
|
+
warnings.push(...result.warnings);
|
|
61
|
+
}
|
|
62
|
+
else if (file.category === 'image' && file.format !== 'vector') {
|
|
63
|
+
const result = await this.extractRasterImage(file);
|
|
64
|
+
text = result.text;
|
|
65
|
+
Object.assign(metadata, result.metadata);
|
|
66
|
+
warnings.push(...result.warnings);
|
|
67
|
+
}
|
|
68
|
+
else if (file.category === 'image' && file.format === 'vector') {
|
|
69
|
+
const result = this.extractSvg(file);
|
|
70
|
+
text = result.text;
|
|
71
|
+
Object.assign(metadata, result.metadata);
|
|
72
|
+
warnings.push(...result.warnings);
|
|
73
|
+
}
|
|
74
|
+
else if (file.format === 'text_clipping') {
|
|
75
|
+
const result = this.extractTextClipping(file);
|
|
76
|
+
text = result.text;
|
|
77
|
+
Object.assign(metadata, result.metadata);
|
|
78
|
+
warnings.push(...result.warnings);
|
|
79
|
+
}
|
|
80
|
+
else if (this.isTextReadable(file)) {
|
|
81
|
+
text = fs.readFileSync(file.absolutePath, 'utf8');
|
|
82
|
+
metadata.extractionMode = 'plain_text';
|
|
83
|
+
metadata.vectorizable = true;
|
|
30
84
|
}
|
|
31
85
|
else {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
warnings.push(...result.warnings);
|
|
38
|
-
}
|
|
39
|
-
else if (file.category === 'data') {
|
|
40
|
-
const result = this.extractData(file);
|
|
41
|
-
text = result.text;
|
|
42
|
-
Object.assign(metadata, result.metadata);
|
|
43
|
-
warnings.push(...result.warnings);
|
|
44
|
-
}
|
|
45
|
-
else if (file.category === 'diagram') {
|
|
46
|
-
const result = this.extractDiagram(file);
|
|
47
|
-
text = result.text;
|
|
48
|
-
Object.assign(metadata, result.metadata);
|
|
49
|
-
warnings.push(...result.warnings);
|
|
50
|
-
}
|
|
51
|
-
else if (file.category === 'document' && file.format === 'pdf') {
|
|
52
|
-
const result = await this.extractPdf(file);
|
|
53
|
-
text = result.text;
|
|
54
|
-
Object.assign(metadata, result.metadata);
|
|
55
|
-
warnings.push(...result.warnings);
|
|
56
|
-
}
|
|
57
|
-
else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
|
|
58
|
-
const result = await this.extractOfficeDocument(file);
|
|
59
|
-
text = result.text;
|
|
60
|
-
Object.assign(metadata, result.metadata);
|
|
61
|
-
warnings.push(...result.warnings);
|
|
62
|
-
}
|
|
63
|
-
else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
|
|
64
|
-
const result = this.extractDelimitedText(file);
|
|
65
|
-
text = result.text;
|
|
66
|
-
Object.assign(metadata, result.metadata);
|
|
67
|
-
warnings.push(...result.warnings);
|
|
68
|
-
}
|
|
69
|
-
else if (file.category === 'spreadsheet') {
|
|
70
|
-
const result = await this.extractSpreadsheet(file);
|
|
71
|
-
text = result.text;
|
|
72
|
-
Object.assign(metadata, result.metadata);
|
|
73
|
-
warnings.push(...result.warnings);
|
|
74
|
-
}
|
|
75
|
-
else if (file.category === 'image' && file.format !== 'vector') {
|
|
76
|
-
const result = await this.extractRasterImage(file);
|
|
77
|
-
text = result.text;
|
|
78
|
-
Object.assign(metadata, result.metadata);
|
|
79
|
-
warnings.push(...result.warnings);
|
|
80
|
-
}
|
|
81
|
-
else if (file.category === 'image' && file.format === 'vector') {
|
|
82
|
-
const result = this.extractSvg(file);
|
|
83
|
-
text = result.text;
|
|
84
|
-
Object.assign(metadata, result.metadata);
|
|
85
|
-
warnings.push(...result.warnings);
|
|
86
|
-
}
|
|
87
|
-
else if (file.format === 'text_clipping') {
|
|
88
|
-
const result = this.extractTextClipping(file);
|
|
89
|
-
text = result.text;
|
|
90
|
-
Object.assign(metadata, result.metadata);
|
|
91
|
-
warnings.push(...result.warnings);
|
|
92
|
-
}
|
|
93
|
-
else if (this.isTextReadable(file)) {
|
|
94
|
-
text = fs.readFileSync(file.absolutePath, 'utf8');
|
|
95
|
-
metadata.extractionMode = 'plain_text';
|
|
96
|
-
metadata.vectorizable = true;
|
|
97
|
-
}
|
|
98
|
-
else {
|
|
99
|
-
text = this.metadataOnlyText(file);
|
|
100
|
-
metadata.extractionMode = 'metadata_only';
|
|
101
|
-
metadata.vectorizable = true;
|
|
102
|
-
metadata.contentCoverage = 'metadata';
|
|
103
|
-
warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取,未解析出正文,未入库`);
|
|
104
|
-
}
|
|
86
|
+
text = this.metadataOnlyText(file);
|
|
87
|
+
metadata.extractionMode = 'metadata_only';
|
|
88
|
+
metadata.vectorizable = true;
|
|
89
|
+
metadata.contentCoverage = 'metadata';
|
|
90
|
+
warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取,未解析出正文,未入库`);
|
|
105
91
|
}
|
|
106
92
|
return {
|
|
107
93
|
text: this.cleanExtractedText(text, file).trim(),
|
|
@@ -110,37 +96,6 @@ export class ContentExtractor {
|
|
|
110
96
|
extractionTimeMs: Date.now() - start,
|
|
111
97
|
};
|
|
112
98
|
}
|
|
113
|
-
tryExternalExtractor(file) {
|
|
114
|
-
const extractors = this.externalExtractors.findAll(file);
|
|
115
|
-
if (extractors.length === 0)
|
|
116
|
-
return undefined;
|
|
117
|
-
const warnings = [];
|
|
118
|
-
for (const extractor of extractors) {
|
|
119
|
-
try {
|
|
120
|
-
const result = extractor.extract(file);
|
|
121
|
-
const text = result.text.trim();
|
|
122
|
-
if (!text) {
|
|
123
|
-
warnings.push(`外部解析器 ${extractor.name} 未提取到正文`);
|
|
124
|
-
continue;
|
|
125
|
-
}
|
|
126
|
-
return {
|
|
127
|
-
text,
|
|
128
|
-
metadata: {
|
|
129
|
-
extractionMode: 'external_advanced_plugin',
|
|
130
|
-
vectorizable: true,
|
|
131
|
-
contentCoverage: 'external_full_text',
|
|
132
|
-
externalExtractor: extractor.id,
|
|
133
|
-
...(result.metadata ?? {}),
|
|
134
|
-
},
|
|
135
|
-
warnings: [...warnings, ...(result.warnings ?? [])],
|
|
136
|
-
};
|
|
137
|
-
}
|
|
138
|
-
catch (error) {
|
|
139
|
-
warnings.push(`外部解析器 ${extractor.name} 失败: ${error instanceof Error ? error.message : String(error)}`);
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
return warnings.length > 0 ? { text: '', metadata: {}, warnings } : undefined;
|
|
143
|
-
}
|
|
144
99
|
extractTextClipping(file) {
|
|
145
100
|
const buffer = fs.readFileSync(file.absolutePath);
|
|
146
101
|
const candidates = [
|
|
@@ -1054,83 +1009,58 @@ try {
|
|
|
1054
1009
|
return undefined;
|
|
1055
1010
|
}
|
|
1056
1011
|
async extractPdf(file) {
|
|
1012
|
+
const hybrid = await this.extractPdfHybridPages(file);
|
|
1013
|
+
if (hybrid.text.trim())
|
|
1014
|
+
return hybrid;
|
|
1057
1015
|
const metadata = { extractionMode: 'pdf_text', vectorizable: true };
|
|
1058
|
-
const warnings = [];
|
|
1059
|
-
// 第一层:pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
|
|
1016
|
+
const warnings = [...hybrid.warnings];
|
|
1060
1017
|
try {
|
|
1061
1018
|
const raw = fs.readFileSync(file.absolutePath);
|
|
1062
1019
|
const text = await this.extractPdfText(raw);
|
|
1063
1020
|
if (text.trim()) {
|
|
1064
1021
|
metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
|
|
1065
1022
|
metadata.pdfExtractor = 'pdfjs-dist';
|
|
1066
|
-
|
|
1067
|
-
if (!this.shouldAugmentPdfWithOcr(markdownText))
|
|
1068
|
-
return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata, warnings };
|
|
1069
|
-
const ocr = await this.extractScannedPdfOcr(file);
|
|
1070
|
-
if (ocr.text.trim()) {
|
|
1071
|
-
metadata.contentCoverage = 'pdf_text_streams_plus_ocr';
|
|
1072
|
-
metadata.ocrAugmented = true;
|
|
1073
|
-
return { text: [this.metadataOnlyText(file), markdownText, '## PDF OCR 备用识别文本', ocr.text].join('\n\n'), metadata: { ...metadata, ocr: ocr.metadata }, warnings: [...warnings, ...ocr.warnings] };
|
|
1074
|
-
}
|
|
1075
|
-
return { text: [this.metadataOnlyText(file), markdownText].join('\n\n'), metadata: { ...metadata, ocrRecommended: true, ocrReason: ocr.metadata.ocrReason ?? 'pdf_text_low_quality' }, warnings: [...warnings, ...ocr.warnings] };
|
|
1023
|
+
return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
|
|
1076
1024
|
}
|
|
1077
1025
|
}
|
|
1078
1026
|
catch (error) {
|
|
1079
1027
|
warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
|
|
1080
1028
|
metadata.parseError = error instanceof Error ? error.message : String(error);
|
|
1081
1029
|
}
|
|
1082
|
-
// 第二层:OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
|
|
1083
|
-
const ocr = await this.extractScannedPdfOcr(file);
|
|
1084
|
-
if (ocr.text.trim())
|
|
1085
|
-
return ocr;
|
|
1086
|
-
// 第三层:仅索引元数据(兜底)
|
|
1087
1030
|
metadata.extractionMode = 'pdf_metadata_only';
|
|
1088
1031
|
metadata.contentCoverage = 'metadata_filename';
|
|
1089
1032
|
metadata.ocrRecommended = true;
|
|
1090
|
-
metadata.ocrReason =
|
|
1033
|
+
metadata.ocrReason = hybrid.metadata.ocrReason ?? 'pdf_text_stream_empty_or_unavailable';
|
|
1091
1034
|
metadata.pdfPageOcrSupported = true;
|
|
1092
1035
|
return {
|
|
1093
1036
|
text: this.metadataOnlyText(file),
|
|
1094
1037
|
metadata,
|
|
1095
|
-
warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据'
|
|
1038
|
+
warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据'],
|
|
1096
1039
|
};
|
|
1097
1040
|
}
|
|
1098
|
-
|
|
1099
|
-
const normalizedLength = this.normalizedTextLength(text);
|
|
1100
|
-
if (normalizedLength < 1200)
|
|
1101
|
-
return true;
|
|
1102
|
-
const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
|
|
1103
|
-
if (lines.length === 0)
|
|
1104
|
-
return true;
|
|
1105
|
-
const shortLineRatio = lines.filter(line => line.length <= 12).length / lines.length;
|
|
1106
|
-
const cjkCount = (text.match(/[\p{Script=Han}]/gu) ?? []).length;
|
|
1107
|
-
return shortLineRatio > 0.65 && cjkCount < 1200;
|
|
1108
|
-
}
|
|
1109
|
-
async extractScannedPdfOcr(file) {
|
|
1041
|
+
async extractPdfHybridPages(file) {
|
|
1110
1042
|
const metadata = {
|
|
1111
|
-
extractionMode: '
|
|
1043
|
+
extractionMode: 'pdf_hybrid_pages',
|
|
1112
1044
|
vectorizable: true,
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
ocrLanguages: 'chi_sim+eng',
|
|
1116
|
-
pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
|
|
1117
|
-
pdfOcrPageLimit: 'all',
|
|
1045
|
+
contentCoverage: 'pdf_page_text_plus_selective_ocr',
|
|
1046
|
+
pdfExtractor: 'pdfjs-dist',
|
|
1118
1047
|
};
|
|
1119
|
-
|
|
1048
|
+
const warnings = [];
|
|
1120
1049
|
let canvasPath;
|
|
1121
1050
|
let pdfjsPath;
|
|
1122
1051
|
let tesseractPath;
|
|
1052
|
+
let sharpPath;
|
|
1123
1053
|
try {
|
|
1124
1054
|
canvasPath = resolvePackage('@napi-rs/canvas');
|
|
1125
1055
|
pdfjsPath = resolvePackage('pdfjs-dist/legacy/build/pdf.mjs');
|
|
1126
1056
|
tesseractPath = resolvePackage('tesseract.js');
|
|
1057
|
+
sharpPath = resolvePackage('sharp');
|
|
1127
1058
|
}
|
|
1128
1059
|
catch (e) {
|
|
1129
1060
|
metadata.ocrRecommended = true;
|
|
1130
|
-
metadata.ocrReason = `
|
|
1131
|
-
return { text: '', metadata, warnings: [
|
|
1061
|
+
metadata.ocrReason = `PDF 混合解析依赖解析失败: ${e.message}`;
|
|
1062
|
+
return { text: '', metadata, warnings: [`内置 PDF 混合解析不可用:${metadata.ocrReason}`] };
|
|
1132
1063
|
}
|
|
1133
|
-
// NODE_PATH 确保子进程能解析 tesseract.js 的依赖
|
|
1134
1064
|
const childEnv = { ...process.env };
|
|
1135
1065
|
const nmRoot = getNodeModulesRoot();
|
|
1136
1066
|
if (nmRoot)
|
|
@@ -1144,52 +1074,133 @@ import path from 'node:path';
|
|
|
1144
1074
|
import { createCanvas } from ${JSON.stringify(canvasPath)};
|
|
1145
1075
|
import * as pdfjs from ${JSON.stringify(pdfjsPath)};
|
|
1146
1076
|
import { createWorker } from ${JSON.stringify(tesseractPath)};
|
|
1077
|
+
import sharp from ${JSON.stringify(sharpPath)};
|
|
1147
1078
|
const filePath = process.argv[1];
|
|
1079
|
+
const minTextLength = Number(process.argv[2]);
|
|
1148
1080
|
const bytes = new Uint8Array(fs.readFileSync(filePath));
|
|
1149
1081
|
const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
|
|
1150
|
-
const
|
|
1151
|
-
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-
|
|
1152
|
-
const
|
|
1153
|
-
const
|
|
1082
|
+
const pageCount = doc.numPages;
|
|
1083
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-hybrid-'));
|
|
1084
|
+
const pageTexts = [];
|
|
1085
|
+
const ocrPages = [];
|
|
1086
|
+
const failedPages = [];
|
|
1087
|
+
let worker;
|
|
1088
|
+
function normalizeTextLength(value) { return String(value || '').replace(/\\s+/gu, '').length; }
|
|
1089
|
+
function isLowQualityText(value) {
|
|
1090
|
+
const text = String(value || '');
|
|
1091
|
+
const normalizedLength = normalizeTextLength(text);
|
|
1092
|
+
if (normalizedLength < minTextLength) return true;
|
|
1093
|
+
const lines = text.split(/\\r?\\n/u).map(line => line.trim()).filter(Boolean);
|
|
1094
|
+
if (lines.length === 0) return true;
|
|
1095
|
+
const shortLineRatio = lines.filter(line => line.length <= 12).length / lines.length;
|
|
1096
|
+
const cjkCount = (text.match(/[\\p{Script=Han}]/gu) || []).length;
|
|
1097
|
+
const replacementCount = (text.match(/[�□]/gu) || []).length;
|
|
1098
|
+
return replacementCount > 5 || (shortLineRatio > 0.65 && cjkCount < 1200);
|
|
1099
|
+
}
|
|
1100
|
+
function itemsToMarkdown(items) {
|
|
1101
|
+
const rows = [];
|
|
1102
|
+
for (const item of items) {
|
|
1103
|
+
if (!item || typeof item.str !== 'string' || !item.str.trim()) continue;
|
|
1104
|
+
const transform = item.transform || [1, 0, 0, 1, 0, 0];
|
|
1105
|
+
rows.push({ text: item.str.trim(), x: transform[4] || 0, y: transform[5] || 0 });
|
|
1106
|
+
}
|
|
1107
|
+
rows.sort((a, b) => Math.abs(b.y - a.y) > 3 ? b.y - a.y : a.x - b.x);
|
|
1108
|
+
const lines = [];
|
|
1109
|
+
let currentY = null;
|
|
1110
|
+
let current = [];
|
|
1111
|
+
for (const row of rows) {
|
|
1112
|
+
if (currentY === null || Math.abs(row.y - currentY) <= 3) {
|
|
1113
|
+
current.push(row);
|
|
1114
|
+
currentY = currentY === null ? row.y : currentY;
|
|
1115
|
+
} else {
|
|
1116
|
+
lines.push(current.sort((a, b) => a.x - b.x).map(x => x.text).join(' '));
|
|
1117
|
+
current = [row];
|
|
1118
|
+
currentY = row.y;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
if (current.length > 0) lines.push(current.sort((a, b) => a.x - b.x).map(x => x.text).join(' '));
|
|
1122
|
+
return lines.join('\\n').trim();
|
|
1123
|
+
}
|
|
1124
|
+
async function recognizePage(page, pageNumber) {
|
|
1125
|
+
if (!worker) worker = await createWorker('chi_sim+eng');
|
|
1126
|
+
const viewport = page.getViewport({ scale: 2.5 });
|
|
1127
|
+
const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
|
|
1128
|
+
const context = canvas.getContext('2d');
|
|
1129
|
+
await page.render({ canvasContext: context, viewport }).promise;
|
|
1130
|
+
const rawPath = path.join(tmpDir, 'page-' + pageNumber + '-raw.png');
|
|
1131
|
+
const processedPath = path.join(tmpDir, 'page-' + pageNumber + '-processed.png');
|
|
1132
|
+
fs.writeFileSync(rawPath, canvas.toBuffer('image/png'));
|
|
1133
|
+
await sharp(rawPath)
|
|
1134
|
+
.resize({ width: Math.min(canvas.width, 4200), withoutEnlargement: true })
|
|
1135
|
+
.grayscale()
|
|
1136
|
+
.normalize()
|
|
1137
|
+
.sharpen()
|
|
1138
|
+
.threshold(170)
|
|
1139
|
+
.png()
|
|
1140
|
+
.toFile(processedPath);
|
|
1141
|
+
const recognized = await worker.recognize(processedPath);
|
|
1142
|
+
return (recognized.data.text || '').trim();
|
|
1143
|
+
}
|
|
1154
1144
|
try {
|
|
1155
|
-
for (let i = 1; i <=
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1145
|
+
for (let i = 1; i <= pageCount; i += 1) {
|
|
1146
|
+
try {
|
|
1147
|
+
const page = await doc.getPage(i);
|
|
1148
|
+
const content = await page.getTextContent();
|
|
1149
|
+
const text = itemsToMarkdown(content.items || []);
|
|
1150
|
+
if (!isLowQualityText(text)) {
|
|
1151
|
+
pageTexts.push('## PDF 第 ' + i + ' 页\\n\\n' + text);
|
|
1152
|
+
continue;
|
|
1153
|
+
}
|
|
1154
|
+
const ocrText = await recognizePage(page, i);
|
|
1155
|
+
if (ocrText) {
|
|
1156
|
+
ocrPages.push(i);
|
|
1157
|
+
pageTexts.push('## PDF 第 ' + i + ' 页(OCR)\\n\\n' + ocrText);
|
|
1158
|
+
} else if (text) {
|
|
1159
|
+
pageTexts.push('## PDF 第 ' + i + ' 页(低质量文本流)\\n\\n' + text);
|
|
1160
|
+
} else {
|
|
1161
|
+
failedPages.push({ page: i, reason: 'empty_text_and_ocr' });
|
|
1162
|
+
}
|
|
1163
|
+
} catch (error) {
|
|
1164
|
+
failedPages.push({ page: i, reason: error instanceof Error ? error.message : String(error) });
|
|
1165
|
+
}
|
|
1167
1166
|
}
|
|
1168
1167
|
} finally {
|
|
1169
|
-
await worker.terminate();
|
|
1168
|
+
if (worker) await worker.terminate();
|
|
1169
|
+
await doc.destroy();
|
|
1170
1170
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
1171
1171
|
}
|
|
1172
|
-
process.stdout.write(JSON.stringify({ pageCount
|
|
1172
|
+
process.stdout.write(JSON.stringify({ pageCount, text: pageTexts.join('\\n\\n'), ocrPages, failedPages }));`,
|
|
1173
1173
|
file.absolutePath,
|
|
1174
|
-
|
|
1174
|
+
'80',
|
|
1175
|
+
], { encoding: 'utf8', timeout: 0, maxBuffer: 80 * 1024 * 1024, env: childEnv });
|
|
1175
1176
|
if (result.status !== 0 || result.error) {
|
|
1176
1177
|
metadata.ocrRecommended = true;
|
|
1177
|
-
metadata.ocrReason = result.error
|
|
1178
|
-
return { text: '', metadata, warnings: [
|
|
1178
|
+
metadata.ocrReason = result.error ? result.error.message : (result.stderr || 'PDF 混合解析子进程失败').trim();
|
|
1179
|
+
return { text: '', metadata, warnings: [`内置 PDF 混合解析失败: ${metadata.ocrReason}`] };
|
|
1179
1180
|
}
|
|
1180
1181
|
try {
|
|
1181
|
-
const parsed = JSON.parse(result.stdout);
|
|
1182
|
-
const
|
|
1183
|
-
|
|
1184
|
-
metadata.
|
|
1185
|
-
metadata.
|
|
1186
|
-
metadata.
|
|
1187
|
-
|
|
1182
|
+
const parsed = JSON.parse(result.stdout || '{}');
|
|
1183
|
+
const ocrPages = parsed.ocrPages ?? [];
|
|
1184
|
+
const failedPages = parsed.failedPages ?? [];
|
|
1185
|
+
metadata.pdfPageCount = parsed.pageCount ?? 0;
|
|
1186
|
+
metadata.ocrAugmented = ocrPages.length > 0;
|
|
1187
|
+
metadata.ocrPages = ocrPages;
|
|
1188
|
+
metadata.textPages = Math.max(metadata.pdfPageCount - ocrPages.length - failedPages.length, 0);
|
|
1189
|
+
metadata.failedPages = failedPages;
|
|
1190
|
+
if (ocrPages.length > 0) {
|
|
1191
|
+
metadata.pdfRenderer = 'pdfjs-dist + @napi-rs/canvas';
|
|
1192
|
+
metadata.ocrProvider = 'tesseract.js';
|
|
1193
|
+
metadata.ocrLanguages = 'chi_sim+eng';
|
|
1194
|
+
metadata.imagePreprocessor = 'sharp';
|
|
1195
|
+
}
|
|
1196
|
+
if (failedPages.length > 0)
|
|
1197
|
+
warnings.push(`PDF 部分页解析失败: ${failedPages.map(p => `${p.page}:${p.reason}`).join('; ')}`);
|
|
1198
|
+
return { text: parsed.text?.trim() ? [this.metadataOnlyText(file), parsed.text.trim()].join('\n\n') : '', metadata, warnings };
|
|
1188
1199
|
}
|
|
1189
|
-
catch {
|
|
1200
|
+
catch (error) {
|
|
1190
1201
|
metadata.ocrRecommended = true;
|
|
1191
|
-
metadata.ocrReason =
|
|
1192
|
-
return { text: '', metadata, warnings: [
|
|
1202
|
+
metadata.ocrReason = `PDF 混合解析结果解析失败: ${error instanceof Error ? error.message : String(error)}`;
|
|
1203
|
+
return { text: '', metadata, warnings: [`内置 PDF 混合解析输出异常:${metadata.ocrReason}`] };
|
|
1193
1204
|
}
|
|
1194
1205
|
}
|
|
1195
1206
|
async extractPdfText(buffer) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@customize-agent/knowledge",
|
|
3
|
-
"version": "4.0.
|
|
3
|
+
"version": "4.0.14",
|
|
4
4
|
"description": "Local knowledge base infrastructure for customize-agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -43,12 +43,13 @@
|
|
|
43
43
|
"dwgdxf": "^2.0.1",
|
|
44
44
|
"dxf-parser": "^1.1.2",
|
|
45
45
|
"fast-glob": "^3.3.3",
|
|
46
|
+
"hnswlib-node": "^3.0.0",
|
|
46
47
|
"jszip": "^3.10.1",
|
|
47
48
|
"mammoth": "^1.12.0",
|
|
49
|
+
"node-gyp": "^12.1.0",
|
|
48
50
|
"pdf-parse": "^2.4.5",
|
|
49
51
|
"pdfjs-dist": "^5.4.394",
|
|
50
|
-
"
|
|
51
|
-
"node-gyp": "^12.1.0",
|
|
52
|
+
"sharp": "0.34.5",
|
|
52
53
|
"tesseract.js": "^7.0.0",
|
|
53
54
|
"word-extractor": "^1.0.4",
|
|
54
55
|
"xlsx": "^0.18.5"
|