@customize-agent/knowledge 4.0.1 → 4.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/bge-tokenizer.d.ts +12 -0
- package/dist/chunking/bge-tokenizer.js +70 -0
- package/dist/chunking/text-chunker.d.ts +20 -0
- package/dist/chunking/text-chunker.js +166 -53
- package/dist/classification/classifier.d.ts +1 -0
- package/dist/classification/classifier.js +1 -1
- package/dist/constants.d.ts +7 -0
- package/dist/constants.js +7 -0
- package/dist/core/change-tracker.d.ts +13 -0
- package/dist/core/change-tracker.js +13 -0
- package/dist/core/file-scanner.d.ts +13 -0
- package/dist/core/file-scanner.js +12 -0
- package/dist/core/index-state-store.d.ts +67 -0
- package/dist/core/index-state-store.js +204 -50
- package/dist/core/knowledge-base-manager.d.ts +24 -2
- package/dist/core/knowledge-base-manager.js +204 -52
- package/dist/core/multi-project-manager.d.ts +10 -0
- package/dist/core/multi-project-manager.js +21 -2
- package/dist/core/project-config.d.ts +4 -0
- package/dist/core/project-config.js +4 -0
- package/dist/core/project-id.d.ts +5 -0
- package/dist/core/project-id.js +5 -0
- package/dist/core/project-registry.d.ts +1 -0
- package/dist/core/project-registry.js +1 -0
- package/dist/dedup/dedup-engine.d.ts +3 -0
- package/dist/dedup/dedup-engine.js +1 -0
- package/dist/dedup/relationship-detector.d.ts +7 -0
- package/dist/dedup/relationship-detector.js +7 -0
- package/dist/embedding/embedding-provider.d.ts +32 -0
- package/dist/embedding/embedding-provider.js +136 -2
- package/dist/extraction/content-extractor.d.ts +32 -2
- package/dist/extraction/content-extractor.js +524 -124
- package/dist/extraction/external-extractor.d.ts +10 -0
- package/dist/extraction/external-extractor.js +6 -0
- package/dist/extraction/module-resolver.js +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +4 -4
- package/dist/search/federation-search.d.ts +8 -0
- package/dist/search/federation-search.js +2 -0
- package/dist/types.d.ts +10 -0
- package/dist/vector/collection-manager.d.ts +3 -0
- package/dist/vector/collection-manager.js +3 -0
- package/dist/vector/hnsw-vector-store.d.ts +21 -0
- package/dist/vector/hnsw-vector-store.js +108 -0
- package/dist/vector/types.d.ts +8 -0
- package/dist/vector/vector-indexer.d.ts +15 -1
- package/dist/vector/vector-indexer.js +34 -5
- package/models/bge-small-zh-v1.5/config.json +31 -0
- package/models/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
- package/models/bge-small-zh-v1.5/special_tokens_map.json +7 -0
- package/models/bge-small-zh-v1.5/tokenizer.json +21278 -0
- package/models/bge-small-zh-v1.5/tokenizer_config.json +15 -0
- package/package.json +16 -9
- package/scripts/install-hnsw.cjs +47 -0
- package/dist/vector/sqlite-vec-store.d.ts +0 -38
- package/dist/vector/sqlite-vec-store.js +0 -203
|
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';
|
|
|
5
5
|
import { pathToFileURL } from 'node:url';
|
|
6
6
|
import { ExternalExtractorRegistry } from './external-extractor.js';
|
|
7
7
|
import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
|
|
8
|
+
/** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
|
|
8
9
|
export class ContentExtractor {
|
|
9
10
|
externalExtractors;
|
|
10
11
|
constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
|
|
@@ -69,12 +70,6 @@ export class ContentExtractor {
|
|
|
69
70
|
Object.assign(metadata, result.metadata);
|
|
70
71
|
warnings.push(...result.warnings);
|
|
71
72
|
}
|
|
72
|
-
else if (file.category === 'archive') {
|
|
73
|
-
const result = await this.extractArchive(file);
|
|
74
|
-
text = result.text;
|
|
75
|
-
Object.assign(metadata, result.metadata);
|
|
76
|
-
warnings.push(...result.warnings);
|
|
77
|
-
}
|
|
78
73
|
else if (file.category === 'image' && file.format !== 'vector') {
|
|
79
74
|
const result = await this.extractRasterImage(file);
|
|
80
75
|
text = result.text;
|
|
@@ -208,7 +203,7 @@ export class ContentExtractor {
|
|
|
208
203
|
if (file.format === 'autocad' && ext === '.dxf') {
|
|
209
204
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
210
205
|
const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
|
|
211
|
-
const textEntities = this.
|
|
206
|
+
const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
|
|
212
207
|
const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
|
|
213
208
|
const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
|
|
214
209
|
const uniqueLayers = Array.from(new Set(layers));
|
|
@@ -221,14 +216,16 @@ export class ContentExtractor {
|
|
|
221
216
|
metadata.blockNames = uniqueBlocks.slice(0, 80);
|
|
222
217
|
metadata.entityTypeCount = uniqueEntityTypes.length;
|
|
223
218
|
metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
|
|
224
|
-
metadata.contentCoverage = '
|
|
219
|
+
metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
|
|
220
|
+
const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
|
|
225
221
|
return {
|
|
226
222
|
text: [
|
|
227
223
|
this.metadataOnlyText(file),
|
|
228
224
|
`CAD DXF 图层: ${uniqueLayers.join(', ')}`,
|
|
229
225
|
`CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
|
|
230
226
|
`CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
|
|
231
|
-
|
|
227
|
+
'CAD 语义图纸节点:',
|
|
228
|
+
...semanticNodes,
|
|
232
229
|
].join('\n'),
|
|
233
230
|
metadata,
|
|
234
231
|
warnings,
|
|
@@ -307,7 +304,7 @@ export class ContentExtractor {
|
|
|
307
304
|
warnings.push('dxf-parser 解析失败,已使用 DXF 文本结构抽取回退');
|
|
308
305
|
}
|
|
309
306
|
const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
|
|
310
|
-
const textEntities = this.
|
|
307
|
+
const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
|
|
311
308
|
const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
|
|
312
309
|
const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
|
|
313
310
|
const uniqueLayers = Array.from(new Set(layers));
|
|
@@ -320,20 +317,71 @@ export class ContentExtractor {
|
|
|
320
317
|
metadata.blockNames = uniqueBlocks.slice(0, 80);
|
|
321
318
|
metadata.entityTypeCount = uniqueEntityTypes.length;
|
|
322
319
|
metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
|
|
323
|
-
metadata.contentCoverage = '
|
|
320
|
+
metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
|
|
324
321
|
metadata.parsedByDxfParser = Boolean(parsed);
|
|
322
|
+
const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
|
|
325
323
|
return {
|
|
326
324
|
text: [
|
|
327
325
|
this.metadataOnlyText(file),
|
|
328
326
|
`CAD DXF 图层: ${uniqueLayers.join(', ')}`,
|
|
329
327
|
`CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
|
|
330
328
|
`CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
|
|
331
|
-
|
|
329
|
+
'CAD 语义图纸节点:',
|
|
330
|
+
...semanticNodes,
|
|
332
331
|
].join('\n'),
|
|
333
332
|
metadata,
|
|
334
333
|
warnings,
|
|
335
334
|
};
|
|
336
335
|
}
|
|
336
|
+
buildCadSemanticNodes(file, layers, blocks, entityTypes, texts) {
|
|
337
|
+
const fileName = path.basename(file.relativePath);
|
|
338
|
+
const defaultLayer = layers[0] ?? '未命名图层';
|
|
339
|
+
const defaultBlock = blocks[0] ?? '全局模型空间';
|
|
340
|
+
const defaultEntity = entityTypes.find(type => /DIMENSION|TEXT|MTEXT|LEADER/u.test(type)) ?? entityTypes[0] ?? 'UNKNOWN';
|
|
341
|
+
const annotations = texts.length > 0 ? texts : [{ text: '未提取到文字标注', layer: defaultLayer, block: defaultBlock, entityType: defaultEntity }];
|
|
342
|
+
return annotations.map((annotation, index) => {
|
|
343
|
+
const nearest = this.findNearestCadAnnotation(annotation, annotations);
|
|
344
|
+
const layer = annotation.layer ?? layers[index % Math.max(1, layers.length)] ?? defaultLayer;
|
|
345
|
+
const block = annotation.block ?? blocks[index % Math.max(1, blocks.length)] ?? defaultBlock;
|
|
346
|
+
const entity = annotation.entityType ?? this.inferCadEntityType(annotation.text, defaultEntity);
|
|
347
|
+
const status = /关键|critical|尺寸|dim|mm|cm|m\b|°|φ|Φ|R\d/iu.test(annotation.text) ? '关键尺寸/约束候选' : '普通标注';
|
|
348
|
+
const position = annotation.x != null && annotation.y != null ? ` | 坐标: (${annotation.x.toFixed(2)}, ${annotation.y.toFixed(2)})` : '';
|
|
349
|
+
return [`图纸节点: ${fileName} | 图层: ${layer} | 块: ${block} | 实体类型: ${entity}${position}`, `└── 标注文本: ${annotation.text} | 关联对象: ${nearest ? `邻近标注 ${nearest.text}` : '空间邻近候选'} | 状态: ${status}`].join('\n');
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
extractDxfTextAnnotations(raw) {
|
|
353
|
+
const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
|
|
354
|
+
return entities.flatMap(section => {
|
|
355
|
+
const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
|
|
356
|
+
if (!text)
|
|
357
|
+
return [];
|
|
358
|
+
return [{
|
|
359
|
+
text,
|
|
360
|
+
layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
|
|
361
|
+
block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
|
|
362
|
+
entityType: section.trim().split(/\s+/u)[0],
|
|
363
|
+
x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
|
|
364
|
+
y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
|
|
365
|
+
}].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
findNearestCadAnnotation(target, annotations) {
|
|
369
|
+
if (target.x == null || target.y == null)
|
|
370
|
+
return undefined;
|
|
371
|
+
return annotations
|
|
372
|
+
.filter(item => item !== target && item.x != null && item.y != null)
|
|
373
|
+
.map(item => ({ item, distance: Math.hypot((item.x ?? 0) - target.x, (item.y ?? 0) - target.y) }))
|
|
374
|
+
.sort((a, b) => a.distance - b.distance)[0]?.item;
|
|
375
|
+
}
|
|
376
|
+
inferCadEntityType(text, fallback) {
|
|
377
|
+
if (/\b\d+(?:\.\d+)?\s*(?:mm|cm|m)\b|φ|Φ|R\d/iu.test(text))
|
|
378
|
+
return '线性尺寸/半径尺寸';
|
|
379
|
+
if (/°|angle|角度/iu.test(text))
|
|
380
|
+
return '角度尺寸';
|
|
381
|
+
if (/note|说明|备注/iu.test(text))
|
|
382
|
+
return '文字说明';
|
|
383
|
+
return fallback;
|
|
384
|
+
}
|
|
337
385
|
async tryConvertDwgWithBundledWasm(filePath) {
|
|
338
386
|
try {
|
|
339
387
|
const mod = await resolveAndImport('dwgdxf');
|
|
@@ -447,36 +495,39 @@ export class ContentExtractor {
|
|
|
447
495
|
}
|
|
448
496
|
extractData(file) {
|
|
449
497
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
450
|
-
const metadata = { extractionMode: 'structured_data', vectorizable: true };
|
|
498
|
+
const metadata = { extractionMode: 'structured_data', semanticExtractionMode: 'structured_data_semantic_paths', vectorizable: true };
|
|
451
499
|
try {
|
|
452
500
|
if (file.format === 'json') {
|
|
453
501
|
const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
|
|
454
|
-
const
|
|
455
|
-
? raw.split(/\r?\n/u).filter(Boolean).
|
|
456
|
-
:
|
|
502
|
+
const records = isJsonl
|
|
503
|
+
? raw.split(/\r?\n/u).filter(Boolean).map((line, index) => ({ path: `line${index + 1}`, value: JSON.parse(line) }))
|
|
504
|
+
: [{ path: '$', value: JSON.parse(raw) }];
|
|
505
|
+
const lines = records.flatMap(record => this.flattenJson(record.value, record.path));
|
|
506
|
+
const objects = records.flatMap(record => this.atomicJsonObjects(record.value, record.path));
|
|
457
507
|
metadata.fieldCount = lines.length;
|
|
458
|
-
metadata.recordCount =
|
|
508
|
+
metadata.recordCount = records.length;
|
|
509
|
+
metadata.objectCount = objects.length;
|
|
459
510
|
metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
|
|
460
|
-
metadata.contentCoverage = isJsonl ? '
|
|
461
|
-
return { text: [this.metadataOnlyText(file), ...lines
|
|
511
|
+
metadata.contentCoverage = isJsonl ? 'jsonl_atomic_objects_paths_values' : 'json_atomic_objects_paths_values';
|
|
512
|
+
return { text: [this.metadataOnlyText(file), '## 路径声明', ...lines, '## 原子对象', ...objects].join('\n'), metadata, warnings: [] };
|
|
462
513
|
}
|
|
463
514
|
}
|
|
464
515
|
catch {
|
|
465
516
|
metadata.parseError = true;
|
|
466
517
|
}
|
|
467
518
|
if (file.format === 'yaml') {
|
|
468
|
-
const
|
|
469
|
-
metadata.fieldCount =
|
|
470
|
-
metadata.dataPaths =
|
|
471
|
-
metadata.contentCoverage = '
|
|
472
|
-
return { text: [this.metadataOnlyText(file), ...
|
|
519
|
+
const lines = this.flattenYamlByIndent(raw).slice(0, 3000);
|
|
520
|
+
metadata.fieldCount = lines.length;
|
|
521
|
+
metadata.dataPaths = lines.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
|
|
522
|
+
metadata.contentCoverage = 'yaml_indented_paths_values';
|
|
523
|
+
return { text: [this.metadataOnlyText(file), '## YAML 路径声明', ...lines].join('\n'), metadata, warnings: [] };
|
|
473
524
|
}
|
|
474
525
|
if (file.format === 'xml') {
|
|
475
|
-
const elements = this.
|
|
526
|
+
const elements = this.flattenXmlPaths(raw).slice(0, 3000);
|
|
476
527
|
metadata.elementTextCount = elements.length;
|
|
477
|
-
metadata.dataPaths = elements.map(line => line.
|
|
478
|
-
metadata.contentCoverage = '
|
|
479
|
-
return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
|
|
528
|
+
metadata.dataPaths = elements.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
|
|
529
|
+
metadata.contentCoverage = 'xml_paths_values';
|
|
530
|
+
return { text: [this.metadataOnlyText(file), '## XML 路径声明', ...elements].join('\n'), metadata, warnings: [] };
|
|
480
531
|
}
|
|
481
532
|
metadata.contentCoverage = 'plain_structured_text';
|
|
482
533
|
return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
|
|
@@ -485,18 +536,19 @@ export class ContentExtractor {
|
|
|
485
536
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
486
537
|
const metadata = { extractionMode: 'diagram_structural', vectorizable: true };
|
|
487
538
|
if (file.format === 'drawio') {
|
|
488
|
-
const
|
|
489
|
-
metadata.nodeTextCount =
|
|
490
|
-
metadata.
|
|
491
|
-
|
|
539
|
+
const graph = this.extractDrawioGraph(raw);
|
|
540
|
+
metadata.nodeTextCount = graph.nodes.length;
|
|
541
|
+
metadata.edgeCount = graph.edges.length;
|
|
542
|
+
metadata.contentCoverage = 'drawio_graph_links';
|
|
543
|
+
return { text: [this.metadataOnlyText(file), 'Draw.io 图链路:', ...graph.links, 'Draw.io 节点:', ...graph.nodes.map(node => `[${node.id}] ${node.label}`)].join('\n'), metadata, warnings: [] };
|
|
492
544
|
}
|
|
493
545
|
if (file.format === 'excalidraw') {
|
|
494
546
|
try {
|
|
495
|
-
const
|
|
496
|
-
|
|
497
|
-
metadata.
|
|
498
|
-
metadata.contentCoverage = '
|
|
499
|
-
return { text: [this.metadataOnlyText(file),
|
|
547
|
+
const graph = this.extractExcalidrawGraph(raw);
|
|
548
|
+
metadata.elementTextCount = graph.nodes.length;
|
|
549
|
+
metadata.edgeCount = graph.links.length;
|
|
550
|
+
metadata.contentCoverage = 'excalidraw_graph_links';
|
|
551
|
+
return { text: [this.metadataOnlyText(file), 'Excalidraw 图链路:', ...graph.links, 'Excalidraw 节点:', ...graph.nodes].join('\n'), metadata, warnings: [] };
|
|
500
552
|
}
|
|
501
553
|
catch {
|
|
502
554
|
metadata.parseError = true;
|
|
@@ -505,6 +557,52 @@ export class ContentExtractor {
|
|
|
505
557
|
metadata.contentCoverage = 'diagram_source_text';
|
|
506
558
|
return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
|
|
507
559
|
}
|
|
560
|
+
extractDrawioGraph(raw) {
|
|
561
|
+
const cells = Array.from(raw.matchAll(/<mxCell\b([^>]*)>/gu), match => this.parseXmlAttributes(match[1] ?? ''));
|
|
562
|
+
const nodes = cells
|
|
563
|
+
.filter(cell => cell.value || cell.label || cell.id)
|
|
564
|
+
.map((cell, index) => ({ id: String(cell.id ?? `node-${index + 1}`), label: this.stripXml(String(cell.value || cell.label || cell.id)) }))
|
|
565
|
+
.filter(node => node.label);
|
|
566
|
+
const nodeById = new Map(nodes.map(node => [node.id, node.label]));
|
|
567
|
+
const edges = cells.filter(cell => cell.edge === '1' && cell.source && cell.target);
|
|
568
|
+
const links = edges.map(edge => {
|
|
569
|
+
const source = nodeById.get(String(edge.source)) ?? String(edge.source);
|
|
570
|
+
const target = nodeById.get(String(edge.target)) ?? String(edge.target);
|
|
571
|
+
const label = this.stripXml(String(edge.value || edge.label || '关系'));
|
|
572
|
+
return `[${source}] ──> (${label}) ──> [${target}]`;
|
|
573
|
+
});
|
|
574
|
+
return { nodes, edges: edges.map(edge => String(edge.id ?? 'edge')), links };
|
|
575
|
+
}
|
|
576
|
+
extractExcalidrawGraph(raw) {
|
|
577
|
+
const parsed = JSON.parse(raw);
|
|
578
|
+
const elements = parsed.elements ?? [];
|
|
579
|
+
const textById = new Map(elements.filter(element => element.text).map(element => [String(element.id), String(element.text)]));
|
|
580
|
+
const shapeLabels = elements
|
|
581
|
+
.filter(element => element.type !== 'arrow' && element.type !== 'line')
|
|
582
|
+
.map((element, index) => {
|
|
583
|
+
const id = String(element.id ?? `node-${index + 1}`);
|
|
584
|
+
const own = element.text ?? textById.get(id);
|
|
585
|
+
const nested = elements.find(candidate => candidate.text && this.isPointInside(candidate, element));
|
|
586
|
+
return { id, label: own ?? nested?.text ?? element.type ?? '节点' };
|
|
587
|
+
});
|
|
588
|
+
const labelById = new Map(shapeLabels.map(node => [node.id, node.label]));
|
|
589
|
+
const links = elements
|
|
590
|
+
.filter(element => element.type === 'arrow' && element.startBinding?.elementId && element.endBinding?.elementId)
|
|
591
|
+
.map(element => `[${labelById.get(element.startBinding.elementId) ?? element.startBinding.elementId}] ──> (箭头) ──> [${labelById.get(element.endBinding.elementId) ?? element.endBinding.elementId}]`);
|
|
592
|
+
const nodes = shapeLabels.map(node => `[${node.id}] ${node.label}`);
|
|
593
|
+
return { nodes, links };
|
|
594
|
+
}
|
|
595
|
+
isPointInside(point, box) {
|
|
596
|
+
if (point.x == null || point.y == null || box.x == null || box.y == null || box.width == null || box.height == null)
|
|
597
|
+
return false;
|
|
598
|
+
return point.x >= box.x && point.x <= box.x + box.width && point.y >= box.y && point.y <= box.y + box.height;
|
|
599
|
+
}
|
|
600
|
+
parseXmlAttributes(input) {
|
|
601
|
+
const attrs = {};
|
|
602
|
+
for (const match of input.matchAll(/([\w:-]+)="([^"]*)"/gu))
|
|
603
|
+
attrs[match[1]] = this.stripXml(match[2] ?? '');
|
|
604
|
+
return attrs;
|
|
605
|
+
}
|
|
508
606
|
parseDelimitedLine(line, delimiter) {
|
|
509
607
|
const values = [];
|
|
510
608
|
let current = '';
|
|
@@ -531,28 +629,38 @@ export class ContentExtractor {
|
|
|
531
629
|
values.push(current.trim());
|
|
532
630
|
return values;
|
|
533
631
|
}
|
|
632
|
+
toMarkdownTable(header, rows) {
|
|
633
|
+
const width = Math.max(header.length, ...rows.map(row => row.length), 1);
|
|
634
|
+
const normalizedHeader = Array.from({ length: width }, (_, index) => header[index] || `COL${index + 1}`);
|
|
635
|
+
const escape = (value) => String(value ?? '').replace(/\|/gu, '\\|').replace(/\r?\n/gu, ' ').trim();
|
|
636
|
+
return [
|
|
637
|
+
`| ${normalizedHeader.map(escape).join(' | ')} |`,
|
|
638
|
+
`| ${normalizedHeader.map(() => '---').join(' | ')} |`,
|
|
639
|
+
...rows.map(row => `| ${Array.from({ length: width }, (_, index) => escape(row[index] ?? '')).join(' | ')} |`),
|
|
640
|
+
].join('\n');
|
|
641
|
+
}
|
|
534
642
|
extractDelimitedText(file) {
|
|
535
643
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
536
644
|
const delimiter = file.format === 'tsv' ? '\t' : ',';
|
|
537
645
|
const rows = raw.split(/\r?\n/u).filter(line => line.trim().length > 0);
|
|
538
646
|
const header = rows[0] ? this.parseDelimitedLine(rows[0], delimiter) : [];
|
|
539
|
-
const
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
});
|
|
647
|
+
const tableRows = rows.slice(1).map(line => this.parseDelimitedLine(line, delimiter));
|
|
648
|
+
const markdown = this.toMarkdownTable(header, tableRows);
|
|
649
|
+
const legacyKv = tableRows.flatMap((values, rowIndex) => values.map((value, colIndex) => {
|
|
650
|
+
const column = header[colIndex] || `COL${colIndex + 1}`;
|
|
651
|
+
return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
|
|
652
|
+
}));
|
|
546
653
|
return {
|
|
547
|
-
text: [this.metadataOnlyText(file),
|
|
654
|
+
text: [this.metadataOnlyText(file), '### 工作表: 默认', markdown, '### 表格路径声明', ...legacyKv].join('\n\n'),
|
|
548
655
|
metadata: {
|
|
549
656
|
extractionMode: 'delimited_text_structured',
|
|
657
|
+
semanticExtractionMode: 'delimited_markdown_table',
|
|
550
658
|
vectorizable: true,
|
|
551
659
|
delimiter: file.format === 'tsv' ? 'tab' : 'comma',
|
|
552
660
|
rowCount: rows.length,
|
|
553
661
|
columnCount: header.length,
|
|
554
662
|
columnNames: header.slice(0, 120),
|
|
555
|
-
contentCoverage: '
|
|
663
|
+
contentCoverage: 'markdown_table',
|
|
556
664
|
},
|
|
557
665
|
warnings: [],
|
|
558
666
|
};
|
|
@@ -567,19 +675,27 @@ export class ContentExtractor {
|
|
|
567
675
|
return this.extractLegacyOfficeBinary(file);
|
|
568
676
|
if (ext === '.docx') {
|
|
569
677
|
try {
|
|
678
|
+
const styledMarkdown = await this.extractDocxStyleTreeMarkdown(file.absolutePath);
|
|
679
|
+
if (styledMarkdown.trim()) {
|
|
680
|
+
return {
|
|
681
|
+
text: styledMarkdown,
|
|
682
|
+
metadata: { extractionMode: 'docx_xml_style_tree_markdown', vectorizable: true, contentCoverage: 'office_style_tree_markdown' },
|
|
683
|
+
warnings: [],
|
|
684
|
+
};
|
|
685
|
+
}
|
|
570
686
|
const mammoth = await resolveAndImport('mammoth');
|
|
571
|
-
const result = await mammoth.
|
|
572
|
-
const text = result.value.trim();
|
|
687
|
+
const result = await mammoth.convertToMarkdown({ path: file.absolutePath });
|
|
688
|
+
const text = String(result.value ?? '').trim();
|
|
573
689
|
if (text) {
|
|
574
690
|
return {
|
|
575
|
-
text,
|
|
576
|
-
metadata: { extractionMode: '
|
|
691
|
+
text: this.normalizeMarkdownHeadings(text),
|
|
692
|
+
metadata: { extractionMode: 'builtin_mammoth_markdown', vectorizable: true, contentCoverage: 'office_markdown_structure' },
|
|
577
693
|
warnings: result.messages.map(m => m.message),
|
|
578
694
|
};
|
|
579
695
|
}
|
|
580
696
|
}
|
|
581
697
|
catch {
|
|
582
|
-
//
|
|
698
|
+
// 降级到下方 Office ZIP 解析
|
|
583
699
|
}
|
|
584
700
|
}
|
|
585
701
|
return this.extractOfficeZip(file);
|
|
@@ -630,6 +746,14 @@ export class ContentExtractor {
|
|
|
630
746
|
warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库'],
|
|
631
747
|
};
|
|
632
748
|
}
|
|
749
|
+
findMergedCellValue(sheet, merges, row, col, XLSX) {
|
|
750
|
+
const merge = merges.find(item => row >= item.s.r && row <= item.e.r && col >= item.s.c && col <= item.e.c);
|
|
751
|
+
if (!merge)
|
|
752
|
+
return undefined;
|
|
753
|
+
const originAddress = XLSX.utils.encode_cell({ r: merge.s.r, c: merge.s.c });
|
|
754
|
+
const origin = sheet[originAddress];
|
|
755
|
+
return origin?.w ?? (origin?.v == null ? undefined : String(origin.v));
|
|
756
|
+
}
|
|
633
757
|
async extractSpreadsheet(file) {
|
|
634
758
|
const ext = path.extname(file.absolutePath).toLowerCase();
|
|
635
759
|
try {
|
|
@@ -644,28 +768,32 @@ export class ContentExtractor {
|
|
|
644
768
|
if (!sheet || !sheet['!ref'])
|
|
645
769
|
continue;
|
|
646
770
|
const range = XLSX.utils.decode_range(sheet['!ref']);
|
|
647
|
-
const lines = [`工作表: ${name}`, `范围: ${sheet['!ref']}`];
|
|
648
771
|
const merges = sheet['!merges'] ?? [];
|
|
649
772
|
mergeCount += merges.length;
|
|
650
|
-
|
|
651
|
-
lines.push(`合并单元格: ${merges.map(item => `${XLSX.utils.encode_cell(item.s)}:${XLSX.utils.encode_cell(item.e)}`).join(', ')}`);
|
|
652
|
-
}
|
|
773
|
+
const matrix = [];
|
|
653
774
|
for (let row = range.s.r; row <= range.e.r; row++) {
|
|
775
|
+
const values = [];
|
|
654
776
|
for (let col = range.s.c; col <= range.e.c; col++) {
|
|
655
777
|
const address = XLSX.utils.encode_cell({ r: row, c: col });
|
|
656
778
|
const cell = sheet[address];
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
const
|
|
664
|
-
|
|
779
|
+
const mergedValue = cell ? undefined : this.findMergedCellValue(sheet, merges, row, col, XLSX);
|
|
780
|
+
if (cell) {
|
|
781
|
+
cellCount++;
|
|
782
|
+
if (cell.f)
|
|
783
|
+
formulaCount++;
|
|
784
|
+
}
|
|
785
|
+
const display = cell?.w ?? String(cell?.v ?? mergedValue ?? '');
|
|
786
|
+
const formula = cell?.f ? ` 公式=${cell.f}` : '';
|
|
787
|
+
values.push(`${display}${formula}`.trim());
|
|
665
788
|
}
|
|
789
|
+
if (values.some(Boolean))
|
|
790
|
+
matrix.push(values);
|
|
791
|
+
}
|
|
792
|
+
if (matrix.length > 0) {
|
|
793
|
+
const header = matrix[0] ?? [];
|
|
794
|
+
const rows = matrix.slice(1);
|
|
795
|
+
sheetTexts.push([`### 工作表: ${name}`, this.toMarkdownTable(header, rows)].join('\n\n'));
|
|
666
796
|
}
|
|
667
|
-
if (lines.length > 2)
|
|
668
|
-
sheetTexts.push(lines.slice(0, 5_000).join('\n'));
|
|
669
797
|
}
|
|
670
798
|
if (sheetTexts.length > 0) {
|
|
671
799
|
return {
|
|
@@ -678,12 +806,47 @@ export class ContentExtractor {
|
|
|
678
806
|
catch {
|
|
679
807
|
if (ext === '.xls')
|
|
680
808
|
return this.extractLegacyOfficeBinary(file);
|
|
681
|
-
//
|
|
809
|
+
// 降级到下方 ZIP 文件内容提取
|
|
682
810
|
}
|
|
683
811
|
if (ext === '.xls')
|
|
684
812
|
return this.extractLegacyOfficeBinary(file);
|
|
685
813
|
return this.extractOfficeZip(file);
|
|
686
814
|
}
|
|
815
|
+
async extractDocxStyleTreeMarkdown(filePath) {
|
|
816
|
+
const jszipMod = await resolveAndImport('jszip');
|
|
817
|
+
const JSZip = jszipMod.default ?? jszipMod;
|
|
818
|
+
const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
|
|
819
|
+
const docXml = await zip.files['word/document.xml']?.async('string');
|
|
820
|
+
if (!docXml)
|
|
821
|
+
return '';
|
|
822
|
+
const paragraphs = Array.from(docXml.matchAll(/<w:p[\s\S]*?<\/w:p>/gu), match => match[0]);
|
|
823
|
+
const lines = [];
|
|
824
|
+
for (const paragraph of paragraphs) {
|
|
825
|
+
const texts = Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
|
|
826
|
+
if (!texts.trim())
|
|
827
|
+
continue;
|
|
828
|
+
const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(paragraph)?.[1] ?? '';
|
|
829
|
+
const bold = /<w:b\b/u.test(paragraph);
|
|
830
|
+
const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(paragraph)?.[1] ?? 0);
|
|
831
|
+
const level = this.docxHeadingLevel(style, bold, size, texts);
|
|
832
|
+
lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
|
|
833
|
+
}
|
|
834
|
+
return lines.join('\n\n');
|
|
835
|
+
}
|
|
836
|
+
docxHeadingLevel(style, bold, size, text) {
|
|
837
|
+
const normalized = style.toLowerCase();
|
|
838
|
+
const heading = /heading(\d)|标题(\d)|h(\d)/iu.exec(normalized);
|
|
839
|
+
const styleLevel = Number(heading?.[1] ?? heading?.[2] ?? heading?.[3] ?? 0);
|
|
840
|
+
if (styleLevel >= 1 && styleLevel <= 6)
|
|
841
|
+
return styleLevel;
|
|
842
|
+
if (text.length <= 100 && size >= 32)
|
|
843
|
+
return 1;
|
|
844
|
+
if (text.length <= 100 && size >= 28)
|
|
845
|
+
return 2;
|
|
846
|
+
if (text.length <= 100 && (size >= 24 || bold))
|
|
847
|
+
return 3;
|
|
848
|
+
return 0;
|
|
849
|
+
}
|
|
687
850
|
async extractOfficeZip(file) {
|
|
688
851
|
const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
|
|
689
852
|
try {
|
|
@@ -709,26 +872,6 @@ export class ContentExtractor {
|
|
|
709
872
|
return { text: '', metadata, warnings: [`Office/表格/演示文件解析失败: ${metadata.parseError},未入库`] };
|
|
710
873
|
}
|
|
711
874
|
}
|
|
712
|
-
async extractArchive(file) {
|
|
713
|
-
const metadata = { extractionMode: 'archive_manifest', vectorizable: true };
|
|
714
|
-
if (path.extname(file.absolutePath).toLowerCase() !== '.zip') {
|
|
715
|
-
metadata.contentCoverage = 'metadata_filename';
|
|
716
|
-
return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
|
|
717
|
-
}
|
|
718
|
-
try {
|
|
719
|
-
const jszipMod = await resolveAndImport('jszip');
|
|
720
|
-
const JSZip = jszipMod.default ?? jszipMod;
|
|
721
|
-
const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
|
|
722
|
-
const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
|
|
723
|
-
metadata.entryCount = Object.keys(zip.files).length;
|
|
724
|
-
metadata.contentCoverage = 'zip_manifest';
|
|
725
|
-
return { text: [this.metadataOnlyText(file), '压缩包文件清单:', ...entries].join('\n'), metadata, warnings: [] };
|
|
726
|
-
}
|
|
727
|
-
catch (error) {
|
|
728
|
-
metadata.parseError = error instanceof Error ? error.message : String(error);
|
|
729
|
-
return { text: '', metadata, warnings: ['压缩包解析失败,内置解析器未提取到文件清单,未入库'] };
|
|
730
|
-
}
|
|
731
|
-
}
|
|
732
875
|
async extractRasterImage(file) {
|
|
733
876
|
const metadata = { extractionMode: 'builtin_tesseract_ocr_isolated', vectorizable: true };
|
|
734
877
|
if (process.env.CUSTOMIZE_AGENT_DISABLE_OCR === '1') {
|
|
@@ -742,6 +885,13 @@ export class ContentExtractor {
|
|
|
742
885
|
metadata.parseError = validationError;
|
|
743
886
|
return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
|
|
744
887
|
}
|
|
888
|
+
const paddle = await this.tryPaddleOcrLayout(file.absolutePath);
|
|
889
|
+
if (paddle) {
|
|
890
|
+
metadata.contentCoverage = 'paddleocr_layout_regions';
|
|
891
|
+
metadata.ocrProvider = 'paddleocr';
|
|
892
|
+
metadata.ocrRegionCount = paddle.regionCount;
|
|
893
|
+
return { text: [this.metadataOnlyText(file), paddle.text].join('\n'), metadata, warnings: [] };
|
|
894
|
+
}
|
|
745
895
|
let tesseractPath;
|
|
746
896
|
try {
|
|
747
897
|
tesseractPath = resolvePackage('tesseract.js');
|
|
@@ -766,7 +916,12 @@ const { createWorker } = tesseractMod;
|
|
|
766
916
|
const worker = await createWorker('chi_sim+eng');
|
|
767
917
|
try {
|
|
768
918
|
const result = await worker.recognize(process.argv[1]);
|
|
769
|
-
|
|
919
|
+
const lines = (result.data.lines || []).map((line, index) => ({
|
|
920
|
+
index: index + 1,
|
|
921
|
+
text: line.text || '',
|
|
922
|
+
bbox: line.bbox || line.baseline || null,
|
|
923
|
+
})).filter(line => line.text.trim());
|
|
924
|
+
process.stdout.write(JSON.stringify({ text: result.data.text || '', lines }));
|
|
770
925
|
} finally {
|
|
771
926
|
await worker.terminate();
|
|
772
927
|
}`,
|
|
@@ -778,17 +933,73 @@ try {
|
|
|
778
933
|
metadata.parseError = message;
|
|
779
934
|
return { text: '', metadata, warnings: [`内置 OCR 解析失败:${message},未入库`] };
|
|
780
935
|
}
|
|
781
|
-
const
|
|
782
|
-
|
|
936
|
+
const parsed = this.parseOcrJson(result.stdout);
|
|
937
|
+
const text = parsed.text.trim();
|
|
938
|
+
const regions = this.formatOcrRegions(parsed.lines);
|
|
939
|
+
metadata.contentCoverage = text ? 'ocr_text_bounding_boxes' : 'metadata_filename';
|
|
783
940
|
metadata.ocrProvider = 'tesseract.js';
|
|
784
941
|
metadata.ocrLanguages = 'chi_sim+eng';
|
|
785
942
|
metadata.ocrTextLength = text.length;
|
|
943
|
+
metadata.ocrLineCount = parsed.lines.length;
|
|
786
944
|
return {
|
|
787
|
-
text: text ? [this.metadataOnlyText(file), `OCR
|
|
945
|
+
text: text ? [this.metadataOnlyText(file), 'OCR 区域文本:', ...regions, `OCR 完整文本:\n${text}`].join('\n') : '',
|
|
788
946
|
metadata,
|
|
789
947
|
warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
|
|
790
948
|
};
|
|
791
949
|
}
|
|
950
|
+
async tryPaddleOcrLayout(filePath) {
|
|
951
|
+
const command = process.env.CUSTOMIZE_PADDLE_OCR_CMD || process.env.PADDLE_OCR_CMD;
|
|
952
|
+
if (!command)
|
|
953
|
+
return undefined;
|
|
954
|
+
const result = spawnSync(command, [filePath], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, shell: true });
|
|
955
|
+
if (result.status !== 0 || !result.stdout.trim())
|
|
956
|
+
return undefined;
|
|
957
|
+
try {
|
|
958
|
+
const parsed = JSON.parse(result.stdout);
|
|
959
|
+
const lines = parsed.map((region, index) => `区域 ${index + 1} [${region.type ?? 'text'}] ${this.formatBoundingBox(region.bbox)}: ${region.text ?? ''}`);
|
|
960
|
+
return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
|
|
961
|
+
}
|
|
962
|
+
catch {
|
|
963
|
+
const lines = result.stdout.split(/\r?\n/u).filter(Boolean);
|
|
964
|
+
return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
parseOcrJson(raw) {
|
|
968
|
+
try {
|
|
969
|
+
const parsed = JSON.parse(raw);
|
|
970
|
+
return {
|
|
971
|
+
text: parsed.text ?? raw,
|
|
972
|
+
lines: (parsed.lines ?? []).map((line, index) => ({ index: line.index ?? index + 1, text: line.text ?? '', bbox: line.bbox })).filter(line => line.text.trim()),
|
|
973
|
+
};
|
|
974
|
+
}
|
|
975
|
+
catch {
|
|
976
|
+
return { text: raw, lines: raw.split(/\r?\n/u).map((text, index) => ({ index: index + 1, text })).filter(line => line.text.trim()) };
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
formatOcrRegions(lines) {
|
|
980
|
+
return lines.map(line => {
|
|
981
|
+
const bbox = this.formatBoundingBox(line.bbox);
|
|
982
|
+
const type = this.classifyOcrRegion(line.text);
|
|
983
|
+
return `区域 ${line.index} [${type}]${bbox ? ` ${bbox}` : ''}: ${line.text}`;
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
classifyOcrRegion(text) {
|
|
987
|
+
if (/\|/.test(text) || /\s{2,}/u.test(text) || /表\s*\d|合计|小计/u.test(text))
|
|
988
|
+
return 'table';
|
|
989
|
+
if (/图\s*\d|figure|image|示意图/iu.test(text))
|
|
990
|
+
return 'image-caption';
|
|
991
|
+
return 'text';
|
|
992
|
+
}
|
|
993
|
+
formatBoundingBox(value) {
|
|
994
|
+
if (!value || typeof value !== 'object')
|
|
995
|
+
return '';
|
|
996
|
+
const record = value;
|
|
997
|
+
const x0 = record.x0 ?? record.left ?? record.x;
|
|
998
|
+
const y0 = record.y0 ?? record.top ?? record.y;
|
|
999
|
+
const x1 = record.x1 ?? record.right;
|
|
1000
|
+
const y1 = record.y1 ?? record.bottom;
|
|
1001
|
+
return [x0, y0, x1, y1].some(item => item != null) ? `[bbox x0=${x0 ?? ''}, y0=${y0 ?? ''}, x1=${x1 ?? ''}, y1=${y1 ?? ''}]` : '';
|
|
1002
|
+
}
|
|
792
1003
|
validateRasterImage(filePath) {
|
|
793
1004
|
const buffer = fs.readFileSync(filePath);
|
|
794
1005
|
if (buffer.length < 12)
|
|
@@ -816,25 +1027,25 @@ try {
|
|
|
816
1027
|
async extractPdf(file) {
|
|
817
1028
|
const metadata = { extractionMode: 'pdf_text', vectorizable: true };
|
|
818
1029
|
const warnings = [];
|
|
819
|
-
//
|
|
1030
|
+
// 第一层:pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
|
|
820
1031
|
try {
|
|
821
1032
|
const raw = fs.readFileSync(file.absolutePath);
|
|
822
1033
|
const text = await this.extractPdfText(raw);
|
|
823
1034
|
if (text.trim()) {
|
|
824
|
-
metadata.contentCoverage = '
|
|
1035
|
+
metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
|
|
825
1036
|
metadata.pdfExtractor = 'pdfjs-dist';
|
|
826
|
-
return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
|
|
1037
|
+
return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
|
|
827
1038
|
}
|
|
828
1039
|
}
|
|
829
1040
|
catch (error) {
|
|
830
1041
|
warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
|
|
831
1042
|
metadata.parseError = error instanceof Error ? error.message : String(error);
|
|
832
1043
|
}
|
|
833
|
-
//
|
|
1044
|
+
// 第二层:OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
|
|
834
1045
|
const ocr = await this.extractScannedPdfOcr(file);
|
|
835
1046
|
if (ocr.text.trim())
|
|
836
1047
|
return ocr;
|
|
837
|
-
//
|
|
1048
|
+
// 第三层:仅索引元数据(兜底)
|
|
838
1049
|
metadata.extractionMode = 'pdf_metadata_only';
|
|
839
1050
|
metadata.contentCoverage = 'metadata_filename';
|
|
840
1051
|
metadata.ocrRecommended = true;
|
|
@@ -846,10 +1057,6 @@ try {
|
|
|
846
1057
|
warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
|
|
847
1058
|
};
|
|
848
1059
|
}
|
|
849
|
-
pdfOcrPageLimit() {
|
|
850
|
-
const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
|
|
851
|
-
return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
|
|
852
|
-
}
|
|
853
1060
|
async extractScannedPdfOcr(file) {
|
|
854
1061
|
const metadata = {
|
|
855
1062
|
extractionMode: 'pdf_page_ocr_embedded',
|
|
@@ -858,9 +1065,8 @@ try {
|
|
|
858
1065
|
ocrProvider: 'tesseract.js',
|
|
859
1066
|
ocrLanguages: 'chi_sim+eng',
|
|
860
1067
|
pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
|
|
861
|
-
pdfOcrPageLimit:
|
|
1068
|
+
pdfOcrPageLimit: 'all',
|
|
862
1069
|
};
|
|
863
|
-
const pageLimitConfig = this.pdfOcrPageLimit();
|
|
864
1070
|
// 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
|
|
865
1071
|
let canvasPath;
|
|
866
1072
|
let pdfjsPath;
|
|
@@ -890,10 +1096,9 @@ import { createCanvas } from ${JSON.stringify(canvasPath)};
|
|
|
890
1096
|
import * as pdfjs from ${JSON.stringify(pdfjsPath)};
|
|
891
1097
|
import { createWorker } from ${JSON.stringify(tesseractPath)};
|
|
892
1098
|
const filePath = process.argv[1];
|
|
893
|
-
const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
|
|
894
1099
|
const bytes = new Uint8Array(fs.readFileSync(filePath));
|
|
895
1100
|
const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
|
|
896
|
-
const pageLimit =
|
|
1101
|
+
const pageLimit = doc.numPages;
|
|
897
1102
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
|
|
898
1103
|
const worker = await createWorker('chi_sim+eng');
|
|
899
1104
|
const pages = [];
|
|
@@ -908,7 +1113,8 @@ try {
|
|
|
908
1113
|
fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
|
|
909
1114
|
const recognized = await worker.recognize(imagePath);
|
|
910
1115
|
const text = (recognized.data.text || '').trim();
|
|
911
|
-
|
|
1116
|
+
const lines = (recognized.data.lines || []).map((line, index) => ({ index: index + 1, text: line.text || '', bbox: line.bbox || null })).filter(line => line.text.trim());
|
|
1117
|
+
if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + lines.map(line => '区域 ' + line.index + ' ' + JSON.stringify(line.bbox || {}) + ': ' + line.text).join('\\n') + '\\n\\n完整文本:\\n' + text);
|
|
912
1118
|
}
|
|
913
1119
|
} finally {
|
|
914
1120
|
await worker.terminate();
|
|
@@ -916,8 +1122,7 @@ try {
|
|
|
916
1122
|
}
|
|
917
1123
|
process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
|
|
918
1124
|
file.absolutePath,
|
|
919
|
-
|
|
920
|
-
], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
|
|
1125
|
+
], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, env: childEnv });
|
|
921
1126
|
if (result.status !== 0 || result.error) {
|
|
922
1127
|
metadata.ocrRecommended = true;
|
|
923
1128
|
metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
|
|
@@ -927,7 +1132,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
927
1132
|
const parsed = JSON.parse(result.stdout);
|
|
928
1133
|
const text = parsed.text?.trim() ?? '';
|
|
929
1134
|
metadata.ocrPageCount = parsed.pageCount ?? 0;
|
|
930
|
-
metadata.pdfOcrPageLimit = parsed.pageLimit ??
|
|
1135
|
+
metadata.pdfOcrPageLimit = parsed.pageLimit ?? 'all';
|
|
931
1136
|
metadata.ocrTextLength = text.length;
|
|
932
1137
|
metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
|
|
933
1138
|
return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
|
|
@@ -939,23 +1144,20 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
939
1144
|
}
|
|
940
1145
|
}
|
|
941
1146
|
async extractPdfText(buffer) {
|
|
942
|
-
//
|
|
1147
|
+
// 第一层:pdfjs-dist 文本提取(处理压缩内容流、CJK 字体、现代 PDF)
|
|
943
1148
|
try {
|
|
944
1149
|
const mod = await resolveAndImport('pdfjs-dist/legacy/build/pdf.mjs');
|
|
945
1150
|
const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
|
|
946
1151
|
const doc = await loadingTask.promise;
|
|
947
1152
|
const pages = [];
|
|
948
|
-
const pageLimit =
|
|
1153
|
+
const pageLimit = doc.numPages;
|
|
949
1154
|
for (let i = 1; i <= pageLimit; i++) {
|
|
950
1155
|
const page = await doc.getPage(i);
|
|
951
1156
|
const content = await page.getTextContent();
|
|
952
|
-
const
|
|
953
|
-
.map((item) =>
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
})
|
|
957
|
-
.filter((s) => s.trim().length > 0)
|
|
958
|
-
.join(' ');
|
|
1157
|
+
const items = content.items
|
|
1158
|
+
.map((item) => this.toPdfTextItem(item))
|
|
1159
|
+
.filter((item) => !!item && item.str.trim().length > 0);
|
|
1160
|
+
const pageText = this.layoutPdfTextItems(items, i);
|
|
959
1161
|
if (pageText.trim())
|
|
960
1162
|
pages.push(pageText.trim());
|
|
961
1163
|
}
|
|
@@ -970,7 +1172,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
970
1172
|
if (process.env.KB_DEBUG === '1')
|
|
971
1173
|
console.warn('[kb] pdfjs-dist extraction failed:', e.message);
|
|
972
1174
|
}
|
|
973
|
-
//
|
|
1175
|
+
// 第二层:pdf-parse(兼容旧版 PDF)
|
|
974
1176
|
try {
|
|
975
1177
|
const mod = await resolveAndImport('pdf-parse');
|
|
976
1178
|
const pdfParse = mod.default;
|
|
@@ -981,9 +1183,9 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
981
1183
|
}
|
|
982
1184
|
}
|
|
983
1185
|
catch {
|
|
984
|
-
//
|
|
1186
|
+
// 降级到下方纯正则提取
|
|
985
1187
|
}
|
|
986
|
-
//
|
|
1188
|
+
// 第三层:raw regex 回退(未压缩的古老 PDF)
|
|
987
1189
|
const raw = buffer.toString('latin1');
|
|
988
1190
|
const matches = Array.from(raw.matchAll(/\(([^()]{2,500})\)\s*T[jJ]/gu), match => match[1] ?? '')
|
|
989
1191
|
.concat(Array.from(raw.matchAll(/\[([^\]]{2,2000})\]\s*TJ/gu), match => match[1] ?? ''));
|
|
@@ -998,16 +1200,166 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
998
1200
|
.join('')
|
|
999
1201
|
.trim();
|
|
1000
1202
|
}
|
|
1203
|
+
toPdfTextItem(item) {
|
|
1204
|
+
if (!item || typeof item !== 'object' || !('str' in item))
|
|
1205
|
+
return undefined;
|
|
1206
|
+
const record = item;
|
|
1207
|
+
const transform = Array.isArray(record.transform) ? record.transform : [];
|
|
1208
|
+
return {
|
|
1209
|
+
str: String(record.str ?? ''),
|
|
1210
|
+
x: Number(transform[4] ?? 0),
|
|
1211
|
+
y: Number(transform[5] ?? 0),
|
|
1212
|
+
width: Number(record.width ?? 0),
|
|
1213
|
+
height: Number(record.height ?? Math.abs(Number(transform[3] ?? 0))),
|
|
1214
|
+
fontName: typeof record.fontName === 'string' ? record.fontName : undefined,
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
layoutPdfTextItems(items, pageNumber) {
|
|
1218
|
+
if (items.length === 0)
|
|
1219
|
+
return '';
|
|
1220
|
+
const rows = this.groupPdfItemsIntoRows(items);
|
|
1221
|
+
const columnSplit = this.detectPdfColumnSplit(rows);
|
|
1222
|
+
const orderedRows = columnSplit == null
|
|
1223
|
+
? rows.sort((a, b) => b.y - a.y || a.x - b.x)
|
|
1224
|
+
: [
|
|
1225
|
+
...rows.filter(row => row.x < columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
|
|
1226
|
+
...rows.filter(row => row.x >= columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
|
|
1227
|
+
];
|
|
1228
|
+
const markdown = this.rowsToPdfMarkdownWithTables(orderedRows);
|
|
1229
|
+
return [`## PDF 第 ${pageNumber} 页`, markdown].join('\n\n');
|
|
1230
|
+
}
|
|
1231
|
+
groupPdfItemsIntoRows(items) {
|
|
1232
|
+
const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1233
|
+
const rows = [];
|
|
1234
|
+
for (const item of sorted) {
|
|
1235
|
+
const row = rows.find(candidate => Math.abs(candidate.y - item.y) <= Math.max(2, item.height * 0.55));
|
|
1236
|
+
if (row)
|
|
1237
|
+
row.items.push(item);
|
|
1238
|
+
else
|
|
1239
|
+
rows.push({ y: item.y, items: [item] });
|
|
1240
|
+
}
|
|
1241
|
+
return rows.map(row => {
|
|
1242
|
+
const rowItems = row.items.sort((a, b) => a.x - b.x);
|
|
1243
|
+
return {
|
|
1244
|
+
text: rowItems.map(item => item.str.trim()).filter(Boolean).join(' ').replace(/\s+/gu, ' '),
|
|
1245
|
+
x: Math.min(...rowItems.map(item => item.x)),
|
|
1246
|
+
y: row.y,
|
|
1247
|
+
height: Math.max(...rowItems.map(item => item.height || 0)),
|
|
1248
|
+
};
|
|
1249
|
+
}).filter(row => row.text);
|
|
1250
|
+
}
|
|
1251
|
+
detectPdfColumnSplit(rows) {
|
|
1252
|
+
if (rows.length < 8)
|
|
1253
|
+
return undefined;
|
|
1254
|
+
const xs = rows.map(row => row.x).sort((a, b) => a - b);
|
|
1255
|
+
const gaps = xs.slice(1).map((x, index) => ({ gap: x - xs[index], left: xs[index], right: x })).sort((a, b) => b.gap - a.gap);
|
|
1256
|
+
const largest = gaps[0];
|
|
1257
|
+
if (!largest || largest.gap < 80)
|
|
1258
|
+
return undefined;
|
|
1259
|
+
const leftCount = rows.filter(row => row.x <= largest.left).length;
|
|
1260
|
+
const rightCount = rows.filter(row => row.x >= largest.right).length;
|
|
1261
|
+
return leftCount >= 3 && rightCount >= 3 ? (largest.left + largest.right) / 2 : undefined;
|
|
1262
|
+
}
|
|
1263
|
+
rowsToPdfMarkdownWithTables(rows) {
|
|
1264
|
+
const output = [];
|
|
1265
|
+
let index = 0;
|
|
1266
|
+
while (index < rows.length) {
|
|
1267
|
+
const tableRows = [];
|
|
1268
|
+
let cursor = index;
|
|
1269
|
+
while (cursor < rows.length) {
|
|
1270
|
+
const cells = this.splitLikelyTableRow(rows[cursor].text);
|
|
1271
|
+
if (cells.length < 2)
|
|
1272
|
+
break;
|
|
1273
|
+
tableRows.push(cells);
|
|
1274
|
+
cursor += 1;
|
|
1275
|
+
}
|
|
1276
|
+
if (tableRows.length >= 2) {
|
|
1277
|
+
output.push('### PDF 表格区域');
|
|
1278
|
+
output.push(this.toMarkdownTable(tableRows[0], tableRows.slice(1)));
|
|
1279
|
+
index = cursor;
|
|
1280
|
+
continue;
|
|
1281
|
+
}
|
|
1282
|
+
output.push(this.pdfRowToMarkdown(rows[index].text, rows[index].height, index));
|
|
1283
|
+
index += 1;
|
|
1284
|
+
}
|
|
1285
|
+
return output.join('\n');
|
|
1286
|
+
}
|
|
1287
|
+
splitLikelyTableRow(text) {
|
|
1288
|
+
const byLargeSpaces = text.split(/\s{2,}/u).map(cell => cell.trim()).filter(Boolean);
|
|
1289
|
+
if (byLargeSpaces.length >= 2)
|
|
1290
|
+
return byLargeSpaces;
|
|
1291
|
+
const byPipes = text.split('|').map(cell => cell.trim()).filter(Boolean);
|
|
1292
|
+
return byPipes.length >= 2 ? byPipes : [];
|
|
1293
|
+
}
|
|
1294
|
+
pdfRowToMarkdown(text, height, index) {
|
|
1295
|
+
if (index === 0 && text.length <= 100)
|
|
1296
|
+
return `# ${text}`;
|
|
1297
|
+
if (height >= 14 && text.length <= 120)
|
|
1298
|
+
return `## ${text}`;
|
|
1299
|
+
if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(text) && text.length <= 120)
|
|
1300
|
+
return `### ${text}`;
|
|
1301
|
+
return text;
|
|
1302
|
+
}
|
|
1303
|
+
toMarkdownDocument(text) {
|
|
1304
|
+
const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
|
|
1305
|
+
return lines.map((line, index) => {
|
|
1306
|
+
if (/^#{1,6}\s/u.test(line) || /^\|/u.test(line))
|
|
1307
|
+
return line;
|
|
1308
|
+
if (line.length <= 80 && !/[。!?.!?]$/u.test(line)) {
|
|
1309
|
+
if (index === 0)
|
|
1310
|
+
return `# ${line}`;
|
|
1311
|
+
if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(line))
|
|
1312
|
+
return `## ${line}`;
|
|
1313
|
+
return `### ${line}`;
|
|
1314
|
+
}
|
|
1315
|
+
return line;
|
|
1316
|
+
}).join('\n\n');
|
|
1317
|
+
}
|
|
1318
|
+
normalizeMarkdownHeadings(text) {
|
|
1319
|
+
return text
|
|
1320
|
+
.split(/\r?\n/u)
|
|
1321
|
+
.map(line => {
|
|
1322
|
+
const trimmed = line.trim();
|
|
1323
|
+
if (!trimmed)
|
|
1324
|
+
return '';
|
|
1325
|
+
if (/^#{1,6}\s/u.test(trimmed) || /^\|/u.test(trimmed))
|
|
1326
|
+
return trimmed;
|
|
1327
|
+
if (trimmed.length <= 80 && /^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(trimmed))
|
|
1328
|
+
return `## ${trimmed}`;
|
|
1329
|
+
return trimmed;
|
|
1330
|
+
})
|
|
1331
|
+
.join('\n');
|
|
1332
|
+
}
|
|
1001
1333
|
extractSvg(file) {
|
|
1002
1334
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
1003
|
-
const
|
|
1004
|
-
const titles = this.matchAll(raw, /<(?:title|desc)\b[^>]*>([\s\S]*?)<\/(?:title|desc)>/giu).map(value => this.stripXml(value)).slice(0, 100);
|
|
1335
|
+
const nodes = this.extractSvgSemanticNodes(raw);
|
|
1005
1336
|
return {
|
|
1006
|
-
text: [this.metadataOnlyText(file),
|
|
1007
|
-
metadata: { extractionMode: 'svg_text_nodes', vectorizable: true, textNodeCount:
|
|
1337
|
+
text: [this.metadataOnlyText(file), 'SVG 层级语义节点:', ...nodes].join('\n'),
|
|
1338
|
+
metadata: { extractionMode: 'svg_text_nodes', semanticExtractionMode: 'svg_semantic_tree_nodes', vectorizable: true, textNodeCount: nodes.length, contentCoverage: 'svg_hierarchical_text_title_desc' },
|
|
1008
1339
|
warnings: [],
|
|
1009
1340
|
};
|
|
1010
1341
|
}
|
|
1342
|
+
extractSvgSemanticNodes(raw) {
|
|
1343
|
+
const nodes = [];
|
|
1344
|
+
const stack = [];
|
|
1345
|
+
const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b([^>]*)>|([^<>]+)/gu;
|
|
1346
|
+
for (const match of raw.matchAll(tokenPattern)) {
|
|
1347
|
+
const tag = match[1];
|
|
1348
|
+
const attrs = match[2] ?? '';
|
|
1349
|
+
const text = match[3]?.replace(/\s+/gu, ' ').trim();
|
|
1350
|
+
const token = match[0];
|
|
1351
|
+
if (tag && token.startsWith('</'))
|
|
1352
|
+
stack.pop();
|
|
1353
|
+
else if (tag && !token.endsWith('/>')) {
|
|
1354
|
+
const id = /\bid="([^"]+)"/u.exec(attrs)?.[1];
|
|
1355
|
+
stack.push(id ? `${tag}#${id}` : tag);
|
|
1356
|
+
}
|
|
1357
|
+
else if (text && ['text', 'title', 'desc'].includes(stack.at(-1)?.split('#')[0] ?? '')) {
|
|
1358
|
+
nodes.push(`${stack.join(' > ')}: ${this.stripXml(text)}`);
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
return nodes;
|
|
1362
|
+
}
|
|
1011
1363
|
isTextReadable(file) {
|
|
1012
1364
|
if (file.category === 'code' || file.category === 'web')
|
|
1013
1365
|
return true;
|
|
@@ -1042,13 +1394,61 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
1042
1394
|
return [`${prefix || 'value'}: ${String(value)}`];
|
|
1043
1395
|
}
|
|
1044
1396
|
if (Array.isArray(value)) {
|
|
1045
|
-
return value.
|
|
1397
|
+
return value.flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
|
|
1046
1398
|
}
|
|
1047
1399
|
if (typeof value === 'object') {
|
|
1048
1400
|
return Object.entries(value).flatMap(([key, item]) => this.flattenJson(item, prefix ? `${prefix}.${key}` : key));
|
|
1049
1401
|
}
|
|
1050
1402
|
return [];
|
|
1051
1403
|
}
|
|
1404
|
+
atomicJsonObjects(value, prefix = '$') {
|
|
1405
|
+
if (value == null || typeof value !== 'object')
|
|
1406
|
+
return [];
|
|
1407
|
+
if (Array.isArray(value))
|
|
1408
|
+
return value.flatMap((item, index) => this.atomicJsonObjects(item, `${prefix}[${index}]`));
|
|
1409
|
+
const entries = Object.entries(value);
|
|
1410
|
+
const current = `${prefix}: ${JSON.stringify(value)}`;
|
|
1411
|
+
const children = entries.flatMap(([key, item]) => this.atomicJsonObjects(item, `${prefix}.${key}`));
|
|
1412
|
+
return [current, ...children];
|
|
1413
|
+
}
|
|
1414
|
+
flattenYamlByIndent(raw) {
|
|
1415
|
+
const stack = [];
|
|
1416
|
+
const lines = [];
|
|
1417
|
+
for (const line of raw.split(/\r?\n/u)) {
|
|
1418
|
+
if (!line.trim() || /^\s*#/u.test(line))
|
|
1419
|
+
continue;
|
|
1420
|
+
const match = /^(\s*)([-\w.]+)\s*:\s*(.*)$/u.exec(line);
|
|
1421
|
+
if (!match)
|
|
1422
|
+
continue;
|
|
1423
|
+
const indent = match[1].length;
|
|
1424
|
+
const key = match[2];
|
|
1425
|
+
const value = match[3].trim();
|
|
1426
|
+
while (stack.length > 0 && stack.at(-1).indent >= indent)
|
|
1427
|
+
stack.pop();
|
|
1428
|
+
const pathName = [...stack.map(item => item.key), key].join('.');
|
|
1429
|
+
if (value)
|
|
1430
|
+
lines.push(`${pathName}: ${value}`);
|
|
1431
|
+
stack.push({ indent, key });
|
|
1432
|
+
}
|
|
1433
|
+
return lines;
|
|
1434
|
+
}
|
|
1435
|
+
flattenXmlPaths(raw) {
|
|
1436
|
+
const lines = [];
|
|
1437
|
+
const stack = [];
|
|
1438
|
+
const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b[^>]*>|([^<>]+)/gu;
|
|
1439
|
+
for (const match of raw.matchAll(tokenPattern)) {
|
|
1440
|
+
const tag = match[1];
|
|
1441
|
+
const text = match[2]?.replace(/\s+/gu, ' ').trim();
|
|
1442
|
+
const token = match[0];
|
|
1443
|
+
if (tag && token.startsWith('</'))
|
|
1444
|
+
stack.pop();
|
|
1445
|
+
else if (tag && !token.endsWith('/>'))
|
|
1446
|
+
stack.push(tag);
|
|
1447
|
+
else if (text && stack.length > 0)
|
|
1448
|
+
lines.push(`${stack.join('.')}: ${this.stripXml(text)}`);
|
|
1449
|
+
}
|
|
1450
|
+
return lines.filter(line => !line.endsWith(':'));
|
|
1451
|
+
}
|
|
1052
1452
|
metadataOnlyText(file) {
|
|
1053
1453
|
const fileName = path.basename(file.relativePath);
|
|
1054
1454
|
const directory = path.dirname(file.relativePath);
|