@customize-agent/knowledge 4.0.1 → 4.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/bge-tokenizer.d.ts +10 -0
- package/dist/chunking/bge-tokenizer.js +68 -0
- package/dist/chunking/text-chunker.d.ts +10 -0
- package/dist/chunking/text-chunker.js +158 -53
- package/dist/classification/classifier.js +0 -1
- package/dist/core/index-state-store.d.ts +47 -0
- package/dist/core/index-state-store.js +184 -50
- package/dist/core/knowledge-base-manager.d.ts +24 -2
- package/dist/core/knowledge-base-manager.js +195 -51
- package/dist/core/multi-project-manager.d.ts +2 -0
- package/dist/core/multi-project-manager.js +13 -2
- package/dist/embedding/embedding-provider.d.ts +22 -0
- package/dist/embedding/embedding-provider.js +116 -2
- package/dist/extraction/content-extractor.d.ts +30 -2
- package/dist/extraction/content-extractor.js +514 -115
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/search/federation-search.d.ts +1 -0
- package/dist/search/federation-search.js +1 -0
- package/dist/vector/hnsw-vector-store.d.ts +20 -0
- package/dist/vector/hnsw-vector-store.js +107 -0
- package/dist/vector/types.d.ts +2 -0
- package/dist/vector/vector-indexer.d.ts +2 -0
- package/dist/vector/vector-indexer.js +13 -1
- package/models/bge-small-zh-v1.5/config.json +31 -0
- package/models/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
- package/models/bge-small-zh-v1.5/special_tokens_map.json +7 -0
- package/models/bge-small-zh-v1.5/tokenizer.json +21278 -0
- package/models/bge-small-zh-v1.5/tokenizer_config.json +15 -0
- package/package.json +11 -4
- package/scripts/install-hnsw.cjs +47 -0
- package/dist/vector/sqlite-vec-store.d.ts +0 -38
- package/dist/vector/sqlite-vec-store.js +0 -203
|
@@ -69,12 +69,6 @@ export class ContentExtractor {
|
|
|
69
69
|
Object.assign(metadata, result.metadata);
|
|
70
70
|
warnings.push(...result.warnings);
|
|
71
71
|
}
|
|
72
|
-
else if (file.category === 'archive') {
|
|
73
|
-
const result = await this.extractArchive(file);
|
|
74
|
-
text = result.text;
|
|
75
|
-
Object.assign(metadata, result.metadata);
|
|
76
|
-
warnings.push(...result.warnings);
|
|
77
|
-
}
|
|
78
72
|
else if (file.category === 'image' && file.format !== 'vector') {
|
|
79
73
|
const result = await this.extractRasterImage(file);
|
|
80
74
|
text = result.text;
|
|
@@ -208,7 +202,7 @@ export class ContentExtractor {
|
|
|
208
202
|
if (file.format === 'autocad' && ext === '.dxf') {
|
|
209
203
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
210
204
|
const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
|
|
211
|
-
const textEntities = this.
|
|
205
|
+
const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
|
|
212
206
|
const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
|
|
213
207
|
const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
|
|
214
208
|
const uniqueLayers = Array.from(new Set(layers));
|
|
@@ -221,14 +215,16 @@ export class ContentExtractor {
|
|
|
221
215
|
metadata.blockNames = uniqueBlocks.slice(0, 80);
|
|
222
216
|
metadata.entityTypeCount = uniqueEntityTypes.length;
|
|
223
217
|
metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
|
|
224
|
-
metadata.contentCoverage = '
|
|
218
|
+
metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
|
|
219
|
+
const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
|
|
225
220
|
return {
|
|
226
221
|
text: [
|
|
227
222
|
this.metadataOnlyText(file),
|
|
228
223
|
`CAD DXF 图层: ${uniqueLayers.join(', ')}`,
|
|
229
224
|
`CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
|
|
230
225
|
`CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
|
|
231
|
-
|
|
226
|
+
'CAD 语义图纸节点:',
|
|
227
|
+
...semanticNodes,
|
|
232
228
|
].join('\n'),
|
|
233
229
|
metadata,
|
|
234
230
|
warnings,
|
|
@@ -307,7 +303,7 @@ export class ContentExtractor {
|
|
|
307
303
|
warnings.push('dxf-parser 解析失败,已使用 DXF 文本结构抽取回退');
|
|
308
304
|
}
|
|
309
305
|
const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
|
|
310
|
-
const textEntities = this.
|
|
306
|
+
const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
|
|
311
307
|
const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
|
|
312
308
|
const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
|
|
313
309
|
const uniqueLayers = Array.from(new Set(layers));
|
|
@@ -320,20 +316,71 @@ export class ContentExtractor {
|
|
|
320
316
|
metadata.blockNames = uniqueBlocks.slice(0, 80);
|
|
321
317
|
metadata.entityTypeCount = uniqueEntityTypes.length;
|
|
322
318
|
metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
|
|
323
|
-
metadata.contentCoverage = '
|
|
319
|
+
metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
|
|
324
320
|
metadata.parsedByDxfParser = Boolean(parsed);
|
|
321
|
+
const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
|
|
325
322
|
return {
|
|
326
323
|
text: [
|
|
327
324
|
this.metadataOnlyText(file),
|
|
328
325
|
`CAD DXF 图层: ${uniqueLayers.join(', ')}`,
|
|
329
326
|
`CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
|
|
330
327
|
`CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
|
|
331
|
-
|
|
328
|
+
'CAD 语义图纸节点:',
|
|
329
|
+
...semanticNodes,
|
|
332
330
|
].join('\n'),
|
|
333
331
|
metadata,
|
|
334
332
|
warnings,
|
|
335
333
|
};
|
|
336
334
|
}
|
|
335
|
+
buildCadSemanticNodes(file, layers, blocks, entityTypes, texts) {
|
|
336
|
+
const fileName = path.basename(file.relativePath);
|
|
337
|
+
const defaultLayer = layers[0] ?? '未命名图层';
|
|
338
|
+
const defaultBlock = blocks[0] ?? '全局模型空间';
|
|
339
|
+
const defaultEntity = entityTypes.find(type => /DIMENSION|TEXT|MTEXT|LEADER/u.test(type)) ?? entityTypes[0] ?? 'UNKNOWN';
|
|
340
|
+
const annotations = texts.length > 0 ? texts : [{ text: '未提取到文字标注', layer: defaultLayer, block: defaultBlock, entityType: defaultEntity }];
|
|
341
|
+
return annotations.map((annotation, index) => {
|
|
342
|
+
const nearest = this.findNearestCadAnnotation(annotation, annotations);
|
|
343
|
+
const layer = annotation.layer ?? layers[index % Math.max(1, layers.length)] ?? defaultLayer;
|
|
344
|
+
const block = annotation.block ?? blocks[index % Math.max(1, blocks.length)] ?? defaultBlock;
|
|
345
|
+
const entity = annotation.entityType ?? this.inferCadEntityType(annotation.text, defaultEntity);
|
|
346
|
+
const status = /关键|critical|尺寸|dim|mm|cm|m\b|°|φ|Φ|R\d/iu.test(annotation.text) ? '关键尺寸/约束候选' : '普通标注';
|
|
347
|
+
const position = annotation.x != null && annotation.y != null ? ` | 坐标: (${annotation.x.toFixed(2)}, ${annotation.y.toFixed(2)})` : '';
|
|
348
|
+
return [`图纸节点: ${fileName} | 图层: ${layer} | 块: ${block} | 实体类型: ${entity}${position}`, `└── 标注文本: ${annotation.text} | 关联对象: ${nearest ? `邻近标注 ${nearest.text}` : '空间邻近候选'} | 状态: ${status}`].join('\n');
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
extractDxfTextAnnotations(raw) {
|
|
352
|
+
const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
|
|
353
|
+
return entities.flatMap(section => {
|
|
354
|
+
const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
|
|
355
|
+
if (!text)
|
|
356
|
+
return [];
|
|
357
|
+
return [{
|
|
358
|
+
text,
|
|
359
|
+
layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
|
|
360
|
+
block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
|
|
361
|
+
entityType: section.trim().split(/\s+/u)[0],
|
|
362
|
+
x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
|
|
363
|
+
y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
|
|
364
|
+
}].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
findNearestCadAnnotation(target, annotations) {
|
|
368
|
+
if (target.x == null || target.y == null)
|
|
369
|
+
return undefined;
|
|
370
|
+
return annotations
|
|
371
|
+
.filter(item => item !== target && item.x != null && item.y != null)
|
|
372
|
+
.map(item => ({ item, distance: Math.hypot((item.x ?? 0) - target.x, (item.y ?? 0) - target.y) }))
|
|
373
|
+
.sort((a, b) => a.distance - b.distance)[0]?.item;
|
|
374
|
+
}
|
|
375
|
+
inferCadEntityType(text, fallback) {
|
|
376
|
+
if (/\b\d+(?:\.\d+)?\s*(?:mm|cm|m)\b|φ|Φ|R\d/iu.test(text))
|
|
377
|
+
return '线性尺寸/半径尺寸';
|
|
378
|
+
if (/°|angle|角度/iu.test(text))
|
|
379
|
+
return '角度尺寸';
|
|
380
|
+
if (/note|说明|备注/iu.test(text))
|
|
381
|
+
return '文字说明';
|
|
382
|
+
return fallback;
|
|
383
|
+
}
|
|
337
384
|
async tryConvertDwgWithBundledWasm(filePath) {
|
|
338
385
|
try {
|
|
339
386
|
const mod = await resolveAndImport('dwgdxf');
|
|
@@ -447,36 +494,39 @@ export class ContentExtractor {
|
|
|
447
494
|
}
|
|
448
495
|
extractData(file) {
|
|
449
496
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
450
|
-
const metadata = { extractionMode: 'structured_data', vectorizable: true };
|
|
497
|
+
const metadata = { extractionMode: 'structured_data', semanticExtractionMode: 'structured_data_semantic_paths', vectorizable: true };
|
|
451
498
|
try {
|
|
452
499
|
if (file.format === 'json') {
|
|
453
500
|
const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
|
|
454
|
-
const
|
|
455
|
-
? raw.split(/\r?\n/u).filter(Boolean).
|
|
456
|
-
:
|
|
501
|
+
const records = isJsonl
|
|
502
|
+
? raw.split(/\r?\n/u).filter(Boolean).map((line, index) => ({ path: `line${index + 1}`, value: JSON.parse(line) }))
|
|
503
|
+
: [{ path: '$', value: JSON.parse(raw) }];
|
|
504
|
+
const lines = records.flatMap(record => this.flattenJson(record.value, record.path));
|
|
505
|
+
const objects = records.flatMap(record => this.atomicJsonObjects(record.value, record.path));
|
|
457
506
|
metadata.fieldCount = lines.length;
|
|
458
|
-
metadata.recordCount =
|
|
507
|
+
metadata.recordCount = records.length;
|
|
508
|
+
metadata.objectCount = objects.length;
|
|
459
509
|
metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
|
|
460
|
-
metadata.contentCoverage = isJsonl ? '
|
|
461
|
-
return { text: [this.metadataOnlyText(file), ...lines
|
|
510
|
+
metadata.contentCoverage = isJsonl ? 'jsonl_atomic_objects_paths_values' : 'json_atomic_objects_paths_values';
|
|
511
|
+
return { text: [this.metadataOnlyText(file), '## 路径声明', ...lines, '## 原子对象', ...objects].join('\n'), metadata, warnings: [] };
|
|
462
512
|
}
|
|
463
513
|
}
|
|
464
514
|
catch {
|
|
465
515
|
metadata.parseError = true;
|
|
466
516
|
}
|
|
467
517
|
if (file.format === 'yaml') {
|
|
468
|
-
const
|
|
469
|
-
metadata.fieldCount =
|
|
470
|
-
metadata.dataPaths =
|
|
471
|
-
metadata.contentCoverage = '
|
|
472
|
-
return { text: [this.metadataOnlyText(file), ...
|
|
518
|
+
const lines = this.flattenYamlByIndent(raw).slice(0, 3000);
|
|
519
|
+
metadata.fieldCount = lines.length;
|
|
520
|
+
metadata.dataPaths = lines.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
|
|
521
|
+
metadata.contentCoverage = 'yaml_indented_paths_values';
|
|
522
|
+
return { text: [this.metadataOnlyText(file), '## YAML 路径声明', ...lines].join('\n'), metadata, warnings: [] };
|
|
473
523
|
}
|
|
474
524
|
if (file.format === 'xml') {
|
|
475
|
-
const elements = this.
|
|
525
|
+
const elements = this.flattenXmlPaths(raw).slice(0, 3000);
|
|
476
526
|
metadata.elementTextCount = elements.length;
|
|
477
|
-
metadata.dataPaths = elements.map(line => line.
|
|
478
|
-
metadata.contentCoverage = '
|
|
479
|
-
return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
|
|
527
|
+
metadata.dataPaths = elements.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
|
|
528
|
+
metadata.contentCoverage = 'xml_paths_values';
|
|
529
|
+
return { text: [this.metadataOnlyText(file), '## XML 路径声明', ...elements].join('\n'), metadata, warnings: [] };
|
|
480
530
|
}
|
|
481
531
|
metadata.contentCoverage = 'plain_structured_text';
|
|
482
532
|
return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
|
|
@@ -485,18 +535,19 @@ export class ContentExtractor {
|
|
|
485
535
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
486
536
|
const metadata = { extractionMode: 'diagram_structural', vectorizable: true };
|
|
487
537
|
if (file.format === 'drawio') {
|
|
488
|
-
const
|
|
489
|
-
metadata.nodeTextCount =
|
|
490
|
-
metadata.
|
|
491
|
-
|
|
538
|
+
const graph = this.extractDrawioGraph(raw);
|
|
539
|
+
metadata.nodeTextCount = graph.nodes.length;
|
|
540
|
+
metadata.edgeCount = graph.edges.length;
|
|
541
|
+
metadata.contentCoverage = 'drawio_graph_links';
|
|
542
|
+
return { text: [this.metadataOnlyText(file), 'Draw.io 图链路:', ...graph.links, 'Draw.io 节点:', ...graph.nodes.map(node => `[${node.id}] ${node.label}`)].join('\n'), metadata, warnings: [] };
|
|
492
543
|
}
|
|
493
544
|
if (file.format === 'excalidraw') {
|
|
494
545
|
try {
|
|
495
|
-
const
|
|
496
|
-
|
|
497
|
-
metadata.
|
|
498
|
-
metadata.contentCoverage = '
|
|
499
|
-
return { text: [this.metadataOnlyText(file),
|
|
546
|
+
const graph = this.extractExcalidrawGraph(raw);
|
|
547
|
+
metadata.elementTextCount = graph.nodes.length;
|
|
548
|
+
metadata.edgeCount = graph.links.length;
|
|
549
|
+
metadata.contentCoverage = 'excalidraw_graph_links';
|
|
550
|
+
return { text: [this.metadataOnlyText(file), 'Excalidraw 图链路:', ...graph.links, 'Excalidraw 节点:', ...graph.nodes].join('\n'), metadata, warnings: [] };
|
|
500
551
|
}
|
|
501
552
|
catch {
|
|
502
553
|
metadata.parseError = true;
|
|
@@ -505,6 +556,52 @@ export class ContentExtractor {
|
|
|
505
556
|
metadata.contentCoverage = 'diagram_source_text';
|
|
506
557
|
return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
|
|
507
558
|
}
|
|
559
|
+
extractDrawioGraph(raw) {
|
|
560
|
+
const cells = Array.from(raw.matchAll(/<mxCell\b([^>]*)>/gu), match => this.parseXmlAttributes(match[1] ?? ''));
|
|
561
|
+
const nodes = cells
|
|
562
|
+
.filter(cell => cell.value || cell.label || cell.id)
|
|
563
|
+
.map((cell, index) => ({ id: String(cell.id ?? `node-${index + 1}`), label: this.stripXml(String(cell.value || cell.label || cell.id)) }))
|
|
564
|
+
.filter(node => node.label);
|
|
565
|
+
const nodeById = new Map(nodes.map(node => [node.id, node.label]));
|
|
566
|
+
const edges = cells.filter(cell => cell.edge === '1' && cell.source && cell.target);
|
|
567
|
+
const links = edges.map(edge => {
|
|
568
|
+
const source = nodeById.get(String(edge.source)) ?? String(edge.source);
|
|
569
|
+
const target = nodeById.get(String(edge.target)) ?? String(edge.target);
|
|
570
|
+
const label = this.stripXml(String(edge.value || edge.label || '关系'));
|
|
571
|
+
return `[${source}] ──> (${label}) ──> [${target}]`;
|
|
572
|
+
});
|
|
573
|
+
return { nodes, edges: edges.map(edge => String(edge.id ?? 'edge')), links };
|
|
574
|
+
}
|
|
575
|
+
extractExcalidrawGraph(raw) {
|
|
576
|
+
const parsed = JSON.parse(raw);
|
|
577
|
+
const elements = parsed.elements ?? [];
|
|
578
|
+
const textById = new Map(elements.filter(element => element.text).map(element => [String(element.id), String(element.text)]));
|
|
579
|
+
const shapeLabels = elements
|
|
580
|
+
.filter(element => element.type !== 'arrow' && element.type !== 'line')
|
|
581
|
+
.map((element, index) => {
|
|
582
|
+
const id = String(element.id ?? `node-${index + 1}`);
|
|
583
|
+
const own = element.text ?? textById.get(id);
|
|
584
|
+
const nested = elements.find(candidate => candidate.text && this.isPointInside(candidate, element));
|
|
585
|
+
return { id, label: own ?? nested?.text ?? element.type ?? '节点' };
|
|
586
|
+
});
|
|
587
|
+
const labelById = new Map(shapeLabels.map(node => [node.id, node.label]));
|
|
588
|
+
const links = elements
|
|
589
|
+
.filter(element => element.type === 'arrow' && element.startBinding?.elementId && element.endBinding?.elementId)
|
|
590
|
+
.map(element => `[${labelById.get(element.startBinding.elementId) ?? element.startBinding.elementId}] ──> (箭头) ──> [${labelById.get(element.endBinding.elementId) ?? element.endBinding.elementId}]`);
|
|
591
|
+
const nodes = shapeLabels.map(node => `[${node.id}] ${node.label}`);
|
|
592
|
+
return { nodes, links };
|
|
593
|
+
}
|
|
594
|
+
isPointInside(point, box) {
|
|
595
|
+
if (point.x == null || point.y == null || box.x == null || box.y == null || box.width == null || box.height == null)
|
|
596
|
+
return false;
|
|
597
|
+
return point.x >= box.x && point.x <= box.x + box.width && point.y >= box.y && point.y <= box.y + box.height;
|
|
598
|
+
}
|
|
599
|
+
parseXmlAttributes(input) {
|
|
600
|
+
const attrs = {};
|
|
601
|
+
for (const match of input.matchAll(/([\w:-]+)="([^"]*)"/gu))
|
|
602
|
+
attrs[match[1]] = this.stripXml(match[2] ?? '');
|
|
603
|
+
return attrs;
|
|
604
|
+
}
|
|
508
605
|
parseDelimitedLine(line, delimiter) {
|
|
509
606
|
const values = [];
|
|
510
607
|
let current = '';
|
|
@@ -531,28 +628,38 @@ export class ContentExtractor {
|
|
|
531
628
|
values.push(current.trim());
|
|
532
629
|
return values;
|
|
533
630
|
}
|
|
631
|
+
toMarkdownTable(header, rows) {
|
|
632
|
+
const width = Math.max(header.length, ...rows.map(row => row.length), 1);
|
|
633
|
+
const normalizedHeader = Array.from({ length: width }, (_, index) => header[index] || `COL${index + 1}`);
|
|
634
|
+
const escape = (value) => String(value ?? '').replace(/\|/gu, '\\|').replace(/\r?\n/gu, ' ').trim();
|
|
635
|
+
return [
|
|
636
|
+
`| ${normalizedHeader.map(escape).join(' | ')} |`,
|
|
637
|
+
`| ${normalizedHeader.map(() => '---').join(' | ')} |`,
|
|
638
|
+
...rows.map(row => `| ${Array.from({ length: width }, (_, index) => escape(row[index] ?? '')).join(' | ')} |`),
|
|
639
|
+
].join('\n');
|
|
640
|
+
}
|
|
534
641
|
extractDelimitedText(file) {
|
|
535
642
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
536
643
|
const delimiter = file.format === 'tsv' ? '\t' : ',';
|
|
537
644
|
const rows = raw.split(/\r?\n/u).filter(line => line.trim().length > 0);
|
|
538
645
|
const header = rows[0] ? this.parseDelimitedLine(rows[0], delimiter) : [];
|
|
539
|
-
const
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
});
|
|
646
|
+
const tableRows = rows.slice(1).map(line => this.parseDelimitedLine(line, delimiter));
|
|
647
|
+
const markdown = this.toMarkdownTable(header, tableRows);
|
|
648
|
+
const legacyKv = tableRows.flatMap((values, rowIndex) => values.map((value, colIndex) => {
|
|
649
|
+
const column = header[colIndex] || `COL${colIndex + 1}`;
|
|
650
|
+
return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
|
|
651
|
+
}));
|
|
546
652
|
return {
|
|
547
|
-
text: [this.metadataOnlyText(file),
|
|
653
|
+
text: [this.metadataOnlyText(file), '### 工作表: 默认', markdown, '### 表格路径声明', ...legacyKv].join('\n\n'),
|
|
548
654
|
metadata: {
|
|
549
655
|
extractionMode: 'delimited_text_structured',
|
|
656
|
+
semanticExtractionMode: 'delimited_markdown_table',
|
|
550
657
|
vectorizable: true,
|
|
551
658
|
delimiter: file.format === 'tsv' ? 'tab' : 'comma',
|
|
552
659
|
rowCount: rows.length,
|
|
553
660
|
columnCount: header.length,
|
|
554
661
|
columnNames: header.slice(0, 120),
|
|
555
|
-
contentCoverage: '
|
|
662
|
+
contentCoverage: 'markdown_table',
|
|
556
663
|
},
|
|
557
664
|
warnings: [],
|
|
558
665
|
};
|
|
@@ -567,13 +674,21 @@ export class ContentExtractor {
|
|
|
567
674
|
return this.extractLegacyOfficeBinary(file);
|
|
568
675
|
if (ext === '.docx') {
|
|
569
676
|
try {
|
|
677
|
+
const styledMarkdown = await this.extractDocxStyleTreeMarkdown(file.absolutePath);
|
|
678
|
+
if (styledMarkdown.trim()) {
|
|
679
|
+
return {
|
|
680
|
+
text: styledMarkdown,
|
|
681
|
+
metadata: { extractionMode: 'docx_xml_style_tree_markdown', vectorizable: true, contentCoverage: 'office_style_tree_markdown' },
|
|
682
|
+
warnings: [],
|
|
683
|
+
};
|
|
684
|
+
}
|
|
570
685
|
const mammoth = await resolveAndImport('mammoth');
|
|
571
|
-
const result = await mammoth.
|
|
572
|
-
const text = result.value.trim();
|
|
686
|
+
const result = await mammoth.convertToMarkdown({ path: file.absolutePath });
|
|
687
|
+
const text = String(result.value ?? '').trim();
|
|
573
688
|
if (text) {
|
|
574
689
|
return {
|
|
575
|
-
text,
|
|
576
|
-
metadata: { extractionMode: '
|
|
690
|
+
text: this.normalizeMarkdownHeadings(text),
|
|
691
|
+
metadata: { extractionMode: 'builtin_mammoth_markdown', vectorizable: true, contentCoverage: 'office_markdown_structure' },
|
|
577
692
|
warnings: result.messages.map(m => m.message),
|
|
578
693
|
};
|
|
579
694
|
}
|
|
@@ -630,6 +745,14 @@ export class ContentExtractor {
|
|
|
630
745
|
warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库'],
|
|
631
746
|
};
|
|
632
747
|
}
|
|
748
|
+
findMergedCellValue(sheet, merges, row, col, XLSX) {
|
|
749
|
+
const merge = merges.find(item => row >= item.s.r && row <= item.e.r && col >= item.s.c && col <= item.e.c);
|
|
750
|
+
if (!merge)
|
|
751
|
+
return undefined;
|
|
752
|
+
const originAddress = XLSX.utils.encode_cell({ r: merge.s.r, c: merge.s.c });
|
|
753
|
+
const origin = sheet[originAddress];
|
|
754
|
+
return origin?.w ?? (origin?.v == null ? undefined : String(origin.v));
|
|
755
|
+
}
|
|
633
756
|
async extractSpreadsheet(file) {
|
|
634
757
|
const ext = path.extname(file.absolutePath).toLowerCase();
|
|
635
758
|
try {
|
|
@@ -644,28 +767,32 @@ export class ContentExtractor {
|
|
|
644
767
|
if (!sheet || !sheet['!ref'])
|
|
645
768
|
continue;
|
|
646
769
|
const range = XLSX.utils.decode_range(sheet['!ref']);
|
|
647
|
-
const lines = [`工作表: ${name}`, `范围: ${sheet['!ref']}`];
|
|
648
770
|
const merges = sheet['!merges'] ?? [];
|
|
649
771
|
mergeCount += merges.length;
|
|
650
|
-
|
|
651
|
-
lines.push(`合并单元格: ${merges.map(item => `${XLSX.utils.encode_cell(item.s)}:${XLSX.utils.encode_cell(item.e)}`).join(', ')}`);
|
|
652
|
-
}
|
|
772
|
+
const matrix = [];
|
|
653
773
|
for (let row = range.s.r; row <= range.e.r; row++) {
|
|
774
|
+
const values = [];
|
|
654
775
|
for (let col = range.s.c; col <= range.e.c; col++) {
|
|
655
776
|
const address = XLSX.utils.encode_cell({ r: row, c: col });
|
|
656
777
|
const cell = sheet[address];
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
const
|
|
664
|
-
|
|
778
|
+
const mergedValue = cell ? undefined : this.findMergedCellValue(sheet, merges, row, col, XLSX);
|
|
779
|
+
if (cell) {
|
|
780
|
+
cellCount++;
|
|
781
|
+
if (cell.f)
|
|
782
|
+
formulaCount++;
|
|
783
|
+
}
|
|
784
|
+
const display = cell?.w ?? String(cell?.v ?? mergedValue ?? '');
|
|
785
|
+
const formula = cell?.f ? ` 公式=${cell.f}` : '';
|
|
786
|
+
values.push(`${display}${formula}`.trim());
|
|
665
787
|
}
|
|
788
|
+
if (values.some(Boolean))
|
|
789
|
+
matrix.push(values);
|
|
790
|
+
}
|
|
791
|
+
if (matrix.length > 0) {
|
|
792
|
+
const header = matrix[0] ?? [];
|
|
793
|
+
const rows = matrix.slice(1);
|
|
794
|
+
sheetTexts.push([`### 工作表: ${name}`, this.toMarkdownTable(header, rows)].join('\n\n'));
|
|
666
795
|
}
|
|
667
|
-
if (lines.length > 2)
|
|
668
|
-
sheetTexts.push(lines.slice(0, 5_000).join('\n'));
|
|
669
796
|
}
|
|
670
797
|
if (sheetTexts.length > 0) {
|
|
671
798
|
return {
|
|
@@ -684,6 +811,41 @@ export class ContentExtractor {
|
|
|
684
811
|
return this.extractLegacyOfficeBinary(file);
|
|
685
812
|
return this.extractOfficeZip(file);
|
|
686
813
|
}
|
|
814
|
+
async extractDocxStyleTreeMarkdown(filePath) {
|
|
815
|
+
const jszipMod = await resolveAndImport('jszip');
|
|
816
|
+
const JSZip = jszipMod.default ?? jszipMod;
|
|
817
|
+
const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
|
|
818
|
+
const docXml = await zip.files['word/document.xml']?.async('string');
|
|
819
|
+
if (!docXml)
|
|
820
|
+
return '';
|
|
821
|
+
const paragraphs = Array.from(docXml.matchAll(/<w:p[\s\S]*?<\/w:p>/gu), match => match[0]);
|
|
822
|
+
const lines = [];
|
|
823
|
+
for (const paragraph of paragraphs) {
|
|
824
|
+
const texts = Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
|
|
825
|
+
if (!texts.trim())
|
|
826
|
+
continue;
|
|
827
|
+
const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(paragraph)?.[1] ?? '';
|
|
828
|
+
const bold = /<w:b\b/u.test(paragraph);
|
|
829
|
+
const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(paragraph)?.[1] ?? 0);
|
|
830
|
+
const level = this.docxHeadingLevel(style, bold, size, texts);
|
|
831
|
+
lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
|
|
832
|
+
}
|
|
833
|
+
return lines.join('\n\n');
|
|
834
|
+
}
|
|
835
|
+
docxHeadingLevel(style, bold, size, text) {
|
|
836
|
+
const normalized = style.toLowerCase();
|
|
837
|
+
const heading = /heading(\d)|标题(\d)|h(\d)/iu.exec(normalized);
|
|
838
|
+
const styleLevel = Number(heading?.[1] ?? heading?.[2] ?? heading?.[3] ?? 0);
|
|
839
|
+
if (styleLevel >= 1 && styleLevel <= 6)
|
|
840
|
+
return styleLevel;
|
|
841
|
+
if (text.length <= 100 && size >= 32)
|
|
842
|
+
return 1;
|
|
843
|
+
if (text.length <= 100 && size >= 28)
|
|
844
|
+
return 2;
|
|
845
|
+
if (text.length <= 100 && (size >= 24 || bold))
|
|
846
|
+
return 3;
|
|
847
|
+
return 0;
|
|
848
|
+
}
|
|
687
849
|
async extractOfficeZip(file) {
|
|
688
850
|
const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
|
|
689
851
|
try {
|
|
@@ -709,26 +871,6 @@ export class ContentExtractor {
|
|
|
709
871
|
return { text: '', metadata, warnings: [`Office/表格/演示文件解析失败: ${metadata.parseError},未入库`] };
|
|
710
872
|
}
|
|
711
873
|
}
|
|
712
|
-
async extractArchive(file) {
|
|
713
|
-
const metadata = { extractionMode: 'archive_manifest', vectorizable: true };
|
|
714
|
-
if (path.extname(file.absolutePath).toLowerCase() !== '.zip') {
|
|
715
|
-
metadata.contentCoverage = 'metadata_filename';
|
|
716
|
-
return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
|
|
717
|
-
}
|
|
718
|
-
try {
|
|
719
|
-
const jszipMod = await resolveAndImport('jszip');
|
|
720
|
-
const JSZip = jszipMod.default ?? jszipMod;
|
|
721
|
-
const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
|
|
722
|
-
const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
|
|
723
|
-
metadata.entryCount = Object.keys(zip.files).length;
|
|
724
|
-
metadata.contentCoverage = 'zip_manifest';
|
|
725
|
-
return { text: [this.metadataOnlyText(file), '压缩包文件清单:', ...entries].join('\n'), metadata, warnings: [] };
|
|
726
|
-
}
|
|
727
|
-
catch (error) {
|
|
728
|
-
metadata.parseError = error instanceof Error ? error.message : String(error);
|
|
729
|
-
return { text: '', metadata, warnings: ['压缩包解析失败,内置解析器未提取到文件清单,未入库'] };
|
|
730
|
-
}
|
|
731
|
-
}
|
|
732
874
|
async extractRasterImage(file) {
|
|
733
875
|
const metadata = { extractionMode: 'builtin_tesseract_ocr_isolated', vectorizable: true };
|
|
734
876
|
if (process.env.CUSTOMIZE_AGENT_DISABLE_OCR === '1') {
|
|
@@ -742,6 +884,13 @@ export class ContentExtractor {
|
|
|
742
884
|
metadata.parseError = validationError;
|
|
743
885
|
return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
|
|
744
886
|
}
|
|
887
|
+
const paddle = await this.tryPaddleOcrLayout(file.absolutePath);
|
|
888
|
+
if (paddle) {
|
|
889
|
+
metadata.contentCoverage = 'paddleocr_layout_regions';
|
|
890
|
+
metadata.ocrProvider = 'paddleocr';
|
|
891
|
+
metadata.ocrRegionCount = paddle.regionCount;
|
|
892
|
+
return { text: [this.metadataOnlyText(file), paddle.text].join('\n'), metadata, warnings: [] };
|
|
893
|
+
}
|
|
745
894
|
let tesseractPath;
|
|
746
895
|
try {
|
|
747
896
|
tesseractPath = resolvePackage('tesseract.js');
|
|
@@ -766,7 +915,12 @@ const { createWorker } = tesseractMod;
|
|
|
766
915
|
const worker = await createWorker('chi_sim+eng');
|
|
767
916
|
try {
|
|
768
917
|
const result = await worker.recognize(process.argv[1]);
|
|
769
|
-
|
|
918
|
+
const lines = (result.data.lines || []).map((line, index) => ({
|
|
919
|
+
index: index + 1,
|
|
920
|
+
text: line.text || '',
|
|
921
|
+
bbox: line.bbox || line.baseline || null,
|
|
922
|
+
})).filter(line => line.text.trim());
|
|
923
|
+
process.stdout.write(JSON.stringify({ text: result.data.text || '', lines }));
|
|
770
924
|
} finally {
|
|
771
925
|
await worker.terminate();
|
|
772
926
|
}`,
|
|
@@ -778,17 +932,73 @@ try {
|
|
|
778
932
|
metadata.parseError = message;
|
|
779
933
|
return { text: '', metadata, warnings: [`内置 OCR 解析失败:${message},未入库`] };
|
|
780
934
|
}
|
|
781
|
-
const
|
|
782
|
-
|
|
935
|
+
const parsed = this.parseOcrJson(result.stdout);
|
|
936
|
+
const text = parsed.text.trim();
|
|
937
|
+
const regions = this.formatOcrRegions(parsed.lines);
|
|
938
|
+
metadata.contentCoverage = text ? 'ocr_text_bounding_boxes' : 'metadata_filename';
|
|
783
939
|
metadata.ocrProvider = 'tesseract.js';
|
|
784
940
|
metadata.ocrLanguages = 'chi_sim+eng';
|
|
785
941
|
metadata.ocrTextLength = text.length;
|
|
942
|
+
metadata.ocrLineCount = parsed.lines.length;
|
|
786
943
|
return {
|
|
787
|
-
text: text ? [this.metadataOnlyText(file), `OCR
|
|
944
|
+
text: text ? [this.metadataOnlyText(file), 'OCR 区域文本:', ...regions, `OCR 完整文本:\n${text}`].join('\n') : '',
|
|
788
945
|
metadata,
|
|
789
946
|
warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
|
|
790
947
|
};
|
|
791
948
|
}
|
|
949
|
+
async tryPaddleOcrLayout(filePath) {
|
|
950
|
+
const command = process.env.CUSTOMIZE_PADDLE_OCR_CMD || process.env.PADDLE_OCR_CMD;
|
|
951
|
+
if (!command)
|
|
952
|
+
return undefined;
|
|
953
|
+
const result = spawnSync(command, [filePath], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, shell: true });
|
|
954
|
+
if (result.status !== 0 || !result.stdout.trim())
|
|
955
|
+
return undefined;
|
|
956
|
+
try {
|
|
957
|
+
const parsed = JSON.parse(result.stdout);
|
|
958
|
+
const lines = parsed.map((region, index) => `区域 ${index + 1} [${region.type ?? 'text'}] ${this.formatBoundingBox(region.bbox)}: ${region.text ?? ''}`);
|
|
959
|
+
return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
|
|
960
|
+
}
|
|
961
|
+
catch {
|
|
962
|
+
const lines = result.stdout.split(/\r?\n/u).filter(Boolean);
|
|
963
|
+
return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
parseOcrJson(raw) {
|
|
967
|
+
try {
|
|
968
|
+
const parsed = JSON.parse(raw);
|
|
969
|
+
return {
|
|
970
|
+
text: parsed.text ?? raw,
|
|
971
|
+
lines: (parsed.lines ?? []).map((line, index) => ({ index: line.index ?? index + 1, text: line.text ?? '', bbox: line.bbox })).filter(line => line.text.trim()),
|
|
972
|
+
};
|
|
973
|
+
}
|
|
974
|
+
catch {
|
|
975
|
+
return { text: raw, lines: raw.split(/\r?\n/u).map((text, index) => ({ index: index + 1, text })).filter(line => line.text.trim()) };
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
formatOcrRegions(lines) {
|
|
979
|
+
return lines.map(line => {
|
|
980
|
+
const bbox = this.formatBoundingBox(line.bbox);
|
|
981
|
+
const type = this.classifyOcrRegion(line.text);
|
|
982
|
+
return `区域 ${line.index} [${type}]${bbox ? ` ${bbox}` : ''}: ${line.text}`;
|
|
983
|
+
});
|
|
984
|
+
}
|
|
985
|
+
classifyOcrRegion(text) {
|
|
986
|
+
if (/\|/.test(text) || /\s{2,}/u.test(text) || /表\s*\d|合计|小计/u.test(text))
|
|
987
|
+
return 'table';
|
|
988
|
+
if (/图\s*\d|figure|image|示意图/iu.test(text))
|
|
989
|
+
return 'image-caption';
|
|
990
|
+
return 'text';
|
|
991
|
+
}
|
|
992
|
+
formatBoundingBox(value) {
|
|
993
|
+
if (!value || typeof value !== 'object')
|
|
994
|
+
return '';
|
|
995
|
+
const record = value;
|
|
996
|
+
const x0 = record.x0 ?? record.left ?? record.x;
|
|
997
|
+
const y0 = record.y0 ?? record.top ?? record.y;
|
|
998
|
+
const x1 = record.x1 ?? record.right;
|
|
999
|
+
const y1 = record.y1 ?? record.bottom;
|
|
1000
|
+
return [x0, y0, x1, y1].some(item => item != null) ? `[bbox x0=${x0 ?? ''}, y0=${y0 ?? ''}, x1=${x1 ?? ''}, y1=${y1 ?? ''}]` : '';
|
|
1001
|
+
}
|
|
792
1002
|
validateRasterImage(filePath) {
|
|
793
1003
|
const buffer = fs.readFileSync(filePath);
|
|
794
1004
|
if (buffer.length < 12)
|
|
@@ -821,9 +1031,9 @@ try {
|
|
|
821
1031
|
const raw = fs.readFileSync(file.absolutePath);
|
|
822
1032
|
const text = await this.extractPdfText(raw);
|
|
823
1033
|
if (text.trim()) {
|
|
824
|
-
metadata.contentCoverage = '
|
|
1034
|
+
metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
|
|
825
1035
|
metadata.pdfExtractor = 'pdfjs-dist';
|
|
826
|
-
return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
|
|
1036
|
+
return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
|
|
827
1037
|
}
|
|
828
1038
|
}
|
|
829
1039
|
catch (error) {
|
|
@@ -846,10 +1056,6 @@ try {
|
|
|
846
1056
|
warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
|
|
847
1057
|
};
|
|
848
1058
|
}
|
|
849
|
-
pdfOcrPageLimit() {
|
|
850
|
-
const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
|
|
851
|
-
return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
|
|
852
|
-
}
|
|
853
1059
|
async extractScannedPdfOcr(file) {
|
|
854
1060
|
const metadata = {
|
|
855
1061
|
extractionMode: 'pdf_page_ocr_embedded',
|
|
@@ -858,9 +1064,8 @@ try {
|
|
|
858
1064
|
ocrProvider: 'tesseract.js',
|
|
859
1065
|
ocrLanguages: 'chi_sim+eng',
|
|
860
1066
|
pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
|
|
861
|
-
pdfOcrPageLimit:
|
|
1067
|
+
pdfOcrPageLimit: 'all',
|
|
862
1068
|
};
|
|
863
|
-
const pageLimitConfig = this.pdfOcrPageLimit();
|
|
864
1069
|
// 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
|
|
865
1070
|
let canvasPath;
|
|
866
1071
|
let pdfjsPath;
|
|
@@ -890,10 +1095,9 @@ import { createCanvas } from ${JSON.stringify(canvasPath)};
|
|
|
890
1095
|
import * as pdfjs from ${JSON.stringify(pdfjsPath)};
|
|
891
1096
|
import { createWorker } from ${JSON.stringify(tesseractPath)};
|
|
892
1097
|
const filePath = process.argv[1];
|
|
893
|
-
const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
|
|
894
1098
|
const bytes = new Uint8Array(fs.readFileSync(filePath));
|
|
895
1099
|
const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
|
|
896
|
-
const pageLimit =
|
|
1100
|
+
const pageLimit = doc.numPages;
|
|
897
1101
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
|
|
898
1102
|
const worker = await createWorker('chi_sim+eng');
|
|
899
1103
|
const pages = [];
|
|
@@ -908,7 +1112,8 @@ try {
|
|
|
908
1112
|
fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
|
|
909
1113
|
const recognized = await worker.recognize(imagePath);
|
|
910
1114
|
const text = (recognized.data.text || '').trim();
|
|
911
|
-
|
|
1115
|
+
const lines = (recognized.data.lines || []).map((line, index) => ({ index: index + 1, text: line.text || '', bbox: line.bbox || null })).filter(line => line.text.trim());
|
|
1116
|
+
if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + lines.map(line => '区域 ' + line.index + ' ' + JSON.stringify(line.bbox || {}) + ': ' + line.text).join('\\n') + '\\n\\n完整文本:\\n' + text);
|
|
912
1117
|
}
|
|
913
1118
|
} finally {
|
|
914
1119
|
await worker.terminate();
|
|
@@ -916,8 +1121,7 @@ try {
|
|
|
916
1121
|
}
|
|
917
1122
|
process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
|
|
918
1123
|
file.absolutePath,
|
|
919
|
-
|
|
920
|
-
], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
|
|
1124
|
+
], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, env: childEnv });
|
|
921
1125
|
if (result.status !== 0 || result.error) {
|
|
922
1126
|
metadata.ocrRecommended = true;
|
|
923
1127
|
metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
|
|
@@ -927,7 +1131,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
927
1131
|
const parsed = JSON.parse(result.stdout);
|
|
928
1132
|
const text = parsed.text?.trim() ?? '';
|
|
929
1133
|
metadata.ocrPageCount = parsed.pageCount ?? 0;
|
|
930
|
-
metadata.pdfOcrPageLimit = parsed.pageLimit ??
|
|
1134
|
+
metadata.pdfOcrPageLimit = parsed.pageLimit ?? 'all';
|
|
931
1135
|
metadata.ocrTextLength = text.length;
|
|
932
1136
|
metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
|
|
933
1137
|
return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
|
|
@@ -945,17 +1149,14 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
945
1149
|
const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
|
|
946
1150
|
const doc = await loadingTask.promise;
|
|
947
1151
|
const pages = [];
|
|
948
|
-
const pageLimit =
|
|
1152
|
+
const pageLimit = doc.numPages;
|
|
949
1153
|
for (let i = 1; i <= pageLimit; i++) {
|
|
950
1154
|
const page = await doc.getPage(i);
|
|
951
1155
|
const content = await page.getTextContent();
|
|
952
|
-
const
|
|
953
|
-
.map((item) =>
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
})
|
|
957
|
-
.filter((s) => s.trim().length > 0)
|
|
958
|
-
.join(' ');
|
|
1156
|
+
const items = content.items
|
|
1157
|
+
.map((item) => this.toPdfTextItem(item))
|
|
1158
|
+
.filter((item) => !!item && item.str.trim().length > 0);
|
|
1159
|
+
const pageText = this.layoutPdfTextItems(items, i);
|
|
959
1160
|
if (pageText.trim())
|
|
960
1161
|
pages.push(pageText.trim());
|
|
961
1162
|
}
|
|
@@ -998,16 +1199,166 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
998
1199
|
.join('')
|
|
999
1200
|
.trim();
|
|
1000
1201
|
}
|
|
1202
|
+
toPdfTextItem(item) {
|
|
1203
|
+
if (!item || typeof item !== 'object' || !('str' in item))
|
|
1204
|
+
return undefined;
|
|
1205
|
+
const record = item;
|
|
1206
|
+
const transform = Array.isArray(record.transform) ? record.transform : [];
|
|
1207
|
+
return {
|
|
1208
|
+
str: String(record.str ?? ''),
|
|
1209
|
+
x: Number(transform[4] ?? 0),
|
|
1210
|
+
y: Number(transform[5] ?? 0),
|
|
1211
|
+
width: Number(record.width ?? 0),
|
|
1212
|
+
height: Number(record.height ?? Math.abs(Number(transform[3] ?? 0))),
|
|
1213
|
+
fontName: typeof record.fontName === 'string' ? record.fontName : undefined,
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
layoutPdfTextItems(items, pageNumber) {
|
|
1217
|
+
if (items.length === 0)
|
|
1218
|
+
return '';
|
|
1219
|
+
const rows = this.groupPdfItemsIntoRows(items);
|
|
1220
|
+
const columnSplit = this.detectPdfColumnSplit(rows);
|
|
1221
|
+
const orderedRows = columnSplit == null
|
|
1222
|
+
? rows.sort((a, b) => b.y - a.y || a.x - b.x)
|
|
1223
|
+
: [
|
|
1224
|
+
...rows.filter(row => row.x < columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
|
|
1225
|
+
...rows.filter(row => row.x >= columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
|
|
1226
|
+
];
|
|
1227
|
+
const markdown = this.rowsToPdfMarkdownWithTables(orderedRows);
|
|
1228
|
+
return [`## PDF 第 ${pageNumber} 页`, markdown].join('\n\n');
|
|
1229
|
+
}
|
|
1230
|
+
groupPdfItemsIntoRows(items) {
|
|
1231
|
+
const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
|
|
1232
|
+
const rows = [];
|
|
1233
|
+
for (const item of sorted) {
|
|
1234
|
+
const row = rows.find(candidate => Math.abs(candidate.y - item.y) <= Math.max(2, item.height * 0.55));
|
|
1235
|
+
if (row)
|
|
1236
|
+
row.items.push(item);
|
|
1237
|
+
else
|
|
1238
|
+
rows.push({ y: item.y, items: [item] });
|
|
1239
|
+
}
|
|
1240
|
+
return rows.map(row => {
|
|
1241
|
+
const rowItems = row.items.sort((a, b) => a.x - b.x);
|
|
1242
|
+
return {
|
|
1243
|
+
text: rowItems.map(item => item.str.trim()).filter(Boolean).join(' ').replace(/\s+/gu, ' '),
|
|
1244
|
+
x: Math.min(...rowItems.map(item => item.x)),
|
|
1245
|
+
y: row.y,
|
|
1246
|
+
height: Math.max(...rowItems.map(item => item.height || 0)),
|
|
1247
|
+
};
|
|
1248
|
+
}).filter(row => row.text);
|
|
1249
|
+
}
|
|
1250
|
+
detectPdfColumnSplit(rows) {
|
|
1251
|
+
if (rows.length < 8)
|
|
1252
|
+
return undefined;
|
|
1253
|
+
const xs = rows.map(row => row.x).sort((a, b) => a - b);
|
|
1254
|
+
const gaps = xs.slice(1).map((x, index) => ({ gap: x - xs[index], left: xs[index], right: x })).sort((a, b) => b.gap - a.gap);
|
|
1255
|
+
const largest = gaps[0];
|
|
1256
|
+
if (!largest || largest.gap < 80)
|
|
1257
|
+
return undefined;
|
|
1258
|
+
const leftCount = rows.filter(row => row.x <= largest.left).length;
|
|
1259
|
+
const rightCount = rows.filter(row => row.x >= largest.right).length;
|
|
1260
|
+
return leftCount >= 3 && rightCount >= 3 ? (largest.left + largest.right) / 2 : undefined;
|
|
1261
|
+
}
|
|
1262
|
+
rowsToPdfMarkdownWithTables(rows) {
|
|
1263
|
+
const output = [];
|
|
1264
|
+
let index = 0;
|
|
1265
|
+
while (index < rows.length) {
|
|
1266
|
+
const tableRows = [];
|
|
1267
|
+
let cursor = index;
|
|
1268
|
+
while (cursor < rows.length) {
|
|
1269
|
+
const cells = this.splitLikelyTableRow(rows[cursor].text);
|
|
1270
|
+
if (cells.length < 2)
|
|
1271
|
+
break;
|
|
1272
|
+
tableRows.push(cells);
|
|
1273
|
+
cursor += 1;
|
|
1274
|
+
}
|
|
1275
|
+
if (tableRows.length >= 2) {
|
|
1276
|
+
output.push('### PDF 表格区域');
|
|
1277
|
+
output.push(this.toMarkdownTable(tableRows[0], tableRows.slice(1)));
|
|
1278
|
+
index = cursor;
|
|
1279
|
+
continue;
|
|
1280
|
+
}
|
|
1281
|
+
output.push(this.pdfRowToMarkdown(rows[index].text, rows[index].height, index));
|
|
1282
|
+
index += 1;
|
|
1283
|
+
}
|
|
1284
|
+
return output.join('\n');
|
|
1285
|
+
}
|
|
1286
|
+
splitLikelyTableRow(text) {
|
|
1287
|
+
const byLargeSpaces = text.split(/\s{2,}/u).map(cell => cell.trim()).filter(Boolean);
|
|
1288
|
+
if (byLargeSpaces.length >= 2)
|
|
1289
|
+
return byLargeSpaces;
|
|
1290
|
+
const byPipes = text.split('|').map(cell => cell.trim()).filter(Boolean);
|
|
1291
|
+
return byPipes.length >= 2 ? byPipes : [];
|
|
1292
|
+
}
|
|
1293
|
+
pdfRowToMarkdown(text, height, index) {
|
|
1294
|
+
if (index === 0 && text.length <= 100)
|
|
1295
|
+
return `# ${text}`;
|
|
1296
|
+
if (height >= 14 && text.length <= 120)
|
|
1297
|
+
return `## ${text}`;
|
|
1298
|
+
if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(text) && text.length <= 120)
|
|
1299
|
+
return `### ${text}`;
|
|
1300
|
+
return text;
|
|
1301
|
+
}
|
|
1302
|
+
toMarkdownDocument(text) {
|
|
1303
|
+
const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
|
|
1304
|
+
return lines.map((line, index) => {
|
|
1305
|
+
if (/^#{1,6}\s/u.test(line) || /^\|/u.test(line))
|
|
1306
|
+
return line;
|
|
1307
|
+
if (line.length <= 80 && !/[。!?.!?]$/u.test(line)) {
|
|
1308
|
+
if (index === 0)
|
|
1309
|
+
return `# ${line}`;
|
|
1310
|
+
if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(line))
|
|
1311
|
+
return `## ${line}`;
|
|
1312
|
+
return `### ${line}`;
|
|
1313
|
+
}
|
|
1314
|
+
return line;
|
|
1315
|
+
}).join('\n\n');
|
|
1316
|
+
}
|
|
1317
|
+
normalizeMarkdownHeadings(text) {
|
|
1318
|
+
return text
|
|
1319
|
+
.split(/\r?\n/u)
|
|
1320
|
+
.map(line => {
|
|
1321
|
+
const trimmed = line.trim();
|
|
1322
|
+
if (!trimmed)
|
|
1323
|
+
return '';
|
|
1324
|
+
if (/^#{1,6}\s/u.test(trimmed) || /^\|/u.test(trimmed))
|
|
1325
|
+
return trimmed;
|
|
1326
|
+
if (trimmed.length <= 80 && /^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(trimmed))
|
|
1327
|
+
return `## ${trimmed}`;
|
|
1328
|
+
return trimmed;
|
|
1329
|
+
})
|
|
1330
|
+
.join('\n');
|
|
1331
|
+
}
|
|
1001
1332
|
extractSvg(file) {
|
|
1002
1333
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
1003
|
-
const
|
|
1004
|
-
const titles = this.matchAll(raw, /<(?:title|desc)\b[^>]*>([\s\S]*?)<\/(?:title|desc)>/giu).map(value => this.stripXml(value)).slice(0, 100);
|
|
1334
|
+
const nodes = this.extractSvgSemanticNodes(raw);
|
|
1005
1335
|
return {
|
|
1006
|
-
text: [this.metadataOnlyText(file),
|
|
1007
|
-
metadata: { extractionMode: 'svg_text_nodes', vectorizable: true, textNodeCount:
|
|
1336
|
+
text: [this.metadataOnlyText(file), 'SVG 层级语义节点:', ...nodes].join('\n'),
|
|
1337
|
+
metadata: { extractionMode: 'svg_text_nodes', semanticExtractionMode: 'svg_semantic_tree_nodes', vectorizable: true, textNodeCount: nodes.length, contentCoverage: 'svg_hierarchical_text_title_desc' },
|
|
1008
1338
|
warnings: [],
|
|
1009
1339
|
};
|
|
1010
1340
|
}
|
|
1341
|
+
extractSvgSemanticNodes(raw) {
|
|
1342
|
+
const nodes = [];
|
|
1343
|
+
const stack = [];
|
|
1344
|
+
const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b([^>]*)>|([^<>]+)/gu;
|
|
1345
|
+
for (const match of raw.matchAll(tokenPattern)) {
|
|
1346
|
+
const tag = match[1];
|
|
1347
|
+
const attrs = match[2] ?? '';
|
|
1348
|
+
const text = match[3]?.replace(/\s+/gu, ' ').trim();
|
|
1349
|
+
const token = match[0];
|
|
1350
|
+
if (tag && token.startsWith('</'))
|
|
1351
|
+
stack.pop();
|
|
1352
|
+
else if (tag && !token.endsWith('/>')) {
|
|
1353
|
+
const id = /\bid="([^"]+)"/u.exec(attrs)?.[1];
|
|
1354
|
+
stack.push(id ? `${tag}#${id}` : tag);
|
|
1355
|
+
}
|
|
1356
|
+
else if (text && ['text', 'title', 'desc'].includes(stack.at(-1)?.split('#')[0] ?? '')) {
|
|
1357
|
+
nodes.push(`${stack.join(' > ')}: ${this.stripXml(text)}`);
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
return nodes;
|
|
1361
|
+
}
|
|
1011
1362
|
isTextReadable(file) {
|
|
1012
1363
|
if (file.category === 'code' || file.category === 'web')
|
|
1013
1364
|
return true;
|
|
@@ -1042,13 +1393,61 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
|
|
|
1042
1393
|
return [`${prefix || 'value'}: ${String(value)}`];
|
|
1043
1394
|
}
|
|
1044
1395
|
if (Array.isArray(value)) {
|
|
1045
|
-
return value.
|
|
1396
|
+
return value.flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
|
|
1046
1397
|
}
|
|
1047
1398
|
if (typeof value === 'object') {
|
|
1048
1399
|
return Object.entries(value).flatMap(([key, item]) => this.flattenJson(item, prefix ? `${prefix}.${key}` : key));
|
|
1049
1400
|
}
|
|
1050
1401
|
return [];
|
|
1051
1402
|
}
|
|
1403
|
+
atomicJsonObjects(value, prefix = '$') {
|
|
1404
|
+
if (value == null || typeof value !== 'object')
|
|
1405
|
+
return [];
|
|
1406
|
+
if (Array.isArray(value))
|
|
1407
|
+
return value.flatMap((item, index) => this.atomicJsonObjects(item, `${prefix}[${index}]`));
|
|
1408
|
+
const entries = Object.entries(value);
|
|
1409
|
+
const current = `${prefix}: ${JSON.stringify(value)}`;
|
|
1410
|
+
const children = entries.flatMap(([key, item]) => this.atomicJsonObjects(item, `${prefix}.${key}`));
|
|
1411
|
+
return [current, ...children];
|
|
1412
|
+
}
|
|
1413
|
+
flattenYamlByIndent(raw) {
|
|
1414
|
+
const stack = [];
|
|
1415
|
+
const lines = [];
|
|
1416
|
+
for (const line of raw.split(/\r?\n/u)) {
|
|
1417
|
+
if (!line.trim() || /^\s*#/u.test(line))
|
|
1418
|
+
continue;
|
|
1419
|
+
const match = /^(\s*)([-\w.]+)\s*:\s*(.*)$/u.exec(line);
|
|
1420
|
+
if (!match)
|
|
1421
|
+
continue;
|
|
1422
|
+
const indent = match[1].length;
|
|
1423
|
+
const key = match[2];
|
|
1424
|
+
const value = match[3].trim();
|
|
1425
|
+
while (stack.length > 0 && stack.at(-1).indent >= indent)
|
|
1426
|
+
stack.pop();
|
|
1427
|
+
const pathName = [...stack.map(item => item.key), key].join('.');
|
|
1428
|
+
if (value)
|
|
1429
|
+
lines.push(`${pathName}: ${value}`);
|
|
1430
|
+
stack.push({ indent, key });
|
|
1431
|
+
}
|
|
1432
|
+
return lines;
|
|
1433
|
+
}
|
|
1434
|
+
flattenXmlPaths(raw) {
|
|
1435
|
+
const lines = [];
|
|
1436
|
+
const stack = [];
|
|
1437
|
+
const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b[^>]*>|([^<>]+)/gu;
|
|
1438
|
+
for (const match of raw.matchAll(tokenPattern)) {
|
|
1439
|
+
const tag = match[1];
|
|
1440
|
+
const text = match[2]?.replace(/\s+/gu, ' ').trim();
|
|
1441
|
+
const token = match[0];
|
|
1442
|
+
if (tag && token.startsWith('</'))
|
|
1443
|
+
stack.pop();
|
|
1444
|
+
else if (tag && !token.endsWith('/>'))
|
|
1445
|
+
stack.push(tag);
|
|
1446
|
+
else if (text && stack.length > 0)
|
|
1447
|
+
lines.push(`${stack.join('.')}: ${this.stripXml(text)}`);
|
|
1448
|
+
}
|
|
1449
|
+
return lines.filter(line => !line.endsWith(':'));
|
|
1450
|
+
}
|
|
1052
1451
|
metadataOnlyText(file) {
|
|
1053
1452
|
const fileName = path.basename(file.relativePath);
|
|
1054
1453
|
const directory = path.dirname(file.relativePath);
|