@customize-agent/knowledge 4.0.1 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/dist/chunking/bge-tokenizer.d.ts +10 -0
  2. package/dist/chunking/bge-tokenizer.js +68 -0
  3. package/dist/chunking/text-chunker.d.ts +10 -0
  4. package/dist/chunking/text-chunker.js +158 -53
  5. package/dist/classification/classifier.js +0 -1
  6. package/dist/core/index-state-store.d.ts +47 -0
  7. package/dist/core/index-state-store.js +184 -50
  8. package/dist/core/knowledge-base-manager.d.ts +24 -2
  9. package/dist/core/knowledge-base-manager.js +195 -51
  10. package/dist/core/multi-project-manager.d.ts +2 -0
  11. package/dist/core/multi-project-manager.js +13 -2
  12. package/dist/embedding/embedding-provider.d.ts +22 -0
  13. package/dist/embedding/embedding-provider.js +116 -2
  14. package/dist/extraction/content-extractor.d.ts +30 -2
  15. package/dist/extraction/content-extractor.js +514 -115
  16. package/dist/index.d.ts +2 -2
  17. package/dist/index.js +2 -2
  18. package/dist/search/federation-search.d.ts +1 -0
  19. package/dist/search/federation-search.js +1 -0
  20. package/dist/vector/hnsw-vector-store.d.ts +20 -0
  21. package/dist/vector/hnsw-vector-store.js +107 -0
  22. package/dist/vector/types.d.ts +2 -0
  23. package/dist/vector/vector-indexer.d.ts +2 -0
  24. package/dist/vector/vector-indexer.js +13 -1
  25. package/models/bge-small-zh-v1.5/config.json +31 -0
  26. package/models/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
  27. package/models/bge-small-zh-v1.5/special_tokens_map.json +7 -0
  28. package/models/bge-small-zh-v1.5/tokenizer.json +21278 -0
  29. package/models/bge-small-zh-v1.5/tokenizer_config.json +15 -0
  30. package/package.json +11 -4
  31. package/scripts/install-hnsw.cjs +47 -0
  32. package/dist/vector/sqlite-vec-store.d.ts +0 -38
  33. package/dist/vector/sqlite-vec-store.js +0 -203
@@ -69,12 +69,6 @@ export class ContentExtractor {
69
69
  Object.assign(metadata, result.metadata);
70
70
  warnings.push(...result.warnings);
71
71
  }
72
- else if (file.category === 'archive') {
73
- const result = await this.extractArchive(file);
74
- text = result.text;
75
- Object.assign(metadata, result.metadata);
76
- warnings.push(...result.warnings);
77
- }
78
72
  else if (file.category === 'image' && file.format !== 'vector') {
79
73
  const result = await this.extractRasterImage(file);
80
74
  text = result.text;
@@ -208,7 +202,7 @@ export class ContentExtractor {
208
202
  if (file.format === 'autocad' && ext === '.dxf') {
209
203
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
210
204
  const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
211
- const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 500);
205
+ const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
212
206
  const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
213
207
  const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
214
208
  const uniqueLayers = Array.from(new Set(layers));
@@ -221,14 +215,16 @@ export class ContentExtractor {
221
215
  metadata.blockNames = uniqueBlocks.slice(0, 80);
222
216
  metadata.entityTypeCount = uniqueEntityTypes.length;
223
217
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
224
- metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
218
+ metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
219
+ const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
225
220
  return {
226
221
  text: [
227
222
  this.metadataOnlyText(file),
228
223
  `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
229
224
  `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
230
225
  `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
231
- `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
226
+ 'CAD 语义图纸节点:',
227
+ ...semanticNodes,
232
228
  ].join('\n'),
233
229
  metadata,
234
230
  warnings,
@@ -307,7 +303,7 @@ export class ContentExtractor {
307
303
  warnings.push('dxf-parser 解析失败,已使用 DXF 文本结构抽取回退');
308
304
  }
309
305
  const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
310
- const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 800);
306
+ const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
311
307
  const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
312
308
  const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
313
309
  const uniqueLayers = Array.from(new Set(layers));
@@ -320,20 +316,71 @@ export class ContentExtractor {
320
316
  metadata.blockNames = uniqueBlocks.slice(0, 80);
321
317
  metadata.entityTypeCount = uniqueEntityTypes.length;
322
318
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
323
- metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
319
+ metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
324
320
  metadata.parsedByDxfParser = Boolean(parsed);
321
+ const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
325
322
  return {
326
323
  text: [
327
324
  this.metadataOnlyText(file),
328
325
  `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
329
326
  `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
330
327
  `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
331
- `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
328
+ 'CAD 语义图纸节点:',
329
+ ...semanticNodes,
332
330
  ].join('\n'),
333
331
  metadata,
334
332
  warnings,
335
333
  };
336
334
  }
335
+ buildCadSemanticNodes(file, layers, blocks, entityTypes, texts) {
336
+ const fileName = path.basename(file.relativePath);
337
+ const defaultLayer = layers[0] ?? '未命名图层';
338
+ const defaultBlock = blocks[0] ?? '全局模型空间';
339
+ const defaultEntity = entityTypes.find(type => /DIMENSION|TEXT|MTEXT|LEADER/u.test(type)) ?? entityTypes[0] ?? 'UNKNOWN';
340
+ const annotations = texts.length > 0 ? texts : [{ text: '未提取到文字标注', layer: defaultLayer, block: defaultBlock, entityType: defaultEntity }];
341
+ return annotations.map((annotation, index) => {
342
+ const nearest = this.findNearestCadAnnotation(annotation, annotations);
343
+ const layer = annotation.layer ?? layers[index % Math.max(1, layers.length)] ?? defaultLayer;
344
+ const block = annotation.block ?? blocks[index % Math.max(1, blocks.length)] ?? defaultBlock;
345
+ const entity = annotation.entityType ?? this.inferCadEntityType(annotation.text, defaultEntity);
346
+ const status = /关键|critical|尺寸|dim|mm|cm|m\b|°|φ|Φ|R\d/iu.test(annotation.text) ? '关键尺寸/约束候选' : '普通标注';
347
+ const position = annotation.x != null && annotation.y != null ? ` | 坐标: (${annotation.x.toFixed(2)}, ${annotation.y.toFixed(2)})` : '';
348
+ return [`图纸节点: ${fileName} | 图层: ${layer} | 块: ${block} | 实体类型: ${entity}${position}`, `└── 标注文本: ${annotation.text} | 关联对象: ${nearest ? `邻近标注 ${nearest.text}` : '空间邻近候选'} | 状态: ${status}`].join('\n');
349
+ });
350
+ }
351
+ extractDxfTextAnnotations(raw) {
352
+ const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
353
+ return entities.flatMap(section => {
354
+ const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
355
+ if (!text)
356
+ return [];
357
+ return [{
358
+ text,
359
+ layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
360
+ block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
361
+ entityType: section.trim().split(/\s+/u)[0],
362
+ x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
363
+ y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
364
+ }].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
365
+ });
366
+ }
367
+ findNearestCadAnnotation(target, annotations) {
368
+ if (target.x == null || target.y == null)
369
+ return undefined;
370
+ return annotations
371
+ .filter(item => item !== target && item.x != null && item.y != null)
372
+ .map(item => ({ item, distance: Math.hypot((item.x ?? 0) - target.x, (item.y ?? 0) - target.y) }))
373
+ .sort((a, b) => a.distance - b.distance)[0]?.item;
374
+ }
375
+ inferCadEntityType(text, fallback) {
376
+ if (/\b\d+(?:\.\d+)?\s*(?:mm|cm|m)\b|φ|Φ|R\d/iu.test(text))
377
+ return '线性尺寸/半径尺寸';
378
+ if (/°|angle|角度/iu.test(text))
379
+ return '角度尺寸';
380
+ if (/note|说明|备注/iu.test(text))
381
+ return '文字说明';
382
+ return fallback;
383
+ }
337
384
  async tryConvertDwgWithBundledWasm(filePath) {
338
385
  try {
339
386
  const mod = await resolveAndImport('dwgdxf');
@@ -447,36 +494,39 @@ export class ContentExtractor {
447
494
  }
448
495
  extractData(file) {
449
496
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
450
- const metadata = { extractionMode: 'structured_data', vectorizable: true };
497
+ const metadata = { extractionMode: 'structured_data', semanticExtractionMode: 'structured_data_semantic_paths', vectorizable: true };
451
498
  try {
452
499
  if (file.format === 'json') {
453
500
  const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
454
- const lines = isJsonl
455
- ? raw.split(/\r?\n/u).filter(Boolean).slice(0, 500).flatMap((line, index) => this.flattenJson(JSON.parse(line), `line${index + 1}`))
456
- : this.flattenJson(JSON.parse(raw));
501
+ const records = isJsonl
502
+ ? raw.split(/\r?\n/u).filter(Boolean).map((line, index) => ({ path: `line${index + 1}`, value: JSON.parse(line) }))
503
+ : [{ path: '$', value: JSON.parse(raw) }];
504
+ const lines = records.flatMap(record => this.flattenJson(record.value, record.path));
505
+ const objects = records.flatMap(record => this.atomicJsonObjects(record.value, record.path));
457
506
  metadata.fieldCount = lines.length;
458
- metadata.recordCount = isJsonl ? raw.split(/\r?\n/u).filter(Boolean).length : 1;
507
+ metadata.recordCount = records.length;
508
+ metadata.objectCount = objects.length;
459
509
  metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
460
- metadata.contentCoverage = isJsonl ? 'jsonl_records_paths_values' : 'json_paths_values';
461
- return { text: [this.metadataOnlyText(file), ...lines.slice(0, 1500)].join('\n'), metadata, warnings: [] };
510
+ metadata.contentCoverage = isJsonl ? 'jsonl_atomic_objects_paths_values' : 'json_atomic_objects_paths_values';
511
+ return { text: [this.metadataOnlyText(file), '## 路径声明', ...lines, '## 原子对象', ...objects].join('\n'), metadata, warnings: [] };
462
512
  }
463
513
  }
464
514
  catch {
465
515
  metadata.parseError = true;
466
516
  }
467
517
  if (file.format === 'yaml') {
468
- const pairs = this.matchAll(raw, /^\s*([\w.-]+)\s*:\s*(.{1,300})$/gmu).slice(0, 1500);
469
- metadata.fieldCount = pairs.length;
470
- metadata.dataPaths = pairs.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
471
- metadata.contentCoverage = 'yaml_key_values';
472
- return { text: [this.metadataOnlyText(file), ...pairs].join('\n'), metadata, warnings: [] };
518
+ const lines = this.flattenYamlByIndent(raw).slice(0, 3000);
519
+ metadata.fieldCount = lines.length;
520
+ metadata.dataPaths = lines.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
521
+ metadata.contentCoverage = 'yaml_indented_paths_values';
522
+ return { text: [this.metadataOnlyText(file), '## YAML 路径声明', ...lines].join('\n'), metadata, warnings: [] };
473
523
  }
474
524
  if (file.format === 'xml') {
475
- const elements = this.matchAll(raw, /<([A-Za-z_][\w:.-]*)\b[^>]*>([^<]{1,200})<\/\1>/gu).slice(0, 1000);
525
+ const elements = this.flattenXmlPaths(raw).slice(0, 3000);
476
526
  metadata.elementTextCount = elements.length;
477
- metadata.dataPaths = elements.map(line => line.match(/^<([A-Za-z_][\w:.-]*)/u)?.[1]).filter(Boolean).slice(0, 200);
478
- metadata.contentCoverage = 'xml_element_text';
479
- return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
527
+ metadata.dataPaths = elements.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
528
+ metadata.contentCoverage = 'xml_paths_values';
529
+ return { text: [this.metadataOnlyText(file), '## XML 路径声明', ...elements].join('\n'), metadata, warnings: [] };
480
530
  }
481
531
  metadata.contentCoverage = 'plain_structured_text';
482
532
  return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
@@ -485,18 +535,19 @@ export class ContentExtractor {
485
535
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
486
536
  const metadata = { extractionMode: 'diagram_structural', vectorizable: true };
487
537
  if (file.format === 'drawio') {
488
- const labels = this.matchAll(raw, /(?:value|label)="([^"]+)"/gu).map(value => this.stripXml(value)).slice(0, 300);
489
- metadata.nodeTextCount = labels.length;
490
- metadata.contentCoverage = 'drawio_labels';
491
- return { text: [this.metadataOnlyText(file), `Draw.io 节点/连线文本:\n${labels.join('\n')}`].join('\n'), metadata, warnings: [] };
538
+ const graph = this.extractDrawioGraph(raw);
539
+ metadata.nodeTextCount = graph.nodes.length;
540
+ metadata.edgeCount = graph.edges.length;
541
+ metadata.contentCoverage = 'drawio_graph_links';
542
+ return { text: [this.metadataOnlyText(file), 'Draw.io 图链路:', ...graph.links, 'Draw.io 节点:', ...graph.nodes.map(node => `[${node.id}] ${node.label}`)].join('\n'), metadata, warnings: [] };
492
543
  }
493
544
  if (file.format === 'excalidraw') {
494
545
  try {
495
- const parsed = JSON.parse(raw);
496
- const texts = (parsed.elements ?? []).filter(element => element.text).map(element => `${element.type ?? 'shape'}: ${element.text}`).slice(0, 300);
497
- metadata.elementTextCount = texts.length;
498
- metadata.contentCoverage = 'excalidraw_text_elements';
499
- return { text: [this.metadataOnlyText(file), `Excalidraw 图形文本:\n${texts.join('\n')}`].join('\n'), metadata, warnings: [] };
546
+ const graph = this.extractExcalidrawGraph(raw);
547
+ metadata.elementTextCount = graph.nodes.length;
548
+ metadata.edgeCount = graph.links.length;
549
+ metadata.contentCoverage = 'excalidraw_graph_links';
550
+ return { text: [this.metadataOnlyText(file), 'Excalidraw 图链路:', ...graph.links, 'Excalidraw 节点:', ...graph.nodes].join('\n'), metadata, warnings: [] };
500
551
  }
501
552
  catch {
502
553
  metadata.parseError = true;
@@ -505,6 +556,52 @@ export class ContentExtractor {
505
556
  metadata.contentCoverage = 'diagram_source_text';
506
557
  return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
507
558
  }
559
+ extractDrawioGraph(raw) {
560
+ const cells = Array.from(raw.matchAll(/<mxCell\b([^>]*)>/gu), match => this.parseXmlAttributes(match[1] ?? ''));
561
+ const nodes = cells
562
+ .filter(cell => cell.value || cell.label || cell.id)
563
+ .map((cell, index) => ({ id: String(cell.id ?? `node-${index + 1}`), label: this.stripXml(String(cell.value || cell.label || cell.id)) }))
564
+ .filter(node => node.label);
565
+ const nodeById = new Map(nodes.map(node => [node.id, node.label]));
566
+ const edges = cells.filter(cell => cell.edge === '1' && cell.source && cell.target);
567
+ const links = edges.map(edge => {
568
+ const source = nodeById.get(String(edge.source)) ?? String(edge.source);
569
+ const target = nodeById.get(String(edge.target)) ?? String(edge.target);
570
+ const label = this.stripXml(String(edge.value || edge.label || '关系'));
571
+ return `[${source}] ──> (${label}) ──> [${target}]`;
572
+ });
573
+ return { nodes, edges: edges.map(edge => String(edge.id ?? 'edge')), links };
574
+ }
575
+ extractExcalidrawGraph(raw) {
576
+ const parsed = JSON.parse(raw);
577
+ const elements = parsed.elements ?? [];
578
+ const textById = new Map(elements.filter(element => element.text).map(element => [String(element.id), String(element.text)]));
579
+ const shapeLabels = elements
580
+ .filter(element => element.type !== 'arrow' && element.type !== 'line')
581
+ .map((element, index) => {
582
+ const id = String(element.id ?? `node-${index + 1}`);
583
+ const own = element.text ?? textById.get(id);
584
+ const nested = elements.find(candidate => candidate.text && this.isPointInside(candidate, element));
585
+ return { id, label: own ?? nested?.text ?? element.type ?? '节点' };
586
+ });
587
+ const labelById = new Map(shapeLabels.map(node => [node.id, node.label]));
588
+ const links = elements
589
+ .filter(element => element.type === 'arrow' && element.startBinding?.elementId && element.endBinding?.elementId)
590
+ .map(element => `[${labelById.get(element.startBinding.elementId) ?? element.startBinding.elementId}] ──> (箭头) ──> [${labelById.get(element.endBinding.elementId) ?? element.endBinding.elementId}]`);
591
+ const nodes = shapeLabels.map(node => `[${node.id}] ${node.label}`);
592
+ return { nodes, links };
593
+ }
594
+ isPointInside(point, box) {
595
+ if (point.x == null || point.y == null || box.x == null || box.y == null || box.width == null || box.height == null)
596
+ return false;
597
+ return point.x >= box.x && point.x <= box.x + box.width && point.y >= box.y && point.y <= box.y + box.height;
598
+ }
599
+ parseXmlAttributes(input) {
600
+ const attrs = {};
601
+ for (const match of input.matchAll(/([\w:-]+)="([^"]*)"/gu))
602
+ attrs[match[1]] = this.stripXml(match[2] ?? '');
603
+ return attrs;
604
+ }
508
605
  parseDelimitedLine(line, delimiter) {
509
606
  const values = [];
510
607
  let current = '';
@@ -531,28 +628,38 @@ export class ContentExtractor {
531
628
  values.push(current.trim());
532
629
  return values;
533
630
  }
631
+ toMarkdownTable(header, rows) {
632
+ const width = Math.max(header.length, ...rows.map(row => row.length), 1);
633
+ const normalizedHeader = Array.from({ length: width }, (_, index) => header[index] || `COL${index + 1}`);
634
+ const escape = (value) => String(value ?? '').replace(/\|/gu, '\\|').replace(/\r?\n/gu, ' ').trim();
635
+ return [
636
+ `| ${normalizedHeader.map(escape).join(' | ')} |`,
637
+ `| ${normalizedHeader.map(() => '---').join(' | ')} |`,
638
+ ...rows.map(row => `| ${Array.from({ length: width }, (_, index) => escape(row[index] ?? '')).join(' | ')} |`),
639
+ ].join('\n');
640
+ }
534
641
  extractDelimitedText(file) {
535
642
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
536
643
  const delimiter = file.format === 'tsv' ? '\t' : ',';
537
644
  const rows = raw.split(/\r?\n/u).filter(line => line.trim().length > 0);
538
645
  const header = rows[0] ? this.parseDelimitedLine(rows[0], delimiter) : [];
539
- const structured = rows.slice(1, 501).flatMap((line, rowIndex) => {
540
- const values = this.parseDelimitedLine(line, delimiter);
541
- return values.map((value, colIndex) => {
542
- const column = header[colIndex] || `COL${colIndex + 1}`;
543
- return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
544
- });
545
- });
646
+ const tableRows = rows.slice(1).map(line => this.parseDelimitedLine(line, delimiter));
647
+ const markdown = this.toMarkdownTable(header, tableRows);
648
+ const legacyKv = tableRows.flatMap((values, rowIndex) => values.map((value, colIndex) => {
649
+ const column = header[colIndex] || `COL${colIndex + 1}`;
650
+ return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
651
+ }));
546
652
  return {
547
- text: [this.metadataOnlyText(file), `表头: ${header.join(' | ')}`, ...structured, raw].join('\n'),
653
+ text: [this.metadataOnlyText(file), '### 工作表: 默认', markdown, '### 表格路径声明', ...legacyKv].join('\n\n'),
548
654
  metadata: {
549
655
  extractionMode: 'delimited_text_structured',
656
+ semanticExtractionMode: 'delimited_markdown_table',
550
657
  vectorizable: true,
551
658
  delimiter: file.format === 'tsv' ? 'tab' : 'comma',
552
659
  rowCount: rows.length,
553
660
  columnCount: header.length,
554
661
  columnNames: header.slice(0, 120),
555
- contentCoverage: 'table_headers_cells_text',
662
+ contentCoverage: 'markdown_table',
556
663
  },
557
664
  warnings: [],
558
665
  };
@@ -567,13 +674,21 @@ export class ContentExtractor {
567
674
  return this.extractLegacyOfficeBinary(file);
568
675
  if (ext === '.docx') {
569
676
  try {
677
+ const styledMarkdown = await this.extractDocxStyleTreeMarkdown(file.absolutePath);
678
+ if (styledMarkdown.trim()) {
679
+ return {
680
+ text: styledMarkdown,
681
+ metadata: { extractionMode: 'docx_xml_style_tree_markdown', vectorizable: true, contentCoverage: 'office_style_tree_markdown' },
682
+ warnings: [],
683
+ };
684
+ }
570
685
  const mammoth = await resolveAndImport('mammoth');
571
- const result = await mammoth.extractRawText({ path: file.absolutePath });
572
- const text = result.value.trim();
686
+ const result = await mammoth.convertToMarkdown({ path: file.absolutePath });
687
+ const text = String(result.value ?? '').trim();
573
688
  if (text) {
574
689
  return {
575
- text,
576
- metadata: { extractionMode: 'builtin_mammoth', vectorizable: true, contentCoverage: 'office_full_text' },
690
+ text: this.normalizeMarkdownHeadings(text),
691
+ metadata: { extractionMode: 'builtin_mammoth_markdown', vectorizable: true, contentCoverage: 'office_markdown_structure' },
577
692
  warnings: result.messages.map(m => m.message),
578
693
  };
579
694
  }
@@ -630,6 +745,14 @@ export class ContentExtractor {
630
745
  warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库'],
631
746
  };
632
747
  }
748
+ findMergedCellValue(sheet, merges, row, col, XLSX) {
749
+ const merge = merges.find(item => row >= item.s.r && row <= item.e.r && col >= item.s.c && col <= item.e.c);
750
+ if (!merge)
751
+ return undefined;
752
+ const originAddress = XLSX.utils.encode_cell({ r: merge.s.r, c: merge.s.c });
753
+ const origin = sheet[originAddress];
754
+ return origin?.w ?? (origin?.v == null ? undefined : String(origin.v));
755
+ }
633
756
  async extractSpreadsheet(file) {
634
757
  const ext = path.extname(file.absolutePath).toLowerCase();
635
758
  try {
@@ -644,28 +767,32 @@ export class ContentExtractor {
644
767
  if (!sheet || !sheet['!ref'])
645
768
  continue;
646
769
  const range = XLSX.utils.decode_range(sheet['!ref']);
647
- const lines = [`工作表: ${name}`, `范围: ${sheet['!ref']}`];
648
770
  const merges = sheet['!merges'] ?? [];
649
771
  mergeCount += merges.length;
650
- if (merges.length > 0) {
651
- lines.push(`合并单元格: ${merges.map(item => `${XLSX.utils.encode_cell(item.s)}:${XLSX.utils.encode_cell(item.e)}`).join(', ')}`);
652
- }
772
+ const matrix = [];
653
773
  for (let row = range.s.r; row <= range.e.r; row++) {
774
+ const values = [];
654
775
  for (let col = range.s.c; col <= range.e.c; col++) {
655
776
  const address = XLSX.utils.encode_cell({ r: row, c: col });
656
777
  const cell = sheet[address];
657
- if (!cell || (cell.v == null && !cell.f))
658
- continue;
659
- cellCount++;
660
- if (cell.f)
661
- formulaCount++;
662
- const display = cell.w ?? String(cell.v ?? '');
663
- const formula = cell.f ? ` 公式=${cell.f}` : '';
664
- lines.push(`${address}: ${display}${formula}`);
778
+ const mergedValue = cell ? undefined : this.findMergedCellValue(sheet, merges, row, col, XLSX);
779
+ if (cell) {
780
+ cellCount++;
781
+ if (cell.f)
782
+ formulaCount++;
783
+ }
784
+ const display = cell?.w ?? String(cell?.v ?? mergedValue ?? '');
785
+ const formula = cell?.f ? ` 公式=${cell.f}` : '';
786
+ values.push(`${display}${formula}`.trim());
665
787
  }
788
+ if (values.some(Boolean))
789
+ matrix.push(values);
790
+ }
791
+ if (matrix.length > 0) {
792
+ const header = matrix[0] ?? [];
793
+ const rows = matrix.slice(1);
794
+ sheetTexts.push([`### 工作表: ${name}`, this.toMarkdownTable(header, rows)].join('\n\n'));
666
795
  }
667
- if (lines.length > 2)
668
- sheetTexts.push(lines.slice(0, 5_000).join('\n'));
669
796
  }
670
797
  if (sheetTexts.length > 0) {
671
798
  return {
@@ -684,6 +811,41 @@ export class ContentExtractor {
684
811
  return this.extractLegacyOfficeBinary(file);
685
812
  return this.extractOfficeZip(file);
686
813
  }
814
+ async extractDocxStyleTreeMarkdown(filePath) {
815
+ const jszipMod = await resolveAndImport('jszip');
816
+ const JSZip = jszipMod.default ?? jszipMod;
817
+ const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
818
+ const docXml = await zip.files['word/document.xml']?.async('string');
819
+ if (!docXml)
820
+ return '';
821
+ const paragraphs = Array.from(docXml.matchAll(/<w:p[\s\S]*?<\/w:p>/gu), match => match[0]);
822
+ const lines = [];
823
+ for (const paragraph of paragraphs) {
824
+ const texts = Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
825
+ if (!texts.trim())
826
+ continue;
827
+ const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(paragraph)?.[1] ?? '';
828
+ const bold = /<w:b\b/u.test(paragraph);
829
+ const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(paragraph)?.[1] ?? 0);
830
+ const level = this.docxHeadingLevel(style, bold, size, texts);
831
+ lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
832
+ }
833
+ return lines.join('\n\n');
834
+ }
835
+ docxHeadingLevel(style, bold, size, text) {
836
+ const normalized = style.toLowerCase();
837
+ const heading = /heading(\d)|标题(\d)|h(\d)/iu.exec(normalized);
838
+ const styleLevel = Number(heading?.[1] ?? heading?.[2] ?? heading?.[3] ?? 0);
839
+ if (styleLevel >= 1 && styleLevel <= 6)
840
+ return styleLevel;
841
+ if (text.length <= 100 && size >= 32)
842
+ return 1;
843
+ if (text.length <= 100 && size >= 28)
844
+ return 2;
845
+ if (text.length <= 100 && (size >= 24 || bold))
846
+ return 3;
847
+ return 0;
848
+ }
687
849
  async extractOfficeZip(file) {
688
850
  const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
689
851
  try {
@@ -709,26 +871,6 @@ export class ContentExtractor {
709
871
  return { text: '', metadata, warnings: [`Office/表格/演示文件解析失败: ${metadata.parseError},未入库`] };
710
872
  }
711
873
  }
712
- async extractArchive(file) {
713
- const metadata = { extractionMode: 'archive_manifest', vectorizable: true };
714
- if (path.extname(file.absolutePath).toLowerCase() !== '.zip') {
715
- metadata.contentCoverage = 'metadata_filename';
716
- return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
717
- }
718
- try {
719
- const jszipMod = await resolveAndImport('jszip');
720
- const JSZip = jszipMod.default ?? jszipMod;
721
- const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
722
- const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
723
- metadata.entryCount = Object.keys(zip.files).length;
724
- metadata.contentCoverage = 'zip_manifest';
725
- return { text: [this.metadataOnlyText(file), '压缩包文件清单:', ...entries].join('\n'), metadata, warnings: [] };
726
- }
727
- catch (error) {
728
- metadata.parseError = error instanceof Error ? error.message : String(error);
729
- return { text: '', metadata, warnings: ['压缩包解析失败,内置解析器未提取到文件清单,未入库'] };
730
- }
731
- }
732
874
  async extractRasterImage(file) {
733
875
  const metadata = { extractionMode: 'builtin_tesseract_ocr_isolated', vectorizable: true };
734
876
  if (process.env.CUSTOMIZE_AGENT_DISABLE_OCR === '1') {
@@ -742,6 +884,13 @@ export class ContentExtractor {
742
884
  metadata.parseError = validationError;
743
885
  return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
744
886
  }
887
+ const paddle = await this.tryPaddleOcrLayout(file.absolutePath);
888
+ if (paddle) {
889
+ metadata.contentCoverage = 'paddleocr_layout_regions';
890
+ metadata.ocrProvider = 'paddleocr';
891
+ metadata.ocrRegionCount = paddle.regionCount;
892
+ return { text: [this.metadataOnlyText(file), paddle.text].join('\n'), metadata, warnings: [] };
893
+ }
745
894
  let tesseractPath;
746
895
  try {
747
896
  tesseractPath = resolvePackage('tesseract.js');
@@ -766,7 +915,12 @@ const { createWorker } = tesseractMod;
766
915
  const worker = await createWorker('chi_sim+eng');
767
916
  try {
768
917
  const result = await worker.recognize(process.argv[1]);
769
- process.stdout.write(result.data.text || '');
918
+ const lines = (result.data.lines || []).map((line, index) => ({
919
+ index: index + 1,
920
+ text: line.text || '',
921
+ bbox: line.bbox || line.baseline || null,
922
+ })).filter(line => line.text.trim());
923
+ process.stdout.write(JSON.stringify({ text: result.data.text || '', lines }));
770
924
  } finally {
771
925
  await worker.terminate();
772
926
  }`,
@@ -778,17 +932,73 @@ try {
778
932
  metadata.parseError = message;
779
933
  return { text: '', metadata, warnings: [`内置 OCR 解析失败:${message},未入库`] };
780
934
  }
781
- const text = result.stdout.trim();
782
- metadata.contentCoverage = text ? 'ocr_text' : 'metadata_filename';
935
+ const parsed = this.parseOcrJson(result.stdout);
936
+ const text = parsed.text.trim();
937
+ const regions = this.formatOcrRegions(parsed.lines);
938
+ metadata.contentCoverage = text ? 'ocr_text_bounding_boxes' : 'metadata_filename';
783
939
  metadata.ocrProvider = 'tesseract.js';
784
940
  metadata.ocrLanguages = 'chi_sim+eng';
785
941
  metadata.ocrTextLength = text.length;
942
+ metadata.ocrLineCount = parsed.lines.length;
786
943
  return {
787
- text: text ? [this.metadataOnlyText(file), `OCR 识别文本:\n${text}`].join('\n') : '',
944
+ text: text ? [this.metadataOnlyText(file), 'OCR 区域文本:', ...regions, `OCR 完整文本:\n${text}`].join('\n') : '',
788
945
  metadata,
789
946
  warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
790
947
  };
791
948
  }
949
+ async tryPaddleOcrLayout(filePath) {
950
+ const command = process.env.CUSTOMIZE_PADDLE_OCR_CMD || process.env.PADDLE_OCR_CMD;
951
+ if (!command)
952
+ return undefined;
953
+ const result = spawnSync(command, [filePath], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, shell: true });
954
+ if (result.status !== 0 || !result.stdout.trim())
955
+ return undefined;
956
+ try {
957
+ const parsed = JSON.parse(result.stdout);
958
+ const lines = parsed.map((region, index) => `区域 ${index + 1} [${region.type ?? 'text'}] ${this.formatBoundingBox(region.bbox)}: ${region.text ?? ''}`);
959
+ return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
960
+ }
961
+ catch {
962
+ const lines = result.stdout.split(/\r?\n/u).filter(Boolean);
963
+ return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
964
+ }
965
+ }
966
+ parseOcrJson(raw) {
967
+ try {
968
+ const parsed = JSON.parse(raw);
969
+ return {
970
+ text: parsed.text ?? raw,
971
+ lines: (parsed.lines ?? []).map((line, index) => ({ index: line.index ?? index + 1, text: line.text ?? '', bbox: line.bbox })).filter(line => line.text.trim()),
972
+ };
973
+ }
974
+ catch {
975
+ return { text: raw, lines: raw.split(/\r?\n/u).map((text, index) => ({ index: index + 1, text })).filter(line => line.text.trim()) };
976
+ }
977
+ }
978
+ formatOcrRegions(lines) {
979
+ return lines.map(line => {
980
+ const bbox = this.formatBoundingBox(line.bbox);
981
+ const type = this.classifyOcrRegion(line.text);
982
+ return `区域 ${line.index} [${type}]${bbox ? ` ${bbox}` : ''}: ${line.text}`;
983
+ });
984
+ }
985
+ classifyOcrRegion(text) {
986
+ if (/\|/.test(text) || /\s{2,}/u.test(text) || /表\s*\d|合计|小计/u.test(text))
987
+ return 'table';
988
+ if (/图\s*\d|figure|image|示意图/iu.test(text))
989
+ return 'image-caption';
990
+ return 'text';
991
+ }
992
+ formatBoundingBox(value) {
993
+ if (!value || typeof value !== 'object')
994
+ return '';
995
+ const record = value;
996
+ const x0 = record.x0 ?? record.left ?? record.x;
997
+ const y0 = record.y0 ?? record.top ?? record.y;
998
+ const x1 = record.x1 ?? record.right;
999
+ const y1 = record.y1 ?? record.bottom;
1000
+ return [x0, y0, x1, y1].some(item => item != null) ? `[bbox x0=${x0 ?? ''}, y0=${y0 ?? ''}, x1=${x1 ?? ''}, y1=${y1 ?? ''}]` : '';
1001
+ }
792
1002
  validateRasterImage(filePath) {
793
1003
  const buffer = fs.readFileSync(filePath);
794
1004
  if (buffer.length < 12)
@@ -821,9 +1031,9 @@ try {
821
1031
  const raw = fs.readFileSync(file.absolutePath);
822
1032
  const text = await this.extractPdfText(raw);
823
1033
  if (text.trim()) {
824
- metadata.contentCoverage = 'pdf_text_streams';
1034
+ metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
825
1035
  metadata.pdfExtractor = 'pdfjs-dist';
826
- return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
1036
+ return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
827
1037
  }
828
1038
  }
829
1039
  catch (error) {
@@ -846,10 +1056,6 @@ try {
846
1056
  warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
847
1057
  };
848
1058
  }
849
- pdfOcrPageLimit() {
850
- const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
851
- return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
852
- }
853
1059
  async extractScannedPdfOcr(file) {
854
1060
  const metadata = {
855
1061
  extractionMode: 'pdf_page_ocr_embedded',
@@ -858,9 +1064,8 @@ try {
858
1064
  ocrProvider: 'tesseract.js',
859
1065
  ocrLanguages: 'chi_sim+eng',
860
1066
  pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
861
- pdfOcrPageLimit: this.pdfOcrPageLimit(),
1067
+ pdfOcrPageLimit: 'all',
862
1068
  };
863
- const pageLimitConfig = this.pdfOcrPageLimit();
864
1069
  // 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
865
1070
  let canvasPath;
866
1071
  let pdfjsPath;
@@ -890,10 +1095,9 @@ import { createCanvas } from ${JSON.stringify(canvasPath)};
890
1095
  import * as pdfjs from ${JSON.stringify(pdfjsPath)};
891
1096
  import { createWorker } from ${JSON.stringify(tesseractPath)};
892
1097
  const filePath = process.argv[1];
893
- const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
894
1098
  const bytes = new Uint8Array(fs.readFileSync(filePath));
895
1099
  const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
896
- const pageLimit = Math.min(doc.numPages, configuredLimit);
1100
+ const pageLimit = doc.numPages;
897
1101
  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
898
1102
  const worker = await createWorker('chi_sim+eng');
899
1103
  const pages = [];
@@ -908,7 +1112,8 @@ try {
908
1112
  fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
909
1113
  const recognized = await worker.recognize(imagePath);
910
1114
  const text = (recognized.data.text || '').trim();
911
- if (text) pages.push('PDF OCR ' + i + ' 页:\\n' + text);
1115
+ const lines = (recognized.data.lines || []).map((line, index) => ({ index: index + 1, text: line.text || '', bbox: line.bbox || null })).filter(line => line.text.trim());
1116
+ if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + lines.map(line => '区域 ' + line.index + ' ' + JSON.stringify(line.bbox || {}) + ': ' + line.text).join('\\n') + '\\n\\n完整文本:\\n' + text);
912
1117
  }
913
1118
  } finally {
914
1119
  await worker.terminate();
@@ -916,8 +1121,7 @@ try {
916
1121
  }
917
1122
  process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
918
1123
  file.absolutePath,
919
- String(pageLimitConfig),
920
- ], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
1124
+ ], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, env: childEnv });
921
1125
  if (result.status !== 0 || result.error) {
922
1126
  metadata.ocrRecommended = true;
923
1127
  metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
@@ -927,7 +1131,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
927
1131
  const parsed = JSON.parse(result.stdout);
928
1132
  const text = parsed.text?.trim() ?? '';
929
1133
  metadata.ocrPageCount = parsed.pageCount ?? 0;
930
- metadata.pdfOcrPageLimit = parsed.pageLimit ?? 5;
1134
+ metadata.pdfOcrPageLimit = parsed.pageLimit ?? 'all';
931
1135
  metadata.ocrTextLength = text.length;
932
1136
  metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
933
1137
  return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
@@ -945,17 +1149,14 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
945
1149
  const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
946
1150
  const doc = await loadingTask.promise;
947
1151
  const pages = [];
948
- const pageLimit = Math.min(doc.numPages, 200);
1152
+ const pageLimit = doc.numPages;
949
1153
  for (let i = 1; i <= pageLimit; i++) {
950
1154
  const page = await doc.getPage(i);
951
1155
  const content = await page.getTextContent();
952
- const pageText = content.items
953
- .map((item) => {
954
- const hasStr = item != null && typeof item === 'object' && 'str' in item;
955
- return hasStr ? String(item.str) : '';
956
- })
957
- .filter((s) => s.trim().length > 0)
958
- .join(' ');
1156
+ const items = content.items
1157
+ .map((item) => this.toPdfTextItem(item))
1158
+ .filter((item) => !!item && item.str.trim().length > 0);
1159
+ const pageText = this.layoutPdfTextItems(items, i);
959
1160
  if (pageText.trim())
960
1161
  pages.push(pageText.trim());
961
1162
  }
@@ -998,16 +1199,166 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
998
1199
  .join('')
999
1200
  .trim();
1000
1201
  }
1202
+ toPdfTextItem(item) {
1203
+ if (!item || typeof item !== 'object' || !('str' in item))
1204
+ return undefined;
1205
+ const record = item;
1206
+ const transform = Array.isArray(record.transform) ? record.transform : [];
1207
+ return {
1208
+ str: String(record.str ?? ''),
1209
+ x: Number(transform[4] ?? 0),
1210
+ y: Number(transform[5] ?? 0),
1211
+ width: Number(record.width ?? 0),
1212
+ height: Number(record.height ?? Math.abs(Number(transform[3] ?? 0))),
1213
+ fontName: typeof record.fontName === 'string' ? record.fontName : undefined,
1214
+ };
1215
+ }
1216
+ layoutPdfTextItems(items, pageNumber) {
1217
+ if (items.length === 0)
1218
+ return '';
1219
+ const rows = this.groupPdfItemsIntoRows(items);
1220
+ const columnSplit = this.detectPdfColumnSplit(rows);
1221
+ const orderedRows = columnSplit == null
1222
+ ? rows.sort((a, b) => b.y - a.y || a.x - b.x)
1223
+ : [
1224
+ ...rows.filter(row => row.x < columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
1225
+ ...rows.filter(row => row.x >= columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
1226
+ ];
1227
+ const markdown = this.rowsToPdfMarkdownWithTables(orderedRows);
1228
+ return [`## PDF 第 ${pageNumber} 页`, markdown].join('\n\n');
1229
+ }
1230
+ groupPdfItemsIntoRows(items) {
1231
+ const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
1232
+ const rows = [];
1233
+ for (const item of sorted) {
1234
+ const row = rows.find(candidate => Math.abs(candidate.y - item.y) <= Math.max(2, item.height * 0.55));
1235
+ if (row)
1236
+ row.items.push(item);
1237
+ else
1238
+ rows.push({ y: item.y, items: [item] });
1239
+ }
1240
+ return rows.map(row => {
1241
+ const rowItems = row.items.sort((a, b) => a.x - b.x);
1242
+ return {
1243
+ text: rowItems.map(item => item.str.trim()).filter(Boolean).join(' ').replace(/\s+/gu, ' '),
1244
+ x: Math.min(...rowItems.map(item => item.x)),
1245
+ y: row.y,
1246
+ height: Math.max(...rowItems.map(item => item.height || 0)),
1247
+ };
1248
+ }).filter(row => row.text);
1249
+ }
1250
+ detectPdfColumnSplit(rows) {
1251
+ if (rows.length < 8)
1252
+ return undefined;
1253
+ const xs = rows.map(row => row.x).sort((a, b) => a - b);
1254
+ const gaps = xs.slice(1).map((x, index) => ({ gap: x - xs[index], left: xs[index], right: x })).sort((a, b) => b.gap - a.gap);
1255
+ const largest = gaps[0];
1256
+ if (!largest || largest.gap < 80)
1257
+ return undefined;
1258
+ const leftCount = rows.filter(row => row.x <= largest.left).length;
1259
+ const rightCount = rows.filter(row => row.x >= largest.right).length;
1260
+ return leftCount >= 3 && rightCount >= 3 ? (largest.left + largest.right) / 2 : undefined;
1261
+ }
1262
+ rowsToPdfMarkdownWithTables(rows) {
1263
+ const output = [];
1264
+ let index = 0;
1265
+ while (index < rows.length) {
1266
+ const tableRows = [];
1267
+ let cursor = index;
1268
+ while (cursor < rows.length) {
1269
+ const cells = this.splitLikelyTableRow(rows[cursor].text);
1270
+ if (cells.length < 2)
1271
+ break;
1272
+ tableRows.push(cells);
1273
+ cursor += 1;
1274
+ }
1275
+ if (tableRows.length >= 2) {
1276
+ output.push('### PDF 表格区域');
1277
+ output.push(this.toMarkdownTable(tableRows[0], tableRows.slice(1)));
1278
+ index = cursor;
1279
+ continue;
1280
+ }
1281
+ output.push(this.pdfRowToMarkdown(rows[index].text, rows[index].height, index));
1282
+ index += 1;
1283
+ }
1284
+ return output.join('\n');
1285
+ }
1286
+ splitLikelyTableRow(text) {
1287
+ const byLargeSpaces = text.split(/\s{2,}/u).map(cell => cell.trim()).filter(Boolean);
1288
+ if (byLargeSpaces.length >= 2)
1289
+ return byLargeSpaces;
1290
+ const byPipes = text.split('|').map(cell => cell.trim()).filter(Boolean);
1291
+ return byPipes.length >= 2 ? byPipes : [];
1292
+ }
1293
+ pdfRowToMarkdown(text, height, index) {
1294
+ if (index === 0 && text.length <= 100)
1295
+ return `# ${text}`;
1296
+ if (height >= 14 && text.length <= 120)
1297
+ return `## ${text}`;
1298
+ if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(text) && text.length <= 120)
1299
+ return `### ${text}`;
1300
+ return text;
1301
+ }
1302
+ toMarkdownDocument(text) {
1303
+ const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
1304
+ return lines.map((line, index) => {
1305
+ if (/^#{1,6}\s/u.test(line) || /^\|/u.test(line))
1306
+ return line;
1307
+ if (line.length <= 80 && !/[。!?.!?]$/u.test(line)) {
1308
+ if (index === 0)
1309
+ return `# ${line}`;
1310
+ if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(line))
1311
+ return `## ${line}`;
1312
+ return `### ${line}`;
1313
+ }
1314
+ return line;
1315
+ }).join('\n\n');
1316
+ }
1317
+ normalizeMarkdownHeadings(text) {
1318
+ return text
1319
+ .split(/\r?\n/u)
1320
+ .map(line => {
1321
+ const trimmed = line.trim();
1322
+ if (!trimmed)
1323
+ return '';
1324
+ if (/^#{1,6}\s/u.test(trimmed) || /^\|/u.test(trimmed))
1325
+ return trimmed;
1326
+ if (trimmed.length <= 80 && /^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(trimmed))
1327
+ return `## ${trimmed}`;
1328
+ return trimmed;
1329
+ })
1330
+ .join('\n');
1331
+ }
1001
1332
  extractSvg(file) {
1002
1333
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
1003
- const texts = this.matchAll(raw, /<text\b[^>]*>([\s\S]*?)<\/text>/giu).map(value => this.stripXml(value)).slice(0, 300);
1004
- const titles = this.matchAll(raw, /<(?:title|desc)\b[^>]*>([\s\S]*?)<\/(?:title|desc)>/giu).map(value => this.stripXml(value)).slice(0, 100);
1334
+ const nodes = this.extractSvgSemanticNodes(raw);
1005
1335
  return {
1006
- text: [this.metadataOnlyText(file), `SVG 标题/描述:\n${titles.join('\n')}`, `SVG 文本节点:\n${texts.join('\n')}`].join('\n'),
1007
- metadata: { extractionMode: 'svg_text_nodes', vectorizable: true, textNodeCount: texts.length, contentCoverage: 'svg_text_title_desc' },
1336
+ text: [this.metadataOnlyText(file), 'SVG 层级语义节点:', ...nodes].join('\n'),
1337
+ metadata: { extractionMode: 'svg_text_nodes', semanticExtractionMode: 'svg_semantic_tree_nodes', vectorizable: true, textNodeCount: nodes.length, contentCoverage: 'svg_hierarchical_text_title_desc' },
1008
1338
  warnings: [],
1009
1339
  };
1010
1340
  }
1341
+ extractSvgSemanticNodes(raw) {
1342
+ const nodes = [];
1343
+ const stack = [];
1344
+ const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b([^>]*)>|([^<>]+)/gu;
1345
+ for (const match of raw.matchAll(tokenPattern)) {
1346
+ const tag = match[1];
1347
+ const attrs = match[2] ?? '';
1348
+ const text = match[3]?.replace(/\s+/gu, ' ').trim();
1349
+ const token = match[0];
1350
+ if (tag && token.startsWith('</'))
1351
+ stack.pop();
1352
+ else if (tag && !token.endsWith('/>')) {
1353
+ const id = /\bid="([^"]+)"/u.exec(attrs)?.[1];
1354
+ stack.push(id ? `${tag}#${id}` : tag);
1355
+ }
1356
+ else if (text && ['text', 'title', 'desc'].includes(stack.at(-1)?.split('#')[0] ?? '')) {
1357
+ nodes.push(`${stack.join(' > ')}: ${this.stripXml(text)}`);
1358
+ }
1359
+ }
1360
+ return nodes;
1361
+ }
1011
1362
  isTextReadable(file) {
1012
1363
  if (file.category === 'code' || file.category === 'web')
1013
1364
  return true;
@@ -1042,13 +1393,61 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
1042
1393
  return [`${prefix || 'value'}: ${String(value)}`];
1043
1394
  }
1044
1395
  if (Array.isArray(value)) {
1045
- return value.slice(0, 50).flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
1396
+ return value.flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
1046
1397
  }
1047
1398
  if (typeof value === 'object') {
1048
1399
  return Object.entries(value).flatMap(([key, item]) => this.flattenJson(item, prefix ? `${prefix}.${key}` : key));
1049
1400
  }
1050
1401
  return [];
1051
1402
  }
1403
+ atomicJsonObjects(value, prefix = '$') {
1404
+ if (value == null || typeof value !== 'object')
1405
+ return [];
1406
+ if (Array.isArray(value))
1407
+ return value.flatMap((item, index) => this.atomicJsonObjects(item, `${prefix}[${index}]`));
1408
+ const entries = Object.entries(value);
1409
+ const current = `${prefix}: ${JSON.stringify(value)}`;
1410
+ const children = entries.flatMap(([key, item]) => this.atomicJsonObjects(item, `${prefix}.${key}`));
1411
+ return [current, ...children];
1412
+ }
1413
+ flattenYamlByIndent(raw) {
1414
+ const stack = [];
1415
+ const lines = [];
1416
+ for (const line of raw.split(/\r?\n/u)) {
1417
+ if (!line.trim() || /^\s*#/u.test(line))
1418
+ continue;
1419
+ const match = /^(\s*)([-\w.]+)\s*:\s*(.*)$/u.exec(line);
1420
+ if (!match)
1421
+ continue;
1422
+ const indent = match[1].length;
1423
+ const key = match[2];
1424
+ const value = match[3].trim();
1425
+ while (stack.length > 0 && stack.at(-1).indent >= indent)
1426
+ stack.pop();
1427
+ const pathName = [...stack.map(item => item.key), key].join('.');
1428
+ if (value)
1429
+ lines.push(`${pathName}: ${value}`);
1430
+ stack.push({ indent, key });
1431
+ }
1432
+ return lines;
1433
+ }
1434
+ flattenXmlPaths(raw) {
1435
+ const lines = [];
1436
+ const stack = [];
1437
+ const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b[^>]*>|([^<>]+)/gu;
1438
+ for (const match of raw.matchAll(tokenPattern)) {
1439
+ const tag = match[1];
1440
+ const text = match[2]?.replace(/\s+/gu, ' ').trim();
1441
+ const token = match[0];
1442
+ if (tag && token.startsWith('</'))
1443
+ stack.pop();
1444
+ else if (tag && !token.endsWith('/>'))
1445
+ stack.push(tag);
1446
+ else if (text && stack.length > 0)
1447
+ lines.push(`${stack.join('.')}: ${this.stripXml(text)}`);
1448
+ }
1449
+ return lines.filter(line => !line.endsWith(':'));
1450
+ }
1052
1451
  metadataOnlyText(file) {
1053
1452
  const fileName = path.basename(file.relativePath);
1054
1453
  const directory = path.dirname(file.relativePath);