@customize-agent/knowledge 4.0.1 → 4.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/chunking/bge-tokenizer.d.ts +12 -0
  2. package/dist/chunking/bge-tokenizer.js +70 -0
  3. package/dist/chunking/text-chunker.d.ts +20 -0
  4. package/dist/chunking/text-chunker.js +166 -53
  5. package/dist/classification/classifier.d.ts +1 -0
  6. package/dist/classification/classifier.js +1 -1
  7. package/dist/constants.d.ts +7 -0
  8. package/dist/constants.js +7 -0
  9. package/dist/core/change-tracker.d.ts +13 -0
  10. package/dist/core/change-tracker.js +13 -0
  11. package/dist/core/file-scanner.d.ts +13 -0
  12. package/dist/core/file-scanner.js +12 -0
  13. package/dist/core/index-state-store.d.ts +67 -0
  14. package/dist/core/index-state-store.js +204 -50
  15. package/dist/core/knowledge-base-manager.d.ts +24 -2
  16. package/dist/core/knowledge-base-manager.js +204 -52
  17. package/dist/core/multi-project-manager.d.ts +10 -0
  18. package/dist/core/multi-project-manager.js +21 -2
  19. package/dist/core/project-config.d.ts +4 -0
  20. package/dist/core/project-config.js +4 -0
  21. package/dist/core/project-id.d.ts +5 -0
  22. package/dist/core/project-id.js +5 -0
  23. package/dist/core/project-registry.d.ts +1 -0
  24. package/dist/core/project-registry.js +1 -0
  25. package/dist/dedup/dedup-engine.d.ts +3 -0
  26. package/dist/dedup/dedup-engine.js +1 -0
  27. package/dist/dedup/relationship-detector.d.ts +7 -0
  28. package/dist/dedup/relationship-detector.js +7 -0
  29. package/dist/embedding/embedding-provider.d.ts +32 -0
  30. package/dist/embedding/embedding-provider.js +136 -2
  31. package/dist/extraction/content-extractor.d.ts +32 -2
  32. package/dist/extraction/content-extractor.js +524 -124
  33. package/dist/extraction/external-extractor.d.ts +10 -0
  34. package/dist/extraction/external-extractor.js +6 -0
  35. package/dist/extraction/module-resolver.js +2 -2
  36. package/dist/index.d.ts +2 -2
  37. package/dist/index.js +4 -4
  38. package/dist/search/federation-search.d.ts +8 -0
  39. package/dist/search/federation-search.js +2 -0
  40. package/dist/types.d.ts +10 -0
  41. package/dist/vector/collection-manager.d.ts +3 -0
  42. package/dist/vector/collection-manager.js +3 -0
  43. package/dist/vector/hnsw-vector-store.d.ts +21 -0
  44. package/dist/vector/hnsw-vector-store.js +108 -0
  45. package/dist/vector/types.d.ts +8 -0
  46. package/dist/vector/vector-indexer.d.ts +15 -1
  47. package/dist/vector/vector-indexer.js +34 -5
  48. package/models/bge-small-zh-v1.5/config.json +31 -0
  49. package/models/bge-small-zh-v1.5/onnx/model_quantized.onnx +0 -0
  50. package/models/bge-small-zh-v1.5/special_tokens_map.json +7 -0
  51. package/models/bge-small-zh-v1.5/tokenizer.json +21278 -0
  52. package/models/bge-small-zh-v1.5/tokenizer_config.json +15 -0
  53. package/package.json +16 -9
  54. package/scripts/install-hnsw.cjs +47 -0
  55. package/dist/vector/sqlite-vec-store.d.ts +0 -38
  56. package/dist/vector/sqlite-vec-store.js +0 -203
@@ -5,6 +5,7 @@ import { tmpdir } from 'node:os';
5
5
  import { pathToFileURL } from 'node:url';
6
6
  import { ExternalExtractorRegistry } from './external-extractor.js';
7
7
  import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
8
+ /** 文件内容提取器,支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
8
9
  export class ContentExtractor {
9
10
  externalExtractors;
10
11
  constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
@@ -69,12 +70,6 @@ export class ContentExtractor {
69
70
  Object.assign(metadata, result.metadata);
70
71
  warnings.push(...result.warnings);
71
72
  }
72
- else if (file.category === 'archive') {
73
- const result = await this.extractArchive(file);
74
- text = result.text;
75
- Object.assign(metadata, result.metadata);
76
- warnings.push(...result.warnings);
77
- }
78
73
  else if (file.category === 'image' && file.format !== 'vector') {
79
74
  const result = await this.extractRasterImage(file);
80
75
  text = result.text;
@@ -208,7 +203,7 @@ export class ContentExtractor {
208
203
  if (file.format === 'autocad' && ext === '.dxf') {
209
204
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
210
205
  const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
211
- const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 500);
206
+ const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 500);
212
207
  const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
213
208
  const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
214
209
  const uniqueLayers = Array.from(new Set(layers));
@@ -221,14 +216,16 @@ export class ContentExtractor {
221
216
  metadata.blockNames = uniqueBlocks.slice(0, 80);
222
217
  metadata.entityTypeCount = uniqueEntityTypes.length;
223
218
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
224
- metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
219
+ metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
220
+ const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
225
221
  return {
226
222
  text: [
227
223
  this.metadataOnlyText(file),
228
224
  `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
229
225
  `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
230
226
  `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
231
- `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
227
+ 'CAD 语义图纸节点:',
228
+ ...semanticNodes,
232
229
  ].join('\n'),
233
230
  metadata,
234
231
  warnings,
@@ -307,7 +304,7 @@ export class ContentExtractor {
307
304
  warnings.push('dxf-parser 解析失败,已使用 DXF 文本结构抽取回退');
308
305
  }
309
306
  const layers = this.matchAll(raw, /\n\s*8\s*\n([^\n]+)/gu).slice(0, 300);
310
- const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 800);
307
+ const textEntities = this.extractDxfTextAnnotations(raw).slice(0, 800);
311
308
  const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
312
309
  const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1200);
313
310
  const uniqueLayers = Array.from(new Set(layers));
@@ -320,20 +317,71 @@ export class ContentExtractor {
320
317
  metadata.blockNames = uniqueBlocks.slice(0, 80);
321
318
  metadata.entityTypeCount = uniqueEntityTypes.length;
322
319
  metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
323
- metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
320
+ metadata.contentCoverage = 'dxf_semantic_layer_block_annotations';
324
321
  metadata.parsedByDxfParser = Boolean(parsed);
322
+ const semanticNodes = this.buildCadSemanticNodes(file, uniqueLayers, uniqueBlocks, uniqueEntityTypes, textEntities);
325
323
  return {
326
324
  text: [
327
325
  this.metadataOnlyText(file),
328
326
  `CAD DXF 图层: ${uniqueLayers.join(', ')}`,
329
327
  `CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
330
328
  `CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
331
- `CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
329
+ 'CAD 语义图纸节点:',
330
+ ...semanticNodes,
332
331
  ].join('\n'),
333
332
  metadata,
334
333
  warnings,
335
334
  };
336
335
  }
336
+ buildCadSemanticNodes(file, layers, blocks, entityTypes, texts) {
337
+ const fileName = path.basename(file.relativePath);
338
+ const defaultLayer = layers[0] ?? '未命名图层';
339
+ const defaultBlock = blocks[0] ?? '全局模型空间';
340
+ const defaultEntity = entityTypes.find(type => /DIMENSION|TEXT|MTEXT|LEADER/u.test(type)) ?? entityTypes[0] ?? 'UNKNOWN';
341
+ const annotations = texts.length > 0 ? texts : [{ text: '未提取到文字标注', layer: defaultLayer, block: defaultBlock, entityType: defaultEntity }];
342
+ return annotations.map((annotation, index) => {
343
+ const nearest = this.findNearestCadAnnotation(annotation, annotations);
344
+ const layer = annotation.layer ?? layers[index % Math.max(1, layers.length)] ?? defaultLayer;
345
+ const block = annotation.block ?? blocks[index % Math.max(1, blocks.length)] ?? defaultBlock;
346
+ const entity = annotation.entityType ?? this.inferCadEntityType(annotation.text, defaultEntity);
347
+ const status = /关键|critical|尺寸|dim|mm|cm|m\b|°|φ|Φ|R\d/iu.test(annotation.text) ? '关键尺寸/约束候选' : '普通标注';
348
+ const position = annotation.x != null && annotation.y != null ? ` | 坐标: (${annotation.x.toFixed(2)}, ${annotation.y.toFixed(2)})` : '';
349
+ return [`图纸节点: ${fileName} | 图层: ${layer} | 块: ${block} | 实体类型: ${entity}${position}`, `└── 标注文本: ${annotation.text} | 关联对象: ${nearest ? `邻近标注 ${nearest.text}` : '空间邻近候选'} | 状态: ${status}`].join('\n');
350
+ });
351
+ }
352
+ extractDxfTextAnnotations(raw) {
353
+ const entities = raw.split(/\n\s*0\s*\n/u).filter(section => /^(?:TEXT|MTEXT|DIMENSION|LEADER)/u.test(section.trim()));
354
+ return entities.flatMap(section => {
355
+ const text = /\n\s*(?:1|3)\s*\n([^\n]+)/u.exec(section)?.[1]?.trim();
356
+ if (!text)
357
+ return [];
358
+ return [{
359
+ text,
360
+ layer: /\n\s*8\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
361
+ block: /\n\s*2\s*\n([^\n]+)/u.exec(section)?.[1]?.trim(),
362
+ entityType: section.trim().split(/\s+/u)[0],
363
+ x: Number(/\n\s*10\s*\n([^\n]+)/u.exec(section)?.[1]),
364
+ y: Number(/\n\s*20\s*\n([^\n]+)/u.exec(section)?.[1]),
365
+ }].map(item => ({ ...item, x: Number.isFinite(item.x) ? item.x : undefined, y: Number.isFinite(item.y) ? item.y : undefined }));
366
+ });
367
+ }
368
+ findNearestCadAnnotation(target, annotations) {
369
+ if (target.x == null || target.y == null)
370
+ return undefined;
371
+ return annotations
372
+ .filter(item => item !== target && item.x != null && item.y != null)
373
+ .map(item => ({ item, distance: Math.hypot((item.x ?? 0) - target.x, (item.y ?? 0) - target.y) }))
374
+ .sort((a, b) => a.distance - b.distance)[0]?.item;
375
+ }
376
+ inferCadEntityType(text, fallback) {
377
+ if (/\b\d+(?:\.\d+)?\s*(?:mm|cm|m)\b|φ|Φ|R\d/iu.test(text))
378
+ return '线性尺寸/半径尺寸';
379
+ if (/°|angle|角度/iu.test(text))
380
+ return '角度尺寸';
381
+ if (/note|说明|备注/iu.test(text))
382
+ return '文字说明';
383
+ return fallback;
384
+ }
337
385
  async tryConvertDwgWithBundledWasm(filePath) {
338
386
  try {
339
387
  const mod = await resolveAndImport('dwgdxf');
@@ -447,36 +495,39 @@ export class ContentExtractor {
447
495
  }
448
496
  extractData(file) {
449
497
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
450
- const metadata = { extractionMode: 'structured_data', vectorizable: true };
498
+ const metadata = { extractionMode: 'structured_data', semanticExtractionMode: 'structured_data_semantic_paths', vectorizable: true };
451
499
  try {
452
500
  if (file.format === 'json') {
453
501
  const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
454
- const lines = isJsonl
455
- ? raw.split(/\r?\n/u).filter(Boolean).slice(0, 500).flatMap((line, index) => this.flattenJson(JSON.parse(line), `line${index + 1}`))
456
- : this.flattenJson(JSON.parse(raw));
502
+ const records = isJsonl
503
+ ? raw.split(/\r?\n/u).filter(Boolean).map((line, index) => ({ path: `line${index + 1}`, value: JSON.parse(line) }))
504
+ : [{ path: '$', value: JSON.parse(raw) }];
505
+ const lines = records.flatMap(record => this.flattenJson(record.value, record.path));
506
+ const objects = records.flatMap(record => this.atomicJsonObjects(record.value, record.path));
457
507
  metadata.fieldCount = lines.length;
458
- metadata.recordCount = isJsonl ? raw.split(/\r?\n/u).filter(Boolean).length : 1;
508
+ metadata.recordCount = records.length;
509
+ metadata.objectCount = objects.length;
459
510
  metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
460
- metadata.contentCoverage = isJsonl ? 'jsonl_records_paths_values' : 'json_paths_values';
461
- return { text: [this.metadataOnlyText(file), ...lines.slice(0, 1500)].join('\n'), metadata, warnings: [] };
511
+ metadata.contentCoverage = isJsonl ? 'jsonl_atomic_objects_paths_values' : 'json_atomic_objects_paths_values';
512
+ return { text: [this.metadataOnlyText(file), '## 路径声明', ...lines, '## 原子对象', ...objects].join('\n'), metadata, warnings: [] };
462
513
  }
463
514
  }
464
515
  catch {
465
516
  metadata.parseError = true;
466
517
  }
467
518
  if (file.format === 'yaml') {
468
- const pairs = this.matchAll(raw, /^\s*([\w.-]+)\s*:\s*(.{1,300})$/gmu).slice(0, 1500);
469
- metadata.fieldCount = pairs.length;
470
- metadata.dataPaths = pairs.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
471
- metadata.contentCoverage = 'yaml_key_values';
472
- return { text: [this.metadataOnlyText(file), ...pairs].join('\n'), metadata, warnings: [] };
519
+ const lines = this.flattenYamlByIndent(raw).slice(0, 3000);
520
+ metadata.fieldCount = lines.length;
521
+ metadata.dataPaths = lines.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
522
+ metadata.contentCoverage = 'yaml_indented_paths_values';
523
+ return { text: [this.metadataOnlyText(file), '## YAML 路径声明', ...lines].join('\n'), metadata, warnings: [] };
473
524
  }
474
525
  if (file.format === 'xml') {
475
- const elements = this.matchAll(raw, /<([A-Za-z_][\w:.-]*)\b[^>]*>([^<]{1,200})<\/\1>/gu).slice(0, 1000);
526
+ const elements = this.flattenXmlPaths(raw).slice(0, 3000);
476
527
  metadata.elementTextCount = elements.length;
477
- metadata.dataPaths = elements.map(line => line.match(/^<([A-Za-z_][\w:.-]*)/u)?.[1]).filter(Boolean).slice(0, 200);
478
- metadata.contentCoverage = 'xml_element_text';
479
- return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
528
+ metadata.dataPaths = elements.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
529
+ metadata.contentCoverage = 'xml_paths_values';
530
+ return { text: [this.metadataOnlyText(file), '## XML 路径声明', ...elements].join('\n'), metadata, warnings: [] };
480
531
  }
481
532
  metadata.contentCoverage = 'plain_structured_text';
482
533
  return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
@@ -485,18 +536,19 @@ export class ContentExtractor {
485
536
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
486
537
  const metadata = { extractionMode: 'diagram_structural', vectorizable: true };
487
538
  if (file.format === 'drawio') {
488
- const labels = this.matchAll(raw, /(?:value|label)="([^"]+)"/gu).map(value => this.stripXml(value)).slice(0, 300);
489
- metadata.nodeTextCount = labels.length;
490
- metadata.contentCoverage = 'drawio_labels';
491
- return { text: [this.metadataOnlyText(file), `Draw.io 节点/连线文本:\n${labels.join('\n')}`].join('\n'), metadata, warnings: [] };
539
+ const graph = this.extractDrawioGraph(raw);
540
+ metadata.nodeTextCount = graph.nodes.length;
541
+ metadata.edgeCount = graph.edges.length;
542
+ metadata.contentCoverage = 'drawio_graph_links';
543
+ return { text: [this.metadataOnlyText(file), 'Draw.io 图链路:', ...graph.links, 'Draw.io 节点:', ...graph.nodes.map(node => `[${node.id}] ${node.label}`)].join('\n'), metadata, warnings: [] };
492
544
  }
493
545
  if (file.format === 'excalidraw') {
494
546
  try {
495
- const parsed = JSON.parse(raw);
496
- const texts = (parsed.elements ?? []).filter(element => element.text).map(element => `${element.type ?? 'shape'}: ${element.text}`).slice(0, 300);
497
- metadata.elementTextCount = texts.length;
498
- metadata.contentCoverage = 'excalidraw_text_elements';
499
- return { text: [this.metadataOnlyText(file), `Excalidraw 图形文本:\n${texts.join('\n')}`].join('\n'), metadata, warnings: [] };
547
+ const graph = this.extractExcalidrawGraph(raw);
548
+ metadata.elementTextCount = graph.nodes.length;
549
+ metadata.edgeCount = graph.links.length;
550
+ metadata.contentCoverage = 'excalidraw_graph_links';
551
+ return { text: [this.metadataOnlyText(file), 'Excalidraw 图链路:', ...graph.links, 'Excalidraw 节点:', ...graph.nodes].join('\n'), metadata, warnings: [] };
500
552
  }
501
553
  catch {
502
554
  metadata.parseError = true;
@@ -505,6 +557,52 @@ export class ContentExtractor {
505
557
  metadata.contentCoverage = 'diagram_source_text';
506
558
  return { text: [this.metadataOnlyText(file), raw].join('\n'), metadata, warnings: [] };
507
559
  }
560
+ extractDrawioGraph(raw) {
561
+ const cells = Array.from(raw.matchAll(/<mxCell\b([^>]*)>/gu), match => this.parseXmlAttributes(match[1] ?? ''));
562
+ const nodes = cells
563
+ .filter(cell => cell.value || cell.label || cell.id)
564
+ .map((cell, index) => ({ id: String(cell.id ?? `node-${index + 1}`), label: this.stripXml(String(cell.value || cell.label || cell.id)) }))
565
+ .filter(node => node.label);
566
+ const nodeById = new Map(nodes.map(node => [node.id, node.label]));
567
+ const edges = cells.filter(cell => cell.edge === '1' && cell.source && cell.target);
568
+ const links = edges.map(edge => {
569
+ const source = nodeById.get(String(edge.source)) ?? String(edge.source);
570
+ const target = nodeById.get(String(edge.target)) ?? String(edge.target);
571
+ const label = this.stripXml(String(edge.value || edge.label || '关系'));
572
+ return `[${source}] ──> (${label}) ──> [${target}]`;
573
+ });
574
+ return { nodes, edges: edges.map(edge => String(edge.id ?? 'edge')), links };
575
+ }
576
+ extractExcalidrawGraph(raw) {
577
+ const parsed = JSON.parse(raw);
578
+ const elements = parsed.elements ?? [];
579
+ const textById = new Map(elements.filter(element => element.text).map(element => [String(element.id), String(element.text)]));
580
+ const shapeLabels = elements
581
+ .filter(element => element.type !== 'arrow' && element.type !== 'line')
582
+ .map((element, index) => {
583
+ const id = String(element.id ?? `node-${index + 1}`);
584
+ const own = element.text ?? textById.get(id);
585
+ const nested = elements.find(candidate => candidate.text && this.isPointInside(candidate, element));
586
+ return { id, label: own ?? nested?.text ?? element.type ?? '节点' };
587
+ });
588
+ const labelById = new Map(shapeLabels.map(node => [node.id, node.label]));
589
+ const links = elements
590
+ .filter(element => element.type === 'arrow' && element.startBinding?.elementId && element.endBinding?.elementId)
591
+ .map(element => `[${labelById.get(element.startBinding.elementId) ?? element.startBinding.elementId}] ──> (箭头) ──> [${labelById.get(element.endBinding.elementId) ?? element.endBinding.elementId}]`);
592
+ const nodes = shapeLabels.map(node => `[${node.id}] ${node.label}`);
593
+ return { nodes, links };
594
+ }
595
+ isPointInside(point, box) {
596
+ if (point.x == null || point.y == null || box.x == null || box.y == null || box.width == null || box.height == null)
597
+ return false;
598
+ return point.x >= box.x && point.x <= box.x + box.width && point.y >= box.y && point.y <= box.y + box.height;
599
+ }
600
+ parseXmlAttributes(input) {
601
+ const attrs = {};
602
+ for (const match of input.matchAll(/([\w:-]+)="([^"]*)"/gu))
603
+ attrs[match[1]] = this.stripXml(match[2] ?? '');
604
+ return attrs;
605
+ }
508
606
  parseDelimitedLine(line, delimiter) {
509
607
  const values = [];
510
608
  let current = '';
@@ -531,28 +629,38 @@ export class ContentExtractor {
531
629
  values.push(current.trim());
532
630
  return values;
533
631
  }
632
+ toMarkdownTable(header, rows) {
633
+ const width = Math.max(header.length, ...rows.map(row => row.length), 1);
634
+ const normalizedHeader = Array.from({ length: width }, (_, index) => header[index] || `COL${index + 1}`);
635
+ const escape = (value) => String(value ?? '').replace(/\|/gu, '\\|').replace(/\r?\n/gu, ' ').trim();
636
+ return [
637
+ `| ${normalizedHeader.map(escape).join(' | ')} |`,
638
+ `| ${normalizedHeader.map(() => '---').join(' | ')} |`,
639
+ ...rows.map(row => `| ${Array.from({ length: width }, (_, index) => escape(row[index] ?? '')).join(' | ')} |`),
640
+ ].join('\n');
641
+ }
534
642
  extractDelimitedText(file) {
535
643
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
536
644
  const delimiter = file.format === 'tsv' ? '\t' : ',';
537
645
  const rows = raw.split(/\r?\n/u).filter(line => line.trim().length > 0);
538
646
  const header = rows[0] ? this.parseDelimitedLine(rows[0], delimiter) : [];
539
- const structured = rows.slice(1, 501).flatMap((line, rowIndex) => {
540
- const values = this.parseDelimitedLine(line, delimiter);
541
- return values.map((value, colIndex) => {
542
- const column = header[colIndex] || `COL${colIndex + 1}`;
543
- return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
544
- });
545
- });
647
+ const tableRows = rows.slice(1).map(line => this.parseDelimitedLine(line, delimiter));
648
+ const markdown = this.toMarkdownTable(header, tableRows);
649
+ const legacyKv = tableRows.flatMap((values, rowIndex) => values.map((value, colIndex) => {
650
+ const column = header[colIndex] || `COL${colIndex + 1}`;
651
+ return `R${rowIndex + 2}C${colIndex + 1} ${column}: ${value}`;
652
+ }));
546
653
  return {
547
- text: [this.metadataOnlyText(file), `表头: ${header.join(' | ')}`, ...structured, raw].join('\n'),
654
+ text: [this.metadataOnlyText(file), '### 工作表: 默认', markdown, '### 表格路径声明', ...legacyKv].join('\n\n'),
548
655
  metadata: {
549
656
  extractionMode: 'delimited_text_structured',
657
+ semanticExtractionMode: 'delimited_markdown_table',
550
658
  vectorizable: true,
551
659
  delimiter: file.format === 'tsv' ? 'tab' : 'comma',
552
660
  rowCount: rows.length,
553
661
  columnCount: header.length,
554
662
  columnNames: header.slice(0, 120),
555
- contentCoverage: 'table_headers_cells_text',
663
+ contentCoverage: 'markdown_table',
556
664
  },
557
665
  warnings: [],
558
666
  };
@@ -567,19 +675,27 @@ export class ContentExtractor {
567
675
  return this.extractLegacyOfficeBinary(file);
568
676
  if (ext === '.docx') {
569
677
  try {
678
+ const styledMarkdown = await this.extractDocxStyleTreeMarkdown(file.absolutePath);
679
+ if (styledMarkdown.trim()) {
680
+ return {
681
+ text: styledMarkdown,
682
+ metadata: { extractionMode: 'docx_xml_style_tree_markdown', vectorizable: true, contentCoverage: 'office_style_tree_markdown' },
683
+ warnings: [],
684
+ };
685
+ }
570
686
  const mammoth = await resolveAndImport('mammoth');
571
- const result = await mammoth.extractRawText({ path: file.absolutePath });
572
- const text = result.value.trim();
687
+ const result = await mammoth.convertToMarkdown({ path: file.absolutePath });
688
+ const text = String(result.value ?? '').trim();
573
689
  if (text) {
574
690
  return {
575
- text,
576
- metadata: { extractionMode: 'builtin_mammoth', vectorizable: true, contentCoverage: 'office_full_text' },
691
+ text: this.normalizeMarkdownHeadings(text),
692
+ metadata: { extractionMode: 'builtin_mammoth_markdown', vectorizable: true, contentCoverage: 'office_markdown_structure' },
577
693
  warnings: result.messages.map(m => m.message),
578
694
  };
579
695
  }
580
696
  }
581
697
  catch {
582
- // fallback below
698
+ // 降级到下方 Office ZIP 解析
583
699
  }
584
700
  }
585
701
  return this.extractOfficeZip(file);
@@ -630,6 +746,14 @@ export class ContentExtractor {
630
746
  warnings: text ? [] : ['旧版 Office 二进制文件未提取到正文,未入库'],
631
747
  };
632
748
  }
749
+ findMergedCellValue(sheet, merges, row, col, XLSX) {
750
+ const merge = merges.find(item => row >= item.s.r && row <= item.e.r && col >= item.s.c && col <= item.e.c);
751
+ if (!merge)
752
+ return undefined;
753
+ const originAddress = XLSX.utils.encode_cell({ r: merge.s.r, c: merge.s.c });
754
+ const origin = sheet[originAddress];
755
+ return origin?.w ?? (origin?.v == null ? undefined : String(origin.v));
756
+ }
633
757
  async extractSpreadsheet(file) {
634
758
  const ext = path.extname(file.absolutePath).toLowerCase();
635
759
  try {
@@ -644,28 +768,32 @@ export class ContentExtractor {
644
768
  if (!sheet || !sheet['!ref'])
645
769
  continue;
646
770
  const range = XLSX.utils.decode_range(sheet['!ref']);
647
- const lines = [`工作表: ${name}`, `范围: ${sheet['!ref']}`];
648
771
  const merges = sheet['!merges'] ?? [];
649
772
  mergeCount += merges.length;
650
- if (merges.length > 0) {
651
- lines.push(`合并单元格: ${merges.map(item => `${XLSX.utils.encode_cell(item.s)}:${XLSX.utils.encode_cell(item.e)}`).join(', ')}`);
652
- }
773
+ const matrix = [];
653
774
  for (let row = range.s.r; row <= range.e.r; row++) {
775
+ const values = [];
654
776
  for (let col = range.s.c; col <= range.e.c; col++) {
655
777
  const address = XLSX.utils.encode_cell({ r: row, c: col });
656
778
  const cell = sheet[address];
657
- if (!cell || (cell.v == null && !cell.f))
658
- continue;
659
- cellCount++;
660
- if (cell.f)
661
- formulaCount++;
662
- const display = cell.w ?? String(cell.v ?? '');
663
- const formula = cell.f ? ` 公式=${cell.f}` : '';
664
- lines.push(`${address}: ${display}${formula}`);
779
+ const mergedValue = cell ? undefined : this.findMergedCellValue(sheet, merges, row, col, XLSX);
780
+ if (cell) {
781
+ cellCount++;
782
+ if (cell.f)
783
+ formulaCount++;
784
+ }
785
+ const display = cell?.w ?? String(cell?.v ?? mergedValue ?? '');
786
+ const formula = cell?.f ? ` 公式=${cell.f}` : '';
787
+ values.push(`${display}${formula}`.trim());
665
788
  }
789
+ if (values.some(Boolean))
790
+ matrix.push(values);
791
+ }
792
+ if (matrix.length > 0) {
793
+ const header = matrix[0] ?? [];
794
+ const rows = matrix.slice(1);
795
+ sheetTexts.push([`### 工作表: ${name}`, this.toMarkdownTable(header, rows)].join('\n\n'));
666
796
  }
667
- if (lines.length > 2)
668
- sheetTexts.push(lines.slice(0, 5_000).join('\n'));
669
797
  }
670
798
  if (sheetTexts.length > 0) {
671
799
  return {
@@ -678,12 +806,47 @@ export class ContentExtractor {
678
806
  catch {
679
807
  if (ext === '.xls')
680
808
  return this.extractLegacyOfficeBinary(file);
681
- // fallback below
809
+ // 降级到下方 ZIP 文件内容提取
682
810
  }
683
811
  if (ext === '.xls')
684
812
  return this.extractLegacyOfficeBinary(file);
685
813
  return this.extractOfficeZip(file);
686
814
  }
815
+ async extractDocxStyleTreeMarkdown(filePath) {
816
+ const jszipMod = await resolveAndImport('jszip');
817
+ const JSZip = jszipMod.default ?? jszipMod;
818
+ const zip = await JSZip.loadAsync(fs.readFileSync(filePath));
819
+ const docXml = await zip.files['word/document.xml']?.async('string');
820
+ if (!docXml)
821
+ return '';
822
+ const paragraphs = Array.from(docXml.matchAll(/<w:p[\s\S]*?<\/w:p>/gu), match => match[0]);
823
+ const lines = [];
824
+ for (const paragraph of paragraphs) {
825
+ const texts = Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
826
+ if (!texts.trim())
827
+ continue;
828
+ const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(paragraph)?.[1] ?? '';
829
+ const bold = /<w:b\b/u.test(paragraph);
830
+ const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(paragraph)?.[1] ?? 0);
831
+ const level = this.docxHeadingLevel(style, bold, size, texts);
832
+ lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
833
+ }
834
+ return lines.join('\n\n');
835
+ }
836
+ docxHeadingLevel(style, bold, size, text) {
837
+ const normalized = style.toLowerCase();
838
+ const heading = /heading(\d)|标题(\d)|h(\d)/iu.exec(normalized);
839
+ const styleLevel = Number(heading?.[1] ?? heading?.[2] ?? heading?.[3] ?? 0);
840
+ if (styleLevel >= 1 && styleLevel <= 6)
841
+ return styleLevel;
842
+ if (text.length <= 100 && size >= 32)
843
+ return 1;
844
+ if (text.length <= 100 && size >= 28)
845
+ return 2;
846
+ if (text.length <= 100 && (size >= 24 || bold))
847
+ return 3;
848
+ return 0;
849
+ }
687
850
  async extractOfficeZip(file) {
688
851
  const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
689
852
  try {
@@ -709,26 +872,6 @@ export class ContentExtractor {
709
872
  return { text: '', metadata, warnings: [`Office/表格/演示文件解析失败: ${metadata.parseError},未入库`] };
710
873
  }
711
874
  }
712
- async extractArchive(file) {
713
- const metadata = { extractionMode: 'archive_manifest', vectorizable: true };
714
- if (path.extname(file.absolutePath).toLowerCase() !== '.zip') {
715
- metadata.contentCoverage = 'metadata_filename';
716
- return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
717
- }
718
- try {
719
- const jszipMod = await resolveAndImport('jszip');
720
- const JSZip = jszipMod.default ?? jszipMod;
721
- const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
722
- const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
723
- metadata.entryCount = Object.keys(zip.files).length;
724
- metadata.contentCoverage = 'zip_manifest';
725
- return { text: [this.metadataOnlyText(file), '压缩包文件清单:', ...entries].join('\n'), metadata, warnings: [] };
726
- }
727
- catch (error) {
728
- metadata.parseError = error instanceof Error ? error.message : String(error);
729
- return { text: '', metadata, warnings: ['压缩包解析失败,内置解析器未提取到文件清单,未入库'] };
730
- }
731
- }
732
875
  async extractRasterImage(file) {
733
876
  const metadata = { extractionMode: 'builtin_tesseract_ocr_isolated', vectorizable: true };
734
877
  if (process.env.CUSTOMIZE_AGENT_DISABLE_OCR === '1') {
@@ -742,6 +885,13 @@ export class ContentExtractor {
742
885
  metadata.parseError = validationError;
743
886
  return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
744
887
  }
888
+ const paddle = await this.tryPaddleOcrLayout(file.absolutePath);
889
+ if (paddle) {
890
+ metadata.contentCoverage = 'paddleocr_layout_regions';
891
+ metadata.ocrProvider = 'paddleocr';
892
+ metadata.ocrRegionCount = paddle.regionCount;
893
+ return { text: [this.metadataOnlyText(file), paddle.text].join('\n'), metadata, warnings: [] };
894
+ }
745
895
  let tesseractPath;
746
896
  try {
747
897
  tesseractPath = resolvePackage('tesseract.js');
@@ -766,7 +916,12 @@ const { createWorker } = tesseractMod;
766
916
  const worker = await createWorker('chi_sim+eng');
767
917
  try {
768
918
  const result = await worker.recognize(process.argv[1]);
769
- process.stdout.write(result.data.text || '');
919
+ const lines = (result.data.lines || []).map((line, index) => ({
920
+ index: index + 1,
921
+ text: line.text || '',
922
+ bbox: line.bbox || line.baseline || null,
923
+ })).filter(line => line.text.trim());
924
+ process.stdout.write(JSON.stringify({ text: result.data.text || '', lines }));
770
925
  } finally {
771
926
  await worker.terminate();
772
927
  }`,
@@ -778,17 +933,73 @@ try {
778
933
  metadata.parseError = message;
779
934
  return { text: '', metadata, warnings: [`内置 OCR 解析失败:${message},未入库`] };
780
935
  }
781
- const text = result.stdout.trim();
782
- metadata.contentCoverage = text ? 'ocr_text' : 'metadata_filename';
936
+ const parsed = this.parseOcrJson(result.stdout);
937
+ const text = parsed.text.trim();
938
+ const regions = this.formatOcrRegions(parsed.lines);
939
+ metadata.contentCoverage = text ? 'ocr_text_bounding_boxes' : 'metadata_filename';
783
940
  metadata.ocrProvider = 'tesseract.js';
784
941
  metadata.ocrLanguages = 'chi_sim+eng';
785
942
  metadata.ocrTextLength = text.length;
943
+ metadata.ocrLineCount = parsed.lines.length;
786
944
  return {
787
- text: text ? [this.metadataOnlyText(file), `OCR 识别文本:\n${text}`].join('\n') : '',
945
+ text: text ? [this.metadataOnlyText(file), 'OCR 区域文本:', ...regions, `OCR 完整文本:\n${text}`].join('\n') : '',
788
946
  metadata,
789
947
  warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
790
948
  };
791
949
  }
950
+ async tryPaddleOcrLayout(filePath) {
951
+ const command = process.env.CUSTOMIZE_PADDLE_OCR_CMD || process.env.PADDLE_OCR_CMD;
952
+ if (!command)
953
+ return undefined;
954
+ const result = spawnSync(command, [filePath], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, shell: true });
955
+ if (result.status !== 0 || !result.stdout.trim())
956
+ return undefined;
957
+ try {
958
+ const parsed = JSON.parse(result.stdout);
959
+ const lines = parsed.map((region, index) => `区域 ${index + 1} [${region.type ?? 'text'}] ${this.formatBoundingBox(region.bbox)}: ${region.text ?? ''}`);
960
+ return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
961
+ }
962
+ catch {
963
+ const lines = result.stdout.split(/\r?\n/u).filter(Boolean);
964
+ return { text: ['OCR 版面分析区域:', ...lines].join('\n'), regionCount: lines.length };
965
+ }
966
+ }
967
+ parseOcrJson(raw) {
968
+ try {
969
+ const parsed = JSON.parse(raw);
970
+ return {
971
+ text: parsed.text ?? raw,
972
+ lines: (parsed.lines ?? []).map((line, index) => ({ index: line.index ?? index + 1, text: line.text ?? '', bbox: line.bbox })).filter(line => line.text.trim()),
973
+ };
974
+ }
975
+ catch {
976
+ return { text: raw, lines: raw.split(/\r?\n/u).map((text, index) => ({ index: index + 1, text })).filter(line => line.text.trim()) };
977
+ }
978
+ }
979
+ formatOcrRegions(lines) {
980
+ return lines.map(line => {
981
+ const bbox = this.formatBoundingBox(line.bbox);
982
+ const type = this.classifyOcrRegion(line.text);
983
+ return `区域 ${line.index} [${type}]${bbox ? ` ${bbox}` : ''}: ${line.text}`;
984
+ });
985
+ }
986
+ classifyOcrRegion(text) {
987
+ if (/\|/.test(text) || /\s{2,}/u.test(text) || /表\s*\d|合计|小计/u.test(text))
988
+ return 'table';
989
+ if (/图\s*\d|figure|image|示意图/iu.test(text))
990
+ return 'image-caption';
991
+ return 'text';
992
+ }
993
+ formatBoundingBox(value) {
994
+ if (!value || typeof value !== 'object')
995
+ return '';
996
+ const record = value;
997
+ const x0 = record.x0 ?? record.left ?? record.x;
998
+ const y0 = record.y0 ?? record.top ?? record.y;
999
+ const x1 = record.x1 ?? record.right;
1000
+ const y1 = record.y1 ?? record.bottom;
1001
+ return [x0, y0, x1, y1].some(item => item != null) ? `[bbox x0=${x0 ?? ''}, y0=${y0 ?? ''}, x1=${x1 ?? ''}, y1=${y1 ?? ''}]` : '';
1002
+ }
792
1003
  validateRasterImage(filePath) {
793
1004
  const buffer = fs.readFileSync(filePath);
794
1005
  if (buffer.length < 12)
@@ -816,25 +1027,25 @@ try {
816
1027
  async extractPdf(file) {
817
1028
  const metadata = { extractionMode: 'pdf_text', vectorizable: true };
818
1029
  const warnings = [];
819
- // Tier 1: pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
1030
+ // 第一层:pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
820
1031
  try {
821
1032
  const raw = fs.readFileSync(file.absolutePath);
822
1033
  const text = await this.extractPdfText(raw);
823
1034
  if (text.trim()) {
824
- metadata.contentCoverage = 'pdf_text_streams';
1035
+ metadata.contentCoverage = 'pdf_text_streams_layout_markdown';
825
1036
  metadata.pdfExtractor = 'pdfjs-dist';
826
- return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
1037
+ return { text: [this.metadataOnlyText(file), this.toMarkdownDocument(text)].join('\n\n'), metadata, warnings };
827
1038
  }
828
1039
  }
829
1040
  catch (error) {
830
1041
  warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
831
1042
  metadata.parseError = error instanceof Error ? error.message : String(error);
832
1043
  }
833
- // Tier 2: OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
1044
+ // 第二层:OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
834
1045
  const ocr = await this.extractScannedPdfOcr(file);
835
1046
  if (ocr.text.trim())
836
1047
  return ocr;
837
- // Tier 3: 仅索引元数据(兜底)
1048
+ // 第三层:仅索引元数据(兜底)
838
1049
  metadata.extractionMode = 'pdf_metadata_only';
839
1050
  metadata.contentCoverage = 'metadata_filename';
840
1051
  metadata.ocrRecommended = true;
@@ -846,10 +1057,6 @@ try {
846
1057
  warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
847
1058
  };
848
1059
  }
849
- pdfOcrPageLimit() {
850
- const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
851
- return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
852
- }
853
1060
  async extractScannedPdfOcr(file) {
854
1061
  const metadata = {
855
1062
  extractionMode: 'pdf_page_ocr_embedded',
@@ -858,9 +1065,8 @@ try {
858
1065
  ocrProvider: 'tesseract.js',
859
1066
  ocrLanguages: 'chi_sim+eng',
860
1067
  pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
861
- pdfOcrPageLimit: this.pdfOcrPageLimit(),
1068
+ pdfOcrPageLimit: 'all',
862
1069
  };
863
- const pageLimitConfig = this.pdfOcrPageLimit();
864
1070
  // 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
865
1071
  let canvasPath;
866
1072
  let pdfjsPath;
@@ -890,10 +1096,9 @@ import { createCanvas } from ${JSON.stringify(canvasPath)};
890
1096
  import * as pdfjs from ${JSON.stringify(pdfjsPath)};
891
1097
  import { createWorker } from ${JSON.stringify(tesseractPath)};
892
1098
  const filePath = process.argv[1];
893
- const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
894
1099
  const bytes = new Uint8Array(fs.readFileSync(filePath));
895
1100
  const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
896
- const pageLimit = Math.min(doc.numPages, configuredLimit);
1101
+ const pageLimit = doc.numPages;
897
1102
  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
898
1103
  const worker = await createWorker('chi_sim+eng');
899
1104
  const pages = [];
@@ -908,7 +1113,8 @@ try {
908
1113
  fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
909
1114
  const recognized = await worker.recognize(imagePath);
910
1115
  const text = (recognized.data.text || '').trim();
911
- if (text) pages.push('PDF OCR ' + i + ' 页:\\n' + text);
1116
+ const lines = (recognized.data.lines || []).map((line, index) => ({ index: index + 1, text: line.text || '', bbox: line.bbox || null })).filter(line => line.text.trim());
1117
+ if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + lines.map(line => '区域 ' + line.index + ' ' + JSON.stringify(line.bbox || {}) + ': ' + line.text).join('\\n') + '\\n\\n完整文本:\\n' + text);
912
1118
  }
913
1119
  } finally {
914
1120
  await worker.terminate();
@@ -916,8 +1122,7 @@ try {
916
1122
  }
917
1123
  process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
918
1124
  file.absolutePath,
919
- String(pageLimitConfig),
920
- ], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
1125
+ ], { encoding: 'utf8', timeout: 0, maxBuffer: 50 * 1024 * 1024, env: childEnv });
921
1126
  if (result.status !== 0 || result.error) {
922
1127
  metadata.ocrRecommended = true;
923
1128
  metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
@@ -927,7 +1132,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
927
1132
  const parsed = JSON.parse(result.stdout);
928
1133
  const text = parsed.text?.trim() ?? '';
929
1134
  metadata.ocrPageCount = parsed.pageCount ?? 0;
930
- metadata.pdfOcrPageLimit = parsed.pageLimit ?? 5;
1135
+ metadata.pdfOcrPageLimit = parsed.pageLimit ?? 'all';
931
1136
  metadata.ocrTextLength = text.length;
932
1137
  metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
933
1138
  return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
@@ -939,23 +1144,20 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
939
1144
  }
940
1145
  }
941
1146
  async extractPdfText(buffer) {
942
- // Tier 1: pdfjs-dist 文本提取(处理压缩内容流、CJK 字体、现代 PDF)
1147
+ // 第一层:pdfjs-dist 文本提取(处理压缩内容流、CJK 字体、现代 PDF)
943
1148
  try {
944
1149
  const mod = await resolveAndImport('pdfjs-dist/legacy/build/pdf.mjs');
945
1150
  const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
946
1151
  const doc = await loadingTask.promise;
947
1152
  const pages = [];
948
- const pageLimit = Math.min(doc.numPages, 200);
1153
+ const pageLimit = doc.numPages;
949
1154
  for (let i = 1; i <= pageLimit; i++) {
950
1155
  const page = await doc.getPage(i);
951
1156
  const content = await page.getTextContent();
952
- const pageText = content.items
953
- .map((item) => {
954
- const hasStr = item != null && typeof item === 'object' && 'str' in item;
955
- return hasStr ? String(item.str) : '';
956
- })
957
- .filter((s) => s.trim().length > 0)
958
- .join(' ');
1157
+ const items = content.items
1158
+ .map((item) => this.toPdfTextItem(item))
1159
+ .filter((item) => !!item && item.str.trim().length > 0);
1160
+ const pageText = this.layoutPdfTextItems(items, i);
959
1161
  if (pageText.trim())
960
1162
  pages.push(pageText.trim());
961
1163
  }
@@ -970,7 +1172,7 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
970
1172
  if (process.env.KB_DEBUG === '1')
971
1173
  console.warn('[kb] pdfjs-dist extraction failed:', e.message);
972
1174
  }
973
- // Tier 2: pdf-parse(兼容旧版 PDF)
1175
+ // 第二层:pdf-parse(兼容旧版 PDF)
974
1176
  try {
975
1177
  const mod = await resolveAndImport('pdf-parse');
976
1178
  const pdfParse = mod.default;
@@ -981,9 +1183,9 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
981
1183
  }
982
1184
  }
983
1185
  catch {
984
- // fallback to raw regex below
1186
+ // 降级到下方纯正则提取
985
1187
  }
986
- // Tier 3: raw regex 回退(未压缩的古老 PDF)
1188
+ // 第三层:raw regex 回退(未压缩的古老 PDF)
987
1189
  const raw = buffer.toString('latin1');
988
1190
  const matches = Array.from(raw.matchAll(/\(([^()]{2,500})\)\s*T[jJ]/gu), match => match[1] ?? '')
989
1191
  .concat(Array.from(raw.matchAll(/\[([^\]]{2,2000})\]\s*TJ/gu), match => match[1] ?? ''));
@@ -998,16 +1200,166 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
998
1200
  .join('')
999
1201
  .trim();
1000
1202
  }
1203
+ toPdfTextItem(item) {
1204
+ if (!item || typeof item !== 'object' || !('str' in item))
1205
+ return undefined;
1206
+ const record = item;
1207
+ const transform = Array.isArray(record.transform) ? record.transform : [];
1208
+ return {
1209
+ str: String(record.str ?? ''),
1210
+ x: Number(transform[4] ?? 0),
1211
+ y: Number(transform[5] ?? 0),
1212
+ width: Number(record.width ?? 0),
1213
+ height: Number(record.height ?? Math.abs(Number(transform[3] ?? 0))),
1214
+ fontName: typeof record.fontName === 'string' ? record.fontName : undefined,
1215
+ };
1216
+ }
1217
+ layoutPdfTextItems(items, pageNumber) {
1218
+ if (items.length === 0)
1219
+ return '';
1220
+ const rows = this.groupPdfItemsIntoRows(items);
1221
+ const columnSplit = this.detectPdfColumnSplit(rows);
1222
+ const orderedRows = columnSplit == null
1223
+ ? rows.sort((a, b) => b.y - a.y || a.x - b.x)
1224
+ : [
1225
+ ...rows.filter(row => row.x < columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
1226
+ ...rows.filter(row => row.x >= columnSplit).sort((a, b) => b.y - a.y || a.x - b.x),
1227
+ ];
1228
+ const markdown = this.rowsToPdfMarkdownWithTables(orderedRows);
1229
+ return [`## PDF 第 ${pageNumber} 页`, markdown].join('\n\n');
1230
+ }
1231
+ groupPdfItemsIntoRows(items) {
1232
+ const sorted = [...items].sort((a, b) => b.y - a.y || a.x - b.x);
1233
+ const rows = [];
1234
+ for (const item of sorted) {
1235
+ const row = rows.find(candidate => Math.abs(candidate.y - item.y) <= Math.max(2, item.height * 0.55));
1236
+ if (row)
1237
+ row.items.push(item);
1238
+ else
1239
+ rows.push({ y: item.y, items: [item] });
1240
+ }
1241
+ return rows.map(row => {
1242
+ const rowItems = row.items.sort((a, b) => a.x - b.x);
1243
+ return {
1244
+ text: rowItems.map(item => item.str.trim()).filter(Boolean).join(' ').replace(/\s+/gu, ' '),
1245
+ x: Math.min(...rowItems.map(item => item.x)),
1246
+ y: row.y,
1247
+ height: Math.max(...rowItems.map(item => item.height || 0)),
1248
+ };
1249
+ }).filter(row => row.text);
1250
+ }
1251
+ detectPdfColumnSplit(rows) {
1252
+ if (rows.length < 8)
1253
+ return undefined;
1254
+ const xs = rows.map(row => row.x).sort((a, b) => a - b);
1255
+ const gaps = xs.slice(1).map((x, index) => ({ gap: x - xs[index], left: xs[index], right: x })).sort((a, b) => b.gap - a.gap);
1256
+ const largest = gaps[0];
1257
+ if (!largest || largest.gap < 80)
1258
+ return undefined;
1259
+ const leftCount = rows.filter(row => row.x <= largest.left).length;
1260
+ const rightCount = rows.filter(row => row.x >= largest.right).length;
1261
+ return leftCount >= 3 && rightCount >= 3 ? (largest.left + largest.right) / 2 : undefined;
1262
+ }
1263
+ rowsToPdfMarkdownWithTables(rows) {
1264
+ const output = [];
1265
+ let index = 0;
1266
+ while (index < rows.length) {
1267
+ const tableRows = [];
1268
+ let cursor = index;
1269
+ while (cursor < rows.length) {
1270
+ const cells = this.splitLikelyTableRow(rows[cursor].text);
1271
+ if (cells.length < 2)
1272
+ break;
1273
+ tableRows.push(cells);
1274
+ cursor += 1;
1275
+ }
1276
+ if (tableRows.length >= 2) {
1277
+ output.push('### PDF 表格区域');
1278
+ output.push(this.toMarkdownTable(tableRows[0], tableRows.slice(1)));
1279
+ index = cursor;
1280
+ continue;
1281
+ }
1282
+ output.push(this.pdfRowToMarkdown(rows[index].text, rows[index].height, index));
1283
+ index += 1;
1284
+ }
1285
+ return output.join('\n');
1286
+ }
1287
+ splitLikelyTableRow(text) {
1288
+ const byLargeSpaces = text.split(/\s{2,}/u).map(cell => cell.trim()).filter(Boolean);
1289
+ if (byLargeSpaces.length >= 2)
1290
+ return byLargeSpaces;
1291
+ const byPipes = text.split('|').map(cell => cell.trim()).filter(Boolean);
1292
+ return byPipes.length >= 2 ? byPipes : [];
1293
+ }
1294
+ pdfRowToMarkdown(text, height, index) {
1295
+ if (index === 0 && text.length <= 100)
1296
+ return `# ${text}`;
1297
+ if (height >= 14 && text.length <= 120)
1298
+ return `## ${text}`;
1299
+ if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(text) && text.length <= 120)
1300
+ return `### ${text}`;
1301
+ return text;
1302
+ }
1303
+ toMarkdownDocument(text) {
1304
+ const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
1305
+ return lines.map((line, index) => {
1306
+ if (/^#{1,6}\s/u.test(line) || /^\|/u.test(line))
1307
+ return line;
1308
+ if (line.length <= 80 && !/[。!?.!?]$/u.test(line)) {
1309
+ if (index === 0)
1310
+ return `# ${line}`;
1311
+ if (/^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(line))
1312
+ return `## ${line}`;
1313
+ return `### ${line}`;
1314
+ }
1315
+ return line;
1316
+ }).join('\n\n');
1317
+ }
1318
+ normalizeMarkdownHeadings(text) {
1319
+ return text
1320
+ .split(/\r?\n/u)
1321
+ .map(line => {
1322
+ const trimmed = line.trim();
1323
+ if (!trimmed)
1324
+ return '';
1325
+ if (/^#{1,6}\s/u.test(trimmed) || /^\|/u.test(trimmed))
1326
+ return trimmed;
1327
+ if (trimmed.length <= 80 && /^(第[一二三四五六七八九十\d]+[章节]|\d+(?:\.\d+)*\s+)/u.test(trimmed))
1328
+ return `## ${trimmed}`;
1329
+ return trimmed;
1330
+ })
1331
+ .join('\n');
1332
+ }
1001
1333
  extractSvg(file) {
1002
1334
  const raw = fs.readFileSync(file.absolutePath, 'utf8');
1003
- const texts = this.matchAll(raw, /<text\b[^>]*>([\s\S]*?)<\/text>/giu).map(value => this.stripXml(value)).slice(0, 300);
1004
- const titles = this.matchAll(raw, /<(?:title|desc)\b[^>]*>([\s\S]*?)<\/(?:title|desc)>/giu).map(value => this.stripXml(value)).slice(0, 100);
1335
+ const nodes = this.extractSvgSemanticNodes(raw);
1005
1336
  return {
1006
- text: [this.metadataOnlyText(file), `SVG 标题/描述:\n${titles.join('\n')}`, `SVG 文本节点:\n${texts.join('\n')}`].join('\n'),
1007
- metadata: { extractionMode: 'svg_text_nodes', vectorizable: true, textNodeCount: texts.length, contentCoverage: 'svg_text_title_desc' },
1337
+ text: [this.metadataOnlyText(file), 'SVG 层级语义节点:', ...nodes].join('\n'),
1338
+ metadata: { extractionMode: 'svg_text_nodes', semanticExtractionMode: 'svg_semantic_tree_nodes', vectorizable: true, textNodeCount: nodes.length, contentCoverage: 'svg_hierarchical_text_title_desc' },
1008
1339
  warnings: [],
1009
1340
  };
1010
1341
  }
1342
+ extractSvgSemanticNodes(raw) {
1343
+ const nodes = [];
1344
+ const stack = [];
1345
+ const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b([^>]*)>|([^<>]+)/gu;
1346
+ for (const match of raw.matchAll(tokenPattern)) {
1347
+ const tag = match[1];
1348
+ const attrs = match[2] ?? '';
1349
+ const text = match[3]?.replace(/\s+/gu, ' ').trim();
1350
+ const token = match[0];
1351
+ if (tag && token.startsWith('</'))
1352
+ stack.pop();
1353
+ else if (tag && !token.endsWith('/>')) {
1354
+ const id = /\bid="([^"]+)"/u.exec(attrs)?.[1];
1355
+ stack.push(id ? `${tag}#${id}` : tag);
1356
+ }
1357
+ else if (text && ['text', 'title', 'desc'].includes(stack.at(-1)?.split('#')[0] ?? '')) {
1358
+ nodes.push(`${stack.join(' > ')}: ${this.stripXml(text)}`);
1359
+ }
1360
+ }
1361
+ return nodes;
1362
+ }
1011
1363
  isTextReadable(file) {
1012
1364
  if (file.category === 'code' || file.category === 'web')
1013
1365
  return true;
@@ -1042,13 +1394,61 @@ process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text:
1042
1394
  return [`${prefix || 'value'}: ${String(value)}`];
1043
1395
  }
1044
1396
  if (Array.isArray(value)) {
1045
- return value.slice(0, 50).flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
1397
+ return value.flatMap((item, index) => this.flattenJson(item, `${prefix}[${index}]`));
1046
1398
  }
1047
1399
  if (typeof value === 'object') {
1048
1400
  return Object.entries(value).flatMap(([key, item]) => this.flattenJson(item, prefix ? `${prefix}.${key}` : key));
1049
1401
  }
1050
1402
  return [];
1051
1403
  }
1404
+ atomicJsonObjects(value, prefix = '$') {
1405
+ if (value == null || typeof value !== 'object')
1406
+ return [];
1407
+ if (Array.isArray(value))
1408
+ return value.flatMap((item, index) => this.atomicJsonObjects(item, `${prefix}[${index}]`));
1409
+ const entries = Object.entries(value);
1410
+ const current = `${prefix}: ${JSON.stringify(value)}`;
1411
+ const children = entries.flatMap(([key, item]) => this.atomicJsonObjects(item, `${prefix}.${key}`));
1412
+ return [current, ...children];
1413
+ }
1414
+ flattenYamlByIndent(raw) {
1415
+ const stack = [];
1416
+ const lines = [];
1417
+ for (const line of raw.split(/\r?\n/u)) {
1418
+ if (!line.trim() || /^\s*#/u.test(line))
1419
+ continue;
1420
+ const match = /^(\s*)([-\w.]+)\s*:\s*(.*)$/u.exec(line);
1421
+ if (!match)
1422
+ continue;
1423
+ const indent = match[1].length;
1424
+ const key = match[2];
1425
+ const value = match[3].trim();
1426
+ while (stack.length > 0 && stack.at(-1).indent >= indent)
1427
+ stack.pop();
1428
+ const pathName = [...stack.map(item => item.key), key].join('.');
1429
+ if (value)
1430
+ lines.push(`${pathName}: ${value}`);
1431
+ stack.push({ indent, key });
1432
+ }
1433
+ return lines;
1434
+ }
1435
+ flattenXmlPaths(raw) {
1436
+ const lines = [];
1437
+ const stack = [];
1438
+ const tokenPattern = /<\/?([A-Za-z_][\w:.-]*)\b[^>]*>|([^<>]+)/gu;
1439
+ for (const match of raw.matchAll(tokenPattern)) {
1440
+ const tag = match[1];
1441
+ const text = match[2]?.replace(/\s+/gu, ' ').trim();
1442
+ const token = match[0];
1443
+ if (tag && token.startsWith('</'))
1444
+ stack.pop();
1445
+ else if (tag && !token.endsWith('/>'))
1446
+ stack.push(tag);
1447
+ else if (text && stack.length > 0)
1448
+ lines.push(`${stack.join('.')}: ${this.stripXml(text)}`);
1449
+ }
1450
+ return lines.filter(line => !line.endsWith(':'));
1451
+ }
1052
1452
  metadataOnlyText(file) {
1053
1453
  const fileName = path.basename(file.relativePath);
1054
1454
  const directory = path.dirname(file.relativePath);