@heripo/document-processor 0.1.16 → 0.1.18

package/dist/index.cjs

@@ -35,6 +35,7 @@ __export(index_exports, {
   CONTINUATION_MARKERS: () => CONTINUATION_MARKERS,
   CaptionParseError: () => CaptionParseError,
   CaptionParser: () => CaptionParser,
+  CaptionProcessingPipeline: () => CaptionProcessingPipeline,
   CaptionValidationError: () => CaptionValidationError,
   CaptionValidator: () => CaptionValidator,
   ChapterConverter: () => ChapterConverter,
@@ -43,12 +44,14 @@ __export(index_exports, {
   PagePattern: () => PagePattern,
   PageRangeParseError: () => PageRangeParseError,
   PageRangeParser: () => PageRangeParser,
+  ResourceConverter: () => ResourceConverter,
   TOC_KEYWORDS: () => TOC_KEYWORDS,
   TextLLMComponent: () => TextLLMComponent,
   TocContentValidationSchema: () => TocContentValidationSchema,
   TocContentValidator: () => TocContentValidator,
   TocEntrySchema: () => TocEntrySchema,
   TocExtractError: () => TocExtractError,
+  TocExtractionPipeline: () => TocExtractionPipeline,
   TocExtractor: () => TocExtractor,
   TocFinder: () => TocFinder,
   TocNotFoundError: () => TocNotFoundError,
@@ -56,7 +59,8 @@ __export(index_exports, {
   TocResponseSchema: () => TocResponseSchema,
   VisionLLMComponent: () => VisionLLMComponent,
   VisionTocExtractionSchema: () => VisionTocExtractionSchema,
-  VisionTocExtractor: () => VisionTocExtractor
+  VisionTocExtractor: () => VisionTocExtractor,
+  extractMaxPageNumber: () => extractMaxPageNumber
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -605,11 +609,7 @@ var RefResolver = class {
    * Build an index mapping self_ref to the actual item
    */
   buildIndex(items, _prefix) {
-    const map = /* @__PURE__ */ new Map();
-    for (const item of items) {
-      map.set(item.self_ref, item);
-    }
-    return map;
+    return new Map(items.map((item) => [item.self_ref, item]));
   }
   /**
    * Resolve a $ref string to the actual item
@@ -835,14 +835,7 @@ var TextCleaner = class {
    */
   static normalizeAndFilterBatch(texts, batchSize = 10) {
     if (batchSize === 0) {
-      const results = [];
-      for (const text of texts) {
-        const normalized = this.normalize(text);
-        if (this.isValidText(normalized)) {
-          results.push(normalized);
-        }
-      }
-      return results;
+      return texts.map((text) => this.normalize(text)).filter((normalized) => this.isValidText(normalized));
     }
     return BatchProcessor.processBatchSync(texts, batchSize, (batch) => {
       const normalized = this.normalizeBatch(batch);
@@ -864,38 +857,22 @@ var MarkdownConverter = class _MarkdownConverter {
     if (refs.length === 0) {
       return "";
     }
-    const lines = [];
-    for (const ref of refs) {
-      const item = refResolver.resolve(ref);
-      if (!item) {
-        continue;
-      }
+    const lines = refs.map((ref) => refResolver.resolve(ref)).filter((item) => item !== null).map((item) => {
       if ("name" in item && (item.name === "list" || item.name === "group")) {
-        const groupMarkdown = _MarkdownConverter.groupToMarkdown(
+        return _MarkdownConverter.groupToMarkdown(
           item,
           refResolver,
           0
         );
-        if (groupMarkdown) {
-          lines.push(groupMarkdown);
-        }
-      } else if ("data" in item && "grid" in item.data) {
-        const tableMarkdown = _MarkdownConverter.tableToMarkdown(
-          item
-        );
-        if (tableMarkdown) {
-          lines.push(tableMarkdown);
-        }
-      } else if ("text" in item && "orig" in item) {
-        const textMarkdown = _MarkdownConverter.textToMarkdown(
-          item,
-          0
-        );
-        if (textMarkdown) {
-          lines.push(textMarkdown);
-        }
       }
-    }
+      if ("data" in item && "grid" in item.data) {
+        return _MarkdownConverter.tableToMarkdown(item);
+      }
+      if ("text" in item && "orig" in item) {
+        return _MarkdownConverter.textToMarkdown(item, 0);
+      }
+      return "";
+    }).filter((line) => line !== "");
     return lines.join("\n\n");
   }
   /**
@@ -911,31 +888,22 @@ var MarkdownConverter = class _MarkdownConverter {
    * - Chapter 2 Methodology ..... 10
    */
   static groupToMarkdown(group, refResolver, indentLevel = 0) {
-    const lines = [];
-    for (const childRef of group.children) {
-      const child = refResolver.resolve(childRef.$ref);
-      if (!child) {
-        continue;
-      }
+    const lines = group.children.map((childRef) => refResolver.resolve(childRef.$ref)).filter((child) => child !== null).map((child) => {
       if ("name" in child && (child.name === "list" || child.name === "group")) {
-        const nestedMarkdown = _MarkdownConverter.groupToMarkdown(
+        return _MarkdownConverter.groupToMarkdown(
           child,
           refResolver,
           indentLevel + 1
         );
-        if (nestedMarkdown) {
-          lines.push(nestedMarkdown);
-        }
-      } else if ("text" in child && "orig" in child) {
-        const textMarkdown = _MarkdownConverter.textToMarkdown(
+      }
+      if ("text" in child && "orig" in child) {
+        return _MarkdownConverter.textToMarkdown(
           child,
           indentLevel
         );
-        if (textMarkdown) {
-          lines.push(textMarkdown);
-        }
       }
-    }
+      return "";
+    }).filter((line) => line !== "");
     return lines.join("\n");
   }
   /**
@@ -953,21 +921,20 @@ var MarkdownConverter = class _MarkdownConverter {
     if (!grid || grid.length === 0) {
       return "";
     }
-    const lines = [];
-    for (let rowIdx = 0; rowIdx < grid.length; rowIdx++) {
-      const row = grid[rowIdx];
+    const lines = grid.flatMap((row, rowIdx) => {
       if (!row || row.length === 0) {
-        continue;
+        return [];
       }
       const cells = row.map(
         (cell) => _MarkdownConverter.escapeTableCell(cell.text)
       );
-      lines.push(`| ${cells.join(" | ")} |`);
+      const rowLine = `| ${cells.join(" | ")} |`;
       if (rowIdx === 0) {
         const separator = row.map(() => "---").join(" | ");
-        lines.push(`| ${separator} |`);
+        return [rowLine, `| ${separator} |`];
       }
-    }
+      return [rowLine];
+    });
     return lines.join("\n");
   }
   /**
@@ -1014,6 +981,20 @@ var MarkdownConverter = class _MarkdownConverter {
   }
 };
 
+// src/utils/toc-markdown-utils.ts
+function extractMaxPageNumber(markdown) {
+  const dotLeaderMatches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
+  const tableCellMatches = [...markdown.matchAll(/\|\s*(\d+)\s*\|\s*$/gm)];
+  const allNumbers = [
+    ...dotLeaderMatches.map((m) => parseInt(m[1], 10)),
+    ...tableCellMatches.map((m) => parseInt(m[1], 10))
+  ];
+  if (allNumbers.length === 0) {
+    return 0;
+  }
+  return Math.max(...allNumbers);
+}
+
 // src/converters/chapter-converter.ts
 var ChapterConverter = class _ChapterConverter {
   static FRONT_MATTER_ID = "ch-000";
@@ -1113,7 +1094,7 @@ var ChapterConverter = class _ChapterConverter {
   flattenChapters(chapters) {
     const result = [];
     const flatten = (chapterList) => {
-      for (const chapter of chapterList) {
+      chapterList.forEach((chapter) => {
         result.push({
           chapter,
           tocPageNo: chapter.pageNo
@@ -1121,7 +1102,7 @@ var ChapterConverter = class _ChapterConverter {
         if (chapter.children && chapter.children.length > 0) {
           flatten(chapter.children);
         }
-      }
+      });
     };
     flatten(chapters);
     return result;
@@ -1148,14 +1129,13 @@ var ChapterConverter = class _ChapterConverter {
       startPage: 1,
       endPage: firstTocPage - 1
     });
-    for (let i = 0; i < sorted.length; i++) {
-      const current = sorted[i];
+    sorted.forEach((current, i) => {
       const next = sorted[i + 1];
       ranges.set(current.chapter.id, {
         startPage: current.tocPageNo,
         endPage: next ? next.tocPageNo - 1 : Number.MAX_SAFE_INTEGER
       });
-    }
+    });
     return ranges;
   }
   /**
@@ -1224,7 +1204,7 @@ var ChapterConverter = class _ChapterConverter {
    */
   assignTextBlocks(chapters, textBlocks, chapterRanges, pageRangeMap) {
     const chapterMap = this.buildChapterMap(chapters);
-    for (const textBlock of textBlocks) {
+    textBlocks.forEach((textBlock) => {
       const actualPageNo = this.pdfPageToActualPage(
         textBlock.pdfPageNo,
         pageRangeMap
@@ -1233,14 +1213,14 @@ var ChapterConverter = class _ChapterConverter {
       if (chapterId && chapterMap.has(chapterId)) {
         chapterMap.get(chapterId).textBlocks.push(textBlock);
       }
-    }
+    });
   }
   /**
    * Link images, tables, and footnotes to chapters based on page ranges
    */
   linkResources(chapters, images, tables, footnotes, chapterRanges, pageRangeMap) {
     const chapterMap = this.buildChapterMap(chapters);
-    for (const image of images) {
+    images.forEach((image) => {
       const actualPageNo = this.pdfPageToActualPage(
         image.pdfPageNo,
         pageRangeMap
@@ -1249,8 +1229,8 @@ var ChapterConverter = class _ChapterConverter {
       if (chapterId && chapterMap.has(chapterId)) {
         chapterMap.get(chapterId).imageIds.push(image.id);
       }
-    }
-    for (const table of tables) {
+    });
+    tables.forEach((table) => {
       const actualPageNo = this.pdfPageToActualPage(
         table.pdfPageNo,
         pageRangeMap
@@ -1259,8 +1239,8 @@ var ChapterConverter = class _ChapterConverter {
       if (chapterId && chapterMap.has(chapterId)) {
         chapterMap.get(chapterId).tableIds.push(table.id);
       }
-    }
-    for (const footnote of footnotes) {
+    });
+    footnotes.forEach((footnote) => {
       const actualPageNo = this.pdfPageToActualPage(
         footnote.pdfPageNo,
         pageRangeMap
@@ -1269,7 +1249,7 @@ var ChapterConverter = class _ChapterConverter {
       if (chapterId && chapterMap.has(chapterId)) {
         chapterMap.get(chapterId).footnoteIds.push(footnote.id);
       }
-    }
+    });
   }
   /**
    * Build flat chapter map for O(1) lookup
@@ -1277,18 +1257,135 @@ var ChapterConverter = class _ChapterConverter {
   buildChapterMap(chapters) {
     const map = /* @__PURE__ */ new Map();
     const addToMap = (chapterList) => {
-      for (const chapter of chapterList) {
+      chapterList.forEach((chapter) => {
         map.set(chapter.id, chapter);
         if (chapter.children && chapter.children.length > 0) {
           addToMap(chapter.children);
         }
-      }
+      });
     };
     addToMap(chapters);
     return map;
   }
 };
 
+// src/converters/resource-converter.ts
+var ResourceConverter = class {
+  logger;
+  idGenerator;
+  captionProcessingPipeline;
+  constructor(logger, idGenerator, captionProcessingPipeline) {
+    this.logger = logger;
+    this.idGenerator = idGenerator;
+    this.captionProcessingPipeline = captionProcessingPipeline;
+  }
+  /**
+   * Convert all resources (images, tables, footnotes)
+   *
+   * Runs image and table conversions in parallel, then footnotes synchronously.
+   */
+  async convertAll(doclingDoc, outputPath) {
+    this.logger.info(
+      "[ResourceConverter] Converting images, tables, and footnotes..."
+    );
+    const [images, tables] = await Promise.all([
+      this.convertImages(doclingDoc, outputPath),
+      this.convertTables(doclingDoc)
+    ]);
+    const footnotes = this.convertFootnotes(doclingDoc);
+    this.logger.info(
+      `[ResourceConverter] Converted ${images.length} images, ${tables.length} tables, and ${footnotes.length} footnotes`
+    );
+    return { images, tables, footnotes };
+  }
+  /**
+   * Convert images from DoclingDocument to ProcessedImage[]
+   */
+  async convertImages(doclingDoc, outputPath) {
+    this.logger.info(
+      `[ResourceConverter] Converting ${doclingDoc.pictures.length} images...`
+    );
+    const captionTexts = doclingDoc.pictures.map(
+      (picture) => this.captionProcessingPipeline.extractCaptionText(picture.captions)
+    );
+    const images = doclingDoc.pictures.map(
+      (picture, index) => ({
+        id: this.idGenerator.generateImageId(),
+        path: `${outputPath}/images/image_${index}.png`,
+        pdfPageNo: picture.prov?.[0]?.page_no ?? 0
+      })
+    );
+    const captionsByIndex = await this.captionProcessingPipeline.processResourceCaptions(
+      captionTexts,
+      "image"
+    );
+    images.forEach((image, i) => {
+      if (captionsByIndex.has(i)) {
+        image.caption = captionsByIndex.get(i);
+      }
+    });
+    return images;
+  }
+  /**
+   * Convert tables from DoclingDocument to ProcessedTable[]
+   */
+  async convertTables(doclingDoc) {
+    this.logger.info(
+      `[ResourceConverter] Converting ${doclingDoc.tables.length} tables...`
+    );
+    const captionTexts = doclingDoc.tables.map(
+      (table) => this.captionProcessingPipeline.extractCaptionText(table.captions)
+    );
+    const tables = doclingDoc.tables.map((table) => {
+      const grid = table.data.grid.map(
+        (row) => row.map((cell) => ({
+          text: cell.text,
+          rowSpan: cell.row_span ?? 1,
+          colSpan: cell.col_span ?? 1,
+          isHeader: cell.column_header || cell.row_header || false
+        }))
+      );
+      return {
+        id: this.idGenerator.generateTableId(),
+        pdfPageNo: table.prov?.[0]?.page_no ?? 0,
+        numRows: grid.length,
+        numCols: grid[0]?.length ?? 0,
+        grid
+      };
+    });
+    const captionsByIndex = await this.captionProcessingPipeline.processResourceCaptions(
+      captionTexts,
+      "table"
+    );
+    tables.forEach((table, i) => {
+      if (captionsByIndex.has(i)) {
+        table.caption = captionsByIndex.get(i);
+      }
+    });
+    return tables;
+  }
+  /**
+   * Convert footnotes from DoclingDocument text items
+   */
+  convertFootnotes(doclingDoc) {
+    const footnoteItems = doclingDoc.texts.filter(
+      (item) => item.label === "footnote"
+    );
+    this.logger.info(
+      `[ResourceConverter] Converting ${footnoteItems.length} footnotes...`
+    );
+    const footnotes = footnoteItems.filter((item) => TextCleaner.isValidText(item.text)).map((item) => ({
+      id: this.idGenerator.generateFootnoteId(),
+      text: TextCleaner.normalize(item.text),
+      pdfPageNo: item.prov?.[0]?.page_no ?? 1
+    }));
+    this.logger.info(
+      `[ResourceConverter] Converted ${footnotes.length} valid footnotes`
+    );
+    return footnotes;
+  }
+};
+
 // src/extractors/toc-extract-error.ts
 var TocExtractError = class _TocExtractError extends Error {
   constructor(message, options) {
@@ -1670,43 +1767,33 @@ var TocFinder = class {
    */
   findByStructure(doc) {
     const candidates = [];
-    for (const group of doc.groups) {
-      const pageNo = this.getGroupFirstPage(group);
-      if (pageNo === void 0 || pageNo > this.maxSearchPages) {
-        continue;
-      }
-      if (this.isGroupTocLike(group, doc)) {
-        const score = this.calculateScore(group, pageNo);
-        candidates.push({
-          result: {
-            itemRefs: [group.self_ref],
-            startPage: pageNo,
-            endPage: pageNo
-          },
-          score
-        });
-      }
-    }
-    for (const table of doc.tables) {
+    doc.groups.map((group) => ({ group, pageNo: this.getGroupFirstPage(group) })).filter(
+      ({ group, pageNo }) => pageNo !== void 0 && pageNo <= this.maxSearchPages && this.isGroupTocLike(group, doc)
+    ).forEach(({ group, pageNo }) => {
+      candidates.push({
+        result: {
+          itemRefs: [group.self_ref],
+          startPage: pageNo,
+          endPage: pageNo
+        },
+        score: this.calculateScore(group, pageNo)
+      });
+    });
+    doc.tables.filter((table) => {
       const pageNo = table.prov[0]?.page_no;
-      if (pageNo === void 0 || pageNo > this.maxSearchPages) {
-        continue;
-      }
-      if (this.isTableTocLike(table)) {
-        let score = this.calculateTableScore(table, pageNo);
-        if (this.isResourceIndexTable(table)) {
-          score -= 100;
-        }
-        candidates.push({
-          result: {
-            itemRefs: [table.self_ref],
-            startPage: pageNo,
-            endPage: pageNo
-          },
-          score
-        });
-      }
-    }
+      return pageNo !== void 0 && pageNo <= this.maxSearchPages && this.isTableTocLike(table);
+    }).forEach((table) => {
+      const pageNo = table.prov[0].page_no;
+      const score = this.calculateTableScore(table, pageNo) - (this.isResourceIndexTable(table) ? 100 : 0);
+      candidates.push({
+        result: {
+          itemRefs: [table.self_ref],
+          startPage: pageNo,
+          endPage: pageNo
+        },
+        score
+      });
+    });
     if (candidates.length === 0) {
       return null;
     }
@@ -1778,17 +1865,10 @@ var TocFinder = class {
     if (group.name !== "list" && group.name !== "group") {
       return false;
     }
-    let pageNumberCount = 0;
     const children = this.refResolver.resolveMany(group.children);
-    for (const child of children) {
-      if (!child) continue;
-      if ("text" in child && "orig" in child) {
-        const textItem = child;
-        if (PAGE_NUMBER_PATTERN.test(textItem.text)) {
-          pageNumberCount++;
-        }
-      }
-    }
+    const pageNumberCount = children.filter(
+      (child) => child !== null && "text" in child && "orig" in child && PAGE_NUMBER_PATTERN.test(child.text)
+    ).length;
     const total = children.filter((c) => c !== null).length;
     return pageNumberCount >= 3 || total > 0 && pageNumberCount / total > 0.5;
   }
@@ -1803,13 +1883,10 @@ var TocFinder = class {
     if (num_rows < 3 || num_cols < 2) {
       return false;
     }
-    let numberCount = 0;
-    for (let row = 1; row < grid.length; row++) {
-      const lastCell = grid[row]?.[num_cols - 1];
-      if (lastCell && /^\d+$/.test(lastCell.text.trim())) {
-        numberCount++;
-      }
-    }
+    const numberCount = grid.slice(1).filter((row) => {
+      const lastCell = row?.[num_cols - 1];
+      return lastCell && /^\d+$/.test(lastCell.text.trim());
+    }).length;
     return numberCount > 0 && numberCount / (num_rows - 1) > 0.5;
   }
   /**
@@ -1821,12 +1898,9 @@ var TocFinder = class {
       (c) => c.start_col_offset_idx === 0
     );
     if (firstColCells.length === 0) return false;
-    let resourceCount = 0;
-    for (const cell of firstColCells) {
-      if (RESOURCE_INDEX_PATTERNS.some((p) => p.test(cell.text.trim()))) {
-        resourceCount++;
-      }
-    }
+    const resourceCount = firstColCells.filter(
+      (cell) => RESOURCE_INDEX_PATTERNS.some((p) => p.test(cell.text.trim()))
+    ).length;
     return resourceCount / firstColCells.length > 0.5;
   }
   /**
@@ -1843,9 +1917,7 @@ var TocFinder = class {
         break;
       }
       const newItems = continuationItems.filter((ref) => !seenRefs.has(ref));
-      for (const ref of newItems) {
-        seenRefs.add(ref);
-      }
+      newItems.forEach((ref) => seenRefs.add(ref));
       itemRefs.unshift(...newItems);
       startPage = pageNo;
       this.logger.info(`[TocFinder] Expanded TOC backward to page ${pageNo}`);
@@ -1856,9 +1928,7 @@ var TocFinder = class {
         break;
       }
       const newItems = continuationItems.filter((ref) => !seenRefs.has(ref));
-      for (const ref of newItems) {
-        seenRefs.add(ref);
-      }
+      newItems.forEach((ref) => seenRefs.add(ref));
       itemRefs.push(...newItems);
       endPage = pageNo;
       this.logger.info(`[TocFinder] Expanded TOC forward to page ${pageNo}`);
@@ -1874,37 +1944,27 @@ var TocFinder = class {
    */
   findContinuationOnPage(doc, pageNo) {
     const refs = [];
-    for (const text of doc.texts) {
-      if (text.prov[0]?.page_no !== pageNo) {
-        continue;
-      }
-      if (this.hasContinuationMarker(text.text)) {
-        const parentRef = text.parent?.$ref;
-        if (parentRef) {
-          const group = this.refResolver.resolveGroup(parentRef);
-          if (group) {
-            refs.push(group.self_ref);
-          }
+    doc.texts.filter(
+      (text) => text.prov[0]?.page_no === pageNo && this.hasContinuationMarker(text.text)
+    ).forEach((text) => {
+      const parentRef = text.parent?.$ref;
+      if (parentRef) {
+        const group = this.refResolver.resolveGroup(parentRef);
+        if (group) {
+          refs.push(group.self_ref);
         }
       }
-    }
-    for (const group of doc.groups) {
-      const groupPage = this.getGroupFirstPage(group);
-      if (groupPage !== pageNo) {
-        continue;
-      }
-      if (this.isGroupTocLike(group, doc) && !refs.includes(group.self_ref)) {
-        refs.push(group.self_ref);
-      }
-    }
-    for (const table of doc.tables) {
-      if (table.prov[0]?.page_no !== pageNo) {
-        continue;
-      }
-      if (this.isTableTocLike(table) && !this.isResourceIndexTable(table) && !refs.includes(table.self_ref)) {
-        refs.push(table.self_ref);
-      }
-    }
+    });
+    doc.groups.filter(
+      (group) => this.getGroupFirstPage(group) === pageNo && this.isGroupTocLike(group, doc) && !refs.includes(group.self_ref)
+    ).forEach((group) => {
+      refs.push(group.self_ref);
+    });
+    doc.tables.filter(
+      (table) => table.prov[0]?.page_no === pageNo && this.isTableTocLike(table) && !this.isResourceIndexTable(table) && !refs.includes(table.self_ref)
+    ).forEach((table) => {
+      refs.push(table.self_ref);
+    });
     return refs;
   }
   /**
@@ -1929,16 +1989,10 @@ var TocFinder = class {
    * Get first page number of a group by checking its children
    */
   getGroupFirstPage(group) {
-    for (const childRef of group.children) {
-      const child = this.refResolver.resolve(childRef.$ref);
-      if (child && "prov" in child) {
-        const prov = child.prov;
-        if (prov && prov[0]?.page_no !== void 0) {
-          return prov[0].page_no;
-        }
-      }
-    }
-    return void 0;
+    const found = group.children.map((childRef) => this.refResolver.resolve(childRef.$ref)).find(
+      (child) => child !== null && "prov" in child && child.prov?.[0]?.page_no !== void 0
+    );
+    return found ? found.prov[0]?.page_no : void 0;
   }
   /**
    * Calculate score for a group candidate
@@ -1949,14 +2003,9 @@ var TocFinder = class {
     score += (this.maxSearchPages - pageNo + 1) * 10;
     score += group.children.length * 2;
     const children = this.refResolver.resolveMany(group.children);
-    for (const child of children) {
-      if (child && "text" in child) {
-        const textItem = child;
-        if (PAGE_NUMBER_PATTERN.test(textItem.text)) {
-          score += 5;
-        }
-      }
-    }
+    score += children.filter(
+      (child) => child !== null && "text" in child && PAGE_NUMBER_PATTERN.test(child.text)
+    ).length * 5;
     return score;
   }
   /**
@@ -3034,9 +3083,7 @@ var PageRangeParser = class extends VisionLLMComponent {
       Object.assign(pageRangeMap, groupResult.pageRangeMap);
       usageList.push(...groupResult.usage);
     }
-    for (const usage of usageList) {
-      this.trackUsage(usage);
-    }
+    usageList.forEach((usage) => this.trackUsage(usage));
     this.postProcess(pageRangeMap);
     this.log(
       "info",
@@ -3104,9 +3151,7 @@ var PageRangeParser = class extends VisionLLMComponent {
         this.SAMPLE_SIZE,
         sampledPages
       );
-      for (const p of samplePageNos) {
-        sampledPages.add(p);
-      }
+      samplePageNos.forEach((p) => sampledPages.add(p));
       this.log(
         "info",
         `Attempt ${attempt + 1}/${this.MAX_PATTERN_RETRIES + 1}: sampling pages ${samplePageNos.join(", ")}`
@@ -3166,10 +3211,9 @@ var PageRangeParser = class extends VisionLLMComponent {
       "warn",
       `Could not detect page pattern after ${consecutiveAllNullAttempts >= MAX_ALL_NULL_ATTEMPTS ? consecutiveAllNullAttempts : this.MAX_PATTERN_RETRIES + 1} attempts for group with ${pageNos.length} pages, marking as failed`
     );
-    const fallbackMap = {};
-    for (const pageNo of pageNos) {
-      fallbackMap[pageNo] = { startPageNo: 0, endPageNo: 0 };
-    }
+    const fallbackMap = Object.fromEntries(
+      pageNos.map((pageNo) => [pageNo, { startPageNo: 0, endPageNo: 0 }])
+    );
     return { pageRangeMap: fallbackMap, usage: usageList };
   }
   /**
@@ -3191,7 +3235,7 @@ var PageRangeParser = class extends VisionLLMComponent {
   async extractMultiplePages(pages, pageNos, model) {
     this.log("info", `Extracting ${pageNos.length} pages in single LLM call`);
     const imageContents = [];
-    for (const pageNo of pageNos) {
+    const imageContentsFromPages = pageNos.map((pageNo) => {
       const page = pages[pageNo - 1];
       const imagePath = path3.resolve(
         this.outputPath,
@@ -3199,12 +3243,13 @@ var PageRangeParser = class extends VisionLLMComponent {
       );
       const imageData = new Uint8Array(fs3.readFileSync(imagePath));
       const mimeType = page.image.mimetype || "image/png";
-      imageContents.push({
+      return {
         type: "image",
         image: imageData,
         mediaType: mimeType
-      });
-    }
+      };
+    });
+    imageContents.push(...imageContentsFromPages);
     const schema = import_zod4.z.object({
       pages: import_zod4.z.array(
         import_zod4.z.object({
@@ -3303,7 +3348,7 @@ var PageRangeParser = class extends VisionLLMComponent {
    */
   applyPattern(pageNos, pattern) {
     const result = {};
-    for (const pdfPageNo of pageNos) {
+    pageNos.forEach((pdfPageNo) => {
       switch (pattern.pattern) {
         case "simple_increment" /* SIMPLE_INCREMENT */:
         case "offset" /* OFFSET */: {
@@ -3325,25 +3370,22 @@ var PageRangeParser = class extends VisionLLMComponent {
         default:
           result[pdfPageNo] = { startPageNo: 0, endPageNo: 0 };
       }
-    }
+    });
     return result;
   }
   /**
    * Convert sample results to page range map (for small groups)
    */
   samplesToMap(samples) {
-    const result = {};
-    for (const sample of samples) {
-      if (sample.startPageNo !== null) {
-        result[sample.pdfPageNo] = {
+    return Object.fromEntries(
+      samples.map((sample) => [
+        sample.pdfPageNo,
+        sample.startPageNo !== null ? {
           startPageNo: sample.startPageNo,
           endPageNo: sample.endPageNo ?? sample.startPageNo
-        };
-      } else {
-        result[sample.pdfPageNo] = { startPageNo: 0, endPageNo: 0 };
-      }
-    }
-    return result;
+        } : { startPageNo: 0, endPageNo: 0 }
+      ])
+    );
   }
   /**
    * Post-process the page range map
@@ -3630,42 +3672,357 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
   }
 };
 
-// src/validators/base-validator.ts
-var BaseValidator = class extends TextLLMComponent {
-  /**
-   * Validator name for logging (kept for backwards compatibility)
-   */
-  validatorName;
-  /**
-   * Constructor for BaseValidator
-   *
-   * @param logger - Logger instance
-   * @param model - Language model to use for validation
-   * @param validatorName - Name of the validator for logging (e.g., "TocContentValidator")
-   * @param options - Optional configuration (maxRetries, temperature)
-   * @param fallbackModel - Optional fallback model for retry on failure
-   * @param aggregator - Optional token usage aggregator for tracking LLM calls
-   */
-  constructor(logger, model, validatorName, options, fallbackModel, aggregator) {
-    super(logger, model, validatorName, options, fallbackModel, aggregator);
-    this.validatorName = validatorName;
+// src/pipelines/caption-processing-pipeline.ts
+var CaptionProcessingPipeline = class {
+  logger;
+  captionParser;
+  captionValidator;
+  refResolver;
+  fallbackModel;
+  enableFallbackRetry;
+  maxRetries;
+  captionParserBatchSize;
+  captionValidatorBatchSize;
+  usageAggregator;
+  abortSignal;
+  constructor(deps) {
+    this.logger = deps.logger;
+    this.captionParser = deps.captionParser;
+    this.captionValidator = deps.captionValidator;
+    this.refResolver = deps.refResolver;
+    this.fallbackModel = deps.fallbackModel;
+    this.enableFallbackRetry = deps.enableFallbackRetry;
+    this.maxRetries = deps.maxRetries;
+    this.captionParserBatchSize = deps.captionParserBatchSize;
+    this.captionValidatorBatchSize = deps.captionValidatorBatchSize;
+    this.usageAggregator = deps.usageAggregator;
+    this.abortSignal = deps.abortSignal;
   }
   /**
-   * Call LLM with LLMCaller
+   * Process resource captions (for images and tables)
    *
-   * This method provides backwards compatibility with existing validators.
-   * It wraps the parent callTextLLM method but allows passing a custom aggregator.
+   * Common caption processing pipeline:
+   * 1. Parse captions in batch
+   * 2. Validate parsed captions
+   * 3. Reparse failed captions with fallback model
    *
-   * @param schema - Zod schema for response validation
-   * @param systemPrompt - System prompt
-   * @param userPrompt - User prompt
-   * @param phase - Phase name for tracking (e.g., 'validation', 'batch-validation')
-   * @param aggregator - Optional token usage aggregator for tracking this call
-   * @returns Parsed and validated LLM response with usage information
+   * @param captionTexts - Array of caption texts to process
+   * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
+   * @returns Parsed captions with index mapping
    */
-  async callLLM(schema, systemPrompt, userPrompt, phase, aggregator) {
-    const result = await LLMCaller.call({
-      schema,
+  async processResourceCaptions(captionTexts, resourceType) {
+    const captionsByIndex = /* @__PURE__ */ new Map();
+    const validCaptionData = [];
+    captionTexts.forEach((text, i) => {
+      if (text !== void 0) {
+        validCaptionData.push({
+          resourceIndex: i,
+          filteredIndex: validCaptionData.length,
+          text
+        });
+      }
+    });
+    const validCaptionTexts = validCaptionData.map((item) => item.text);
+    const parsedCaptions = validCaptionTexts.length > 0 ? await this.captionParser.parseBatch(
+      validCaptionTexts,
+      this.captionParserBatchSize
+    ) : [];
+    let finalValidCaptionData = validCaptionData;
+    let finalParsedCaptions = parsedCaptions;
+    if (parsedCaptions.length !== validCaptionData.length) {
+      this.logger.warn(
+        `[CaptionProcessingPipeline] Caption parsing length mismatch for ${resourceType}: expected ${validCaptionData.length}, got ${parsedCaptions.length}. Attempting recovery by matching fullText...`
+      );
+      const parsedMap = new Map(
+        parsedCaptions.map((parsed) => [parsed.fullText, parsed])
+      );
+      const recoveredData = validCaptionData.filter((item) => {
+        if (parsedMap.has(item.text)) {
+          return true;
+        }
+        this.logger.warn(
+          `[CaptionProcessingPipeline] Skipping ${resourceType} caption at index ${item.resourceIndex}: "${item.text}" (not found in parsed results)`
+        );
+        return false;
+      });
+      const recoveredCaptions = recoveredData.map((item) => parsedMap.get(item.text)).filter((caption) => caption !== void 0);
+      if (recoveredCaptions.length !== recoveredData.length) {
+        throw new Error(
+          `[CaptionProcessingPipeline] Failed to recover from length mismatch: recovered ${recoveredCaptions.length} captions for ${recoveredData.length} valid items`
+        );
+      }
+      finalValidCaptionData = recoveredData;
+      finalParsedCaptions = recoveredCaptions;
+      this.logger.info(
+        `[CaptionProcessingPipeline] Successfully recovered ${finalParsedCaptions.length} ${resourceType} captions after length mismatch`
+      );
+    }
+    finalParsedCaptions.forEach((caption, i) => {
+      const resourceIndex = finalValidCaptionData[i].resourceIndex;
+      captionsByIndex.set(resourceIndex, caption);
+    });
+    if (finalParsedCaptions.length > 0) {
+      const finalValidCaptionTexts = finalValidCaptionData.map(
+        (item) => item.text
+      );
+      const validationResults = await this.captionValidator.validateBatch(
+        finalParsedCaptions,
+        finalValidCaptionTexts,
+        this.captionValidatorBatchSize
+      );
+      const failedIndices = validationResults.map((isValid, index) => isValid ? -1 : index).filter((index) => index !== -1);
+      if (failedIndices.length > 0) {
+        failedIndices.forEach((filteredIndex) => {
+          const captionData = finalValidCaptionData[filteredIndex];
+          const originalText = captionData.text;
+          const parsedNum = finalParsedCaptions[filteredIndex].num;
+          const resourceIndex = captionData.resourceIndex;
+          this.logger.warn(
+            `[CaptionProcessingPipeline] Invalid ${resourceType} caption [${resourceIndex}]: "${originalText}" | parsed num="${parsedNum}"`
+          );
+        });
+        if (this.enableFallbackRetry) {
+          this.logger.info(
+            `[CaptionProcessingPipeline] Reparsing ${failedIndices.length} failed ${resourceType} captions with fallback model...`
+          );
+          const failedCaptionTexts = failedIndices.map(
+            (filteredIndex) => finalValidCaptionData[filteredIndex].text
+          );
+          const fallbackCaptionParser = new CaptionParser(
+            this.logger,
+            this.fallbackModel,
+            {
+              maxRetries: this.maxRetries,
+              componentName: "CaptionParser-fallback",
+              abortSignal: this.abortSignal
+            },
+            void 0,
+            // no fallback for the fallback
+            this.usageAggregator
+          );
+          const reparsedCaptions = await fallbackCaptionParser.parseBatch(
+            failedCaptionTexts,
+            0
+            // sequential processing
+          );
+          failedIndices.forEach((filteredIndex, i) => {
+            const resourceIndex = finalValidCaptionData[filteredIndex].resourceIndex;
+            captionsByIndex.set(resourceIndex, reparsedCaptions[i]);
+          });
+          this.logger.info(
+            `[CaptionProcessingPipeline] Reparsed ${reparsedCaptions.length} ${resourceType} captions`
+          );
+        } else {
+          this.logger.warn(
+            `[CaptionProcessingPipeline] ${failedIndices.length} ${resourceType} captions failed validation (kept as-is, fallback retry disabled)`
+          );
+        }
+      }
+    }
+    return captionsByIndex;
+  }
+  /**
+   * Extract caption text from resource
+   *
+   * Handles both string references and $ref resolution
+   */
+  extractCaptionText(captions) {
+    if (!captions?.[0]) {
+      return void 0;
+    }
+    const captionRef = captions[0];
+    if (typeof captionRef === "string") {
+      return captionRef;
+    }
+    if (this.refResolver && "$ref" in captionRef) {
+      const resolved = this.refResolver.resolveText(captionRef.$ref);
+      return resolved?.text;
+    }
+    return void 0;
+  }
+};
+
+// src/pipelines/toc-extraction-pipeline.ts
+var TocExtractionPipeline = class {
+  logger;
+  tocFinder;
+  tocExtractor;
+  tocContentValidator;
+  visionTocExtractor;
+  refResolver;
+  usageAggregator;
+  constructor(deps) {
+    this.logger = deps.logger;
+    this.tocFinder = deps.tocFinder;
+    this.tocExtractor = deps.tocExtractor;
+    this.tocContentValidator = deps.tocContentValidator;
+    this.visionTocExtractor = deps.visionTocExtractor;
+    this.refResolver = deps.refResolver;
+    this.usageAggregator = deps.usageAggregator;
+  }
+  /**
+   * Execute the TOC extraction pipeline
+   *
+   * @param doclingDoc - Document containing texts, pages, etc.
+   * @param _filteredTexts - Pre-filtered texts (reserved for future use)
+   * @returns Extracted TOC entries
+   * @throws {TocNotFoundError} When TOC cannot be found or extracted
+   */
+  async extract(doclingDoc, _filteredTexts) {
+    this.logger.info("[TocExtractionPipeline] Extracting TOC...");
+    let markdown = null;
+    try {
+      const tocArea = this.tocFinder.find(doclingDoc);
+      this.logger.info(
+        `[TocExtractionPipeline] Found TOC area: pages ${tocArea.startPage}-${tocArea.endPage}`
+      );
+      markdown = MarkdownConverter.convert(tocArea.itemRefs, this.refResolver);
+      this.logger.info(
+        `[TocExtractionPipeline] Converted TOC to Markdown (${markdown.length} chars)`
+      );
+      const validation = await this.tocContentValidator.validate(markdown);
+      if (!this.tocContentValidator.isValid(validation)) {
+        this.logger.warn(
+          `[TocExtractionPipeline] TOC validation failed: ${validation.reason}`
+        );
+        markdown = null;
+      } else {
+        const validMarkdown = this.tocContentValidator.getValidMarkdown(validation);
+        if (validMarkdown) {
+          if (validation.contentType === "mixed") {
+            this.logger.info(
+              `[TocExtractionPipeline] Mixed TOC detected, using extracted main TOC (${validMarkdown.length} chars)`
+            );
+          }
+          markdown = validMarkdown;
+          this.logger.info(
+            `[TocExtractionPipeline] TOC validation passed (confidence: ${validation.confidence})`
+          );
+        } else {
+          markdown = null;
+        }
+      }
+    } catch (error) {
+      if (error instanceof TocNotFoundError) {
+        this.logger.info(
+          "[TocExtractionPipeline] Rule-based TOC not found, will try vision fallback"
+        );
+      } else {
+        throw error;
+      }
+    }
+    let fromVision = false;
+    const totalPages = Object.keys(doclingDoc.pages).length;
+    if (!markdown) {
+      fromVision = true;
+      this.logger.info("[TocExtractionPipeline] Using vision fallback for TOC");
+      markdown = await this.visionTocExtractor.extract(totalPages);
+      if (!markdown) {
+        const reason = "Both rule-based search and vision fallback failed to locate TOC";
+        this.logger.error(
+          `[TocExtractionPipeline] TOC extraction failed: ${reason}`
+        );
+        throw new TocNotFoundError(
+          `Table of contents not found in the document. ${reason}.`
+        );
+      }
+      this.logger.info(
+        `[TocExtractionPipeline] Vision extracted TOC markdown (${markdown.length} chars)`
+      );
+    }
+    const maxTocPageNo = extractMaxPageNumber(markdown);
+    const effectiveTotalPages = maxTocPageNo > totalPages ? void 0 : totalPages;
+    let tocResult;
+    try {
+      tocResult = await this.tocExtractor.extract(markdown, {
+        totalPages: effectiveTotalPages
+      });
+    } catch (error) {
+      if (error instanceof TocValidationError) {
+        this.logger.warn(
+          `[TocExtractionPipeline] TOC extraction validation failed: ${error.message}`
+        );
+        tocResult = { entries: [], usages: [] };
+      } else {
+        throw error;
+      }
+    }
+    tocResult.usages.forEach((usage) => this.usageAggregator.track(usage));
+    if (tocResult.entries.length === 0 && !fromVision) {
+      this.logger.warn(
+        "[TocExtractionPipeline] Text-based TOC extraction yielded 0 entries, retrying with vision"
+      );
+      const visionMarkdown = await this.visionTocExtractor.extract(totalPages);
+      if (visionMarkdown) {
+        this.logger.info(
+          `[TocExtractionPipeline] Vision extracted TOC markdown (${visionMarkdown.length} chars)`
+        );
+        const visionMaxPageNo = extractMaxPageNumber(visionMarkdown);
+        const visionEffectivePages = visionMaxPageNo > totalPages ? void 0 : totalPages;
+        try {
+          const visionResult = await this.tocExtractor.extract(visionMarkdown, {
+            totalPages: visionEffectivePages
+          });
+          visionResult.usages.forEach(
+            (usage) => this.usageAggregator.track(usage)
+          );
+          if (visionResult.entries.length > 0) {
+            tocResult = visionResult;
+          }
+        } catch {
+        }
+      }
+    }
+    if (tocResult.entries.length === 0) {
+      const reason = "TOC area was detected but LLM could not extract any structured entries";
+      this.logger.error(
+        `[TocExtractionPipeline] TOC extraction failed: ${reason}`
+      );
+      throw new TocNotFoundError(`${reason}.`);
+    }
+    this.logger.info(
+      `[TocExtractionPipeline] Extracted ${tocResult.entries.length} top-level TOC entries`
+    );
+    return tocResult.entries;
+  }
+};
+
+// src/validators/base-validator.ts
+var BaseValidator = class extends TextLLMComponent {
+  /**
+   * Validator name for logging (kept for backwards compatibility)
+   */
+  validatorName;
+  /**
+   * Constructor for BaseValidator
+   *
+   * @param logger - Logger instance
+   * @param model - Language model to use for validation
+   * @param validatorName - Name of the validator for logging (e.g., "TocContentValidator")
+   * @param options - Optional configuration (maxRetries, temperature)
+   * @param fallbackModel - Optional fallback model for retry on failure
+   * @param aggregator - Optional token usage aggregator for tracking LLM calls
+   */
+  constructor(logger, model, validatorName, options, fallbackModel, aggregator) {
+    super(logger, model, validatorName, options, fallbackModel, aggregator);
+    this.validatorName = validatorName;
+  }
+  /**
+   * Call LLM with LLMCaller
+   *
+   * This method provides backwards compatibility with existing validators.
+   * It wraps the parent callTextLLM method but allows passing a custom aggregator.
+   *
+   * @param schema - Zod schema for response validation
+   * @param systemPrompt - System prompt
+   * @param userPrompt - User prompt
+   * @param phase - Phase name for tracking (e.g., 'validation', 'batch-validation')
+   * @param aggregator - Optional token usage aggregator for tracking this call
+   * @returns Parsed and validated LLM response with usage information
+   */
+  async callLLM(schema, systemPrompt, userPrompt, phase, aggregator) {
+    const result = await LLMCaller.call({
+      schema,
       systemPrompt,
       userPrompt,
       primaryModel: this.model,
@@ -4078,7 +4435,9 @@ var DocumentProcessor = class {
   visionTocExtractor;
   captionParser;
   chapterConverter;
-  textCleaner = TextCleaner;
+  resourceConverter;
+  tocExtractionPipeline;
+  captionProcessingPipeline;
   usageAggregator = new LLMTokenUsageAggregator();
   constructor(options) {
     this.logger = options.logger;
@@ -4161,16 +4520,16 @@ var DocumentProcessor = class {
     this.emitTokenUsage();
     this.checkAborted();
     const startTimeToc = Date.now();
-    const tocEntries = await this.extractTableOfContents(doclingDoc, filtered);
+    const tocEntries = await this.tocExtractionPipeline.extract(
+      doclingDoc,
+      filtered
+    );
     const tocTime = Date.now() - startTimeToc;
     this.logger.info(`[DocumentProcessor] TOC extraction took ${tocTime}ms`);
     this.emitTokenUsage();
     this.checkAborted();
     const startTimeResources = Date.now();
-    const { images, tables, footnotes } = await this.convertResources(
-      doclingDoc,
-      outputPath
-    );
+    const { images, tables, footnotes } = await this.resourceConverter.convertAll(doclingDoc, outputPath);
     const resourcesTime = Date.now() - startTimeResources;
     this.logger.info(
       `[DocumentProcessor] Resource conversion took ${resourcesTime}ms`
@@ -4275,7 +4634,37 @@ var DocumentProcessor = class {
     );
     this.logger.info("[DocumentProcessor] - ChapterConverter");
     this.chapterConverter = new ChapterConverter(this.logger, this.idGenerator);
-    this.logger.info("[DocumentProcessor] All processors initialized");
+    this.logger.info("[DocumentProcessor] - TocExtractionPipeline");
+    this.tocExtractionPipeline = new TocExtractionPipeline({
+      logger: this.logger,
+      tocFinder: this.tocFinder,
+      tocExtractor: this.tocExtractor,
+      tocContentValidator: this.tocContentValidator,
+      visionTocExtractor: this.visionTocExtractor,
+      refResolver: this.refResolver,
+      usageAggregator: this.usageAggregator
+    });
+    this.logger.info("[DocumentProcessor] - CaptionProcessingPipeline");
+    this.captionProcessingPipeline = new CaptionProcessingPipeline({
+      logger: this.logger,
+      captionParser: this.captionParser,
+      captionValidator: this.captionValidator,
+      refResolver: this.refResolver,
+      fallbackModel: this.fallbackModel,
+      enableFallbackRetry: this.enableFallbackRetry,
+      maxRetries: this.maxRetries,
+      captionParserBatchSize: this.captionParserBatchSize,
+      captionValidatorBatchSize: this.captionValidatorBatchSize,
+      usageAggregator: this.usageAggregator,
+      abortSignal: this.abortSignal
+    });
+    this.logger.info("[DocumentProcessor] - ResourceConverter");
+    this.resourceConverter = new ResourceConverter(
+      this.logger,
+      this.idGenerator,
+      this.captionProcessingPipeline
+    );
+    this.logger.info("[DocumentProcessor] All processors initialized");
   }
   /**
    * Normalize and filter texts using TextCleaner
@@ -4286,7 +4675,7 @@ var DocumentProcessor = class {
   normalizeAndFilterTexts(doclingDoc) {
     this.logger.info("[DocumentProcessor] Normalizing and filtering texts...");
     const texts = doclingDoc.texts.map((text) => text.text);
-    const filtered = this.textCleaner.normalizeAndFilterBatch(
+    const filtered = TextCleaner.normalizeAndFilterBatch(
       texts,
       this.textCleanerBatchSize
     );
@@ -4310,58 +4699,6 @@ var DocumentProcessor = class {
     );
     return pageRangeMap;
   }
-  /**
-   * Convert images, tables, and footnotes
-   *
-   * Runs conversions:
-   * - Images conversion (with caption extraction)
-   * - Tables conversion (with caption extraction, excluding TOC tables)
-   * - Footnotes conversion (synchronous, from text items with label='footnote')
-   */
-  async convertResources(doclingDoc, outputPath) {
-    this.logger.info(
-      "[DocumentProcessor] Converting images, tables, and footnotes..."
-    );
-    const [images, tables] = await Promise.all([
-      this.convertImages(doclingDoc, outputPath),
-      this.convertTables(doclingDoc)
-    ]);
-    const footnotes = this.convertFootnotes(doclingDoc);
-    this.logger.info(
-      `[DocumentProcessor] Converted ${images.length} images, ${tables.length} tables, and ${footnotes.length} footnotes`
-    );
-    return { images, tables, footnotes };
-  }
-  /**
-   * Convert footnotes
-   *
-   * Extracts footnotes from DoclingDocument text items with label='footnote'
-   */
-  convertFootnotes(doclingDoc) {
-    const footnoteItems = doclingDoc.texts.filter(
-      (item) => item.label === "footnote"
-    );
-    this.logger.info(
-      `[DocumentProcessor] Converting ${footnoteItems.length} footnotes...`
-    );
-    const footnotes = [];
-    for (const item of footnoteItems) {
-      if (!this.textCleaner.isValidText(item.text)) {
-        continue;
-      }
-      const pdfPageNo = item.prov?.[0]?.page_no ?? 1;
-      const footnoteId = this.idGenerator.generateFootnoteId();
-      footnotes.push({
-        id: footnoteId,
-        text: this.textCleaner.normalize(item.text),
-        pdfPageNo
-      });
-    }
-    this.logger.info(
-      `[DocumentProcessor] Converted ${footnotes.length} valid footnotes`
-    );
-    return footnotes;
-  }
   /**
    * Assemble the final ProcessedDocument
    *
@@ -4382,386 +4719,6 @@ var DocumentProcessor = class {
     );
     return processedDoc;
   }
-  /**
-   * Extract table of contents (TOC)
-   *
-   * Uses rule-based extraction with LLM validation and vision fallback:
-   * 1. TocFinder - find TOC area in document (rule-based)
-   * 2. MarkdownConverter - convert TOC items to Markdown
-   * 3. TocContentValidator - validate if content is actually a TOC (LLM)
-   * 4. If invalid: VisionTocExtractor - extract from page images (vision LLM fallback)
-   * 5. TocExtractor - LLM-based structured extraction
-   */
-  async extractTableOfContents(doclingDoc, _filteredTexts) {
-    this.logger.info("[DocumentProcessor] Extracting TOC...");
-    let markdown = null;
-    try {
-      const tocArea = this.tocFinder.find(doclingDoc);
-      this.logger.info(
-        `[DocumentProcessor] Found TOC area: pages ${tocArea.startPage}-${tocArea.endPage}`
-      );
-      markdown = MarkdownConverter.convert(tocArea.itemRefs, this.refResolver);
-      this.logger.info(
-        `[DocumentProcessor] Converted TOC to Markdown (${markdown.length} chars)`
-      );
-      const validation = await this.tocContentValidator.validate(markdown);
-      if (!this.tocContentValidator.isValid(validation)) {
-        this.logger.warn(
-          `[DocumentProcessor] TOC validation failed: ${validation.reason}`
-        );
-        markdown = null;
-      } else {
-        const validMarkdown = this.tocContentValidator.getValidMarkdown(validation);
-        if (validMarkdown) {
-          if (validation.contentType === "mixed") {
-            this.logger.info(
-              `[DocumentProcessor] Mixed TOC detected, using extracted main TOC (${validMarkdown.length} chars)`
-            );
-          }
-          markdown = validMarkdown;
-          this.logger.info(
-            `[DocumentProcessor] TOC validation passed (confidence: ${validation.confidence})`
-          );
-        } else {
-          markdown = null;
-        }
-      }
-    } catch (error) {
-      if (error instanceof TocNotFoundError) {
-        this.logger.info(
-          "[DocumentProcessor] Rule-based TOC not found, will try vision fallback"
-        );
-      } else {
-        throw error;
-      }
-    }
-    let fromVision = false;
-    if (!markdown) {
-      fromVision = true;
-      this.logger.info("[DocumentProcessor] Using vision fallback for TOC");
-      const totalPages2 = Object.keys(doclingDoc.pages).length;
-      markdown = await this.visionTocExtractor.extract(totalPages2);
-      if (!markdown) {
-        const reason = "Both rule-based search and vision fallback failed to locate TOC";
-        this.logger.error(
-          `[DocumentProcessor] TOC extraction failed: ${reason}`
-        );
-        throw new TocNotFoundError(
-          `Table of contents not found in the document. ${reason}.`
-        );
-      }
-      this.logger.info(
-        `[DocumentProcessor] Vision extracted TOC markdown (${markdown.length} chars)`
-      );
-    }
-    const totalPages = Object.keys(doclingDoc.pages).length;
-    const maxTocPageNo = this.extractMaxPageNumber(markdown);
-    const effectiveTotalPages = maxTocPageNo > totalPages ? void 0 : totalPages;
-    let tocResult;
-    try {
-      tocResult = await this.tocExtractor.extract(markdown, {
-        totalPages: effectiveTotalPages
-      });
-    } catch (error) {
-      if (error instanceof TocValidationError) {
-        this.logger.warn(
-          `[DocumentProcessor] TOC extraction validation failed: ${error.message}`
-        );
-        tocResult = { entries: [], usages: [] };
-      } else {
-        throw error;
-      }
-    }
-    for (const usage of tocResult.usages) {
-      this.usageAggregator.track(usage);
-    }
-    if (tocResult.entries.length === 0 && !fromVision) {
-      this.logger.warn(
-        "[DocumentProcessor] Text-based TOC extraction yielded 0 entries, retrying with vision"
-      );
-      const visionMarkdown = await this.visionTocExtractor.extract(totalPages);
-      if (visionMarkdown) {
-        this.logger.info(
-          `[DocumentProcessor] Vision extracted TOC markdown (${visionMarkdown.length} chars)`
-        );
-        const visionMaxPageNo = this.extractMaxPageNumber(visionMarkdown);
-        const visionEffectivePages = visionMaxPageNo > totalPages ? void 0 : totalPages;
-        try {
-          const visionResult = await this.tocExtractor.extract(
-            visionMarkdown,
-            {
-              totalPages: visionEffectivePages
-            }
-          );
-          for (const usage of visionResult.usages) {
-            this.usageAggregator.track(usage);
-          }
-          if (visionResult.entries.length > 0) {
-            tocResult = visionResult;
-          }
-        } catch {
-        }
-      }
-    }
-    if (tocResult.entries.length === 0) {
-      const reason = "TOC area was detected but LLM could not extract any structured entries";
-      this.logger.error(`[DocumentProcessor] TOC extraction failed: ${reason}`);
-      throw new TocNotFoundError(`${reason}.`);
-    }
-    this.logger.info(
-      `[DocumentProcessor] Extracted ${tocResult.entries.length} top-level TOC entries`
-    );
-    return tocResult.entries;
-  }
-  /**
-   * Extract the maximum page number from TOC markdown
-   *
-   * Parses page numbers from dot-leader patterns (e.g., "..... 175")
-   * and table cell patterns (e.g., "| title | 175 |") to detect compiled
-   * volume scenarios where TOC page numbers exceed the sub-document's page count.
-   */
-  extractMaxPageNumber(markdown) {
-    const dotLeaderMatches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
-    const tableCellMatches = [...markdown.matchAll(/\|\s*(\d+)\s*\|\s*$/gm)];
-    const allNumbers = [
-      ...dotLeaderMatches.map((m) => parseInt(m[1], 10)),
-      ...tableCellMatches.map((m) => parseInt(m[1], 10))
-    ];
-    if (allNumbers.length === 0) {
-      return 0;
-    }
-    return Math.max(...allNumbers);
-  }
-  /**
-   * Process resource captions (for images and tables)
-   *
-   * Common caption processing pipeline:
-   * 1. Parse captions in batch
-   * 2. Validate parsed captions
-   * 3. Reparse failed captions with fallback model
-   *
-   * @param captionTexts - Array of caption texts to process
-   * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
-   * @returns Parsed captions with index mapping
-   */
-  async processResourceCaptions(captionTexts, resourceType) {
-    const captionsByIndex = /* @__PURE__ */ new Map();
-    const validCaptionData = [];
-    for (let i = 0; i < captionTexts.length; i++) {
-      const text = captionTexts[i];
-      if (text !== void 0) {
-        validCaptionData.push({
-          resourceIndex: i,
-          filteredIndex: validCaptionData.length,
-          text
-        });
-      }
-    }
-    const validCaptionTexts = validCaptionData.map((item) => item.text);
-    const parsedCaptions = validCaptionTexts.length > 0 ? await this.captionParser.parseBatch(
-      validCaptionTexts,
-      this.captionParserBatchSize
-    ) : [];
-    let finalValidCaptionData = validCaptionData;
-    let finalParsedCaptions = parsedCaptions;
-    if (parsedCaptions.length !== validCaptionData.length) {
-      this.logger.warn(
-        `[DocumentProcessor] Caption parsing length mismatch for ${resourceType}: expected ${validCaptionData.length}, got ${parsedCaptions.length}. Attempting recovery by matching fullText...`
-      );
-      const parsedMap = /* @__PURE__ */ new Map();
-      for (const parsed of parsedCaptions) {
-        parsedMap.set(parsed.fullText, parsed);
-      }
-      const recoveredData = [];
-      for (const item of validCaptionData) {
-        if (parsedMap.has(item.text)) {
-          recoveredData.push(item);
-        } else {
-          this.logger.warn(
-            `[DocumentProcessor] Skipping ${resourceType} caption at index ${item.resourceIndex}: "${item.text}" (not found in parsed results)`
-          );
-        }
-      }
-      const recoveredCaptions = [];
-      for (const item of recoveredData) {
-        const caption = parsedMap.get(item.text);
-        if (caption) {
-          recoveredCaptions.push(caption);
-        }
-      }
-      if (recoveredCaptions.length !== recoveredData.length) {
-        throw new Error(
-          `[DocumentProcessor] Failed to recover from length mismatch: recovered ${recoveredCaptions.length} captions for ${recoveredData.length} valid items`
-        );
-      }
-      finalValidCaptionData = recoveredData;
-      finalParsedCaptions = recoveredCaptions;
-      this.logger.info(
-        `[DocumentProcessor] Successfully recovered ${finalParsedCaptions.length} ${resourceType} captions after length mismatch`
-      );
-    }
-    for (let i = 0; i < finalParsedCaptions.length; i++) {
-      const resourceIndex = finalValidCaptionData[i].resourceIndex;
-      captionsByIndex.set(resourceIndex, finalParsedCaptions[i]);
-    }
-    if (finalParsedCaptions.length > 0) {
-      const finalValidCaptionTexts = finalValidCaptionData.map(
-        (item) => item.text
-      );
-      const validationResults = await this.captionValidator.validateBatch(
-        finalParsedCaptions,
-        finalValidCaptionTexts,
-        this.captionValidatorBatchSize
-      );
-      const failedIndices = validationResults.map((isValid, index) => isValid ? -1 : index).filter((index) => index !== -1);
-      if (failedIndices.length > 0) {
-        for (const filteredIndex of failedIndices) {
-          const captionData = finalValidCaptionData[filteredIndex];
-          const originalText = captionData.text;
-          const parsedNum = finalParsedCaptions[filteredIndex].num;
-          const resourceIndex = captionData.resourceIndex;
-          this.logger.warn(
-            `[DocumentProcessor] Invalid ${resourceType} caption [${resourceIndex}]: "${originalText}" | parsed num="${parsedNum}"`
-          );
-        }
-        if (this.enableFallbackRetry) {
-          this.logger.info(
-            `[DocumentProcessor] Reparsing ${failedIndices.length} failed ${resourceType} captions with fallback model...`
-          );
-          const failedCaptionTexts = failedIndices.map(
-            (filteredIndex) => finalValidCaptionData[filteredIndex].text
-          );
-          const fallbackCaptionParser = new CaptionParser(
-            this.logger,
-            this.fallbackModel,
-            {
-              maxRetries: this.maxRetries,
-              componentName: "CaptionParser-fallback",
-              abortSignal: this.abortSignal
-            },
-            void 0,
-            // no fallback for the fallback
-            this.usageAggregator
-          );
-          const reparsedCaptions = await fallbackCaptionParser.parseBatch(
-            failedCaptionTexts,
-            0
-            // sequential processing
-          );
-          for (let i = 0; i < failedIndices.length; i++) {
-            const filteredIndex = failedIndices[i];
-            const resourceIndex = finalValidCaptionData[filteredIndex].resourceIndex;
-            captionsByIndex.set(resourceIndex, reparsedCaptions[i]);
-          }
-          this.logger.info(
-            `[DocumentProcessor] Reparsed ${reparsedCaptions.length} ${resourceType} captions`
-          );
-        } else {
-          this.logger.warn(
-            `[DocumentProcessor] ${failedIndices.length} ${resourceType} captions failed validation (kept as-is, fallback retry disabled)`
-          );
-        }
-      }
-    }
-    return captionsByIndex;
-  }
-  /**
-   * Extract caption text from resource
-   *
-   * Handles both string references and $ref resolution
-   */
-  extractCaptionText(captions) {
-    if (!captions?.[0]) {
-      return void 0;
-    }
-    const captionRef = captions[0];
-    if (typeof captionRef === "string") {
-      return captionRef;
-    }
-    if (this.refResolver && "$ref" in captionRef) {
-      const resolved = this.refResolver.resolveText(captionRef.$ref);
-      return resolved?.text;
-    }
-    return void 0;
-  }
-  /**
-   * Convert images
-   *
-   * Converts pictures from DoclingDocument to ProcessedImage
-   */
-  async convertImages(doclingDoc, outputPath) {
-    this.logger.info(
-      `[DocumentProcessor] Converting ${doclingDoc.pictures.length} images...`
-    );
-    const images = [];
-    const captionTexts = [];
-    for (const picture of doclingDoc.pictures) {
-      const pdfPageNo = picture.prov?.[0]?.page_no ?? 0;
-      const imageId = this.idGenerator?.generateImageId() ?? `img-${images.length + 1}`;
-      const captionText = this.extractCaptionText(picture.captions);
-      captionTexts.push(captionText);
-      images.push({
-        id: imageId,
-        path: `${outputPath}/images/image_${images.length}.png`,
-        pdfPageNo
-        // caption will be assigned later
-      });
-    }
-    const captionsByIndex = await this.processResourceCaptions(
-      captionTexts,
-      "image"
-    );
-    for (let i = 0; i < images.length; i++) {
-      if (captionsByIndex.has(i)) {
-        images[i].caption = captionsByIndex.get(i);
-      }
-    }
-    return images;
-  }
-  /**
-   * Convert tables
-   *
-   * Converts tables from DoclingDocument to ProcessedTable
-   */
-  async convertTables(doclingDoc) {
-    this.logger.info(
-      `[DocumentProcessor] Converting ${doclingDoc.tables.length} tables...`
-    );
-    const tables = [];
-    const captionTexts = [];
-    for (const table of doclingDoc.tables) {
-      const pdfPageNo = table.prov?.[0]?.page_no ?? 0;
-      const tableId = this.idGenerator?.generateTableId() ?? `tbl-${tables.length + 1}`;
-      const grid = table.data.grid.map(
-        (row) => row.map((cell) => ({
-          text: cell.text,
-          rowSpan: cell.row_span ?? 1,
-          colSpan: cell.col_span ?? 1,
-          isHeader: cell.column_header || cell.row_header || false
-        }))
-      );
-      const captionText = this.extractCaptionText(table.captions);
-      captionTexts.push(captionText);
-      tables.push({
-        id: tableId,
-        pdfPageNo,
-        numRows: grid.length,
-        numCols: grid[0]?.length ?? 0,
-        grid
-        // caption will be assigned later
-      });
-    }
-    const captionsByIndex = await this.processResourceCaptions(
-      captionTexts,
-      "table"
-    );
-    for (let i = 0; i < tables.length; i++) {
-      if (captionsByIndex.has(i)) {
-        tables[i].caption = captionsByIndex.get(i);
-      }
-    }
-    return tables;
-  }
   /**
    * Convert chapters and link resources
    *
@@ -4796,6 +4753,7 @@ var DocumentProcessor = class {
   CONTINUATION_MARKERS,
   CaptionParseError,
   CaptionParser,
+  CaptionProcessingPipeline,
   CaptionValidationError,
   CaptionValidator,
   ChapterConverter,
@@ -4804,12 +4762,14 @@ var DocumentProcessor = class {
   PagePattern,
   PageRangeParseError,
   PageRangeParser,
+  ResourceConverter,
   TOC_KEYWORDS,
   TextLLMComponent,
   TocContentValidationSchema,
   TocContentValidator,
   TocEntrySchema,
   TocExtractError,
+  TocExtractionPipeline,
   TocExtractor,
   TocFinder,
   TocNotFoundError,
@@ -4817,6 +4777,7 @@ var DocumentProcessor = class {
   TocResponseSchema,
   VisionLLMComponent,
   VisionTocExtractionSchema,
-  VisionTocExtractor
+  VisionTocExtractor,
+  extractMaxPageNumber
 });
 //# sourceMappingURL=index.cjs.map