@heripo/document-processor 0.1.16 → 0.1.18

package/dist/index.d.cts

@@ -60,8 +60,8 @@ interface DocumentProcessorOptions {
      */
     maxRetries?: number;
     /**
-     * Enable fallback retry mechanism - automatically retries with fallback model on failure (default: true)
-     * Set to false to disable automatic fallback retry and fail immediately on component-specific model errors
+     * Enable fallback retry mechanism - automatically retries with fallback model on failure (default: false)
+     * Set to true to enable automatic fallback retry with fallback model on component-specific model errors
      */
     enableFallbackRetry?: boolean;
     /**
@@ -155,7 +155,9 @@ declare class DocumentProcessor {
     private visionTocExtractor?;
     private captionParser?;
     private chapterConverter?;
-    private textCleaner;
+    private resourceConverter?;
+    private tocExtractionPipeline?;
+    private captionProcessingPipeline?;
     private readonly usageAggregator;
     constructor(options: DocumentProcessorOptions);
     /**
@@ -214,77 +216,12 @@ declare class DocumentProcessor {
      * Token usage is automatically tracked by PageRangeParser into the shared aggregator.
      */
     private parsePageRanges;
-    /**
-     * Convert images, tables, and footnotes
-     *
-     * Runs conversions:
-     * - Images conversion (with caption extraction)
-     * - Tables conversion (with caption extraction, excluding TOC tables)
-     * - Footnotes conversion (synchronous, from text items with label='footnote')
-     */
-    private convertResources;
-    /**
-     * Convert footnotes
-     *
-     * Extracts footnotes from DoclingDocument text items with label='footnote'
-     */
-    private convertFootnotes;
     /**
      * Assemble the final ProcessedDocument
      *
      * Creates the ProcessedDocument structure with all converted components
      */
     private assembleProcessedDocument;
-    /**
-     * Extract table of contents (TOC)
-     *
-     * Uses rule-based extraction with LLM validation and vision fallback:
-     * 1. TocFinder - find TOC area in document (rule-based)
-     * 2. MarkdownConverter - convert TOC items to Markdown
-     * 3. TocContentValidator - validate if content is actually a TOC (LLM)
-     * 4. If invalid: VisionTocExtractor - extract from page images (vision LLM fallback)
-     * 5. TocExtractor - LLM-based structured extraction
-     */
-    private extractTableOfContents;
-    /**
-     * Extract the maximum page number from TOC markdown
-     *
-     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
-     * and table cell patterns (e.g., "| title | 175 |") to detect compiled
-     * volume scenarios where TOC page numbers exceed the sub-document's page count.
-     */
-    private extractMaxPageNumber;
-    /**
-     * Process resource captions (for images and tables)
-     *
-     * Common caption processing pipeline:
-     * 1. Parse captions in batch
-     * 2. Validate parsed captions
-     * 3. Reparse failed captions with fallback model
-     *
-     * @param captionTexts - Array of caption texts to process
-     * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
-     * @returns Parsed captions with index mapping
-     */
-    private processResourceCaptions;
-    /**
-     * Extract caption text from resource
-     *
-     * Handles both string references and $ref resolution
-     */
-    private extractCaptionText;
-    /**
-     * Convert images
-     *
-     * Converts pictures from DoclingDocument to ProcessedImage
-     */
-    private convertImages;
-    /**
-     * Convert tables
-     *
-     * Converts tables from DoclingDocument to ProcessedTable
-     */
-    private convertTables;
     /**
      * Convert chapters and link resources
      *
@@ -924,6 +861,18 @@ declare class IdGenerator {
     private padNumber;
 }
 
+/**
+ * Extract the maximum page number from TOC markdown
+ *
+ * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+ * and table cell patterns (e.g., "| title | 175 |") to detect compiled
+ * volume scenarios where TOC page numbers exceed the sub-document's page count.
+ *
+ * @param markdown - TOC markdown content
+ * @returns Maximum page number found, or 0 if none found
+ */
+declare function extractMaxPageNumber(markdown: string): number;
+
 /**
  * TOC keyword patterns for different languages
  * Korean: 목차, 차례, 목 차
@@ -1508,4 +1457,142 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+/**
+ * Dependencies required by CaptionProcessingPipeline
+ */
+interface CaptionProcessingPipelineDeps {
+    logger: LoggerMethods;
+    captionParser: CaptionParser;
+    captionValidator: CaptionValidator;
+    refResolver?: RefResolver;
+    fallbackModel: LanguageModel;
+    enableFallbackRetry: boolean;
+    maxRetries: number;
+    captionParserBatchSize: number;
+    captionValidatorBatchSize: number;
+    usageAggregator: LLMTokenUsageAggregator;
+    abortSignal?: AbortSignal;
+}
+/**
+ * CaptionProcessingPipeline
+ *
+ * Processes resource captions through a multi-step pipeline:
+ * 1. Parse captions in batch (CaptionParser)
+ * 2. Handle length mismatch recovery
+ * 3. Validate parsed captions (CaptionValidator)
+ * 4. Reparse failed captions with fallback model
+ */
+declare class CaptionProcessingPipeline {
+    private readonly logger;
+    private readonly captionParser;
+    private readonly captionValidator;
+    private readonly refResolver?;
+    private readonly fallbackModel;
+    private readonly enableFallbackRetry;
+    private readonly maxRetries;
+    private readonly captionParserBatchSize;
+    private readonly captionValidatorBatchSize;
+    private readonly usageAggregator;
+    private readonly abortSignal?;
+    constructor(deps: CaptionProcessingPipelineDeps);
+    /**
+     * Process resource captions (for images and tables)
+     *
+     * Common caption processing pipeline:
+     * 1. Parse captions in batch
+     * 2. Validate parsed captions
+     * 3. Reparse failed captions with fallback model
+     *
+     * @param captionTexts - Array of caption texts to process
+     * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
+     * @returns Parsed captions with index mapping
+     */
+    processResourceCaptions(captionTexts: Array<string | undefined>, resourceType: string): Promise<Map<number, Caption>>;
+    /**
+     * Extract caption text from resource
+     *
+     * Handles both string references and $ref resolution
+     */
+    extractCaptionText(captions: Array<string | {
+        $ref: string;
+    }> | undefined): string | undefined;
+}
+
+/**
+ * Dependencies required by TocExtractionPipeline
+ */
+interface TocExtractionPipelineDeps {
+    logger: LoggerMethods;
+    tocFinder: TocFinder;
+    tocExtractor: TocExtractor;
+    tocContentValidator: TocContentValidator;
+    visionTocExtractor: VisionTocExtractor;
+    refResolver: RefResolver;
+    usageAggregator: LLMTokenUsageAggregator;
+}
+/**
+ * TocExtractionPipeline
+ *
+ * Orchestrates the multi-stage TOC extraction process:
+ * 1. Rule-based extraction (TocFinder + MarkdownConverter)
+ * 2. Content validation (TocContentValidator)
+ * 3. Vision fallback (VisionTocExtractor)
+ * 4. Structured extraction with validation retry (TocExtractor)
+ * 5. Vision fallback retry when text-based extraction yields 0 entries
+ */
+declare class TocExtractionPipeline {
+    private readonly logger;
+    private readonly tocFinder;
+    private readonly tocExtractor;
+    private readonly tocContentValidator;
+    private readonly visionTocExtractor;
+    private readonly refResolver;
+    private readonly usageAggregator;
+    constructor(deps: TocExtractionPipelineDeps);
+    /**
+     * Execute the TOC extraction pipeline
+     *
+     * @param doclingDoc - Document containing texts, pages, etc.
+     * @param _filteredTexts - Pre-filtered texts (reserved for future use)
+     * @returns Extracted TOC entries
+     * @throws {TocNotFoundError} When TOC cannot be found or extracted
+     */
+    extract(doclingDoc: DoclingDocument, _filteredTexts: string[]): Promise<TocEntry[]>;
+}
+
+/**
+ * ResourceConverter
+ *
+ * Converts DoclingDocument resources (images, tables, footnotes) to processed format.
+ * Uses CaptionProcessingPipeline for caption extraction and validation.
+ */
+declare class ResourceConverter {
+    private readonly logger;
+    private readonly idGenerator;
+    private readonly captionProcessingPipeline;
+    constructor(logger: LoggerMethods, idGenerator: IdGenerator, captionProcessingPipeline: CaptionProcessingPipeline);
+    /**
+     * Convert all resources (images, tables, footnotes)
+     *
+     * Runs image and table conversions in parallel, then footnotes synchronously.
+     */
+    convertAll(doclingDoc: DoclingDocument, outputPath: string): Promise<{
+        images: ProcessedImage[];
+        tables: ProcessedTable[];
+        footnotes: ProcessedFootnote[];
+    }>;
+    /**
+     * Convert images from DoclingDocument to ProcessedImage[]
+     */
+    convertImages(doclingDoc: DoclingDocument, outputPath: string): Promise<ProcessedImage[]>;
+    /**
+     * Convert tables from DoclingDocument to ProcessedTable[]
+     */
+    convertTables(doclingDoc: DoclingDocument): Promise<ProcessedTable[]>;
+    /**
+     * Convert footnotes from DoclingDocument text items
+     */
+    convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
+}
+
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };

package/dist/index.d.ts

@@ -60,8 +60,8 @@ interface DocumentProcessorOptions {
      */
     maxRetries?: number;
     /**
-     * Enable fallback retry mechanism - automatically retries with fallback model on failure (default: true)
-     * Set to false to disable automatic fallback retry and fail immediately on component-specific model errors
+     * Enable fallback retry mechanism - automatically retries with fallback model on failure (default: false)
+     * Set to true to enable automatic fallback retry with fallback model on component-specific model errors
      */
     enableFallbackRetry?: boolean;
     /**
@@ -155,7 +155,9 @@ declare class DocumentProcessor {
     private visionTocExtractor?;
     private captionParser?;
     private chapterConverter?;
-    private textCleaner;
+    private resourceConverter?;
+    private tocExtractionPipeline?;
+    private captionProcessingPipeline?;
     private readonly usageAggregator;
     constructor(options: DocumentProcessorOptions);
     /**
@@ -214,77 +216,12 @@ declare class DocumentProcessor {
      * Token usage is automatically tracked by PageRangeParser into the shared aggregator.
      */
     private parsePageRanges;
-    /**
-     * Convert images, tables, and footnotes
-     *
-     * Runs conversions:
-     * - Images conversion (with caption extraction)
-     * - Tables conversion (with caption extraction, excluding TOC tables)
-     * - Footnotes conversion (synchronous, from text items with label='footnote')
-     */
-    private convertResources;
-    /**
-     * Convert footnotes
-     *
-     * Extracts footnotes from DoclingDocument text items with label='footnote'
-     */
-    private convertFootnotes;
     /**
      * Assemble the final ProcessedDocument
      *
      * Creates the ProcessedDocument structure with all converted components
      */
     private assembleProcessedDocument;
-    /**
-     * Extract table of contents (TOC)
-     *
-     * Uses rule-based extraction with LLM validation and vision fallback:
-     * 1. TocFinder - find TOC area in document (rule-based)
-     * 2. MarkdownConverter - convert TOC items to Markdown
-     * 3. TocContentValidator - validate if content is actually a TOC (LLM)
-     * 4. If invalid: VisionTocExtractor - extract from page images (vision LLM fallback)
-     * 5. TocExtractor - LLM-based structured extraction
-     */
-    private extractTableOfContents;
-    /**
-     * Extract the maximum page number from TOC markdown
-     *
-     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
-     * and table cell patterns (e.g., "| title | 175 |") to detect compiled
-     * volume scenarios where TOC page numbers exceed the sub-document's page count.
-     */
-    private extractMaxPageNumber;
-    /**
-     * Process resource captions (for images and tables)
-     *
-     * Common caption processing pipeline:
-     * 1. Parse captions in batch
-     * 2. Validate parsed captions
-     * 3. Reparse failed captions with fallback model
-     *
-     * @param captionTexts - Array of caption texts to process
-     * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
-     * @returns Parsed captions with index mapping
-     */
-    private processResourceCaptions;
-    /**
-     * Extract caption text from resource
-     *
-     * Handles both string references and $ref resolution
-     */
-    private extractCaptionText;
-    /**
-     * Convert images
-     *
-     * Converts pictures from DoclingDocument to ProcessedImage
-     */
-    private convertImages;
-    /**
-     * Convert tables
-     *
-     * Converts tables from DoclingDocument to ProcessedTable
-     */
-    private convertTables;
     /**
      * Convert chapters and link resources
      *
@@ -924,6 +861,18 @@ declare class IdGenerator {
     private padNumber;
 }
 
+/**
+ * Extract the maximum page number from TOC markdown
+ *
+ * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+ * and table cell patterns (e.g., "| title | 175 |") to detect compiled
+ * volume scenarios where TOC page numbers exceed the sub-document's page count.
+ *
+ * @param markdown - TOC markdown content
+ * @returns Maximum page number found, or 0 if none found
+ */
+declare function extractMaxPageNumber(markdown: string): number;
+
 /**
  * TOC keyword patterns for different languages
  * Korean: 목차, 차례, 목 차
@@ -1508,4 +1457,142 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+/**
+ * Dependencies required by CaptionProcessingPipeline
+ */
+interface CaptionProcessingPipelineDeps {
+    logger: LoggerMethods;
+    captionParser: CaptionParser;
+    captionValidator: CaptionValidator;
+    refResolver?: RefResolver;
+    fallbackModel: LanguageModel;
+    enableFallbackRetry: boolean;
+    maxRetries: number;
+    captionParserBatchSize: number;
+    captionValidatorBatchSize: number;
+    usageAggregator: LLMTokenUsageAggregator;
+    abortSignal?: AbortSignal;
+}
+/**
+ * CaptionProcessingPipeline
+ *
+ * Processes resource captions through a multi-step pipeline:
+ * 1. Parse captions in batch (CaptionParser)
+ * 2. Handle length mismatch recovery
+ * 3. Validate parsed captions (CaptionValidator)
+ * 4. Reparse failed captions with fallback model
+ */
+declare class CaptionProcessingPipeline {
+    private readonly logger;
+    private readonly captionParser;
+    private readonly captionValidator;
+    private readonly refResolver?;
+    private readonly fallbackModel;
+    private readonly enableFallbackRetry;
+    private readonly maxRetries;
+    private readonly captionParserBatchSize;
+    private readonly captionValidatorBatchSize;
+    private readonly usageAggregator;
+    private readonly abortSignal?;
+    constructor(deps: CaptionProcessingPipelineDeps);
+    /**
+     * Process resource captions (for images and tables)
+     *
+     * Common caption processing pipeline:
+     * 1. Parse captions in batch
+     * 2. Validate parsed captions
+     * 3. Reparse failed captions with fallback model
+     *
+     * @param captionTexts - Array of caption texts to process
+     * @param resourceType - Type of resource for logging (e.g., 'image', 'table')
+     * @returns Parsed captions with index mapping
+     */
+    processResourceCaptions(captionTexts: Array<string | undefined>, resourceType: string): Promise<Map<number, Caption>>;
+    /**
+     * Extract caption text from resource
+     *
+     * Handles both string references and $ref resolution
+     */
+    extractCaptionText(captions: Array<string | {
+        $ref: string;
+    }> | undefined): string | undefined;
+}
+
+/**
+ * Dependencies required by TocExtractionPipeline
+ */
+interface TocExtractionPipelineDeps {
+    logger: LoggerMethods;
+    tocFinder: TocFinder;
+    tocExtractor: TocExtractor;
+    tocContentValidator: TocContentValidator;
+    visionTocExtractor: VisionTocExtractor;
+    refResolver: RefResolver;
+    usageAggregator: LLMTokenUsageAggregator;
+}
+/**
+ * TocExtractionPipeline
+ *
+ * Orchestrates the multi-stage TOC extraction process:
+ * 1. Rule-based extraction (TocFinder + MarkdownConverter)
+ * 2. Content validation (TocContentValidator)
+ * 3. Vision fallback (VisionTocExtractor)
+ * 4. Structured extraction with validation retry (TocExtractor)
+ * 5. Vision fallback retry when text-based extraction yields 0 entries
+ */
+declare class TocExtractionPipeline {
+    private readonly logger;
+    private readonly tocFinder;
+    private readonly tocExtractor;
+    private readonly tocContentValidator;
+    private readonly visionTocExtractor;
+    private readonly refResolver;
+    private readonly usageAggregator;
+    constructor(deps: TocExtractionPipelineDeps);
+    /**
+     * Execute the TOC extraction pipeline
+     *
+     * @param doclingDoc - Document containing texts, pages, etc.
+     * @param _filteredTexts - Pre-filtered texts (reserved for future use)
+     * @returns Extracted TOC entries
+     * @throws {TocNotFoundError} When TOC cannot be found or extracted
+     */
+    extract(doclingDoc: DoclingDocument, _filteredTexts: string[]): Promise<TocEntry[]>;
+}
+
+/**
+ * ResourceConverter
+ *
+ * Converts DoclingDocument resources (images, tables, footnotes) to processed format.
+ * Uses CaptionProcessingPipeline for caption extraction and validation.
+ */
+declare class ResourceConverter {
+    private readonly logger;
+    private readonly idGenerator;
+    private readonly captionProcessingPipeline;
+    constructor(logger: LoggerMethods, idGenerator: IdGenerator, captionProcessingPipeline: CaptionProcessingPipeline);
+    /**
+     * Convert all resources (images, tables, footnotes)
+     *
+     * Runs image and table conversions in parallel, then footnotes synchronously.
+     */
+    convertAll(doclingDoc: DoclingDocument, outputPath: string): Promise<{
+        images: ProcessedImage[];
+        tables: ProcessedTable[];
+        footnotes: ProcessedFootnote[];
+    }>;
+    /**
+     * Convert images from DoclingDocument to ProcessedImage[]
+     */
+    convertImages(doclingDoc: DoclingDocument, outputPath: string): Promise<ProcessedImage[]>;
+    /**
+     * Convert tables from DoclingDocument to ProcessedTable[]
+     */
+    convertTables(doclingDoc: DoclingDocument): Promise<ProcessedTable[]>;
+    /**
+     * Convert footnotes from DoclingDocument text items
+     */
+    convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
+}
+
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };