@heripo/document-processor 0.1.6 → 0.1.8

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { DoclingDocument, HanjaAssessment, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { TokenUsageReport, DoclingDocument, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -14,7 +14,7 @@ interface DocumentProcessorOptions {
     logger: LoggerMethods;
     /**
      * Fallback model - used as fallback when component-specific models are not provided or fail.
-     * This is the only required model. Should be set to a frontier model (e.g., Claude Opus 4.5, GPT-5.2)
+     * This is the only required model. Should be set to a frontier model (e.g., Claude Opus 4.6, GPT-5.2)
      * to ensure reliable fallback performance across all components.
      */
     fallbackModel: LanguageModel;
@@ -43,11 +43,6 @@ interface DocumentProcessorOptions {
      * Falls back to 'fallbackModel' if not provided.
      */
     captionParserModel?: LanguageModel;
-    /**
-     * Model for HanjaQualitySampler - evaluates KCJ character quality in OCR output.
-     * Requires vision capabilities. Falls back to 'fallbackModel' if not provided.
-     */
-    hanjaQualitySamplerModel?: LanguageModel;
     /**
      * Batch size for TextCleaner text normalization (synchronous processing)
      */
@@ -74,6 +69,12 @@ interface DocumentProcessorOptions {
      * When aborted, processing stops at the next checkpoint between stages.
      */
     abortSignal?: AbortSignal;
+    /**
+     * Callback fired after each major processing phase completes.
+     * Receives the current cumulative token usage report.
+     * Useful for real-time token usage monitoring during processing.
+     */
+    onTokenUsage?: (report: TokenUsageReport) => void;
 }
 /**
  * DocumentProcessor
@@ -137,13 +138,13 @@ declare class DocumentProcessor {
     private readonly validatorModel;
     private readonly visionTocExtractorModel;
     private readonly captionParserModel;
-    private readonly hanjaQualitySamplerModel;
     private readonly textCleanerBatchSize;
     private readonly captionParserBatchSize;
     private readonly captionValidatorBatchSize;
     private readonly maxRetries;
     private readonly enableFallbackRetry;
     private readonly abortSignal?;
+    private readonly onTokenUsage?;
     private idGenerator;
     private refResolver?;
     private pageRangeParser?;
@@ -158,23 +159,18 @@ declare class DocumentProcessor {
     private readonly usageAggregator;
     constructor(options: DocumentProcessorOptions);
     /**
-     * Check if abort has been requested and throw error if so
+     * Emit current token usage report via callback
      *
-     * @throws {Error} with name 'AbortError' if aborted
+     * Calls the onTokenUsage callback with the current cumulative report
+     * from the usage aggregator. Safe to call even if no callback is set.
      */
-    private checkAborted;
+    private emitTokenUsage;
     /**
-     * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
-     *
-     * Samples pages containing KCJ characters and uses Vision LLM to compare
-     * the OCR text against the original page images. Returns an assessment
-     * indicating whether the document should be re-parsed with VLM pipeline.
+     * Check if abort has been requested and throw error if so
      *
-     * @param doclingDoc - Original document extracted from Docling SDK
-     * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
-     * @returns Assessment result with severity and recommendation
+     * @throws {Error} with name 'AbortError' if aborted
      */
-    assessHanjaQuality(doclingDoc: DoclingDocument, outputPath: string): Promise<HanjaAssessment>;
+    private checkAborted;
     /**
      * Converts DoclingDocument to ProcessedDocument with token usage tracking.
      *
@@ -250,6 +246,14 @@ declare class DocumentProcessor {
      * 5. TocExtractor - LLM-based structured extraction
      */
     private extractTableOfContents;
+    /**
+     * Extract the maximum page number from TOC markdown
+     *
+     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+     * and table cell patterns (e.g., "| title | 175 |") to detect compiled
+     * volume scenarios where TOC page numbers exceed the sub-document's page count.
+     */
+    private extractMaxPageNumber;
     /**
      * Process resource captions (for images and tables)
      *
@@ -992,6 +996,11 @@ declare class TocFinder {
      * Find the TOC container (group or table) from a parent reference
      */
     private findTocContainer;
+    /**
+     * Find adjacent TOC item among body siblings following the keyword text.
+     * Used when findTocContainer returns null (e.g., parent is #/body).
+     */
+    private findSiblingTocItem;
     /**
      * Check if a group contains TOC-like structure
      */
@@ -1000,6 +1009,11 @@ declare class TocFinder {
      * Check if a table contains TOC-like structure
      */
     private isTableTocLike;
+    /**
+     * Check if a table is a resource index (drawing/photo/figure list)
+     * rather than a main TOC table
+     */
+    private isResourceIndexTable;
     /**
      * Expand TOC area to consecutive pages (both backward and forward)
      */
@@ -1032,7 +1046,7 @@ declare class TocFinder {
 }
 
 /**
- * Zod schema for recursive TocEntry structure
+ * Zod schema for TocEntry structure with fixed 4-level depth
  */
 declare const TocEntrySchema: z.ZodType<TocEntry>;
 /**
@@ -1500,64 +1514,4 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-/**
- * HanjaQualitySampler
- *
- * Evaluates the quality of KCJ (Chinese/Japanese/Korean) character recognition
- * in OCR-processed documents. Samples a subset of pages containing KCJ text and
- * uses Vision LLM to compare the OCR output against the original page images.
- *
- * This is used to determine whether a document should be re-parsed using the
- * VLM pipeline for better KCJ character accuracy.
- */
-declare class HanjaQualitySampler extends VisionLLMComponent {
-    constructor(logger: LoggerMethods, model: LanguageModel, outputPath: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
-    /**
-     * Assess the quality of KCJ character recognition in the document
-     *
-     * @param doclingDoc - DoclingDocument to assess
-     * @returns Assessment result indicating whether VLM re-parse is needed
-     */
-    assess(doclingDoc: DoclingDocument): Promise<HanjaAssessment>;
-    /**
-     * Get the eligible page range after trimming front/back edges.
-     * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
-     */
-    private getEligiblePageRange;
-    /**
-     * Get page numbers that are "image-only" (contain pictures but minimal text).
-     * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
-     */
-    private getImageOnlyPages;
-    /**
-     * Get all pages with substantial text content.
-     * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
-     */
-    private getTextPages;
-    /**
-     * Select sample pages with highest text density.
-     * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
-     * Falls back to all text pages if filtering leaves too few candidates.
-     */
-    private selectSamplePages;
-    /**
-     * Evaluate sampled pages using Vision LLM
-     */
-    private evaluatePages;
-    /**
-     * Evaluate a single page by comparing OCR text against the page image
-     */
-    private evaluateSinglePage;
-    /**
-     * Aggregate individual page results into final assessment
-     */
-    private aggregateResults;
-    /**
-     * Get page number from a text item
-     */
-    private getPageNo;
-    protected buildSystemPrompt(): string;
-    protected buildUserPrompt(ocrText: string): string;
-}
-
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, HanjaQualitySampler, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { DoclingDocument, HanjaAssessment, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { TokenUsageReport, DoclingDocument, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -14,7 +14,7 @@ interface DocumentProcessorOptions {
     logger: LoggerMethods;
     /**
      * Fallback model - used as fallback when component-specific models are not provided or fail.
-     * This is the only required model. Should be set to a frontier model (e.g., Claude Opus 4.5, GPT-5.2)
+     * This is the only required model. Should be set to a frontier model (e.g., Claude Opus 4.6, GPT-5.2)
      * to ensure reliable fallback performance across all components.
      */
     fallbackModel: LanguageModel;
@@ -43,11 +43,6 @@ interface DocumentProcessorOptions {
      * Falls back to 'fallbackModel' if not provided.
      */
     captionParserModel?: LanguageModel;
-    /**
-     * Model for HanjaQualitySampler - evaluates KCJ character quality in OCR output.
-     * Requires vision capabilities. Falls back to 'fallbackModel' if not provided.
-     */
-    hanjaQualitySamplerModel?: LanguageModel;
     /**
      * Batch size for TextCleaner text normalization (synchronous processing)
      */
@@ -74,6 +69,12 @@ interface DocumentProcessorOptions {
      * When aborted, processing stops at the next checkpoint between stages.
      */
     abortSignal?: AbortSignal;
+    /**
+     * Callback fired after each major processing phase completes.
+     * Receives the current cumulative token usage report.
+     * Useful for real-time token usage monitoring during processing.
+     */
+    onTokenUsage?: (report: TokenUsageReport) => void;
 }
 /**
  * DocumentProcessor
@@ -137,13 +138,13 @@ declare class DocumentProcessor {
     private readonly validatorModel;
     private readonly visionTocExtractorModel;
     private readonly captionParserModel;
-    private readonly hanjaQualitySamplerModel;
     private readonly textCleanerBatchSize;
     private readonly captionParserBatchSize;
     private readonly captionValidatorBatchSize;
     private readonly maxRetries;
     private readonly enableFallbackRetry;
     private readonly abortSignal?;
+    private readonly onTokenUsage?;
     private idGenerator;
     private refResolver?;
     private pageRangeParser?;
@@ -158,23 +159,18 @@ declare class DocumentProcessor {
     private readonly usageAggregator;
     constructor(options: DocumentProcessorOptions);
     /**
-     * Check if abort has been requested and throw error if so
+     * Emit current token usage report via callback
      *
-     * @throws {Error} with name 'AbortError' if aborted
+     * Calls the onTokenUsage callback with the current cumulative report
+     * from the usage aggregator. Safe to call even if no callback is set.
      */
-    private checkAborted;
+    private emitTokenUsage;
     /**
-     * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
-     *
-     * Samples pages containing KCJ characters and uses Vision LLM to compare
-     * the OCR text against the original page images. Returns an assessment
-     * indicating whether the document should be re-parsed with VLM pipeline.
+     * Check if abort has been requested and throw error if so
      *
-     * @param doclingDoc - Original document extracted from Docling SDK
-     * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
-     * @returns Assessment result with severity and recommendation
+     * @throws {Error} with name 'AbortError' if aborted
      */
-    assessHanjaQuality(doclingDoc: DoclingDocument, outputPath: string): Promise<HanjaAssessment>;
+    private checkAborted;
     /**
      * Converts DoclingDocument to ProcessedDocument with token usage tracking.
      *
@@ -250,6 +246,14 @@ declare class DocumentProcessor {
      * 5. TocExtractor - LLM-based structured extraction
      */
     private extractTableOfContents;
+    /**
+     * Extract the maximum page number from TOC markdown
+     *
+     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+     * and table cell patterns (e.g., "| title | 175 |") to detect compiled
+     * volume scenarios where TOC page numbers exceed the sub-document's page count.
+     */
+    private extractMaxPageNumber;
     /**
      * Process resource captions (for images and tables)
      *
@@ -992,6 +996,11 @@ declare class TocFinder {
      * Find the TOC container (group or table) from a parent reference
      */
     private findTocContainer;
+    /**
+     * Find adjacent TOC item among body siblings following the keyword text.
+     * Used when findTocContainer returns null (e.g., parent is #/body).
+     */
+    private findSiblingTocItem;
     /**
      * Check if a group contains TOC-like structure
      */
@@ -1000,6 +1009,11 @@ declare class TocFinder {
      * Check if a table contains TOC-like structure
      */
     private isTableTocLike;
+    /**
+     * Check if a table is a resource index (drawing/photo/figure list)
+     * rather than a main TOC table
+     */
+    private isResourceIndexTable;
     /**
      * Expand TOC area to consecutive pages (both backward and forward)
      */
@@ -1032,7 +1046,7 @@ declare class TocFinder {
 }
 
 /**
- * Zod schema for recursive TocEntry structure
+ * Zod schema for TocEntry structure with fixed 4-level depth
  */
 declare const TocEntrySchema: z.ZodType<TocEntry>;
 /**
@@ -1500,64 +1514,4 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-/**
- * HanjaQualitySampler
- *
- * Evaluates the quality of KCJ (Chinese/Japanese/Korean) character recognition
- * in OCR-processed documents. Samples a subset of pages containing KCJ text and
- * uses Vision LLM to compare the OCR output against the original page images.
- *
- * This is used to determine whether a document should be re-parsed using the
- * VLM pipeline for better KCJ character accuracy.
- */
-declare class HanjaQualitySampler extends VisionLLMComponent {
-    constructor(logger: LoggerMethods, model: LanguageModel, outputPath: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
-    /**
-     * Assess the quality of KCJ character recognition in the document
-     *
-     * @param doclingDoc - DoclingDocument to assess
-     * @returns Assessment result indicating whether VLM re-parse is needed
-     */
-    assess(doclingDoc: DoclingDocument): Promise<HanjaAssessment>;
-    /**
-     * Get the eligible page range after trimming front/back edges.
-     * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
-     */
-    private getEligiblePageRange;
-    /**
-     * Get page numbers that are "image-only" (contain pictures but minimal text).
-     * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
-     */
-    private getImageOnlyPages;
-    /**
-     * Get all pages with substantial text content.
-     * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
-     */
-    private getTextPages;
-    /**
-     * Select sample pages with highest text density.
-     * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
-     * Falls back to all text pages if filtering leaves too few candidates.
-     */
-    private selectSamplePages;
-    /**
-     * Evaluate sampled pages using Vision LLM
-     */
-    private evaluatePages;
-    /**
-     * Evaluate a single page by comparing OCR text against the page image
-     */
-    private evaluateSinglePage;
-    /**
-     * Aggregate individual page results into final assessment
-     */
-    private aggregateResults;
-    /**
-     * Get page number from a text item
-     */
-    private getPageNo;
-    protected buildSystemPrompt(): string;
-    protected buildUserPrompt(ocrText: string): string;
-}
-
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, HanjaQualitySampler, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };