@heripo/pdf-parser 0.1.29 → 0.1.31

package/dist/index.d.cts

@@ -1,37 +1,58 @@
 import { LoggerMethods } from '@heripo/logger';
-import { OcrStrategy, TokenUsageReport, ReviewAssistanceProgressEvent } from '@heripo/model';
+import { TokenUsageReport, ReviewAssistanceProgressEvent } from '@heripo/model';
 export { ReviewAssistanceProgressEvent, ReviewAssistanceProgressStatus, ReviewAssistanceProgressSubstage } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { ConversionOptions } from 'docling-sdk';
 import { LLMTokenUsageAggregator } from '@heripo/shared';
 
-interface ReviewAssistanceOptions {
-    /** Enable page-level review assistance. Defaults to false. */
-    enabled?: boolean;
-    /** Minimum final confidence for direct mutation. Defaults to 0.85. */
+type ReviewAssistanceTaskId = 'text_ocr_hanja' | 'text_integrity' | 'text_role_footnote' | 'tables' | 'pictures_captions' | 'layout_bbox_order';
+
+interface PDFCorrectionModelOptions {
+    textCorrection: LanguageModel;
+    pageGate: LanguageModel;
+    reviewAssistance: LanguageModel;
+    reviewAssistanceTasks?: Partial<Record<ReviewAssistanceTaskId, LanguageModel>>;
+    tableCorrection?: LanguageModel;
+}
+interface PDFCorrectionConcurrencyOptions {
+    pages?: number;
+    reviewTasks?: number;
+    tables?: number;
+}
+interface PDFCorrectionMaxRetriesOptions {
+    textCorrection?: number;
+    pageGate?: number;
+    reviewAssistance?: number;
+    tableCorrection?: number;
+}
+interface PDFCorrectionPageGateOptions {
+    structuralNoiseThreshold?: number;
+}
+interface PDFCorrectionOptions {
+    models: PDFCorrectionModelOptions;
+    concurrency?: PDFCorrectionConcurrencyOptions;
+    maxRetries?: PDFCorrectionMaxRetriesOptions;
+    localModelConcurrency?: number;
+    workItemTimeoutMs?: number;
+    outputLanguage?: string;
+    pageGate?: PDFCorrectionPageGateOptions;
     autoApplyThreshold?: number;
-    /** Minimum final confidence for sidecar proposals. Defaults to 0.5. */
     proposalThreshold?: number;
-    /** Maximum retries per page-level VLM call. Defaults to 3. */
-    maxRetries?: number;
-    /** VLM generation temperature. Defaults to 0. */
     temperature?: number;
-    /** Language for human-readable review reasons. Defaults to English. */
-    outputLanguage?: string;
 }
-interface NormalizedReviewAssistanceOptions {
-    enabled: boolean;
-    concurrency: number;
+interface NormalizedPDFCorrectionOptions {
+    models: PDFCorrectionModelOptions;
+    concurrency: Required<PDFCorrectionConcurrencyOptions>;
+    maxRetries: Required<PDFCorrectionMaxRetriesOptions>;
+    localModelConcurrency: number;
+    workItemTimeoutMs: number;
+    outputLanguage: string;
+    pageGate: Required<PDFCorrectionPageGateOptions>;
     autoApplyThreshold: number;
     proposalThreshold: number;
-    maxRetries: number;
     temperature: number;
-    outputLanguage: string;
 }
-type ReviewAssistanceOptionInput = boolean | ReviewAssistanceOptions | undefined;
-declare const REVIEW_ASSISTANCE_DEFAULTS: NormalizedReviewAssistanceOptions;
-declare function normalizeReviewAssistanceOptions(value: ReviewAssistanceOptionInput): NormalizedReviewAssistanceOptions;
-declare function isReviewAssistanceEnabled(value: ReviewAssistanceOptionInput): boolean;
+declare const PDF_CORRECTION_DEFAULTS: Omit<NormalizedPDFCorrectionOptions, 'models'>;
 
 /**
  * Callback function invoked after PDF conversion completes
@@ -42,30 +63,16 @@ type ConversionCompleteCallback = (outputPath: string) => Promise<void> | void;
  * Extended options for PDF conversion.
  */
 type PDFConvertOptions = Omit<ConversionOptions, 'to_formats' | 'image_export_mode' | 'ocr_engine' | 'accelerator_options' | 'ocr_options' | 'generate_picture_images' | 'generate_page_images' | 'images_scale' | 'force_ocr' | 'pipeline' | 'vlm_pipeline_model_local' | 'vlm_pipeline_model_api'> & {
+    correction: PDFCorrectionOptions;
     num_threads?: number;
     /**
      * Force pre-conversion to image-based PDF before processing.
      * Requires ImageMagick and Ghostscript.
      */
     forceImagePdf?: boolean;
-    /** Vision model for OCR strategy sampling (enables new strategy-based flow) */
-    strategySamplerModel?: LanguageModel;
-    /** Vision model for VLM page processing (required when strategy selects VLM) */
-    vlmProcessorModel?: LanguageModel;
-    /** Concurrency for VLM page processing (default: 1) */
-    vlmConcurrency?: number;
-    /**
-     * Optional page-level review assistance.
-     * Boolean true enables defaults; object form still requires enabled: true.
-     */
-    reviewAssistance?: boolean | ReviewAssistanceOptions;
-    /** Skip sampling and default to ocrmac */
-    skipSampling?: boolean;
-    /** Force a specific OCR method, bypassing sampling */
-    forcedMethod?: 'ocrmac' | 'vlm';
-    /** Token usage aggregator for tracking across sampling and VLM processing */
+    /** Token usage aggregator for tracking post-Docling correction calls */
     aggregator?: LLMTokenUsageAggregator;
-    /** Callback fired after each batch of VLM pages completes, with cumulative token usage */
+    /** Callback fired after correction work completes, with cumulative token usage */
     onTokenUsage?: (report: TokenUsageReport) => void;
     /** Callback fired when Review Assistance advances through parser substages */
     onReviewAssistanceProgress?: (event: ReviewAssistanceProgressEvent) => void;
@@ -77,16 +84,11 @@ type PDFConvertOptions = Omit<ConversionOptions, 'to_formats' | 'image_export_mo
     chunkSize?: number;
     /** Max retry attempts per failed chunk (default: CHUNKED_CONVERSION.DEFAULT_MAX_RETRIES) */
     chunkMaxRetries?: number;
+    /** Vision model for document language detection when the PDF text layer is insufficient */
+    languageDetectionModel?: LanguageModel;
     /** LLM model for document type validation (opt-in: skipped when not set) */
     documentValidationModel?: LanguageModel;
 };
-/** Result of strategy-based conversion */
-interface ConvertWithStrategyResult {
-    /** The OCR strategy that was determined */
-    strategy: OcrStrategy;
-    /** Token usage report from sampling and/or VLM processing (null when no LLM usage occurs) */
-    tokenUsageReport: TokenUsageReport | null;
-}
 
 type Options = {
     logger: LoggerMethods;
@@ -334,4 +336,4 @@ declare class VlmResponseValidator {
     private static detectRepetitivePattern;
 }
 
-export { type ConversionCompleteCallback, type ConvertWithStrategyResult, ImagePdfFallbackError, InvalidDocumentTypeError, type NormalizedReviewAssistanceOptions, type PDFConvertOptions, PDFParser, REVIEW_ASSISTANCE_DEFAULTS, type ReviewAssistanceOptions, type VlmPageQuality, type VlmQualityIssue, type VlmQualityIssueType, VlmResponseValidator, type VlmValidationResult, isReviewAssistanceEnabled, normalizeReviewAssistanceOptions };
+export { type ConversionCompleteCallback, ImagePdfFallbackError, InvalidDocumentTypeError, type NormalizedPDFCorrectionOptions, type PDFConvertOptions, type PDFCorrectionConcurrencyOptions, type PDFCorrectionMaxRetriesOptions, type PDFCorrectionModelOptions, type PDFCorrectionOptions, type PDFCorrectionPageGateOptions, PDFParser, PDF_CORRECTION_DEFAULTS, type ReviewAssistanceTaskId, type VlmPageQuality, type VlmQualityIssue, type VlmQualityIssueType, VlmResponseValidator, type VlmValidationResult };

package/dist/index.d.ts

@@ -1,37 +1,58 @@
 import { LoggerMethods } from '@heripo/logger';
-import { OcrStrategy, TokenUsageReport, ReviewAssistanceProgressEvent } from '@heripo/model';
+import { TokenUsageReport, ReviewAssistanceProgressEvent } from '@heripo/model';
 export { ReviewAssistanceProgressEvent, ReviewAssistanceProgressStatus, ReviewAssistanceProgressSubstage } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { ConversionOptions } from 'docling-sdk';
 import { LLMTokenUsageAggregator } from '@heripo/shared';
 
-interface ReviewAssistanceOptions {
-    /** Enable page-level review assistance. Defaults to false. */
-    enabled?: boolean;
-    /** Minimum final confidence for direct mutation. Defaults to 0.85. */
+type ReviewAssistanceTaskId = 'text_ocr_hanja' | 'text_integrity' | 'text_role_footnote' | 'tables' | 'pictures_captions' | 'layout_bbox_order';
+
+interface PDFCorrectionModelOptions {
+    textCorrection: LanguageModel;
+    pageGate: LanguageModel;
+    reviewAssistance: LanguageModel;
+    reviewAssistanceTasks?: Partial<Record<ReviewAssistanceTaskId, LanguageModel>>;
+    tableCorrection?: LanguageModel;
+}
+interface PDFCorrectionConcurrencyOptions {
+    pages?: number;
+    reviewTasks?: number;
+    tables?: number;
+}
+interface PDFCorrectionMaxRetriesOptions {
+    textCorrection?: number;
+    pageGate?: number;
+    reviewAssistance?: number;
+    tableCorrection?: number;
+}
+interface PDFCorrectionPageGateOptions {
+    structuralNoiseThreshold?: number;
+}
+interface PDFCorrectionOptions {
+    models: PDFCorrectionModelOptions;
+    concurrency?: PDFCorrectionConcurrencyOptions;
+    maxRetries?: PDFCorrectionMaxRetriesOptions;
+    localModelConcurrency?: number;
+    workItemTimeoutMs?: number;
+    outputLanguage?: string;
+    pageGate?: PDFCorrectionPageGateOptions;
     autoApplyThreshold?: number;
-    /** Minimum final confidence for sidecar proposals. Defaults to 0.5. */
     proposalThreshold?: number;
-    /** Maximum retries per page-level VLM call. Defaults to 3. */
-    maxRetries?: number;
-    /** VLM generation temperature. Defaults to 0. */
     temperature?: number;
-    /** Language for human-readable review reasons. Defaults to English. */
-    outputLanguage?: string;
 }
-interface NormalizedReviewAssistanceOptions {
-    enabled: boolean;
-    concurrency: number;
+interface NormalizedPDFCorrectionOptions {
+    models: PDFCorrectionModelOptions;
+    concurrency: Required<PDFCorrectionConcurrencyOptions>;
+    maxRetries: Required<PDFCorrectionMaxRetriesOptions>;
+    localModelConcurrency: number;
+    workItemTimeoutMs: number;
+    outputLanguage: string;
+    pageGate: Required<PDFCorrectionPageGateOptions>;
     autoApplyThreshold: number;
     proposalThreshold: number;
-    maxRetries: number;
     temperature: number;
-    outputLanguage: string;
 }
-type ReviewAssistanceOptionInput = boolean | ReviewAssistanceOptions | undefined;
-declare const REVIEW_ASSISTANCE_DEFAULTS: NormalizedReviewAssistanceOptions;
-declare function normalizeReviewAssistanceOptions(value: ReviewAssistanceOptionInput): NormalizedReviewAssistanceOptions;
-declare function isReviewAssistanceEnabled(value: ReviewAssistanceOptionInput): boolean;
+declare const PDF_CORRECTION_DEFAULTS: Omit<NormalizedPDFCorrectionOptions, 'models'>;
 
 /**
  * Callback function invoked after PDF conversion completes
@@ -42,30 +63,16 @@ type ConversionCompleteCallback = (outputPath: string) => Promise<void> | void;
  * Extended options for PDF conversion.
  */
 type PDFConvertOptions = Omit<ConversionOptions, 'to_formats' | 'image_export_mode' | 'ocr_engine' | 'accelerator_options' | 'ocr_options' | 'generate_picture_images' | 'generate_page_images' | 'images_scale' | 'force_ocr' | 'pipeline' | 'vlm_pipeline_model_local' | 'vlm_pipeline_model_api'> & {
+    correction: PDFCorrectionOptions;
     num_threads?: number;
     /**
      * Force pre-conversion to image-based PDF before processing.
      * Requires ImageMagick and Ghostscript.
      */
     forceImagePdf?: boolean;
-    /** Vision model for OCR strategy sampling (enables new strategy-based flow) */
-    strategySamplerModel?: LanguageModel;
-    /** Vision model for VLM page processing (required when strategy selects VLM) */
-    vlmProcessorModel?: LanguageModel;
-    /** Concurrency for VLM page processing (default: 1) */
-    vlmConcurrency?: number;
-    /**
-     * Optional page-level review assistance.
-     * Boolean true enables defaults; object form still requires enabled: true.
-     */
-    reviewAssistance?: boolean | ReviewAssistanceOptions;
-    /** Skip sampling and default to ocrmac */
-    skipSampling?: boolean;
-    /** Force a specific OCR method, bypassing sampling */
-    forcedMethod?: 'ocrmac' | 'vlm';
-    /** Token usage aggregator for tracking across sampling and VLM processing */
+    /** Token usage aggregator for tracking post-Docling correction calls */
     aggregator?: LLMTokenUsageAggregator;
-    /** Callback fired after each batch of VLM pages completes, with cumulative token usage */
+    /** Callback fired after correction work completes, with cumulative token usage */
     onTokenUsage?: (report: TokenUsageReport) => void;
     /** Callback fired when Review Assistance advances through parser substages */
     onReviewAssistanceProgress?: (event: ReviewAssistanceProgressEvent) => void;
@@ -77,16 +84,11 @@ type PDFConvertOptions = Omit<ConversionOptions, 'to_formats' | 'image_export_mo
     chunkSize?: number;
     /** Max retry attempts per failed chunk (default: CHUNKED_CONVERSION.DEFAULT_MAX_RETRIES) */
     chunkMaxRetries?: number;
+    /** Vision model for document language detection when the PDF text layer is insufficient */
+    languageDetectionModel?: LanguageModel;
     /** LLM model for document type validation (opt-in: skipped when not set) */
     documentValidationModel?: LanguageModel;
 };
-/** Result of strategy-based conversion */
-interface ConvertWithStrategyResult {
-    /** The OCR strategy that was determined */
-    strategy: OcrStrategy;
-    /** Token usage report from sampling and/or VLM processing (null when no LLM usage occurs) */
-    tokenUsageReport: TokenUsageReport | null;
-}
 
 type Options = {
     logger: LoggerMethods;
@@ -334,4 +336,4 @@ declare class VlmResponseValidator {
     private static detectRepetitivePattern;
 }
 
-export { type ConversionCompleteCallback, type ConvertWithStrategyResult, ImagePdfFallbackError, InvalidDocumentTypeError, type NormalizedReviewAssistanceOptions, type PDFConvertOptions, PDFParser, REVIEW_ASSISTANCE_DEFAULTS, type ReviewAssistanceOptions, type VlmPageQuality, type VlmQualityIssue, type VlmQualityIssueType, VlmResponseValidator, type VlmValidationResult, isReviewAssistanceEnabled, normalizeReviewAssistanceOptions };
+export { type ConversionCompleteCallback, ImagePdfFallbackError, InvalidDocumentTypeError, type NormalizedPDFCorrectionOptions, type PDFConvertOptions, type PDFCorrectionConcurrencyOptions, type PDFCorrectionMaxRetriesOptions, type PDFCorrectionModelOptions, type PDFCorrectionOptions, type PDFCorrectionPageGateOptions, PDFParser, PDF_CORRECTION_DEFAULTS, type ReviewAssistanceTaskId, type VlmPageQuality, type VlmQualityIssue, type VlmQualityIssueType, VlmResponseValidator, type VlmValidationResult };