@heripo/pdf-parser 0.1.38 → 0.1.40

package/dist/index.d.cts

@@ -1,365 +0,0 @@
-import { LoggerMethods } from '@heripo/logger';
-import { TokenUsageReport, ReviewAssistanceProgressEvent } from '@heripo/model';
-export { ReviewAssistanceProgressEvent, ReviewAssistanceProgressStatus, ReviewAssistanceProgressSubstage } from '@heripo/model';
-import { LanguageModel } from 'ai';
-import { ConversionOptions } from 'docling-sdk';
-import { LLMTokenUsageAggregator } from '@heripo/shared';
-
-type ReviewAssistanceTaskId = 'text_ocr_hanja' | 'text_integrity' | 'text_role_footnote' | 'tables' | 'pictures_captions' | 'layout_bbox_order';
-
-interface PDFCorrectionModelOptions {
-    textCorrection: LanguageModel;
-    /** textCorrection 실패(예: lmstudio 400) 시 LLMCaller fallback. */
-    textCorrectionFallback?: LanguageModel;
-    pageGate: LanguageModel;
-    /** pageGate(page-eligibility) 실패 시 fallback. */
-    pageGateFallback?: LanguageModel;
-    reviewAssistance: LanguageModel;
-    /** review-assistance 공통 fallback (task별 미지정 시 사용). */
-    reviewAssistanceFallback?: LanguageModel;
-    reviewAssistanceTasks?: Partial<Record<ReviewAssistanceTaskId, LanguageModel>>;
-    /** review-assistance task별 fallback. */
-    reviewAssistanceTasksFallback?: Partial<Record<ReviewAssistanceTaskId, LanguageModel>>;
-    tableCorrection?: LanguageModel;
-    /** tableCorrection 실패 시 fallback. */
-    tableCorrectionFallback?: LanguageModel;
-}
-interface PDFCorrectionConcurrencyOptions {
-    pages?: number;
-    reviewTasks?: number;
-    tables?: number;
-}
-interface PDFCorrectionMaxRetriesOptions {
-    textCorrection?: number;
-    pageGate?: number;
-    reviewAssistance?: number;
-    tableCorrection?: number;
-}
-interface PDFCorrectionPageGateOptions {
-    structuralNoiseThreshold?: number;
-}
-interface PDFCorrectionOptions {
-    models: PDFCorrectionModelOptions;
-    concurrency?: PDFCorrectionConcurrencyOptions;
-    maxRetries?: PDFCorrectionMaxRetriesOptions;
-    modelConcurrency?: number;
-    workItemTimeoutMs?: number;
-    outputLanguage?: string;
-    pageGate?: PDFCorrectionPageGateOptions;
-    autoApplyThreshold?: number;
-    proposalThreshold?: number;
-    /**
-     * When true, every valid review-assistance command auto-applies regardless of
-     * confidence threshold or structural block reason (no manual-review routing).
-     * Enabled by the engine demo; defaults to false everywhere else.
-     */
-    forceAutoApply?: boolean;
-    /**
-     * When false, the review-assistance stage (the per-page eligibility gate plus
-     * the structural review-assistance runner) is skipped entirely and only the
-     * text-correction stage runs. Defaults to true.
-     */
-    reviewAssistanceEnabled?: boolean;
-    temperature?: number;
-}
-interface NormalizedPDFCorrectionOptions {
-    models: PDFCorrectionModelOptions;
-    concurrency: Required<PDFCorrectionConcurrencyOptions>;
-    maxRetries: Required<PDFCorrectionMaxRetriesOptions>;
-    modelConcurrency: number;
-    workItemTimeoutMs: number;
-    outputLanguage: string;
-    pageGate: Required<PDFCorrectionPageGateOptions>;
-    autoApplyThreshold: number;
-    proposalThreshold: number;
-    forceAutoApply: boolean;
-    reviewAssistanceEnabled: boolean;
-    temperature: number;
-}
-declare const PDF_CORRECTION_DEFAULTS: Omit<NormalizedPDFCorrectionOptions, 'models'>;
-
-/**
- * Callback function invoked after PDF conversion completes
- * @param outputPath Absolute path to the output directory containing result files
- */
-type ConversionCompleteCallback = (outputPath: string) => Promise<void> | void;
-/**
- * Extended options for PDF conversion.
- */
-type PDFConvertOptions = Omit<ConversionOptions, 'to_formats' | 'image_export_mode' | 'ocr_engine' | 'accelerator_options' | 'ocr_options' | 'generate_picture_images' | 'generate_page_images' | 'images_scale' | 'force_ocr' | 'pipeline' | 'vlm_pipeline_model_local' | 'vlm_pipeline_model_api'> & {
-    correction: PDFCorrectionOptions;
-    num_threads?: number;
-    /**
-     * Force pre-conversion to image-based PDF before processing.
-     * Requires ImageMagick and Ghostscript.
-     */
-    forceImagePdf?: boolean;
-    /** Token usage aggregator for tracking post-Docling correction calls */
-    aggregator?: LLMTokenUsageAggregator;
-    /** Callback fired after correction work completes, with cumulative token usage */
-    onTokenUsage?: (report: TokenUsageReport) => void;
-    /** Callback fired when Review Assistance advances through parser substages */
-    onReviewAssistanceProgress?: (event: ReviewAssistanceProgressEvent) => void;
-    /** Document processing timeout in seconds for the Docling server (default: server default) */
-    document_timeout?: number;
-    /** Enable chunked conversion for large PDFs (local files only) */
-    chunkedConversion?: boolean;
-    /** Pages per chunk (default: CHUNKED_CONVERSION.DEFAULT_CHUNK_SIZE) */
-    chunkSize?: number;
-    /** Max retry attempts per failed chunk (default: CHUNKED_CONVERSION.DEFAULT_MAX_RETRIES) */
-    chunkMaxRetries?: number;
-    /** Vision model for document language detection when the PDF text layer is insufficient */
-    languageDetectionModel?: LanguageModel;
-    /** languageDetectionModel 실패 시 fallback. */
-    languageDetectionFallbackModel?: LanguageModel;
-    /** LLM model for document type validation (opt-in: skipped when not set) */
-    documentValidationModel?: LanguageModel;
-};
-
-type Options = {
-    logger: LoggerMethods;
-    timeout?: number;
-    venvPath?: string;
-    killExistingProcess?: boolean;
-    /**
-     * Enable fallback to image-based PDF when conversion fails.
-     * Only works with local server mode (port option).
-     * Requires ImageMagick and Ghostscript to be installed.
-     */
-    enableImagePdfFallback?: boolean;
-} & ({
-    port?: number;
-} | {
-    baseUrl: string;
-});
-/**
- * PDFParser - A PDF document parser using docling-serve
- *
- * ## System Requirements
- * Before using PDFParser, ensure your system meets these requirements:
- *
- * ### Operating System
- * - macOS 10.15 (Catalina) or later
- *
- * ### Required Software
- * - `python3` (version 3.9 - 3.12)
- *   - Python 3.13+ is NOT compatible with docling-serve
- *   - Recommended: Python 3.11 or 3.12
- *   - Install specific version: `pyenv install 3.12.0 && pyenv global 3.12.0`
- * - `jq` - JSON processor
- *   - Install: `brew install jq`
- * - `poppler` - PDF text extraction tools (pdftotext, pdfinfo)
- *   - Install: `brew install poppler`
- * - `lsof` - List open files (usually pre-installed on macOS)
- *
- * ## Initialization Process
- * When `init()` is called, the following setup occurs automatically:
- *
- * ### If using external server (baseUrl provided):
- * 1. Connects to the provided baseUrl
- * 2. Waits for server health check (up to 120 seconds)
- *
- * ### If using local server (default):
- * 1. **Python Environment Setup**
- *    - Verifies Python version compatibility (3.9-3.12)
- *    - Creates Python virtual environment at `venvPath` (default: `.venv`)
- *    - Verifies virtual environment Python version
- *
- * 2. **Dependency Installation**
- *    - Upgrades pip to latest version
- *    - Installs setuptools and wheel
- *    - Installs pyarrow (binary-only to avoid compilation)
- *    - Installs docling-serve package
- *
- * 3. **Server Management**
- *    - Checks if specified port is already in use
- *    - If `killExistingProcess` is true, kills any process using the port
- *    - If port is in use and `killExistingProcess` is false, reuses existing server
- *    - Otherwise, starts new docling-serve instance on specified port
- *    - Waits for server to become ready (health check, up to 120 seconds)
- *
- * ## Notes
- * - First initialization may take several minutes due to Python package downloads
- * - Subsequent initializations are faster if virtual environment already exists
- * - The virtual environment and packages are reused across sessions
- * - Server process runs in background and needs to be managed separately if needed
- */
-declare class PDFParser {
-    private readonly logger;
-    private readonly port?;
-    private readonly baseUrl?;
-    private readonly timeout;
-    private readonly venvPath;
-    private readonly killExistingProcess;
-    private readonly enableImagePdfFallback;
-    private client;
-    private environment?;
-    private recoveryPromise;
-    constructor(options: Options);
-    init(): Promise<void>;
-    /**
-     * Check if an error is a connection refused error (ECONNREFUSED).
-     * This typically indicates the Docling server has crashed.
-     */
-    private isConnectionRefusedError;
-    private canRecoverLocalServer;
-    /**
-     * Restart the Docling server after it has crashed.
-     * This kills any existing process on the port, starts a new server,
-     * and waits for it to become ready.
-     *
-     * Note: This method is only called when canRecover is true,
-     * which guarantees this.port is defined.
-     */
-    private restartServer;
-    private recoverServer;
-    isReady(): Promise<boolean>;
-    ensureReady(): Promise<void>;
-    private waitForServerReady;
-    parse(url: string, reportId: string, onComplete: ConversionCompleteCallback, cleanupAfterCallback: boolean, options: PDFConvertOptions, abortSignal?: AbortSignal): Promise<TokenUsageReport | null>;
-    /**
-     * Execute a function with automatic server recovery on ECONNREFUSED errors.
-     * Recovery is only attempted in local server mode (no baseUrl, port defined).
-     */
-    private executeWithRecovery;
-    /**
-     * Dispose the parser instance.
-     * - Sets the internal client to null
-     * - If a local docling server was started (no baseUrl), kills the process on the configured port
-     */
-    dispose(): Promise<void>;
-}
-
-/**
- * Error thrown when both original PDF conversion and image PDF fallback fail.
- * Contains both errors for debugging purposes.
- */
-declare class ImagePdfFallbackError extends Error {
-    readonly originalError: Error;
-    readonly fallbackError: Error;
-    readonly name = "ImagePdfFallbackError";
-    constructor(originalError: Error, fallbackError: Error);
-}
-
-/**
- * Error thrown when the uploaded PDF does not appear to be
- * an archaeological investigation report.
- */
-declare class InvalidDocumentTypeError extends Error {
-    readonly reason: string;
-    readonly name = "InvalidDocumentTypeError";
-    readonly code = "INVALID_DOCUMENT_TYPE";
-    constructor(reason: string);
-}
-
-/**
- * Intermediate format produced by VLM page-by-page processing.
- * Intentionally kept simple so VLM prompts stay short and accurate.
- * The DoclingDocumentAssembler converts these into a full DoclingDocument.
- */
-/** Allowed element types matching DoclingDocument text labels */
-declare const VLM_ELEMENT_TYPES: readonly ["text", "section_header", "caption", "footnote", "page_header", "page_footer", "list_item", "picture", "table"];
-type VlmElementType = (typeof VLM_ELEMENT_TYPES)[number];
-/**
- * Normalized bounding box with coordinates in the range 0.0 to 1.0,
- * using top-left origin (standard image coordinates).
- */
-interface VlmBBox {
-    /** Left edge (0.0 = left boundary) */
-    l: number;
-    /** Top edge (0.0 = top boundary) */
-    t: number;
-    /** Right edge (1.0 = right boundary) */
-    r: number;
-    /** Bottom edge (1.0 = bottom boundary) */
-    b: number;
-}
-/** A single content element detected on a page by VLM */
-interface VlmPageElement {
-    /** Element type */
-    type: VlmElementType;
-    /** Text content (empty string for picture elements) */
-    content: string;
-    /** Heading depth for section_header (1 = top-level) */
-    level?: number;
-    /** List marker for list_item (e.g., "1)", "a.", "\u2022") */
-    marker?: string;
-    /** Reading order within the page (top-to-bottom, left-to-right) */
-    order: number;
-    /**
-     * Bounding box in normalized coordinates (0.0-1.0, top-left origin).
-     * - Text elements: optional (included in prov if present)
-     * - Picture elements: **required** (used for image cropping)
-     */
-    bbox?: VlmBBox;
-}
-/** Types of quality issues detected in VLM responses */
-declare const VLM_QUALITY_ISSUE_TYPES: readonly ["placeholder_text", "script_anomaly", "meta_description", "repetitive_pattern"];
-type VlmQualityIssueType = (typeof VLM_QUALITY_ISSUE_TYPES)[number];
-/** Quality metadata for a processed page */
-interface VlmPageQuality {
-    /** Whether the page passed quality validation (after possible retry) */
-    isValid: boolean;
-    /** Whether a quality retry was performed for this page */
-    retried: boolean;
-    /** Issue types detected (may persist even after retry) */
-    issueTypes: VlmQualityIssueType[];
-}
-
-/** A single quality issue found during validation */
-interface VlmQualityIssue {
-    /** Type of issue detected */
-    type: VlmQualityIssueType;
-    /** Human-readable description of the issue */
-    message: string;
-    /** Element reading-order indices that triggered the issue */
-    affectedElements: number[];
-}
-/** Result of VLM response quality validation */
-interface VlmValidationResult {
-    /** Whether the response passes quality validation */
-    isValid: boolean;
-    /** List of quality issues found (empty if valid) */
-    issues: VlmQualityIssue[];
-}
-/**
- * Lightweight, stateless validator for VLM page extraction responses.
- *
- * Detects four categories of hallucination without any additional VLM calls:
- * 1. Placeholder text (Lorem ipsum and variants)
- * 2. Script anomaly (expected Korean but got Latin-only text)
- * 3. Meta description (VLM described the image instead of transcribing text)
- * 4. Repetitive pattern (repeated character patterns like `: : : : :`)
- */
-declare class VlmResponseValidator {
-    /**
-     * Validate VLM page result quality.
-     *
-     * @param elements - Extracted page elements to validate
-     * @param documentLanguages - BCP 47 language tags (e.g., ['ko-KR', 'en-US'])
-     * @returns Validation result with issues list
-     */
-    static validate(elements: VlmPageElement[], documentLanguages?: string[]): VlmValidationResult;
-    /**
-     * Detect known placeholder / filler text in elements.
-     */
-    private static detectPlaceholderText;
-    /**
-     * Detect script anomaly: expected Korean content but found mostly Latin text.
-     * Counts Hangul + CJK characters and flags if the ratio is below threshold.
-     */
-    private static detectScriptAnomaly;
-    /**
-     * Detect meta description: VLM described the image/resolution instead
-     * of transcribing actual text content.
-     */
-    private static detectMetaDescription;
-    /**
-     * Detect repetitive character patterns (e.g., `: : : : :` or `= = = = =`).
-     * Flags when the same character repeats with spaces 5+ times and the
-     * repetitive portion exceeds 30% of total content.
-     */
-    private static detectRepetitivePattern;
-}
-
-export { type ConversionCompleteCallback, ImagePdfFallbackError, InvalidDocumentTypeError, type NormalizedPDFCorrectionOptions, type PDFConvertOptions, type PDFCorrectionConcurrencyOptions, type PDFCorrectionMaxRetriesOptions, type PDFCorrectionModelOptions, type PDFCorrectionOptions, type PDFCorrectionPageGateOptions, PDFParser, PDF_CORRECTION_DEFAULTS, type ReviewAssistanceTaskId, type VlmPageQuality, type VlmQualityIssue, type VlmQualityIssueType, VlmResponseValidator, type VlmValidationResult };