@heripo/pdf-parser 0.1.27 → 0.1.29

package/dist/index.js

@@ -6525,15 +6525,12 @@ import { normalizeToBcp47 } from "@heripo/model";
 import { readFileSync as readFileSync3 } from "fs";
 import { z as z3 } from "zod/v4";
 
-// src/prompts/korean-hanja-mix-prompt.ts
-var KOREAN_HANJA_MIX_PROMPT = `Look at this page image carefully. Does it contain any Hanja (\u6F22\u5B57/Chinese characters) mixed with Korean text?
+// src/prompts/korean-document-detection-prompt.ts
+var KOREAN_DOCUMENT_DETECTION_PROMPT = `Look at this page image carefully and identify all languages present on this page.
 
-Hanja examples: \u907A\u8E5F, \u767C\u6398, \u8ABF\u67FB, \u5831\u544A\u66F8, \u6587\u5316\u8CA1
-Note: Hanja are Chinese characters used in Korean documents, different from modern Korean (\uD55C\uAE00).
+Return an array of ocrmac-compatible language tags ordered by prevalence (primary language first).
+Use ko-KR when Korean text is present, even when the page also contains Hanja (\u6F22\u5B57/Chinese characters), English, numbers, tables, or captions.
 
-Answer whether any Hanja characters are present on this page.
-
-Also identify all languages present on this page. Return an array of ocrmac-compatible language tags ordered by prevalence (primary language first).
 Supported tags: ar-SA, ars-SA, cs-CZ, da-DK, de-DE, en-US, es-ES, fr-FR, id-ID, it-IT, ja-JP, ko-KR, ms-MY, nb-NO, nl-NL, nn-NO, no-NO, pl-PL, pt-BR, ro-RO, ru-RU, sv-SE, th-TH, tr-TR, uk-UA, vi-VT, yue-Hans, yue-Hant, zh-Hans, zh-Hant.
 Examples: ["ko-KR", "en-US"], ["ja-JP"], ["zh-Hant", "en-US"]`;
 
@@ -6541,12 +6538,8 @@ Examples: ["ko-KR", "en-US"], ["ja-JP"], ["zh-Hant", "en-US"]`;
 var EDGE_TRIM_RATIO = 0.1;
 var DEFAULT_MAX_SAMPLE_PAGES = 15;
 var DEFAULT_MAX_RETRIES = 3;
-var CJK_REGEX = /[\u4E00-\u9FFF]/;
 var HANGUL_REGEX = /[\uAC00-\uD7AF]/;
-var koreanHanjaMixSchema = z3.object({
-  hasKoreanHanjaMix: z3.boolean().describe(
-    "Whether the page contains any Hanja (\u6F22\u5B57/Chinese characters) mixed with Korean text"
-  ),
+var koreanDocumentDetectionSchema = z3.object({
   detectedLanguages: z3.array(z3.string()).describe(
     'BCP 47 language tags of languages found on this page, ordered by prevalence (e.g., ["ko-KR", "en-US"])'
   )
@@ -6565,14 +6558,14 @@ var OcrStrategySampler = class {
    *
    * @param pdfPath - Path to the PDF file
    * @param outputDir - Directory for temporary rendered pages
-   * @param model - Vision language model for Korean-Hanja mix detection
+   * @param model - Vision language model for Korean document detection
    * @param options - Sampling options
    * @returns OcrStrategy with method ('ocrmac' or 'vlm') and metadata
    */
   async sample(pdfPath, outputDir, model, options) {
     const maxSamplePages = options?.maxSamplePages ?? DEFAULT_MAX_SAMPLE_PAGES;
     this.logger.info("[OcrStrategySampler] Starting OCR strategy sampling...");
-    const preCheckResult = await this.preCheckHanjaFromTextLayer(pdfPath);
+    const preCheckResult = await this.preCheckKoreanFromTextLayer(pdfPath);
     if (preCheckResult) {
       return preCheckResult;
     }
@@ -6603,9 +6596,9 @@ var OcrStrategySampler = class {
       model,
       options
     );
-    if (sampleResult.foundMix) {
+    if (sampleResult.foundKorean) {
       this.logger.info(
-        `[OcrStrategySampler] Korean-Hanja mix detected on page ${sampleResult.mixPageNo} \u2192 VLM strategy`
+        `[OcrStrategySampler] Korean document detected on page ${sampleResult.koreanPageNo} \u2192 VLM strategy`
       );
       const detectedLanguages2 = this.aggregateLanguages(
         sampleResult.languageFrequency
@@ -6613,13 +6606,13 @@ var OcrStrategySampler = class {
       return {
         method: "vlm",
         detectedLanguages: detectedLanguages2,
-        reason: `Korean-Hanja mix detected on page ${sampleResult.mixPageNo}`,
+        reason: `Korean document detected on page ${sampleResult.koreanPageNo}`,
         sampledPages: sampleResult.sampledCount,
         totalPages: renderResult.pageCount
       };
     }
     this.logger.info(
-      "[OcrStrategySampler] No Korean-Hanja mix detected \u2192 ocrmac strategy"
+      "[OcrStrategySampler] No Korean language detected \u2192 ocrmac strategy"
     );
     const detectedLanguages = this.aggregateLanguages(
       sampleResult.languageFrequency
@@ -6627,13 +6620,13 @@ var OcrStrategySampler = class {
     return {
       method: "ocrmac",
       detectedLanguages,
-      reason: `No Korean-Hanja mix detected in ${sampleResult.sampledCount} sampled pages`,
+      reason: `No Korean language detected in ${sampleResult.sampledCount} sampled pages`,
       sampledPages: sampleResult.sampledCount,
       totalPages: renderResult.pageCount
     };
   }
   /**
-   * Sample pages for Korean-Hanja mix detection with early exit.
+   * Sample pages for Korean language detection with early exit.
    */
   async samplePages(sampleIndices, pageFiles, model, options) {
     const languageFrequency = /* @__PURE__ */ new Map();
@@ -6649,72 +6642,52 @@ var OcrStrategySampler = class {
       for (const lang of pageAnalysis.detectedLanguages) {
         languageFrequency.set(lang, (languageFrequency.get(lang) ?? 0) + 1);
       }
-      if (pageAnalysis.hasKoreanHanjaMix) {
+      if (pageAnalysis.detectedLanguages.includes("ko-KR")) {
         return {
-          foundMix: true,
-          mixPageNo: idx + 1,
+          foundKorean: true,
+          koreanPageNo: idx + 1,
           sampledCount: i + 1,
           languageFrequency
         };
       }
     }
     return {
-      foundMix: false,
+      foundKorean: false,
       sampledCount: sampleIndices.length,
       languageFrequency
     };
   }
   /**
-   * Pre-check for Hangul-Hanja mix in PDF text layer using pdftotext.
-   * Extracts full document text in a single process and checks at document level.
-   * Only makes a definitive decision for Korean (Hangul) documents:
-   * - Hangul + Hanja (anywhere in document) → VLM (confirmed Korean-Hanja mix)
-   * - Hangul only → ocrmac with ko-KR (confirmed Korean)
+   * Pre-check for Korean text in PDF text layer using pdftotext.
+   * Extracts full document text in a single process and checks at document level:
+   * - Hangul anywhere in document → VLM (confirmed Korean document)
    * - No Hangul (English, Japanese, etc.) → null (delegates to VLM for language detection)
    */
-  async preCheckHanjaFromTextLayer(pdfPath) {
+  async preCheckKoreanFromTextLayer(pdfPath) {
     try {
       const totalPages = await this.textExtractor.getPageCount(pdfPath);
       if (totalPages === 0) return null;
       const fullText = await this.textExtractor.extractFullText(pdfPath);
       if (fullText.trim().length === 0) {
         this.logger.debug(
-          "[OcrStrategySampler] No Hangul in text layer, falling back to VLM sampling"
+          "[OcrStrategySampler] No text in text layer, falling back to VLM sampling"
         );
         return null;
       }
       const hasHangul = HANGUL_REGEX.test(fullText);
-      const hasHanja = CJK_REGEX.test(fullText);
       if (!hasHangul) {
         this.logger.debug(
           "[OcrStrategySampler] No Hangul in text layer, falling back to VLM sampling"
         );
         return null;
       }
-      if (hasHanja) {
-        const pageTextArray = fullText.split("\f");
-        const koreanHanjaMixPages = pageTextArray.flatMap(
-          (text, i) => CJK_REGEX.test(text) ? [i + 1] : []
-        );
-        this.logger.info(
-          `[OcrStrategySampler] Hangul-Hanja mix detected in text layer \u2192 VLM strategy (${koreanHanjaMixPages.length} pages with Hanja)`
-        );
-        return {
-          method: "vlm",
-          detectedLanguages: ["ko-KR"],
-          reason: "Hangul-Hanja mix found in PDF text layer",
-          koreanHanjaMixPages,
-          sampledPages: totalPages,
-          totalPages
-        };
-      }
       this.logger.info(
-        "[OcrStrategySampler] No Hangul-Hanja mix in text layer \u2192 ocrmac strategy"
+        "[OcrStrategySampler] Korean text detected in text layer \u2192 VLM strategy"
       );
       return {
-        method: "ocrmac",
+        method: "vlm",
         detectedLanguages: ["ko-KR"],
-        reason: `No Hangul-Hanja mix in PDF text layer (${totalPages} pages checked)`,
+        reason: `Korean text found in PDF text layer (${totalPages} pages checked)`,
         sampledPages: totalPages,
         totalPages
       };
@@ -6755,21 +6728,24 @@ var OcrStrategySampler = class {
     );
   }
   /**
-   * Analyze a single sample page for Korean-Hanja mixed script and primary language.
+   * Analyze a single sample page for language.
    * Normalizes raw VLM language responses to valid BCP 47 tags, filtering out invalid ones.
    *
-   * @returns Object with Korean-Hanja detection result and normalized detected languages
+   * @returns Object with normalized detected languages
    */
   async analyzeSamplePage(pageFile, pageNo, model, options) {
     this.logger.debug(
-      `[OcrStrategySampler] Analyzing page ${pageNo} for Korean-Hanja mix and language...`
+      `[OcrStrategySampler] Analyzing page ${pageNo} for language...`
     );
     const imageData = new Uint8Array(readFileSync3(pageFile));
     const messages = [
       {
         role: "user",
         content: [
-          { type: "text", text: KOREAN_HANJA_MIX_PROMPT },
+          {
+            type: "text",
+            text: KOREAN_DOCUMENT_DETECTION_PROMPT
+          },
           {
             type: "image",
             image: imageData,
@@ -6779,7 +6755,7 @@ var OcrStrategySampler = class {
       }
     ];
     const result = await LLMCaller.callVision({
-      schema: koreanHanjaMixSchema,
+      schema: koreanDocumentDetectionSchema,
       messages,
       primaryModel: model,
       fallbackModel: options?.fallbackModel,
@@ -6787,7 +6763,7 @@ var OcrStrategySampler = class {
       temperature: options?.temperature ?? 0,
       abortSignal: options?.abortSignal,
       component: "OcrStrategySampler",
-      phase: "korean-hanja-mix-detection"
+      phase: "korean-document-detection"
     });
     if (options?.aggregator) {
       options.aggregator.track(result.usage);
@@ -6795,10 +6771,9 @@ var OcrStrategySampler = class {
     const output = result.output;
     const normalizedLanguages = output.detectedLanguages.map(normalizeToBcp47).filter((tag) => tag !== null);
     this.logger.debug(
-      `[OcrStrategySampler] Page ${pageNo}: hasKoreanHanjaMix=${output.hasKoreanHanjaMix}, detectedLanguages=${normalizedLanguages.join(",")}`
+      `[OcrStrategySampler] Page ${pageNo}: detectedLanguages=${normalizedLanguages.join(",")}`
     );
     return {
-      hasKoreanHanjaMix: output.hasKoreanHanjaMix,
       detectedLanguages: normalizedLanguages
     };
   }
@@ -7047,16 +7022,7 @@ var VlmTextCorrector = class {
         pagesFailed: 0
       };
     }
-    let pageNumbers;
-    if (options?.koreanHanjaMixPages && options.koreanHanjaMixPages.length > 0) {
-      const hanjaSet = new Set(options.koreanHanjaMixPages);
-      pageNumbers = allPageNumbers.filter((p) => hanjaSet.has(p));
-      this.logger.info(
-        `[VlmTextCorrector] Filtering to ${pageNumbers.length} Korean-Hanja mix pages out of ${allPageNumbers.length} total`
-      );
-    } else {
-      pageNumbers = allPageNumbers;
-    }
+    const pageNumbers = allPageNumbers;
     const concurrency = options?.concurrency ?? DEFAULT_CONCURRENCY;
     this.logger.info(
       `[VlmTextCorrector] Processing ${pageNumbers.length} pages (concurrency: ${concurrency})...`
@@ -7261,7 +7227,7 @@ var VlmConversionPipeline = class {
    * Wrap the original callback with VLM text correction.
    * Returns a new callback that runs VLM correction before calling the original.
    */
-  wrapCallback(pdfPath, options, originalCallback, abortSignal, detectedLanguages, koreanHanjaMixPages) {
+  wrapCallback(pdfPath, options, originalCallback, abortSignal, detectedLanguages) {
     if (!options.vlmProcessorModel) {
       throw new Error("vlmProcessorModel is required when OCR strategy is VLM");
     }
@@ -7293,8 +7259,7 @@ var VlmConversionPipeline = class {
           abortSignal,
           onTokenUsage: options.onTokenUsage,
           documentLanguages: detectedLanguages,
-          pageTexts,
-          koreanHanjaMixPages
+          pageTexts
         }
       );
       await originalCallback(outputDir);
@@ -7446,8 +7411,7 @@ var PDFConverter = class {
         trackedOptions,
         postCorrectionCallback,
         abortSignal,
-        strategy.detectedLanguages,
-        strategy.koreanHanjaMixPages
+        strategy.detectedLanguages
       );
       const vlmOptions = strategy.detectedLanguages ? { ...trackedOptions, ocr_lang: strategy.detectedLanguages } : trackedOptions;
       await this.convert(
@@ -7982,7 +7946,7 @@ var META_DESCRIPTION_PATTERNS_EN = [
 var REPETITIVE_PATTERN_RATIO_THRESHOLD = 0.3;
 var REPETITIVE_PATTERN_MIN_REPEATS = 5;
 var HANGUL_REGEX2 = /[\uAC00-\uD7AF\u1100-\u11FF]/g;
-var CJK_REGEX2 = /[\u4E00-\u9FFF]/g;
+var CJK_REGEX = /[\u4E00-\u9FFF]/g;
 var VlmResponseValidator = class {
   /**
    * Validate VLM page result quality.
@@ -8050,7 +8014,7 @@ var VlmResponseValidator = class {
       return null;
     }
     const hangulCount = allContent.match(HANGUL_REGEX2)?.length ?? 0;
-    const cjkCount = allContent.match(CJK_REGEX2)?.length ?? 0;
+    const cjkCount = allContent.match(CJK_REGEX)?.length ?? 0;
     const koreanCjkCount = hangulCount + cjkCount;
     const ratio = koreanCjkCount / nonWhitespace.length;
     if (ratio < KOREAN_SCRIPT_RATIO_THRESHOLD) {