@heripo/document-processor 0.1.5 → 0.1.7

package/dist/index.cjs

@@ -39,6 +39,7 @@ __export(index_exports, {
   CaptionValidator: () => CaptionValidator,
   ChapterConverter: () => ChapterConverter,
   DocumentProcessor: () => DocumentProcessor,
+  HanjaQualitySampler: () => HanjaQualitySampler,
   PAGE_NUMBER_PATTERN: () => PAGE_NUMBER_PATTERN,
   PagePattern: () => PagePattern,
   PageRangeParseError: () => PageRangeParseError,
@@ -3408,6 +3409,283 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
   }
 };
 
+// src/samplers/hanja-quality-sampler.ts
+var import_zod5 = require("zod");
+var MIN_TEXT_LENGTH = 100;
+var MAX_SAMPLE_PAGES = 10;
+var EDGE_TRIM_RATIO = 0.1;
+var IMAGE_PAGE_TEXT_THRESHOLD = 50;
+var CORRUPTION_THRESHOLD = 0.5;
+var MIN_SEVERE_CORRUPTED_COUNT = 3;
+var HanjaQualityResponseSchema = import_zod5.z.object({
+  isCorrupted: import_zod5.z.boolean().describe(
+    "Whether the KCJ (Hanja) characters in the OCR text are significantly corrupted compared to the page image"
+  ),
+  corruptedCharCount: import_zod5.z.number().int().nonnegative().describe(
+    "Approximate number of corrupted or incorrectly recognized KCJ characters"
+  ),
+  totalKcjCharCount: import_zod5.z.number().int().nonnegative().describe("Total number of KCJ characters found in the OCR text"),
+  explanation: import_zod5.z.string().describe("Brief explanation of the assessment result")
+});
+var HanjaQualitySampler = class extends VisionLLMComponent {
+  constructor(logger, model, outputPath, maxRetries, fallbackModel, aggregator, abortSignal) {
+    super(
+      logger,
+      model,
+      "HanjaQualitySampler",
+      outputPath,
+      { maxRetries, abortSignal },
+      fallbackModel,
+      aggregator
+    );
+  }
+  /**
+   * Assess the quality of KCJ character recognition in the document
+   *
+   * @param doclingDoc - DoclingDocument to assess
+   * @returns Assessment result indicating whether VLM re-parse is needed
+   */
+  async assess(doclingDoc) {
+    const totalPages = Object.keys(doclingDoc.pages).length;
+    this.log("info", "Starting KCJ quality assessment...");
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const imageOnlyPages = this.getImageOnlyPages(doclingDoc);
+    this.log(
+      "info",
+      `Total pages: ${totalPages}, eligible range: (${frontCutoff}, ${backCutoff}], image-only pages excluded: ${imageOnlyPages.size}`
+    );
+    const textPages = this.getTextPages(doclingDoc);
+    if (textPages.length === 0) {
+      this.log("info", "No text pages found for assessment");
+      return {
+        needsVlmReparse: false,
+        severity: "none",
+        kcjPageCount: 0,
+        sampledPageCount: 0,
+        corruptedRatio: 0,
+        reason: "No text pages found for assessment"
+      };
+    }
+    this.log(
+      "info",
+      `Found ${textPages.length} text pages (min length: ${MIN_TEXT_LENGTH})`
+    );
+    const sampled = this.selectSamplePages(
+      textPages,
+      totalPages,
+      imageOnlyPages
+    );
+    this.log(
+      "info",
+      `Sampling ${sampled.length} pages: [${sampled.map((p) => p.pageNo).join(", ")}]`
+    );
+    const results = await this.evaluatePages(sampled);
+    const assessment = this.aggregateResults(
+      textPages.length,
+      sampled.length,
+      results
+    );
+    this.log(
+      "info",
+      `Assessment complete: severity=${assessment.severity}, corrupted=${assessment.corruptedRatio.toFixed(2)}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
+  /**
+   * Get the eligible page range after trimming front/back edges.
+   * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
+   */
+  getEligiblePageRange(totalPages) {
+    const trimCount = Math.ceil(totalPages * EDGE_TRIM_RATIO);
+    return {
+      frontCutoff: trimCount,
+      backCutoff: totalPages - trimCount
+    };
+  }
+  /**
+   * Get page numbers that are "image-only" (contain pictures but minimal text).
+   * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
+   */
+  getImageOnlyPages(doclingDoc) {
+    const picturePages = /* @__PURE__ */ new Set();
+    for (const picture of doclingDoc.pictures) {
+      const pageNo = picture.prov?.[0]?.page_no;
+      if (pageNo != null) {
+        picturePages.add(pageNo);
+      }
+    }
+    if (picturePages.size === 0) return /* @__PURE__ */ new Set();
+    const pageTextLength = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      if (picturePages.has(pageNo)) {
+        pageTextLength.set(
+          pageNo,
+          (pageTextLength.get(pageNo) ?? 0) + text.text.length
+        );
+      }
+    }
+    const imageOnlyPages = /* @__PURE__ */ new Set();
+    for (const pageNo of picturePages) {
+      const textLength = pageTextLength.get(pageNo) ?? 0;
+      if (textLength < IMAGE_PAGE_TEXT_THRESHOLD) {
+        imageOnlyPages.add(pageNo);
+      }
+    }
+    return imageOnlyPages;
+  }
+  /**
+   * Get all pages with substantial text content.
+   * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
+   */
+  getTextPages(doclingDoc) {
+    const pageMap = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      const entry = pageMap.get(pageNo) ?? { textLength: 0, texts: [] };
+      entry.textLength += text.text.length;
+      entry.texts.push(text.text);
+      pageMap.set(pageNo, entry);
+    }
+    return Array.from(pageMap.entries()).filter(([, data]) => data.textLength >= MIN_TEXT_LENGTH).map(([pageNo, data]) => ({
+      pageNo,
+      textLength: data.textLength,
+      texts: data.texts
+    }));
+  }
+  /**
+   * Select sample pages with highest text density.
+   * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
+   * Falls back to all text pages if filtering leaves too few candidates.
+   */
+  selectSamplePages(textPages, totalPages, imageOnlyPages) {
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const filtered = totalPages > 0 ? textPages.filter(
+      (p) => p.pageNo > frontCutoff && p.pageNo <= backCutoff && !imageOnlyPages.has(p.pageNo)
+    ) : textPages;
+    const candidates = filtered.length > 0 ? filtered : textPages;
+    if (candidates !== filtered) {
+      this.log(
+        "warn",
+        `All text pages were filtered out by edge/image exclusion. Falling back to all ${textPages.length} text pages.`
+      );
+    }
+    const sorted = [...candidates].sort((a, b) => b.textLength - a.textLength);
+    const count = Math.min(MAX_SAMPLE_PAGES, sorted.length);
+    return sorted.slice(0, count);
+  }
+  /**
+   * Evaluate sampled pages using Vision LLM
+   */
+  async evaluatePages(pages) {
+    const results = [];
+    for (const page of pages) {
+      const result = await this.evaluateSinglePage(page);
+      results.push(result);
+    }
+    return results;
+  }
+  /**
+   * Evaluate a single page by comparing OCR text against the page image
+   */
+  async evaluateSinglePage(page) {
+    const imagePath = `pages/page_${page.pageNo - 1}.png`;
+    let imageContent;
+    try {
+      imageContent = this.buildImageContent(imagePath);
+    } catch {
+      this.log(
+        "warn",
+        `Failed to load page image for page ${page.pageNo}, marking as not corrupted`
+      );
+      return {
+        isCorrupted: false,
+        corruptedCharCount: 0,
+        totalKcjCharCount: 0,
+        explanation: `Page image not available for page ${page.pageNo}`
+      };
+    }
+    const ocrText = page.texts.join("\n");
+    const messages = [
+      {
+        role: "user",
+        content: [
+          imageContent,
+          {
+            type: "text",
+            text: this.buildUserPrompt(ocrText)
+          }
+        ]
+      }
+    ];
+    const { output } = await this.callVisionLLM(
+      HanjaQualityResponseSchema,
+      messages,
+      `page-${page.pageNo}`
+    );
+    this.log(
+      "info",
+      `Page ${page.pageNo}: corrupted=${output.isCorrupted}, ${output.corruptedCharCount}/${output.totalKcjCharCount} KCJ chars corrupted`
+    );
+    return output;
+  }
+  /**
+   * Aggregate individual page results into final assessment
+   */
+  aggregateResults(totalTextPages, sampledCount, results) {
+    const corruptedCount = results.filter((r) => r.isCorrupted).length;
+    const corruptedRatio = sampledCount > 0 ? corruptedCount / sampledCount : 0;
+    let severity;
+    let needsVlmReparse;
+    if (corruptedRatio >= CORRUPTION_THRESHOLD || corruptedCount >= MIN_SEVERE_CORRUPTED_COUNT) {
+      severity = "severe";
+      needsVlmReparse = true;
+    } else if (corruptedCount > 0) {
+      severity = "minor";
+      needsVlmReparse = false;
+    } else {
+      severity = "none";
+      needsVlmReparse = false;
+    }
+    const reason = corruptedCount === 0 ? "No Hanja character corruption detected" : `${corruptedCount}/${sampledCount} sampled pages have corrupted Hanja characters (ratio: ${corruptedRatio.toFixed(2)})`;
+    return {
+      needsVlmReparse,
+      severity,
+      kcjPageCount: totalTextPages,
+      sampledPageCount: sampledCount,
+      corruptedRatio,
+      reason
+    };
+  }
+  /**
+   * Get page number from a text item
+   */
+  getPageNo(text) {
+    return text.prov?.[0]?.page_no ?? 1;
+  }
+  buildSystemPrompt() {
+    return "";
+  }
+  buildUserPrompt(ocrText) {
+    return `You are evaluating the quality of OCR text extraction for Hanja (\u6F22\u5B57) characters used in Korean archaeological reports.
+
+Look at the page image and find any Hanja (\u6F22\u5B57) characters visible in the original document. Then compare them against the OCR text below.
+
+## Evaluation Criteria
+- Find Hanja (\u6F22\u5B57) characters in the page image first
+- Check if each Hanja character is correctly recognized in the OCR text
+- A character is "corrupted" if a Hanja character in the image was replaced by Korean hangul, garbled text, symbols, or a completely different character in the OCR text
+- Common corruption patterns: Hanja replaced by similar-looking hangul (e.g., \u7C98\u571F \u2192 \uADF8\uC73C), wrong character substitution, partial recognition, question marks or boxes
+- Minor font rendering differences are NOT corruption
+- If no Hanja characters are visible in the page image, report as not corrupted with 0 total characters
+
+## OCR Text to Evaluate
+${ocrText}
+
+Compare the Hanja characters visible in the page image against the OCR text above and assess whether they were correctly recognized.`;
+  }
+};
+
 // src/validators/base-validator.ts
 var BaseValidator = class extends TextLLMComponent {
   /**
@@ -3467,13 +3745,13 @@ var BaseValidator = class extends TextLLMComponent {
 };
 
 // src/validators/toc-content-validator.ts
-var import_zod5 = require("zod");
-var TocContentValidationSchema = import_zod5.z.object({
-  isValid: import_zod5.z.boolean().describe("Whether valid main document TOC was found"),
-  confidence: import_zod5.z.number().min(0).max(1).describe("Confidence score between 0 and 1"),
-  contentType: import_zod5.z.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
-  extractedTocMarkdown: import_zod5.z.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
-  reason: import_zod5.z.string().describe("Brief explanation in English")
+var import_zod6 = require("zod");
+var TocContentValidationSchema = import_zod6.z.object({
+  isValid: import_zod6.z.boolean().describe("Whether valid main document TOC was found"),
+  confidence: import_zod6.z.number().min(0).max(1).describe("Confidence score between 0 and 1"),
+  contentType: import_zod6.z.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
+  extractedTocMarkdown: import_zod6.z.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
+  reason: import_zod6.z.string().describe("Brief explanation in English")
 });
 var TocContentValidator = class extends BaseValidator {
   confidenceThreshold;
@@ -3624,14 +3902,14 @@ ${markdown}`;
 };
 
 // src/validators/caption-validator.ts
-var import_zod6 = require("zod");
-var CaptionValidationItemSchema = import_zod6.z.object({
-  index: import_zod6.z.number().int().describe("Index of the caption in the input array"),
-  isValid: import_zod6.z.boolean().describe("Whether the parsed caption is correct"),
-  reason: import_zod6.z.string().nullable().describe("Brief explanation if invalid, null if valid")
+var import_zod7 = require("zod");
+var CaptionValidationItemSchema = import_zod7.z.object({
+  index: import_zod7.z.number().int().describe("Index of the caption in the input array"),
+  isValid: import_zod7.z.boolean().describe("Whether the parsed caption is correct"),
+  reason: import_zod7.z.string().nullable().describe("Brief explanation if invalid, null if valid")
 });
-var CaptionValidationBatchSchema = import_zod6.z.object({
-  results: import_zod6.z.array(CaptionValidationItemSchema)
+var CaptionValidationBatchSchema = import_zod7.z.object({
+  results: import_zod7.z.array(CaptionValidationItemSchema)
 });
 var CaptionValidator = class extends BaseValidator {
   constructor(logger, model, options, fallbackModel, aggregator) {
@@ -3839,6 +4117,7 @@ var DocumentProcessor = class {
   validatorModel;
   visionTocExtractorModel;
   captionParserModel;
+  hanjaQualitySamplerModel;
   textCleanerBatchSize;
   captionParserBatchSize;
   captionValidatorBatchSize;
@@ -3865,6 +4144,7 @@ var DocumentProcessor = class {
     this.validatorModel = options.validatorModel ?? options.fallbackModel;
     this.visionTocExtractorModel = options.visionTocExtractorModel ?? options.fallbackModel;
     this.captionParserModel = options.captionParserModel ?? options.fallbackModel;
+    this.hanjaQualitySamplerModel = options.hanjaQualitySamplerModel ?? options.fallbackModel;
     this.textCleanerBatchSize = options.textCleanerBatchSize;
     this.captionParserBatchSize = options.captionParserBatchSize;
     this.captionValidatorBatchSize = options.captionValidatorBatchSize;
@@ -3884,6 +4164,36 @@ var DocumentProcessor = class {
       throw error;
     }
   }
+  /**
+   * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
+   *
+   * Samples pages containing KCJ characters and uses Vision LLM to compare
+   * the OCR text against the original page images. Returns an assessment
+   * indicating whether the document should be re-parsed with VLM pipeline.
+   *
+   * @param doclingDoc - Original document extracted from Docling SDK
+   * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
+   * @returns Assessment result with severity and recommendation
+   */
+  async assessHanjaQuality(doclingDoc, outputPath) {
+    this.logger.info(
+      "[DocumentProcessor] Starting Hanja quality assessment..."
+    );
+    const sampler = new HanjaQualitySampler(
+      this.logger,
+      this.hanjaQualitySamplerModel,
+      outputPath,
+      this.maxRetries,
+      this.enableFallbackRetry ? this.fallbackModel : void 0,
+      this.usageAggregator,
+      this.abortSignal
+    );
+    const assessment = await sampler.assess(doclingDoc);
+    this.logger.info(
+      `[DocumentProcessor] Hanja assessment: severity=${assessment.severity}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
   /**
    * Converts DoclingDocument to ProcessedDocument with token usage tracking.
    *
@@ -4217,8 +4527,10 @@ var DocumentProcessor = class {
       );
     }
     const totalPages = Object.keys(doclingDoc.pages).length;
+    const maxTocPageNo = this.extractMaxPageNumber(markdown);
+    const effectiveTotalPages = maxTocPageNo > totalPages ? void 0 : totalPages;
     const tocResult = await this.tocExtractor.extract(markdown, {
-      totalPages
+      totalPages: effectiveTotalPages
     });
     for (const usage of tocResult.usages) {
       this.usageAggregator.track(usage);
@@ -4233,6 +4545,20 @@ var DocumentProcessor = class {
     );
     return tocResult.entries;
   }
+  /**
+   * Extract the maximum page number from TOC markdown
+   *
+   * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+   * to detect compiled volume scenarios where TOC page numbers exceed
+   * the sub-document's page count.
+   */
+  extractMaxPageNumber(markdown) {
+    const matches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
+    if (matches.length === 0) {
+      return 0;
+    }
+    return Math.max(...matches.map((m) => parseInt(m[1], 10)));
+  }
   /**
    * Process resource captions (for images and tables)
    *
@@ -4501,6 +4827,7 @@ var DocumentProcessor = class {
   CaptionValidator,
   ChapterConverter,
   DocumentProcessor,
+  HanjaQualitySampler,
   PAGE_NUMBER_PATTERN,
   PagePattern,
   PageRangeParseError,