@heripo/document-processor 0.1.5 → 0.1.7

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { DoclingDocument, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { DoclingDocument, HanjaAssessment, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -43,6 +43,11 @@ interface DocumentProcessorOptions {
      * Falls back to 'fallbackModel' if not provided.
      */
     captionParserModel?: LanguageModel;
+    /**
+     * Model for HanjaQualitySampler - evaluates KCJ character quality in OCR output.
+     * Requires vision capabilities. Falls back to 'fallbackModel' if not provided.
+     */
+    hanjaQualitySamplerModel?: LanguageModel;
     /**
      * Batch size for TextCleaner text normalization (synchronous processing)
      */
@@ -132,6 +137,7 @@ declare class DocumentProcessor {
     private readonly validatorModel;
     private readonly visionTocExtractorModel;
     private readonly captionParserModel;
+    private readonly hanjaQualitySamplerModel;
     private readonly textCleanerBatchSize;
     private readonly captionParserBatchSize;
     private readonly captionValidatorBatchSize;
@@ -157,6 +163,18 @@ declare class DocumentProcessor {
      * @throws {Error} with name 'AbortError' if aborted
      */
     private checkAborted;
+    /**
+     * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
+     *
+     * Samples pages containing KCJ characters and uses Vision LLM to compare
+     * the OCR text against the original page images. Returns an assessment
+     * indicating whether the document should be re-parsed with VLM pipeline.
+     *
+     * @param doclingDoc - Original document extracted from Docling SDK
+     * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
+     * @returns Assessment result with severity and recommendation
+     */
+    assessHanjaQuality(doclingDoc: DoclingDocument, outputPath: string): Promise<HanjaAssessment>;
     /**
      * Converts DoclingDocument to ProcessedDocument with token usage tracking.
      *
@@ -232,6 +250,14 @@ declare class DocumentProcessor {
      * 5. TocExtractor - LLM-based structured extraction
      */
     private extractTableOfContents;
+    /**
+     * Extract the maximum page number from TOC markdown
+     *
+     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+     * to detect compiled volume scenarios where TOC page numbers exceed
+     * the sub-document's page count.
+     */
+    private extractMaxPageNumber;
     /**
      * Process resource captions (for images and tables)
      *
@@ -1482,4 +1508,64 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+/**
+ * HanjaQualitySampler
+ *
+ * Evaluates the quality of KCJ (Chinese/Japanese/Korean) character recognition
+ * in OCR-processed documents. Samples a subset of pages containing KCJ text and
+ * uses Vision LLM to compare the OCR output against the original page images.
+ *
+ * This is used to determine whether a document should be re-parsed using the
+ * VLM pipeline for better KCJ character accuracy.
+ */
+declare class HanjaQualitySampler extends VisionLLMComponent {
+    constructor(logger: LoggerMethods, model: LanguageModel, outputPath: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
+    /**
+     * Assess the quality of KCJ character recognition in the document
+     *
+     * @param doclingDoc - DoclingDocument to assess
+     * @returns Assessment result indicating whether VLM re-parse is needed
+     */
+    assess(doclingDoc: DoclingDocument): Promise<HanjaAssessment>;
+    /**
+     * Get the eligible page range after trimming front/back edges.
+     * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
+     */
+    private getEligiblePageRange;
+    /**
+     * Get page numbers that are "image-only" (contain pictures but minimal text).
+     * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
+     */
+    private getImageOnlyPages;
+    /**
+     * Get all pages with substantial text content.
+     * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
+     */
+    private getTextPages;
+    /**
+     * Select sample pages with highest text density.
+     * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
+     * Falls back to all text pages if filtering leaves too few candidates.
+     */
+    private selectSamplePages;
+    /**
+     * Evaluate sampled pages using Vision LLM
+     */
+    private evaluatePages;
+    /**
+     * Evaluate a single page by comparing OCR text against the page image
+     */
+    private evaluateSinglePage;
+    /**
+     * Aggregate individual page results into final assessment
+     */
+    private aggregateResults;
+    /**
+     * Get page number from a text item
+     */
+    private getPageNo;
+    protected buildSystemPrompt(): string;
+    protected buildUserPrompt(ocrText: string): string;
+}
+
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, HanjaQualitySampler, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { DoclingDocument, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { DoclingDocument, HanjaAssessment, DocumentProcessResult, Caption, PageRange, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -43,6 +43,11 @@ interface DocumentProcessorOptions {
      * Falls back to 'fallbackModel' if not provided.
      */
     captionParserModel?: LanguageModel;
+    /**
+     * Model for HanjaQualitySampler - evaluates KCJ character quality in OCR output.
+     * Requires vision capabilities. Falls back to 'fallbackModel' if not provided.
+     */
+    hanjaQualitySamplerModel?: LanguageModel;
     /**
      * Batch size for TextCleaner text normalization (synchronous processing)
      */
@@ -132,6 +137,7 @@ declare class DocumentProcessor {
     private readonly validatorModel;
     private readonly visionTocExtractorModel;
     private readonly captionParserModel;
+    private readonly hanjaQualitySamplerModel;
     private readonly textCleanerBatchSize;
     private readonly captionParserBatchSize;
     private readonly captionValidatorBatchSize;
@@ -157,6 +163,18 @@ declare class DocumentProcessor {
      * @throws {Error} with name 'AbortError' if aborted
      */
     private checkAborted;
+    /**
+     * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
+     *
+     * Samples pages containing KCJ characters and uses Vision LLM to compare
+     * the OCR text against the original page images. Returns an assessment
+     * indicating whether the document should be re-parsed with VLM pipeline.
+     *
+     * @param doclingDoc - Original document extracted from Docling SDK
+     * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
+     * @returns Assessment result with severity and recommendation
+     */
+    assessHanjaQuality(doclingDoc: DoclingDocument, outputPath: string): Promise<HanjaAssessment>;
     /**
      * Converts DoclingDocument to ProcessedDocument with token usage tracking.
      *
@@ -232,6 +250,14 @@ declare class DocumentProcessor {
      * 5. TocExtractor - LLM-based structured extraction
      */
     private extractTableOfContents;
+    /**
+     * Extract the maximum page number from TOC markdown
+     *
+     * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+     * to detect compiled volume scenarios where TOC page numbers exceed
+     * the sub-document's page count.
+     */
+    private extractMaxPageNumber;
     /**
      * Process resource captions (for images and tables)
      *
@@ -1482,4 +1508,64 @@ declare class ChapterConverter {
     private buildChapterMap;
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };
+/**
+ * HanjaQualitySampler
+ *
+ * Evaluates the quality of KCJ (Chinese/Japanese/Korean) character recognition
+ * in OCR-processed documents. Samples a subset of pages containing KCJ text and
+ * uses Vision LLM to compare the OCR output against the original page images.
+ *
+ * This is used to determine whether a document should be re-parsed using the
+ * VLM pipeline for better KCJ character accuracy.
+ */
+declare class HanjaQualitySampler extends VisionLLMComponent {
+    constructor(logger: LoggerMethods, model: LanguageModel, outputPath: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
+    /**
+     * Assess the quality of KCJ character recognition in the document
+     *
+     * @param doclingDoc - DoclingDocument to assess
+     * @returns Assessment result indicating whether VLM re-parse is needed
+     */
+    assess(doclingDoc: DoclingDocument): Promise<HanjaAssessment>;
+    /**
+     * Get the eligible page range after trimming front/back edges.
+     * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
+     */
+    private getEligiblePageRange;
+    /**
+     * Get page numbers that are "image-only" (contain pictures but minimal text).
+     * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
+     */
+    private getImageOnlyPages;
+    /**
+     * Get all pages with substantial text content.
+     * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
+     */
+    private getTextPages;
+    /**
+     * Select sample pages with highest text density.
+     * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
+     * Falls back to all text pages if filtering leaves too few candidates.
+     */
+    private selectSamplePages;
+    /**
+     * Evaluate sampled pages using Vision LLM
+     */
+    private evaluatePages;
+    /**
+     * Evaluate a single page by comparing OCR text against the page image
+     */
+    private evaluateSinglePage;
+    /**
+     * Aggregate individual page results into final assessment
+     */
+    private aggregateResults;
+    /**
+     * Get page number from a text item
+     */
+    private getPageNo;
+    protected buildSystemPrompt(): string;
+    protected buildUserPrompt(ocrText: string): string;
+}
+
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, HanjaQualitySampler, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions };

package/dist/index.js

@@ -3346,6 +3346,283 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
   }
 };
 
+// src/samplers/hanja-quality-sampler.ts
+import { z as z5 } from "zod";
+var MIN_TEXT_LENGTH = 100;
+var MAX_SAMPLE_PAGES = 10;
+var EDGE_TRIM_RATIO = 0.1;
+var IMAGE_PAGE_TEXT_THRESHOLD = 50;
+var CORRUPTION_THRESHOLD = 0.5;
+var MIN_SEVERE_CORRUPTED_COUNT = 3;
+var HanjaQualityResponseSchema = z5.object({
+  isCorrupted: z5.boolean().describe(
+    "Whether the KCJ (Hanja) characters in the OCR text are significantly corrupted compared to the page image"
+  ),
+  corruptedCharCount: z5.number().int().nonnegative().describe(
+    "Approximate number of corrupted or incorrectly recognized KCJ characters"
+  ),
+  totalKcjCharCount: z5.number().int().nonnegative().describe("Total number of KCJ characters found in the OCR text"),
+  explanation: z5.string().describe("Brief explanation of the assessment result")
+});
+var HanjaQualitySampler = class extends VisionLLMComponent {
+  constructor(logger, model, outputPath, maxRetries, fallbackModel, aggregator, abortSignal) {
+    super(
+      logger,
+      model,
+      "HanjaQualitySampler",
+      outputPath,
+      { maxRetries, abortSignal },
+      fallbackModel,
+      aggregator
+    );
+  }
+  /**
+   * Assess the quality of KCJ character recognition in the document
+   *
+   * @param doclingDoc - DoclingDocument to assess
+   * @returns Assessment result indicating whether VLM re-parse is needed
+   */
+  async assess(doclingDoc) {
+    const totalPages = Object.keys(doclingDoc.pages).length;
+    this.log("info", "Starting KCJ quality assessment...");
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const imageOnlyPages = this.getImageOnlyPages(doclingDoc);
+    this.log(
+      "info",
+      `Total pages: ${totalPages}, eligible range: (${frontCutoff}, ${backCutoff}], image-only pages excluded: ${imageOnlyPages.size}`
+    );
+    const textPages = this.getTextPages(doclingDoc);
+    if (textPages.length === 0) {
+      this.log("info", "No text pages found for assessment");
+      return {
+        needsVlmReparse: false,
+        severity: "none",
+        kcjPageCount: 0,
+        sampledPageCount: 0,
+        corruptedRatio: 0,
+        reason: "No text pages found for assessment"
+      };
+    }
+    this.log(
+      "info",
+      `Found ${textPages.length} text pages (min length: ${MIN_TEXT_LENGTH})`
+    );
+    const sampled = this.selectSamplePages(
+      textPages,
+      totalPages,
+      imageOnlyPages
+    );
+    this.log(
+      "info",
+      `Sampling ${sampled.length} pages: [${sampled.map((p) => p.pageNo).join(", ")}]`
+    );
+    const results = await this.evaluatePages(sampled);
+    const assessment = this.aggregateResults(
+      textPages.length,
+      sampled.length,
+      results
+    );
+    this.log(
+      "info",
+      `Assessment complete: severity=${assessment.severity}, corrupted=${assessment.corruptedRatio.toFixed(2)}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
+  /**
+   * Get the eligible page range after trimming front/back edges.
+   * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
+   */
+  getEligiblePageRange(totalPages) {
+    const trimCount = Math.ceil(totalPages * EDGE_TRIM_RATIO);
+    return {
+      frontCutoff: trimCount,
+      backCutoff: totalPages - trimCount
+    };
+  }
+  /**
+   * Get page numbers that are "image-only" (contain pictures but minimal text).
+   * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
+   */
+  getImageOnlyPages(doclingDoc) {
+    const picturePages = /* @__PURE__ */ new Set();
+    for (const picture of doclingDoc.pictures) {
+      const pageNo = picture.prov?.[0]?.page_no;
+      if (pageNo != null) {
+        picturePages.add(pageNo);
+      }
+    }
+    if (picturePages.size === 0) return /* @__PURE__ */ new Set();
+    const pageTextLength = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      if (picturePages.has(pageNo)) {
+        pageTextLength.set(
+          pageNo,
+          (pageTextLength.get(pageNo) ?? 0) + text.text.length
+        );
+      }
+    }
+    const imageOnlyPages = /* @__PURE__ */ new Set();
+    for (const pageNo of picturePages) {
+      const textLength = pageTextLength.get(pageNo) ?? 0;
+      if (textLength < IMAGE_PAGE_TEXT_THRESHOLD) {
+        imageOnlyPages.add(pageNo);
+      }
+    }
+    return imageOnlyPages;
+  }
+  /**
+   * Get all pages with substantial text content.
+   * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
+   */
+  getTextPages(doclingDoc) {
+    const pageMap = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      const entry = pageMap.get(pageNo) ?? { textLength: 0, texts: [] };
+      entry.textLength += text.text.length;
+      entry.texts.push(text.text);
+      pageMap.set(pageNo, entry);
+    }
+    return Array.from(pageMap.entries()).filter(([, data]) => data.textLength >= MIN_TEXT_LENGTH).map(([pageNo, data]) => ({
+      pageNo,
+      textLength: data.textLength,
+      texts: data.texts
+    }));
+  }
+  /**
+   * Select sample pages with highest text density.
+   * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
+   * Falls back to all text pages if filtering leaves too few candidates.
+   */
+  selectSamplePages(textPages, totalPages, imageOnlyPages) {
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const filtered = totalPages > 0 ? textPages.filter(
+      (p) => p.pageNo > frontCutoff && p.pageNo <= backCutoff && !imageOnlyPages.has(p.pageNo)
+    ) : textPages;
+    const candidates = filtered.length > 0 ? filtered : textPages;
+    if (candidates !== filtered) {
+      this.log(
+        "warn",
+        `All text pages were filtered out by edge/image exclusion. Falling back to all ${textPages.length} text pages.`
+      );
+    }
+    const sorted = [...candidates].sort((a, b) => b.textLength - a.textLength);
+    const count = Math.min(MAX_SAMPLE_PAGES, sorted.length);
+    return sorted.slice(0, count);
+  }
+  /**
+   * Evaluate sampled pages using Vision LLM
+   */
+  async evaluatePages(pages) {
+    const results = [];
+    for (const page of pages) {
+      const result = await this.evaluateSinglePage(page);
+      results.push(result);
+    }
+    return results;
+  }
+  /**
+   * Evaluate a single page by comparing OCR text against the page image
+   */
+  async evaluateSinglePage(page) {
+    const imagePath = `pages/page_${page.pageNo - 1}.png`;
+    let imageContent;
+    try {
+      imageContent = this.buildImageContent(imagePath);
+    } catch {
+      this.log(
+        "warn",
+        `Failed to load page image for page ${page.pageNo}, marking as not corrupted`
+      );
+      return {
+        isCorrupted: false,
+        corruptedCharCount: 0,
+        totalKcjCharCount: 0,
+        explanation: `Page image not available for page ${page.pageNo}`
+      };
+    }
+    const ocrText = page.texts.join("\n");
+    const messages = [
+      {
+        role: "user",
+        content: [
+          imageContent,
+          {
+            type: "text",
+            text: this.buildUserPrompt(ocrText)
+          }
+        ]
+      }
+    ];
+    const { output } = await this.callVisionLLM(
+      HanjaQualityResponseSchema,
+      messages,
+      `page-${page.pageNo}`
+    );
+    this.log(
+      "info",
+      `Page ${page.pageNo}: corrupted=${output.isCorrupted}, ${output.corruptedCharCount}/${output.totalKcjCharCount} KCJ chars corrupted`
+    );
+    return output;
+  }
+  /**
+   * Aggregate individual page results into final assessment
+   */
+  aggregateResults(totalTextPages, sampledCount, results) {
+    const corruptedCount = results.filter((r) => r.isCorrupted).length;
+    const corruptedRatio = sampledCount > 0 ? corruptedCount / sampledCount : 0;
+    let severity;
+    let needsVlmReparse;
+    if (corruptedRatio >= CORRUPTION_THRESHOLD || corruptedCount >= MIN_SEVERE_CORRUPTED_COUNT) {
+      severity = "severe";
+      needsVlmReparse = true;
+    } else if (corruptedCount > 0) {
+      severity = "minor";
+      needsVlmReparse = false;
+    } else {
+      severity = "none";
+      needsVlmReparse = false;
+    }
+    const reason = corruptedCount === 0 ? "No Hanja character corruption detected" : `${corruptedCount}/${sampledCount} sampled pages have corrupted Hanja characters (ratio: ${corruptedRatio.toFixed(2)})`;
+    return {
+      needsVlmReparse,
+      severity,
+      kcjPageCount: totalTextPages,
+      sampledPageCount: sampledCount,
+      corruptedRatio,
+      reason
+    };
+  }
+  /**
+   * Get page number from a text item
+   */
+  getPageNo(text) {
+    return text.prov?.[0]?.page_no ?? 1;
+  }
+  buildSystemPrompt() {
+    return "";
+  }
+  buildUserPrompt(ocrText) {
+    return `You are evaluating the quality of OCR text extraction for Hanja (\u6F22\u5B57) characters used in Korean archaeological reports.
+
+Look at the page image and find any Hanja (\u6F22\u5B57) characters visible in the original document. Then compare them against the OCR text below.
+
+## Evaluation Criteria
+- Find Hanja (\u6F22\u5B57) characters in the page image first
+- Check if each Hanja character is correctly recognized in the OCR text
+- A character is "corrupted" if a Hanja character in the image was replaced by Korean hangul, garbled text, symbols, or a completely different character in the OCR text
+- Common corruption patterns: Hanja replaced by similar-looking hangul (e.g., \u7C98\u571F \u2192 \uADF8\uC73C), wrong character substitution, partial recognition, question marks or boxes
+- Minor font rendering differences are NOT corruption
+- If no Hanja characters are visible in the page image, report as not corrupted with 0 total characters
+
+## OCR Text to Evaluate
+${ocrText}
+
+Compare the Hanja characters visible in the page image against the OCR text above and assess whether they were correctly recognized.`;
+  }
+};
+
 // src/validators/base-validator.ts
 var BaseValidator = class extends TextLLMComponent {
   /**
@@ -3405,13 +3682,13 @@ var BaseValidator = class extends TextLLMComponent {
 };
 
 // src/validators/toc-content-validator.ts
-import { z as z5 } from "zod";
-var TocContentValidationSchema = z5.object({
-  isValid: z5.boolean().describe("Whether valid main document TOC was found"),
-  confidence: z5.number().min(0).max(1).describe("Confidence score between 0 and 1"),
-  contentType: z5.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
-  extractedTocMarkdown: z5.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
-  reason: z5.string().describe("Brief explanation in English")
+import { z as z6 } from "zod";
+var TocContentValidationSchema = z6.object({
+  isValid: z6.boolean().describe("Whether valid main document TOC was found"),
+  confidence: z6.number().min(0).max(1).describe("Confidence score between 0 and 1"),
+  contentType: z6.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
+  extractedTocMarkdown: z6.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
+  reason: z6.string().describe("Brief explanation in English")
 });
 var TocContentValidator = class extends BaseValidator {
   confidenceThreshold;
@@ -3562,14 +3839,14 @@ ${markdown}`;
 };
 
 // src/validators/caption-validator.ts
-import { z as z6 } from "zod";
-var CaptionValidationItemSchema = z6.object({
-  index: z6.number().int().describe("Index of the caption in the input array"),
-  isValid: z6.boolean().describe("Whether the parsed caption is correct"),
-  reason: z6.string().nullable().describe("Brief explanation if invalid, null if valid")
+import { z as z7 } from "zod";
+var CaptionValidationItemSchema = z7.object({
+  index: z7.number().int().describe("Index of the caption in the input array"),
+  isValid: z7.boolean().describe("Whether the parsed caption is correct"),
+  reason: z7.string().nullable().describe("Brief explanation if invalid, null if valid")
 });
-var CaptionValidationBatchSchema = z6.object({
-  results: z6.array(CaptionValidationItemSchema)
+var CaptionValidationBatchSchema = z7.object({
+  results: z7.array(CaptionValidationItemSchema)
 });
 var CaptionValidator = class extends BaseValidator {
   constructor(logger, model, options, fallbackModel, aggregator) {
@@ -3777,6 +4054,7 @@ var DocumentProcessor = class {
   validatorModel;
   visionTocExtractorModel;
   captionParserModel;
+  hanjaQualitySamplerModel;
   textCleanerBatchSize;
   captionParserBatchSize;
   captionValidatorBatchSize;
@@ -3803,6 +4081,7 @@ var DocumentProcessor = class {
     this.validatorModel = options.validatorModel ?? options.fallbackModel;
     this.visionTocExtractorModel = options.visionTocExtractorModel ?? options.fallbackModel;
     this.captionParserModel = options.captionParserModel ?? options.fallbackModel;
+    this.hanjaQualitySamplerModel = options.hanjaQualitySamplerModel ?? options.fallbackModel;
     this.textCleanerBatchSize = options.textCleanerBatchSize;
     this.captionParserBatchSize = options.captionParserBatchSize;
     this.captionValidatorBatchSize = options.captionValidatorBatchSize;
@@ -3822,6 +4101,36 @@ var DocumentProcessor = class {
       throw error;
     }
   }
+  /**
+   * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
+   *
+   * Samples pages containing KCJ characters and uses Vision LLM to compare
+   * the OCR text against the original page images. Returns an assessment
+   * indicating whether the document should be re-parsed with VLM pipeline.
+   *
+   * @param doclingDoc - Original document extracted from Docling SDK
+   * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
+   * @returns Assessment result with severity and recommendation
+   */
+  async assessHanjaQuality(doclingDoc, outputPath) {
+    this.logger.info(
+      "[DocumentProcessor] Starting Hanja quality assessment..."
+    );
+    const sampler = new HanjaQualitySampler(
+      this.logger,
+      this.hanjaQualitySamplerModel,
+      outputPath,
+      this.maxRetries,
+      this.enableFallbackRetry ? this.fallbackModel : void 0,
+      this.usageAggregator,
+      this.abortSignal
+    );
+    const assessment = await sampler.assess(doclingDoc);
+    this.logger.info(
+      `[DocumentProcessor] Hanja assessment: severity=${assessment.severity}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
   /**
    * Converts DoclingDocument to ProcessedDocument with token usage tracking.
    *
@@ -4155,8 +4464,10 @@ var DocumentProcessor = class {
       );
     }
     const totalPages = Object.keys(doclingDoc.pages).length;
+    const maxTocPageNo = this.extractMaxPageNumber(markdown);
+    const effectiveTotalPages = maxTocPageNo > totalPages ? void 0 : totalPages;
     const tocResult = await this.tocExtractor.extract(markdown, {
-      totalPages
+      totalPages: effectiveTotalPages
     });
     for (const usage of tocResult.usages) {
       this.usageAggregator.track(usage);
@@ -4171,6 +4482,20 @@ var DocumentProcessor = class {
     );
     return tocResult.entries;
   }
+  /**
+   * Extract the maximum page number from TOC markdown
+   *
+   * Parses page numbers from dot-leader patterns (e.g., "..... 175")
+   * to detect compiled volume scenarios where TOC page numbers exceed
+   * the sub-document's page count.
+   */
+  extractMaxPageNumber(markdown) {
+    const matches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
+    if (matches.length === 0) {
+      return 0;
+    }
+    return Math.max(...matches.map((m) => parseInt(m[1], 10)));
+  }
   /**
    * Process resource captions (for images and tables)
    *
@@ -4438,6 +4763,7 @@ export {
   CaptionValidator,
   ChapterConverter,
   DocumentProcessor,
+  HanjaQualitySampler,
   PAGE_NUMBER_PATTERN,
   PagePattern,
   PageRangeParseError,