@heripo/document-processor 0.1.4 → 0.1.6

package/dist/index.js

@@ -1206,7 +1206,8 @@ var TocValidationError = class extends TocExtractError {
 // src/extractors/toc-validator.ts
 var DEFAULT_OPTIONS = {
   totalPages: Infinity,
-  maxTitleLength: 200
+  maxTitleLength: 200,
+  maxFirstEntryPageRatio: 0.3
 };
 var TocValidator = class {
   options;
@@ -1227,6 +1228,7 @@ var TocValidator = class {
   validate(entries) {
     this.issues = [];
     this.validateEntries(entries, "", null, /* @__PURE__ */ new Set());
+    this.validateFirstEntryPagePosition(entries);
     const errorCount = this.issues.length;
     return {
       valid: errorCount === 0,
@@ -1243,8 +1245,12 @@ var TocValidator = class {
   validateOrThrow(entries) {
     const result = this.validate(entries);
     if (!result.valid) {
+      const details = result.issues.map(
+        (issue) => `  [${issue.code}] ${issue.message} (path: ${issue.path}, entry: "${issue.entry.title}" page ${issue.entry.pageNo})`
+      ).join("\n");
       throw new TocValidationError(
-        `TOC validation failed with ${result.errorCount} error(s)`,
+        `TOC validation failed with ${result.errorCount} error(s):
+${details}`,
         result
       );
     }
@@ -1359,6 +1365,33 @@ var TocValidator = class {
       });
     }
   }
+  /**
+   * V007: Validate first entry page position (completeness check)
+   *
+   * If the first level-1 entry starts too late in the document,
+   * earlier entries might be missing from the TOC.
+   */
+  validateFirstEntryPagePosition(entries) {
+    if (entries.length === 0) {
+      return;
+    }
+    if (!isFinite(this.options.totalPages)) {
+      return;
+    }
+    const firstEntry = entries[0];
+    const threshold = Math.max(
+      50,
+      Math.floor(this.options.totalPages * this.options.maxFirstEntryPageRatio)
+    );
+    if (firstEntry.pageNo > threshold) {
+      this.addIssue({
+        code: "V007",
+        message: `TOC may be incomplete - first entry starts at page ${firstEntry.pageNo}, expected within first ${threshold} pages. Earlier entries might be missing.`,
+        path: "[0]",
+        entry: firstEntry
+      });
+    }
+  }
   /**
    * Add issue to the list
    */
@@ -1577,22 +1610,42 @@ var TocFinder = class {
     return numberCount > 0 && numberCount / (num_rows - 1) > 0.5;
   }
   /**
-   * Expand TOC area to consecutive pages
+   * Expand TOC area to consecutive pages (both backward and forward)
    */
   expandToConsecutivePages(initial, doc) {
     const itemRefs = [...initial.itemRefs];
+    const seenRefs = new Set(itemRefs);
+    let startPage = initial.startPage;
     let endPage = initial.endPage;
+    for (let pageNo = initial.startPage - 1; pageNo >= 1; pageNo--) {
+      const continuationItems = this.findContinuationOnPage(doc, pageNo);
+      if (continuationItems.length === 0) {
+        break;
+      }
+      const newItems = continuationItems.filter((ref) => !seenRefs.has(ref));
+      for (const ref of newItems) {
+        seenRefs.add(ref);
+      }
+      itemRefs.unshift(...newItems);
+      startPage = pageNo;
+      this.logger.info(`[TocFinder] Expanded TOC backward to page ${pageNo}`);
+    }
     for (let pageNo = initial.endPage + 1; pageNo <= this.maxSearchPages; pageNo++) {
       const continuationItems = this.findContinuationOnPage(doc, pageNo);
       if (continuationItems.length === 0) {
         break;
       }
-      itemRefs.push(...continuationItems);
+      const newItems = continuationItems.filter((ref) => !seenRefs.has(ref));
+      for (const ref of newItems) {
+        seenRefs.add(ref);
+      }
+      itemRefs.push(...newItems);
       endPage = pageNo;
+      this.logger.info(`[TocFinder] Expanded TOC forward to page ${pageNo}`);
     }
     return {
       itemRefs,
-      startPage: initial.startPage,
+      startPage,
       endPage
     };
   }
@@ -1810,12 +1863,22 @@ var TextLLMComponent = class extends BaseLLMComponent {
 };
 
 // src/extractors/toc-extractor.ts
+var MAX_VALIDATION_RETRIES = 3;
+var VALIDATION_CODE_DESCRIPTIONS = {
+  V001: "Page numbers must be in non-decreasing order within the same level. A decrease usually means a hierarchy or page number error.",
+  V002: "Page number is out of valid range (must be >= 1 and <= total pages).",
+  V003: "Title is empty or contains only whitespace.",
+  V004: "Title exceeds the maximum allowed length.",
+  V005: "Child page number is before parent page number. Children must start on or after the parent page.",
+  V006: "Duplicate entry detected (same title and page number).",
+  V007: "First TOC entry starts too late in the document. Earlier entries may be missing."
+};
 var TocEntrySchema = z.lazy(
   () => z.object({
     title: z.string().describe("Chapter or section title"),
     level: z.number().int().min(1).describe("Hierarchy depth (1 = top level)"),
     pageNo: z.number().int().min(1).describe("Starting page number"),
-    children: z.array(TocEntrySchema).optional().describe("Child sections")
+    children: z.array(TocEntrySchema).describe("Child sections (use empty array [] if none)")
   })
 );
 var TocResponseSchema = z.object({
@@ -1838,12 +1901,15 @@ var TocExtractor = class extends TextLLMComponent {
   /**
    * Extract TOC structure from Markdown
    *
+   * When validation fails, retries with correction feedback up to MAX_VALIDATION_RETRIES times.
+   *
    * @param markdown - Markdown representation of TOC area
-   * @returns Object with entries array and token usage information
+   * @param validationOverrides - Optional overrides for validation options (merged with constructor options)
+   * @returns Object with entries array and token usage array (initial extraction + any corrections)
    * @throws {TocParseError} When LLM fails to parse structure
-   * @throws {TocValidationError} When validation fails
+   * @throws {TocValidationError} When validation fails after all retries
    */
-  async extract(markdown) {
+  async extract(markdown, validationOverrides) {
     this.log("info", `Starting TOC extraction (${markdown.length} chars)`);
     if (!markdown.trim()) {
       this.log("error", "Cannot extract TOC from empty markdown content");
@@ -1858,18 +1924,52 @@ var TocExtractor = class extends TextLLMComponent {
         this.buildUserPrompt(markdown),
         "extraction"
       );
-      const entries = this.normalizeEntries(result.output.entries);
+      const usages = [result.usage];
+      let entries = this.normalizeEntries(result.output.entries);
       if (!this.skipValidation) {
-        this.validateEntries(entries);
+        let validationError = this.tryValidateEntries(
+          entries,
+          validationOverrides
+        );
+        for (let attempt = 1; attempt <= MAX_VALIDATION_RETRIES && validationError !== null; attempt++) {
+          this.log(
+            "warn",
+            `Validation failed (attempt ${attempt}/${MAX_VALIDATION_RETRIES}), retrying with correction feedback`
+          );
+          const correctionPrompt = this.buildCorrectionPrompt(
+            markdown,
+            entries,
+            validationError.validationResult.issues
+          );
+          const correctionResult = await this.callTextLLM(
+            TocResponseSchema,
+            this.buildSystemPrompt(),
+            correctionPrompt,
+            `correction-${attempt}`
+          );
+          usages.push(correctionResult.usage);
+          entries = this.normalizeEntries(correctionResult.output.entries);
+          validationError = this.tryValidateEntries(
+            entries,
+            validationOverrides
+          );
+        }
+        if (validationError !== null) {
+          this.log(
+            "error",
+            `Validation failed after ${MAX_VALIDATION_RETRIES} retries:
+${validationError.getSummary()}`
+          );
+          throw validationError;
+        }
       }
       this.log(
         "info",
-        `Extraction completed: ${entries.length} top-level entries`
+        `Extraction completed: ${entries.length} top-level entries (${usages.length} LLM call(s))`
       );
-      return { entries, usage: result.usage };
+      return { entries, usages };
     } catch (error) {
       if (error instanceof TocValidationError) {
-        this.log("error", `Validation failed: ${error.message}`);
         throw error;
       }
       const message = error instanceof Error ? error.message : String(error);
@@ -1880,16 +1980,69 @@ var TocExtractor = class extends TextLLMComponent {
     }
   }
   /**
-   * Validate extracted entries
+   * Validate extracted entries and return error or null
    *
-   * @throws {TocValidationError} When validation fails
+   * Unlike validateOrThrow, this returns the error instead of throwing,
+   * allowing the retry loop to handle it.
+   *
+   * @returns TocValidationError if validation fails, null if valid
    */
-  validateEntries(entries) {
+  tryValidateEntries(entries, overrides) {
     if (entries.length === 0) {
-      return;
+      return null;
     }
-    const validator = new TocValidator(this.validationOptions);
-    validator.validateOrThrow(entries);
+    const options = { ...this.validationOptions, ...overrides };
+    const validator = new TocValidator(options);
+    const result = validator.validate(entries);
+    if (!result.valid) {
+      const details = result.issues.map(
+        (issue) => `  [${issue.code}] ${issue.message} (path: ${issue.path}, entry: "${issue.entry.title}" page ${issue.entry.pageNo})`
+      ).join("\n");
+      return new TocValidationError(
+        `TOC validation failed with ${result.errorCount} error(s):
+${details}`,
+        result
+      );
+    }
+    return null;
+  }
+  /**
+   * Build correction prompt with validation error feedback
+   *
+   * Includes the original markdown, previous extraction result,
+   * validation errors, and guidance for fixing common mistakes.
+   */
+  buildCorrectionPrompt(markdown, previousEntries, issues) {
+    const errorLines = issues.map((issue) => {
+      const desc = VALIDATION_CODE_DESCRIPTIONS[issue.code] ?? "Unknown validation error.";
+      return `- [${issue.code}] ${issue.message}
+  Path: ${issue.path}
+  Entry: "${issue.entry.title}" (page ${issue.entry.pageNo})
+  Rule: ${desc}`;
+    });
+    return `Your previous TOC extraction had validation errors. Please fix them and re-extract.
+
+## Validation Errors
+
+${errorLines.join("\n\n")}
+
+## Common Mistakes to Avoid
+
+1. **Hierarchy confusion**: Entries with the same numbering prefix (e.g., "4)") can belong to different hierarchy levels depending on context. Use indentation and surrounding entries to determine the correct parent-child relationship.
+2. **Page number misread**: Carefully distinguish Roman numerals (VI=6) from Arabic numerals. "VI. \uACE0\uCC30" at page 277 is NOT "V. \uACE0\uCC30" at page 27.
+3. **Page order**: Within the same parent, sibling entries must have non-decreasing page numbers. If a page number decreases, the entry likely belongs to a different hierarchy level.
+
+## Original Markdown
+
+${markdown}
+
+## Your Previous Extraction (with errors)
+
+${JSON.stringify(previousEntries, null, 2)}
+
+## Instructions
+
+Re-extract the TOC structure from the original markdown above. Fix all validation errors listed above. Return the corrected entries.`;
   }
   /**
    * Build system prompt for TOC extraction
@@ -1907,11 +2060,12 @@ var TocExtractor = class extends TextLLMComponent {
    - Level 3: Subsections (e.g., "1.1.1", "a.", "(1)")
    - Use indentation and numbering patterns to infer level
 
-3. **Page Number**: Extract the page number from each entry. Convert Roman numerals to Arabic numerals if present (e.g., "iv" \u2192 4).
+3. **Page Number**: Extract the page number from each entry. Use only Arabic numerals for page numbers.
 
 4. **Children**: Nest child entries under parent entries based on their hierarchy level.
 
-5. **IMPORTANT - Extract Main TOC Only**: Only extract the main document table of contents. EXCLUDE the following supplementary indices:
+5. **IMPORTANT - Extract Main TOC Only**: Only extract the main document table of contents. EXCLUDE the following:
+   - **Front matter with Roman numeral pages**: Entries whose page numbers are Roman numerals (i, ii, xxi, etc.) such as \uC77C\uB7EC\uB450\uAE30, \uBC1C\uAC04\uC0AC, \uC11C\uBB38, \uBC94\uB840, Preface, Foreword, Editorial Notes. These use a separate page numbering system and are not part of the main content.
    - Photo/image indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28, \uD654\uBCF4 \uBAA9\uCC28, Photo Index, List of Photos, List of Figures)
    - Drawing/diagram indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28, \uC0BD\uB3C4 \uBAA9\uCC28, Drawing Index, List of Drawings)
    - Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28, Table Index, List of Tables)
@@ -1938,11 +2092,11 @@ Output:
       "level": 1,
       "pageNo": 1,
       "children": [
-        { "title": "1. \uC5F0\uAD6C \uBC30\uACBD", "level": 2, "pageNo": 3 },
-        { "title": "2. \uC5F0\uAD6C \uBAA9\uC801", "level": 2, "pageNo": 5 }
+        { "title": "1. \uC5F0\uAD6C \uBC30\uACBD", "level": 2, "pageNo": 3, "children": [] },
+        { "title": "2. \uC5F0\uAD6C \uBAA9\uC801", "level": 2, "pageNo": 5, "children": [] }
       ]
     },
-    { "title": "\uC81C2\uC7A5 \uBC29\uBC95\uB860", "level": 1, "pageNo": 10 }
+    { "title": "\uC81C2\uC7A5 \uBC29\uBC95\uB860", "level": 1, "pageNo": 10, "children": [] }
   ]
 }`;
   }
@@ -2585,7 +2739,7 @@ var PagePattern = /* @__PURE__ */ ((PagePattern2) => {
 var PageRangeParser = class extends VisionLLMComponent {
   // Configuration constants
   SAMPLE_SIZE = 3;
-  MAX_PATTERN_RETRIES = 6;
+  MAX_PATTERN_RETRIES = 19;
   SIZE_TOLERANCE = 5;
   constructor(logger, model, outputPath, maxRetries = 3, fallbackModel, aggregator, abortSignal) {
     super(
@@ -3192,6 +3346,283 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
   }
 };
 
+// src/samplers/hanja-quality-sampler.ts
+import { z as z5 } from "zod";
+var MIN_TEXT_LENGTH = 100;
+var MAX_SAMPLE_PAGES = 10;
+var EDGE_TRIM_RATIO = 0.1;
+var IMAGE_PAGE_TEXT_THRESHOLD = 50;
+var CORRUPTION_THRESHOLD = 0.5;
+var MIN_SEVERE_CORRUPTED_COUNT = 3;
+var HanjaQualityResponseSchema = z5.object({
+  isCorrupted: z5.boolean().describe(
+    "Whether the KCJ (Hanja) characters in the OCR text are significantly corrupted compared to the page image"
+  ),
+  corruptedCharCount: z5.number().int().nonnegative().describe(
+    "Approximate number of corrupted or incorrectly recognized KCJ characters"
+  ),
+  totalKcjCharCount: z5.number().int().nonnegative().describe("Total number of KCJ characters found in the OCR text"),
+  explanation: z5.string().describe("Brief explanation of the assessment result")
+});
+var HanjaQualitySampler = class extends VisionLLMComponent {
+  constructor(logger, model, outputPath, maxRetries, fallbackModel, aggregator, abortSignal) {
+    super(
+      logger,
+      model,
+      "HanjaQualitySampler",
+      outputPath,
+      { maxRetries, abortSignal },
+      fallbackModel,
+      aggregator
+    );
+  }
+  /**
+   * Assess the quality of KCJ character recognition in the document
+   *
+   * @param doclingDoc - DoclingDocument to assess
+   * @returns Assessment result indicating whether VLM re-parse is needed
+   */
+  async assess(doclingDoc) {
+    const totalPages = Object.keys(doclingDoc.pages).length;
+    this.log("info", "Starting KCJ quality assessment...");
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const imageOnlyPages = this.getImageOnlyPages(doclingDoc);
+    this.log(
+      "info",
+      `Total pages: ${totalPages}, eligible range: (${frontCutoff}, ${backCutoff}], image-only pages excluded: ${imageOnlyPages.size}`
+    );
+    const textPages = this.getTextPages(doclingDoc);
+    if (textPages.length === 0) {
+      this.log("info", "No text pages found for assessment");
+      return {
+        needsVlmReparse: false,
+        severity: "none",
+        kcjPageCount: 0,
+        sampledPageCount: 0,
+        corruptedRatio: 0,
+        reason: "No text pages found for assessment"
+      };
+    }
+    this.log(
+      "info",
+      `Found ${textPages.length} text pages (min length: ${MIN_TEXT_LENGTH})`
+    );
+    const sampled = this.selectSamplePages(
+      textPages,
+      totalPages,
+      imageOnlyPages
+    );
+    this.log(
+      "info",
+      `Sampling ${sampled.length} pages: [${sampled.map((p) => p.pageNo).join(", ")}]`
+    );
+    const results = await this.evaluatePages(sampled);
+    const assessment = this.aggregateResults(
+      textPages.length,
+      sampled.length,
+      results
+    );
+    this.log(
+      "info",
+      `Assessment complete: severity=${assessment.severity}, corrupted=${assessment.corruptedRatio.toFixed(2)}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
+  /**
+   * Get the eligible page range after trimming front/back edges.
+   * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
+   */
+  getEligiblePageRange(totalPages) {
+    const trimCount = Math.ceil(totalPages * EDGE_TRIM_RATIO);
+    return {
+      frontCutoff: trimCount,
+      backCutoff: totalPages - trimCount
+    };
+  }
+  /**
+   * Get page numbers that are "image-only" (contain pictures but minimal text).
+   * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
+   */
+  getImageOnlyPages(doclingDoc) {
+    const picturePages = /* @__PURE__ */ new Set();
+    for (const picture of doclingDoc.pictures) {
+      const pageNo = picture.prov?.[0]?.page_no;
+      if (pageNo != null) {
+        picturePages.add(pageNo);
+      }
+    }
+    if (picturePages.size === 0) return /* @__PURE__ */ new Set();
+    const pageTextLength = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      if (picturePages.has(pageNo)) {
+        pageTextLength.set(
+          pageNo,
+          (pageTextLength.get(pageNo) ?? 0) + text.text.length
+        );
+      }
+    }
+    const imageOnlyPages = /* @__PURE__ */ new Set();
+    for (const pageNo of picturePages) {
+      const textLength = pageTextLength.get(pageNo) ?? 0;
+      if (textLength < IMAGE_PAGE_TEXT_THRESHOLD) {
+        imageOnlyPages.add(pageNo);
+      }
+    }
+    return imageOnlyPages;
+  }
+  /**
+   * Get all pages with substantial text content.
+   * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
+   */
+  getTextPages(doclingDoc) {
+    const pageMap = /* @__PURE__ */ new Map();
+    for (const text of doclingDoc.texts) {
+      const pageNo = this.getPageNo(text);
+      const entry = pageMap.get(pageNo) ?? { textLength: 0, texts: [] };
+      entry.textLength += text.text.length;
+      entry.texts.push(text.text);
+      pageMap.set(pageNo, entry);
+    }
+    return Array.from(pageMap.entries()).filter(([, data]) => data.textLength >= MIN_TEXT_LENGTH).map(([pageNo, data]) => ({
+      pageNo,
+      textLength: data.textLength,
+      texts: data.texts
+    }));
+  }
+  /**
+   * Select sample pages with highest text density.
+   * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
+   * Falls back to all text pages if filtering leaves too few candidates.
+   */
+  selectSamplePages(textPages, totalPages, imageOnlyPages) {
+    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
+    const filtered = totalPages > 0 ? textPages.filter(
+      (p) => p.pageNo > frontCutoff && p.pageNo <= backCutoff && !imageOnlyPages.has(p.pageNo)
+    ) : textPages;
+    const candidates = filtered.length > 0 ? filtered : textPages;
+    if (candidates !== filtered) {
+      this.log(
+        "warn",
+        `All text pages were filtered out by edge/image exclusion. Falling back to all ${textPages.length} text pages.`
+      );
+    }
+    const sorted = [...candidates].sort((a, b) => b.textLength - a.textLength);
+    const count = Math.min(MAX_SAMPLE_PAGES, sorted.length);
+    return sorted.slice(0, count);
+  }
+  /**
+   * Evaluate sampled pages using Vision LLM
+   */
+  async evaluatePages(pages) {
+    const results = [];
+    for (const page of pages) {
+      const result = await this.evaluateSinglePage(page);
+      results.push(result);
+    }
+    return results;
+  }
+  /**
+   * Evaluate a single page by comparing OCR text against the page image
+   */
+  async evaluateSinglePage(page) {
+    const imagePath = `pages/page_${page.pageNo - 1}.png`;
+    let imageContent;
+    try {
+      imageContent = this.buildImageContent(imagePath);
+    } catch {
+      this.log(
+        "warn",
+        `Failed to load page image for page ${page.pageNo}, marking as not corrupted`
+      );
+      return {
+        isCorrupted: false,
+        corruptedCharCount: 0,
+        totalKcjCharCount: 0,
+        explanation: `Page image not available for page ${page.pageNo}`
+      };
+    }
+    const ocrText = page.texts.join("\n");
+    const messages = [
+      {
+        role: "user",
+        content: [
+          imageContent,
+          {
+            type: "text",
+            text: this.buildUserPrompt(ocrText)
+          }
+        ]
+      }
+    ];
+    const { output } = await this.callVisionLLM(
+      HanjaQualityResponseSchema,
+      messages,
+      `page-${page.pageNo}`
+    );
+    this.log(
+      "info",
+      `Page ${page.pageNo}: corrupted=${output.isCorrupted}, ${output.corruptedCharCount}/${output.totalKcjCharCount} KCJ chars corrupted`
+    );
+    return output;
+  }
+  /**
+   * Aggregate individual page results into final assessment
+   */
+  aggregateResults(totalTextPages, sampledCount, results) {
+    const corruptedCount = results.filter((r) => r.isCorrupted).length;
+    const corruptedRatio = sampledCount > 0 ? corruptedCount / sampledCount : 0;
+    let severity;
+    let needsVlmReparse;
+    if (corruptedRatio >= CORRUPTION_THRESHOLD || corruptedCount >= MIN_SEVERE_CORRUPTED_COUNT) {
+      severity = "severe";
+      needsVlmReparse = true;
+    } else if (corruptedCount > 0) {
+      severity = "minor";
+      needsVlmReparse = false;
+    } else {
+      severity = "none";
+      needsVlmReparse = false;
+    }
+    const reason = corruptedCount === 0 ? "No Hanja character corruption detected" : `${corruptedCount}/${sampledCount} sampled pages have corrupted Hanja characters (ratio: ${corruptedRatio.toFixed(2)})`;
+    return {
+      needsVlmReparse,
+      severity,
+      kcjPageCount: totalTextPages,
+      sampledPageCount: sampledCount,
+      corruptedRatio,
+      reason
+    };
+  }
+  /**
+   * Get page number from a text item
+   */
+  getPageNo(text) {
+    return text.prov?.[0]?.page_no ?? 1;
+  }
+  buildSystemPrompt() {
+    return "";
+  }
+  buildUserPrompt(ocrText) {
+    return `You are evaluating the quality of OCR text extraction for Hanja (\u6F22\u5B57) characters used in Korean archaeological reports.
+
+Look at the page image and find any Hanja (\u6F22\u5B57) characters visible in the original document. Then compare them against the OCR text below.
+
+## Evaluation Criteria
+- Find Hanja (\u6F22\u5B57) characters in the page image first
+- Check if each Hanja character is correctly recognized in the OCR text
+- A character is "corrupted" if a Hanja character in the image was replaced by Korean hangul, garbled text, symbols, or a completely different character in the OCR text
+- Common corruption patterns: Hanja replaced by similar-looking hangul (e.g., \u7C98\u571F \u2192 \uADF8\uC73C), wrong character substitution, partial recognition, question marks or boxes
+- Minor font rendering differences are NOT corruption
+- If no Hanja characters are visible in the page image, report as not corrupted with 0 total characters
+
+## OCR Text to Evaluate
+${ocrText}
+
+Compare the Hanja characters visible in the page image against the OCR text above and assess whether they were correctly recognized.`;
+  }
+};
+
 // src/validators/base-validator.ts
 var BaseValidator = class extends TextLLMComponent {
   /**
@@ -3251,13 +3682,13 @@ var BaseValidator = class extends TextLLMComponent {
 };
 
 // src/validators/toc-content-validator.ts
-import { z as z5 } from "zod";
-var TocContentValidationSchema = z5.object({
-  isValid: z5.boolean().describe("Whether valid main document TOC was found"),
-  confidence: z5.number().min(0).max(1).describe("Confidence score between 0 and 1"),
-  contentType: z5.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
-  extractedTocMarkdown: z5.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
-  reason: z5.string().describe("Brief explanation in English")
+import { z as z6 } from "zod";
+var TocContentValidationSchema = z6.object({
+  isValid: z6.boolean().describe("Whether valid main document TOC was found"),
+  confidence: z6.number().min(0).max(1).describe("Confidence score between 0 and 1"),
+  contentType: z6.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
+  extractedTocMarkdown: z6.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
+  reason: z6.string().describe("Brief explanation in English")
 });
 var TocContentValidator = class extends BaseValidator {
   confidenceThreshold;
@@ -3408,14 +3839,14 @@ ${markdown}`;
 };
 
 // src/validators/caption-validator.ts
-import { z as z6 } from "zod";
-var CaptionValidationItemSchema = z6.object({
-  index: z6.number().int().describe("Index of the caption in the input array"),
-  isValid: z6.boolean().describe("Whether the parsed caption is correct"),
-  reason: z6.string().nullable().describe("Brief explanation if invalid, null if valid")
+import { z as z7 } from "zod";
+var CaptionValidationItemSchema = z7.object({
+  index: z7.number().int().describe("Index of the caption in the input array"),
+  isValid: z7.boolean().describe("Whether the parsed caption is correct"),
+  reason: z7.string().nullable().describe("Brief explanation if invalid, null if valid")
 });
-var CaptionValidationBatchSchema = z6.object({
-  results: z6.array(CaptionValidationItemSchema)
+var CaptionValidationBatchSchema = z7.object({
+  results: z7.array(CaptionValidationItemSchema)
 });
 var CaptionValidator = class extends BaseValidator {
   constructor(logger, model, options, fallbackModel, aggregator) {
@@ -3623,6 +4054,7 @@ var DocumentProcessor = class {
   validatorModel;
   visionTocExtractorModel;
   captionParserModel;
+  hanjaQualitySamplerModel;
   textCleanerBatchSize;
   captionParserBatchSize;
   captionValidatorBatchSize;
@@ -3649,6 +4081,7 @@ var DocumentProcessor = class {
     this.validatorModel = options.validatorModel ?? options.fallbackModel;
     this.visionTocExtractorModel = options.visionTocExtractorModel ?? options.fallbackModel;
     this.captionParserModel = options.captionParserModel ?? options.fallbackModel;
+    this.hanjaQualitySamplerModel = options.hanjaQualitySamplerModel ?? options.fallbackModel;
     this.textCleanerBatchSize = options.textCleanerBatchSize;
     this.captionParserBatchSize = options.captionParserBatchSize;
     this.captionValidatorBatchSize = options.captionValidatorBatchSize;
@@ -3668,6 +4101,36 @@ var DocumentProcessor = class {
       throw error;
     }
   }
+  /**
+   * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
+   *
+   * Samples pages containing KCJ characters and uses Vision LLM to compare
+   * the OCR text against the original page images. Returns an assessment
+   * indicating whether the document should be re-parsed with VLM pipeline.
+   *
+   * @param doclingDoc - Original document extracted from Docling SDK
+   * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
+   * @returns Assessment result with severity and recommendation
+   */
+  async assessHanjaQuality(doclingDoc, outputPath) {
+    this.logger.info(
+      "[DocumentProcessor] Starting Hanja quality assessment..."
+    );
+    const sampler = new HanjaQualitySampler(
+      this.logger,
+      this.hanjaQualitySamplerModel,
+      outputPath,
+      this.maxRetries,
+      this.enableFallbackRetry ? this.fallbackModel : void 0,
+      this.usageAggregator,
+      this.abortSignal
+    );
+    const assessment = await sampler.assess(doclingDoc);
+    this.logger.info(
+      `[DocumentProcessor] Hanja assessment: severity=${assessment.severity}, needsVlmReparse=${assessment.needsVlmReparse}`
+    );
+    return assessment;
+  }
   /**
    * Converts DoclingDocument to ProcessedDocument with token usage tracking.
    *
@@ -3985,8 +4448,8 @@ var DocumentProcessor = class {
     }
     if (!markdown) {
       this.logger.info("[DocumentProcessor] Using vision fallback for TOC");
-      const totalPages = Object.keys(doclingDoc.pages).length;
-      markdown = await this.visionTocExtractor.extract(totalPages);
+      const totalPages2 = Object.keys(doclingDoc.pages).length;
+      markdown = await this.visionTocExtractor.extract(totalPages2);
       if (!markdown) {
         const reason = "Both rule-based search and vision fallback failed to locate TOC";
         this.logger.error(
@@ -4000,8 +4463,13 @@ var DocumentProcessor = class {
         `[DocumentProcessor] Vision extracted TOC markdown (${markdown.length} chars)`
       );
     }
-    const tocResult = await this.tocExtractor.extract(markdown);
-    this.usageAggregator.track(tocResult.usage);
+    const totalPages = Object.keys(doclingDoc.pages).length;
+    const tocResult = await this.tocExtractor.extract(markdown, {
+      totalPages
+    });
+    for (const usage of tocResult.usages) {
+      this.usageAggregator.track(usage);
+    }
     if (tocResult.entries.length === 0) {
       const reason = "TOC area was detected but LLM could not extract any structured entries";
       this.logger.error(`[DocumentProcessor] TOC extraction failed: ${reason}`);
@@ -4279,6 +4747,7 @@ export {
   CaptionValidator,
   ChapterConverter,
   DocumentProcessor,
+  HanjaQualitySampler,
   PAGE_NUMBER_PATTERN,
   PagePattern,
   PageRangeParseError,