@heripo/document-processor 0.1.7 → 0.1.9

package/dist/index.cjs

@@ -39,7 +39,6 @@ __export(index_exports, {
   CaptionValidator: () => CaptionValidator,
   ChapterConverter: () => ChapterConverter,
   DocumentProcessor: () => DocumentProcessor,
-  HanjaQualitySampler: () => HanjaQualitySampler,
   PAGE_NUMBER_PATTERN: () => PAGE_NUMBER_PATTERN,
   PagePattern: () => PagePattern,
   PageRangeParseError: () => PageRangeParseError,
@@ -136,6 +135,15 @@ var BatchProcessor = class {
     return results.flat();
   }
 };
+function detectProvider(model) {
+  const providerId = model.provider;
+  if (!providerId || typeof providerId !== "string") return "unknown";
+  if (providerId.includes("openai")) return "openai";
+  if (providerId.includes("google")) return "google";
+  if (providerId.includes("anthropic")) return "anthropic";
+  if (providerId.includes("together")) return "togetherai";
+  return "unknown";
+}
 var LLMCaller = class {
   /**
    * Extract model name from LanguageModel object
@@ -164,10 +172,95 @@ var LLMCaller = class {
       totalTokens: response.usage?.totalTokens ?? 0
     };
   }
+  /**
+   * Maximum number of retries when structured output generation fails.
+   * Total attempts = MAX_STRUCTURED_OUTPUT_RETRIES + 1.
+   *
+   * Applied to both:
+   * - `Output.object()` path: retries on NoObjectGeneratedError (schema mismatch)
+   * - Tool call path: retries when model does not produce a tool call
+   */
+  static MAX_STRUCTURED_OUTPUT_RETRIES = 10;
+  /**
+   * Generate structured output via forced tool call.
+   *
+   * Used for providers (Together AI, unknown) that do not reliably support
+   * `Output.object()`. Forces the model to call a tool whose inputSchema
+   * is the target Zod schema, then extracts the parsed input.
+   *
+   * Retries up to MAX_STRUCTURED_OUTPUT_RETRIES times when the model does not
+   * produce a tool call, for a total of MAX_STRUCTURED_OUTPUT_RETRIES + 1 attempts.
+   *
+   * @throws NoObjectGeneratedError when all attempts fail to produce a tool call
+   */
+  static async generateViaToolCall(model, schema, promptParams) {
+    const submitTool = (0, import_ai.tool)({
+      description: "Submit the structured result",
+      inputSchema: schema
+    });
+    let lastResult;
+    for (let attempt = 0; attempt <= this.MAX_STRUCTURED_OUTPUT_RETRIES; attempt++) {
+      lastResult = await (0, import_ai.generateText)({
+        ...promptParams,
+        model,
+        tools: { submitResult: submitTool },
+        toolChoice: { type: "tool", toolName: "submitResult" },
+        stopWhen: (0, import_ai.hasToolCall)("submitResult")
+      });
+      const toolCall = lastResult.toolCalls?.[0];
+      if (toolCall) {
+        return {
+          output: toolCall.input,
+          usage: lastResult.usage
+        };
+      }
+    }
+    throw new import_ai.NoObjectGeneratedError({
+      message: "Model did not produce a tool call for structured output",
+      text: lastResult.text ?? "",
+      response: lastResult.response,
+      usage: lastResult.usage,
+      finishReason: lastResult.finishReason
+    });
+  }
+  /**
+   * Generate structured output with provider-aware strategy.
+   *
+   * Strategy per provider:
+   * - OpenAI / Anthropic / Google Gemini: `Output.object()` with schema retry
+   * - Together AI / unknown: forced tool call pattern
+   *
+   * Retries up to MAX_STRUCTURED_OUTPUT_RETRIES times on NoObjectGeneratedError
+   * (schema mismatch), re-throwing the last error if all attempts fail.
+   */
+  static async generateStructuredOutput(model, schema, promptParams) {
+    const providerType = detectProvider(model);
+    if (providerType === "togetherai" || providerType === "unknown") {
+      return this.generateViaToolCall(model, schema, promptParams);
+    }
+    let lastError;
+    for (let attempt = 0; attempt <= this.MAX_STRUCTURED_OUTPUT_RETRIES; attempt++) {
+      try {
+        return await (0, import_ai.generateText)({
+          model,
+          output: import_ai.Output.object({ schema }),
+          ...promptParams
+        });
+      } catch (error) {
+        if (import_ai.NoObjectGeneratedError.isInstance(error)) {
+          lastError = error;
+          continue;
+        }
+        throw error;
+      }
+    }
+    throw lastError;
+  }
   /**
    * Execute LLM call with fallback support
    *
    * Common execution logic for both text and vision calls.
+   * Logs additional details when NoObjectGeneratedError occurs.
    */
   static async executeWithFallback(config, generateFn) {
     const primaryModelName = this.extractModelName(config.primaryModel);
@@ -202,6 +295,8 @@ var LLMCaller = class {
    * 2. If all fail and fallbackModel provided, try fallback up to maxRetries times
    * 3. Throw error if all attempts exhausted
    *
+   * Provider-aware strategy is automatically applied based on the model's provider field.
+   *
    * @template TOutput - Output type from schema validation
    * @param config - LLM call configuration
    * @returns Result with parsed object and usage information
@@ -210,11 +305,7 @@ var LLMCaller = class {
   static async call(config) {
     return this.executeWithFallback(
       config,
-      (model) => (0, import_ai.generateText)({
-        model,
-        output: import_ai.Output.object({
-          schema: config.schema
-        }),
+      (model) => this.generateStructuredOutput(model, config.schema, {
         system: config.systemPrompt,
         prompt: config.userPrompt,
         temperature: config.temperature,
@@ -227,6 +318,7 @@ var LLMCaller = class {
    * Call LLM for vision tasks with message format support
    *
    * Same retry and fallback logic as call(), but using message format instead of system/user prompts.
+   * Provider-aware strategy is automatically applied based on the model's provider field.
    *
    * @template TOutput - Output type from schema validation
    * @param config - LLM vision call configuration
@@ -236,11 +328,7 @@ var LLMCaller = class {
   static async callVision(config) {
     return this.executeWithFallback(
       config,
-      (model) => (0, import_ai.generateText)({
-        model,
-        output: import_ai.Output.object({
-          schema: config.schema
-        }),
+      (model) => this.generateStructuredOutput(model, config.schema, {
         messages: config.messages,
         temperature: config.temperature,
         maxRetries: config.maxRetries,
@@ -1496,6 +1584,20 @@ var CONTINUATION_MARKERS = [
   "continued"
 ];
 var PAGE_NUMBER_PATTERN = /\.{2,}\s*\d+\s*$|…+\s*\d+\s*$|\s+\d+\s*$/;
+var RESOURCE_INDEX_PATTERNS = [
+  /^\[도면\s*\d+\]/,
+  /^\[사진\s*\d+\]/,
+  /^\[표\s*\d+\]/,
+  /^\[그림\s*\d+\]/,
+  /^도면\s*\d+[.:)\s]/,
+  /^사진\s*\d+[.:)\s]/,
+  /^표\s*\d+[.:)\s]/,
+  /^Fig\.\s*\d+/i,
+  /^Photo\.?\s*\d+/i,
+  /^Plate\s*\d+/i,
+  /^Map\s*\d+/i,
+  /^圖\s*\d+/
+];
 var TocFinder = class {
   constructor(logger, refResolver, options) {
     this.logger = logger;
@@ -1556,6 +1658,10 @@ var TocFinder = class {
       if (result) {
         return this.expandToConsecutivePages(result, doc);
       }
+      const siblingResult = this.findSiblingTocItem(doc, text.self_ref, pageNo);
+      if (siblingResult) {
+        return this.expandToConsecutivePages(siblingResult, doc);
+      }
     }
     return null;
   }
@@ -1587,7 +1693,10 @@ var TocFinder = class {
         continue;
       }
       if (this.isTableTocLike(table)) {
-        const score = this.calculateTableScore(table, pageNo);
+        let score = this.calculateTableScore(table, pageNo);
+        if (this.isResourceIndexTable(table)) {
+          score -= 100;
+        }
         candidates.push({
           result: {
             itemRefs: [table.self_ref],
@@ -1631,6 +1740,37 @@ var TocFinder = class {
     }
     return null;
   }
+  /**
+   * Find adjacent TOC item among body siblings following the keyword text.
+   * Used when findTocContainer returns null (e.g., parent is #/body).
+   */
+  findSiblingTocItem(doc, textRef, pageNo) {
+    const idx = doc.body.children.findIndex((c) => c.$ref === textRef);
+    if (idx < 0) return null;
+    for (let i = idx + 1; i < Math.min(idx + 4, doc.body.children.length); i++) {
+      const ref = doc.body.children[i].$ref;
+      const table = this.refResolver.resolveTable(ref);
+      if (table && table.prov[0]?.page_no === pageNo && this.isTableTocLike(table)) {
+        return {
+          itemRefs: [table.self_ref],
+          startPage: pageNo,
+          endPage: pageNo
+        };
+      }
+      const group = this.refResolver.resolveGroup(ref);
+      if (group) {
+        const groupPage = this.getGroupFirstPage(group);
+        if (groupPage === pageNo && this.isGroupTocLike(group, doc)) {
+          return {
+            itemRefs: [group.self_ref],
+            startPage: pageNo,
+            endPage: pageNo
+          };
+        }
+      }
+    }
+    return null;
+  }
   /**
    * Check if a group contains TOC-like structure
    */
@@ -1672,6 +1812,23 @@ var TocFinder = class {
     }
     return numberCount > 0 && numberCount / (num_rows - 1) > 0.5;
   }
+  /**
+   * Check if a table is a resource index (drawing/photo/figure list)
+   * rather than a main TOC table
+   */
+  isResourceIndexTable(table) {
+    const firstColCells = table.data.table_cells.filter(
+      (c) => c.start_col_offset_idx === 0
+    );
+    if (firstColCells.length === 0) return false;
+    let resourceCount = 0;
+    for (const cell of firstColCells) {
+      if (RESOURCE_INDEX_PATTERNS.some((p) => p.test(cell.text.trim()))) {
+        resourceCount++;
+      }
+    }
+    return resourceCount / firstColCells.length > 0.5;
+  }
   /**
    * Expand TOC area to consecutive pages (both backward and forward)
    */
@@ -1744,7 +1901,7 @@ var TocFinder = class {
       if (table.prov[0]?.page_no !== pageNo) {
         continue;
       }
-      if (this.isTableTocLike(table) && !refs.includes(table.self_ref)) {
+      if (this.isTableTocLike(table) && !this.isResourceIndexTable(table) && !refs.includes(table.self_ref)) {
         refs.push(table.self_ref);
       }
     }
@@ -1936,13 +2093,22 @@ var VALIDATION_CODE_DESCRIPTIONS = {
   V006: "Duplicate entry detected (same title and page number).",
   V007: "First TOC entry starts too late in the document. Earlier entries may be missing."
 };
-var TocEntrySchema = import_zod.z.lazy(
-  () => import_zod.z.object({
+function buildTocEntrySchema(maxDepth) {
+  const base = {
     title: import_zod.z.string().describe("Chapter or section title"),
     level: import_zod.z.number().int().min(1).describe("Hierarchy depth (1 = top level)"),
-    pageNo: import_zod.z.number().int().min(1).describe("Starting page number"),
-    children: import_zod.z.array(TocEntrySchema).describe("Child sections (use empty array [] if none)")
-  })
+    pageNo: import_zod.z.number().int().min(1).describe("Starting page number")
+  };
+  if (maxDepth <= 0) {
+    return import_zod.z.object(base);
+  }
+  return import_zod.z.object({
+    ...base,
+    children: import_zod.z.array(buildTocEntrySchema(maxDepth - 1)).describe("Child sections (use empty array [] if none)")
+  });
+}
+var TocEntrySchema = buildTocEntrySchema(
+  4
 );
 var TocResponseSchema = import_zod.z.object({
   entries: import_zod.z.array(TocEntrySchema).describe("Extracted TOC entries")
@@ -2129,9 +2295,9 @@ Re-extract the TOC structure from the original markdown above. Fix all validatio
 
 5. **IMPORTANT - Extract Main TOC Only**: Only extract the main document table of contents. EXCLUDE the following:
    - **Front matter with Roman numeral pages**: Entries whose page numbers are Roman numerals (i, ii, xxi, etc.) such as \uC77C\uB7EC\uB450\uAE30, \uBC1C\uAC04\uC0AC, \uC11C\uBB38, \uBC94\uB840, Preface, Foreword, Editorial Notes. These use a separate page numbering system and are not part of the main content.
-   - Photo/image indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28, \uD654\uBCF4 \uBAA9\uCC28, Photo Index, List of Photos, List of Figures)
-   - Drawing/diagram indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28, \uC0BD\uB3C4 \uBAA9\uCC28, Drawing Index, List of Drawings)
-   - Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28, Table Index, List of Tables)
+   - Photo/image indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28, \uD654\uBCF4 \uBAA9\uCC28, \u5BEB\u771E \u76EE\u6B21, \u5BEB\u771E\u76EE\u6B21, Photo Index, List of Photos, List of Figures)
+   - Drawing/diagram indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28, \uC0BD\uB3C4 \uBAA9\uCC28, \u5716\u9762 \u76EE\u6B21, \u5716\u9762\u76EE\u6B21, Drawing Index, List of Drawings)
+   - Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28, \u8868 \u76EE\u6B21, \u8868\u76EE\u6B21, Table Index, List of Tables)
    - Appendix indices (\uBD80\uB85D \uBAA9\uCC28, Appendix Index)
    - Any other supplementary material indices
 
@@ -2452,19 +2618,34 @@ ${continuation.trim()}`;
 
 I am providing ${pageCount} document page images (pages ${startPage}-${endPage}).
 
-## Where to Look for TOC:
+## How to Identify TOC Pages:
 - TOC typically appears in the first 10-20 pages of a document
-- Look for pages with headings like "\uBAA9\uCC28", "\uCC28\uB840", "Contents", "Table of Contents"
-- Look for structured lists with chapter titles and page numbers
+- Look for pages with a heading that indicates "Table of Contents":
+  - Korean: "\uBAA9\uCC28", "\uCC28\uB840", "\uBAA9 \uCC28"
+  - Hanja (Chinese characters used in Korean documents): "\u76EE\u6B21", "\u76EE \u6B21"
+  - English: "Contents", "Table of Contents"
+- The heading may appear in decorative or stylized fonts on a textured background
+- Below the heading, look for structured lists with chapter/section titles and page numbers connected by dots or leaders
+
+## How to Distinguish Main TOC from Supplementary Indices:
+- **Main TOC** (EXTRACT THIS): The heading is simply "\uBAA9\uCC28", "\u76EE\u6B21", "Contents" with NO prefix qualifier. It lists chapters/sections of the document body.
+- **Supplementary indices** (DO NOT EXTRACT): The heading has a prefix qualifier specifying a resource type:
+  - Photo indices: "\uC0AC\uC9C4 \uBAA9\uCC28", "\uC0AC\uC9C4\uBAA9\uCC28", "\u5BEB\u771E \u76EE\u6B21", "\u5BEB\u771E\u76EE\u6B21"
+  - Drawing/figure indices: "\uB3C4\uBA74 \uBAA9\uCC28", "\uB3C4\uBA74\uBAA9\uCC28", "\u5716\u9762 \u76EE\u6B21", "\u5716\u9762\u76EE\u6B21"
+  - Table indices: "\uD45C \uBAA9\uCC28", "\uD45C\uBAA9\uCC28", "\u8868 \u76EE\u6B21", "\u8868\u76EE\u6B21"
+- Key rule: If the heading contains a qualifier word BEFORE "\uBAA9\uCC28"/"\u76EE\u6B21"/"Contents", it is a supplementary index. Only extract content from pages whose heading is the unqualified "\uBAA9\uCC28"/"\u76EE\u6B21"/"Contents".
 
 ## What to Extract:
-Extract the TOC content as markdown format that matches this exact structure:
+Extract the TOC content as markdown format with this exact structure:
 - Use "- " prefix for each list item
 - Use 2-space indentation for hierarchy levels
 - Include "..... " followed by page number at the end of each entry
 - Preserve original chapter/section numbering from the document
+- Preserve the original language of titles (Korean, Hanja, English, or mixed)
 
-## Output Format Example:
+## Output Format Examples:
+
+### Example 1: Standard Korean TOC
 \`\`\`
 - \uC81C1\uC7A5 \uC11C\uB860 ..... 1
   - 1. \uC5F0\uAD6C \uBC30\uACBD ..... 3
@@ -2475,21 +2656,30 @@ Extract the TOC content as markdown format that matches this exact structure:
 - \uC81C3\uC7A5 \uC5F0\uAD6C \uACB0\uACFC ..... 25
 \`\`\`
 
+### Example 2: Mixed Hanja-Korean TOC (common in archaeological reports)
+\`\`\`
+- \u2160. \u8ABF\u67FB\u6982\u8981 ..... 175
+- \u2161. \u8ABF\u67FB\u5730\u57DF\uC758 \u74B0\u5883 ..... 177
+- \u2162. \u8ABF\u67FB\u5185\u5BB9 ..... 199
+  - 1. \uC870\uC0AC\uC9C0\uC5ED \uD1F4\uC801\uC591\uC0C1 ..... 199
+  - 2. \uC720\uC801 \uC870\uC0AC\uB0B4\uC6A9 ..... 200
+- \u2163. \u8ABF\u67FB\u7D50\u679C ..... 228
+\`\`\`
+
 ## Important Rules:
-1. Extract ONLY the main document TOC
-2. DO NOT include supplementary indices:
-   - Photo indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28)
-   - Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28)
-   - Figure indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28)
-3. If no TOC is found, set hasToc to false and tocMarkdown to null
-4. Set continuesOnNextPage to true if the TOC appears to continue beyond the visible pages
+1. Extract ONLY the main document TOC (the one with unqualified heading "\uBAA9\uCC28"/"\u76EE\u6B21"/"Contents")
+2. DO NOT include supplementary indices (those with prefixed headings like "\uC0AC\uC9C4 \uBAA9\uCC28", "\u5BEB\u771E \u76EE\u6B21", "\u5716\u9762 \u76EE\u6B21", "\u8868 \u76EE\u6B21", etc.)
+3. Page numbers can be any valid positive integer - documents in compiled volumes may have high page numbers (e.g., 175, 228, 500+). This is normal and expected.
+4. Some pages in the batch may be photographs, illustrations, blank pages, or other non-text content with no TOC. Skip these pages.
+5. If no TOC is found, set hasToc to false and tocMarkdown to null
+6. Set continuesOnNextPage to true if the TOC appears to continue beyond the visible pages
 
 Please examine these pages and:
-1. Determine if any page contains a Table of Contents (TOC)
+1. Determine if any page contains a main document Table of Contents
 2. If found, extract the complete TOC in markdown format
 3. Indicate if the TOC continues beyond these pages
 
-Remember: Extract the main document TOC only. Ignore photo/table/figure indices.`;
+Remember: Extract the main document TOC only. Ignore photo/table/figure/drawing indices.`;
   }
 };
 
@@ -2975,7 +3165,10 @@ var PageRangeParser = class extends VisionLLMComponent {
     const imageContents = [];
     for (const pageNo of pageNos) {
       const page = pages[pageNo - 1];
-      const imagePath = path3.resolve(this.outputPath, page.image.uri);
+      const imagePath = path3.resolve(
+        this.outputPath,
+        `pages/page_${pageNo - 1}.png`
+      );
       const imageBuffer = fs3.readFileSync(imagePath);
       const base64Image = imageBuffer.toString("base64");
       const mimeType = page.image.mimetype || "image/png";
@@ -3409,283 +3602,6 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
   }
 };
 
-// src/samplers/hanja-quality-sampler.ts
-var import_zod5 = require("zod");
-var MIN_TEXT_LENGTH = 100;
-var MAX_SAMPLE_PAGES = 10;
-var EDGE_TRIM_RATIO = 0.1;
-var IMAGE_PAGE_TEXT_THRESHOLD = 50;
-var CORRUPTION_THRESHOLD = 0.5;
-var MIN_SEVERE_CORRUPTED_COUNT = 3;
-var HanjaQualityResponseSchema = import_zod5.z.object({
-  isCorrupted: import_zod5.z.boolean().describe(
-    "Whether the KCJ (Hanja) characters in the OCR text are significantly corrupted compared to the page image"
-  ),
-  corruptedCharCount: import_zod5.z.number().int().nonnegative().describe(
-    "Approximate number of corrupted or incorrectly recognized KCJ characters"
-  ),
-  totalKcjCharCount: import_zod5.z.number().int().nonnegative().describe("Total number of KCJ characters found in the OCR text"),
-  explanation: import_zod5.z.string().describe("Brief explanation of the assessment result")
-});
-var HanjaQualitySampler = class extends VisionLLMComponent {
-  constructor(logger, model, outputPath, maxRetries, fallbackModel, aggregator, abortSignal) {
-    super(
-      logger,
-      model,
-      "HanjaQualitySampler",
-      outputPath,
-      { maxRetries, abortSignal },
-      fallbackModel,
-      aggregator
-    );
-  }
-  /**
-   * Assess the quality of KCJ character recognition in the document
-   *
-   * @param doclingDoc - DoclingDocument to assess
-   * @returns Assessment result indicating whether VLM re-parse is needed
-   */
-  async assess(doclingDoc) {
-    const totalPages = Object.keys(doclingDoc.pages).length;
-    this.log("info", "Starting KCJ quality assessment...");
-    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
-    const imageOnlyPages = this.getImageOnlyPages(doclingDoc);
-    this.log(
-      "info",
-      `Total pages: ${totalPages}, eligible range: (${frontCutoff}, ${backCutoff}], image-only pages excluded: ${imageOnlyPages.size}`
-    );
-    const textPages = this.getTextPages(doclingDoc);
-    if (textPages.length === 0) {
-      this.log("info", "No text pages found for assessment");
-      return {
-        needsVlmReparse: false,
-        severity: "none",
-        kcjPageCount: 0,
-        sampledPageCount: 0,
-        corruptedRatio: 0,
-        reason: "No text pages found for assessment"
-      };
-    }
-    this.log(
-      "info",
-      `Found ${textPages.length} text pages (min length: ${MIN_TEXT_LENGTH})`
-    );
-    const sampled = this.selectSamplePages(
-      textPages,
-      totalPages,
-      imageOnlyPages
-    );
-    this.log(
-      "info",
-      `Sampling ${sampled.length} pages: [${sampled.map((p) => p.pageNo).join(", ")}]`
-    );
-    const results = await this.evaluatePages(sampled);
-    const assessment = this.aggregateResults(
-      textPages.length,
-      sampled.length,
-      results
-    );
-    this.log(
-      "info",
-      `Assessment complete: severity=${assessment.severity}, corrupted=${assessment.corruptedRatio.toFixed(2)}, needsVlmReparse=${assessment.needsVlmReparse}`
-    );
-    return assessment;
-  }
-  /**
-   * Get the eligible page range after trimming front/back edges.
-   * Returns cutoff values where eligible pages satisfy: pageNo > frontCutoff && pageNo <= backCutoff
-   */
-  getEligiblePageRange(totalPages) {
-    const trimCount = Math.ceil(totalPages * EDGE_TRIM_RATIO);
-    return {
-      frontCutoff: trimCount,
-      backCutoff: totalPages - trimCount
-    };
-  }
-  /**
-   * Get page numbers that are "image-only" (contain pictures but minimal text).
-   * These pages (e.g., photo plates/도판) are unsuitable for KCJ quality assessment.
-   */
-  getImageOnlyPages(doclingDoc) {
-    const picturePages = /* @__PURE__ */ new Set();
-    for (const picture of doclingDoc.pictures) {
-      const pageNo = picture.prov?.[0]?.page_no;
-      if (pageNo != null) {
-        picturePages.add(pageNo);
-      }
-    }
-    if (picturePages.size === 0) return /* @__PURE__ */ new Set();
-    const pageTextLength = /* @__PURE__ */ new Map();
-    for (const text of doclingDoc.texts) {
-      const pageNo = this.getPageNo(text);
-      if (picturePages.has(pageNo)) {
-        pageTextLength.set(
-          pageNo,
-          (pageTextLength.get(pageNo) ?? 0) + text.text.length
-        );
-      }
-    }
-    const imageOnlyPages = /* @__PURE__ */ new Set();
-    for (const pageNo of picturePages) {
-      const textLength = pageTextLength.get(pageNo) ?? 0;
-      if (textLength < IMAGE_PAGE_TEXT_THRESHOLD) {
-        imageOnlyPages.add(pageNo);
-      }
-    }
-    return imageOnlyPages;
-  }
-  /**
-   * Get all pages with substantial text content.
-   * Pages with fewer characters than MIN_TEXT_LENGTH (e.g., page numbers, headers only) are excluded.
-   */
-  getTextPages(doclingDoc) {
-    const pageMap = /* @__PURE__ */ new Map();
-    for (const text of doclingDoc.texts) {
-      const pageNo = this.getPageNo(text);
-      const entry = pageMap.get(pageNo) ?? { textLength: 0, texts: [] };
-      entry.textLength += text.text.length;
-      entry.texts.push(text.text);
-      pageMap.set(pageNo, entry);
-    }
-    return Array.from(pageMap.entries()).filter(([, data]) => data.textLength >= MIN_TEXT_LENGTH).map(([pageNo, data]) => ({
-      pageNo,
-      textLength: data.textLength,
-      texts: data.texts
-    }));
-  }
-  /**
-   * Select sample pages with highest text density.
-   * Prefers pages in the eligible range (not edge-trimmed) and not image-only.
-   * Falls back to all text pages if filtering leaves too few candidates.
-   */
-  selectSamplePages(textPages, totalPages, imageOnlyPages) {
-    const { frontCutoff, backCutoff } = this.getEligiblePageRange(totalPages);
-    const filtered = totalPages > 0 ? textPages.filter(
-      (p) => p.pageNo > frontCutoff && p.pageNo <= backCutoff && !imageOnlyPages.has(p.pageNo)
-    ) : textPages;
-    const candidates = filtered.length > 0 ? filtered : textPages;
-    if (candidates !== filtered) {
-      this.log(
-        "warn",
-        `All text pages were filtered out by edge/image exclusion. Falling back to all ${textPages.length} text pages.`
-      );
-    }
-    const sorted = [...candidates].sort((a, b) => b.textLength - a.textLength);
-    const count = Math.min(MAX_SAMPLE_PAGES, sorted.length);
-    return sorted.slice(0, count);
-  }
-  /**
-   * Evaluate sampled pages using Vision LLM
-   */
-  async evaluatePages(pages) {
-    const results = [];
-    for (const page of pages) {
-      const result = await this.evaluateSinglePage(page);
-      results.push(result);
-    }
-    return results;
-  }
-  /**
-   * Evaluate a single page by comparing OCR text against the page image
-   */
-  async evaluateSinglePage(page) {
-    const imagePath = `pages/page_${page.pageNo - 1}.png`;
-    let imageContent;
-    try {
-      imageContent = this.buildImageContent(imagePath);
-    } catch {
-      this.log(
-        "warn",
-        `Failed to load page image for page ${page.pageNo}, marking as not corrupted`
-      );
-      return {
-        isCorrupted: false,
-        corruptedCharCount: 0,
-        totalKcjCharCount: 0,
-        explanation: `Page image not available for page ${page.pageNo}`
-      };
-    }
-    const ocrText = page.texts.join("\n");
-    const messages = [
-      {
-        role: "user",
-        content: [
-          imageContent,
-          {
-            type: "text",
-            text: this.buildUserPrompt(ocrText)
-          }
-        ]
-      }
-    ];
-    const { output } = await this.callVisionLLM(
-      HanjaQualityResponseSchema,
-      messages,
-      `page-${page.pageNo}`
-    );
-    this.log(
-      "info",
-      `Page ${page.pageNo}: corrupted=${output.isCorrupted}, ${output.corruptedCharCount}/${output.totalKcjCharCount} KCJ chars corrupted`
-    );
-    return output;
-  }
-  /**
-   * Aggregate individual page results into final assessment
-   */
-  aggregateResults(totalTextPages, sampledCount, results) {
-    const corruptedCount = results.filter((r) => r.isCorrupted).length;
-    const corruptedRatio = sampledCount > 0 ? corruptedCount / sampledCount : 0;
-    let severity;
-    let needsVlmReparse;
-    if (corruptedRatio >= CORRUPTION_THRESHOLD || corruptedCount >= MIN_SEVERE_CORRUPTED_COUNT) {
-      severity = "severe";
-      needsVlmReparse = true;
-    } else if (corruptedCount > 0) {
-      severity = "minor";
-      needsVlmReparse = false;
-    } else {
-      severity = "none";
-      needsVlmReparse = false;
-    }
-    const reason = corruptedCount === 0 ? "No Hanja character corruption detected" : `${corruptedCount}/${sampledCount} sampled pages have corrupted Hanja characters (ratio: ${corruptedRatio.toFixed(2)})`;
-    return {
-      needsVlmReparse,
-      severity,
-      kcjPageCount: totalTextPages,
-      sampledPageCount: sampledCount,
-      corruptedRatio,
-      reason
-    };
-  }
-  /**
-   * Get page number from a text item
-   */
-  getPageNo(text) {
-    return text.prov?.[0]?.page_no ?? 1;
-  }
-  buildSystemPrompt() {
-    return "";
-  }
-  buildUserPrompt(ocrText) {
-    return `You are evaluating the quality of OCR text extraction for Hanja (\u6F22\u5B57) characters used in Korean archaeological reports.
-
-Look at the page image and find any Hanja (\u6F22\u5B57) characters visible in the original document. Then compare them against the OCR text below.
-
-## Evaluation Criteria
-- Find Hanja (\u6F22\u5B57) characters in the page image first
-- Check if each Hanja character is correctly recognized in the OCR text
-- A character is "corrupted" if a Hanja character in the image was replaced by Korean hangul, garbled text, symbols, or a completely different character in the OCR text
-- Common corruption patterns: Hanja replaced by similar-looking hangul (e.g., \u7C98\u571F \u2192 \uADF8\uC73C), wrong character substitution, partial recognition, question marks or boxes
-- Minor font rendering differences are NOT corruption
-- If no Hanja characters are visible in the page image, report as not corrupted with 0 total characters
-
-## OCR Text to Evaluate
-${ocrText}
-
-Compare the Hanja characters visible in the page image against the OCR text above and assess whether they were correctly recognized.`;
-  }
-};
-
 // src/validators/base-validator.ts
 var BaseValidator = class extends TextLLMComponent {
   /**
@@ -3745,13 +3661,13 @@ var BaseValidator = class extends TextLLMComponent {
 };
 
 // src/validators/toc-content-validator.ts
-var import_zod6 = require("zod");
-var TocContentValidationSchema = import_zod6.z.object({
-  isValid: import_zod6.z.boolean().describe("Whether valid main document TOC was found"),
-  confidence: import_zod6.z.number().min(0).max(1).describe("Confidence score between 0 and 1"),
-  contentType: import_zod6.z.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
-  extractedTocMarkdown: import_zod6.z.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
-  reason: import_zod6.z.string().describe("Brief explanation in English")
+var import_zod5 = require("zod");
+var TocContentValidationSchema = import_zod5.z.object({
+  isValid: import_zod5.z.boolean().describe("Whether valid main document TOC was found"),
+  confidence: import_zod5.z.number().min(0).max(1).describe("Confidence score between 0 and 1"),
+  contentType: import_zod5.z.enum(["pure_toc", "mixed", "resource_only", "invalid"]).describe("Type of content detected"),
+  extractedTocMarkdown: import_zod5.z.string().nullable().describe("Extracted main TOC markdown when mixed; null otherwise"),
+  reason: import_zod5.z.string().describe("Brief explanation in English")
 });
 var TocContentValidator = class extends BaseValidator {
   confidenceThreshold;
@@ -3856,9 +3772,9 @@ When classifying as "mixed", you MUST extract ONLY the main TOC portion and retu
 
 ### 3. resource_only
 The content contains ONLY resource indices such as:
-- Photo/image indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28, Photo Index, List of Figures, List of Photos)
-- Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28, Table Index, List of Tables)
-- Drawing/diagram indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28, Drawing Index, List of Drawings)
+- Photo/image indices (\uC0AC\uC9C4 \uBAA9\uCC28, \uC0AC\uC9C4\uBAA9\uCC28, \u5BEB\u771E \u76EE\u6B21, \u5BEB\u771E\u76EE\u6B21, Photo Index, List of Figures, List of Photos)
+- Table indices (\uD45C \uBAA9\uCC28, \uD45C\uBAA9\uCC28, \u8868 \u76EE\u6B21, \u8868\u76EE\u6B21, Table Index, List of Tables)
+- Drawing/diagram indices (\uB3C4\uBA74 \uBAA9\uCC28, \uB3C4\uBA74\uBAA9\uCC28, \u5716\u9762 \u76EE\u6B21, \u5716\u9762\u76EE\u6B21, Drawing Index, List of Drawings)
 - Appendix indices (\uBD80\uB85D \uBAA9\uCC28, Appendix Index)
 
 ### 4. invalid
@@ -3902,14 +3818,14 @@ ${markdown}`;
 };
 
 // src/validators/caption-validator.ts
-var import_zod7 = require("zod");
-var CaptionValidationItemSchema = import_zod7.z.object({
-  index: import_zod7.z.number().int().describe("Index of the caption in the input array"),
-  isValid: import_zod7.z.boolean().describe("Whether the parsed caption is correct"),
-  reason: import_zod7.z.string().nullable().describe("Brief explanation if invalid, null if valid")
+var import_zod6 = require("zod");
+var CaptionValidationItemSchema = import_zod6.z.object({
+  index: import_zod6.z.number().int().describe("Index of the caption in the input array"),
+  isValid: import_zod6.z.boolean().describe("Whether the parsed caption is correct"),
+  reason: import_zod6.z.string().nullable().describe("Brief explanation if invalid, null if valid")
 });
-var CaptionValidationBatchSchema = import_zod7.z.object({
-  results: import_zod7.z.array(CaptionValidationItemSchema)
+var CaptionValidationBatchSchema = import_zod6.z.object({
+  results: import_zod6.z.array(CaptionValidationItemSchema)
 });
 var CaptionValidator = class extends BaseValidator {
   constructor(logger, model, options, fallbackModel, aggregator) {
@@ -4117,13 +4033,13 @@ var DocumentProcessor = class {
   validatorModel;
   visionTocExtractorModel;
   captionParserModel;
-  hanjaQualitySamplerModel;
   textCleanerBatchSize;
   captionParserBatchSize;
   captionValidatorBatchSize;
   maxRetries;
   enableFallbackRetry;
   abortSignal;
+  onTokenUsage;
   idGenerator = new IdGenerator();
   refResolver;
   pageRangeParser;
@@ -4144,13 +4060,22 @@ var DocumentProcessor = class {
     this.validatorModel = options.validatorModel ?? options.fallbackModel;
     this.visionTocExtractorModel = options.visionTocExtractorModel ?? options.fallbackModel;
     this.captionParserModel = options.captionParserModel ?? options.fallbackModel;
-    this.hanjaQualitySamplerModel = options.hanjaQualitySamplerModel ?? options.fallbackModel;
     this.textCleanerBatchSize = options.textCleanerBatchSize;
     this.captionParserBatchSize = options.captionParserBatchSize;
     this.captionValidatorBatchSize = options.captionValidatorBatchSize;
     this.maxRetries = options.maxRetries ?? 3;
     this.enableFallbackRetry = options.enableFallbackRetry ?? false;
     this.abortSignal = options.abortSignal;
+    this.onTokenUsage = options.onTokenUsage;
+  }
+  /**
+   * Emit current token usage report via callback
+   *
+   * Calls the onTokenUsage callback with the current cumulative report
+   * from the usage aggregator. Safe to call even if no callback is set.
+   */
+  emitTokenUsage() {
+    this.onTokenUsage?.(this.usageAggregator.getReport());
   }
   /**
    * Check if abort has been requested and throw error if so
@@ -4164,36 +4089,6 @@ var DocumentProcessor = class {
       throw error;
     }
   }
-  /**
-   * Assess the quality of KCJ (Hanja) character recognition in the OCR output.
-   *
-   * Samples pages containing KCJ characters and uses Vision LLM to compare
-   * the OCR text against the original page images. Returns an assessment
-   * indicating whether the document should be re-parsed with VLM pipeline.
-   *
-   * @param doclingDoc - Original document extracted from Docling SDK
-   * @param outputPath - Path containing pages subdirectory (pages/page_0.png, etc.)
-   * @returns Assessment result with severity and recommendation
-   */
-  async assessHanjaQuality(doclingDoc, outputPath) {
-    this.logger.info(
-      "[DocumentProcessor] Starting Hanja quality assessment..."
-    );
-    const sampler = new HanjaQualitySampler(
-      this.logger,
-      this.hanjaQualitySamplerModel,
-      outputPath,
-      this.maxRetries,
-      this.enableFallbackRetry ? this.fallbackModel : void 0,
-      this.usageAggregator,
-      this.abortSignal
-    );
-    const assessment = await sampler.assess(doclingDoc);
-    this.logger.info(
-      `[DocumentProcessor] Hanja assessment: severity=${assessment.severity}, needsVlmReparse=${assessment.needsVlmReparse}`
-    );
-    return assessment;
-  }
   /**
    * Converts DoclingDocument to ProcessedDocument with token usage tracking.
    *
@@ -4235,11 +4130,13 @@ var DocumentProcessor = class {
     this.logger.info(
       `[DocumentProcessor] Page range parsing took ${pageRangeTime}ms`
     );
+    this.emitTokenUsage();
     this.checkAborted();
     const startTimeToc = Date.now();
     const tocEntries = await this.extractTableOfContents(doclingDoc, filtered);
     const tocTime = Date.now() - startTimeToc;
     this.logger.info(`[DocumentProcessor] TOC extraction took ${tocTime}ms`);
+    this.emitTokenUsage();
     this.checkAborted();
     const startTimeResources = Date.now();
     const { images, tables, footnotes } = await this.convertResources(
@@ -4250,6 +4147,7 @@ var DocumentProcessor = class {
     this.logger.info(
       `[DocumentProcessor] Resource conversion took ${resourcesTime}ms`
     );
+    this.emitTokenUsage();
     this.checkAborted();
     const startTimeChapters = Date.now();
     const chapters = await this.convertChapters(
@@ -4509,7 +4407,9 @@ var DocumentProcessor = class {
         throw error;
       }
     }
+    let fromVision = false;
     if (!markdown) {
+      fromVision = true;
       this.logger.info("[DocumentProcessor] Using vision fallback for TOC");
       const totalPages2 = Object.keys(doclingDoc.pages).length;
       markdown = await this.visionTocExtractor.extract(totalPages2);
@@ -4529,12 +4429,52 @@ var DocumentProcessor = class {
     const totalPages = Object.keys(doclingDoc.pages).length;
     const maxTocPageNo = this.extractMaxPageNumber(markdown);
     const effectiveTotalPages = maxTocPageNo > totalPages ? void 0 : totalPages;
-    const tocResult = await this.tocExtractor.extract(markdown, {
-      totalPages: effectiveTotalPages
-    });
+    let tocResult;
+    try {
+      tocResult = await this.tocExtractor.extract(markdown, {
+        totalPages: effectiveTotalPages
+      });
+    } catch (error) {
+      if (error instanceof TocValidationError) {
+        this.logger.warn(
+          `[DocumentProcessor] TOC extraction validation failed: ${error.message}`
+        );
+        tocResult = { entries: [], usages: [] };
+      } else {
+        throw error;
+      }
+    }
     for (const usage of tocResult.usages) {
       this.usageAggregator.track(usage);
     }
+    if (tocResult.entries.length === 0 && !fromVision) {
+      this.logger.warn(
+        "[DocumentProcessor] Text-based TOC extraction yielded 0 entries, retrying with vision"
+      );
+      const visionMarkdown = await this.visionTocExtractor.extract(totalPages);
+      if (visionMarkdown) {
+        this.logger.info(
+          `[DocumentProcessor] Vision extracted TOC markdown (${visionMarkdown.length} chars)`
+        );
+        const visionMaxPageNo = this.extractMaxPageNumber(visionMarkdown);
+        const visionEffectivePages = visionMaxPageNo > totalPages ? void 0 : totalPages;
+        try {
+          const visionResult = await this.tocExtractor.extract(
+            visionMarkdown,
+            {
+              totalPages: visionEffectivePages
+            }
+          );
+          for (const usage of visionResult.usages) {
+            this.usageAggregator.track(usage);
+          }
+          if (visionResult.entries.length > 0) {
+            tocResult = visionResult;
+          }
+        } catch {
+        }
+      }
+    }
     if (tocResult.entries.length === 0) {
       const reason = "TOC area was detected but LLM could not extract any structured entries";
       this.logger.error(`[DocumentProcessor] TOC extraction failed: ${reason}`);
@@ -4549,15 +4489,20 @@ var DocumentProcessor = class {
    * Extract the maximum page number from TOC markdown
    *
    * Parses page numbers from dot-leader patterns (e.g., "..... 175")
-   * to detect compiled volume scenarios where TOC page numbers exceed
-   * the sub-document's page count.
+   * and table cell patterns (e.g., "| title | 175 |") to detect compiled
+   * volume scenarios where TOC page numbers exceed the sub-document's page count.
    */
   extractMaxPageNumber(markdown) {
-    const matches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
-    if (matches.length === 0) {
+    const dotLeaderMatches = [...markdown.matchAll(/\.{2,}\s*(\d+)/g)];
+    const tableCellMatches = [...markdown.matchAll(/\|\s*(\d+)\s*\|\s*$/gm)];
+    const allNumbers = [
+      ...dotLeaderMatches.map((m) => parseInt(m[1], 10)),
+      ...tableCellMatches.map((m) => parseInt(m[1], 10))
+    ];
+    if (allNumbers.length === 0) {
       return 0;
     }
-    return Math.max(...matches.map((m) => parseInt(m[1], 10)));
+    return Math.max(...allNumbers);
   }
   /**
    * Process resource captions (for images and tables)
@@ -4827,7 +4772,6 @@ var DocumentProcessor = class {
   CaptionValidator,
   ChapterConverter,
   DocumentProcessor,
-  HanjaQualitySampler,
   PAGE_NUMBER_PATTERN,
   PagePattern,
   PageRangeParseError,