@heripo/document-processor 0.1.48 → 0.2.0

package/README.ko.md

@@ -265,6 +265,8 @@ DocumentProcessor는 다음 5단계 파이프라인으로 문서를 처리합니
 - PDF 페이지와 문서 논리적 페이지 매핑
 - 스캔 오류로 인한 페이지 번호 불일치 처리
 
+PDF 물리 페이지는 1부터 시작하며, 논리 페이지는 본문 1쪽을 기준으로 연속해서 매깁니다. 앞쪽은 음수와 0을 사용하며, 자동 매핑·검토된 입력·목차·챕터 배정에서 이 값을 보존합니다. 추출 실패는 내부적으로 정상 번호인 `0`과 구분합니다. 알려진 페이지의 패턴으로도 번호를 추론할 수 없으면 `PageRangeParseError`를 발생시키며, 실패한 페이지를 논리 `0`으로 출력하지 않습니다.
+
 ### 3. TOC 추출 (5단계 파이프라인)
 
 #### Stage 1: TocFinder (규칙 기반)

package/README.md

@@ -280,6 +280,8 @@ DocumentProcessor processes documents through a 5-stage pipeline:
 - PDF page to document logical page mapping
 - Handle page number mismatches due to scanning errors
 
+Physical PDF pages start at 1; logical pages count continuously relative to body page 1. Front matter can use negative values and zero, which are preserved in automatic mapping, reviewed inputs, TOC entries, and chapter assignment. Extraction failures are tracked separately from valid logical zero. If known page patterns cannot resolve a missing number, parsing throws `PageRangeParseError` rather than outputting logical zero for an unknown page.
+
 ### 3. TOC Extraction (5-Stage Pipeline)
 
 #### Stage 1: TocFinder (Rule-Based)

package/dist/index.d.ts

@@ -19,7 +19,7 @@ interface TocEntry {
      */
     level: number;
     /**
-     * Starting page number
+     * Starting logical page number (signed integer, relative to body page 1)
      */
     pageNo: number;
     /**
@@ -620,7 +620,7 @@ declare enum PagePattern {
  * 2. For each group:
  *    - If ≤3 pages: send all to LLM at once
  *    - If >3 pages: random sample 3 pages, detect pattern, apply to all
- * 3. Post-process: handle drops, normalize negatives, backfill failed pages
+ * 3. Post-process: handle drops and infer missing logical pages, including zero and negatives
  */
 declare class PageRangeParser extends VisionLLMComponent {
     private readonly SAMPLE_SIZE;
@@ -713,11 +713,7 @@ declare class PageRangeParser extends VisionLLMComponent {
      */
     private detectAndHandleDrops;
     /**
-     * Normalize negative page numbers to 0
-     */
-    private normalizeNegatives;
-    /**
-     * Backfill pages marked with 0 using detected pattern
+     * Infer unresolved pages from the logical numbering pattern
      */
     private backfillFailedPages;
     /**
@@ -1493,7 +1489,7 @@ declare class ChapterConverter {
      * Uses next chapter's start page as end boundary
      *
      * Front Matter (ch-000) gets special handling:
-     * - startPage: 1
+     * - startPage: the first logical page
      * - endPage: first TOC entry's page - 1 (or 0 if TOC starts at page 1)
      */
     private calculatePageRanges;

package/dist/index.js

@@ -6,6 +6,8 @@ import {
   hasToolCall,
   tool
 } from "ai";
+import { wrapLanguageModel } from "ai";
+import { wrapLanguageModel as wrapLanguageModel2 } from "ai";
 var BatchProcessor = class {
   /**
    * Splits an array into batches of specified size.
@@ -135,9 +137,23 @@ var LLMCaller = class {
         cacheWriteTokens: allWrites - (write1h ?? 0)
       },
       ...write1h !== void 0 && { cacheWrite1hTokens: write1h },
+      ...usage?.outputTokenDetails?.reasoningTokens != null && {
+        reasoningTokens: usage.outputTokenDetails.reasoningTokens
+      },
       metadata: config.metadata
     };
   }
+  /** Finish reason and structured-output attempts, when known. */
+  static callDetails(response) {
+    return {
+      ...response.finishReason !== void 0 && {
+        finishReason: response.finishReason
+      },
+      ...response.structuredAttempts !== void 0 && {
+        structuredAttempts: response.structuredAttempts
+      }
+    };
+  }
   /**
    * Maximum number of retries when structured output generation fails.
    * Total attempts = MAX_STRUCTURED_OUTPUT_RETRIES + 1.
@@ -178,7 +194,9 @@ var LLMCaller = class {
         return {
           output: toolCall.input,
           usage: lastResult.usage,
-          steps: lastResult.steps
+          steps: lastResult.steps,
+          finishReason: lastResult.finishReason,
+          structuredAttempts: attempt + 1
         };
       }
     }
@@ -212,11 +230,18 @@ var LLMCaller = class {
     let lastError;
     for (let attempt = 0; attempt <= this.MAX_STRUCTURED_OUTPUT_RETRIES; attempt++) {
       try {
-        return await generateText({
+        const result = await generateText({
           model,
           output: Output.object({ schema }),
           ...promptParams
         });
+        return {
+          output: result.output,
+          usage: result.usage,
+          steps: result.steps,
+          finishReason: result.finishReason,
+          structuredAttempts: attempt + 1
+        };
       } catch (error) {
         if (NoObjectGeneratedError.isInstance(error)) {
           lastError = error;
@@ -240,7 +265,8 @@ var LLMCaller = class {
       return {
         output: response.output,
         usage: this.buildUsage(config, primaryModelName, response, false),
-        usedFallback: false
+        usedFallback: false,
+        ...this.callDetails(response)
       };
     } catch (primaryError) {
       if (config.abortSignal?.aborted) {
@@ -254,7 +280,8 @@ var LLMCaller = class {
       return {
         output: response.output,
         usage: this.buildUsage(config, fallbackModelName, response, true),
-        usedFallback: true
+        usedFallback: true,
+        ...this.callDetails(response)
       };
     }
   }
@@ -277,8 +304,17 @@ var LLMCaller = class {
     return this.executeWithFallback(
       config,
       (model) => this.generateStructuredOutput(model, config.schema, {
-        instructions: config.systemPrompt,
+        instructions: config.cacheSystemPrompt ? {
+          role: "system",
+          content: config.systemPrompt,
+          providerOptions: {
+            anthropic: { cacheControl: { type: "ephemeral" } }
+          }
+        } : config.systemPrompt,
         prompt: config.userPrompt,
+        ...config.providerOptions && {
+          providerOptions: config.providerOptions
+        },
         temperature: config.temperature,
         maxRetries: config.maxRetries,
         abortSignal: config.abortSignal
@@ -548,6 +584,7 @@ var LLMTokenUsageAggregator = class {
     this.usage = {};
   }
 };
+var encoder = new TextEncoder();
 
 // src/utils/ref-resolver.ts
 var RefResolver = class {
@@ -1037,7 +1074,7 @@ var ChapterConverter = class _ChapterConverter {
    */
   convert(tocEntries, textItems, pageRangeMap, images, tables, footnotes) {
     this.logger.info("[ChapterConverter] Starting chapter conversion...");
-    const frontMatter = this.createFrontMatterChapter();
+    const frontMatter = this.createFrontMatterChapter(pageRangeMap);
     const tocChapters = this.buildChapterTree(tocEntries);
     this.logger.info(
       `[ChapterConverter] Built ${tocChapters.length} TOC chapters + Front Matter`
@@ -1069,12 +1106,15 @@ var ChapterConverter = class _ChapterConverter {
   /**
    * Create Front Matter chapter for pre-TOC content
    */
-  createFrontMatterChapter() {
+  createFrontMatterChapter(pageRangeMap) {
+    const logicalPages = Object.values(pageRangeMap).map(
+      (range) => range.startPageNo
+    );
     return {
       id: _ChapterConverter.FRONT_MATTER_ID,
       originTitle: _ChapterConverter.FRONT_MATTER_TITLE,
       title: _ChapterConverter.FRONT_MATTER_TITLE,
-      pageNo: 1,
+      pageNo: logicalPages.length > 0 ? Math.min(...logicalPages) : 1,
       level: 1,
       sourceRefs: [],
       textBlocks: [],
@@ -1133,7 +1173,7 @@ var ChapterConverter = class _ChapterConverter {
    * Uses next chapter's start page as end boundary
    *
    * Front Matter (ch-000) gets special handling:
-   * - startPage: 1
+   * - startPage: the first logical page
    * - endPage: first TOC entry's page - 1 (or 0 if TOC starts at page 1)
    */
   calculatePageRanges(flatChapters, tocEntries) {
@@ -1147,7 +1187,7 @@ var ChapterConverter = class _ChapterConverter {
     );
     const sorted = [...tocChapters].sort((a, b) => a.tocPageNo - b.tocPageNo);
     ranges.set(_ChapterConverter.FRONT_MATTER_ID, {
-      startPage: 1,
+      startPage: flatChapters[0].tocPageNo,
       endPage: firstTocPage - 1
     });
     sorted.forEach((current, i) => {
@@ -1211,7 +1251,7 @@ var ChapterConverter = class _ChapterConverter {
    */
   findChapterForPage(actualPageNo, chapterRanges) {
     let bestMatch = null;
-    let bestStartPage = -1;
+    let bestStartPage = -Infinity;
     for (const [chapterId, range] of chapterRanges) {
       if (actualPageNo >= range.startPage && actualPageNo <= range.endPage) {
         if (range.startPage > bestStartPage) {
@@ -1612,7 +1652,7 @@ ${details}`,
    * Recursively validate entries
    */
   validateEntries(entries, parentPath, parentEntry, seenKeys) {
-    let prevPageNo = parentEntry?.pageNo ?? 0;
+    let prevPageNo = parentEntry?.pageNo ?? -Infinity;
     for (let i = 0; i < entries.length; i++) {
       const entry = entries[i];
       const path4 = parentPath ? `${parentPath}.children[${i}]` : `[${i}]`;
@@ -1662,10 +1702,10 @@ ${details}`,
    * V002: Validate page number range
    */
   validatePageRange(entry, path4) {
-    if (entry.pageNo < 1) {
+    if (!Number.isInteger(entry.pageNo)) {
       this.addIssue({
         code: "V002",
-        message: `Page number must be >= 1, got ${entry.pageNo}`,
+        message: `Page number must be an integer, got ${entry.pageNo}`,
         path: path4,
         entry
       });
@@ -2242,7 +2282,7 @@ var TextLLMComponent = class extends BaseLLMComponent {
 var DEFAULT_MAX_VALIDATION_RETRIES = 3;
 var VALIDATION_CODE_DESCRIPTIONS = {
   V001: "Page numbers must be in non-decreasing order within the same level. A decrease usually means a hierarchy or page number error.",
-  V002: "Page number is out of valid range (must be >= 1 and <= total pages).",
+  V002: "Page number must be an integer within the configured logical page range.",
   V003: "Title is empty or contains only whitespace.",
   V004: "Title exceeds the maximum allowed length.",
   V005: "Child page number is before parent page number. Children must start on or after the parent page.",
@@ -2253,7 +2293,7 @@ function buildTocEntrySchema(maxDepth) {
   const base = {
     title: z.string().describe("Chapter or section title"),
     level: z.number().int().min(1).describe("Hierarchy depth (1 = top level)"),
-    pageNo: z.number().int().min(1).describe("Starting page number")
+    pageNo: z.number().int().describe("Starting logical page number")
   };
   if (maxDepth <= 0) {
     return z.object(base);
@@ -3180,7 +3220,7 @@ var PageRangeParser = class extends VisionLLMComponent {
       "info",
       `Found ${sizeGroups.length} size group(s), total ${pages.length} pages`
     );
-    const pageRangeMap = {};
+    const extractedPageRanges = {};
     const usageList = [];
     for (let i = 0; i < sizeGroups.length; i++) {
       const group = sizeGroups[i];
@@ -3189,11 +3229,20 @@ var PageRangeParser = class extends VisionLLMComponent {
         `Processing group ${i + 1}/${sizeGroups.length}: ${group.pageNos.length} pages`
       );
       const groupResult = await this.processGroup(pages, group, this.model);
-      Object.assign(pageRangeMap, groupResult.pageRangeMap);
+      Object.assign(extractedPageRanges, groupResult.pageRangeMap);
       usageList.push(...groupResult.usage);
     }
     usageList.forEach((usage) => this.trackUsage(usage));
-    this.postProcess(pageRangeMap);
+    this.postProcess(extractedPageRanges);
+    const pageRangeMap = {};
+    for (const [pdfPage, range] of Object.entries(extractedPageRanges)) {
+      if (range === null) {
+        throw new PageRangeParseError(
+          `Could not determine a logical page number for PDF page ${pdfPage}`
+        );
+      }
+      pageRangeMap[Number(pdfPage)] = range;
+    }
     this.log(
       "info",
       `Completed: ${Object.keys(pageRangeMap).length} pages mapped`
@@ -3321,7 +3370,7 @@ var PageRangeParser = class extends VisionLLMComponent {
       `Could not detect page pattern after ${consecutiveAllNullAttempts >= MAX_ALL_NULL_ATTEMPTS ? consecutiveAllNullAttempts : this.MAX_PATTERN_RETRIES + 1} attempts for group with ${pageNos.length} pages, marking as failed`
     );
     const fallbackMap = Object.fromEntries(
-      pageNos.map((pageNo) => [pageNo, { startPageNo: 0, endPageNo: 0 }])
+      pageNos.map((pageNo) => [pageNo, null])
     );
     return { pageRangeMap: fallbackMap, usage: usageList };
   }
@@ -3363,8 +3412,8 @@ var PageRangeParser = class extends VisionLLMComponent {
       pages: z4.array(
         z4.object({
           imageIndex: z4.number().describe("0-based index of the image in the request"),
-          startPageNo: z4.number().nullable().describe("Start page number (null if not found)"),
-          endPageNo: z4.number().nullable().describe(
+          startPageNo: z4.number().int().nullable().describe("Start page number (null if not found)"),
+          endPageNo: z4.number().int().nullable().describe(
             "End page number for double-sided scans (null for single page)"
           )
         })
@@ -3477,7 +3526,7 @@ var PageRangeParser = class extends VisionLLMComponent {
           break;
         }
         default:
-          result[pdfPageNo] = { startPageNo: 0, endPageNo: 0 };
+          result[pdfPageNo] = null;
       }
     });
     return result;
@@ -3492,7 +3541,7 @@ var PageRangeParser = class extends VisionLLMComponent {
         sample.startPageNo !== null ? {
           startPageNo: sample.startPageNo,
           endPageNo: sample.endPageNo ?? sample.startPageNo
-        } : { startPageNo: 0, endPageNo: 0 }
+        } : null
       ])
     );
   }
@@ -3502,7 +3551,6 @@ var PageRangeParser = class extends VisionLLMComponent {
   postProcess(pageRangeMap) {
     this.detectAndHandleOutliers(pageRangeMap);
     this.detectAndHandleDrops(pageRangeMap);
-    this.normalizeNegatives(pageRangeMap);
     this.backfillFailedPages(pageRangeMap);
   }
   /**
@@ -3528,8 +3576,9 @@ var PageRangeParser = class extends VisionLLMComponent {
     let hasOutliers = false;
     for (let i = 0; i < normalSequenceStart; i++) {
       const pdfPage = pdfPages[i];
-      const pageNo = pageRangeMap[pdfPage].startPageNo;
-      if (pageNo === 0) continue;
+      const range = pageRangeMap[pdfPage];
+      if (range === null) continue;
+      const pageNo = range.startPageNo;
       const pdfDiff = normalStartPdfPage - pdfPage;
       const isDoubleSided = this.isDoubleSidedRange(
         pageRangeMap[normalStartPdfPage]
@@ -3540,7 +3589,7 @@ var PageRangeParser = class extends VisionLLMComponent {
           "info",
           `Outlier detected: PDF ${pdfPage}=${pageNo} (expected ~${expectedPageNo})`
         );
-        pageRangeMap[pdfPage] = { startPageNo: 0, endPageNo: 0 };
+        pageRangeMap[pdfPage] = null;
         hasOutliers = true;
       }
     }
@@ -3567,7 +3616,7 @@ var PageRangeParser = class extends VisionLLMComponent {
         const nextPdfPage = pdfPages[startIdx + i + 1];
         const currRange = pageRangeMap[currPdfPage];
         const nextRange = pageRangeMap[nextPdfPage];
-        if (currRange.startPageNo === 0 || nextRange.startPageNo === 0) {
+        if (currRange === null || nextRange === null) {
           isValidSequence = false;
           break;
         }
@@ -3608,40 +3657,32 @@ var PageRangeParser = class extends VisionLLMComponent {
     for (let i = 1; i < pdfPages.length; i++) {
       const prevPdfPage = pdfPages[i - 1];
       const currPdfPage = pdfPages[i];
-      const prevPageNo = pageRangeMap[prevPdfPage].startPageNo;
-      const currPageNo = pageRangeMap[currPdfPage].startPageNo;
-      if (prevPageNo === 0 || currPageNo === 0) continue;
-      if (currPageNo > 0 && prevPageNo > currPageNo && prevPageNo - currPageNo > 1) {
+      const prevRange = pageRangeMap[prevPdfPage];
+      const currRange = pageRangeMap[currPdfPage];
+      if (prevRange === null || currRange === null) continue;
+      const prevPageNo = prevRange.startPageNo;
+      const currPageNo = currRange.startPageNo;
+      if (prevPageNo > currPageNo && prevPageNo - currPageNo > 1) {
         this.log(
           "info",
           `Page drop detected: PDF ${prevPdfPage}=${prevPageNo} -> PDF ${currPdfPage}=${currPageNo}`
         );
-        const isDoubleSided = this.isDoubleSidedRange(
-          pageRangeMap[currPdfPage]
-        );
+        const isDoubleSided = this.isDoubleSidedRange(currRange);
         for (let j = i - 1; j >= 0; j--) {
           const pdfPage = pdfPages[j];
           const distance = currPdfPage - pdfPage;
           if (isDoubleSided) {
             const expectedStartPageNo = currPageNo - distance * 2;
-            if (expectedStartPageNo < 1) {
-              pageRangeMap[pdfPage] = { startPageNo: 0, endPageNo: 0 };
-            } else {
-              pageRangeMap[pdfPage] = {
-                startPageNo: expectedStartPageNo,
-                endPageNo: expectedStartPageNo + 1
-              };
-            }
+            pageRangeMap[pdfPage] = {
+              startPageNo: expectedStartPageNo,
+              endPageNo: expectedStartPageNo + 1
+            };
           } else {
             const expectedPageNo = currPageNo - distance;
-            if (expectedPageNo < 1) {
-              pageRangeMap[pdfPage] = { startPageNo: 0, endPageNo: 0 };
-            } else {
-              pageRangeMap[pdfPage] = {
-                startPageNo: expectedPageNo,
-                endPageNo: expectedPageNo
-              };
-            }
+            pageRangeMap[pdfPage] = {
+              startPageNo: expectedPageNo,
+              endPageNo: expectedPageNo
+            };
           }
           this.log(
             "info",
@@ -3652,32 +3693,19 @@ var PageRangeParser = class extends VisionLLMComponent {
     }
   }
   /**
-   * Normalize negative page numbers to 0
-   */
-  normalizeNegatives(pageRangeMap) {
-    for (const [pdfPageStr, range] of Object.entries(pageRangeMap)) {
-      if (range.startPageNo < 0 || range.endPageNo < 0) {
-        this.log("info", `Normalizing negative: PDF ${pdfPageStr} -> 0`);
-        pageRangeMap[Number(pdfPageStr)] = { startPageNo: 0, endPageNo: 0 };
-      }
-    }
-  }
-  /**
-   * Backfill pages marked with 0 using detected pattern
+   * Infer unresolved pages from the logical numbering pattern
    */
   backfillFailedPages(pageRangeMap) {
     const pdfPages = Object.keys(pageRangeMap).map(Number).sort((a, b) => a - b);
-    const failedPages = pdfPages.filter(
-      (p) => pageRangeMap[p].startPageNo === 0
-    );
+    const failedPages = pdfPages.filter((p) => pageRangeMap[p] === null);
     if (failedPages.length === 0) return;
-    const successfulPages = pdfPages.filter((p) => pageRangeMap[p].startPageNo > 0).map((p) => ({
+    const successfulPages = pdfPages.filter((p) => pageRangeMap[p] !== null).map((p) => ({
       pdfPage: p,
       pageNo: pageRangeMap[p].startPageNo,
       isDoubleSided: this.isDoubleSidedRange(pageRangeMap[p])
     }));
-    if (successfulPages.length < 2) {
-      this.log("warn", "Not enough successful pages for backfill");
+    if (successfulPages.length === 0) {
+      this.log("warn", "No known logical pages for backfill");
       return;
     }
     const doubleSidedCount = successfulPages.filter(
@@ -3695,16 +3723,9 @@ var PageRangeParser = class extends VisionLLMComponent {
       );
       for (const pdfPage of failedPages) {
         const expectedStartPageNo = pdfPage * 2 + avgOffset;
-        if (expectedStartPageNo < 1) {
-          this.log(
-            "info",
-            `Backfill skipped for PDF ${pdfPage} (would be ${expectedStartPageNo})`
-          );
-          continue;
-        }
         this.log(
           "info",
-          `Backfill PDF ${pdfPage}: 0 -> ${expectedStartPageNo}-${expectedStartPageNo + 1}`
+          `Backfill PDF ${pdfPage}: unknown -> ${expectedStartPageNo}-${expectedStartPageNo + 1}`
         );
         pageRangeMap[pdfPage] = {
           startPageNo: expectedStartPageNo,
@@ -3722,14 +3743,10 @@ var PageRangeParser = class extends VisionLLMComponent {
       );
       for (const pdfPage of failedPages) {
         const expectedPageNo = pdfPage + avgOffset;
-        if (expectedPageNo < 1) {
-          this.log(
-            "info",
-            `Backfill skipped for PDF ${pdfPage} (would be ${expectedPageNo})`
-          );
-          continue;
-        }
-        this.log("info", `Backfill PDF ${pdfPage}: 0 -> ${expectedPageNo}`);
+        this.log(
+          "info",
+          `Backfill PDF ${pdfPage}: unknown -> ${expectedPageNo}`
+        );
         pageRangeMap[pdfPage] = {
           startPageNo: expectedPageNo,
           endPageNo: expectedPageNo