@heripo/document-processor 0.1.21 → 0.1.23

package/dist/index.js

@@ -1224,12 +1224,12 @@ var ResourceConverter = class {
    *
    * Runs image and table conversions in parallel, then footnotes synchronously.
    */
-  async convertAll(doclingDoc, outputPath) {
+  async convertAll(doclingDoc, artifactDir) {
     this.logger.info(
       "[ResourceConverter] Converting images, tables, and footnotes..."
     );
     const [images, tables] = await Promise.all([
-      this.convertImages(doclingDoc, outputPath),
+      this.convertImages(doclingDoc, artifactDir),
       this.convertTables(doclingDoc)
     ]);
     const footnotes = this.convertFootnotes(doclingDoc);
@@ -1241,7 +1241,7 @@ var ResourceConverter = class {
   /**
    * Convert images from DoclingDocument to ProcessedImage[]
    */
-  async convertImages(doclingDoc, outputPath) {
+  async convertImages(doclingDoc, artifactDir) {
     this.logger.info(
       `[ResourceConverter] Converting ${doclingDoc.pictures.length} images...`
     );
@@ -1251,7 +1251,7 @@ var ResourceConverter = class {
     const images = doclingDoc.pictures.map(
       (picture, index) => ({
         id: this.idGenerator.generateImageId(),
-        path: `${outputPath}/images/image_${index}.png`,
+        path: `${artifactDir}/images/image_${index}.png`,
         pdfPageNo: picture.prov?.[0]?.page_no ?? 0
       })
     );
@@ -2072,7 +2072,7 @@ var TextLLMComponent = class extends BaseLLMComponent {
 };
 
 // src/extractors/toc-extractor.ts
-var MAX_VALIDATION_RETRIES = 3;
+var DEFAULT_MAX_VALIDATION_RETRIES = 3;
 var VALIDATION_CODE_DESCRIPTIONS = {
   V001: "Page numbers must be in non-decreasing order within the same level. A decrease usually means a hierarchy or page number error.",
   V002: "Page number is out of valid range (must be >= 1 and <= total pages).",
@@ -2105,6 +2105,7 @@ var TocResponseSchema = z.object({
 var TocExtractor = class extends TextLLMComponent {
   validationOptions;
   skipValidation;
+  maxValidationRetries;
   constructor(logger, model, options, fallbackModel, abortSignal) {
     super(
       logger,
@@ -2115,11 +2116,12 @@ var TocExtractor = class extends TextLLMComponent {
     );
     this.validationOptions = options?.validation;
     this.skipValidation = options?.skipValidation ?? false;
+    this.maxValidationRetries = options?.maxValidationRetries ?? DEFAULT_MAX_VALIDATION_RETRIES;
   }
   /**
    * Extract TOC structure from Markdown
    *
-   * When validation fails, retries with correction feedback up to MAX_VALIDATION_RETRIES times.
+   * When validation fails, retries with correction feedback up to maxValidationRetries times.
    *
    * @param markdown - Markdown representation of TOC area
    * @param validationOverrides - Optional overrides for validation options (merged with constructor options)
@@ -2149,10 +2151,10 @@ var TocExtractor = class extends TextLLMComponent {
           entries,
           validationOverrides
         );
-        for (let attempt = 1; attempt <= MAX_VALIDATION_RETRIES && validationError !== null; attempt++) {
+        for (let attempt = 1; attempt <= this.maxValidationRetries && validationError !== null; attempt++) {
           this.log(
             "warn",
-            `Validation failed (attempt ${attempt}/${MAX_VALIDATION_RETRIES}), retrying with correction feedback`
+            `Validation failed (attempt ${attempt}/${this.maxValidationRetries}), retrying with correction feedback`
           );
           const correctionPrompt = this.buildCorrectionPrompt(
             markdown,
@@ -2175,7 +2177,7 @@ var TocExtractor = class extends TextLLMComponent {
         if (validationError !== null) {
           this.log(
             "error",
-            `Validation failed after ${MAX_VALIDATION_RETRIES} retries:
+            `Validation failed after ${this.maxValidationRetries} retries:
 ${validationError.getSummary()}`
           );
           throw validationError;
@@ -2367,10 +2369,10 @@ import { z as z2 } from "zod";
 import * as fs from "fs";
 import * as path from "path";
 var VisionLLMComponent = class extends BaseLLMComponent {
-  outputPath;
-  constructor(logger, model, componentName, outputPath, options, fallbackModel, aggregator) {
+  artifactDir;
+  constructor(logger, model, componentName, artifactDir, options, fallbackModel, aggregator) {
     super(logger, model, componentName, options, fallbackModel, aggregator);
-    this.outputPath = outputPath;
+    this.artifactDir = artifactDir;
   }
   /**
    * Call LLM with vision capabilities using LLMCaller.callVision()
@@ -2402,12 +2404,12 @@ var VisionLLMComponent = class extends BaseLLMComponent {
   /**
    * Build image content object for vision LLM messages
    *
-   * @param imagePath - Path to the image file (relative to outputPath or absolute)
+   * @param imagePath - Path to the image file (relative to artifactDir or absolute)
    * @param mimeType - MIME type of the image (default: 'image/png')
    * @returns ImageContent object for LLM message
    */
   buildImageContent(imagePath, mimeType = "image/png") {
-    const absolutePath = path.isAbsolute(imagePath) ? imagePath : path.resolve(this.outputPath, imagePath);
+    const absolutePath = path.isAbsolute(imagePath) ? imagePath : path.resolve(this.artifactDir, imagePath);
     const imageData = new Uint8Array(fs.readFileSync(absolutePath));
     return {
       type: "image",
@@ -2426,12 +2428,12 @@ var VisionTocExtractionSchema = z2.object({
 var VisionTocExtractor = class extends VisionLLMComponent {
   firstBatchSize;
   secondBatchSize;
-  constructor(logger, model, outputPath, options, fallbackModel, aggregator) {
+  constructor(logger, model, artifactDir, options, fallbackModel, aggregator) {
     super(
       logger,
       model,
       "VisionTocExtractor",
-      outputPath,
+      artifactDir,
       options,
       fallbackModel,
       aggregator ?? new LLMTokenUsageAggregator()
@@ -2564,7 +2566,7 @@ var VisionTocExtractor = class extends VisionLLMComponent {
     const imageContents = [];
     for (let pageNo = startPage; pageNo <= endPage; pageNo++) {
       const imagePath = path2.resolve(
-        this.outputPath,
+        this.artifactDir,
         `pages/page_${pageNo - 1}.png`
       );
       const imageData = new Uint8Array(fs2.readFileSync(imagePath));
@@ -2974,12 +2976,12 @@ var PageRangeParser = class extends VisionLLMComponent {
   SAMPLE_SIZE = 3;
   MAX_PATTERN_RETRIES = 19;
   SIZE_TOLERANCE = 5;
-  constructor(logger, model, outputPath, maxRetries = 3, fallbackModel, aggregator, abortSignal) {
+  constructor(logger, model, artifactDir, maxRetries = 3, fallbackModel, aggregator, abortSignal) {
     super(
       logger,
       model,
       "PageRangeParser",
-      outputPath,
+      artifactDir,
       { maxRetries, abortSignal },
       fallbackModel,
       aggregator ?? new LLMTokenUsageAggregator()
@@ -3178,7 +3180,7 @@ var PageRangeParser = class extends VisionLLMComponent {
     const imageContentsFromPages = pageNos.map((pageNo) => {
       const page = pages[pageNo - 1];
       const imagePath = path3.resolve(
-        this.outputPath,
+        this.artifactDir,
         `pages/page_${pageNo - 1}.png`
       );
       const imageData = new Uint8Array(fs3.readFileSync(imagePath));
@@ -4362,6 +4364,7 @@ var DocumentProcessor = class {
   captionParserBatchSize;
   captionValidatorBatchSize;
   maxRetries;
+  maxValidationRetries;
   enableFallbackRetry;
   abortSignal;
   onTokenUsage;
@@ -4391,6 +4394,7 @@ var DocumentProcessor = class {
     this.captionParserBatchSize = options.captionParserBatchSize;
     this.captionValidatorBatchSize = options.captionValidatorBatchSize;
     this.maxRetries = options.maxRetries ?? 3;
+    this.maxValidationRetries = options.maxValidationRetries ?? 3;
     this.enableFallbackRetry = options.enableFallbackRetry ?? false;
     this.abortSignal = options.abortSignal;
     this.onTokenUsage = options.onTokenUsage;
@@ -4422,7 +4426,7 @@ var DocumentProcessor = class {
    * Conversion process:
    * 1. Initialize processors and resolvers
    * 2. Normalize and filter texts
-   * 3. Clean texts and parse page ranges (parallel)
+   * 3. Clean texts and resolve page ranges
    * 4. Extract table of contents
    * 5. Convert images and tables (parallel)
    * 6. Convert chapters and link resources
@@ -4431,19 +4435,20 @@ var DocumentProcessor = class {
    *
    * @param doclingDoc - Original document extracted from Docling SDK
    * @param reportId - Report unique identifier
-   * @param outputPath - Path containing images and pages subdirectories (images/image_0.png, pages/page_0.png, etc.)
+   * @param artifactDir - Directory containing parser artifacts such as images/, pages/, and result.json
+   * @param processOptions - Per-document processing inputs such as manually verified page range mappings and TOC entries
    * @returns Document processing result with ProcessedDocument and token usage report
    *
    * @throws {TocExtractError} When TOC extraction fails
    * @throws {PageRangeParseError} When page range parsing fails
    * @throws {ConversionError} When error occurs during conversion
    */
-  async process(doclingDoc, reportId, outputPath) {
+  async process(doclingDoc, reportId, artifactDir, processOptions = {}) {
     this.logger.info("[DocumentProcessor] Starting document processing...");
     this.logger.info("[DocumentProcessor] Report ID:", reportId);
     this.usageAggregator.reset();
     this.checkAborted();
-    this.initializeProcessors(doclingDoc, outputPath);
+    this.initializeProcessors(doclingDoc, artifactDir);
     const startTimeFilter = Date.now();
     const filtered = this.normalizeAndFilterTexts(doclingDoc);
     const filteringTime = Date.now() - startTimeFilter;
@@ -4452,24 +4457,31 @@ var DocumentProcessor = class {
     );
     this.checkAborted();
     const startTimePageRange = Date.now();
-    const pageRangeMap = await this.parsePageRanges(doclingDoc);
+    const pageRangeMap = processOptions.pageRangeMap !== void 0 ? processOptions.pageRangeMap : await this.parsePageRanges(doclingDoc);
+    if (processOptions.pageRangeMap !== void 0) {
+      this.logger.info(
+        `[DocumentProcessor] Using injected page range map with ${Object.keys(pageRangeMap).length} entries`
+      );
+    }
     const pageRangeTime = Date.now() - startTimePageRange;
     this.logger.info(
-      `[DocumentProcessor] Page range parsing took ${pageRangeTime}ms`
+      `[DocumentProcessor] Page range resolution took ${pageRangeTime}ms`
     );
     this.emitTokenUsage();
     this.checkAborted();
     const startTimeToc = Date.now();
-    const tocEntries = await this.tocExtractionPipeline.extract(
-      doclingDoc,
-      filtered
-    );
+    const tocEntries = processOptions.tocEntries !== void 0 ? processOptions.tocEntries : await this.tocExtractionPipeline.extract(doclingDoc, filtered);
+    if (processOptions.tocEntries !== void 0) {
+      this.logger.info(
+        `[DocumentProcessor] Using injected TOC entries with ${tocEntries.length} top-level entries`
+      );
+    }
     const tocTime = Date.now() - startTimeToc;
     this.logger.info(`[DocumentProcessor] TOC extraction took ${tocTime}ms`);
     this.emitTokenUsage();
     this.checkAborted();
     const startTimeResources = Date.now();
-    const { images, tables, footnotes } = await this.resourceConverter.convertAll(doclingDoc, outputPath);
+    const { images, tables, footnotes } = await this.resourceConverter.convertAll(doclingDoc, artifactDir);
     const resourcesTime = Date.now() - startTimeResources;
     this.logger.info(
       `[DocumentProcessor] Resource conversion took ${resourcesTime}ms`
@@ -4513,7 +4525,7 @@ var DocumentProcessor = class {
    *
    * Sets up RefResolver, PageRangeParser, TocFinder, and TocExtractor
    */
-  initializeProcessors(doclingDoc, outputPath) {
+  initializeProcessors(doclingDoc, artifactDir) {
     this.logger.info("[DocumentProcessor] Initializing processors...");
     this.logger.info("[DocumentProcessor] - RefResolver");
     this.refResolver = new RefResolver(this.logger, doclingDoc);
@@ -4521,7 +4533,7 @@ var DocumentProcessor = class {
     this.pageRangeParser = new PageRangeParser(
       this.logger,
       this.pageRangeParserModel,
-      outputPath,
+      artifactDir,
       this.maxRetries,
       this.enableFallbackRetry ? this.fallbackModel : void 0,
       this.usageAggregator,
@@ -4534,7 +4546,8 @@ var DocumentProcessor = class {
       this.logger,
       this.tocExtractorModel,
       {
-        maxRetries: this.maxRetries
+        maxRetries: this.maxRetries,
+        maxValidationRetries: this.maxValidationRetries
       },
       this.enableFallbackRetry ? this.fallbackModel : void 0,
       this.abortSignal
@@ -4559,7 +4572,7 @@ var DocumentProcessor = class {
     this.visionTocExtractor = new VisionTocExtractor(
       this.logger,
       this.visionTocExtractorModel,
-      outputPath,
+      artifactDir,
       { maxRetries: this.maxRetries, abortSignal: this.abortSignal },
       this.enableFallbackRetry ? this.fallbackModel : void 0,
       this.usageAggregator