@heripo/document-processor 0.1.28 → 0.1.30

package/dist/index.cjs

@@ -41,6 +41,7 @@ __export(index_exports, {
   ChapterConverter: () => ChapterConverter,
   DocumentProcessor: () => DocumentProcessor,
   PAGE_NUMBER_PATTERN: () => PAGE_NUMBER_PATTERN,
+  PROCESSED_DOCUMENT_SCHEMA_VERSION: () => PROCESSED_DOCUMENT_SCHEMA_VERSION,
   PagePattern: () => PagePattern,
   PageRangeParseError: () => PageRangeParseError,
   PageRangeParser: () => PageRangeParser,
@@ -619,6 +620,52 @@ var RefResolver = class {
   buildIndex(items, _prefix) {
     return new Map(items.map((item) => [item.self_ref, item]));
   }
+  /**
+   * Check whether a Docling self reference exists in the indexed document.
+   */
+  hasRef(ref) {
+    const match = ref.match(/^#\/(\w+)\//);
+    if (!match) {
+      return false;
+    }
+    const collection = match[1];
+    if (collection === "texts") {
+      return this.textMap.has(ref);
+    }
+    if (collection === "pictures") {
+      return this.pictureMap.has(ref);
+    }
+    if (collection === "tables") {
+      return this.tableMap.has(ref);
+    }
+    if (collection === "groups") {
+      return this.groupMap.has(ref);
+    }
+    return false;
+  }
+  /**
+   * Assert that a Docling self reference exists in the indexed document.
+   */
+  assertRef(ref, context) {
+    if (this.hasRef(ref)) {
+      return;
+    }
+    const message = `[RefResolver] Missing source reference in ${context}: ${ref}`;
+    this.logger.warn(message);
+    throw new Error(message);
+  }
+  /**
+   * Assert that every Docling self reference exists in the indexed document.
+   */
+  assertRefs(refs, context) {
+    const missingRefs = refs.filter((ref) => !this.hasRef(ref));
+    if (missingRefs.length === 0) {
+      return;
+    }
+    const message = `[RefResolver] Missing source references in ${context}: ${missingRefs.join(", ")}`;
+    this.logger.warn(message);
+    throw new Error(message);
+  }
   /**
    * Resolve a $ref string to the actual item
    * @param ref - Reference string (e.g., "#/texts/0")
@@ -710,6 +757,7 @@ var IdGenerator = class {
   imageCounter = 0;
   tableCounter = 0;
   footnoteCounter = 0;
+  textBlockCounter = 0;
   /**
    * Generate a chapter ID
    * @returns A chapter ID in the format "ch-001"
@@ -742,6 +790,14 @@ var IdGenerator = class {
     this.footnoteCounter++;
     return `ftn-${this.padNumber(this.footnoteCounter)}`;
   }
+  /**
+   * Generate a text block ID
+   * @returns A text block ID in the format "txt-001"
+   */
+  generateTextBlockId() {
+    this.textBlockCounter++;
+    return `txt-${this.padNumber(this.textBlockCounter)}`;
+  }
   /**
    * Reset all counters to zero
    */
@@ -750,6 +806,7 @@ var IdGenerator = class {
     this.imageCounter = 0;
     this.tableCounter = 0;
     this.footnoteCounter = 0;
+    this.textBlockCounter = 0;
   }
   /**
    * Get current counter values (for testing/debugging)
@@ -759,7 +816,8 @@ var IdGenerator = class {
       chapter: this.chapterCounter,
       image: this.imageCounter,
       table: this.tableCounter,
-      footnote: this.footnoteCounter
+      footnote: this.footnoteCounter,
+      textBlock: this.textBlockCounter
     };
   }
   /**
@@ -1065,6 +1123,7 @@ var ChapterConverter = class _ChapterConverter {
       title: _ChapterConverter.FRONT_MATTER_TITLE,
       pageNo: 1,
       level: 1,
+      sourceRefs: [],
       textBlocks: [],
       imageIds: [],
       tableIds: [],
@@ -1084,6 +1143,7 @@ var ChapterConverter = class _ChapterConverter {
         title: TextCleaner.normalize(entry.title),
         pageNo: entry.pageNo,
         level: entry.level,
+        sourceRefs: entry.sourceRefs ?? [],
         textBlocks: [],
         imageIds: [],
         tableIds: [],
@@ -1174,6 +1234,8 @@ var ChapterConverter = class _ChapterConverter {
     ).map((item) => {
       const pdfPageNo = item.prov?.[0]?.page_no ?? 1;
       return {
+        id: this.idGenerator.generateTextBlockId(),
+        sourceRef: item.self_ref,
         text: TextCleaner.normalize(item.text),
         pdfPageNo
       };
@@ -1278,6 +1340,84 @@ var ChapterConverter = class _ChapterConverter {
 };
 
 // src/converters/resource-converter.ts
+function getFiniteNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function getRowSpan(cell) {
+  const startRow = getFiniteNumber(cell.start_row_offset_idx);
+  const endRow = getFiniteNumber(cell.end_row_offset_idx);
+  const offsetSpan = startRow !== void 0 && endRow !== void 0 ? endRow - startRow : 0;
+  return Math.max(1, offsetSpan || getFiniteNumber(cell.row_span) || 1);
+}
+function getColSpan(cell) {
+  const startCol = getFiniteNumber(cell.start_col_offset_idx);
+  const endCol = getFiniteNumber(cell.end_col_offset_idx);
+  const offsetSpan = startCol !== void 0 && endCol !== void 0 ? endCol - startCol : 0;
+  return Math.max(1, offsetSpan || getFiniteNumber(cell.col_span) || 1);
+}
+function convertTableCell(cell) {
+  return {
+    text: cell.text ?? "",
+    rowSpan: getRowSpan(cell),
+    colSpan: getColSpan(cell),
+    isHeader: cell.column_header || cell.row_header || false
+  };
+}
+function getAnchorColumn(cell, rowIndex, colIndex) {
+  const startRow = getFiniteNumber(cell.start_row_offset_idx) ?? rowIndex;
+  const startCol = getFiniteNumber(cell.start_col_offset_idx) ?? colIndex;
+  return startRow === rowIndex ? startCol : void 0;
+}
+function buildGridFromRawGrid(rawGrid) {
+  return rawGrid.map((row, rowIndex) => {
+    const cellsByColumn = /* @__PURE__ */ new Map();
+    row.forEach((cell, colIndex) => {
+      const anchorColumn = getAnchorColumn(cell, rowIndex, colIndex);
+      if (anchorColumn === void 0 || cellsByColumn.has(anchorColumn)) {
+        return;
+      }
+      cellsByColumn.set(anchorColumn, convertTableCell(cell));
+    });
+    return Array.from(cellsByColumn.entries()).sort(([left], [right]) => left - right).map(([, cell]) => cell);
+  });
+}
+function buildGridFromTableCells(tableCells, rowCount) {
+  if (!tableCells || tableCells.length === 0) return [];
+  const maxRow = tableCells.reduce(
+    (max, cell) => Math.max(max, getFiniteNumber(cell.start_row_offset_idx) ?? 0),
+    rowCount - 1
+  ) + 1;
+  const rows = Array.from({ length: Math.max(0, maxRow) }, () => []);
+  for (const cell of tableCells) {
+    const row = getFiniteNumber(cell.start_row_offset_idx) ?? 0;
+    const col = getFiniteNumber(cell.start_col_offset_idx) ?? 0;
+    if (row < 0 || row >= rows.length) continue;
+    rows[row].push({ col, cell: convertTableCell(cell) });
+  }
+  return rows.map(
+    (row) => row.sort((a, b) => a.col - b.col).map(({ cell }) => cell)
+  );
+}
+function getGridColumnCount(grid) {
+  return grid.reduce(
+    (max, row) => Math.max(
+      max,
+      row.reduce((sum, cell) => sum + Math.max(1, cell.colSpan), 0)
+    ),
+    0
+  );
+}
+function buildProcessedTableGrid(table) {
+  const rawGrid = table.data.grid ?? [];
+  const rawNumRows = getFiniteNumber(table.data.num_rows);
+  const rawNumCols = getFiniteNumber(table.data.num_cols);
+  const grid = rawGrid.length > 0 ? buildGridFromRawGrid(rawGrid) : buildGridFromTableCells(table.data.table_cells, rawNumRows ?? 0);
+  return {
+    grid,
+    numRows: rawNumRows !== void 0 && rawNumRows > 0 ? rawNumRows : grid.length,
+    numCols: rawNumCols !== void 0 && rawNumCols > 0 ? rawNumCols : getGridColumnCount(grid)
+  };
+}
 var ResourceConverter = class {
   logger;
   idGenerator;
@@ -1313,12 +1453,17 @@ var ResourceConverter = class {
     this.logger.info(
       `[ResourceConverter] Converting ${doclingDoc.pictures.length} images...`
     );
-    const captionTexts = doclingDoc.pictures.map(
-      (picture) => this.captionProcessingPipeline.extractCaptionText(picture.captions)
+    const captionSources = doclingDoc.pictures.map(
+      (picture) => this.captionProcessingPipeline.extractCaptionSource(picture.captions)
+    );
+    const captionTexts = captionSources.map(
+      (captionSource) => captionSource.text
     );
     const images = doclingDoc.pictures.map(
       (picture, index) => ({
         id: this.idGenerator.generateImageId(),
+        sourceRef: picture.self_ref,
+        captionSourceRefs: captionSources[index].sourceRefs,
         path: `${artifactDir}/images/image_${index}.png`,
         pdfPageNo: picture.prov?.[0]?.page_no ?? 0
       })
@@ -1341,23 +1486,21 @@ var ResourceConverter = class {
     this.logger.info(
       `[ResourceConverter] Converting ${doclingDoc.tables.length} tables...`
     );
-    const captionTexts = doclingDoc.tables.map(
-      (table) => this.captionProcessingPipeline.extractCaptionText(table.captions)
+    const captionSources = doclingDoc.tables.map(
+      (table) => this.captionProcessingPipeline.extractCaptionSource(table.captions)
     );
-    const tables = doclingDoc.tables.map((table) => {
-      const grid = table.data.grid.map(
-        (row) => row.map((cell) => ({
-          text: cell.text,
-          rowSpan: cell.row_span ?? 1,
-          colSpan: cell.col_span ?? 1,
-          isHeader: cell.column_header || cell.row_header || false
-        }))
-      );
+    const captionTexts = captionSources.map(
+      (captionSource) => captionSource.text
+    );
+    const tables = doclingDoc.tables.map((table, index) => {
+      const { grid, numRows, numCols } = buildProcessedTableGrid(table);
       return {
         id: this.idGenerator.generateTableId(),
+        sourceRef: table.self_ref,
+        captionSourceRefs: captionSources[index].sourceRefs,
         pdfPageNo: table.prov?.[0]?.page_no ?? 0,
-        numRows: grid.length,
-        numCols: grid[0]?.length ?? 0,
+        numRows,
+        numCols,
         grid
       };
     });
@@ -1384,6 +1527,7 @@ var ResourceConverter = class {
     );
     const footnotes = footnoteItems.filter((item) => TextCleaner.isValidText(item.text)).map((item) => ({
       id: this.idGenerator.generateFootnoteId(),
+      sourceRef: item.self_ref,
       text: TextCleaner.normalize(item.text),
       pdfPageNo: item.prov?.[0]?.page_no ?? 1
     }));
@@ -3683,7 +3827,8 @@ Remember: Look for SMALL numbers in MARGINS only. Ignore figure/table/photo numb
 };
 
 // src/pipelines/caption-processing-pipeline.ts
-var CaptionProcessingPipeline = class {
+var CaptionProcessingPipeline = class _CaptionProcessingPipeline {
+  static CAPTION_JOIN_SEPARATOR = " ";
   logger;
   captionParser;
   captionValidator;
@@ -3832,23 +3977,51 @@ var CaptionProcessingPipeline = class {
     return captionsByIndex;
   }
   /**
-   * Extract caption text from resource
+   * Extract combined caption text from resource captions.
+   *
+   * Returns every caption joined with a single space; empty/whitespace-only
+   * entries are skipped. Returns `undefined` when nothing resolves to text.
    *
-   * Handles both string references and $ref resolution
+   * @deprecated Use {@link extractCaptionSource} to also receive the
+   * caption `$ref` list. This helper is retained as a thin wrapper for
+   * legacy callers.
    */
   extractCaptionText(captions) {
-    if (!captions?.[0]) {
-      return void 0;
-    }
-    const captionRef = captions[0];
-    if (typeof captionRef === "string") {
-      return captionRef;
-    }
-    if (this.refResolver && "$ref" in captionRef) {
-      const resolved = this.refResolver.resolveText(captionRef.$ref);
-      return resolved?.text;
+    return this.extractCaptionSource(captions).text;
+  }
+  /**
+   * Extract caption text and source references from resource captions.
+   *
+   * Iterates captions in order. String captions contribute to text only;
+   * `$ref` captions also push the ref onto `sourceRefs` regardless of whether
+   * the resolver finds text. Resolved/raw text is trimmed, and empty parts
+   * are skipped before joining.
+   */
+  extractCaptionSource(captions) {
+    const sourceRefs = [];
+    const textParts = [];
+    captions?.forEach((caption) => {
+      if (typeof caption === "string") {
+        const text2 = caption.trim();
+        if (text2.length > 0) {
+          textParts.push(text2);
+        }
+        return;
+      }
+      sourceRefs.push(caption.$ref);
+      const resolved = this.refResolver?.resolveText(caption.$ref);
+      const text = resolved?.text.trim();
+      if (text && text.length > 0) {
+        textParts.push(text);
+      }
+    });
+    if (textParts.length === 0) {
+      return { sourceRefs };
     }
-    return void 0;
+    return {
+      text: textParts.join(_CaptionProcessingPipeline.CAPTION_JOIN_SEPARATOR),
+      sourceRefs
+    };
   }
 };
 
@@ -4420,6 +4593,7 @@ var CaptionValidationError = class extends Error {
 };
 
 // src/document-processor.ts
+var PROCESSED_DOCUMENT_SCHEMA_VERSION = "processed-document.v2";
 var DocumentProcessor = class {
   logger;
   fallbackModel;
@@ -4570,18 +4744,26 @@ var DocumentProcessor = class {
       `[DocumentProcessor] Chapter conversion took ${chaptersTime}ms`
     );
     const startTimeAssemble = Date.now();
-    const processedDoc = this.assembleProcessedDocument(
+    const processedDoc = this.assembleProcessedDocument({
       reportId,
       pageRangeMap,
       chapters,
       images,
       tables,
-      footnotes
-    );
+      footnotes,
+      source: processOptions.source
+    });
     const assembleTime = Date.now() - startTimeAssemble;
     this.logger.info(
       `[DocumentProcessor] Document assembly took ${assembleTime}ms`
     );
+    const sourceRefValidationMode = this.resolveSourceRefValidationMode(processOptions);
+    if (sourceRefValidationMode !== "off") {
+      this.validateProcessedDocumentSourceRefs(
+        processedDoc,
+        sourceRefValidationMode
+      );
+    }
     this.logger.info("[DocumentProcessor] Document processing completed");
     return {
       document: processedDoc,
@@ -4720,15 +4902,93 @@ var DocumentProcessor = class {
     );
     return pageRangeMap;
   }
+  resolveSourceRefValidationMode(processOptions) {
+    if (processOptions.sourceRefValidationMode !== void 0) {
+      return processOptions.sourceRefValidationMode;
+    }
+    return processOptions.validateSourceRefs === true ? "error" : "off";
+  }
+  validateProcessedDocumentSourceRefs(processedDoc, mode) {
+    if (!this.refResolver) {
+      throw new Error(
+        "[DocumentProcessor] Cannot validate source references before RefResolver initialization"
+      );
+    }
+    const issues = this.collectMissingSourceRefs(
+      processedDoc,
+      this.refResolver
+    );
+    if (issues.length === 0) {
+      this.logger.info(
+        "[DocumentProcessor] Source reference validation passed"
+      );
+      return;
+    }
+    const details = issues.map((issue) => `${issue.context} (${issue.ref})`).join("; ");
+    const message = `[DocumentProcessor] Source reference validation found ${issues.length} missing reference(s): ${details}`;
+    if (mode === "warn") {
+      this.logger.warn(message);
+      return;
+    }
+    this.logger.error(message);
+    throw new Error(message);
+  }
+  collectMissingSourceRefs(processedDoc, refResolver) {
+    const issues = [];
+    const addRef = (ref, context) => {
+      if (ref !== void 0 && !refResolver.hasRef(ref)) {
+        issues.push({ context, ref });
+      }
+    };
+    const addRefs = (refs, context) => {
+      refs?.forEach((ref, index) => {
+        addRef(ref, `${context}[${index}]`);
+      });
+    };
+    const visitChapter = (chapter) => {
+      addRefs(chapter.sourceRefs, `chapter ${chapter.id} sourceRefs`);
+      chapter.textBlocks.forEach((textBlock, index) => {
+        addRef(
+          textBlock.sourceRef,
+          `chapter ${chapter.id} textBlock ${textBlock.id ?? index} sourceRef`
+        );
+      });
+      chapter.children?.forEach(visitChapter);
+    };
+    processedDoc.chapters.forEach(visitChapter);
+    processedDoc.images.forEach((image) => {
+      addRef(image.sourceRef, `image ${image.id} sourceRef`);
+      addRefs(image.captionSourceRefs, `image ${image.id} captionSourceRefs`);
+    });
+    processedDoc.tables.forEach((table) => {
+      addRef(table.sourceRef, `table ${table.id} sourceRef`);
+      addRefs(table.captionSourceRefs, `table ${table.id} captionSourceRefs`);
+    });
+    processedDoc.footnotes.forEach((footnote) => {
+      addRef(footnote.sourceRef, `footnote ${footnote.id} sourceRef`);
+    });
+    return issues;
+  }
   /**
    * Assemble the final ProcessedDocument
    *
    * Creates the ProcessedDocument structure with all converted components
    */
-  assembleProcessedDocument(reportId, pageRangeMap, chapters, images, tables, footnotes) {
+  assembleProcessedDocument(input) {
     this.logger.info("[DocumentProcessor] Assembling ProcessedDocument...");
+    const {
+      reportId,
+      pageRangeMap,
+      chapters,
+      images,
+      tables,
+      footnotes,
+      source
+    } = input;
     const processedDoc = {
       reportId,
+      schemaVersion: PROCESSED_DOCUMENT_SCHEMA_VERSION,
+      ...source !== void 0 ? { source } : {},
       pageRangeMap,
       chapters,
       images,
@@ -4780,6 +5040,7 @@ var DocumentProcessor = class {
   ChapterConverter,
   DocumentProcessor,
   PAGE_NUMBER_PATTERN,
+  PROCESSED_DOCUMENT_SCHEMA_VERSION,
   PagePattern,
   PageRangeParseError,
   PageRangeParser,