@heripo/document-processor 0.1.28 → 0.1.30

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { TokenUsageReport, DoclingDocument, PageRange, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { TokenUsageReport, DoclingDocument, PageRange, ProcessedDocumentSource, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter, DoclingReference } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -22,6 +22,10 @@ interface TocEntry {
      * Starting page number
      */
     pageNo: number;
+    /**
+     * Docling references for source nodes used to derive this TOC entry
+     */
+    sourceRefs?: string[];
     /**
      * Child TOC entries
      */
@@ -58,6 +62,8 @@ interface PageSizeGroup {
     pageNos: number[];
 }
 
+declare const PROCESSED_DOCUMENT_SCHEMA_VERSION = "processed-document.v2";
+type SourceRefValidationMode = 'off' | 'warn' | 'error';
 /**
  * DocumentProcessor Options
  */
@@ -148,6 +154,30 @@ interface DocumentProcessorProcessOptions {
      * When provided, automatic TOC extraction pipeline execution is skipped.
      */
     tocEntries?: TocEntry[];
+    /**
+     * Caller-supplied source artifact metadata for the Docling input.
+     */
+    source?: ProcessedDocumentSource;
+    /**
+     * Validate generated source references against the input Docling document.
+     *
+     * When true, missing references are treated as errors unless
+     * `sourceRefValidationMode` is provided.
+     *
+     * @experimental
+     */
+    validateSourceRefs?: boolean;
+    /**
+     * Controls how missing generated source references are handled.
+     *
+     * Explicit mode takes precedence over `validateSourceRefs`. Use `warn` for
+     * rollout visibility without failing processing, and `error` for strict
+     * provenance checks.
+     *
+     * @default 'off'
+     * @experimental
+     */
+    sourceRefValidationMode?: SourceRefValidationMode;
 }
 /**
  * DocumentProcessor
@@ -292,6 +322,9 @@ declare class DocumentProcessor {
      * Token usage is automatically tracked by PageRangeParser into the shared aggregator.
      */
     private parsePageRanges;
+    private resolveSourceRefValidationMode;
+    private validateProcessedDocumentSourceRefs;
+    private collectMissingSourceRefs;
     /**
      * Assemble the final ProcessedDocument
      *
@@ -789,6 +822,18 @@ declare class RefResolver {
      * Build an index mapping self_ref to the actual item
      */
     private buildIndex;
+    /**
+     * Check whether a Docling self reference exists in the indexed document.
+     */
+    hasRef(ref: string): boolean;
+    /**
+     * Assert that a Docling self reference exists in the indexed document.
+     */
+    assertRef(ref: string, context: string): void;
+    /**
+     * Assert that every Docling self reference exists in the indexed document.
+     */
+    assertRefs(refs: string[], context: string): void;
     /**
      * Resolve a $ref string to the actual item
      * @param ref - Reference string (e.g., "#/texts/0")
@@ -836,6 +881,7 @@ declare class RefResolver {
  * - Chapters: ch-001, ch-002, ...
  * - Images: img-001, img-002, ...
  * - Tables: tbl-001, tbl-002, ...
+ * - Text blocks: txt-001, txt-002, ...
  *
  * Each type maintains its own independent counter.
  */
@@ -844,6 +890,7 @@ declare class IdGenerator {
     private imageCounter;
     private tableCounter;
     private footnoteCounter;
+    private textBlockCounter;
     /**
      * Generate a chapter ID
      * @returns A chapter ID in the format "ch-001"
@@ -864,6 +911,11 @@ declare class IdGenerator {
      * @returns A footnote ID in the format "ftn-001"
      */
     generateFootnoteId(): string;
+    /**
+     * Generate a text block ID
+     * @returns A text block ID in the format "txt-001"
+     */
+    generateTextBlockId(): string;
     /**
      * Reset all counters to zero
      */
@@ -876,6 +928,7 @@ declare class IdGenerator {
         image: number;
         table: number;
         footnote: number;
+        textBlock: number;
     };
     /**
      * Pad a number to 3 digits with leading zeros
@@ -1500,6 +1553,17 @@ interface CaptionProcessingPipelineDeps {
     usageAggregator: LLMTokenUsageAggregator;
     abortSignal?: AbortSignal;
 }
+/**
+ * Caption text and source references extracted from Docling caption links.
+ *
+ * `text` is omitted when none of the captions resolved to non-empty text;
+ * `sourceRefs` is always present and contains every `$ref` caption in order
+ * (string captions contribute to `text` only).
+ */
+interface CaptionSourceExtraction {
+    text?: string;
+    sourceRefs: string[];
+}
 /**
  * CaptionProcessingPipeline
  *
@@ -1510,6 +1574,7 @@ interface CaptionProcessingPipelineDeps {
  * 4. Reparse failed captions with fallback model
  */
 declare class CaptionProcessingPipeline {
+    private static readonly CAPTION_JOIN_SEPARATOR;
     private readonly logger;
     private readonly captionParser;
     private readonly captionValidator;
@@ -1536,13 +1601,25 @@ declare class CaptionProcessingPipeline {
      */
     processResourceCaptions(captionTexts: Array<string | undefined>, resourceType: string): Promise<Map<number, Caption>>;
     /**
-     * Extract caption text from resource
+     * Extract combined caption text from resource captions.
      *
-     * Handles both string references and $ref resolution
+     * Returns every caption joined with a single space; empty/whitespace-only
+     * entries are skipped. Returns `undefined` when nothing resolves to text.
+     *
+     * @deprecated Use {@link extractCaptionSource} to also receive the
+     * caption `$ref` list. This helper is retained as a thin wrapper for
+     * legacy callers.
      */
-    extractCaptionText(captions: Array<string | {
-        $ref: string;
-    }> | undefined): string | undefined;
+    extractCaptionText(captions: Array<string | DoclingReference> | undefined): string | undefined;
+    /**
+     * Extract caption text and source references from resource captions.
+     *
+     * Iterates captions in order. String captions contribute to text only;
+     * `$ref` captions also push the ref onto `sourceRefs` regardless of whether
+     * the resolver finds text. Resolved/raw text is trimmed, and empty parts
+     * are skipped before joining.
+     */
+    extractCaptionSource(captions: Array<string | DoclingReference> | undefined): CaptionSourceExtraction;
 }
 
 /**
@@ -1622,4 +1699,4 @@ declare class ResourceConverter {
     convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, type CaptionSourceExtraction, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PROCESSED_DOCUMENT_SCHEMA_VERSION, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, type SourceRefValidationMode, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import { LoggerMethods } from '@heripo/logger';
-import { TokenUsageReport, DoclingDocument, PageRange, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
+import { TokenUsageReport, DoclingDocument, PageRange, ProcessedDocumentSource, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter, DoclingReference } from '@heripo/model';
 import { LanguageModel } from 'ai';
 import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
 import { z } from 'zod';
@@ -22,6 +22,10 @@ interface TocEntry {
      * Starting page number
      */
     pageNo: number;
+    /**
+     * Docling references for source nodes used to derive this TOC entry
+     */
+    sourceRefs?: string[];
     /**
      * Child TOC entries
      */
@@ -58,6 +62,8 @@ interface PageSizeGroup {
     pageNos: number[];
 }
 
+declare const PROCESSED_DOCUMENT_SCHEMA_VERSION = "processed-document.v2";
+type SourceRefValidationMode = 'off' | 'warn' | 'error';
 /**
  * DocumentProcessor Options
  */
@@ -148,6 +154,30 @@ interface DocumentProcessorProcessOptions {
      * When provided, automatic TOC extraction pipeline execution is skipped.
      */
     tocEntries?: TocEntry[];
+    /**
+     * Caller-supplied source artifact metadata for the Docling input.
+     */
+    source?: ProcessedDocumentSource;
+    /**
+     * Validate generated source references against the input Docling document.
+     *
+     * When true, missing references are treated as errors unless
+     * `sourceRefValidationMode` is provided.
+     *
+     * @experimental
+     */
+    validateSourceRefs?: boolean;
+    /**
+     * Controls how missing generated source references are handled.
+     *
+     * Explicit mode takes precedence over `validateSourceRefs`. Use `warn` for
+     * rollout visibility without failing processing, and `error` for strict
+     * provenance checks.
+     *
+     * @default 'off'
+     * @experimental
+     */
+    sourceRefValidationMode?: SourceRefValidationMode;
 }
 /**
  * DocumentProcessor
@@ -292,6 +322,9 @@ declare class DocumentProcessor {
      * Token usage is automatically tracked by PageRangeParser into the shared aggregator.
      */
     private parsePageRanges;
+    private resolveSourceRefValidationMode;
+    private validateProcessedDocumentSourceRefs;
+    private collectMissingSourceRefs;
     /**
      * Assemble the final ProcessedDocument
      *
@@ -789,6 +822,18 @@ declare class RefResolver {
      * Build an index mapping self_ref to the actual item
      */
     private buildIndex;
+    /**
+     * Check whether a Docling self reference exists in the indexed document.
+     */
+    hasRef(ref: string): boolean;
+    /**
+     * Assert that a Docling self reference exists in the indexed document.
+     */
+    assertRef(ref: string, context: string): void;
+    /**
+     * Assert that every Docling self reference exists in the indexed document.
+     */
+    assertRefs(refs: string[], context: string): void;
     /**
      * Resolve a $ref string to the actual item
      * @param ref - Reference string (e.g., "#/texts/0")
@@ -836,6 +881,7 @@ declare class RefResolver {
  * - Chapters: ch-001, ch-002, ...
  * - Images: img-001, img-002, ...
  * - Tables: tbl-001, tbl-002, ...
+ * - Text blocks: txt-001, txt-002, ...
  *
  * Each type maintains its own independent counter.
  */
@@ -844,6 +890,7 @@ declare class IdGenerator {
     private imageCounter;
     private tableCounter;
     private footnoteCounter;
+    private textBlockCounter;
     /**
      * Generate a chapter ID
      * @returns A chapter ID in the format "ch-001"
@@ -864,6 +911,11 @@ declare class IdGenerator {
      * @returns A footnote ID in the format "ftn-001"
      */
     generateFootnoteId(): string;
+    /**
+     * Generate a text block ID
+     * @returns A text block ID in the format "txt-001"
+     */
+    generateTextBlockId(): string;
     /**
      * Reset all counters to zero
      */
@@ -876,6 +928,7 @@ declare class IdGenerator {
         image: number;
         table: number;
         footnote: number;
+        textBlock: number;
     };
     /**
      * Pad a number to 3 digits with leading zeros
@@ -1500,6 +1553,17 @@ interface CaptionProcessingPipelineDeps {
     usageAggregator: LLMTokenUsageAggregator;
     abortSignal?: AbortSignal;
 }
+/**
+ * Caption text and source references extracted from Docling caption links.
+ *
+ * `text` is omitted when none of the captions resolved to non-empty text;
+ * `sourceRefs` is always present and contains every `$ref` caption in order
+ * (string captions contribute to `text` only).
+ */
+interface CaptionSourceExtraction {
+    text?: string;
+    sourceRefs: string[];
+}
 /**
  * CaptionProcessingPipeline
  *
@@ -1510,6 +1574,7 @@ interface CaptionProcessingPipelineDeps {
  * 4. Reparse failed captions with fallback model
  */
 declare class CaptionProcessingPipeline {
+    private static readonly CAPTION_JOIN_SEPARATOR;
     private readonly logger;
     private readonly captionParser;
     private readonly captionValidator;
@@ -1536,13 +1601,25 @@ declare class CaptionProcessingPipeline {
      */
     processResourceCaptions(captionTexts: Array<string | undefined>, resourceType: string): Promise<Map<number, Caption>>;
     /**
-     * Extract caption text from resource
+     * Extract combined caption text from resource captions.
      *
-     * Handles both string references and $ref resolution
+     * Returns every caption joined with a single space; empty/whitespace-only
+     * entries are skipped. Returns `undefined` when nothing resolves to text.
+     *
+     * @deprecated Use {@link extractCaptionSource} to also receive the
+     * caption `$ref` list. This helper is retained as a thin wrapper for
+     * legacy callers.
      */
-    extractCaptionText(captions: Array<string | {
-        $ref: string;
-    }> | undefined): string | undefined;
+    extractCaptionText(captions: Array<string | DoclingReference> | undefined): string | undefined;
+    /**
+     * Extract caption text and source references from resource captions.
+     *
+     * Iterates captions in order. String captions contribute to text only;
+     * `$ref` captions also push the ref onto `sourceRefs` regardless of whether
+     * the resolver finds text. Resolved/raw text is trimmed, and empty parts
+     * are skipped before joining.
+     */
+    extractCaptionSource(captions: Array<string | DoclingReference> | undefined): CaptionSourceExtraction;
 }
 
 /**
@@ -1622,4 +1699,4 @@ declare class ResourceConverter {
     convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
 }
 
-export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
+export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, type CaptionSourceExtraction, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PROCESSED_DOCUMENT_SCHEMA_VERSION, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, type SourceRefValidationMode, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };