@heripo/document-processor 0.1.22 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +81 -4
- package/README.md +81 -4
- package/dist/index.cjs +48 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +96 -69
- package/dist/index.d.ts +96 -69
- package/dist/index.js +48 -35
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/dist/index.d.cts
CHANGED
|
@@ -1,9 +1,63 @@
|
|
|
1
1
|
import { LoggerMethods } from '@heripo/logger';
|
|
2
|
-
import { TokenUsageReport, DoclingDocument, DocumentProcessResult, Caption,
|
|
2
|
+
import { TokenUsageReport, DoclingDocument, PageRange, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
|
|
3
3
|
import { LanguageModel } from 'ai';
|
|
4
4
|
import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Table of Contents Entry
|
|
9
|
+
*
|
|
10
|
+
* Tree node representing the table of contents structure of a document.
|
|
11
|
+
*/
|
|
12
|
+
interface TocEntry {
|
|
13
|
+
/**
|
|
14
|
+
* Chapter title
|
|
15
|
+
*/
|
|
16
|
+
title: string;
|
|
17
|
+
/**
|
|
18
|
+
* Hierarchy depth (1, 2, 3...)
|
|
19
|
+
*/
|
|
20
|
+
level: number;
|
|
21
|
+
/**
|
|
22
|
+
* Starting page number
|
|
23
|
+
*/
|
|
24
|
+
pageNo: number;
|
|
25
|
+
/**
|
|
26
|
+
* Child TOC entries
|
|
27
|
+
*/
|
|
28
|
+
children?: TocEntry[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* TOC Area Search Result
|
|
32
|
+
*/
|
|
33
|
+
interface TocAreaResult {
|
|
34
|
+
/**
|
|
35
|
+
* Group or table item references corresponding to the table of contents
|
|
36
|
+
*/
|
|
37
|
+
itemRefs: string[];
|
|
38
|
+
/**
|
|
39
|
+
* TOC start page
|
|
40
|
+
*/
|
|
41
|
+
startPage: number;
|
|
42
|
+
/**
|
|
43
|
+
* TOC end page
|
|
44
|
+
*/
|
|
45
|
+
endPage: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Page Size Information
|
|
49
|
+
*/
|
|
50
|
+
interface PageSizeGroup {
|
|
51
|
+
/**
|
|
52
|
+
* Size identifier (width x height)
|
|
53
|
+
*/
|
|
54
|
+
sizeKey: string;
|
|
55
|
+
/**
|
|
56
|
+
* PDF page numbers with this size specification
|
|
57
|
+
*/
|
|
58
|
+
pageNos: number[];
|
|
59
|
+
}
|
|
60
|
+
|
|
7
61
|
/**
|
|
8
62
|
* DocumentProcessor Options
|
|
9
63
|
*/
|
|
@@ -59,6 +113,10 @@ interface DocumentProcessorOptions {
|
|
|
59
113
|
* Maximum retry count (default: 3)
|
|
60
114
|
*/
|
|
61
115
|
maxRetries?: number;
|
|
116
|
+
/**
|
|
117
|
+
* Maximum retry count for TOC validation correction feedback (default: 3)
|
|
118
|
+
*/
|
|
119
|
+
maxValidationRetries?: number;
|
|
62
120
|
/**
|
|
63
121
|
* Enable fallback retry mechanism - automatically retries with fallback model on failure (default: false)
|
|
64
122
|
* Set to true to enable automatic fallback retry with fallback model on component-specific model errors
|
|
@@ -76,6 +134,21 @@ interface DocumentProcessorOptions {
|
|
|
76
134
|
*/
|
|
77
135
|
onTokenUsage?: (report: TokenUsageReport) => void;
|
|
78
136
|
}
|
|
137
|
+
/**
|
|
138
|
+
* Per-document processing inputs.
|
|
139
|
+
*/
|
|
140
|
+
interface DocumentProcessorProcessOptions {
|
|
141
|
+
/**
|
|
142
|
+
* Precomputed PDF page number to actual document page range mapping.
|
|
143
|
+
* When provided, automatic PageRangeParser execution is skipped.
|
|
144
|
+
*/
|
|
145
|
+
pageRangeMap?: Record<number, PageRange>;
|
|
146
|
+
/**
|
|
147
|
+
* Precomputed table of contents entries.
|
|
148
|
+
* When provided, automatic TOC extraction pipeline execution is skipped.
|
|
149
|
+
*/
|
|
150
|
+
tocEntries?: TocEntry[];
|
|
151
|
+
}
|
|
79
152
|
/**
|
|
80
153
|
* DocumentProcessor
|
|
81
154
|
*
|
|
@@ -121,12 +194,13 @@ interface DocumentProcessorOptions {
|
|
|
121
194
|
* captionParserBatchSize: 10, // LLM caption parsing
|
|
122
195
|
* captionValidatorBatchSize: 10, // LLM caption validation
|
|
123
196
|
* maxRetries: 3,
|
|
197
|
+
* maxValidationRetries: 3,
|
|
124
198
|
* });
|
|
125
199
|
*
|
|
126
200
|
* const result = await processor.process(
|
|
127
201
|
* doclingDoc,
|
|
128
202
|
* 'report-001',
|
|
129
|
-
*
|
|
203
|
+
* artifactDir
|
|
130
204
|
* );
|
|
131
205
|
* ```
|
|
132
206
|
*/
|
|
@@ -142,6 +216,7 @@ declare class DocumentProcessor {
|
|
|
142
216
|
private readonly captionParserBatchSize;
|
|
143
217
|
private readonly captionValidatorBatchSize;
|
|
144
218
|
private readonly maxRetries;
|
|
219
|
+
private readonly maxValidationRetries;
|
|
145
220
|
private readonly enableFallbackRetry;
|
|
146
221
|
private readonly abortSignal?;
|
|
147
222
|
private readonly onTokenUsage?;
|
|
@@ -179,7 +254,7 @@ declare class DocumentProcessor {
|
|
|
179
254
|
* Conversion process:
|
|
180
255
|
* 1. Initialize processors and resolvers
|
|
181
256
|
* 2. Normalize and filter texts
|
|
182
|
-
* 3. Clean texts and
|
|
257
|
+
* 3. Clean texts and resolve page ranges
|
|
183
258
|
* 4. Extract table of contents
|
|
184
259
|
* 5. Convert images and tables (parallel)
|
|
185
260
|
* 6. Convert chapters and link resources
|
|
@@ -188,14 +263,15 @@ declare class DocumentProcessor {
|
|
|
188
263
|
*
|
|
189
264
|
* @param doclingDoc - Original document extracted from Docling SDK
|
|
190
265
|
* @param reportId - Report unique identifier
|
|
191
|
-
* @param
|
|
266
|
+
* @param artifactDir - Directory containing parser artifacts such as images/, pages/, and result.json
|
|
267
|
+
* @param processOptions - Per-document processing inputs such as manually verified page range mappings and TOC entries
|
|
192
268
|
* @returns Document processing result with ProcessedDocument and token usage report
|
|
193
269
|
*
|
|
194
270
|
* @throws {TocExtractError} When TOC extraction fails
|
|
195
271
|
* @throws {PageRangeParseError} When page range parsing fails
|
|
196
272
|
* @throws {ConversionError} When error occurs during conversion
|
|
197
273
|
*/
|
|
198
|
-
process(doclingDoc: DoclingDocument, reportId: string,
|
|
274
|
+
process(doclingDoc: DoclingDocument, reportId: string, artifactDir: string, processOptions?: DocumentProcessorProcessOptions): Promise<DocumentProcessResult>;
|
|
199
275
|
/**
|
|
200
276
|
* Initialize all processors and resolvers
|
|
201
277
|
*
|
|
@@ -361,8 +437,8 @@ interface ImageContent {
|
|
|
361
437
|
* Subclasses: PageRangeParser, VisionTocExtractor
|
|
362
438
|
*/
|
|
363
439
|
declare abstract class VisionLLMComponent extends BaseLLMComponent {
|
|
364
|
-
protected readonly
|
|
365
|
-
constructor(logger: LoggerMethods, model: LanguageModel, componentName: string,
|
|
440
|
+
protected readonly artifactDir: string;
|
|
441
|
+
constructor(logger: LoggerMethods, model: LanguageModel, componentName: string, artifactDir: string, options?: VisionLLMComponentOptions, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator);
|
|
366
442
|
/**
|
|
367
443
|
* Call LLM with vision capabilities using LLMCaller.callVision()
|
|
368
444
|
*
|
|
@@ -382,67 +458,13 @@ declare abstract class VisionLLMComponent extends BaseLLMComponent {
|
|
|
382
458
|
/**
|
|
383
459
|
* Build image content object for vision LLM messages
|
|
384
460
|
*
|
|
385
|
-
* @param imagePath - Path to the image file (relative to
|
|
461
|
+
* @param imagePath - Path to the image file (relative to artifactDir or absolute)
|
|
386
462
|
* @param mimeType - MIME type of the image (default: 'image/png')
|
|
387
463
|
* @returns ImageContent object for LLM message
|
|
388
464
|
*/
|
|
389
465
|
protected buildImageContent(imagePath: string, mimeType?: string): ImageContent;
|
|
390
466
|
}
|
|
391
467
|
|
|
392
|
-
/**
|
|
393
|
-
* Table of Contents Entry
|
|
394
|
-
*
|
|
395
|
-
* Tree node representing the table of contents structure of a document.
|
|
396
|
-
*/
|
|
397
|
-
interface TocEntry {
|
|
398
|
-
/**
|
|
399
|
-
* Chapter title
|
|
400
|
-
*/
|
|
401
|
-
title: string;
|
|
402
|
-
/**
|
|
403
|
-
* Hierarchy depth (1, 2, 3...)
|
|
404
|
-
*/
|
|
405
|
-
level: number;
|
|
406
|
-
/**
|
|
407
|
-
* Starting page number
|
|
408
|
-
*/
|
|
409
|
-
pageNo: number;
|
|
410
|
-
/**
|
|
411
|
-
* Child TOC entries
|
|
412
|
-
*/
|
|
413
|
-
children?: TocEntry[];
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* TOC Area Search Result
|
|
417
|
-
*/
|
|
418
|
-
interface TocAreaResult {
|
|
419
|
-
/**
|
|
420
|
-
* Group or table item references corresponding to the table of contents
|
|
421
|
-
*/
|
|
422
|
-
itemRefs: string[];
|
|
423
|
-
/**
|
|
424
|
-
* TOC start page
|
|
425
|
-
*/
|
|
426
|
-
startPage: number;
|
|
427
|
-
/**
|
|
428
|
-
* TOC end page
|
|
429
|
-
*/
|
|
430
|
-
endPage: number;
|
|
431
|
-
}
|
|
432
|
-
/**
|
|
433
|
-
* Page Size Information
|
|
434
|
-
*/
|
|
435
|
-
interface PageSizeGroup {
|
|
436
|
-
/**
|
|
437
|
-
* Size identifier (width x height)
|
|
438
|
-
*/
|
|
439
|
-
sizeKey: string;
|
|
440
|
-
/**
|
|
441
|
-
* PDF page numbers with this size specification
|
|
442
|
-
*/
|
|
443
|
-
pageNos: number[];
|
|
444
|
-
}
|
|
445
|
-
|
|
446
468
|
/**
|
|
447
469
|
* CaptionParser options
|
|
448
470
|
*/
|
|
@@ -571,7 +593,7 @@ declare class PageRangeParser extends VisionLLMComponent {
|
|
|
571
593
|
private readonly SAMPLE_SIZE;
|
|
572
594
|
private readonly MAX_PATTERN_RETRIES;
|
|
573
595
|
private readonly SIZE_TOLERANCE;
|
|
574
|
-
constructor(logger: LoggerMethods, model: LanguageModel,
|
|
596
|
+
constructor(logger: LoggerMethods, model: LanguageModel, artifactDir: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
|
|
575
597
|
/**
|
|
576
598
|
* Main parse method
|
|
577
599
|
*
|
|
@@ -1012,6 +1034,10 @@ interface TocExtractorOptions extends BaseLLMComponentOptions {
|
|
|
1012
1034
|
* Whether to skip validation entirely (default: false)
|
|
1013
1035
|
*/
|
|
1014
1036
|
skipValidation?: boolean;
|
|
1037
|
+
/**
|
|
1038
|
+
* Maximum retry count for TOC validation correction feedback (default: 3)
|
|
1039
|
+
*/
|
|
1040
|
+
maxValidationRetries?: number;
|
|
1015
1041
|
}
|
|
1016
1042
|
/**
|
|
1017
1043
|
* TocExtractor
|
|
@@ -1020,16 +1046,17 @@ interface TocExtractorOptions extends BaseLLMComponentOptions {
|
|
|
1020
1046
|
* Extends TextLLMComponent for standardized LLM call handling.
|
|
1021
1047
|
*
|
|
1022
1048
|
* When validation fails, automatically retries with correction feedback
|
|
1023
|
-
* up to
|
|
1049
|
+
* up to maxValidationRetries times before throwing.
|
|
1024
1050
|
*/
|
|
1025
1051
|
declare class TocExtractor extends TextLLMComponent {
|
|
1026
1052
|
private readonly validationOptions?;
|
|
1027
1053
|
private readonly skipValidation;
|
|
1054
|
+
private readonly maxValidationRetries;
|
|
1028
1055
|
constructor(logger: LoggerMethods, model: LanguageModel, options?: TocExtractorOptions, fallbackModel?: LanguageModel, abortSignal?: AbortSignal);
|
|
1029
1056
|
/**
|
|
1030
1057
|
* Extract TOC structure from Markdown
|
|
1031
1058
|
*
|
|
1032
|
-
* When validation fails, retries with correction feedback up to
|
|
1059
|
+
* When validation fails, retries with correction feedback up to maxValidationRetries times.
|
|
1033
1060
|
*
|
|
1034
1061
|
* @param markdown - Markdown representation of TOC area
|
|
1035
1062
|
* @param validationOverrides - Optional overrides for validation options (merged with constructor options)
|
|
@@ -1111,7 +1138,7 @@ interface VisionTocExtractorOptions extends VisionLLMComponentOptions {
|
|
|
1111
1138
|
declare class VisionTocExtractor extends VisionLLMComponent {
|
|
1112
1139
|
private readonly firstBatchSize;
|
|
1113
1140
|
private readonly secondBatchSize;
|
|
1114
|
-
constructor(logger: LoggerMethods, model: LanguageModel,
|
|
1141
|
+
constructor(logger: LoggerMethods, model: LanguageModel, artifactDir: string, options?: VisionTocExtractorOptions, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator);
|
|
1115
1142
|
/**
|
|
1116
1143
|
* Extract TOC from page images
|
|
1117
1144
|
*
|
|
@@ -1576,7 +1603,7 @@ declare class ResourceConverter {
|
|
|
1576
1603
|
*
|
|
1577
1604
|
* Runs image and table conversions in parallel, then footnotes synchronously.
|
|
1578
1605
|
*/
|
|
1579
|
-
convertAll(doclingDoc: DoclingDocument,
|
|
1606
|
+
convertAll(doclingDoc: DoclingDocument, artifactDir: string): Promise<{
|
|
1580
1607
|
images: ProcessedImage[];
|
|
1581
1608
|
tables: ProcessedTable[];
|
|
1582
1609
|
footnotes: ProcessedFootnote[];
|
|
@@ -1584,7 +1611,7 @@ declare class ResourceConverter {
|
|
|
1584
1611
|
/**
|
|
1585
1612
|
* Convert images from DoclingDocument to ProcessedImage[]
|
|
1586
1613
|
*/
|
|
1587
|
-
convertImages(doclingDoc: DoclingDocument,
|
|
1614
|
+
convertImages(doclingDoc: DoclingDocument, artifactDir: string): Promise<ProcessedImage[]>;
|
|
1588
1615
|
/**
|
|
1589
1616
|
* Convert tables from DoclingDocument to ProcessedTable[]
|
|
1590
1617
|
*/
|
|
@@ -1595,4 +1622,4 @@ declare class ResourceConverter {
|
|
|
1595
1622
|
convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
|
|
1596
1623
|
}
|
|
1597
1624
|
|
|
1598
|
-
export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
|
|
1625
|
+
export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,63 @@
|
|
|
1
1
|
import { LoggerMethods } from '@heripo/logger';
|
|
2
|
-
import { TokenUsageReport, DoclingDocument, DocumentProcessResult, Caption,
|
|
2
|
+
import { TokenUsageReport, DoclingDocument, PageRange, DocumentProcessResult, Caption, DoclingTextItem, DoclingPictureItem, DoclingTableItem, DoclingGroupItem, ProcessedImage, ProcessedTable, ProcessedFootnote, Chapter } from '@heripo/model';
|
|
3
3
|
import { LanguageModel } from 'ai';
|
|
4
4
|
import { LLMTokenUsageAggregator, ExtendedTokenUsage } from '@heripo/shared';
|
|
5
5
|
import { z } from 'zod';
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Table of Contents Entry
|
|
9
|
+
*
|
|
10
|
+
* Tree node representing the table of contents structure of a document.
|
|
11
|
+
*/
|
|
12
|
+
interface TocEntry {
|
|
13
|
+
/**
|
|
14
|
+
* Chapter title
|
|
15
|
+
*/
|
|
16
|
+
title: string;
|
|
17
|
+
/**
|
|
18
|
+
* Hierarchy depth (1, 2, 3...)
|
|
19
|
+
*/
|
|
20
|
+
level: number;
|
|
21
|
+
/**
|
|
22
|
+
* Starting page number
|
|
23
|
+
*/
|
|
24
|
+
pageNo: number;
|
|
25
|
+
/**
|
|
26
|
+
* Child TOC entries
|
|
27
|
+
*/
|
|
28
|
+
children?: TocEntry[];
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* TOC Area Search Result
|
|
32
|
+
*/
|
|
33
|
+
interface TocAreaResult {
|
|
34
|
+
/**
|
|
35
|
+
* Group or table item references corresponding to the table of contents
|
|
36
|
+
*/
|
|
37
|
+
itemRefs: string[];
|
|
38
|
+
/**
|
|
39
|
+
* TOC start page
|
|
40
|
+
*/
|
|
41
|
+
startPage: number;
|
|
42
|
+
/**
|
|
43
|
+
* TOC end page
|
|
44
|
+
*/
|
|
45
|
+
endPage: number;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Page Size Information
|
|
49
|
+
*/
|
|
50
|
+
interface PageSizeGroup {
|
|
51
|
+
/**
|
|
52
|
+
* Size identifier (width x height)
|
|
53
|
+
*/
|
|
54
|
+
sizeKey: string;
|
|
55
|
+
/**
|
|
56
|
+
* PDF page numbers with this size specification
|
|
57
|
+
*/
|
|
58
|
+
pageNos: number[];
|
|
59
|
+
}
|
|
60
|
+
|
|
7
61
|
/**
|
|
8
62
|
* DocumentProcessor Options
|
|
9
63
|
*/
|
|
@@ -59,6 +113,10 @@ interface DocumentProcessorOptions {
|
|
|
59
113
|
* Maximum retry count (default: 3)
|
|
60
114
|
*/
|
|
61
115
|
maxRetries?: number;
|
|
116
|
+
/**
|
|
117
|
+
* Maximum retry count for TOC validation correction feedback (default: 3)
|
|
118
|
+
*/
|
|
119
|
+
maxValidationRetries?: number;
|
|
62
120
|
/**
|
|
63
121
|
* Enable fallback retry mechanism - automatically retries with fallback model on failure (default: false)
|
|
64
122
|
* Set to true to enable automatic fallback retry with fallback model on component-specific model errors
|
|
@@ -76,6 +134,21 @@ interface DocumentProcessorOptions {
|
|
|
76
134
|
*/
|
|
77
135
|
onTokenUsage?: (report: TokenUsageReport) => void;
|
|
78
136
|
}
|
|
137
|
+
/**
|
|
138
|
+
* Per-document processing inputs.
|
|
139
|
+
*/
|
|
140
|
+
interface DocumentProcessorProcessOptions {
|
|
141
|
+
/**
|
|
142
|
+
* Precomputed PDF page number to actual document page range mapping.
|
|
143
|
+
* When provided, automatic PageRangeParser execution is skipped.
|
|
144
|
+
*/
|
|
145
|
+
pageRangeMap?: Record<number, PageRange>;
|
|
146
|
+
/**
|
|
147
|
+
* Precomputed table of contents entries.
|
|
148
|
+
* When provided, automatic TOC extraction pipeline execution is skipped.
|
|
149
|
+
*/
|
|
150
|
+
tocEntries?: TocEntry[];
|
|
151
|
+
}
|
|
79
152
|
/**
|
|
80
153
|
* DocumentProcessor
|
|
81
154
|
*
|
|
@@ -121,12 +194,13 @@ interface DocumentProcessorOptions {
|
|
|
121
194
|
* captionParserBatchSize: 10, // LLM caption parsing
|
|
122
195
|
* captionValidatorBatchSize: 10, // LLM caption validation
|
|
123
196
|
* maxRetries: 3,
|
|
197
|
+
* maxValidationRetries: 3,
|
|
124
198
|
* });
|
|
125
199
|
*
|
|
126
200
|
* const result = await processor.process(
|
|
127
201
|
* doclingDoc,
|
|
128
202
|
* 'report-001',
|
|
129
|
-
*
|
|
203
|
+
* artifactDir
|
|
130
204
|
* );
|
|
131
205
|
* ```
|
|
132
206
|
*/
|
|
@@ -142,6 +216,7 @@ declare class DocumentProcessor {
|
|
|
142
216
|
private readonly captionParserBatchSize;
|
|
143
217
|
private readonly captionValidatorBatchSize;
|
|
144
218
|
private readonly maxRetries;
|
|
219
|
+
private readonly maxValidationRetries;
|
|
145
220
|
private readonly enableFallbackRetry;
|
|
146
221
|
private readonly abortSignal?;
|
|
147
222
|
private readonly onTokenUsage?;
|
|
@@ -179,7 +254,7 @@ declare class DocumentProcessor {
|
|
|
179
254
|
* Conversion process:
|
|
180
255
|
* 1. Initialize processors and resolvers
|
|
181
256
|
* 2. Normalize and filter texts
|
|
182
|
-
* 3. Clean texts and
|
|
257
|
+
* 3. Clean texts and resolve page ranges
|
|
183
258
|
* 4. Extract table of contents
|
|
184
259
|
* 5. Convert images and tables (parallel)
|
|
185
260
|
* 6. Convert chapters and link resources
|
|
@@ -188,14 +263,15 @@ declare class DocumentProcessor {
|
|
|
188
263
|
*
|
|
189
264
|
* @param doclingDoc - Original document extracted from Docling SDK
|
|
190
265
|
* @param reportId - Report unique identifier
|
|
191
|
-
* @param
|
|
266
|
+
* @param artifactDir - Directory containing parser artifacts such as images/, pages/, and result.json
|
|
267
|
+
* @param processOptions - Per-document processing inputs such as manually verified page range mappings and TOC entries
|
|
192
268
|
* @returns Document processing result with ProcessedDocument and token usage report
|
|
193
269
|
*
|
|
194
270
|
* @throws {TocExtractError} When TOC extraction fails
|
|
195
271
|
* @throws {PageRangeParseError} When page range parsing fails
|
|
196
272
|
* @throws {ConversionError} When error occurs during conversion
|
|
197
273
|
*/
|
|
198
|
-
process(doclingDoc: DoclingDocument, reportId: string,
|
|
274
|
+
process(doclingDoc: DoclingDocument, reportId: string, artifactDir: string, processOptions?: DocumentProcessorProcessOptions): Promise<DocumentProcessResult>;
|
|
199
275
|
/**
|
|
200
276
|
* Initialize all processors and resolvers
|
|
201
277
|
*
|
|
@@ -361,8 +437,8 @@ interface ImageContent {
|
|
|
361
437
|
* Subclasses: PageRangeParser, VisionTocExtractor
|
|
362
438
|
*/
|
|
363
439
|
declare abstract class VisionLLMComponent extends BaseLLMComponent {
|
|
364
|
-
protected readonly
|
|
365
|
-
constructor(logger: LoggerMethods, model: LanguageModel, componentName: string,
|
|
440
|
+
protected readonly artifactDir: string;
|
|
441
|
+
constructor(logger: LoggerMethods, model: LanguageModel, componentName: string, artifactDir: string, options?: VisionLLMComponentOptions, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator);
|
|
366
442
|
/**
|
|
367
443
|
* Call LLM with vision capabilities using LLMCaller.callVision()
|
|
368
444
|
*
|
|
@@ -382,67 +458,13 @@ declare abstract class VisionLLMComponent extends BaseLLMComponent {
|
|
|
382
458
|
/**
|
|
383
459
|
* Build image content object for vision LLM messages
|
|
384
460
|
*
|
|
385
|
-
* @param imagePath - Path to the image file (relative to
|
|
461
|
+
* @param imagePath - Path to the image file (relative to artifactDir or absolute)
|
|
386
462
|
* @param mimeType - MIME type of the image (default: 'image/png')
|
|
387
463
|
* @returns ImageContent object for LLM message
|
|
388
464
|
*/
|
|
389
465
|
protected buildImageContent(imagePath: string, mimeType?: string): ImageContent;
|
|
390
466
|
}
|
|
391
467
|
|
|
392
|
-
/**
|
|
393
|
-
* Table of Contents Entry
|
|
394
|
-
*
|
|
395
|
-
* Tree node representing the table of contents structure of a document.
|
|
396
|
-
*/
|
|
397
|
-
interface TocEntry {
|
|
398
|
-
/**
|
|
399
|
-
* Chapter title
|
|
400
|
-
*/
|
|
401
|
-
title: string;
|
|
402
|
-
/**
|
|
403
|
-
* Hierarchy depth (1, 2, 3...)
|
|
404
|
-
*/
|
|
405
|
-
level: number;
|
|
406
|
-
/**
|
|
407
|
-
* Starting page number
|
|
408
|
-
*/
|
|
409
|
-
pageNo: number;
|
|
410
|
-
/**
|
|
411
|
-
* Child TOC entries
|
|
412
|
-
*/
|
|
413
|
-
children?: TocEntry[];
|
|
414
|
-
}
|
|
415
|
-
/**
|
|
416
|
-
* TOC Area Search Result
|
|
417
|
-
*/
|
|
418
|
-
interface TocAreaResult {
|
|
419
|
-
/**
|
|
420
|
-
* Group or table item references corresponding to the table of contents
|
|
421
|
-
*/
|
|
422
|
-
itemRefs: string[];
|
|
423
|
-
/**
|
|
424
|
-
* TOC start page
|
|
425
|
-
*/
|
|
426
|
-
startPage: number;
|
|
427
|
-
/**
|
|
428
|
-
* TOC end page
|
|
429
|
-
*/
|
|
430
|
-
endPage: number;
|
|
431
|
-
}
|
|
432
|
-
/**
|
|
433
|
-
* Page Size Information
|
|
434
|
-
*/
|
|
435
|
-
interface PageSizeGroup {
|
|
436
|
-
/**
|
|
437
|
-
* Size identifier (width x height)
|
|
438
|
-
*/
|
|
439
|
-
sizeKey: string;
|
|
440
|
-
/**
|
|
441
|
-
* PDF page numbers with this size specification
|
|
442
|
-
*/
|
|
443
|
-
pageNos: number[];
|
|
444
|
-
}
|
|
445
|
-
|
|
446
468
|
/**
|
|
447
469
|
* CaptionParser options
|
|
448
470
|
*/
|
|
@@ -571,7 +593,7 @@ declare class PageRangeParser extends VisionLLMComponent {
|
|
|
571
593
|
private readonly SAMPLE_SIZE;
|
|
572
594
|
private readonly MAX_PATTERN_RETRIES;
|
|
573
595
|
private readonly SIZE_TOLERANCE;
|
|
574
|
-
constructor(logger: LoggerMethods, model: LanguageModel,
|
|
596
|
+
constructor(logger: LoggerMethods, model: LanguageModel, artifactDir: string, maxRetries?: number, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator, abortSignal?: AbortSignal);
|
|
575
597
|
/**
|
|
576
598
|
* Main parse method
|
|
577
599
|
*
|
|
@@ -1012,6 +1034,10 @@ interface TocExtractorOptions extends BaseLLMComponentOptions {
|
|
|
1012
1034
|
* Whether to skip validation entirely (default: false)
|
|
1013
1035
|
*/
|
|
1014
1036
|
skipValidation?: boolean;
|
|
1037
|
+
/**
|
|
1038
|
+
* Maximum retry count for TOC validation correction feedback (default: 3)
|
|
1039
|
+
*/
|
|
1040
|
+
maxValidationRetries?: number;
|
|
1015
1041
|
}
|
|
1016
1042
|
/**
|
|
1017
1043
|
* TocExtractor
|
|
@@ -1020,16 +1046,17 @@ interface TocExtractorOptions extends BaseLLMComponentOptions {
|
|
|
1020
1046
|
* Extends TextLLMComponent for standardized LLM call handling.
|
|
1021
1047
|
*
|
|
1022
1048
|
* When validation fails, automatically retries with correction feedback
|
|
1023
|
-
* up to
|
|
1049
|
+
* up to maxValidationRetries times before throwing.
|
|
1024
1050
|
*/
|
|
1025
1051
|
declare class TocExtractor extends TextLLMComponent {
|
|
1026
1052
|
private readonly validationOptions?;
|
|
1027
1053
|
private readonly skipValidation;
|
|
1054
|
+
private readonly maxValidationRetries;
|
|
1028
1055
|
constructor(logger: LoggerMethods, model: LanguageModel, options?: TocExtractorOptions, fallbackModel?: LanguageModel, abortSignal?: AbortSignal);
|
|
1029
1056
|
/**
|
|
1030
1057
|
* Extract TOC structure from Markdown
|
|
1031
1058
|
*
|
|
1032
|
-
* When validation fails, retries with correction feedback up to
|
|
1059
|
+
* When validation fails, retries with correction feedback up to maxValidationRetries times.
|
|
1033
1060
|
*
|
|
1034
1061
|
* @param markdown - Markdown representation of TOC area
|
|
1035
1062
|
* @param validationOverrides - Optional overrides for validation options (merged with constructor options)
|
|
@@ -1111,7 +1138,7 @@ interface VisionTocExtractorOptions extends VisionLLMComponentOptions {
|
|
|
1111
1138
|
declare class VisionTocExtractor extends VisionLLMComponent {
|
|
1112
1139
|
private readonly firstBatchSize;
|
|
1113
1140
|
private readonly secondBatchSize;
|
|
1114
|
-
constructor(logger: LoggerMethods, model: LanguageModel,
|
|
1141
|
+
constructor(logger: LoggerMethods, model: LanguageModel, artifactDir: string, options?: VisionTocExtractorOptions, fallbackModel?: LanguageModel, aggregator?: LLMTokenUsageAggregator);
|
|
1115
1142
|
/**
|
|
1116
1143
|
* Extract TOC from page images
|
|
1117
1144
|
*
|
|
@@ -1576,7 +1603,7 @@ declare class ResourceConverter {
|
|
|
1576
1603
|
*
|
|
1577
1604
|
* Runs image and table conversions in parallel, then footnotes synchronously.
|
|
1578
1605
|
*/
|
|
1579
|
-
convertAll(doclingDoc: DoclingDocument,
|
|
1606
|
+
convertAll(doclingDoc: DoclingDocument, artifactDir: string): Promise<{
|
|
1580
1607
|
images: ProcessedImage[];
|
|
1581
1608
|
tables: ProcessedTable[];
|
|
1582
1609
|
footnotes: ProcessedFootnote[];
|
|
@@ -1584,7 +1611,7 @@ declare class ResourceConverter {
|
|
|
1584
1611
|
/**
|
|
1585
1612
|
* Convert images from DoclingDocument to ProcessedImage[]
|
|
1586
1613
|
*/
|
|
1587
|
-
convertImages(doclingDoc: DoclingDocument,
|
|
1614
|
+
convertImages(doclingDoc: DoclingDocument, artifactDir: string): Promise<ProcessedImage[]>;
|
|
1588
1615
|
/**
|
|
1589
1616
|
* Convert tables from DoclingDocument to ProcessedTable[]
|
|
1590
1617
|
*/
|
|
@@ -1595,4 +1622,4 @@ declare class ResourceConverter {
|
|
|
1595
1622
|
convertFootnotes(doclingDoc: DoclingDocument): ProcessedFootnote[];
|
|
1596
1623
|
}
|
|
1597
1624
|
|
|
1598
|
-
export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
|
|
1625
|
+
export { BaseLLMComponent, type BaseLLMComponentOptions, BaseValidator, type BaseValidatorOptions, CONTINUATION_MARKERS, CaptionParseError, CaptionParser, type CaptionParserOptions, CaptionProcessingPipeline, type CaptionProcessingPipelineDeps, CaptionValidationError, CaptionValidator, type CaptionValidatorOptions, ChapterConverter, DocumentProcessor, type DocumentProcessorOptions, type DocumentProcessorProcessOptions, type ImageContent, PAGE_NUMBER_PATTERN, PagePattern, PageRangeParseError, PageRangeParser, type PageSizeGroup, ResourceConverter, TOC_KEYWORDS, TextLLMComponent, type TocAreaResult, type TocContentValidationResult, TocContentValidationSchema, TocContentValidator, type TocContentValidatorOptions, type TocEntry, TocEntrySchema, TocExtractError, TocExtractionPipeline, type TocExtractionPipelineDeps, TocExtractor, type TocExtractorOptions, TocFinder, type TocFinderOptions, TocNotFoundError, TocParseError, type TocResponse, TocResponseSchema, VisionLLMComponent, type VisionLLMComponentOptions, type VisionTocExtractionResult, VisionTocExtractionSchema, VisionTocExtractor, type VisionTocExtractorOptions, extractMaxPageNumber };
|