ocr-provenance-mcp 1.0.0

package/dist/services/chunking/heading-normalizer.js

@@ -0,0 +1,101 @@
+/**
+ * Heading Level Normalizer for Section-Aware Chunking
+ *
+ * Fixes inconsistent heading levels from Datalab OCR by detecting
+ * repeating heading patterns (e.g., "ARTICLE N") and normalizing
+ * their heading levels to the mode (most common) level within each group.
+ *
+ * @module services/chunking/heading-normalizer
+ */
+/**
+ * Patterns that identify structural heading groups in legal/organizational documents.
+ * Each regex matches the heading text (not the markdown # prefix).
+ * Bold-wrapped text (e.g., **ARTICLE 1**) is stripped before matching.
+ */
+const HEADING_PATTERNS = [
+    { name: 'ARTICLE', regex: /^ARTICLE\s+\d+/i },
+    { name: 'SECTION', regex: /^SECTION\s+\d+(\.\d+)*/i },
+    { name: 'CHAPTER', regex: /^CHAPTER\s+\d+/i },
+    { name: 'PART', regex: /^PART\s+\d+/i },
+    { name: 'TITLE', regex: /^TITLE\s+\d+/i },
+    { name: 'APPENDIX', regex: /^APPENDIX\s+[A-Z0-9]/i },
+    { name: 'SCHEDULE', regex: /^SCHEDULE\s+[A-Z0-9]/i },
+    { name: 'EXHIBIT', regex: /^EXHIBIT\s+[A-Z0-9]/i },
+];
+/**
+ * Strip bold markers (**text**) from heading text for pattern matching.
+ */
+function stripBold(text) {
+    return text.replace(/^\*\*(.+)\*\*$/, '$1').trim();
+}
+/**
+ * Compute the mode (most frequent value) of a number array.
+ * Ties are broken by preferring the smaller value.
+ */
+function computeMode(values) {
+    const counts = new Map();
+    for (const v of values) {
+        counts.set(v, (counts.get(v) ?? 0) + 1);
+    }
+    let modeValue = values[0];
+    let modeCount = 0;
+    for (const [val, count] of counts) {
+        if (count > modeCount || (count === modeCount && val < modeValue)) {
+            modeValue = val;
+            modeCount = count;
+        }
+    }
+    return modeValue;
+}
+/**
+ * Normalize heading levels in-place for consistent section hierarchy.
+ *
+ * Groups headings by structural patterns (ARTICLE N, Section N.N, etc.),
+ * then normalizes each group to use the mode heading level. This fixes
+ * Datalab OCR inconsistencies where identical structural headings get
+ * assigned different levels (e.g., ARTICLE 1 as H1 but ARTICLE 5 as H3).
+ *
+ * Only mutates `block.headingLevel` - never modifies `block.text`.
+ *
+ * @param blocks - Parsed markdown blocks (mutated in-place)
+ * @param config - Normalization configuration
+ * @returns The same blocks array (for chaining convenience)
+ */
+export function normalizeHeadingLevels(blocks, config) {
+    if (!config.enabled) {
+        return blocks;
+    }
+    const minCount = config.minPatternCount ?? 3;
+    // Build pattern groups
+    const groups = HEADING_PATTERNS.map(p => ({
+        name: p.name,
+        blockIndices: [],
+        levels: [],
+    }));
+    for (let i = 0; i < blocks.length; i++) {
+        const block = blocks[i];
+        if (block.type !== 'heading' || block.headingLevel === null || block.headingText === null) {
+            continue;
+        }
+        const cleanText = stripBold(block.headingText);
+        for (let g = 0; g < HEADING_PATTERNS.length; g++) {
+            if (HEADING_PATTERNS[g].regex.test(cleanText)) {
+                groups[g].blockIndices.push(i);
+                groups[g].levels.push(block.headingLevel);
+                break; // A heading belongs to at most one pattern group
+            }
+        }
+    }
+    // Normalize groups that meet the minimum count threshold
+    for (const group of groups) {
+        if (group.blockIndices.length < minCount) {
+            continue;
+        }
+        const targetLevel = computeMode(group.levels);
+        for (const blockIdx of group.blockIndices) {
+            blocks[blockIdx].headingLevel = targetLevel;
+        }
+    }
+    return blocks;
+}
+//# sourceMappingURL=heading-normalizer.js.map

package/dist/services/chunking/heading-normalizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heading-normalizer.js","sourceRoot":"","sources":["../../../src/services/chunking/heading-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAsBH;;;;GAIG;AACH,MAAM,gBAAgB,GAA2C;IAC/D,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE;IAC7C,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACrD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,iBAAiB,EAAE;IAC7C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE;IACvC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,eAAe,EAAE;IACzC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,uBAAuB,EAAE;IACpD,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,uBAAuB,EAAE;IACpD,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,sBAAsB,EAAE;CACnD,CAAC;AAEF;;GAEG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,MAAgB;IACnC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,IAAI,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,KAAK,GAAG,SAAS,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,GAAG,GAAG,SAAS,CAAC,EAAE,CAAC;YAClE,SAAS,GAAG,GAAG,CAAC;YAChB,SAAS,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAuB,EACvB,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,IAAI,CAAC,CAAC;IAE7C,uBAAuB;IACvB,MAAM,MAAM,GAAmB,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,YAAY,EAAE,EAAE;QAChB,MAAM,EAAE,EAAE;KACX,CAAC,CAAC,CAAC;IAEJ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,YAAY,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;YAC1F,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACjD,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC1C,MAAM,CAAC,iDAAiD;YAC1D,CAAC;QACH,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YACzC,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE9C,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;YAC1C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,GAAG,WAAW,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/services/chunking/json-block-analyzer.d.ts

@@ -0,0 +1,163 @@
+/**
+ * JSON Block Analyzer for Section-Aware Chunking
+ *
+ * Analyzes Datalab JSON block hierarchy to identify atomic (unsplittable)
+ * regions such as tables, figures, and code blocks. These regions inform
+ * the hybrid chunker where it must NOT split text.
+ *
+ * @module services/chunking/json-block-analyzer
+ */
+import { PageOffset } from '../../models/document.js';
+/** A region in the markdown text that should not be split */
+export interface AtomicRegion {
+    startOffset: number;
+    endOffset: number;
+    blockType: string;
+    pageNumber: number | null;
+}
+/**
+ * Find atomic (unsplittable) regions in the markdown text by analyzing JSON blocks.
+ *
+ * Walks the Datalab JSON block tree, locates Table, TableGroup, Figure, FigureGroup,
+ * and Code blocks, then finds their approximate positions in the markdown text using
+ * fuzzy text matching. Returns sorted, non-overlapping regions.
+ *
+ * @param jsonBlocks - The JSON block hierarchy from Datalab OCR (may be null)
+ * @param markdownText - The full markdown text to search within
+ * @param pageOffsets - Page offset information for page number assignment
+ * @returns Sorted array of AtomicRegion representing unsplittable text spans
+ */
+export declare function findAtomicRegions(jsonBlocks: Record<string, unknown> | null, markdownText: string, pageOffsets: PageOffset[]): AtomicRegion[];
+/**
+ * Check if a character offset falls within an atomic region.
+ *
+ * Uses binary search on the sorted regions array for efficient lookup.
+ *
+ * @param offset - The character offset to check
+ * @param regions - Sorted array of AtomicRegion (from findAtomicRegions)
+ * @returns The containing AtomicRegion, or null if offset is not in any region
+ */
+export declare function isOffsetInAtomicRegion(offset: number, regions: AtomicRegion[]): AtomicRegion | null;
+/** Statistics about block types found in the JSON block hierarchy */
+export interface BlockTypeStats {
+    total_blocks: number;
+    text_blocks: number;
+    table_blocks: number;
+    figure_blocks: number;
+    code_blocks: number;
+    list_blocks: number;
+    header_blocks: number;
+    footer_blocks: number;
+    heading_blocks: number;
+    page_count: number;
+    tables_per_page: number;
+    figures_per_page: number;
+    text_density: number;
+}
+/**
+ * Walk the JSON block tree and count block types to produce statistics.
+ *
+ * Recognizes: Text, Table, TableGroup, Figure, FigureGroup, Code,
+ * ListItem, List, PageHeader, PageFooter, SectionHeader, Title, Page.
+ *
+ * @param jsonBlocks - The JSON block hierarchy from Datalab OCR (may be null)
+ * @returns BlockTypeStats with counts and derived ratios
+ */
+export declare function computeBlockTypeStats(jsonBlocks: Record<string, unknown> | null): BlockTypeStats | null;
+/**
+ * Confidence scores for block types, used to compute chunk quality from
+ * the block types present in a chunk. Higher values indicate more structured
+ * and typically more reliable content.
+ */
+export declare const BLOCK_TYPE_CONFIDENCE: Record<string, number>;
+/**
+ * Compute a confidence score for a chunk based on the block types it contains.
+ *
+ * Returns the average confidence across all content types in the chunk.
+ * Unknown block types default to 0.7. An empty content types array also
+ * defaults to 0.7.
+ *
+ * @param contentTypes - Array of block type strings from the chunk
+ * @returns Confidence score between 0 and 1
+ */
+export declare function computeBlockConfidence(contentTypes: string[]): number;
+/** Information about headers and footers detected in the JSON block tree */
+export interface HeaderFooterInfo {
+    headerTexts: string[];
+    footerTexts: string[];
+    repeatedHeaders: string[];
+    repeatedFooters: string[];
+}
+/**
+ * Detect repeated headers and footers from the JSON block tree.
+ *
+ * Walks the block tree for each page, collecting PageHeader and PageFooter
+ * block texts. A text is considered "repeated" if it appears on >50% of pages
+ * with at least 2 occurrences.
+ *
+ * @param jsonBlocks - The JSON block hierarchy from Datalab OCR (may be null)
+ * @returns HeaderFooterInfo with all and repeated header/footer texts
+ */
+export declare function detectRepeatedHeadersFooters(jsonBlocks: Record<string, unknown> | null): HeaderFooterInfo;
+/**
+ * Check if a chunk's text closely matches any of the repeated header/footer texts.
+ * Uses normalized comparison (lowercased, whitespace-collapsed).
+ *
+ * @param chunkText - The chunk text to check
+ * @param repeatedTexts - Array of repeated header/footer texts
+ * @returns true if the chunk text matches a repeated header/footer
+ */
+export declare function isRepeatedHeaderFooter(chunkText: string, repeatedTexts: string[]): boolean;
+/** Extracted structure information for a table block */
+export interface TableStructure {
+    startOffset: number;
+    endOffset: number;
+    columnHeaders: string[];
+    rowCount: number;
+    columnCount: number;
+    pageNumber: number | null;
+    /** Human-readable summary of table content */
+    summary: string;
+    /** Values from the first data row (for summary generation) */
+    firstRowValues: string[];
+    /** Caption text from preceding block (e.g., "Table 1: Budget Summary") */
+    caption?: string;
+    /** Index of a prior table this continues (cross-page table detection) */
+    continuationOf?: number;
+}
+/**
+ * Extract table structures from the JSON block tree.
+ *
+ * Walks json_blocks for Table/TableGroup blocks, extracts column headers
+ * from the first row, and maps to markdown text offsets.
+ *
+ * @param jsonBlocks - The JSON block hierarchy from Datalab OCR (may be null)
+ * @param markdownText - The full markdown text to search within
+ * @param pageOffsets - Page offset information for page number assignment
+ * @returns Array of TableStructure with column headers and position info
+ */
+export declare function extractTableStructures(jsonBlocks: Record<string, unknown> | null, markdownText: string, pageOffsets: PageOffset[]): TableStructure[];
+/**
+ * Extract column headers from the first pipe-delimited row of markdown table text.
+ * Fallback when JSON block children don't contain TableRow elements.
+ */
+export declare function extractHeadersFromMarkdown(tableMarkdown: string): string[];
+/**
+ * Count table dimensions from markdown pipe-delimited text.
+ * Counts data rows (excludes header and separator rows).
+ */
+export declare function countTableDimensionsFromMarkdown(tableMarkdown: string): {
+    rowCount: number;
+    columnCount: number;
+};
+/**
+ * Extract values from the first data row (after header and separator) of markdown table.
+ */
+export declare function extractFirstDataRow(tableMarkdown: string): string[];
+/**
+ * Generate a human-readable summary of table content.
+ * Format: "Table with N rows and columns: col1, col2. Sample: val1, val2"
+ * Max 200 chars.
+ */
+export declare function generateTableSummary(columnHeaders: string[], rowCount: number, firstRowValues: string[], caption?: string): string;
+//# sourceMappingURL=json-block-analyzer.d.ts.map

package/dist/services/chunking/json-block-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-block-analyzer.d.ts","sourceRoot":"","sources":["../../../src/services/chunking/json-block-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAEtD,6DAA6D;AAC7D,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAWD;;;;;;;;;;;GAWG;AACH,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,EAC1C,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,UAAU,EAAE,GACxB,YAAY,EAAE,CA6BhB;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,GAAG,IAAI,CAuBnG;AA8cD,qEAAqE;AACrE,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GACzC,cAAc,GAAG,IAAI,CAyFvB;AAMD;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAYxD,CAAC;AAEF;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,MAAM,CAIrE;AAMD,4EAA4E;AAC5E,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAmCD;;;;;;;;;GASG;AACH,wBAAgB,4BAA4B,CAC1C,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,GACzC,gBAAgB,CA4DlB;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,MAAM,EAAE,GACtB,OAAO,CAmBT;AAMD,wDAAwD;AACxD,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,8CAA8C;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,0EAA0E;IAC1E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,EAC1C,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,UAAU,EAAE,GACxB,cAAc,EAAE,CAgFlB;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,CAc1E;AAED;;;GAGG;AACH,wBAAgB,gCAAgC,CAAC,aAAa,EAAE,MAAM,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAA;CAAE,CAyBjH;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBnE;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,MAAM,EAAE,EACvB,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,EAAE,EACxB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CAuBR"}