ocr-provenance-mcp 1.0.0

package/dist/services/chunking/chunker.js

@@ -0,0 +1,566 @@
+/**
+ * Hybrid Section-Aware Chunking Service for OCR Provenance MCP System
+ *
+ * Uses markdown structure (headings, paragraphs, tables), JSON block data
+ * (for atomic region detection), and page offsets (for page tracking) to
+ * produce semantically coherent chunks with provenance records (chain_depth=2).
+ *
+ * @module services/chunking/chunker
+ */
+import { DEFAULT_CHUNKING_CONFIG, getOverlapCharacters, } from '../../models/chunk.js';
+import { ProvenanceType, } from '../../models/provenance.js';
+import { parseMarkdownBlocks, buildSectionHierarchy, getPageNumberForOffset, } from './markdown-parser.js';
+import { findAtomicRegions, isOffsetInAtomicRegion, extractTableStructures, extractHeadersFromMarkdown, countTableDimensionsFromMarkdown, extractFirstDataRow, generateTableSummary, } from './json-block-analyzer.js';
+import { normalizeHeadingLevels } from './heading-normalizer.js';
+import { mergeHeadingOnlyChunks } from './chunk-merger.js';
+/**
+ * Strip HTML tags from text and collapse whitespace.
+ * Used to clean table chunk text before FTS5 indexing.
+ */
+function stripHtmlForFTS(text) {
+    return text.replace(/<[^>]+>/g, '').replace(/\s{2,}/g, ' ');
+}
+/**
+ * Create table metadata directly from chunk text by parsing pipe-delimited markdown.
+ * Used as a fallback when offset-based table structure matching fails
+ * (common with DOCX tables where locateBlockInMarkdown cannot find the table).
+ */
+function createTableMetadataFromText(chunkText) {
+    // Only attempt if text contains pipe-delimited table patterns
+    const pipeLines = chunkText.split('\n').filter(l => l.includes('|'));
+    if (pipeLines.length < 2) {
+        return null;
+    }
+    const columnHeaders = extractHeadersFromMarkdown(chunkText);
+    const { rowCount, columnCount } = countTableDimensionsFromMarkdown(chunkText);
+    const firstRowValues = extractFirstDataRow(chunkText);
+    const summary = generateTableSummary(columnHeaders, rowCount, firstRowValues);
+    return {
+        columnHeaders,
+        rowCount,
+        columnCount: columnCount > 0 ? columnCount : columnHeaders.length,
+        summary,
+    };
+}
+function createEmptyAccumulator(startOffset) {
+    return {
+        text: '',
+        blocks: [],
+        startOffset,
+        contentTypes: new Set(),
+    };
+}
+function accumulatorHasContent(acc) {
+    return acc.text.trim().length > 0;
+}
+function addBlockToAccumulator(acc, block) {
+    if (acc.text.length > 0) {
+        acc.text += '\n\n';
+    }
+    acc.text += block.text;
+    acc.blocks.push(block);
+    acc.contentTypes.add(mapBlockTypeToContentType(block.type));
+}
+function mapBlockTypeToContentType(blockType) {
+    switch (blockType) {
+        case 'heading': return 'heading';
+        case 'table': return 'table';
+        case 'code': return 'code';
+        case 'list': return 'list';
+        case 'paragraph': return 'text';
+        default: return 'text';
+    }
+}
+// ---------------------------------------------------------------------------
+// Sentence boundary detection
+// ---------------------------------------------------------------------------
+/**
+ * Find a sentence boundary position for splitting text.
+ *
+ * Scans backward from `maxPos` looking for sentence-ending punctuation,
+ * paragraph breaks, line breaks, or spaces. Returns the position just
+ * after the boundary character (i.e., the start of the next sentence).
+ *
+ * @param text - The text to scan
+ * @param maxPos - Maximum position (typically chunkSize)
+ * @returns Position to split at
+ */
+function findSentenceBoundary(text, maxPos) {
+    const searchStart = Math.max(0, maxPos - 500);
+    // Priority 1: Sentence endings (. ? !) followed by whitespace
+    for (let i = maxPos; i >= searchStart; i--) {
+        const ch = text[i];
+        if ((ch === '.' || ch === '?' || ch === '!') && i + 1 < text.length) {
+            const next = text[i + 1];
+            if (next === ' ' || next === '\n') {
+                return i + 1; // Split after the punctuation
+            }
+        }
+    }
+    // Priority 2: Paragraph break (\n\n)
+    for (let i = maxPos; i >= searchStart + 1; i--) {
+        if (text[i] === '\n' && text[i - 1] === '\n') {
+            return i + 1;
+        }
+    }
+    // Priority 3: Line break (\n)
+    for (let i = maxPos; i >= searchStart; i--) {
+        if (text[i] === '\n') {
+            return i + 1;
+        }
+    }
+    // Priority 4: Any space
+    for (let i = maxPos; i >= searchStart; i--) {
+        if (text[i] === ' ') {
+            return i + 1;
+        }
+    }
+    // Last resort: force split at maxPos
+    return maxPos;
+}
+// ---------------------------------------------------------------------------
+// Page info determination
+// ---------------------------------------------------------------------------
+/**
+ * Determine page number and page range for a character span.
+ */
+function determinePageInfoForSpan(startOffset, endOffset, pageOffsets) {
+    if (pageOffsets.length === 0) {
+        return { pageNumber: null, pageRange: null };
+    }
+    const startPage = getPageNumberForOffset(startOffset, pageOffsets);
+    const endPage = getPageNumberForOffset(Math.max(startOffset, endOffset - 1), pageOffsets);
+    if (startPage === null) {
+        return { pageNumber: null, pageRange: null };
+    }
+    if (endPage === null || startPage === endPage) {
+        return { pageNumber: startPage, pageRange: null };
+    }
+    return {
+        pageNumber: startPage,
+        pageRange: `${startPage}-${endPage}`,
+    };
+}
+// ---------------------------------------------------------------------------
+// Main hybrid chunking function
+// ---------------------------------------------------------------------------
+/**
+ * Hybrid section-aware chunking.
+ *
+ * Uses markdown structure (headings, paragraphs, tables), JSON block data
+ * (for atomic region detection), and page offsets (for page tracking) to
+ * produce semantically coherent chunks.
+ *
+ * @param text - Full markdown text from OCR output
+ * @param pageOffsets - Page offset information for page number assignment
+ * @param jsonBlocks - JSON block hierarchy from Datalab OCR (may be null)
+ * @param config - Chunking configuration (default: 2000 chars, 10% overlap)
+ * @returns Array of ChunkResult with section context, content types, and page info
+ */
+export function chunkHybridSectionAware(text, pageOffsets, jsonBlocks, config = DEFAULT_CHUNKING_CONFIG) {
+    // 1. Empty text returns empty array
+    if (text.length === 0) {
+        return [];
+    }
+    // 2. Parse markdown blocks
+    const blocks = parseMarkdownBlocks(text, pageOffsets);
+    // 3. If blocks is empty but text is not, something is wrong
+    if (blocks.length === 0) {
+        throw new Error(`Markdown parser returned no blocks for non-empty text (${text.length} chars)`);
+    }
+    // 3.5. Normalize heading levels if configured
+    if (config.headingNormalization) {
+        normalizeHeadingLevels(blocks, config.headingNormalization);
+    }
+    // 4. Build section hierarchy
+    const sections = buildSectionHierarchy(blocks);
+    // 5. Find atomic regions from JSON blocks
+    const atomicRegions = findAtomicRegions(jsonBlocks, text, pageOffsets);
+    // 5.5. Extract table structures for column header context (Task 7.2)
+    const tableStructures = extractTableStructures(jsonBlocks, text, pageOffsets);
+    /**
+     * Find a table structure whose offset range overlaps a given block.
+     */
+    function findTableStructureForBlock(block) {
+        for (const ts of tableStructures) {
+            // Check if block overlaps this table structure
+            if (block.startOffset < ts.endOffset && block.endOffset > ts.startOffset) {
+                return ts;
+            }
+        }
+        return null;
+    }
+    /**
+     * Find table metadata for a chunk based on its offset range.
+     * Returns metadata if the chunk overlaps with a known table structure.
+     */
+    function findTableMetadata(offset, length) {
+        const end = offset + length;
+        for (const ts of tableStructures) {
+            // Check if chunk overlaps with table structure
+            if (offset <= ts.endOffset && end >= ts.startOffset) {
+                return {
+                    columnHeaders: ts.columnHeaders,
+                    rowCount: ts.rowCount,
+                    columnCount: ts.columnCount,
+                    summary: ts.summary,
+                    caption: ts.caption,
+                    continuationOf: ts.continuationOf,
+                };
+            }
+        }
+        return null;
+    }
+    /**
+     * Build a column header prefix string for a table chunk.
+     * Format: "[Table: col1 | col2 | col3] "
+     */
+    function buildTableHeaderPrefix(ts) {
+        if (ts.columnHeaders.length === 0)
+            return '';
+        return `[Table: ${ts.columnHeaders.join(' | ')}] `;
+    }
+    // 6. Walk blocks, accumulating into chunks
+    const chunks = [];
+    let accumulator = createEmptyAccumulator(0);
+    let currentSectionPath = null;
+    let currentHeadingText = null;
+    let currentHeadingLevel = null;
+    let chunkIndex = 0;
+    const overlapSize = getOverlapCharacters(config);
+    /**
+     * Flush the accumulator as a chunk and reset it.
+     */
+    function flushAccumulator(isAtomic) {
+        if (!accumulatorHasContent(accumulator)) {
+            return;
+        }
+        const chunkText = accumulator.text;
+        const startOff = accumulator.startOffset;
+        const endOff = startOff + chunkText.length;
+        const pageInfo = determinePageInfoForSpan(startOff, endOff, pageOffsets);
+        // Check if this flushed chunk overlaps with a table structure
+        const hasTableContent = accumulator.contentTypes.has('table');
+        const tableMetaForFlushed = hasTableContent
+            ? (findTableMetadata(startOff, endOff - startOff) ?? createTableMetadataFromText(chunkText))
+            : null;
+        chunks.push({
+            index: chunkIndex++,
+            text: chunkText,
+            startOffset: startOff,
+            endOffset: endOff,
+            overlapWithPrevious: 0, // Set in post-processing
+            overlapWithNext: 0, // Set in post-processing
+            pageNumber: pageInfo.pageNumber,
+            pageRange: pageInfo.pageRange,
+            headingContext: currentHeadingText,
+            headingLevel: currentHeadingLevel,
+            sectionPath: currentSectionPath,
+            contentTypes: Array.from(accumulator.contentTypes),
+            isAtomic,
+            tableMetadata: tableMetaForFlushed,
+        });
+    }
+    /**
+     * Emit a single block as an atomic chunk (table, code, or JSON-detected region).
+     * For table blocks, prepends column header context if available (Task 7.2).
+     */
+    function emitAtomicChunk(block) {
+        // Guard: reject empty blocks (same as flushAccumulator's accumulatorHasContent)
+        if (block.text.trim().length === 0) {
+            console.error(`[chunker] Skipping empty atomic block at offset ${block.startOffset}-${block.endOffset} ` +
+                `(type=${block.type}, raw length=${block.text.length})`);
+            return;
+        }
+        const startOff = block.startOffset;
+        const endOff = block.endOffset;
+        const pageInfo = determinePageInfoForSpan(startOff, endOff, pageOffsets);
+        // Task 7.2: Prepend column header context for table chunks
+        let chunkText = block.text;
+        if (block.type === 'table') {
+            // Strip HTML tags from table text for clean FTS indexing
+            chunkText = stripHtmlForFTS(chunkText);
+            const ts = findTableStructureForBlock(block);
+            if (ts) {
+                const prefix = buildTableHeaderPrefix(ts);
+                if (prefix.length > 0) {
+                    chunkText = prefix + chunkText;
+                }
+            }
+        }
+        // Post-processing guard: HTML stripping or prefix may leave empty text
+        if (chunkText.trim().length === 0) {
+            console.error(`[chunker] Atomic chunk became empty after processing at offset ${startOff}-${endOff} ` +
+                `(type=${block.type}, original length=${block.text.length})`);
+            return;
+        }
+        const tableMetaForAtomicChunk = block.type === 'table'
+            ? (findTableMetadata(startOff, endOff - startOff) ?? createTableMetadataFromText(chunkText))
+            : null;
+        chunks.push({
+            index: chunkIndex++,
+            text: chunkText,
+            startOffset: startOff,
+            endOffset: endOff,
+            overlapWithPrevious: 0,
+            overlapWithNext: 0,
+            pageNumber: pageInfo.pageNumber,
+            pageRange: pageInfo.pageRange,
+            headingContext: currentHeadingText,
+            headingLevel: currentHeadingLevel,
+            sectionPath: currentSectionPath,
+            contentTypes: [mapBlockTypeToContentType(block.type)],
+            isAtomic: true,
+            tableMetadata: tableMetaForAtomicChunk,
+        });
+    }
+    /**
+     * Emit an atomic block with size awareness: if the block exceeds
+     * maxChunkSize, split it at line boundaries (row breaks for tables,
+     * line breaks for code). Each sub-chunk inherits atomic status.
+     * For table blocks, prepends column header context to each sub-chunk (Task 7.2).
+     */
+    function emitSizedAtomicChunk(block) {
+        if (block.text.length <= config.maxChunkSize) {
+            emitAtomicChunk(block);
+            return;
+        }
+        // Task 7.2: Get table header prefix for table blocks
+        let tablePrefix = '';
+        if (block.type === 'table') {
+            const ts = findTableStructureForBlock(block);
+            if (ts) {
+                tablePrefix = buildTableHeaderPrefix(ts);
+            }
+        }
+        // Split oversized atomic block at line boundaries
+        // Strip HTML tags from table text for clean FTS indexing
+        const blockText = block.type === 'table' ? stripHtmlForFTS(block.text) : block.text;
+        let pos = 0;
+        while (pos < blockText.length) {
+            let endPos;
+            if (blockText.length - pos <= config.maxChunkSize) {
+                // Remaining text fits in one chunk
+                endPos = blockText.length;
+            }
+            else {
+                // Find last newline before maxChunkSize boundary
+                endPos = blockText.lastIndexOf('\n', pos + config.maxChunkSize);
+                if (endPos <= pos) {
+                    // No newline found within range, force split at maxChunkSize
+                    endPos = pos + config.maxChunkSize;
+                }
+            }
+            let chunkText = blockText.slice(pos, endPos);
+            if (chunkText.trim().length > 0) {
+                // Task 7.2: Prepend column header context to each table sub-chunk
+                if (tablePrefix.length > 0) {
+                    chunkText = tablePrefix + chunkText;
+                }
+                const startOff = block.startOffset + pos;
+                const endOff = block.startOffset + endPos;
+                const pageInfo = determinePageInfoForSpan(startOff, endOff, pageOffsets);
+                const tableMetaForSubChunk = block.type === 'table'
+                    ? (findTableMetadata(startOff, endOff - startOff) ?? createTableMetadataFromText(chunkText))
+                    : null;
+                chunks.push({
+                    index: chunkIndex++,
+                    text: chunkText,
+                    startOffset: startOff,
+                    endOffset: endOff,
+                    overlapWithPrevious: 0,
+                    overlapWithNext: 0,
+                    pageNumber: pageInfo.pageNumber,
+                    pageRange: pageInfo.pageRange,
+                    headingContext: currentHeadingText,
+                    headingLevel: currentHeadingLevel,
+                    sectionPath: currentSectionPath,
+                    contentTypes: [mapBlockTypeToContentType(block.type)],
+                    isAtomic: true,
+                    tableMetadata: tableMetaForSubChunk,
+                });
+            }
+            // Advance past the split point (skip newline if present)
+            pos = endPos < blockText.length && blockText[endPos] === '\n' ? endPos + 1 : endPos;
+        }
+    }
+    for (let blockIdx = 0; blockIdx < blocks.length; blockIdx++) {
+        const block = blocks[blockIdx];
+        // Skip empty and page_marker blocks
+        if (block.type === 'empty' || block.type === 'page_marker') {
+            continue;
+        }
+        // Get section info for this block
+        const sectionNode = sections.get(blockIdx);
+        if (sectionNode) {
+            currentSectionPath = sectionNode.path;
+        }
+        if (block.type === 'heading') {
+            // Flush accumulator before starting new section
+            flushAccumulator(false);
+            // Update heading context
+            currentHeadingText = block.headingText;
+            currentHeadingLevel = block.headingLevel;
+            // Start new accumulator with the heading
+            accumulator = createEmptyAccumulator(block.startOffset);
+            addBlockToAccumulator(accumulator, block);
+        }
+        else if (block.type === 'table' || block.type === 'code') {
+            // Size-aware atomic treatment: only emit as atomic if block is large enough
+            // to produce meaningful standalone embeddings. Small tables/code blocks are
+            // merged into surrounding content for better embedding quality.
+            const minAtomicSize = Math.floor(config.chunkSize / 4);
+            if (block.text.length >= minAtomicSize) {
+                // Large table/code → atomic chunk (with oversized splitting)
+                flushAccumulator(false);
+                emitSizedAtomicChunk(block);
+                accumulator = createEmptyAccumulator(block.endOffset);
+            }
+            else {
+                // Small table/code → treat as regular content, merge into accumulator
+                addBlockToAccumulator(accumulator, block);
+                // Check if accumulator exceeds chunk size (same logic as paragraph branch)
+                if (accumulator.text.length > config.chunkSize) {
+                    const splitPos = findSentenceBoundary(accumulator.text, config.chunkSize);
+                    const fullText = accumulator.text;
+                    const savedStartOffset = accumulator.startOffset;
+                    const savedContentTypes = new Set(accumulator.contentTypes);
+                    accumulator.text = fullText.slice(0, splitPos);
+                    flushAccumulator(false);
+                    const remainder = fullText.slice(splitPos);
+                    accumulator = createEmptyAccumulator(savedStartOffset + splitPos);
+                    accumulator.text = remainder;
+                    accumulator.contentTypes = savedContentTypes;
+                }
+            }
+        }
+        else {
+            // Regular content (paragraph, list)
+            // Check if this block overlaps an atomic region from JSON blocks
+            const atomicRegion = isOffsetInAtomicRegion(block.startOffset, atomicRegions);
+            if (atomicRegion) {
+                // This was detected as part of an atomic region by JSON analysis
+                flushAccumulator(false);
+                emitAtomicChunk(block);
+                accumulator = createEmptyAccumulator(block.endOffset);
+            }
+            else {
+                // Add to accumulator
+                addBlockToAccumulator(accumulator, block);
+                // Check if accumulator exceeds chunk size
+                if (accumulator.text.length > config.chunkSize) {
+                    // Need to split - find sentence boundary
+                    const splitPos = findSentenceBoundary(accumulator.text, config.chunkSize);
+                    // Save state before flush
+                    const fullText = accumulator.text;
+                    const savedStartOffset = accumulator.startOffset;
+                    const savedContentTypes = new Set(accumulator.contentTypes);
+                    // Truncate accumulator text to split point and flush
+                    accumulator.text = fullText.slice(0, splitPos);
+                    flushAccumulator(false);
+                    // Keep the remainder in a new accumulator
+                    const remainder = fullText.slice(splitPos);
+                    accumulator = createEmptyAccumulator(savedStartOffset + splitPos);
+                    accumulator.text = remainder;
+                    accumulator.contentTypes = savedContentTypes;
+                    // If remainder still exceeds maxChunkSize, keep splitting
+                    while (accumulator.text.length > config.chunkSize) {
+                        const innerSplitPos = findSentenceBoundary(accumulator.text, config.chunkSize);
+                        const innerFullText = accumulator.text;
+                        const innerStartOffset = accumulator.startOffset;
+                        const innerContentTypes = new Set(accumulator.contentTypes);
+                        accumulator.text = innerFullText.slice(0, innerSplitPos);
+                        flushAccumulator(false);
+                        const innerRemainder = innerFullText.slice(innerSplitPos);
+                        accumulator = createEmptyAccumulator(innerStartOffset + innerSplitPos);
+                        accumulator.text = innerRemainder;
+                        accumulator.contentTypes = innerContentTypes;
+                    }
+                }
+            }
+        }
+    }
+    // Flush any remaining content
+    flushAccumulator(false);
+    // 8.5. Merge heading-only tiny chunks with neighbors
+    const mergedChunks = mergeHeadingOnlyChunks(chunks, config.minChunkSize ?? 100);
+    // Replace chunks array contents with merged results
+    chunks.length = 0;
+    chunks.push(...mergedChunks);
+    // 9. Set overlap values for non-atomic chunks
+    for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        if (chunk.isAtomic) {
+            // Atomic chunks never participate in overlap
+            chunk.overlapWithPrevious = 0;
+            chunk.overlapWithNext = 0;
+            continue;
+        }
+        // Set overlapWithPrevious for non-first, non-atomic chunks
+        if (i > 0 && !chunks[i - 1].isAtomic) {
+            chunk.overlapWithPrevious = overlapSize;
+        }
+        // Set overlapWithNext for non-last, non-atomic chunks
+        if (i < chunks.length - 1 && !chunks[i + 1].isAtomic) {
+            chunk.overlapWithNext = overlapSize;
+        }
+    }
+    return chunks;
+}
+// ---------------------------------------------------------------------------
+// Provenance creation
+// ---------------------------------------------------------------------------
+/**
+ * Create provenance parameters for a chunk.
+ *
+ * Generates a CreateProvenanceParams object suitable for creating
+ * a CHUNK provenance record (chain_depth=2).
+ *
+ * @param params - Chunk provenance parameters
+ * @returns CreateProvenanceParams ready for insertProvenance
+ */
+export function createChunkProvenance(params) {
+    const { chunk, chunkTextHash, ocrProvenanceId, documentProvenanceId, ocrContentHash, fileHash, totalChunks, processingDurationMs, config = DEFAULT_CHUNKING_CONFIG, } = params;
+    // Build location information
+    const location = {
+        chunk_index: chunk.index,
+        character_start: chunk.startOffset,
+        character_end: chunk.endOffset,
+    };
+    // Add page info only if available
+    if (chunk.pageNumber !== null) {
+        location.page_number = chunk.pageNumber;
+    }
+    if (chunk.pageRange !== null) {
+        location.page_range = chunk.pageRange;
+    }
+    return {
+        type: ProvenanceType.CHUNK,
+        source_type: 'CHUNKING',
+        source_id: ocrProvenanceId,
+        root_document_id: documentProvenanceId,
+        content_hash: chunkTextHash,
+        input_hash: ocrContentHash,
+        file_hash: fileHash,
+        processor: 'chunker',
+        processor_version: '2.0.0',
+        processing_params: {
+            chunk_size: config.chunkSize,
+            overlap_percent: config.overlapPercent,
+            max_chunk_size: config.maxChunkSize,
+            strategy: 'hybrid_section',
+            chunk_index: chunk.index,
+            total_chunks: totalChunks,
+            character_start: chunk.startOffset,
+            character_end: chunk.endOffset,
+            heading_context: chunk.headingContext ?? null,
+            section_path: chunk.sectionPath ?? null,
+            is_atomic: chunk.isAtomic,
+            content_types: chunk.contentTypes,
+        },
+        processing_duration_ms: processingDurationMs ?? null,
+        location,
+    };
+}
+export { DEFAULT_CHUNKING_CONFIG } from '../../models/chunk.js';
+//# sourceMappingURL=chunker.js.map

package/dist/services/chunking/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/services/chunking/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAGL,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EACL,cAAc,GAIf,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,sBAAsB,GAEvB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,iBAAiB,EACjB,sBAAsB,EACtB,sBAAsB,EACtB,0BAA0B,EAC1B,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,GAErB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AAE3D;;;GAGG;AACH,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;AAC9D,CAAC;AAED;;;;GAIG;AACH,SAAS,2BAA2B,CAAC,SAAiB;IACpD,8DAA8D;IAC9D,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACrE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,aAAa,GAAG,0BAA0B,CAAC,SAAS,CAAC,CAAC;IAC5D,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,gCAAgC,CAAC,SAAS,CAAC,CAAC;IAC9E,MAAM,cAAc,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,oBAAoB,CAAC,aAAa,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;IAE9E,OAAO;QACL,aAAa;QACb,QAAQ;QACR,WAAW,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM;QACjE,OAAO;KACR,CAAC;AACJ,CAAC;AAqCD,SAAS,sBAAsB,CAAC,WAAmB;IACjD,OAAO;QACL,IAAI,EAAE,EAAE;QACR,MAAM,EAAE,EAAE;QACV,WAAW;QACX,YAAY,EAAE,IAAI,GAAG,EAAE;KACxB,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAgB;IAC7C,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAgB,EAAE,KAAoB;IACnE,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC;IACrB,CAAC;IACD,GAAG,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;IACvB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,yBAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,yBAAyB,CAAC,SAAiB;IAClD,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,SAAS,CAAC,CAAC,OAAO,SAAS,CAAC;QACjC,KAAK,OAAO,CAAC,CAAC,OAAO,OAAO,CAAC;QAC7B,KAAK,MAAM,CAAC,CAAC,OAAO,MAAM,CAAC;QAC3B,KAAK,MAAM,CAAC,CAAC,OAAO,MAAM,CAAC;QAC3B,KAAK,WAAW,CAAC,CAAC,OAAO,MAAM,CAAC;QAChC,OAAO,CAAC,CAAC,OAAO,MAAM,CAAC;IACzB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,SAAS,oBAAoB,CAAC,IAAY,EAAE,MAAc;IACxD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,GAAG,CAAC,CAAC;IAE9C,8DAA8D;IAC9D,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,IAAI,CAAC,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YACpE,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACzB,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,8BAA8B;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC7C,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,wBAAwB;IACxB,KAAK,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IAED,qCAAqC;IACrC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E;;GAEG;AACH,SAAS,wBAAwB,CAC/B,WAAmB,EACnB,SAAiB,EACjB,WAAyB;IAEzB,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,SAAS,GAAG,sBAAsB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,sBAAsB,CACpC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,GAAG,CAAC,CAAC,EACpC,WAAW,CACZ,CAAC;IAEF,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QAC9C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACpD,CAAC;IAED,OAAO;QACL,UAAU,EAAE,SAAS;QACrB,SAAS,EAAE,GAAG,SAAS,IAAI,OAAO,EAAE;KACrC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,gCAAgC;AAChC,8EAA8E;AAE9E;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,uBAAuB,CACrC,IAAY,EACZ,WAAyB,EACzB,UAA0C,EAC1C,SAAyB,uBAAuB;IAEhD,oCAAoC;IACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,2BAA2B;IAC3B,MAAM,MAAM,GAAG,mBAAmB,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAEtD,4DAA4D;IAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CACb,0DAA0D,IAAI,CAAC,MAAM,SAAS,CAC/E,CAAC;IACJ,CAAC;IAED,8CAA8C;IAC9C,IAAI,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAChC,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAC9D,CAAC;IAED,6BAA6B;IAC7B,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAE/C,0CAA0C;IAC1C,MAAM,aAAa,GAAG,iBAAiB,CAAC,UAAU,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IAEvE,qEAAqE;IACrE,MAAM,eAAe,GAAG,sBAAsB,CAAC,UAAU,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IAE9E;;OAEG;IACH,SAAS,0BAA0B,CAAC,KAAoB;QACtD,KAAK,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;YACjC,+CAA+C;YAC/C,IAAI,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;gBACzE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,SAAS,iBAAiB,CACxB,MAAc,EACd,MAAc;QAEd,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;QAC5B,KAAK,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;YACjC,+CAA+C;YAC/C,IAAI,MAAM,IAAI,EAAE,CAAC,SAAS,IAAI,GAAG,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBACpD,OAAO;oBACL,aAAa,EAAE,EAAE,CAAC,aAAa;oBAC/B,QAAQ,EAAE,EAAE,CAAC,QAAQ;oBACrB,WAAW,EAAE,EAAE,CAAC,WAAW;oBAC3B,OAAO,EAAE,EAAE,CAAC,OAAO;oBACnB,OAAO,EAAE,EAAE,CAAC,OAAO;oBACnB,cAAc,EAAE,EAAE,CAAC,cAAc;iBAClC,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACH,SAAS,sBAAsB,CAAC,EAAkB;QAChD,IAAI,EAAE,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAC7C,OAAO,WAAW,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACrD,CAAC;IAED,2CAA2C;IAC3C,MAAM,MAAM,GAAkB,EAAE,CAAC;IACjC,IAAI,WAAW,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAC5C,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,IAAI,kBAAkB,GAAkB,IAAI,CAAC;IAC7C,IAAI,mBAAmB,GAAkB,IAAI,CAAC;IAC9C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,WAAW,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAEjD;;OAEG;IACH,SAAS,gBAAgB,CAAC,QAAiB;QACzC,IAAI,CAAC,qBAAqB,CAAC,WAAW,CAAC,EAAE,CAAC;YACxC,OAAO;QACT,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC;QACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,WAAW,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;QAE3C,MAAM,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEzE,8DAA8D;QAC9D,MAAM,eAAe,GAAG,WAAW,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC9D,MAAM,mBAAmB,GAAG,eAAe;YACzC,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;YAC5F,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,UAAU,EAAE;YACnB,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,QAAQ;YACrB,SAAS,EAAE,MAAM;YACjB,mBAAmB,EAAE,CAAC,EAAE,yBAAyB;YACjD,eAAe,EAAE,CAAC,EAAM,yBAAyB;YACjD,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,cAAc,EAAE,kBAAkB;YAClC,YAAY,EAAE,mBAAmB;YACjC,WAAW,EAAE,kBAAkB;YAC/B,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC;YAClD,QAAQ;YACR,aAAa,EAAE,mBAAmB;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,SAAS,eAAe,CAAC,KAAoB;QAC3C,gFAAgF;QAChF,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CACX,mDAAmD,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,SAAS,GAAG;gBAC1F,SAAS,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CACxD,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;QAC/B,MAAM,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;QAEzE,2DAA2D;QAC3D,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC;QAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,yDAAyD;YACzD,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;YACvC,MAAM,EAAE,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,EAAE,EAAE,CAAC;gBACP,MAAM,MAAM,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;gBAC1C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;gBACjC,CAAC;YACH,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,CAAC,KAAK,CACX,kEAAkE,QAAQ,IAAI,MAAM,GAAG;gBACvF,SAAS,KAAK,CAAC,IAAI,qBAAqB,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAC7D,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,uBAAuB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO;YACpD,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;YAC5F,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,UAAU,EAAE;YACnB,IAAI,EAAE,SAAS;YACf,WAAW,EAAE,QAAQ;YACrB,SAAS,EAAE,MAAM;YACjB,mBAAmB,EAAE,CAAC;YACtB,eAAe,EAAE,CAAC;YAClB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,cAAc,EAAE,kBAAkB;YAClC,YAAY,EAAE,mBAAmB;YACjC,WAAW,EAAE,kBAAkB;YAC/B,YAAY,EAAE,CAAC,yBAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACrD,QAAQ,EAAE,IAAI;YACd,aAAa,EAAE,uBAAuB;SACvC,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,SAAS,oBAAoB,CAAC,KAAoB;QAChD,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YAC7C,eAAe,CAAC,KAAK,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;QAED,qDAAqD;QACrD,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,0BAA0B,CAAC,KAAK,CAAC,CAAC;YAC7C,IAAI,EAAE,EAAE,CAAC;gBACP,WAAW,GAAG,sBAAsB,CAAC,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,kDAAkD;QAClD,yDAAyD;QACzD,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACpF,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,OAAO,GAAG,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,MAAc,CAAC;YAEnB,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;gBAClD,mCAAmC;gBACnC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,MAAM,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;gBAChE,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBAClB,6DAA6D;oBAC7D,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,YAAY,CAAC;gBACrC,CAAC;YACH,CAAC;YAED,IAAI,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,kEAAkE;gBAClE,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3B,SAAS,GAAG,WAAW,GAAG,SAAS,CAAC;gBACtC,CAAC;gBAED,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,GAAG,GAAG,CAAC;gBACzC,MAAM,MAAM,GAAG,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;gBAC1C,MAAM,QAAQ,GAAG,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;gBAEzE,MAAM,oBAAoB,GAAG,KAAK,CAAC,IAAI,KAAK,OAAO;oBACjD,CAAC,CAAC,CAAC,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,CAAC,IAAI,2BAA2B,CAAC,SAAS,CAAC,CAAC;oBAC5F,CAAC,CAAC,IAAI,CAAC;gBAET,MAAM,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,UAAU,EAAE;oBACnB,IAAI,EAAE,SAAS;oBACf,WAAW,EAAE,QAAQ;oBACrB,SAAS,EAAE,MAAM;oBACjB,mBAAmB,EAAE,CAAC;oBACtB,eAAe,EAAE,CAAC;oBAClB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,SAAS,EAAE,QAAQ,CAAC,SAAS;oBAC7B,cAAc,EAAE,kBAAkB;oBAClC,YAAY,EAAE,mBAAmB;oBACjC,WAAW,EAAE,kBAAkB;oBAC/B,YAAY,EAAE,CAAC,yBAAyB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACrD,QAAQ,EAAE,IAAI;oBACd,aAAa,EAAE,oBAAoB;iBACpC,CAAC,CAAC;YACL,CAAC;YAED,yDAAyD;YACzD,GAAG,GAAG,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACtF,CAAC;IACH,CAAC;IAED,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAE/B,oCAAoC;QACpC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3D,SAAS;QACX,CAAC;QAED,kCAAkC;QAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC3C,IAAI,WAAW,EAAE,CAAC;YAChB,kBAAkB,GAAG,WAAW,CAAC,IAAI,CAAC;QACxC,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC7B,gDAAgD;YAChD,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAExB,yBAAyB;YACzB,kBAAkB,GAAG,KAAK,CAAC,WAAW,CAAC;YACvC,mBAAmB,GAAG,KAAK,CAAC,YAAY,CAAC;YAEzC,yCAAyC;YACzC,WAAW,GAAG,sBAAsB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACxD,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAE5C,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3D,4EAA4E;YAC5E,4EAA4E;YAC5E,gEAAgE;YAChE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;YAEvD,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;gBACvC,6DAA6D;gBAC7D,gBAAgB,CAAC,KAAK,CAAC,CAAC;gBACxB,oBAAoB,CAAC,KAAK,CAAC,CAAC;gBAC5B,WAAW,GAAG,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACxD,CAAC;iBAAM,CAAC;gBACN,sEAAsE;gBACtE,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;gBAE1C,2EAA2E;gBAC3E,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC/C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC1E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC;oBAClC,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,CAAC;oBACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;oBAE5D,WAAW,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBAC/C,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBAExB,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC3C,WAAW,GAAG,sBAAsB,CAAC,gBAAgB,GAAG,QAAQ,CAAC,CAAC;oBAClE,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;oBAC7B,WAAW,CAAC,YAAY,GAAG,iBAAiB,CAAC;gBAC/C,CAAC;YACH,CAAC;QAEH,CAAC;aAAM,CAAC;YACN,oCAAoC;YACpC,iEAAiE;YACjE,MAAM,YAAY,GAAG,sBAAsB,CAAC,KAAK,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC;YAC9E,IAAI,YAAY,EAAE,CAAC;gBACjB,iEAAiE;gBACjE,gBAAgB,CAAC,KAAK,CAAC,CAAC;gBACxB,eAAe,CAAC,KAAK,CAAC,CAAC;gBACvB,WAAW,GAAG,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACxD,CAAC;iBAAM,CAAC;gBACN,qBAAqB;gBACrB,qBAAqB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;gBAE1C,0CAA0C;gBAC1C,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC/C,yCAAyC;oBACzC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;oBAE1E,0BAA0B;oBAC1B,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC;oBAClC,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,CAAC;oBACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;oBAE5D,qDAAqD;oBACrD,WAAW,CAAC,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;oBAC/C,gBAAgB,CAAC,KAAK,CAAC,CAAC;oBAExB,0CAA0C;oBAC1C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;oBAC3C,WAAW,GAAG,sBAAsB,CAAC,gBAAgB,GAAG,QAAQ,CAAC,CAAC;oBAClE,WAAW,CAAC,IAAI,GAAG,SAAS,CAAC;oBAC7B,WAAW,CAAC,YAAY,GAAG,iBAAiB,CAAC;oBAE7C,0DAA0D;oBAC1D,OAAO,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;wBAClD,MAAM,aAAa,GAAG,oBAAoB,CACxC,WAAW,CAAC,IAAI,EAChB,MAAM,CAAC,SAAS,CACjB,CAAC;wBACF,MAAM,aAAa,GAAG,WAAW,CAAC,IAAI,CAAC;wBACvC,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,CAAC;wBACjD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;wBAE5D,WAAW,CAAC,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;wBACzD,gBAAgB,CAAC,KAAK,CAAC,CAAC;wBAExB,MAAM,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;wBAC1D,WAAW,GAAG,sBAAsB,CAAC,gBAAgB,GAAG,aAAa,CAAC,CAAC;wBACvE,WAAW,CAAC,IAAI,GAAG,cAAc,CAAC;wBAClC,WAAW,CAAC,YAAY,GAAG,iBAAiB,CAAC;oBAC/C,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,8BAA8B;IAC9B,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAExB,qDAAqD;IACrD,MAAM,YAAY,GAAG,sBAAsB,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC;IAChF,oDAAoD;IACpD,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,MAAM,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAE7B,8CAA8C;IAC9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,6CAA6C;YAC7C,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;YAC9B,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,2DAA2D;QAC3D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrC,KAAK,CAAC,mBAAmB,GAAG,WAAW,CAAC;QAC1C,CAAC;QAED,sDAAsD;QACtD,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YACrD,KAAK,CAAC,eAAe,GAAG,WAAW,CAAC;QACtC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAA6B;IACjE,MAAM,EACJ,KAAK,EACL,aAAa,EACb,eAAe,EACf,oBAAoB,EACpB,cAAc,EACd,QAAQ,EACR,WAAW,EACX,oBAAoB,EACpB,MAAM,GAAG,uBAAuB,GACjC,GAAG,MAAM,CAAC;IAEX,6BAA6B;IAC7B,MAAM,QAAQ,GAAuB;QACnC,WAAW,EAAE,KAAK,CAAC,KAAK;QACxB,eAAe,EAAE,KAAK,CAAC,WAAW;QAClC,aAAa,EAAE,KAAK,CAAC,SAAS;KAC/B,CAAC;IAEF,kCAAkC;IAClC,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;QAC9B,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;IAC1C,CAAC;IACD,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;QAC7B,QAAQ,CAAC,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC;IACxC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,cAAc,CAAC,KAAK;QAC1B,WAAW,EAAE,UAAwB;QACrC,SAAS,EAAE,eAAe;QAC1B,gBAAgB,EAAE,oBAAoB;QACtC,YAAY,EAAE,aAAa;QAC3B,UAAU,EAAE,cAAc;QAC1B,SAAS,EAAE,QAAQ;QACnB,SAAS,EAAE,SAAS;QACpB,iBAAiB,EAAE,OAAO;QAC1B,iBAAiB,EAAE;YACjB,UAAU,EAAE,MAAM,CAAC,SAAS;YAC5B,eAAe,EAAE,MAAM,CAAC,cAAc;YACtC,cAAc,EAAE,MAAM,CAAC,YAAY;YACnC,QAAQ,EAAE,gBAAgB;YAC1B,WAAW,EAAE,KAAK,CAAC,KAAK;YACxB,YAAY,EAAE,WAAW;YACzB,eAAe,EAAE,KAAK,CAAC,WAAW;YAClC,aAAa,EAAE,KAAK,CAAC,SAAS;YAC9B,eAAe,EAAE,KAAK,CAAC,cAAc,IAAI,IAAI;YAC7C,YAAY,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;YACvC,SAAS,EAAE,KAAK,CAAC,QAAQ;YACzB,aAAa,EAAE,KAAK,CAAC,YAAY;SAClC;QACD,sBAAsB,EAAE,oBAAoB,IAAI,IAAI;QACpD,QAAQ;KACT,CAAC;AACJ,CAAC;AAID,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/services/chunking/heading-normalizer.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Heading Level Normalizer for Section-Aware Chunking
+ *
+ * Fixes inconsistent heading levels from Datalab OCR by detecting
+ * repeating heading patterns (e.g., "ARTICLE N") and normalizing
+ * their heading levels to the mode (most common) level within each group.
+ *
+ * @module services/chunking/heading-normalizer
+ */
+import { MarkdownBlock } from './markdown-parser.js';
+/** Configuration for heading normalization */
+export interface HeadingNormalizationConfig {
+    /** Enable heading normalization (default: false) */
+    enabled: boolean;
+    /** Minimum pattern group size to trigger normalization (default: 3) */
+    minPatternCount?: number;
+}
+/**
+ * Normalize heading levels in-place for consistent section hierarchy.
+ *
+ * Groups headings by structural patterns (ARTICLE N, Section N.N, etc.),
+ * then normalizes each group to use the mode heading level. This fixes
+ * Datalab OCR inconsistencies where identical structural headings get
+ * assigned different levels (e.g., ARTICLE 1 as H1 but ARTICLE 5 as H3).
+ *
+ * Only mutates `block.headingLevel` - never modifies `block.text`.
+ *
+ * @param blocks - Parsed markdown blocks (mutated in-place)
+ * @param config - Normalization configuration
+ * @returns The same blocks array (for chaining convenience)
+ */
+export declare function normalizeHeadingLevels(blocks: MarkdownBlock[], config: HeadingNormalizationConfig): MarkdownBlock[];
+//# sourceMappingURL=heading-normalizer.d.ts.map

package/dist/services/chunking/heading-normalizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heading-normalizer.d.ts","sourceRoot":"","sources":["../../../src/services/chunking/heading-normalizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,8CAA8C;AAC9C,MAAM,WAAW,0BAA0B;IACzC,oDAAoD;IACpD,OAAO,EAAE,OAAO,CAAC;IACjB,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAwDD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,aAAa,EAAE,EACvB,MAAM,EAAE,0BAA0B,GACjC,aAAa,EAAE,CA6CjB"}