@futdevpro/nts-dynamo 1.15.156 → 1.15.160

package/build/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.js

@@ -1,468 +1,468 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DyNTS_DAI_DocChunking_Util = void 0;
-const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
-const global_settings_const_1 = require("../../../../../_collections/global-settings.const");
-const dai_doc_chunk_data_model_1 = require("../_models/data-models/dai-doc-chunk.data-model");
-/**
- * ClickUp dokumentum chunking szolgáltatás
- * @author AI
- * @description ClickUp dokumentumok intelligens chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
- */
-class DyNTS_DAI_DocChunking_Util {
-    static defaultErrorUserMsg = `We encountered an unhandled Control Service Error, ` +
-        `\nplease contact the responsible development team.`;
-    /**
-     * Unified break points configuration
-     */
-    static breakPoints = [
-        { pattern: '\n# ', level: 1, includeLength: 0 }, // Header 1 - cut before marker
-        { pattern: '\n## ', level: 2, includeLength: 0 }, // Header 2 - cut before marker
-        { pattern: '\n### ', level: 3, includeLength: 0 }, // Header 3 - cut before marker
-        { pattern: '\n#### ', level: 4, includeLength: 0 }, // Header 4 - cut before marker
-        { pattern: '\n\n', level: 0, includeLength: 2 }, // Double line break - include
-        { pattern: '\n', level: 0, includeLength: 1 }, // Single line break - include
-        { pattern: '. ', level: 0, includeLength: 2 }, // Sentence end - include
-        { pattern: ' ', level: 0, includeLength: 1 }, // Word break - include
-    ];
-    static debugLog = false;
-    /**
-     * Markdown tartalom chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
-     * @param page - ClickUp dokumentum oldal
-     * @param issuer - Kérés kezdeményezője
-     * @returns Promise<CU_DocChunk[]> - Létrehozott chunk-ok listája
-     */
-    static async chunkMdContent(page, getPageLink, issuer, debugLog) {
-        try {
-            if (debugLog) {
-                fsm_dynamo_1.DyFM_Log.info(`Chunking page "${page.name}"...`);
-            }
-            if (page.content.includes('#####')) {
-                fsm_dynamo_1.DyFM_Log.warn('lvl5+ headers are not supported yet for page: ' + page.name);
-            }
-            if (!page.documentId) {
-                throw new fsm_dynamo_1.DyFM_Error({
-                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page documentId is not set'), issuer),
-                    errorCode: 'CCAP-CUC-CC2',
-                });
-            }
-            if (!page._id) {
-                throw new fsm_dynamo_1.DyFM_Error({
-                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page _id is not set'), issuer),
-                    errorCode: 'CCAP-CUC-CC3',
-                });
-            }
-            /* DyFM_Log.log(`Chunking page: ${page.name}\n\n`, JSON.stringify(page.content, null, 2)); */
-            let leftovers = page.content;
-            const maxChunkSize = global_settings_const_1.DyNTS_global_settings.docChunking.maxChunkSize;
-            const maxChunkCount = global_settings_const_1.DyNTS_global_settings.docChunking.maxChunkCount;
-            const chunks = [];
-            let chunkIndex = 0;
-            let currentPosition = 0; // Track current position in original content
-            // Parse all headers to build hierarchy
-            const headerHierarchy = this.parseHeaderHierarchy(leftovers);
-            // Debug: Log parsed headers
-            /* console.log('\n=== Parsed Header Hierarchy ===');
-            headerHierarchy.forEach((header, index) => {
-              console.log(`${index}: Level ${header.level} at position ${header.index}: "${header.header}"`);
-            });
-            console.log('=== End Header Hierarchy ===\n'); */
-            // Track the last headers of each level as we progress
-            const lastHeaders = new Map();
-            while (leftovers.length > 0 && chunkIndex < maxChunkCount) {
-                // Ha a teljes tartalom belefér egy chunk-ba, adjuk vissza az egészet
-                if (leftovers.length <= maxChunkSize) {
-                    const chunkContent = leftovers.trim();
-                    if (chunkContent.length > 0) {
-                        //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
-                        //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
-                        const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders); // headerHierarchy.map(header => header.header);
-                        //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
-                        const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
-                            externalDocumentId: page.externalDocumentId,
-                            documentId: page.documentId,
-                            externalPageId: page.externalPageId,
-                            pageId: page._id,
-                            filePathParents: page.parentedPath,
-                            chunkHeadParents: headParents,
-                            allFlaggedParents: [...page.allFlaggedParents, ...headParents],
-                            /* path: [ ...page.parentedPath, ...headParents ].join('/'), */
-                            path: page.allFlaggedParentsMerged,
-                            documentName: page.externalDocumentId,
-                            pageName: page.name,
-                            pageLink: getPageLink(page, issuer),
-                            chunkIndex: chunkIndex,
-                            chunkContent: chunkContent,
-                            /* chunkOriginalContent: chunkContent,
-              
-                            chunkContent: chunkWithHierarchy, */
-                        });
-                        if (this.debugLog || debugLog) {
-                            fsm_dynamo_1.DyFM_Log.H_info('Full content (before assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
-                        }
-                        /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
-                        if (this.debugLog || debugLog) {
-                            fsm_dynamo_1.DyFM_Log.H_info('Full content (after assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
-                                chunk: chunk.chunkContent,
-                            });
-                        }
-                        /* await DyFM_Async.wait(100_000); */
-                        chunks.push(chunk);
-                    }
-                    break;
-                }
-                // Keressük meg a legjobb break point-ot a maxChunkSize-n belül
-                const breakPoint = this.findBestBreakPoint(leftovers, maxChunkSize);
-                if (breakPoint && breakPoint.index > 0) {
-                    // Ha találtunk break point-ot, vágjunk a marker előtt
-                    const chunkContent = leftovers.substring(0, breakPoint.index).trim();
-                    // Debug: Log current state
-                    /* console.log(`\n--- Creating chunk ${chunkIndex} ---`);
-                    console.log(`Current position: ${currentPosition}`);
-                    console.log(`Last headers:`, Object.fromEntries(lastHeaders));
-                    console.log(`Chunk content preview:`, chunkContent.substring(0, 100) + '...'); */
-                    // Check if chunk has headers
-                    //const hasHeaders: boolean = headerHierarchy.some(header => 
-                    //  header.index >= currentPosition && header.index < currentPosition + breakPoint.index
-                    //);
-                    /* console.log(`Chunk has headers: ${hasHeaders}`); */
-                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
-                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
-                    const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders); // headerHierarchy.map(header => header.header);
-                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
-                    /* console.log(`Chunk path:`, chunkPath);
-                    console.log(`Chunk with hierarchy preview:`, chunkWithHierarchy.substring(0, 100) + '...'); */
-                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
-                        externalDocumentId: page.externalDocumentId,
-                        externalPageId: page.externalPageId,
-                        documentId: page.documentId,
-                        pageId: page._id,
-                        filePathParents: page.parentedPath,
-                        chunkHeadParents: headParents,
-                        allFlaggedParents: [...page.allFlaggedParents, ...headParents],
-                        path: [...page.parentedPath, ...headParents].join('/'),
-                        documentName: page.externalDocumentId,
-                        pageName: page.name,
-                        pageLink: getPageLink(page, issuer),
-                        chunkIndex: chunkIndex,
-                        chunkContent: chunkContent,
-                        /* chunkOriginalContent: chunkContent,
-            
-                        chunkContent: chunkWithHierarchy, */
-                    });
-                    if (this.debugLog || debugLog) {
-                        fsm_dynamo_1.DyFM_Log.H_info('Break point (before assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
-                    }
-                    /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
-                    if (this.debugLog || debugLog) {
-                        fsm_dynamo_1.DyFM_Log.H_info('Break point (after assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
-                            chunk: chunk.chunkContent,
-                        });
-                    }
-                    chunks.push(chunk);
-                    chunkIndex++;
-                    currentPosition += breakPoint.index;
-                    leftovers = leftovers.substring(breakPoint.index);
-                    // Update last headers after creating the chunk
-                    this.updateLastHeaders(headerHierarchy, currentPosition, lastHeaders);
-                    /* console.log(`Updated last headers:`, Object.fromEntries(lastHeaders));
-                    console.log(`--- End chunk ${chunkIndex - 1} ---\n`); */
-                }
-                else {
-                    // Ha nem találtunk break point-ot, használjuk a maxChunkSize-t
-                    const chunkContent = leftovers.substring(0, maxChunkSize).trim();
-                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
-                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
-                    const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders); // headerHierarchy.map(header => header.header);
-                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
-                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
-                        externalDocumentId: page.externalDocumentId,
-                        externalPageId: page.externalPageId,
-                        documentId: page.documentId,
-                        pageId: page._id,
-                        filePathParents: page.parentedPath,
-                        chunkHeadParents: headParents,
-                        allFlaggedParents: [...page.allFlaggedParents, ...headParents],
-                        path: [...page.parentedPath, ...headParents].join('/'),
-                        documentName: page.externalDocumentId,
-                        pageName: page.name,
-                        pageLink: getPageLink(page, issuer),
-                        chunkIndex: chunkIndex,
-                        chunkContent: chunkContent,
-                        /* chunkOriginalContent: chunkContent,
-            
-                        chunkContent: chunkWithHierarchy, */
-                    });
-                    if (this.debugLog || debugLog) {
-                        fsm_dynamo_1.DyFM_Log.H_info('Max chunk size', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
-                    }
-                    /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
-                    if (this.debugLog || debugLog) {
-                        fsm_dynamo_1.DyFM_Log.H_info(`Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
-                            chunk: chunk.chunkContent,
-                        });
-                    }
-                    chunks.push(chunk);
-                    chunkIndex++;
-                    currentPosition += maxChunkSize;
-                    leftovers = leftovers.substring(maxChunkSize);
-                    // Update last headers after creating the chunk
-                    this.updateLastHeaders(headerHierarchy, currentPosition, lastHeaders);
-                }
-                // Safety check to prevent infinite loops
-                if (leftovers.length > 0 && leftovers.length >= page.content.length) {
-                    throw new fsm_dynamo_1.DyFM_Error({
-                        ...this.getDefaultErrorSettings('chunkMdContent', new Error('Infinite loop detected in chunking'), issuer),
-                        errorCode: 'CCAP-CUC-CC1',
-                    });
-                }
-            }
-            // Log warning if we hit the chunk count limit
-            if (chunkIndex >= maxChunkCount && leftovers.length > 0) {
-                fsm_dynamo_1.DyFM_Log.warn(`⚠️  Reached maximum chunk count (${maxChunkCount}). Remaining content: ${leftovers.length} characters`, {
-                    pageName: page.name,
-                    pageSize: page.content.length,
-                });
-            }
-            /* DyFM_Log.info(`✅ Created ${chunks.length} chunks`); */
-            return chunks;
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('chunkMdContent', error, issuer),
-                errorCode: 'CCAP-CUC-CP0',
-            });
-        }
-    }
-    /* private static assembleChunkWithHeaders(chunk: DyNTS_DAI_DocChunk, issuer: string): string {
-      try {
-        const fileHeaders = chunk.filePathParents.map(
-          (header, index) => `${'>'.repeat(index + 1)} **${header}**`
-        ).join('\n');
-        const headers = chunk.chunkHeadParents.filter(
-          path => !chunk.chunkOriginalContent.includes(path)
-        ).join('\n');
-        return `${fileHeaders}\n\n${headers}\n\n${chunk.chunkOriginalContent}`;
-      } catch (error) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('assembleChunkWithHeaders', error, issuer),
-          errorCode: 'CCAP-CUC-ACH0',
-        });
-      }
-    } */
-    static reassembleChunksToContent(chunks, issuer) {
-        try {
-            let pageContent = '';
-            for (const chunk of chunks) {
-                if (!chunk) {
-                    throw new fsm_dynamo_1.DyFM_Error({
-                        ...this.getDefaultErrorSettings('reassembleChunksToContent', new Error('Chunk is null, you`ve passing null chunks to the function'), issuer),
-                        errorCode: 'CCAP-CUC-RCTC0',
-                        additionalContent: {
-                            chunks: chunks,
-                        },
-                    });
-                }
-                pageContent += chunk.chunkContent;
-            }
-            return pageContent;
-        }
-        catch (error) {
-            throw new fsm_dynamo_1.DyFM_Error({
-                ...this.getDefaultErrorSettings('reassembleChunks', error, issuer),
-                errorCode: 'CCAP-CUC-RC0',
-            });
-        }
-    }
-    /**
-     * Find the best break point within the chunk size
-     * @param text - The text to search in
-     * @param maxIndex - Maximum index to search up to
-     * @returns The best break point or null if not found
-     */
-    static findBestBreakPoint(text, maxIndex) {
-        for (const breakPoint of this.breakPoints) {
-            const index = this.findLastOccurrence(text, breakPoint.pattern, maxIndex);
-            if (index > 0) {
-                return {
-                    index: index + breakPoint.includeLength,
-                    level: breakPoint.level
-                };
-            }
-        }
-        return null;
-    }
-    /**
-     * Parse header hierarchy from the content
-     * @param content - The content to parse
-     * @returns Array of header positions and levels
-     */
-    static parseHeaderHierarchy(content) {
-        const headers = [];
-        const lines = content.split('\n');
-        let currentIndex = 0;
-        for (const line of lines) {
-            const trimmedLine = line.trim();
-            // Check if line starts with # and determine the level
-            if (trimmedLine.startsWith('#')) {
-                const match = trimmedLine.match(/^(#+)\s+(.+)$/);
-                if (match) {
-                    const level = match[1].length; // Count the number of # symbols
-                    const headerText = trimmedLine;
-                    if (level >= 1 && level <= 4) { // Only support levels 1-4
-                        headers.push({
-                            index: currentIndex,
-                            level: level,
-                            header: headerText
-                        });
-                    }
-                }
-            }
-            currentIndex += line.length + 1; // +1 for the newline
-        }
-        return headers;
-    }
-    /**
-     * Update the last headers map based on current position
-     * @param headerHierarchy - The parsed header hierarchy
-     * @param currentPosition - Current position in content
-     * @param lastHeaders - Map to store last headers of each level
-     */
-    static updateLastHeaders(headerHierarchy, currentPosition, lastHeaders) {
-        // Find all headers that come before current position (not including current position)
-        for (const header of headerHierarchy) {
-            if (header.index < currentPosition) {
-                // Set this header as the last header for its level
-                lastHeaders.set(header.level, header.header);
-                // Clear all lower level headers (higher level numbers) when any header comes
-                // For example, if we see a level 2 header, clear levels 3, 4
-                // If we see a level 1 header, clear levels 2, 3, 4
-                for (let level = header.level + 1; level <= 4; level++) {
-                    lastHeaders.delete(level);
-                }
-            }
-        }
-    }
-    /**
-     * Visszaadja a chunk-hoz tartozó releváns szülő fejléceket a hierarchia alapján.
-     * @param headerHierarchy - A feldolgozott fejléc-hierarchia
-     * @param startIndex - A chunk kezdőindexe az eredeti tartalomban
-     * @param endIndex - A chunk végindexe az eredeti tartalomban
-     * @param lastHeaders - Az utolsó ismert fejlécek szintenként
-     * @returns A releváns szülő fejlécek tömbje
-     */
-    static getRelevantHeaders(headerHierarchy, startIndex, endIndex, lastHeaders) {
-        // Keressük meg az első fejlécet a chunkban
-        const firstHeaderInChunk = headerHierarchy.find(header => header.index >= startIndex && header.index < endIndex);
-        if (!firstHeaderInChunk) {
-            // Nincs fejléc a chunkban, a legmagasabb szintű utolsó fejlécet használjuk
-            let highestLevel = 0;
-            let highestHeader = '';
-            for (const [level, header] of lastHeaders.entries()) {
-                if (level > highestLevel) {
-                    highestLevel = level;
-                    highestHeader = header;
-                }
-            }
-            return highestHeader ? [highestHeader] : [];
-        }
-        // Szülő fejlécek összegyűjtése az első fejléc szintje alapján
-        const relevantHeaders = [];
-        for (let level = 1; level < firstHeaderInChunk.level; level++) {
-            const parentHeader = lastHeaders.get(level);
-            if (parentHeader) {
-                relevantHeaders.push(parentHeader);
-            }
-        }
-        return relevantHeaders;
-    }
-    /**
-     * Build hierarchical chunk path based on header structure
-     * @param pageName - The page name
-     * @param headerHierarchy - The parsed header hierarchy
-     * @param startIndex - Start index of the chunk in original content
-     * @param endIndex - End index of the chunk in original content
-     * @param lastHeaders - Map of last headers for each level
-     * @returns Hierarchical path array
-     */
-    static buildChunkPath(pageName, headerHierarchy, startIndex, endIndex, lastHeaders) {
-        // Find the first header within this chunk
-        const firstHeaderInChunk = headerHierarchy.find(header => header.index >= startIndex && header.index < endIndex);
-        if (!firstHeaderInChunk) {
-            // No header in chunk, use the highest level from lastHeaders as parent
-            const pathParts = [pageName];
-            // Find the highest level header from lastHeaders
-            let highestLevel = 0;
-            let highestHeader = '';
-            for (const [level, header] of lastHeaders.entries()) {
-                if (level > highestLevel) {
-                    highestLevel = level;
-                    highestHeader = header;
-                }
-            }
-            if (highestHeader) {
-                pathParts.push(highestHeader);
-            }
-            return pathParts;
-        }
-        // Build the path hierarchy
-        const pathParts = [pageName];
-        // Add parent headers from lastHeaders (these are the actual parent headers)
-        for (let level = 1; level < firstHeaderInChunk.level; level++) {
-            const parentHeader = lastHeaders.get(level);
-            if (parentHeader) {
-                pathParts.push(parentHeader); // Include the full header with markdown symbols
-            }
-        }
-        // Add the first header in chunk
-        pathParts.push(firstHeaderInChunk.header); // Include the full header with markdown symbols
-        return pathParts;
-    }
-    /**
-     * Add header hierarchy to the beginning of a chunk
-     * @param chunkContent - The chunk content
-     * @param headerHierarchy - The parsed header hierarchy
-     * @param startIndex - Start index of the chunk in original content
-     * @param endIndex - End index of the chunk in original content
-     * @param lastHeaders - Map of last headers for each level
-     * @returns Content with header hierarchy added
-     */
-    static addHeaderHierarchy(chunkContent, headerHierarchy, startIndex, endIndex, lastHeaders) {
-        const relevantHeaders = this.getRelevantHeaders(headerHierarchy, startIndex, endIndex, lastHeaders);
-        if (relevantHeaders.length > 0) {
-            return relevantHeaders.join('\n') + '\n\n' + chunkContent;
-        }
-        return chunkContent;
-    }
-    /**
-     * Helper method to find the last occurrence of a pattern within a specified range
-     * @param text - The text to search in
-     * @param pattern - The pattern to search for
-     * @param maxIndex - Maximum index to search up to
-     * @returns The index of the last occurrence, or -1 if not found
-     */
-    static findLastOccurrence(text, pattern, maxIndex) {
-        const searchText = text.substring(0, maxIndex);
-        const lastIndex = searchText.lastIndexOf(pattern);
-        return lastIndex;
-    }
-    static getDefaultErrorSettings(fnName, error, issuer, 
-    /** @deprecated we wont support the separate user message in the future, use the message instead */
-    useMessageAsUserMessage = true) {
-        return {
-            status: error?.___status ?? 500,
-            message: error?.message ??
-                error?._message ??
-                `${fnName} was UNSUCCESSFUL (${global_settings_const_1.DyNTS_global_settings.systemShortCodeName})`,
-            addECToUserMsg: !error?.__userMessage,
-            userMessage: error?.__userMessage ??
-                (useMessageAsUserMessage ? error?.message : this.defaultErrorUserMsg),
-            /* errorCode: errorCode, */
-            issuer: issuer,
-            issuerService: this.constructor?.name,
-            error: error,
-        };
-    }
-}
-exports.DyNTS_DAI_DocChunking_Util = DyNTS_DAI_DocChunking_Util;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_DAI_DocChunking_Util = void 0;
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+const global_settings_const_1 = require("../../../../../_collections/global-settings.const");
+const dai_doc_chunk_data_model_1 = require("../_models/data-models/dai-doc-chunk.data-model");
+/**
+ * ClickUp dokumentum chunking szolgáltatás
+ * @author AI
+ * @description ClickUp dokumentumok intelligens chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
+ */
+class DyNTS_DAI_DocChunking_Util {
+    static defaultErrorUserMsg = `We encountered an unhandled Control Service Error, ` +
+        `\nplease contact the responsible development team.`;
+    /**
+     * Unified break points configuration
+     */
+    static breakPoints = [
+        { pattern: '\n# ', level: 1, includeLength: 0 }, // Header 1 - cut before marker
+        { pattern: '\n## ', level: 2, includeLength: 0 }, // Header 2 - cut before marker
+        { pattern: '\n### ', level: 3, includeLength: 0 }, // Header 3 - cut before marker
+        { pattern: '\n#### ', level: 4, includeLength: 0 }, // Header 4 - cut before marker
+        { pattern: '\n\n', level: 0, includeLength: 2 }, // Double line break - include
+        { pattern: '\n', level: 0, includeLength: 1 }, // Single line break - include
+        { pattern: '. ', level: 0, includeLength: 2 }, // Sentence end - include
+        { pattern: ' ', level: 0, includeLength: 1 }, // Word break - include
+    ];
+    static debugLog = false;
+    /**
+     * Markdown tartalom chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
+     * @param page - ClickUp dokumentum oldal
+     * @param issuer - Kérés kezdeményezője
+     * @returns Promise<CU_DocChunk[]> - Létrehozott chunk-ok listája
+     */
+    static async chunkMdContent(page, getPageLink, issuer, debugLog) {
+        try {
+            if (debugLog) {
+                fsm_dynamo_1.DyFM_Log.info(`Chunking page "${page.name}"...`);
+            }
+            if (page.content.includes('#####')) {
+                fsm_dynamo_1.DyFM_Log.warn('lvl5+ headers are not supported yet for page: ' + page.name);
+            }
+            if (!page.documentId) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page documentId is not set'), issuer),
+                    errorCode: 'CCAP-CUC-CC2',
+                });
+            }
+            if (!page._id) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page _id is not set'), issuer),
+                    errorCode: 'CCAP-CUC-CC3',
+                });
+            }
+            /* DyFM_Log.log(`Chunking page: ${page.name}\n\n`, JSON.stringify(page.content, null, 2)); */
+            let leftovers = page.content;
+            const maxChunkSize = global_settings_const_1.DyNTS_global_settings.docChunking.maxChunkSize;
+            const maxChunkCount = global_settings_const_1.DyNTS_global_settings.docChunking.maxChunkCount;
+            const chunks = [];
+            let chunkIndex = 0;
+            let currentPosition = 0; // Track current position in original content
+            // Parse all headers to build hierarchy
+            const headerHierarchy = this.parseHeaderHierarchy(leftovers);
+            // Debug: Log parsed headers
+            /* console.log('\n=== Parsed Header Hierarchy ===');
+            headerHierarchy.forEach((header, index) => {
+              console.log(`${index}: Level ${header.level} at position ${header.index}: "${header.header}"`);
+            });
+            console.log('=== End Header Hierarchy ===\n'); */
+            // Track the last headers of each level as we progress
+            const lastHeaders = new Map();
+            while (leftovers.length > 0 && chunkIndex < maxChunkCount) {
+                // Ha a teljes tartalom belefér egy chunk-ba, adjuk vissza az egészet
+                if (leftovers.length <= maxChunkSize) {
+                    const chunkContent = leftovers.trim();
+                    if (chunkContent.length > 0) {
+                        //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
+                        //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
+                        const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders); // headerHierarchy.map(header => header.header);
+                        //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
+                        const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                            externalDocumentId: page.externalDocumentId,
+                            documentId: page.documentId,
+                            externalPageId: page.externalPageId,
+                            pageId: page._id,
+                            filePathParents: page.parentedPath,
+                            chunkHeadParents: headParents,
+                            allFlaggedParents: [...page.allFlaggedParents, ...headParents],
+                            /* path: [ ...page.parentedPath, ...headParents ].join('/'), */
+                            path: page.allFlaggedParentsMerged,
+                            documentName: page.externalDocumentId,
+                            pageName: page.name,
+                            pageLink: getPageLink(page, issuer),
+                            chunkIndex: chunkIndex,
+                            chunkContent: chunkContent,
+                            /* chunkOriginalContent: chunkContent,
+              
+                            chunkContent: chunkWithHierarchy, */
+                        });
+                        if (this.debugLog || debugLog) {
+                            fsm_dynamo_1.DyFM_Log.H_info('Full content (before assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
+                        }
+                        /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
+                        if (this.debugLog || debugLog) {
+                            fsm_dynamo_1.DyFM_Log.H_info('Full content (after assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
+                                chunk: chunk.chunkContent,
+                            });
+                        }
+                        /* await DyFM_Async.wait(100_000); */
+                        chunks.push(chunk);
+                    }
+                    break;
+                }
+                // Keressük meg a legjobb break point-ot a maxChunkSize-n belül
+                const breakPoint = this.findBestBreakPoint(leftovers, maxChunkSize);
+                if (breakPoint && breakPoint.index > 0) {
+                    // Ha találtunk break point-ot, vágjunk a marker előtt
+                    const chunkContent = leftovers.substring(0, breakPoint.index).trim();
+                    // Debug: Log current state
+                    /* console.log(`\n--- Creating chunk ${chunkIndex} ---`);
+                    console.log(`Current position: ${currentPosition}`);
+                    console.log(`Last headers:`, Object.fromEntries(lastHeaders));
+                    console.log(`Chunk content preview:`, chunkContent.substring(0, 100) + '...'); */
+                    // Check if chunk has headers
+                    //const hasHeaders: boolean = headerHierarchy.some(header => 
+                    //  header.index >= currentPosition && header.index < currentPosition + breakPoint.index
+                    //);
+                    /* console.log(`Chunk has headers: ${hasHeaders}`); */
+                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
+                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
+                    const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders); // headerHierarchy.map(header => header.header);
+                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
+                    /* console.log(`Chunk path:`, chunkPath);
+                    console.log(`Chunk with hierarchy preview:`, chunkWithHierarchy.substring(0, 100) + '...'); */
+                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                        externalDocumentId: page.externalDocumentId,
+                        externalPageId: page.externalPageId,
+                        documentId: page.documentId,
+                        pageId: page._id,
+                        filePathParents: page.parentedPath,
+                        chunkHeadParents: headParents,
+                        allFlaggedParents: [...page.allFlaggedParents, ...headParents],
+                        path: [...page.parentedPath, ...headParents].join('/'),
+                        documentName: page.externalDocumentId,
+                        pageName: page.name,
+                        pageLink: getPageLink(page, issuer),
+                        chunkIndex: chunkIndex,
+                        chunkContent: chunkContent,
+                        /* chunkOriginalContent: chunkContent,
+            
+                        chunkContent: chunkWithHierarchy, */
+                    });
+                    if (this.debugLog || debugLog) {
+                        fsm_dynamo_1.DyFM_Log.H_info('Break point (before assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
+                    }
+                    /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
+                    if (this.debugLog || debugLog) {
+                        fsm_dynamo_1.DyFM_Log.H_info('Break point (after assembleChunkWithHeaders)', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
+                            chunk: chunk.chunkContent,
+                        });
+                    }
+                    chunks.push(chunk);
+                    chunkIndex++;
+                    currentPosition += breakPoint.index;
+                    leftovers = leftovers.substring(breakPoint.index);
+                    // Update last headers after creating the chunk
+                    this.updateLastHeaders(headerHierarchy, currentPosition, lastHeaders);
+                    /* console.log(`Updated last headers:`, Object.fromEntries(lastHeaders));
+                    console.log(`--- End chunk ${chunkIndex - 1} ---\n`); */
+                }
+                else {
+                    // Ha nem találtunk break point-ot, használjuk a maxChunkSize-t
+                    const chunkContent = leftovers.substring(0, maxChunkSize).trim();
+                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
+                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
+                    const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders); // headerHierarchy.map(header => header.header);
+                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
+                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                        externalDocumentId: page.externalDocumentId,
+                        externalPageId: page.externalPageId,
+                        documentId: page.documentId,
+                        pageId: page._id,
+                        filePathParents: page.parentedPath,
+                        chunkHeadParents: headParents,
+                        allFlaggedParents: [...page.allFlaggedParents, ...headParents],
+                        path: [...page.parentedPath, ...headParents].join('/'),
+                        documentName: page.externalDocumentId,
+                        pageName: page.name,
+                        pageLink: getPageLink(page, issuer),
+                        chunkIndex: chunkIndex,
+                        chunkContent: chunkContent,
+                        /* chunkOriginalContent: chunkContent,
+            
+                        chunkContent: chunkWithHierarchy, */
+                    });
+                    if (this.debugLog || debugLog) {
+                        fsm_dynamo_1.DyFM_Log.H_info('Max chunk size', `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, { chunk: chunk.chunkContent });
+                    }
+                    /* chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer); */
+                    if (this.debugLog || debugLog) {
+                        fsm_dynamo_1.DyFM_Log.H_info(`Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, {
+                            chunk: chunk.chunkContent,
+                        });
+                    }
+                    chunks.push(chunk);
+                    chunkIndex++;
+                    currentPosition += maxChunkSize;
+                    leftovers = leftovers.substring(maxChunkSize);
+                    // Update last headers after creating the chunk
+                    this.updateLastHeaders(headerHierarchy, currentPosition, lastHeaders);
+                }
+                // Safety check to prevent infinite loops
+                if (leftovers.length > 0 && leftovers.length >= page.content.length) {
+                    throw new fsm_dynamo_1.DyFM_Error({
+                        ...this.getDefaultErrorSettings('chunkMdContent', new Error('Infinite loop detected in chunking'), issuer),
+                        errorCode: 'CCAP-CUC-CC1',
+                    });
+                }
+            }
+            // Log warning if we hit the chunk count limit
+            if (chunkIndex >= maxChunkCount && leftovers.length > 0) {
+                fsm_dynamo_1.DyFM_Log.warn(`⚠️  Reached maximum chunk count (${maxChunkCount}). Remaining content: ${leftovers.length} characters`, {
+                    pageName: page.name,
+                    pageSize: page.content.length,
+                });
+            }
+            /* DyFM_Log.info(`✅ Created ${chunks.length} chunks`); */
+            return chunks;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('chunkMdContent', error, issuer),
+                errorCode: 'CCAP-CUC-CP0',
+            });
+        }
+    }
+    /* private static assembleChunkWithHeaders(chunk: DyNTS_DAI_DocChunk, issuer: string): string {
+      try {
+        const fileHeaders = chunk.filePathParents.map(
+          (header, index) => `${'>'.repeat(index + 1)} **${header}**`
+        ).join('\n');
+        const headers = chunk.chunkHeadParents.filter(
+          path => !chunk.chunkOriginalContent.includes(path)
+        ).join('\n');
+        return `${fileHeaders}\n\n${headers}\n\n${chunk.chunkOriginalContent}`;
+      } catch (error) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('assembleChunkWithHeaders', error, issuer),
+          errorCode: 'CCAP-CUC-ACH0',
+        });
+      }
+    } */
+    static reassembleChunksToContent(chunks, issuer) {
+        try {
+            let pageContent = '';
+            for (const chunk of chunks) {
+                if (!chunk) {
+                    throw new fsm_dynamo_1.DyFM_Error({
+                        ...this.getDefaultErrorSettings('reassembleChunksToContent', new Error('Chunk is null, you`ve passing null chunks to the function'), issuer),
+                        errorCode: 'CCAP-CUC-RCTC0',
+                        additionalContent: {
+                            chunks: chunks,
+                        },
+                    });
+                }
+                pageContent += chunk.chunkContent;
+            }
+            return pageContent;
+        }
+        catch (error) {
+            throw new fsm_dynamo_1.DyFM_Error({
+                ...this.getDefaultErrorSettings('reassembleChunks', error, issuer),
+                errorCode: 'CCAP-CUC-RC0',
+            });
+        }
+    }
+    /**
+     * Find the best break point within the chunk size
+     * @param text - The text to search in
+     * @param maxIndex - Maximum index to search up to
+     * @returns The best break point or null if not found
+     */
+    static findBestBreakPoint(text, maxIndex) {
+        for (const breakPoint of this.breakPoints) {
+            const index = this.findLastOccurrence(text, breakPoint.pattern, maxIndex);
+            if (index > 0) {
+                return {
+                    index: index + breakPoint.includeLength,
+                    level: breakPoint.level
+                };
+            }
+        }
+        return null;
+    }
+    /**
+     * Parse header hierarchy from the content
+     * @param content - The content to parse
+     * @returns Array of header positions and levels
+     */
+    static parseHeaderHierarchy(content) {
+        const headers = [];
+        const lines = content.split('\n');
+        let currentIndex = 0;
+        for (const line of lines) {
+            const trimmedLine = line.trim();
+            // Check if line starts with # and determine the level
+            if (trimmedLine.startsWith('#')) {
+                const match = trimmedLine.match(/^(#+)\s+(.+)$/);
+                if (match) {
+                    const level = match[1].length; // Count the number of # symbols
+                    const headerText = trimmedLine;
+                    if (level >= 1 && level <= 4) { // Only support levels 1-4
+                        headers.push({
+                            index: currentIndex,
+                            level: level,
+                            header: headerText
+                        });
+                    }
+                }
+            }
+            currentIndex += line.length + 1; // +1 for the newline
+        }
+        return headers;
+    }
+    /**
+     * Update the last headers map based on current position
+     * @param headerHierarchy - The parsed header hierarchy
+     * @param currentPosition - Current position in content
+     * @param lastHeaders - Map to store last headers of each level
+     */
+    static updateLastHeaders(headerHierarchy, currentPosition, lastHeaders) {
+        // Find all headers that come before current position (not including current position)
+        for (const header of headerHierarchy) {
+            if (header.index < currentPosition) {
+                // Set this header as the last header for its level
+                lastHeaders.set(header.level, header.header);
+                // Clear all lower level headers (higher level numbers) when any header comes
+                // For example, if we see a level 2 header, clear levels 3, 4
+                // If we see a level 1 header, clear levels 2, 3, 4
+                for (let level = header.level + 1; level <= 4; level++) {
+                    lastHeaders.delete(level);
+                }
+            }
+        }
+    }
+    /**
+     * Visszaadja a chunk-hoz tartozó releváns szülő fejléceket a hierarchia alapján.
+     * @param headerHierarchy - A feldolgozott fejléc-hierarchia
+     * @param startIndex - A chunk kezdőindexe az eredeti tartalomban
+     * @param endIndex - A chunk végindexe az eredeti tartalomban
+     * @param lastHeaders - Az utolsó ismert fejlécek szintenként
+     * @returns A releváns szülő fejlécek tömbje
+     */
+    static getRelevantHeaders(headerHierarchy, startIndex, endIndex, lastHeaders) {
+        // Keressük meg az első fejlécet a chunkban
+        const firstHeaderInChunk = headerHierarchy.find(header => header.index >= startIndex && header.index < endIndex);
+        if (!firstHeaderInChunk) {
+            // Nincs fejléc a chunkban, a legmagasabb szintű utolsó fejlécet használjuk
+            let highestLevel = 0;
+            let highestHeader = '';
+            for (const [level, header] of lastHeaders.entries()) {
+                if (level > highestLevel) {
+                    highestLevel = level;
+                    highestHeader = header;
+                }
+            }
+            return highestHeader ? [highestHeader] : [];
+        }
+        // Szülő fejlécek összegyűjtése az első fejléc szintje alapján
+        const relevantHeaders = [];
+        for (let level = 1; level < firstHeaderInChunk.level; level++) {
+            const parentHeader = lastHeaders.get(level);
+            if (parentHeader) {
+                relevantHeaders.push(parentHeader);
+            }
+        }
+        return relevantHeaders;
+    }
+    /**
+     * Build hierarchical chunk path based on header structure
+     * @param pageName - The page name
+     * @param headerHierarchy - The parsed header hierarchy
+     * @param startIndex - Start index of the chunk in original content
+     * @param endIndex - End index of the chunk in original content
+     * @param lastHeaders - Map of last headers for each level
+     * @returns Hierarchical path array
+     */
+    static buildChunkPath(pageName, headerHierarchy, startIndex, endIndex, lastHeaders) {
+        // Find the first header within this chunk
+        const firstHeaderInChunk = headerHierarchy.find(header => header.index >= startIndex && header.index < endIndex);
+        if (!firstHeaderInChunk) {
+            // No header in chunk, use the highest level from lastHeaders as parent
+            const pathParts = [pageName];
+            // Find the highest level header from lastHeaders
+            let highestLevel = 0;
+            let highestHeader = '';
+            for (const [level, header] of lastHeaders.entries()) {
+                if (level > highestLevel) {
+                    highestLevel = level;
+                    highestHeader = header;
+                }
+            }
+            if (highestHeader) {
+                pathParts.push(highestHeader);
+            }
+            return pathParts;
+        }
+        // Build the path hierarchy
+        const pathParts = [pageName];
+        // Add parent headers from lastHeaders (these are the actual parent headers)
+        for (let level = 1; level < firstHeaderInChunk.level; level++) {
+            const parentHeader = lastHeaders.get(level);
+            if (parentHeader) {
+                pathParts.push(parentHeader); // Include the full header with markdown symbols
+            }
+        }
+        // Add the first header in chunk
+        pathParts.push(firstHeaderInChunk.header); // Include the full header with markdown symbols
+        return pathParts;
+    }
+    /**
+     * Add header hierarchy to the beginning of a chunk
+     * @param chunkContent - The chunk content
+     * @param headerHierarchy - The parsed header hierarchy
+     * @param startIndex - Start index of the chunk in original content
+     * @param endIndex - End index of the chunk in original content
+     * @param lastHeaders - Map of last headers for each level
+     * @returns Content with header hierarchy added
+     */
+    static addHeaderHierarchy(chunkContent, headerHierarchy, startIndex, endIndex, lastHeaders) {
+        const relevantHeaders = this.getRelevantHeaders(headerHierarchy, startIndex, endIndex, lastHeaders);
+        if (relevantHeaders.length > 0) {
+            return relevantHeaders.join('\n') + '\n\n' + chunkContent;
+        }
+        return chunkContent;
+    }
+    /**
+     * Helper method to find the last occurrence of a pattern within a specified range
+     * @param text - The text to search in
+     * @param pattern - The pattern to search for
+     * @param maxIndex - Maximum index to search up to
+     * @returns The index of the last occurrence, or -1 if not found
+     */
+    static findLastOccurrence(text, pattern, maxIndex) {
+        const searchText = text.substring(0, maxIndex);
+        const lastIndex = searchText.lastIndexOf(pattern);
+        return lastIndex;
+    }
+    static getDefaultErrorSettings(fnName, error, issuer, 
+    /** @deprecated we wont support the separate user message in the future, use the message instead */
+    useMessageAsUserMessage = true) {
+        return {
+            status: error?.___status ?? 500,
+            message: error?.message ??
+                error?._message ??
+                `${fnName} was UNSUCCESSFUL (${global_settings_const_1.DyNTS_global_settings.systemShortCodeName})`,
+            addECToUserMsg: !error?.__userMessage,
+            userMessage: error?.__userMessage ??
+                (useMessageAsUserMessage ? error?.message : this.defaultErrorUserMsg),
+            /* errorCode: errorCode, */
+            issuer: issuer,
+            issuerService: this.constructor?.name,
+            error: error,
+        };
+    }
+}
+exports.DyNTS_DAI_DocChunking_Util = DyNTS_DAI_DocChunking_Util;
 //# sourceMappingURL=dai-chunking.util.js.map