@futdevpro/nts-dynamo 1.12.15 → 1.14.1

package/build/_modules/ai/_models/ai-input-interfaces.d.ts

@@ -0,0 +1,61 @@
+import { DyFM_AI_Message } from '@futdevpro/fsm-dynamo/ai';
+import { DyFM_AI_CallSettings } from '@futdevpro/fsm-dynamo/ai';
+export interface DyFM_AI_Base_Input {
+    issuer: string;
+    settings?: DyFM_AI_CallSettings;
+    debugLog?: boolean;
+}
+export interface DyFM_AI_Question_Input extends DyFM_AI_Base_Input {
+    question: string;
+    replaceThisInLog?: string;
+}
+export interface DyFM_AI_ListSelect_Input extends DyFM_AI_Question_Input {
+    selectFrom: string[];
+}
+export interface DyFM_AI_GenericSelect_Input<T> extends DyFM_AI_Question_Input {
+    selectFrom: T[];
+}
+export interface DyFM_AI_MultiSelect_Input extends DyFM_AI_Question_Input {
+    options: string[];
+}
+export interface DyFM_AI_GenericMultiSelect_Input<T> extends DyFM_AI_Question_Input {
+    options: T[];
+}
+export interface DyFM_AI_JSONKeysDescription_Input extends DyFM_AI_Question_Input {
+    keysDescription: string;
+}
+export interface DyFM_AI_JSONExactKeys_Input extends DyFM_AI_Question_Input {
+    keys: string[];
+}
+export interface DyFM_AI_SimpleMessage_Input extends DyFM_AI_Base_Input {
+    message: string;
+}
+export interface DyFM_AI_ConversationBase_Input extends DyFM_AI_Base_Input {
+    conversation: DyFM_AI_Message[];
+    replaceThisInLog?: string;
+}
+export interface DyFM_AI_ConversationQuestion_Input extends DyFM_AI_ConversationBase_Input {
+    question: string;
+}
+export interface DyFM_AI_ConversationSelect_Input extends DyFM_AI_ConversationQuestion_Input {
+    selectFrom: string[];
+}
+export interface DyFM_AI_ConversationGenericSelect_Input<T> extends DyFM_AI_ConversationQuestion_Input {
+    selectFrom: T[];
+}
+export interface DyFM_AI_ConversationMultiSelect_Input extends DyFM_AI_ConversationQuestion_Input {
+    options: string[];
+}
+export interface DyFM_AI_ConversationGenericMultiSelect_Input<T> extends DyFM_AI_ConversationQuestion_Input {
+    options: T[];
+}
+export interface DyFM_AI_ConversationJSONKeysDescription_Input extends DyFM_AI_ConversationQuestion_Input {
+    keysDescription: string;
+}
+export interface DyFM_AI_ConversationJSONExactKeys_Input extends DyFM_AI_ConversationQuestion_Input {
+    keys: string[];
+}
+export interface DyFM_AI_ConversationGetAnswer_Input extends DyFM_AI_ConversationBase_Input {
+    newMessage?: string;
+}
+//# sourceMappingURL=ai-input-interfaces.d.ts.map

package/build/_modules/ai/_models/ai-input-interfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-input-interfaces.d.ts","sourceRoot":"","sources":["../../../../src/_modules/ai/_models/ai-input-interfaces.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAEhE,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,sBAAuB,SAAQ,kBAAkB;IAChE,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,wBAAyB,SAAQ,sBAAsB;IACtE,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,2BAA2B,CAAC,CAAC,CAAE,SAAQ,sBAAsB;IAC5E,UAAU,EAAE,CAAC,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,yBAA0B,SAAQ,sBAAsB;IACvE,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,gCAAgC,CAAC,CAAC,CAAE,SAAQ,sBAAsB;IACjF,OAAO,EAAE,CAAC,EAAE,CAAC;CACd;AAED,MAAM,WAAW,iCAAkC,SAAQ,sBAAsB;IAC/E,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,2BAA4B,SAAQ,sBAAsB;IACzE,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,2BAA4B,SAAQ,kBAAkB;IACrE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,8BAA+B,SAAQ,kBAAkB;IACxE,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kCAAmC,SAAQ,8BAA8B;IACxF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gCAAiC,SAAQ,kCAAkC;IAC1F,UAAU,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,uCAAuC,CAAC,CAAC,CAAE,SAAQ,kCAAkC;IACpG,UAAU,EAAE,CAAC,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,qCAAsC,SAAQ,kCAAkC;IAC/F,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,MAAM,WAAW,4CAA4C,CAAC,CAAC,CAAE,SAAQ,kCAAkC;IACzG,OAAO,EAAE,CAAC,EAAE,CAAC;CACd;AAED,MAAM,WAAW,6CAA8C,SAAQ,kCAAkC;IACvG,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,uCAAwC,SAAQ,kCAAkC;IACjG,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,mCAAoC,SAAQ,8BAA8B;IACzF,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}

package/build/_modules/{open-ai/_models/interfaces/oai-document-page.interface.js → ai/_models/ai-input-interfaces.js}

@@ -1,3 +1,3 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=oai-document-page.interface.js.map
+//# sourceMappingURL=ai-input-interfaces.js.map

package/build/_modules/ai/_models/ai-input-interfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-input-interfaces.js","sourceRoot":"","sources":["../../../../src/_modules/ai/_models/ai-input-interfaces.ts"],"names":[],"mappings":""}

package/build/_modules/{open-ai/_collections/oai-chunking.util.d.ts → ai/_modules/document-ai/_collections/dai-chunking.util.d.ts}

@@ -1,13 +1,17 @@
 import { DyFM_AnyError, DyFM_Error_Settings } from '@futdevpro/fsm-dynamo';
-import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
-import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
+import { DyNTS_DAI_DocChunk } from '../_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
 /**
  * ClickUp dokumentum chunking szolgáltatás
  * @author AI
  * @description ClickUp dokumentumok intelligens chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
  */
-export declare class DyNTS_OAI_Chunking_Util {
+export declare class DyNTS_DAI_DocChunking_Util {
     protected static readonly defaultErrorUserMsg: string;
+    /**
+     * Unified break points configuration
+     */
+    private static readonly breakPoints;
     protected static readonly debugLog: boolean;
     /**
      * Markdown tartalom chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
@@ -15,12 +19,8 @@ export declare class DyNTS_OAI_Chunking_Util {
      * @param issuer - Kérés kezdeményezője
      * @returns Promise<CU_DocChunk[]> - Létrehozott chunk-ok listája
      */
-    static chunkMdContent<T_Chunk extends DyNTS_OAI_Chunk, T_Page extends DyNTS_OAI_DocumentPage<T_Chunk>>(page: T_Page, getPageLink: (page: T_Page, issuer: string) => string, issuer: string, debugLog?: boolean): Promise<T_Chunk[]>;
-    static reassembleChunksToContent(chunks: DyNTS_OAI_Chunk[], issuer: string): string;
-    /**
-     * Unified break points configuration
-     */
-    private static readonly breakPoints;
+    static chunkMdContent<T_Chunk extends DyNTS_DAI_DocChunk, T_Page extends DyNTS_DAI_DocPage<T_Chunk>>(page: T_Page, getPageLink: (page: T_Page, issuer: string) => string, issuer: string, debugLog?: boolean): Promise<T_Chunk[]>;
+    static reassembleChunksToContent(chunks: DyNTS_DAI_DocChunk[], issuer: string): string;
     /**
      * Find the best break point within the chunk size
      * @param text - The text to search in
@@ -80,4 +80,4 @@ export declare class DyNTS_OAI_Chunking_Util {
     private static findLastOccurrence;
     protected static getDefaultErrorSettings(fnName: string, error: DyFM_AnyError, issuer: string): DyFM_Error_Settings;
 }
-//# sourceMappingURL=oai-chunking.util.d.ts.map
+//# sourceMappingURL=dai-chunking.util.d.ts.map

package/build/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-chunking.util.d.ts","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAwB,aAAa,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAEjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,iDAAiD,CAAC;AACrF,OAAO,EAAE,iBAAiB,EAAE,MAAM,gDAAgD,CAAC;AAcnF;;;;GAIG;AACH,qBAAa,0BAA0B;IAErC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAEE;IAEvD;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CASjC;IAEF,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAS;IAEpD;;;;;OAKG;WACU,cAAc,CACzB,OAAO,SAAS,kBAAkB,EAClC,MAAM,SAAS,iBAAiB,CAAC,OAAO,CAAC,EAEvC,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,MAAM,EACrD,MAAM,EAAE,MAAM,EACd,QAAQ,CAAC,EAAE,OAAO,GACnB,OAAO,CAAC,OAAO,EAAE,CAAC;IA+RrB,MAAM,CAAC,yBAAyB,CAC9B,MAAM,EAAE,kBAAkB,EAAE,EAC5B,MAAM,EAAE,MAAM,GACb,MAAM;IAgCT;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAcjC;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,oBAAoB;IAgCnC;;;;;OAKG;IACH,OAAO,CAAC,MAAM,CAAC,iBAAiB;IAqBhC;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAqCjC;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,cAAc;IAoD7B;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAgBjC;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAOjC,SAAS,CAAC,MAAM,CAAC,uBAAuB,CACtC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,aAAa,EACpB,MAAM,EAAE,MAAM,GAEb,mBAAmB;CAcvB"}

package/build/_modules/{open-ai/_collections/oai-chunking.util.js → ai/_modules/document-ai/_collections/dai-chunking.util.js}

@@ -1,17 +1,30 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DyNTS_OAI_Chunking_Util = void 0;
+exports.DyNTS_DAI_DocChunking_Util = void 0;
 const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
-const oai_doc_chunk_data_model_1 = require("../_models/oai-doc-chunk.data-model");
-const global_settings_const_1 = require("../../../_collections/global-settings.const");
+const global_settings_const_1 = require("../../../../../_collections/global-settings.const");
+const dai_doc_chunk_data_model_1 = require("../_models/data-models/dai-doc-chunk.data-model");
 /**
  * ClickUp dokumentum chunking szolgáltatás
  * @author AI
  * @description ClickUp dokumentumok intelligens chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
  */
-class DyNTS_OAI_Chunking_Util {
+class DyNTS_DAI_DocChunking_Util {
     static defaultErrorUserMsg = `We encountered an unhandled Control Service Error, ` +
         `\nplease contact the responsible development team.`;
+    /**
+     * Unified break points configuration
+     */
+    static breakPoints = [
+        { pattern: '\n# ', level: 1, includeLength: 0 }, // Header 1 - cut before marker
+        { pattern: '\n## ', level: 2, includeLength: 0 }, // Header 2 - cut before marker
+        { pattern: '\n### ', level: 3, includeLength: 0 }, // Header 3 - cut before marker
+        { pattern: '\n#### ', level: 4, includeLength: 0 }, // Header 4 - cut before marker
+        { pattern: '\n\n', level: 0, includeLength: 2 }, // Double line break - include
+        { pattern: '\n', level: 0, includeLength: 1 }, // Single line break - include
+        { pattern: '. ', level: 0, includeLength: 2 }, // Sentence end - include
+        { pattern: ' ', level: 0, includeLength: 1 }, // Word break - include
+    ];
     static debugLog = false;
     /**
      * Markdown tartalom chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
@@ -27,6 +40,18 @@ class DyNTS_OAI_Chunking_Util {
             if (page.content.includes('#####')) {
                 fsm_dynamo_1.DyFM_Log.warn('lvl5+ headers are not supported yet for page: ' + page.name);
             }
+            if (!page.documentId) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page documentId is not set'), issuer),
+                    errorCode: 'CCAP-CUC-CC2',
+                });
+            }
+            if (!page._id) {
+                throw new fsm_dynamo_1.DyFM_Error({
+                    ...this.getDefaultErrorSettings('chunkMdContent', new Error('Page _id is not set'), issuer),
+                    errorCode: 'CCAP-CUC-CC3',
+                });
+            }
             /* DyFM_Log.log(`Chunking page: ${page.name}\n\n`, JSON.stringify(page.content, null, 2)); */
             let leftovers = page.content;
             const maxChunkSize = global_settings_const_1.DyNTS_global_settings.docChunking.maxChunkSize;
@@ -49,22 +74,24 @@ class DyNTS_OAI_Chunking_Util {
                 if (leftovers.length <= maxChunkSize) {
                     const chunkContent = leftovers.trim();
                     if (chunkContent.length > 0) {
-                        const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
-                        const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
+                        //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
+                        //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
                         const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders); // headerHierarchy.map(header => header.header);
-                        const allFlaggedParents = [...page.allFlaggedParents, ...headParents];
-                        const chunk = new oai_doc_chunk_data_model_1.DyNTS_OAI_Chunk({
+                        //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
+                        const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                            documentId: page.documentId,
+                            pageId: page._id,
                             filePathParents: page.parentedPath,
                             chunkHeadParents: headParents,
                             allFlaggedParents: [...page.allFlaggedParents, ...headParents],
                             path: [...page.parentedPath, ...headParents].join('/'),
-                            documentName: page.documentId,
+                            documentName: page.externalDocumentId,
                             pageName: page.name,
                             pageLink: getPageLink(page, issuer),
                             chunkIndex: chunkIndex,
                             chunkContent: chunkContent,
                             /* chunkOriginalContent: chunkContent,
-                
+              
                             chunkContent: chunkWithHierarchy, */
                         });
                         if (this.debugLog || debugLog) {
@@ -83,7 +110,7 @@ class DyNTS_OAI_Chunking_Util {
                 }
                 // Keressük meg a legjobb break point-ot a maxChunkSize-n belül
                 const breakPoint = this.findBestBreakPoint(leftovers, maxChunkSize);
-                if (breakPoint.index > 0) {
+                if (breakPoint && breakPoint.index > 0) {
                     // Ha találtunk break point-ot, vágjunk a marker előtt
                     const chunkContent = leftovers.substring(0, breakPoint.index).trim();
                     // Debug: Log current state
@@ -92,20 +119,24 @@ class DyNTS_OAI_Chunking_Util {
                     console.log(`Last headers:`, Object.fromEntries(lastHeaders));
                     console.log(`Chunk content preview:`, chunkContent.substring(0, 100) + '...'); */
                     // Check if chunk has headers
-                    const hasHeaders = headerHierarchy.some(header => header.index >= currentPosition && header.index < currentPosition + breakPoint.index);
+                    //const hasHeaders: boolean = headerHierarchy.some(header => 
+                    //  header.index >= currentPosition && header.index < currentPosition + breakPoint.index
+                    //);
                     /* console.log(`Chunk has headers: ${hasHeaders}`); */
-                    const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
-                    const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
+                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
+                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
                     const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders); // headerHierarchy.map(header => header.header);
-                    const allFlaggedParents = [...page.allFlaggedParents, ...headParents];
+                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
                     /* console.log(`Chunk path:`, chunkPath);
                     console.log(`Chunk with hierarchy preview:`, chunkWithHierarchy.substring(0, 100) + '...'); */
-                    const chunk = new oai_doc_chunk_data_model_1.DyNTS_OAI_Chunk({
+                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                        documentId: page.documentId,
+                        pageId: page._id,
                         filePathParents: page.parentedPath,
                         chunkHeadParents: headParents,
                         allFlaggedParents: [...page.allFlaggedParents, ...headParents],
                         path: [...page.parentedPath, ...headParents].join('/'),
-                        documentName: page.documentId,
+                        documentName: page.externalDocumentId,
                         pageName: page.name,
                         pageLink: getPageLink(page, issuer),
                         chunkIndex: chunkIndex,
@@ -135,16 +166,18 @@ class DyNTS_OAI_Chunking_Util {
                 else {
                     // Ha nem találtunk break point-ot, használjuk a maxChunkSize-t
                     const chunkContent = leftovers.substring(0, maxChunkSize).trim();
-                    const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
-                    const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
+                    //const chunkWithHierarchy: string = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
+                    //const chunkPath: string[] = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
                     const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders); // headerHierarchy.map(header => header.header);
-                    const allFlaggedParents = [...page.allFlaggedParents, ...headParents];
-                    const chunk = new oai_doc_chunk_data_model_1.DyNTS_OAI_Chunk({
+                    //const allFlaggedParents: string[] = [ ...page.allFlaggedParents, ...headParents ];
+                    const chunk = new dai_doc_chunk_data_model_1.DyNTS_DAI_DocChunk({
+                        documentId: page.documentId,
+                        pageId: page._id,
                         filePathParents: page.parentedPath,
                         chunkHeadParents: headParents,
                         allFlaggedParents: [...page.allFlaggedParents, ...headParents],
                         path: [...page.parentedPath, ...headParents].join('/'),
-                        documentName: page.documentId,
+                        documentName: page.externalDocumentId,
                         pageName: page.name,
                         pageLink: getPageLink(page, issuer),
                         chunkIndex: chunkIndex,
@@ -194,7 +227,7 @@ class DyNTS_OAI_Chunking_Util {
             });
         }
     }
-    /* private static assembleChunkWithHeaders(chunk: DyNTS_OAI_Chunk, issuer: string): string {
+    /* private static assembleChunkWithHeaders(chunk: DyNTS_DAI_DocChunk, issuer: string): string {
       try {
         const fileHeaders = chunk.filePathParents.map(
           (header, index) => `${'>'.repeat(index + 1)} **${header}**`
@@ -234,19 +267,6 @@ class DyNTS_OAI_Chunking_Util {
             });
         }
     }
-    /**
-     * Unified break points configuration
-     */
-    static breakPoints = [
-        { pattern: '\n# ', level: 1, includeLength: 0 }, // Header 1 - cut before marker
-        { pattern: '\n## ', level: 2, includeLength: 0 }, // Header 2 - cut before marker
-        { pattern: '\n### ', level: 3, includeLength: 0 }, // Header 3 - cut before marker
-        { pattern: '\n#### ', level: 4, includeLength: 0 }, // Header 4 - cut before marker
-        { pattern: '\n\n', level: 0, includeLength: 2 }, // Double line break - include
-        { pattern: '\n', level: 0, includeLength: 1 }, // Single line break - include
-        { pattern: '. ', level: 0, includeLength: 2 }, // Sentence end - include
-        { pattern: ' ', level: 0, includeLength: 1 }, // Word break - include
-    ];
     /**
      * Find the best break point within the chunk size
      * @param text - The text to search in
@@ -434,8 +454,5 @@ class DyNTS_OAI_Chunking_Util {
         };
     }
 }
-exports.DyNTS_OAI_Chunking_Util = DyNTS_OAI_Chunking_Util;
-function isLast(element, array) {
-    return array.indexOf(element) === array.length - 1;
-}
-//# sourceMappingURL=oai-chunking.util.js.map
+exports.DyNTS_DAI_DocChunking_Util = DyNTS_DAI_DocChunking_Util;
+//# sourceMappingURL=dai-chunking.util.js.map

package/build/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-chunking.util.js","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.ts"],"names":[],"mappings":";;;AAAA,sDAAiG;AACjG,6FAA0F;AAC1F,8FAAqF;AAerF;;;;GAIG;AACH,MAAa,0BAA0B;IAE3B,MAAM,CAAU,mBAAmB,GAC3C,qDAAqD;QACrD,oDAAoD,CAAC;IAEvD;;OAEG;IACK,MAAM,CAAU,WAAW,GAAgE;QACjG,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,+BAA+B;QAChF,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,+BAA+B;QACjF,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,+BAA+B;QAClF,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,+BAA+B;QACnF,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,8BAA8B;QAC/E,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,8BAA8B;QAC7E,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,yBAAyB;QACxE,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,uBAAuB;KACtE,CAAC;IAEQ,MAAM,CAAU,QAAQ,GAAY,KAAK,CAAC;IAEpD;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAIvB,IAAY,EACZ,WAAqD,EACrD,MAAc,EACd,QAAkB;QAEpB,IAAI,CAAC;YACH,IAAI,QAAQ,EAAE,CAAC;gBACb,qBAAQ,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,MAAM,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,qBAAQ,CAAC,IAAI,CAAC,gDAAgD,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9E,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,EAAE,IAAI,KAAK,CAAC,4BAA4B,CAAC,EAAE,MAAM,CAAC;oBAClG,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC;YACL,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACd,MAAM,IAAI,uBAAU,CAAC;oBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,EAAE,IAAI,KAAK,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;oBAC3F,SAAS,EAAE,cAAc;iBAC1B,CAAC,CAAC;YACL,CAAC;YAED,6FAA6F;YAE7F,IAAI,SAAS,GAAW,IAAI,CAAC,OAAO,CAAC;YACrC,MAAM,YAAY,GAAW,6CAAqB,CAAC,WAAW,CAAC,YAAY,CAAC;YAC5E,MAAM,aAAa,GAAW,6CAAqB,CAAC,WAAW,CAAC,aAAa,CAAC;YAC9E,MAAM,MAAM,GAAc,EAAE,CAAC;YAC7B,IAAI,UAAU,GAAW,CAAC,CAAC;YAC3B,IAAI,eAAe,GAAW,CAAC,CAAC,CAAC,6CAA6C;YAE9E,uCAAuC;YACvC,MAAM,eAAe,GAAmC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAE7F,4BAA4B;YAC5B;;;;6DAIiD;YAEjD,sDAAsD;YACtD,MAAM,WAAW,GAAwB,IAAI,GAAG,EAAE,CAAC;YAEnD,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;gBAC1D,qEAAqE;gBACrE,IAAI,SAAS,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;oBACrC,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;oBAEtC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,8JAA8J;wBAC9J,gJAAgJ;wBAChJ,MAAM,WAAW,GAAa,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,eAAe,EAAE,eAAe,GAAG,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,gDAAgD;wBAC1L,oFAAoF;wBAEpF,MAAM,KAAK,GAAY,IAAI,6CAAkB,CAAC;4BAC5C,UAAU,EAAE,IAAI,CAAC,UAAU;4BAC3B,MAAM,EAAE,IAAI,CAAC,GAAG;4BAEhB,eAAe,EAAE,IAAI,CAAC,YAAY;4BAClC,gBAAgB,EAAE,WAAW;4BAE7B,iBAAiB,EAAE,CAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,GAAG,WAAW,CAAE;4BAChE,IAAI,EAAE,CAAE,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,WAAW,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC;4BAExD,YAAY,EAAE,IAAI,CAAC,kBAAkB;4BACrC,QAAQ,EAAE,IAAI,CAAC,IAAI;4BACnB,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC;4BACnC,UAAU,EAAE,UAAU;4BAEtB,YAAY,EAAE,YAAY;4BAC1B;;gEAEoC;yBACrC,CAAY,CAAC;wBAEd,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;4BAC9B,qBAAQ,CAAC,MAAM,CACb,gDAAgD,EAChD,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,CAC9B,CAAC;wBACJ,CAAC;wBAED,wEAAwE;wBACxE,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;4BAC9B,qBAAQ,CAAC,MAAM,CACb,+CAA+C,EAC/C,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE;gCACE,KAAK,EAAE,KAAK,CAAC,YAAY;6BAC1B,CACF,CAAC;wBACJ,CAAC;wBAED,qCAAqC;wBACrC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACrB,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,+DAA+D;gBAC/D,MAAM,UAAU,GAA4C,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;gBAE7G,IAAI,UAAU,IAAI,UAAU,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBACvC,sDAAsD;oBACtD,MAAM,YAAY,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;oBAE7E,2BAA2B;oBAC3B;;;qGAGiF;oBAEjF,6BAA6B;oBAC7B,6DAA6D;oBAC7D,wFAAwF;oBACxF,IAAI;oBACJ,sDAAsD;oBAEtD,8JAA8J;oBAC9J,gJAAgJ;oBAChJ,MAAM,WAAW,GAAa,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,eAAe,EAAE,eAAe,GAAG,UAAU,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,gDAAgD;oBAC1L,oFAAoF;oBAEpF;kHAC8F;oBAE9F,MAAM,KAAK,GAAY,IAAI,6CAAkB,CAAC;wBAC5C,UAAU,EAAE,IAAI,CAAC,UAAU;wBAC3B,MAAM,EAAE,IAAI,CAAC,GAAG;wBAEhB,eAAe,EAAE,IAAI,CAAC,YAAY;wBAClC,gBAAgB,EAAE,WAAW;wBAE7B,iBAAiB,EAAE,CAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,GAAG,WAAW,CAAE;wBAChE,IAAI,EAAE,CAAE,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,WAAW,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC;wBAExD,YAAY,EAAE,IAAI,CAAC,kBAAkB;wBACrC,QAAQ,EAAE,IAAI,CAAC,IAAI;wBACnB,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC;wBACnC,UAAU,EAAE,UAAU;wBAEtB,YAAY,EAAE,YAAY;wBAC1B;;4DAEoC;qBACrC,CAAY,CAAC;oBAEd,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;wBAC9B,qBAAQ,CAAC,MAAM,CACb,+CAA+C,EAC/C,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,CAC9B,CAAC;oBACJ,CAAC;oBAED,wEAAwE;oBACxE,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;wBAC9B,qBAAQ,CAAC,MAAM,CACb,8CAA8C,EAC9C,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE;4BACE,KAAK,EAAE,KAAK,CAAC,YAAY;yBAC1B,CACF,CAAC;oBACJ,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,UAAU,EAAE,CAAC;oBACb,eAAe,IAAI,UAAU,CAAC,KAAK,CAAC;oBACpC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;oBAElD,+CAA+C;oBAC/C,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;oBAEtE;4EACwD;gBAC1D,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,MAAM,YAAY,GAAW,SAAS,CAAC,SAAS,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;oBACzE,0JAA0J;oBAC1J,4IAA4I;oBAC5I,MAAM,WAAW,GAAa,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,eAAe,EAAE,eAAe,GAAG,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,gDAAgD;oBACtL,oFAAoF;oBAEpF,MAAM,KAAK,GAAY,IAAI,6CAAkB,CAAC;wBAC5C,UAAU,EAAE,IAAI,CAAC,UAAU;wBAC3B,MAAM,EAAE,IAAI,CAAC,GAAG;wBAEhB,eAAe,EAAE,IAAI,CAAC,YAAY;wBAClC,gBAAgB,EAAE,WAAW;wBAE7B,iBAAiB,EAAE,CAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,GAAG,WAAW,CAAE;wBAChE,IAAI,EAAE,CAAE,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,WAAW,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC;wBAExD,YAAY,EAAE,IAAI,CAAC,kBAAkB;wBACrC,QAAQ,EAAE,IAAI,CAAC,IAAI;wBACnB,QAAQ,EAAE,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC;wBACnC,UAAU,EAAE,UAAU;wBAEtB,YAAY,EAAE,YAAY;wBAC1B;;4DAEoC;qBACrC,CAAY,CAAC;oBAEd,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;wBAC9B,qBAAQ,CAAC,MAAM,CACb,gBAAgB,EAChB,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE,EAAE,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE,CAC9B,CAAC;oBACJ,CAAC;oBAED,wEAAwE;oBACxE,IAAI,IAAI,CAAC,QAAQ,IAAI,QAAQ,EAAE,CAAC;wBAC9B,qBAAQ,CAAC,MAAM,CACb,SAAS,UAAU,KAAK,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EACtE;4BACE,KAAK,EAAE,KAAK,CAAC,YAAY;yBAC1B,CACF,CAAC;oBACJ,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,UAAU,EAAE,CAAC;oBACb,eAAe,IAAI,YAAY,CAAC;oBAChC,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;oBAE9C,+CAA+C;oBAC/C,IAAI,CAAC,iBAAiB,CAAC,eAAe,EAAE,eAAe,EAAE,WAAW,CAAC,CAAC;gBACxE,CAAC;gBAED,yCAAyC;gBACzC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;oBACpE,MAAM,IAAI,uBAAU,CAAC;wBACnB,GAAG,IAAI,CAAC,uBAAuB,CAC7B,gBAAgB,EAChB,IAAI,KAAK,CAAC,oCAAoC,CAAC,EAC/C,MAAM,CACP;wBACD,SAAS,EAAE,cAAc;qBAC1B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,8CAA8C;YAC9C,IAAI,UAAU,IAAI,aAAa,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxD,qBAAQ,CAAC,IAAI,CACX,oCAAoC,aAAa,yBAAyB,SAAS,CAAC,MAAM,aAAa,EACvG;oBACE,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;iBAC9B,CACF,CAAC;YACJ,CAAC;YAED,yDAAyD;YACzD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,gBAAgB,EAAE,KAAK,EAAE,MAAM,CAAC;gBAChE,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;QAeI;IAEJ,MAAM,CAAC,yBAAyB,CAC9B,MAA4B,EAC5B,MAAc;QAEd,IAAI,CAAC;YACH,IAAI,WAAW,GAAW,EAAE,CAAC;YAE7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,MAAM,IAAI,uBAAU,CAAC;wBACnB,GAAG,IAAI,CAAC,uBAAuB,CAC7B,2BAA2B,EAC3B,IAAI,KAAK,CAAC,2DAA2D,CAAC,EACtE,MAAM,CACP;wBACD,SAAS,EAAE,gBAAgB;wBAE3B,iBAAiB,EAAE;4BACjB,MAAM,EAAE,MAAM;yBACf;qBACF,CAAC,CAAC;gBACL,CAAC;gBAED,WAAW,IAAI,KAAK,CAAC,YAAY,CAAC;YACpC,CAAC;YAED,OAAO,WAAW,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,uBAAU,CAAC;gBACnB,GAAG,IAAI,CAAC,uBAAuB,CAAC,kBAAkB,EAAE,KAAK,EAAE,MAAM,CAAC;gBAClE,SAAS,EAAE,cAAc;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,kBAAkB,CAAC,IAAY,EAAE,QAAgB;QAC9D,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAW,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAElF,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,OAAO;oBACL,KAAK,EAAE,KAAK,GAAG,UAAU,CAAC,aAAa;oBACvC,KAAK,EAAE,UAAU,CAAC,KAAK;iBACxB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,oBAAoB,CAAC,OAAe;QACjD,MAAM,OAAO,GAAmC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAa,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,YAAY,GAAW,CAAC,CAAC;QAE7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,WAAW,GAAW,IAAI,CAAC,IAAI,EAAE,CAAC;YAExC,sDAAsD;YACtD,IAAI,WAAW,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,GAA4B,WAAW,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAE1E,IAAI,KAAK,EAAE,CAAC;oBACV,MAAM,KAAK,GAAW,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,gCAAgC;oBACvE,MAAM,UAAU,GAAW,WAAW,CAAC;oBAEvC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC,0BAA0B;wBACxD,OAAO,CAAC,IAAI,CAAC;4BACX,KAAK,EAAE,YAAY;4BACnB,KAAK,EAAE,KAAK;4BACZ,MAAM,EAAE,UAAU;yBACnB,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,YAAY,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,qBAAqB;QACxD,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,iBAAiB,CAC9B,eAA+C,EAC/C,eAAuB,EACvB,WAAgC;QAEhC,sFAAsF;QACtF,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,KAAK,GAAG,eAAe,EAAE,CAAC;gBACnC,mDAAmD;gBACnD,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;gBAE7C,6EAA6E;gBAC7E,6DAA6D;gBAC7D,mDAAmD;gBACnD,KAAK,IAAI,KAAK,GAAW,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC/D,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACK,MAAM,CAAC,kBAAkB,CAC/B,eAA+C,EAC/C,UAAkB,EAClB,QAAgB,EAChB,WAAgC;QAEhC,2CAA2C;QAC3C,MAAM,kBAAkB,GAA6C,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACjG,MAAM,CAAC,KAAK,IAAI,UAAU,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CACtD,CAAC;QAEF,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,2EAA2E;YAC3E,IAAI,YAAY,GAAW,CAAC,CAAC;YAC7B,IAAI,aAAa,GAAW,EAAE,CAAC;YAE/B,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpD,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBACzB,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,MAAM,CAAC;gBACzB,CAAC;YACH,CAAC;YACD,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,CAAC;QAED,8DAA8D;QAC9D,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,KAAK,IAAI,KAAK,GAAW,CAAC,EAAE,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YACtE,MAAM,YAAY,GAAuB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEhE,IAAI,YAAY,EAAE,CAAC;gBACjB,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC;IAED;;;;;;;;OAQG;IACK,MAAM,CAAC,cAAc,CAC3B,QAAgB,EAChB,eAA+C,EAC/C,UAAkB,EAClB,QAAgB,EAChB,WAAgC;QAEhC,0CAA0C;QAC1C,MAAM,kBAAkB,GAA6C,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACjG,MAAM,CAAC,KAAK,IAAI,UAAU,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CACtD,CAAC;QAEF,IAAI,CAAC,kBAAkB,EAAE,CAAC;YACxB,uEAAuE;YACvE,MAAM,SAAS,GAAa,CAAC,QAAQ,CAAC,CAAC;YAEvC,iDAAiD;YACjD,IAAI,YAAY,GAAW,CAAC,CAAC;YAC7B,IAAI,aAAa,GAAW,EAAE,CAAC;YAE/B,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;gBACpD,IAAI,KAAK,GAAG,YAAY,EAAE,CAAC;oBACzB,YAAY,GAAG,KAAK,CAAC;oBACrB,aAAa,GAAG,MAAM,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,IAAI,aAAa,EAAE,CAAC;gBAClB,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,2BAA2B;QAC3B,MAAM,SAAS,GAAa,CAAC,QAAQ,CAAC,CAAC;QAEvC,4EAA4E;QAC5E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC;YAC9D,MAAM,YAAY,GAAuB,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAEhE,IAAI,YAAY,EAAE,CAAC;gBACjB,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,gDAAgD;YAChF,CAAC;QACH,CAAC;QAED,gCAAgC;QAChC,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,gDAAgD;QAE3F,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;OAQG;IACK,MAAM,CAAC,kBAAkB,CAC/B,YAAoB,EACpB,eAA+C,EAC/C,UAAkB,EAClB,QAAgB,EAChB,WAAgC;QAEhC,MAAM,eAAe,GAAa,IAAI,CAAC,kBAAkB,CAAC,eAAe,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAE9G,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,YAAY,CAAC;QAC5D,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACK,MAAM,CAAC,kBAAkB,CAAC,IAAY,EAAE,OAAe,EAAE,QAAgB;QAC/E,MAAM,UAAU,GAAW,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,SAAS,GAAW,UAAU,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAE1D,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,MAAM,CAAC,uBAAuB,CACtC,MAAc,EACd,KAAoB,EACpB,MAAc;QAGd,OAAO;YACL,MAAM,EAAG,KAAoB,EAAE,SAAS,IAAI,GAAG;YAC/C,OAAO,EAAG,KAAe,EAAE,OAAO;gBAC/B,KAAoB,EAAE,QAAQ;gBAC/B,GAAG,MAAM,sBAAsB,6CAAqB,CAAC,mBAAmB,GAAG;YAC7E,cAAc,EAAE,CAAE,KAAoB,EAAE,aAAa;YACrD,WAAW,EAAG,KAAoB,EAAE,aAAa,IAAI,IAAI,CAAC,mBAAmB;YAC7E,2BAA2B;YAC3B,MAAM,EAAE,MAAM;YACd,aAAa,EAAE,IAAI,CAAC,WAAW,EAAE,IAAI;YACrC,KAAK,EAAE,KAAK;SACb,CAAC;IACJ,CAAC;;AA7lBH,gEA8lBC"}

package/build/_modules/ai/_modules/document-ai/_collections/dai-document.util.d.ts

@@ -0,0 +1,29 @@
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
+/**
+ * Generic document utility functions
+ * Provider-agnostic document operations
+ */
+export declare class DyNTS_DAI_Document_Util {
+    static getClickupPageLink(pageName: string, workspaceId: string, documentId: string, pageId: string): string;
+    /**
+     * Validate document page structure
+     */
+    static validateDocumentPage(page: DyNTS_DAI_DocPage): boolean;
+    /**
+     * Normalize document page content
+     */
+    static normalizeContent(content: string): string;
+    /**
+     * Extract metadata from document content
+     */
+    static extractMetadata(content: string): Record<string, any>;
+    /**
+     * Check if content is empty or whitespace only
+     */
+    static isEmpty(content: string): boolean;
+    /**
+     * Get content preview (first N characters)
+     */
+    static getPreview(content: string, maxLength?: number): string;
+}
+//# sourceMappingURL=dai-document.util.d.ts.map

package/build/_modules/ai/_modules/document-ai/_collections/dai-document.util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-document.util.d.ts","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gDAAgD,CAAC;AAEnF;;;GAGG;AACH,qBAAa,uBAAuB;IAElC,MAAM,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM;IAM5G;;OAEG;IACH,MAAM,CAAC,oBAAoB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO;IAc7D;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAOhD;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAsB5D;;OAEG;IACH,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAIxC;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,GAAG,MAAM;CAOpE"}

package/build/_modules/ai/_modules/document-ai/_collections/dai-document.util.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_DAI_Document_Util = void 0;
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+/**
+ * Generic document utility functions
+ * Provider-agnostic document operations
+ */
+class DyNTS_DAI_Document_Util {
+    static getClickupPageLink(pageName, workspaceId, documentId, pageId) {
+        // https://app.clickup.com/9012753333/v/dc/8ck7cxn-692/8ck7cxn-1052
+        const link = `https://app.clickup.com/${workspaceId}/v/dc/${documentId}/${pageId}`;
+        return `[${pageName}](${link})`;
+    }
+    /**
+     * Validate document page structure
+     */
+    static validateDocumentPage(page) {
+        if (!page.name) {
+            fsm_dynamo_1.DyFM_Log.error('Document page validation failed: missing name');
+            return false;
+        }
+        if (!page.content) {
+            fsm_dynamo_1.DyFM_Log.error('Document page validation failed: missing content');
+            return false;
+        }
+        return true;
+    }
+    /**
+     * Normalize document page content
+     */
+    static normalizeContent(content) {
+        return content
+            .replace(/\r\n/g, '\n') // Normalize line endings
+            .replace(/\r/g, '\n') // Handle old Mac line endings
+            .trim(); // Remove leading/trailing whitespace
+    }
+    /**
+     * Extract metadata from document content
+     */
+    static extractMetadata(content) {
+        const metadata = {};
+        // Extract title from first header
+        const titleMatch = content.match(/^#\s+(.+)$/m);
+        if (titleMatch) {
+            metadata.title = titleMatch[1].trim();
+        }
+        // Extract word count
+        metadata.wordCount = content.split(/\s+/).length;
+        // Extract character count
+        metadata.characterCount = content.length;
+        // Extract header count
+        const headerMatches = content.match(/^#{1,6}\s+.+$/gm);
+        metadata.headerCount = headerMatches ? headerMatches.length : 0;
+        return metadata;
+    }
+    /**
+     * Check if content is empty or whitespace only
+     */
+    static isEmpty(content) {
+        return !content || content.trim().length === 0;
+    }
+    /**
+     * Get content preview (first N characters)
+     */
+    static getPreview(content, maxLength = 200) {
+        if (content.length <= maxLength) {
+            return content;
+        }
+        return content.substring(0, maxLength) + '...';
+    }
+}
+exports.DyNTS_DAI_Document_Util = DyNTS_DAI_Document_Util;
+//# sourceMappingURL=dai-document.util.js.map

package/build/_modules/ai/_modules/document-ai/_collections/dai-document.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-document.util.js","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts"],"names":[],"mappings":";;;AAAA,sDAAiD;AAGjD;;;GAGG;AACH,MAAa,uBAAuB;IAElC,MAAM,CAAC,kBAAkB,CAAC,QAAgB,EAAE,WAAmB,EAAE,UAAkB,EAAE,MAAc;QAC5F,mEAAmE;QACxE,MAAM,IAAI,GAAW,2BAA2B,WAAW,SAAS,UAAU,IAAI,MAAM,EAAE,CAAC;QAC3F,OAAO,IAAI,QAAQ,KAAK,IAAI,GAAG,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,oBAAoB,CAAC,IAAuB;QACjD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,qBAAQ,CAAC,KAAK,CAAC,+CAA+C,CAAC,CAAC;YAChE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,qBAAQ,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;YACnE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,gBAAgB,CAAC,OAAe;QACrC,OAAO,OAAO;aACX,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAE,yBAAyB;aACjD,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAI,8BAA8B;aACtD,IAAI,EAAE,CAAC,CAAiB,qCAAqC;IAClE,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,eAAe,CAAC,OAAe;QACpC,MAAM,QAAQ,GAAwB,EAAE,CAAC;QAEzC,kCAAkC;QAClC,MAAM,UAAU,GAA4B,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QACzE,IAAI,UAAU,EAAE,CAAC;YACf,QAAQ,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC;QAED,qBAAqB;QACrB,QAAQ,CAAC,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;QAEjD,0BAA0B;QAC1B,QAAQ,CAAC,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;QAEzC,uBAAuB;QACvB,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;QACvD,QAAQ,CAAC,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAEhE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,OAAO,CAAC,OAAe;QAC5B,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;IACjD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,OAAe,EAAE,YAAoB,GAAG;QACxD,IAAI,OAAO,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAChC,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,OAAO,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,GAAG,KAAK,CAAC;IACjD,CAAC;CACF;AA7ED,0DA6EC"}

package/build/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.d.ts

@@ -0,0 +1,7 @@
+export declare enum DyNTS_DAI_CompareResult_Type {
+    equal = "equal",
+    modified = "modified",
+    new = "new",
+    deleted = "deleted"
+}
+//# sourceMappingURL=dai-compare-result-type.enum.d.ts.map

package/build/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-compare-result-type.enum.d.ts","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts"],"names":[],"mappings":"AAEA,oBAAY,4BAA4B;IACtC,KAAK,UAAU;IACf,QAAQ,aAAa;IACrB,GAAG,QAAQ;IACX,OAAO,YAAY;CACpB"}

package/build/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_DAI_CompareResult_Type = void 0;
+var DyNTS_DAI_CompareResult_Type;
+(function (DyNTS_DAI_CompareResult_Type) {
+    DyNTS_DAI_CompareResult_Type["equal"] = "equal";
+    DyNTS_DAI_CompareResult_Type["modified"] = "modified";
+    DyNTS_DAI_CompareResult_Type["new"] = "new";
+    DyNTS_DAI_CompareResult_Type["deleted"] = "deleted";
+})(DyNTS_DAI_CompareResult_Type || (exports.DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type = {}));
+//# sourceMappingURL=dai-compare-result-type.enum.js.map

package/build/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-compare-result-type.enum.js","sourceRoot":"","sources":["../../../../../../src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts"],"names":[],"mappings":";;;AAEA,IAAY,4BAKX;AALD,WAAY,4BAA4B;IACtC,+CAAe,CAAA;IACf,qDAAqB,CAAA;IACrB,2CAAW,CAAA;IACX,mDAAmB,CAAA;AACrB,CAAC,EALW,4BAA4B,4CAA5B,4BAA4B,QAKvC"}

package/build/_modules/{open-ai/_models/oai-doc-chunk.data-model.d.ts → ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.d.ts}

@@ -1,28 +1,41 @@
-import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
-export declare class DyNTS_OAI_Chunk extends DyFM_Metadata {
+import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+/**
+ * Generic chunk model for document processing
+ * Provider-agnostic version of document chunks
+ */
+export declare class DyNTS_DAI_DocChunk extends DyFM_Metadata {
+    documentId: string;
+    /**
+     * this is the document id in the outside source
+     * eg.: the id in Clickup, Confluence, etc.
+     * */
+    externalDocumentId?: string;
+    pageId: string;
+    /**
+     * this is the page id in the outside source
+     * eg.: the id in Clickup, Confluence, etc.
+     * */
+    externalPageId?: string;
     /** Path to the page where the chunk is located ending with the page name */
     filePathParents: string[];
-    /** Headers that are parents of the chunk */
+    /** Headers that are parents of the chunk (all headers, all parent pages, and document name) */
     chunkHeadParents: string[];
     /**
      * All parents of the chunk, flagged, from the highest level to the lowest local level
      * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc.
-     * */
+     */
     allFlaggedParents: string[];
     /**
      * allFlaggedParentsMerged
      * All parents of the chunk, flagged, merged into a with '/' as separator
      * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
-     * or: System:FDP/Project:FDP Documentations/Document:Specifications/Services/Service:Organizer/Module:Chat/Page:Chat
-     * */
+     */
     path: string;
     /**
      * The page name with the full path
      * System/Project/Side/Module/Document/Page
      * */
-    /**
-     * The document name
-     * */
+    /** The document name */
     documentName: string;
     pageName: string;
     pageLink: string;
@@ -33,8 +46,7 @@ export declare class DyNTS_OAI_Chunk extends DyFM_Metadata {
      */
     chunkContent: string;
     chunkParentedContentVectorized?: number[];
-    constructor(set: DyNTS_OAI_Chunk);
+    constructor(set?: DyNTS_DAI_DocChunk);
 }
-export declare const chunk_propertyParams: DyFM_DataProperties<DyNTS_OAI_Chunk>;
-export declare const DyNTS_OAI_defaultChunk_dataParams: DyFM_DataModel_Params<DyNTS_OAI_Chunk>;
-//# sourceMappingURL=oai-doc-chunk.data-model.d.ts.map
+export declare const DyNTS_DAI_docChunk_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocChunk>;
+//# sourceMappingURL=dai-doc-chunk.data-model.d.ts.map

package/build/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dai-doc-chunk.data-model.d.ts","sourceRoot":"","sources":["../../../../../../../src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAuB,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAIlG;;;GAGG;AACH,qBAAa,kBAAmB,SAAQ,aAAa;IACnD,UAAU,EAAE,MAAM,CAAC;IACnB;;;SAGK;IACL,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf;;;SAGK;IACL,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB,4EAA4E;IAC5E,eAAe,EAAE,MAAM,EAAE,CAAM;IAC/B,+FAA+F;IAC/F,gBAAgB,EAAE,MAAM,EAAE,CAAM;IAEhC;;;OAGG;IACH,iBAAiB,EAAE,MAAM,EAAE,CAAM;IACjC;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IACb;;;SAGK;IAGL,wBAAwB;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,UAAU,EAAE,MAAM,CAAC;IAInB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,8BAA8B,CAAC,EAAE,MAAM,EAAE,CAAC;gBAGxC,GAAG,CAAC,EAAE,kBAAkB;CAQ3B;AAED,eAAO,MAAM,6BAA6B,EAAE,qBAAqB,CAAC,kBAAkB,CAwEhF,CAAC"}