@futdevpro/nts-dynamo 1.11.5 → 1.11.6

package/src/_modules/open-ai/_collections/oai-chunking.util.ts

@@ -1,4 +1,4 @@
-import { DyFM_AnyError, DyFM_Error, DyFM_Error_Settings, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyFM_AnyError, DyFM_Error, DyFM_Error_Settings, DyFM_Log, DyFM_wait } from '@futdevpro/fsm-dynamo';
 import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
 import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
 import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
@@ -14,6 +14,8 @@ export class DyNTS_OAI_Chunking_Util {
     `We encountered an unhandled Control Service Error, ` +
     `\nplease contact the responsible development team.`; 
 
+  protected static readonly debugLog: boolean = false;
+
   /**
    * Markdown tartalom chunking-ja a leghosszabb lehetséges chunk-ok létrehozásához
    * @param page - ClickUp dokumentum oldal
@@ -62,15 +64,15 @@ export class DyNTS_OAI_Chunking_Util {
           if (chunkContent.length > 0) {
             const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
             const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders);
-            const headParents = headerHierarchy.map(header => header.header);
+            const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + leftovers.length, lastHeaders); // headerHierarchy.map(header => header.header);
             const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
             
             const chunk: T_Chunk = new DyNTS_OAI_Chunk({
               filePathParents: page.path,
               chunkHeadParents: headParents,
 
-              allFlaggedParents: allFlaggedParents,
-              allFlaggedParentsMerged: allFlaggedParents.join('/'),
+              allFlaggedParents: [ ...page.allFlaggedParents, ...headParents ],
+              path: [ ...page.path, ...headParents ].join('/'),
 
               documentName: page.documentId,
               pageName: page.name,
@@ -79,9 +81,26 @@ export class DyNTS_OAI_Chunking_Util {
               
               chunkOriginalContent: chunkContent,
 
-              chunkParentedContent: chunkWithHierarchy,
+              chunkContent: chunkWithHierarchy,
             }) as T_Chunk;
-            chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
+            if (this.debugLog) {
+              DyFM_Log.H_info(
+                'Full content (before assembleChunkWithHeaders)',
+                `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+                { chunk: chunk.chunkContent }
+              );
+            }
+            chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer);
+            if (this.debugLog) {
+              DyFM_Log.H_info(
+                'Full content (after assembleChunkWithHeaders)',
+                `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+                { 
+                  chunk: chunk.chunkContent,
+                }
+              );
+            }
+            await DyFM_wait(100_000);
             chunks.push(chunk);
           }
           break;
@@ -108,7 +127,7 @@ export class DyNTS_OAI_Chunking_Util {
           
           const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
           const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders);
-          const headParents = headerHierarchy.map(header => header.header);
+          const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + breakPoint.index, lastHeaders); // headerHierarchy.map(header => header.header);
           const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
           
           /* console.log(`Chunk path:`, chunkPath);
@@ -118,8 +137,8 @@ export class DyNTS_OAI_Chunking_Util {
             filePathParents: page.path,
             chunkHeadParents: headParents,
 
-            allFlaggedParents: allFlaggedParents,
-            allFlaggedParentsMerged: allFlaggedParents.join('/'),
+            allFlaggedParents: [ ...page.allFlaggedParents, ...headParents ],
+            path: [ ...page.path, ...headParents ].join('/'),
 
             documentName: page.documentId,
             pageName: page.name,
@@ -128,9 +147,25 @@ export class DyNTS_OAI_Chunking_Util {
 
             chunkOriginalContent: chunkContent,
 
-            chunkParentedContent: chunkWithHierarchy,
+            chunkContent: chunkWithHierarchy,
           }) as T_Chunk;
-          chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
+          if (this.debugLog) {
+            DyFM_Log.H_info(
+              'Break point (before assembleChunkWithHeaders)',
+              `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+              { chunk: chunk.chunkContent }
+            );
+          }
+          chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer);
+          if (this.debugLog) {
+            DyFM_Log.H_info(
+              'Break point (after assembleChunkWithHeaders)',
+              `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+              { 
+                chunk: chunk.chunkContent,
+              }
+            );
+          }
           chunks.push(chunk);
           chunkIndex++;
           currentPosition += breakPoint.index;
@@ -146,15 +181,15 @@ export class DyNTS_OAI_Chunking_Util {
           const chunkContent = leftovers.substring(0, maxChunkSize).trim();
           const chunkWithHierarchy = this.addHeaderHierarchy(chunkContent, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
           const chunkPath = this.buildChunkPath(page.name, headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders);
-          const headParents = headerHierarchy.map(header => header.header);
+          const headParents = this.getRelevantHeaders(headerHierarchy, currentPosition, currentPosition + maxChunkSize, lastHeaders); // headerHierarchy.map(header => header.header);
           const allFlaggedParents = [ ...page.allFlaggedParents, ...headParents ];
           
           const chunk: T_Chunk = new DyNTS_OAI_Chunk({
             filePathParents: page.path,
             chunkHeadParents: headParents,
 
-            allFlaggedParents: allFlaggedParents,
-            allFlaggedParentsMerged: allFlaggedParents.join('/'),
+            allFlaggedParents: [ ...page.allFlaggedParents, ...headParents ],
+            path: [ ...page.path, ...headParents ].join('/'),
 
             documentName: page.documentId,
             pageName: page.name,
@@ -163,9 +198,24 @@ export class DyNTS_OAI_Chunking_Util {
 
             chunkOriginalContent: chunkContent,
 
-            chunkParentedContent: chunkWithHierarchy,
+            chunkContent: chunkWithHierarchy,
           }) as T_Chunk;
-          chunk.chunkParentedContent = this.assembleChunkWithHeaders(chunk, issuer);
+          if (this.debugLog) {
+            DyFM_Log.H_info(
+              'Max chunk size',
+              `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+              { chunk: chunk.chunkContent }
+            );
+          }
+          chunk.chunkContent = this.assembleChunkWithHeaders(chunk, issuer);
+          if (this.debugLog) {
+            DyFM_Log.H_info(
+              `Chunk ${chunkIndex}: ${chunk.path.replaceAll('/', '/\n').split('/')}`, 
+              { 
+                chunk: chunk.chunkContent,
+              }
+            );
+          }
           chunks.push(chunk);
           chunkIndex++;
           currentPosition += maxChunkSize;
@@ -357,6 +407,49 @@ export class DyNTS_OAI_Chunking_Util {
     }
   }
 
+  /**
+   * Visszaadja a chunk-hoz tartozó releváns szülő fejléceket a hierarchia alapján.
+   * @param headerHierarchy - A feldolgozott fejléc-hierarchia
+   * @param startIndex - A chunk kezdőindexe az eredeti tartalomban
+   * @param endIndex - A chunk végindexe az eredeti tartalomban
+   * @param lastHeaders - Az utolsó ismert fejlécek szintenként
+   * @returns A releváns szülő fejlécek tömbje
+   */
+  private static getRelevantHeaders(
+    headerHierarchy: Array<{index: number, level: number, header: string}>,
+    startIndex: number,
+    endIndex: number,
+    lastHeaders: Map<number, string>
+  ): string[] {
+    // Keressük meg az első fejlécet a chunkban
+    const firstHeaderInChunk = headerHierarchy.find(header => 
+      header.index >= startIndex && header.index < endIndex
+    );
+
+    if (!firstHeaderInChunk) {
+      // Nincs fejléc a chunkban, a legmagasabb szintű utolsó fejlécet használjuk
+      let highestLevel = 0;
+      let highestHeader = '';
+      for (const [level, header] of lastHeaders.entries()) {
+        if (level > highestLevel) {
+          highestLevel = level;
+          highestHeader = header;
+        }
+      }
+      return highestHeader ? [highestHeader] : [];
+    }
+
+    // Szülő fejlécek összegyűjtése az első fejléc szintje alapján
+    const relevantHeaders: string[] = [];
+    for (let level = 1; level < firstHeaderInChunk.level; level++) {
+      const parentHeader = lastHeaders.get(level);
+      if (parentHeader) {
+        relevantHeaders.push(parentHeader);
+      }
+    }
+    return relevantHeaders;
+  }
+
   /**
    * Build hierarchical chunk path based on header structure
    * @param pageName - The page name
@@ -432,51 +525,10 @@ export class DyNTS_OAI_Chunking_Util {
     endIndex: number,
     lastHeaders: Map<number, string>
   ): string {
-    // Find the first header within this chunk to determine its level
-    const firstHeaderInChunk = headerHierarchy.find(header => 
-      header.index >= startIndex && header.index < endIndex
-    );
-
-    if (!firstHeaderInChunk) {
-      // No header in chunk, use the highest level from lastHeaders as parent
-      const relevantHeaders: string[] = [];
-      
-      // Find the highest level header from lastHeaders
-      let highestLevel = 0;
-      let highestHeader = '';
-      for (const [level, header] of lastHeaders.entries()) {
-        if (level > highestLevel) {
-          highestLevel = level;
-          highestHeader = header;
-        }
-      }
-      
-      if (highestHeader) {
-        relevantHeaders.push(highestHeader);
-      }
-
-      if (relevantHeaders.length > 0) {
-        return relevantHeaders.join('\n') + '\n\n' + chunkContent;
-      }
-
-      return chunkContent;
-    }
-
-    // Build hierarchy from stored last headers (only parent headers, not from chunk content)
-    const relevantHeaders: string[] = [];
-    
-    // Add parent headers based on the first header in chunk level
-    for (let level = 1; level < firstHeaderInChunk.level; level++) {
-      const parentHeader = lastHeaders.get(level);
-      if (parentHeader) {
-        relevantHeaders.push(parentHeader);
-      }
-    }
-
+    const relevantHeaders = this.getRelevantHeaders(headerHierarchy, startIndex, endIndex, lastHeaders);
     if (relevantHeaders.length > 0) {
       return relevantHeaders.join('\n') + '\n\n' + chunkContent;
     }
-
     return chunkContent;
   }
 
@@ -512,4 +564,9 @@ export class DyNTS_OAI_Chunking_Util {
       error: error,
     };
   }
-} 
+} 
+
+
+function isLast<T>(element: T, array: T[]): boolean {
+  return array.indexOf(element) === array.length - 1;
+}

package/src/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.ts

@@ -1,5 +1,7 @@
 import { DyNTS_OAI_Chunk } from '../oai-doc-chunk.data-model';
-import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../..';
+import { DyNTS_OAI_ChunkCompareResult } from './oai-chunk-compare-result.interface';
+import { DyNTS_OAI_CompareResult_Type } from '../../_enums/oai-compare-result-type.enum';
+import { DyNTS_OAI_DocumentPage } from './oai-document-page.interface';
 
 export interface DyNTS_OAI_PageCompareResult<
   T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk,

package/src/_modules/open-ai/_models/oai-doc-chunk.data-model.ts

@@ -16,10 +16,21 @@ export class DyNTS_OAI_Chunk extends DyFM_Metadata {
    * */
   allFlaggedParents: string[] = [];
   /** 
+   * allFlaggedParentsMerged
    * All parents of the chunk, flagged, merged into a with '/' as separator
+   * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
+   * or: System:FDP/Project:FDP Documentations/Document:Specifications/Services/Service:Organizer/Module:Chat/Page:Chat
+   * */
+  path: string;
+  /** 
+   * The page name with the full path
+   * System/Project/Side/Module/Document/Page
+   * */
+  /* pageNameWithFullPath: string; */
+
+  /** 
+   * The document name
    * */
-  allFlaggedParentsMerged: string;
-  
   documentName: string;
   pageName: string;
   pageLink: string;
@@ -28,7 +39,10 @@ export class DyNTS_OAI_Chunk extends DyFM_Metadata {
 
   chunkOriginalContent: string;
 
-  chunkParentedContent: string;
+  /**
+   * chunk Parented Content
+   */
+  chunkContent: string;
   chunkParentedContentVectorized?: number[];
 
   constructor(
@@ -54,7 +68,7 @@ export const chunk_propertyParams: DyFM_DataProperties<DyNTS_OAI_Chunk> = {
     type: 'string[]', 
     required: true,
   },
-  allFlaggedParentsMerged: {
+  path: {
     type: 'string', index: true,
     required: true,
   },
@@ -68,8 +82,7 @@ export const chunk_propertyParams: DyFM_DataProperties<DyNTS_OAI_Chunk> = {
     required: true,
   },
   pageLink: {
-    type: 'string', index: true,
-    required: true,
+    type: 'string',
   },
   chunkIndex: {
     type: 'number', index: true,
@@ -81,21 +94,22 @@ export const chunk_propertyParams: DyFM_DataProperties<DyNTS_OAI_Chunk> = {
     required: true,
   },
 
-  chunkParentedContent: {
-    type: 'string', // index: true,
+  chunkContent: {
+    type: 'string',
     required: true,
   },
   chunkParentedContentVectorized: {
     type: 'number[]',
     required: true,
     embeddingModel: DyFM_OpenAIModel.textEmbedding_3Large,
-    vectorizedFrom: [ 'chunkParentedContent' ],
+    vectorizedFrom: [ 'path', 'chunkContent' ],
     vectorizeUseIndex: 'chunkParentedContentVectorized',
     vectorizeAlways: true,
   },
 }
 
-export const DyNTS_OAI_defaultChunk_dataParams: DyFM_DataModel_Params<DyNTS_OAI_Chunk> = new DyFM_DataModel_Params<DyNTS_OAI_Chunk>({
-  dataName: 'doc_chunk',
-  properties: chunk_propertyParams,
-});
+export const DyNTS_OAI_defaultChunk_dataParams: DyFM_DataModel_Params<DyNTS_OAI_Chunk> = 
+  new DyFM_DataModel_Params<DyNTS_OAI_Chunk>({
+    dataName: 'doc_chunk',
+    properties: chunk_propertyParams,
+  });

package/src/_modules/open-ai/_services/oai-chunk.service-base.ts

@@ -1,9 +1,11 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata, DyFM_wait } from '@futdevpro/fsm-dynamo';
 import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
 import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
 import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
 import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
-import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../../discord-assistant';
+import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
+import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
+import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
 import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
 
 
@@ -12,6 +14,8 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
   T_Page extends DyNTS_OAI_DocumentPage<T_Chunk> = DyNTS_OAI_DocumentPage<T_Chunk>, 
 > extends DyNTS_OAI_VectorDataService<T_Chunk> {
 
+  debugLog: boolean = true;
+
   constructor(
     dataParams: DyFM_DataModel_Params<T_Chunk>,
     openAiSettings: DyFM_OpenAI_Settings,
@@ -38,6 +42,7 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
       if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
         // delete extra chunks
         for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
+
           if (i < pageCompareResult.newChunks.length) {
             pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
           } else {
@@ -54,6 +59,18 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
           if (
             pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_OAI_CompareResult_Type.equal
           ) {
+            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
+            chunk.na */
+
+            chunk.path = chunk.allFlaggedParents.join('/');
+
+            if (this.debugLog) {
+              DyFM_Log.H_info(`✅ saving Chunk: ${pageCompareResult.subjectPage.name}; ${chunk.chunkIndex}`, {
+                chunk: chunk,
+              });
+              await DyFM_wait(10_000);
+            }
+
             await this.saveData(chunk, false, true);
             DyFM_Log.info(`✅ Chunk saved: ${pageCompareResult.subjectPage.name}; ${chunk.chunkIndex}`)
           }
@@ -71,8 +88,18 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
 
   async comparePage(page: T_Page, issuer: string): Promise<DyNTS_OAI_PageCompareResult<T_Chunk, T_Page>> {
     try {
+      if (!page.allFlaggedParentsMerged) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'comparePage', 
+            new Error(`allFlaggedParentsMerged is not set (${page.name})`),
+          ),
+          errorCode: 'DCH-CSB-CP0',
+        });
+      }
+
       const oldChunks = await this.findDataList({
-        allFlaggedParentsMerged: page.allFlaggedParentsMerged,
+        path: page.allFlaggedParentsMerged,
       } as DyFM_DBFilterSimple<T_Chunk>);
 
       const compareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[] = await this.compareChunks(
@@ -143,8 +170,17 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
         return [];
       }
 
+      if (chunks.some(chunk => !chunk.path)) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'compareChunks', new Error('allFlaggedParentsMerged is not set'),
+          ),
+          errorCode: 'DCH-CSB-CC0',
+        });
+      }
+
       oldChunks ??= await this.findDataList({
-        allFlaggedParentsMerged: chunks[0].allFlaggedParentsMerged,
+        path: chunks[0].path,
       } as DyFM_DBFilterSimple<T_Chunk>);
       oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
 
@@ -168,7 +204,7 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
   ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>> {
     try {
       oldChunk ??= await this.findData({
-        allFlaggedParentsMerged: newChunk.allFlaggedParentsMerged,
+        path: newChunk.path,
         chunkIndex: newChunk.chunkIndex,
       } as DyFM_DBFilterSimple<T_Chunk>);
 
@@ -187,7 +223,7 @@ export class DyNTS_OAI_Chunk_DataServiceBase<
         };
       }
 
-      if (oldChunk.chunkParentedContent === newChunk.chunkParentedContent) {
+      if (oldChunk.chunkContent === newChunk.chunkContent) {
         return {
           result: DyNTS_OAI_CompareResult_Type.equal,
           oldChunk: oldChunk,

package/src/_modules/open-ai/_services/oai-vector-data.service.ts

@@ -251,9 +251,11 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
     /* alwaysVectorize?: boolean */
   ): Promise<string> {
     if (vectorizedProperty.vectorizedFrom.length === 1) {
-      const value: unknown = data[vectorizedProperty.vectorizedFrom[0]];
+      const value: unknown = data?.[vectorizedProperty.vectorizedFrom[0]];
 
-      if (typeof value === 'string') {
+      if (value === undefined) {
+        return undefined;
+      } else if (typeof value === 'string') {
         return value;
       } else if (Array.isArray(value) || typeof value === 'object') {
         return JSON.stringify(value);
@@ -264,7 +266,9 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
       const object: any = {};
 
       vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
-        object[key] = data[key];
+        if (data?.[key] !== undefined) {
+          object[key] = data[key];
+        }
       });
 
       return JSON.stringify(object);

package/src/_services/base/data.service.ts

@@ -1190,13 +1190,14 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
                 'validateForSave',
                 new Error(
                   `validateForSave failed, "${element.key}" is missing! ` +
-                  `(${this.dataParams.dataName})`
+                  `(index or required in "${this.dataParams.dataName}" dataParams) `
                 )
               ),
 
               status: 522, 
               errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-DS0-VD1`,
               userMessage: this.defaultValidationErrorUserMsg,
+              __localStack: this.dataParams.stackLocation,
               additionalContent: {
                 data: data,
               },
@@ -1219,6 +1220,7 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
               status: 522, 
               errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-DS0-VD2`,
               userMessage: this.defaultValidationErrorUserMsg,
+              __localStack: this.dataParams.stackLocation,
               additionalContent: {
                 data: data,
               },