@futdevpro/nts-dynamo 1.14.6 → 1.14.8

package/src/_modules/ai/_modules/open-ai/_services/oai-doc-chunk-data.service.ts

@@ -20,7 +20,7 @@ export class DyNTS_OAI_DocChunkDataService<
   T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
 > extends DyNTS_OAI_VectorDataService<T_Chunk> {
 
-  override debugLog: boolean = true;
+  /* override debugLog: boolean = false; */
 
   constructor(
     issuer: string,
@@ -68,10 +68,21 @@ export class DyNTS_OAI_DocChunkDataService<
         const oldChunk = oldChunks[i];
         if (oldChunk) {
           newChunk._id = oldChunk._id;
-          const savedChunk: T_Chunk = await this.saveData(newChunk, true);
-          const chunkCompareResult: DyNTS_DAI_DocChunkCompareResult<T_Chunk> = await this.compareChunk(savedChunk, oldChunk);
-          chunkCompareResults.push(chunkCompareResult);
+          // Compare first to determine if we need to save
+          const chunkCompareResult: DyNTS_DAI_DocChunkCompareResult<T_Chunk> = await this.compareChunk(newChunk, oldChunk);
+          
+          if (chunkCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+            // No changes detected, reuse existing chunk without saving
+            chunkCompareResult.newChunk = oldChunk;
+            chunkCompareResults.push(chunkCompareResult);
+          } else {
+            // Changes detected, save the updated chunk
+            const savedChunk: T_Chunk = await this.saveData(newChunk, true);
+            chunkCompareResult.newChunk = savedChunk;
+            chunkCompareResults.push(chunkCompareResult);
+          }
         } else {
+          // New chunk, save it
           const savedChunk: T_Chunk = await this.saveData(newChunk, true);
           chunkCompareResults.push({
             result: DyNTS_DAI_CompareResult_Type.new,
@@ -146,31 +157,20 @@ export class DyNTS_OAI_DocChunkDataService<
         DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
       }
 
-      /* oldChunk ??= await this.findData({
-        path: newChunk.path,
-        chunkIndex: newChunk.chunkIndex,
-      } as DyFM_DBFilterSimple<T_Chunk>); */
-
-      /* DyFM_Log.H_info(`✅ Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`, {
-        newChunk: newChunk,
-        oldChunk: oldChunk,
-        new: !oldChunk,
-        equal: oldChunk?.chunkParentedContent === newChunk.chunkParentedContent,
-        modified: oldChunk && oldChunk.chunkParentedContent !== newChunk.chunkParentedContent,
-      }); */
-
-      /* if (!oldChunk) {
-        // itt kéne vektor search-el utána keresni, 
-        // hogy a page link-je megtalálható-e (avagy csak átnevezésre került-e)
-        // hogy az adott tartalom, nem-e csak átnevezésre, szétbontásra vagy összevonásra került-e
-        return {
-          result: DyNTS_DAI_CompareResult_Type.new,
-          newChunk: newChunk,
-          oldChunk: null,
-        };
-      } */
+      // Compare all relevant fields for comprehensive change detection
+      const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
+      const pathEqual = oldChunk.path === newChunk.path;
+      const documentNameEqual = oldChunk.documentName === newChunk.documentName;
+      const pageNameEqual = oldChunk.pageName === newChunk.pageName;
+      const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
+      
+      // Compare allFlaggedParents arrays
+      const allFlaggedParentsEqual = this.compareStringArrays(
+        oldChunk.allFlaggedParents || [], 
+        newChunk.allFlaggedParents || []
+      );
 
-      if (oldChunk.chunkContent === newChunk.chunkContent) {
+      if (contentEqual && pathEqual && documentNameEqual && pageNameEqual && pageLinkEqual && allFlaggedParentsEqual) {
         return {
           result: DyNTS_DAI_CompareResult_Type.equal,
           newChunk: newChunk,
@@ -190,6 +190,13 @@ export class DyNTS_OAI_DocChunkDataService<
       });
     }
   }
+
+  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
+    if (arr1.length !== arr2.length) {
+      return false;
+    }
+    return arr1.every((item, index) => item === arr2[index]);
+  }
   
   async chunkPage(page: T_Page): Promise<T_Chunk[]> {
     try {

package/src/_modules/ai/_modules/open-ai/_services/oai-doc-page.data-service.ts

@@ -106,12 +106,6 @@ export class DyNTS_OAI_DocPageDataService<
 
   async processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
     try {
-      // ugyan a logikus az lenne, ha először compare-elnénk, és csak az után mentenénk, 
-      //   de akkor nem lenne id, amit a sub-element-ekhez kellenek, ezért...
-      // - először kikérjük a régi adatot, 
-      // - aztán mentünk, hogy legyen id, 
-      // - és utána compare-eljük
-
       let existingPage: T_Page;
       if (page._id) {
         existingPage = await this.findData({
@@ -131,30 +125,47 @@ export class DyNTS_OAI_DocPageDataService<
         } as DyFM_DBFilterSimple<T_Page>);
       }
 
-      // save the page to ensure its id (create new obj, to protect from chunks deletion)
-      let pageDataToSave: T_Page;
+      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
+      
       if (existingPage) {
-        pageDataToSave = existingPage;
-        Object.assign(pageDataToSave, page);
+        // Compare first to determine if we need to save
+        pageCompareResult = await this.comparePage(page, existingPage);
+        
+        if (pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+          // No changes detected, reuse existing page without saving
+          pageCompareResult.newPage = existingPage;
+          pageCompareResult.oldPage = existingPage;
+        } else {
+          // Changes detected, save the updated page
+          let pageDataToSave: T_Page;
+          pageDataToSave = existingPage;
+          Object.assign(pageDataToSave, page);
+          delete pageDataToSave.chunks;
+
+          const savedPage: T_Page = await this.saveData(pageDataToSave, true);
+
+          page.chunks.forEach((chunk: T_Chunk) => {
+            chunk.documentId = savedPage.documentId;
+            chunk.pageId = savedPage._id;
+          });
+          page._id = savedPage._id;
+          pageCompareResult.newPage = page;
+        }
       } else {
-        pageDataToSave = {
+        // New page, save it first to get ID
+        const pageDataToSave: T_Page = {
           ...page,
         };
-      }
-      delete pageDataToSave.chunks;
+        delete pageDataToSave.chunks;
 
-      const savedPage: T_Page = await this.saveData(pageDataToSave, true);
+        const savedPage: T_Page = await this.saveData(pageDataToSave, true);
 
-      page.chunks.forEach((chunk: T_Chunk) => {
-        chunk.documentId = savedPage.documentId;
-        chunk.pageId = savedPage._id;
-      });
-      page._id = savedPage._id;
+        page.chunks.forEach((chunk: T_Chunk) => {
+          chunk.documentId = savedPage.documentId;
+          chunk.pageId = savedPage._id;
+        });
+        page._id = savedPage._id;
 
-      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
-      if (existingPage) {
-        pageCompareResult = await this.comparePage(page, existingPage);
-      } else {
         pageCompareResult = {
           result: DyNTS_DAI_CompareResult_Type.new,
           newPage: page,
@@ -263,6 +274,7 @@ export class DyNTS_OAI_DocPageDataService<
       const messages: string[] = [];
       const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
 
+      // Compare all relevant fields for comprehensive change detection
       if (newPage.name !== oldPage.name) {
         pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
         messages.push(`Page name modified`);
@@ -271,7 +283,42 @@ export class DyNTS_OAI_DocPageDataService<
       if (newPage.content !== oldPage.content) {
         pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
         messages.push(`Page content modified`);
+      }
 
+      // Compare allFlaggedParents arrays
+      const allFlaggedParentsEqual = this.compareStringArrays(
+        newPage.allFlaggedParents || [], 
+        oldPage.allFlaggedParents || []
+      );
+      if (!allFlaggedParentsEqual) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page allFlaggedParents modified`);
+      }
+
+      // Compare allFlaggedParentsMerged
+      if (newPage.allFlaggedParentsMerged !== oldPage.allFlaggedParentsMerged) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page allFlaggedParentsMerged modified`);
+      }
+
+      // Compare filePath
+      if (newPage.filePath !== oldPage.filePath) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page filePath modified`);
+      }
+
+      // Compare parentedPath arrays
+      const parentedPathEqual = this.compareStringArrays(
+        newPage.parentedPath || [], 
+        oldPage.parentedPath || []
+      );
+      if (!parentedPathEqual) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page parentedPath modified`);
+      }
+
+      // Only process chunks if page content actually changed
+      if (pageCompareResult === DyNTS_DAI_CompareResult_Type.modified) {
         const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
           this.getDocChunkDataService();
 
@@ -300,6 +347,13 @@ export class DyNTS_OAI_DocPageDataService<
     }
   }
 
+  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
+    if (arr1.length !== arr2.length) {
+      return false;
+    }
+    return arr1.every((item, index) => item === arr2[index]);
+  }
+
   /* async comparePage_deep(page: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
     try {
       if (this.debugLog) {

package/src/_modules/ai/_modules/open-ai/_services/oai-document.data-service.ts

@@ -53,6 +53,10 @@ export class DyNTS_OAI_DocumentDataService<
     DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[]
   > {
     try {
+      if (!documents || documents.length === 0) {
+        return [];
+      }
+
       const compareResults: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[] = [];
 
       await DyFM_Array.asyncForEachAllAtOnce(
@@ -96,7 +100,7 @@ export class DyNTS_OAI_DocumentDataService<
         messages.unshift(`${pagesToDelete.length} pages deleted`);
 
         await this.deleteData(deletedDocument._id, this.absoluteDeletesOnProcess);
-        messages.unshift(`Document "${deletedDocument.name}" deleted`);
+        messages.unshift(`Document "${deletedDocument?.name}" deleted`);
 
         compareResults.push({
           result: DyNTS_DAI_CompareResult_Type.deleted,
@@ -124,13 +128,6 @@ export class DyNTS_OAI_DocumentDataService<
     DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
   > {
     try {
-      // ugyan a logikus az lenne, ha először compare-elnénk, és csak az után mentenénk, 
-      //   de akkor nem lenne id, amit a sub-element-ekhez kellenek, ezért...
-      // - először kikérjük a régi adatot, 
-      // - aztán mentünk, hogy legyen id, 
-      // - és utána compare-eljük
-
-
       let existingDocument: T_Document; 
 
       if (document._id) {
@@ -150,30 +147,47 @@ export class DyNTS_OAI_DocumentDataService<
         });
       }
 
-      // save the document to ensure its id (create new obj, to protect from pages deletion)
-      let docDataToSave: T_Document;
+      let documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>;
+      
       if (existingDocument) {
-        docDataToSave = existingDocument;
-        Object.assign(docDataToSave, document);
+        // Compare first to determine if we need to save
+        documentCompareResult = await this.compareDocument_deep(document, existingDocument);
+        
+        if (documentCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+          // No changes detected, reuse existing document without saving
+          documentCompareResult.newDocument = existingDocument;
+          documentCompareResult.oldDocument = existingDocument;
+        } else {
+          // Changes detected, save the updated document
+          let docDataToSave: T_Document;
+          docDataToSave = existingDocument;
+          Object.assign(docDataToSave, document);
+          delete docDataToSave.pages;
+
+          const savedDocument: T_Document = await this.saveData(docDataToSave, true);
+
+          // set the id to the pages and the document
+          document.pages.forEach((page: T_Page) => {
+            page.documentId = savedDocument._id;
+          });
+          document._id = savedDocument._id;
+          documentCompareResult.newDocument = document;
+        }
       } else {
-        docDataToSave = {
+        // New document, save it first to get ID
+        const docDataToSave: T_Document = {
           ...document,
         };
-      };
-      delete docDataToSave.pages;
+        delete docDataToSave.pages;
 
-      const savedDocument: T_Document = await this.saveData(docDataToSave, true);
+        const savedDocument: T_Document = await this.saveData(docDataToSave, true);
 
-      // set the id to the pages and the document
-      document.pages.forEach((page: T_Page) => {
-        page.documentId = savedDocument._id;
-      });
-      document._id = savedDocument._id;
+        // set the id to the pages and the document
+        document.pages.forEach((page: T_Page) => {
+          page.documentId = savedDocument._id;
+        });
+        document._id = savedDocument._id;
 
-      let documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>;
-      if (existingDocument) {
-        documentCompareResult = await this.compareDocument_deep(document, existingDocument);
-      } else {
         const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
           this.getDocPageDataService();
 
@@ -187,8 +201,8 @@ export class DyNTS_OAI_DocumentDataService<
           oldDocument: null,
           pageCompareResults: pagesCompareResults,
           reports: [
-            `New document: ${document.name}`,
-            `${document.pages.length} new pages in ${document.name}`,
+            `New document: "${document?.name}"`,
+            `${document?.pages?.length ?? 0} new pages in "${document?.name}"`,
           ],
         };
       }
@@ -206,7 +220,7 @@ export class DyNTS_OAI_DocumentDataService<
   > {
     try {
       if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} document: ${newDocument.name}`);
+        DyFM_Log.info(`🔍 Comparing "${this.dataParams.dataName}" document: "${newDocument?.name}"`);
       }
 
       if (!oldDocument) {
@@ -253,13 +267,13 @@ export class DyNTS_OAI_DocumentDataService<
         ).length;
 
         if (modifiedPagesCount > 0) {
-          messages.push(`${modifiedPagesCount} pages modified`);
+          messages.push(`${modifiedPagesCount} modified pages`);
         }
         if (newPagesCount > 0) {
-          messages.push(`${newPagesCount} pages new`);
+          messages.push(`${newPagesCount} new pages`);
         }
         if (deletedPagesCount > 0) {
-          messages.push(`${deletedPagesCount} pages deleted`);
+          messages.push(`${deletedPagesCount} deleted pages`);
         }
       } else {
         pagesResult = DyNTS_DAI_CompareResult_Type.new;
@@ -312,18 +326,34 @@ export class DyNTS_OAI_DocumentDataService<
         });
       }
 
-      let result: DyNTS_DAI_CompareResult_Type;
+      let result: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
       const messages: string[] = [];
 
-      if (!oldDocument) {
-        result = DyNTS_DAI_CompareResult_Type.new;
-        messages.push(`New document: ${newDocument.name}`);
-      } else if (newDocument.name !== oldDocument?.name) {
+      // Compare document name
+      if (newDocument.name !== oldDocument.name) {
         result = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Document name changed from "${oldDocument?.name}" to "${newDocument.name}"`);
-      } else {
-        result = DyNTS_DAI_CompareResult_Type.equal;
-        messages.push(`Equal document: ${newDocument.name}`);
+        messages.push(`Document name changed from "${oldDocument.name}" to "${newDocument.name}"`);
+      }
+
+      // Compare externalDocumentId
+      if (newDocument.externalDocumentId !== oldDocument.externalDocumentId) {
+        result = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Document externalDocumentId changed`);
+      }
+
+      // Compare externalMetadata if present
+      if (newDocument.externalMetadata || oldDocument.externalMetadata) {
+        const newMetadataStr = newDocument.externalMetadata ? JSON.stringify(newDocument.externalMetadata) : '';
+        const oldMetadataStr = oldDocument.externalMetadata ? JSON.stringify(oldDocument.externalMetadata) : '';
+        
+        if (newMetadataStr !== oldMetadataStr) {
+          result = DyNTS_DAI_CompareResult_Type.modified;
+          messages.push(`Document externalMetadata changed`);
+        }
+      }
+
+      if (result === DyNTS_DAI_CompareResult_Type.equal) {
+        messages.push(`Document name or not changed: "${newDocument.name}"`);
       }
 
       const result_flat: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = {
@@ -348,29 +378,35 @@ export class DyNTS_OAI_DocumentDataService<
     documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>, 
   ): Promise<void> {
     try {
-      DyFM_Log.log(`✅ Document processed: ${documentCompareResult.newDocument.name}`);
+      DyFM_Log.log(
+        `\n✅ Document processed: "${
+          documentCompareResult?.newDocument?.name ?? documentCompareResult?.oldDocument?.name ?? 'unknown'
+        }"`
+      );
 
       switch (documentCompareResult.result) {
         case DyNTS_DAI_CompareResult_Type.new:
-          DyFM_Log.info(` ✨ New document`);
+          DyFM_Log.info(`  ✨ New document`);
           break;
           
         case DyNTS_DAI_CompareResult_Type.equal:
-          DyFM_Log.log(` 🟰 Equal document`);
+          DyFM_Log.success(`  🟰 Equal document`);
           break;
 
         case DyNTS_DAI_CompareResult_Type.modified:
-          DyFM_Log.info(` 🔄 Modified document`);
+          DyFM_Log.info(`  🔄 Modified document`);
           break;
 
         case DyNTS_DAI_CompareResult_Type.deleted:
-          DyFM_Log.warn(` ❌ Deleted document`);
+          DyFM_Log.warn(`  ❌ Deleted document`);
           break;
       }
 
       for (const report of documentCompareResult.reports) {
         DyFM_Log.log(`  - ${report}`);
       }
+
+      DyFM_Log.log('');
     } catch (error) {
       throw new DyFM_Error({
         ...this.getDefaultErrorSettings('reportDocumentProcessing', error),

package/src/_modules/ai/_modules/open-ai/_services/oai-vector-data.service.ts

@@ -1,3 +1,4 @@
+import { DyFM_Log } from '@futdevpro/fsm-dynamo';
 
 import {
   DyFM_Array,
@@ -62,7 +63,7 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
   readonly vectorizedProperties: DyFM_DataProperty_Params<any, T>[];
   /* readonly vectorizeProperties: { from: string, to: string }[]; */
 
-  debugLog: boolean = true;
+  debugLog: boolean = false;
   swallowVectorizedProperties: boolean = true;
 
   embedding_CS: DyNTS_OAI_Embedding_ControlService;
@@ -244,7 +245,10 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
       }
 
       if (this.debugLog) {
-        console.log(`vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"`);
+        DyFM_Log.log(
+          `vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
+          (newData as any)?.name ? `: "${(newData as any)?.name}"` : ''
+        );
       }
 
       newData[vectorizedProperty.key] = await this.vectorize(
@@ -412,13 +416,13 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
       }
 
       if (this.debugLog) {
-        console.log(`vectorSearch...`);
+        DyFM_Log.log(`vectorSearch...`);
       }
       
       const result = await this.dataDBService.aggregate([ vectorSearchFilter ]);
 
       if (this.debugLog) {
-        console.log(`vectorSearch: ${result.length} results`);
+        DyFM_Log.log(`vectorSearch: ${result.length} results`);
       }
 
       return result;

package/src/_modules/discord-bot/_services/dibo-io.control-service.ts

@@ -69,7 +69,7 @@ export abstract class DyNTS_DiBo_IO_ControlService extends DyNTS_SingletonServic
   async handleNewMessage(message: Message, issuer: string): Promise<void> {
     try {
       if (DyNTS_DiBo_global_settings.debugLevel >= 1) {
-        DyFM_Log.H_log(`incoming message: (${message.author.displayName}) ${message.content}`)
+        DyFM_Log.H_log(`incoming message (${message.author.displayName}):\n${message.content}`)
       }
 
       const isForYou = await this.getMessageIsForBotToHandle(message, issuer);