@futdevpro/nts-dynamo 1.12.15 → 1.14.1

package/src/_modules/ai/_modules/open-ai/_services/oai-doc-chunk-data.service.ts

@@ -0,0 +1,284 @@
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata, DyFM_Async } from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
+import { DyNTS_DAI_DocChunking_Util } from '../../document-ai/_collections/dai-chunking.util';
+import { DyNTS_DAI_DocChunk, DyNTS_DAI_docChunk_dataParams } from '../../document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from '../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../document-ai/_enums/dai-compare-result-type.enum';
+import { DyNTS_DAI_DocPage } from '../../document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_DocPageCompareResult } from '../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
+import { DyNTS_DAI_Document } from '../../document-ai/_models/data-models/dai-document.data-model';
+import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
+import { DyNTS_OAI_DocPageDataService } from './oai-doc-page.data-service';
+import { DyNTS_DAI_Document_Util } from '../../document-ai/_collections/dai-document.util';
+
+
+export class DyNTS_OAI_DocChunkDataService<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, 
+  T_ExternalDocumentId = string,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
+> extends DyNTS_OAI_VectorDataService<T_Chunk> {
+
+  override debugLog: boolean = true;
+
+  constructor(
+    issuer: string,
+    openAiSettings: DyFM_OAI_Settings,
+    dataParams: DyFM_DataModel_Params<T_Chunk> = DyNTS_DAI_docChunk_dataParams as DyFM_DataModel_Params<T_Chunk>,
+  ) {
+    super(
+      null, 
+      dataParams, 
+      openAiSettings,
+      issuer
+    );
+  }
+
+  getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.openAISettings,
+    );
+  }
+
+  getPageDataService(): DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.openAISettings,
+    );
+  }
+
+//  async compareChunks(
+//    newChunks: T_Chunk[], 
+//    oldChunks: T_Chunk[]
+//  ): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
+//    try {
+//      if (this.debugLog) {
+//        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunks: ${newChunks.length}`);
+//      }
+//
+//      if (newChunks.length === 0) {
+//        return [];
+//      }
+//
+//      /* if (newChunks.some(chunk => !chunk.path)) {
+//        throw new DyFM_Error({
+//          ...this.getDefaultErrorSettings(
+//            'compareChunks', new Error(`${this.dataParams.dataName} path is not set for some chunks`),
+//          ),
+//          errorCode: 'DCH-CSB-CCS1',
+//        });
+//      } */
+//
+//      /* oldChunks ??= await this.findDataList({
+//        path: newChunks[0].path,
+//      } as DyFM_DBFilterSimple<T_Chunk>);
+//      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex); */
+//
+//      return await DyFM_Array.asyncForEachAllAtOnce(
+//        newChunks,
+//        async (chunk) => {
+//          return await this.compareChunk(chunk, oldChunks[chunk.chunkIndex]);
+//        }
+//      )
+//    } catch (error) {
+//      throw new DyFM_Error({
+//        ...this.getDefaultErrorSettings('compareChunks', error),
+//        errorCode: 'DCH-CSB-CCS0',
+//      });
+//    }
+//  }
+
+  async processPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
+    try {
+      const newChunks: T_Chunk[] = await this.chunkPage(page);
+
+      const oldChunks: T_Chunk[] = await this.findDataList({
+        pageId: page._id,
+      } as DyFM_DBFilterSimple<T_Chunk>);
+      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
+
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+      for (let i = 0; i < newChunks.length; i++) {
+        const newChunk = newChunks[i];
+        newChunk.documentId = page.documentId;
+        newChunk.pageId = page._id;
+        newChunk.chunkIndex = i;
+
+        const oldChunk = oldChunks[i];
+        if (oldChunk) {
+          newChunk._id = oldChunk._id;
+          const savedChunk: T_Chunk = await this.saveData(newChunk, true);
+          const chunkCompareResult: DyNTS_DAI_DocChunkCompareResult<T_Chunk> = await this.compareChunk(savedChunk, oldChunk);
+          chunkCompareResults.push(chunkCompareResult);
+        } else {
+          const savedChunk: T_Chunk = await this.saveData(newChunk, true);
+          chunkCompareResults.push({
+            result: DyNTS_DAI_CompareResult_Type.new,
+            newChunk: savedChunk,
+            oldChunk: null,
+          });
+        }
+      }
+
+      for (let i = newChunks.length; i < oldChunks.length; i++) {
+        const oldChunk = oldChunks[i];
+        await this.deleteData(oldChunk._id);
+        chunkCompareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.deleted,
+          newChunk: null,
+          oldChunk: oldChunk,
+        });
+      }
+
+      return chunkCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPage', error),
+        errorCode: 'DCH-CSB-CCS0',
+      });
+    }
+  }
+
+  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
+    try {
+      const chunks: T_Chunk[] = await this.chunkPage(page);
+
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+      
+      for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        chunk.documentId = page.documentId;
+        chunk.pageId = page._id;
+        chunk.chunkIndex = i;
+
+        const savedChunk: T_Chunk = await this.saveData(chunk, true);
+
+        chunkCompareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newChunk: savedChunk,
+          oldChunk: null,
+        });
+      }
+
+      return chunkCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewChunks', error),
+        errorCode: 'DCH-CSB-CCS0',
+      });
+    }
+  }
+
+  async compareChunk(
+    newChunk: T_Chunk, 
+    oldChunk: T_Chunk
+  ): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>> {
+    try {
+      if (!oldChunk) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
+          errorCode: 'DCH-CSB-CC1',
+        });
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
+      }
+
+      /* oldChunk ??= await this.findData({
+        path: newChunk.path,
+        chunkIndex: newChunk.chunkIndex,
+      } as DyFM_DBFilterSimple<T_Chunk>); */
+
+      /* DyFM_Log.H_info(`✅ Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`, {
+        newChunk: newChunk,
+        oldChunk: oldChunk,
+        new: !oldChunk,
+        equal: oldChunk?.chunkParentedContent === newChunk.chunkParentedContent,
+        modified: oldChunk && oldChunk.chunkParentedContent !== newChunk.chunkParentedContent,
+      }); */
+
+      /* if (!oldChunk) {
+        // itt kéne vektor search-el utána keresni, 
+        // hogy a page link-je megtalálható-e (avagy csak átnevezésre került-e)
+        // hogy az adott tartalom, nem-e csak átnevezésre, szétbontásra vagy összevonásra került-e
+        return {
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newChunk: newChunk,
+          oldChunk: null,
+        };
+      } */
+
+      if (oldChunk.chunkContent === newChunk.chunkContent) {
+        return {
+          result: DyNTS_DAI_CompareResult_Type.equal,
+          newChunk: newChunk,
+          oldChunk: oldChunk,
+        };
+      }
+
+      return {
+        result: DyNTS_DAI_CompareResult_Type.modified,
+        newChunk: newChunk,
+        oldChunk: oldChunk,
+      };
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareChunk', error),
+        errorCode: 'DCH-CSB-CC0',
+      });
+    }
+  }
+  
+  async chunkPage(page: T_Page): Promise<T_Chunk[]> {
+    try {
+      page.chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(
+        page, 
+        (page: T_Page) => this.getPageLink(page), 
+        this.issuer
+      );
+
+      if (this.debugLog) {
+        DyFM_Log.H_info(
+          `Processing file: ${page.name}`, 
+          { 
+            'Page name': page.name, 
+            'Page filePath': page.filePath, 
+            'Page parentedPath': page.parentedPath, 
+            'Page content': page.content, 
+            'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
+            chunks: page.chunks.map(chunk => { 
+              return {
+                'Chunk name': chunk.documentName,
+                'Chunk path': chunk.path,
+                'Chunk index': chunk.chunkIndex,
+                'Chunk content': chunk.chunkContent,
+                'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
+              }
+            })
+          }
+        );
+      }
+
+      return page.chunks;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('chunkDocPages', error),
+        errorCode: 'CCAP-CUD-CDP0',
+      });
+    }
+  }
+
+  clickupWorkspaceId: string;
+  getPageLink(page: T_Page): string {
+    return DyNTS_DAI_Document_Util.getClickupPageLink(
+      page.name, 
+      this.clickupWorkspaceId, 
+      page.documentId, 
+      page.externalPageId
+    );
+  }
+
+}