@futdevpro/nts-dynamo 1.12.14 → 1.14.1

package/src/_modules/ai/_modules/open-ai/_services/oai-doc-page.data-service.ts

@@ -0,0 +1,508 @@
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { DyNTS_DataService } from '../../../../../_services/base/data.service';
+import { DyNTS_DAI_DocPageCompareResult } from '../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
+import { DyNTS_DAI_DocChunking_Util } from '../../document-ai/_collections/dai-chunking.util';
+import { DyNTS_DAI_DocChunk } from '../../document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from '../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_DocPage, DyNTS_DAI_docPage_dataParams } from '../../document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../document-ai/_models/data-models/dai-document.data-model';
+
+import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
+import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
+import { DyNTS_DAI_CompareResult_Type } from '../../document-ai/_enums/dai-compare-result-type.enum';
+
+
+export class DyNTS_OAI_DocPageDataService<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
+  T_ExternalDocumentId = string,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
+> extends DyNTS_DataService<T_Page> {
+
+  debugLog: boolean = true;
+  absoluteDeletesOnProcess: boolean = true;
+
+  constructor(
+    issuer: string,
+    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
+    data?: T_Page,
+    dataParams: DyFM_DataModel_Params<T_Page> = DyNTS_DAI_docPage_dataParams as DyFM_DataModel_Params<T_Page>,
+  ) {
+    super(
+      data,
+      dataParams,
+      issuer,
+    );
+  }
+
+  getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  async processPages(documentId: string, pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        compareResults.push(await this.processPage(page));
+      }
+
+      // All pages that are not in the list, and are in the database saved to the same document
+      const deletedPages: T_Page[] = await this.findDataList({
+        documentId: documentId,
+        _id: {
+          $nin: pages.map(page => page._id),
+        },
+      } as DyFM_DBFilter<T_Page>);
+
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      for (const deletedPage of deletedPages) {
+        const messages: string[] = [];
+        
+        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
+          documentId: documentId,
+          pageId: deletedPage._id,
+        } as DyFM_DBFilterSimple<T_Chunk>);
+
+        for (const chunk of chunksToDelete) {
+          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
+        }
+        messages.push(`${chunksToDelete.length} chunks deleted`);
+
+        await this.deleteData(deletedPage._id, this.absoluteDeletesOnProcess);
+        messages.unshift(`Page "${deletedPage.name}" deleted (chunks: ${chunksToDelete.length})`);
+
+        compareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.deleted,
+          newPage: null,
+          oldPage: deletedPage,
+          chunkCompareResults: [],
+          reports: messages,
+        });
+      }
+
+      return compareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPages', error),
+        errorCode: 'DCH-CSB-PP0',
+      });
+    }
+  }
+
+  async processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      // ugyan a logikus az lenne, ha először compare-elnénk, és csak az után mentenénk, 
+      //   de akkor nem lenne id, amit a sub-element-ekhez kellenek, ezért...
+      // - először kikérjük a régi adatot, 
+      // - aztán mentünk, hogy legyen id, 
+      // - és utána compare-eljük
+
+      let existingPage: T_Page;
+      if (page._id) {
+        existingPage = await this.findData({
+          _id: page._id,
+        } as DyFM_DBFilterSimple<T_Page>);
+      } else {
+        if (!page.externalPageId) {
+          throw new DyFM_Error({
+            ...this.getDefaultErrorSettings('processPage', new Error('externalPageId is required')),
+            errorCode: 'DCH-CSB-PP0',
+          });
+        }
+
+        existingPage = await this.findData({
+          documentId: page.documentId,
+          externalPageId: page.externalPageId,
+        } as DyFM_DBFilterSimple<T_Page>);
+      }
+
+      // save the page to ensure its id
+      let pageDataToSave: T_Page;
+      if (existingPage) {
+        pageDataToSave = existingPage;
+        Object.assign(pageDataToSave, page);
+      } else {
+        pageDataToSave = {
+          ...page,
+        };
+      }
+      delete pageDataToSave.chunks;
+
+      const savedPage: T_Page = await this.saveData(pageDataToSave, true);
+
+      page.chunks.forEach((chunk: T_Chunk) => {
+        chunk.documentId = savedPage.documentId;
+        chunk.pageId = savedPage._id;
+      });
+      page._id = savedPage._id;
+
+      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
+      if (existingPage) {
+        pageCompareResult = await this.comparePage(page, existingPage);
+      } else {
+        pageCompareResult = {
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newPage: page,
+          oldPage: null,
+          chunkCompareResults: [],
+          reports: [
+            `New page: ${page.name}`,
+            `${page.chunks.length} new chunks in ${page.name}`,
+          ],
+        };
+
+        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+          this.getDocChunkDataService();
+
+        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
+          savedPage,
+        );
+
+        pageCompareResult.chunkCompareResults = chunksCompareResults;
+      }
+
+      return pageCompareResult;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPage', error),
+        errorCode: 'DCH-CSB-PP0',
+      });
+    }
+  }
+
+  async processNewPages(pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const pageCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.processNewPage(page);
+        pageCompareResults.push(pageCompareResult);
+      }
+
+      return pageCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewPages', error),
+        errorCode: 'DCH-CSB-PP0',
+      });
+    }
+  }
+
+  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      // save the page to ensure its id
+      const savedPage: T_Page = await this.saveData(page, true);
+
+      // create chunks
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
+        savedPage,
+      );
+      /* const chunks: T_Chunk[] = await docChunk_DS.processNewChunks(page); */
+
+      /* page.chunks.forEach((chunk: T_Chunk) => {
+        chunk.documentId = savedPage.documentId;
+        chunk.pageId = savedPage._id;
+      });
+      page._id = savedPage._id; */
+
+      const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
+        result: DyNTS_DAI_CompareResult_Type.new,
+        newPage: page,
+        oldPage: null,
+        chunkCompareResults: chunksCompareResults,
+        reports: [
+          `New page: ${page.name}`,
+          `${page.chunks.length} new chunks in ${page.name}`,
+        ],
+      };
+
+      return pageCompareResult;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewPage', error),
+        errorCode: 'DCH-CSB-PP0',
+      });
+    }
+  }
+
+  /** 
+   * this is a deep compare, that will update chunks if needed 
+   * */
+  async comparePage(newPage: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      if (!oldPage) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('comparePage', new Error('oldPage is required')),
+          errorCode: 'DCH-CSB-CP1',
+        });
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${newPage.name}`);
+      }
+
+      let pageCompareResult: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
+      const messages: string[] = [];
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+
+      if (newPage.content !== oldPage.content) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page content modified`);
+
+        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+          this.getDocChunkDataService();
+
+        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processPage(
+          newPage,
+        );
+
+        chunkCompareResults.push(...chunksCompareResults);
+      }
+
+      if (newPage.name !== oldPage.name) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page name modified`);
+      }
+
+      const result: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
+        result: pageCompareResult,
+        newPage: newPage,
+        oldPage: oldPage,
+        chunkCompareResults: chunkCompareResults,
+        reports: messages,
+      };
+      
+
+      return result;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePage', error),
+        errorCode: 'DCH-CSB-CP0',
+      });
+    }
+  }
+
+  /* async comparePage_deep(page: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
+      }
+
+      const pageCompareResult_flat: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = 
+        await this.comparePage(page, oldPage);
+      const messages: string[] = pageCompareResult_flat.reports;
+
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
+        page.chunks, oldPage.chunks);
+
+
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePage_deep', error),
+        errorCode: 'DCH-CSB-CP0',
+      });
+    }
+  } */
+
+
+
+//  async comparePage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+//    try {
+//      if (this.debugLog) {
+//        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
+//      }
+//
+//      if (!page?.allFlaggedParentsMerged) {
+//        DyFM_Log.T_error(
+//          `❌ ${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`, JSON.stringify(page, null, 2)
+//        );
+//
+//        throw new DyFM_Error({
+//          ...this.getDefaultErrorSettings(
+//            'comparePage', 
+//            new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`),
+//          ),
+//          errorCode: 'DCH-CSB-CP0',
+//        });
+//      }
+//
+//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+//        this.getDocChunkDataService();
+//
+//      const oldChunks: T_Chunk[] = await docChunk_DS.findDataList({
+//        path: page.allFlaggedParentsMerged,
+//      } as DyFM_DBFilterSimple<T_Chunk>);
+//
+//      const compareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
+//        page.chunks,
+//        oldChunks
+//      );
+//
+//      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
+//        compareResults: compareResults,
+//        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
+//        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
+//        modified: compareResults.some(
+//          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
+//        ),
+//      }); */
+//
+//      if (
+//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)
+//      ) {
+//        if (this.debugLog) {
+//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is equal: ${page.name}`);
+//        }
+//
+//        return {
+//          result: DyNTS_DAI_CompareResult_Type.equal,
+//          newPage: page,
+//          oldPageContent: page.content,
+//          chunkCompareResults: compareResults,
+//          newChunks: page.chunks,
+//          oldChunks: oldChunks,
+//        };
+//      }
+//
+//      if (
+//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.new)
+//      ) {
+//        if (this.debugLog) {
+//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is new: ${page.name}`);
+//        }
+//
+//        return {
+//          result: DyNTS_DAI_CompareResult_Type.new,
+//          newPage: page,
+//          oldPageContent: '',
+//          chunkCompareResults: compareResults,
+//          newChunks: page.chunks,
+//          oldChunks: oldChunks,
+//        };
+//      }
+//
+//      if (this.debugLog) {
+//        DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is modified: ${page.name}`);
+//      }
+//
+//      return {
+//        result: DyNTS_DAI_CompareResult_Type.modified,
+//        newPage: page,
+//        oldPageContent: DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(
+//          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
+//          this.issuer
+//        ),
+//        chunkCompareResults: compareResults,
+//        newChunks: page.chunks,
+//        oldChunks: oldChunks,
+//      };
+//    } catch (error) {
+//      throw new DyFM_Error({
+//        ...this.getDefaultErrorSettings('comparePage', error),
+//        errorCode: 'DCH-CSB-CP0',
+//      });
+//    }
+//  }
+
+  
+//  async updatePage(
+//    pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>, 
+//    issuer: string
+//  ): Promise<void> {
+//    try {
+//      // save chunks 
+//      // (only new and modified pages 
+//      //   (but all the page element should be updated 
+//      //     (delete extra elements)))
+//
+//      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
+//        // delete extra chunks
+//        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
+//
+//          if (i < pageCompareResult.newChunks.length) {
+//            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
+//          } else {
+//            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
+//              DyFM_Error.logSimple(`❌ ${this.dataParams.dataName} Chunk not deleted`, error);
+//            });
+//          }
+//        }
+//      }
+//
+//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+//        this.getDocChunkDataService(issuer);
+//
+//      await DyFM_Array.asyncForEachAllAtOnce(
+//        pageCompareResult.newChunks,
+//        async (chunk: T_Chunk) => {
+//          if (
+//            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_DAI_CompareResult_Type.equal
+//          ) {
+//            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
+//            chunk.na */
+//
+//            chunk.path = chunk.allFlaggedParents.join('/');
+//
+//            if (this.debugLog) {
+//              DyFM_Log.H_info(`✅ saving ${this.dataParams.dataName} Chunk: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`, {
+//                chunk: chunk,
+//              });
+//
+//              /* DyFM_Log.H_warn(`STOP`);
+//              await DyFM_Async.wait(10_000 * minute); */
+//            }
+//
+//            await docChunk_DS.saveData(chunk, false, true);
+//            DyFM_Log.info(`✅ ${this.dataParams.dataName} Chunk saved: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`)
+//          }
+//        }
+//      );
+//
+//      DyFM_Log.success(`✅ ${this.dataParams.dataName} Chunks updated(SAVED) for page: ${pageCompareResult.newPage.name}`);
+//    } catch (error) {
+//      throw new DyFM_Error({
+//        ...this.getDefaultErrorSettings('processPage', error),
+//        errorCode: 'DCH-CSB-PP0',
+//      });
+//    }
+//  }
+
+  /* async comparePages(pages: T_Page[], oldPages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        const compareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.comparePage(page);
+        compareResults.push(compareResult);
+      }
+
+      return compareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePages', error),
+        errorCode: 'DCH-CSB-CP0',
+      });
+    }
+  } */
+}