@futdevpro/nts-dynamo 1.15.157 → 1.15.160

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.ts

@@ -1,572 +0,0 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Object } from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
-import { DyNTS_DAI_DocChunking_Util } from '../../../document-ai/_collections/dai-chunking.util';
-import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
-import { DyNTS_DAI_DocChunkCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
-import { DyNTS_DAI_DocPage, DyNTS_DAI_docPage_dataParams } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
-import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../../document-ai/_models/data-models/dai-document.data-model';
-
-import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
-import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
-import { DyNTS_DAI_CompareResult_Type } from '../../../document-ai/_enums/dai-compare-result-type.enum';
-
-
-export class DyNTS_OAI_DocPageDataService<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
-  T_ExternalDocumentId = string,
-  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
-> extends DyNTS_DataService<T_Page> {
-
-  debugLog: boolean = true;
-  absoluteDeletesOnProcess: boolean = true;
-
-  constructor(
-    issuer: string,
-    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
-    data?: T_Page,
-    dataParams: DyFM_DataModel_Params<T_Page> = DyNTS_DAI_docPage_dataParams as DyFM_DataModel_Params<T_Page>,
-  ) {
-    super(
-      data,
-      dataParams,
-      issuer,
-    );
-  }
-
-  getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  async processPages(documentId: string, pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        page.documentId = documentId;
-        compareResults.push(await this.processPage(page));
-      }
-
-      // All pages that are not in the list, and are in the database saved to the same document
-      const deletedPages: T_Page[] = await this.findDataList({
-        documentId: documentId,
-        externalPageId: {
-          $nin: pages.map(page => page.externalPageId),
-        },
-        /* _id: {
-          $nin: pages.map(page => page._id),
-        }, */
-      } as DyFM_DBFilter<T_Page>);
-
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      for (const deletedPage of deletedPages) {
-        const messages: string[] = [];
-        
-        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
-          documentId: documentId,
-          pageId: deletedPage._id,
-        } as DyFM_DBFilterSimple<T_Chunk>);
-
-        for (const chunk of chunksToDelete) {
-          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
-        }
-        messages.push(`${chunksToDelete.length} chunks deleted`);
-
-        await this.deleteData(deletedPage._id, this.absoluteDeletesOnProcess);
-        messages.unshift(`Page "${deletedPage.name}" deleted (chunks: ${chunksToDelete.length})`);
-
-        compareResults.push({
-          result: DyNTS_DAI_CompareResult_Type.deleted,
-          newPage: null,
-          oldPage: deletedPage,
-          chunkCompareResults: [],
-          reports: messages,
-        });
-      }
-
-      return compareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processPages', error),
-        errorCode: 'DCH-CDP-PPS0',
-      });
-    }
-  }
-
-  async processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      let existingPage: T_Page;
-      if (page._id) {
-        existingPage = await this.findData({
-          _id: page._id,
-        } as DyFM_DBFilterSimple<T_Page>);
-      } else {
-        if (!page.externalPageId) {
-          throw new DyFM_Error({
-            ...this.getDefaultErrorSettings('processPage', new Error('externalPageId is required')),
-            errorCode: 'DCH-CDP-PP1',
-          });
-        }
-
-        existingPage = await this.findData({
-          /* documentId: page.documentId, */
-          externalPageId: page.externalPageId,
-        } as DyFM_DBFilterSimple<T_Page>);
-      }
-
-      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
-      
-      if (existingPage) {
-        // Compare first to determine if we need to save
-        pageCompareResult = await this.comparePage(page, existingPage);
-        
-        // Always process chunks after page comparison, regardless of page equality
-        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-          this.getDocChunkDataService();
-
-        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processPage(
-          existingPage,
-        );
-
-        pageCompareResult.chunkCompareResults = chunksCompareResults;
-
-        // Recalculate result based on both page metadata and chunk changes
-        if (
-          pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal && 
-          chunksCompareResults.some(r => r.result !== DyNTS_DAI_CompareResult_Type.equal)
-        ) {
-          pageCompareResult.result = DyNTS_DAI_CompareResult_Type.modified;
-          pageCompareResult.reports.push('Chunks modified');
-        }
-
-        if (pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
-          // No changes detected, reuse existing page without saving
-          pageCompareResult.newPage = existingPage;
-          pageCompareResult.oldPage = existingPage;
-        } else {
-          // Changes detected, save the updated page
-          const pageDataToSave: T_Page = { ...existingPage };
-          DyFM_Object.cleanAssign(pageDataToSave, page);
-          delete pageDataToSave.chunks;
-
-          const savedPage: T_Page = await this.saveData(pageDataToSave, true);
-
-          page.chunks.forEach((chunk: T_Chunk) => {
-            chunk.documentId = savedPage.documentId;
-            chunk.pageId = savedPage._id;
-          });
-          page._id = savedPage._id;
-          pageCompareResult.newPage = page;
-        }
-      } else {
-        // New page, save it first to get ID
-        const pageDataToSave: T_Page = {
-          ...page,
-        };
-        delete pageDataToSave.chunks;
-
-        const savedPage: T_Page = await this.saveData(pageDataToSave, true);
-
-        page.chunks.forEach((chunk: T_Chunk) => {
-          chunk.documentId = savedPage.documentId;
-          chunk.pageId = savedPage._id;
-        });
-        page._id = savedPage._id;
-
-        pageCompareResult = {
-          result: DyNTS_DAI_CompareResult_Type.new,
-          newPage: page,
-          oldPage: null,  
-          chunkCompareResults: [],
-          reports: [
-            `New page: ${page.name}`,
-            `${page.chunks.length} new chunks in ${page.name}`,
-          ],
-        };
-
-        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-          this.getDocChunkDataService();
-
-        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
-          savedPage,
-        );
-
-        pageCompareResult.chunkCompareResults = chunksCompareResults;
-      }
-
-      return pageCompareResult;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processPage', error),
-        errorCode: 'DCH-CDP-PP0',
-      });
-    }
-  }
-
-  async processNewPages(pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const pageCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.processNewPage(page);
-        pageCompareResults.push(pageCompareResult);
-      }
-
-      return pageCompareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processNewPages', error),
-        errorCode: 'DCH-CDP-PNPS0',
-      });
-    }
-  }
-
-  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      // save the page to ensure its id
-      const savedPage: T_Page = await this.saveData(page, true);
-
-      // create chunks
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
-        savedPage,
-      );
-      /* const chunks: T_Chunk[] = await docChunk_DS.processNewChunks(page); */
-
-      /* page.chunks.forEach((chunk: T_Chunk) => {
-        chunk.documentId = savedPage.documentId;
-        chunk.pageId = savedPage._id;
-      });
-      page._id = savedPage._id; */
-
-      const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
-        result: DyNTS_DAI_CompareResult_Type.new,
-        newPage: page,
-        oldPage: null,
-        chunkCompareResults: chunksCompareResults,
-        reports: [
-          `New page: ${page.name}`,
-          `${page.chunks.length} new chunks in ${page.name}`,
-        ],
-      };
-
-      return pageCompareResult;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processNewPage', error),
-        errorCode: 'DCH-CDP-PNP0',
-      });
-    }
-  }
-
-  /** 
-   * this is a deep compare, that will update chunks if needed 
-   * */
-  async comparePage(newPage: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      if (!oldPage) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('comparePage', new Error('oldPage is required')),
-          errorCode: 'DCH-CDP-CP1',
-        });
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${newPage.name}`);
-      }
-
-      let pageCompareResult: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
-      const messages: string[] = [];
-      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
-
-      // Compare all relevant fields for comprehensive change detection
-      if (newPage.name !== oldPage.name) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page name modified`);
-      }
-
-      if (newPage.content !== oldPage.content) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page content modified`);
-      }
-
-      // Compare allFlaggedParents arrays
-      const allFlaggedParentsEqual = this.compareStringArrays(
-        newPage.allFlaggedParents || [], 
-        oldPage.allFlaggedParents || []
-      );
-      if (!allFlaggedParentsEqual) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page allFlaggedParents modified`);
-      }
-
-      // Compare allFlaggedParentsMerged
-      if (newPage.allFlaggedParentsMerged !== oldPage.allFlaggedParentsMerged) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page allFlaggedParentsMerged modified`);
-      }
-
-      // Compare filePath
-      if (newPage.filePath !== oldPage.filePath) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page filePath modified`);
-      }
-
-      // Compare parentedPath arrays
-      const parentedPathEqual = this.compareStringArrays(
-        newPage.parentedPath || [], 
-        oldPage.parentedPath || []
-      );
-      if (!parentedPathEqual) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page parentedPath modified`);
-      }
-
-      const result: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
-        result: pageCompareResult,
-        newPage: newPage,
-        oldPage: oldPage,
-        chunkCompareResults: chunkCompareResults,
-        reports: messages,
-      };
-      
-
-      return result;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePage', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  }
-
-  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
-    if (arr1.length !== arr2.length) {
-      return false;
-    }
-    return arr1.every((item, index) => item === arr2[index]);
-  }
-
-  /* async comparePage_deep(page: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
-      }
-
-      const pageCompareResult_flat: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = 
-        await this.comparePage(page, oldPage);
-      const messages: string[] = pageCompareResult_flat.reports;
-
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
-        page.chunks, oldPage.chunks);
-
-
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePage_deep', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  } */
-
-
-
-//  async comparePage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-//    try {
-//      if (this.debugLog) {
-//        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
-//      }
-//
-//      if (!page?.allFlaggedParentsMerged) {
-//        DyFM_Log.T_error(
-//          `❌ ${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`, JSON.stringify(page, null, 2)
-//        );
-//
-//        throw new DyFM_Error({
-//          ...this.getDefaultErrorSettings(
-//            'comparePage', 
-//            new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`),
-//          ),
-//          errorCode: 'DCH-CSB-CP0',
-//        });
-//      }
-//
-//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-//        this.getDocChunkDataService();
-//
-//      const oldChunks: T_Chunk[] = await docChunk_DS.findDataList({
-//        path: page.allFlaggedParentsMerged,
-//      } as DyFM_DBFilterSimple<T_Chunk>);
-//
-//      const compareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
-//        page.chunks,
-//        oldChunks
-//      );
-//
-//      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
-//        compareResults: compareResults,
-//        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
-//        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
-//        modified: compareResults.some(
-//          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
-//        ),
-//      }); */
-//
-//      if (
-//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)
-//      ) {
-//        if (this.debugLog) {
-//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is equal: ${page.name}`);
-//        }
-//
-//        return {
-//          result: DyNTS_DAI_CompareResult_Type.equal,
-//          newPage: page,
-//          oldPageContent: page.content,
-//          chunkCompareResults: compareResults,
-//          newChunks: page.chunks,
-//          oldChunks: oldChunks,
-//        };
-//      }
-//
-//      if (
-//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.new)
-//      ) {
-//        if (this.debugLog) {
-//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is new: ${page.name}`);
-//        }
-//
-//        return {
-//          result: DyNTS_DAI_CompareResult_Type.new,
-//          newPage: page,
-//          oldPageContent: '',
-//          chunkCompareResults: compareResults,
-//          newChunks: page.chunks,
-//          oldChunks: oldChunks,
-//        };
-//      }
-//
-//      if (this.debugLog) {
-//        DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is modified: ${page.name}`);
-//      }
-//
-//      return {
-//        result: DyNTS_DAI_CompareResult_Type.modified,
-//        newPage: page,
-//        oldPageContent: DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(
-//          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
-//          this.issuer
-//        ),
-//        chunkCompareResults: compareResults,
-//        newChunks: page.chunks,
-//        oldChunks: oldChunks,
-//      };
-//    } catch (error) {
-//      throw new DyFM_Error({
-//        ...this.getDefaultErrorSettings('comparePage', error),
-//        errorCode: 'DCH-CDP-CP0',
-//      });
-//    }
-//  }
-
-  
-//  async updatePage(
-//    pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>, 
-//    issuer: string
-//  ): Promise<void> {
-//    try {
-//      // save chunks 
-//      // (only new and modified pages 
-//      //   (but all the page element should be updated 
-//      //     (delete extra elements)))
-//
-//      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
-//        // delete extra chunks
-//        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
-//
-//          if (i < pageCompareResult.newChunks.length) {
-//            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
-//          } else {
-//            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
-//              DyFM_Error.logSimple(`❌ ${this.dataParams.dataName} Chunk not deleted`, error);
-//            });
-//          }
-//        }
-//      }
-//
-//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-//        this.getDocChunkDataService(issuer);
-//
-//      await DyFM_Array.asyncForEachAllAtOnce(
-//        pageCompareResult.newChunks,
-//        async (chunk: T_Chunk) => {
-//          if (
-//            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_DAI_CompareResult_Type.equal
-//          ) {
-//            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
-//            chunk.na */
-//
-//            chunk.path = chunk.allFlaggedParents.join('/');
-//
-//            if (this.debugLog) {
-//              DyFM_Log.H_info(`✅ saving ${this.dataParams.dataName} Chunk: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`, {
-//                chunk: chunk,
-//              });
-//
-//              /* DyFM_Log.H_warn(`STOP`);
-//              await DyFM_Async.wait(10_000 * minute); */
-//            }
-//
-//            await docChunk_DS.saveData(chunk, false, true);
-//            DyFM_Log.info(`✅ ${this.dataParams.dataName} Chunk saved: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`)
-//          }
-//        }
-//      );
-//
-//      DyFM_Log.success(`✅ ${this.dataParams.dataName} Chunks updated(SAVED) for page: ${pageCompareResult.newPage.name}`);
-//    } catch (error) {
-//      throw new DyFM_Error({
-//        ...this.getDefaultErrorSettings('processPage', error),
-//        errorCode: 'DCH-CDP-PP0',
-//      });
-//    }
-//  }
-
-  /* async comparePages(pages: T_Page[], oldPages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        const compareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.comparePage(page);
-        compareResults.push(compareResult);
-      }
-
-      return compareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePages', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  } */
-}