@futdevpro/nts-dynamo 1.15.150 → 1.15.153

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service.ts

@@ -1,572 +1,572 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Object } from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
-import { DyNTS_DAI_DocChunking_Util } from '../../../document-ai/_collections/dai-chunking.util';
-import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
-import { DyNTS_DAI_DocChunkCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
-import { DyNTS_DAI_DocPage, DyNTS_DAI_docPage_dataParams } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
-import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../../document-ai/_models/data-models/dai-document.data-model';
-
-import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
-import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
-import { DyNTS_DAI_CompareResult_Type } from '../../../document-ai/_enums/dai-compare-result-type.enum';
-
-
-export class DyNTS_OAI_DocPageDataService<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
-  T_ExternalDocumentId = string,
-  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
-> extends DyNTS_DataService<T_Page> {
-
-  debugLog: boolean = true;
-  absoluteDeletesOnProcess: boolean = true;
-
-  constructor(
-    issuer: string,
-    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
-    data?: T_Page,
-    dataParams: DyFM_DataModel_Params<T_Page> = DyNTS_DAI_docPage_dataParams as DyFM_DataModel_Params<T_Page>,
-  ) {
-    super(
-      data,
-      dataParams,
-      issuer,
-    );
-  }
-
-  getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  async processPages(documentId: string, pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        page.documentId = documentId;
-        compareResults.push(await this.processPage(page));
-      }
-
-      // All pages that are not in the list, and are in the database saved to the same document
-      const deletedPages: T_Page[] = await this.findDataList({
-        documentId: documentId,
-        externalPageId: {
-          $nin: pages.map(page => page.externalPageId),
-        },
-        /* _id: {
-          $nin: pages.map(page => page._id),
-        }, */
-      } as DyFM_DBFilter<T_Page>);
-
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      for (const deletedPage of deletedPages) {
-        const messages: string[] = [];
-        
-        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
-          documentId: documentId,
-          pageId: deletedPage._id,
-        } as DyFM_DBFilterSimple<T_Chunk>);
-
-        for (const chunk of chunksToDelete) {
-          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
-        }
-        messages.push(`${chunksToDelete.length} chunks deleted`);
-
-        await this.deleteData(deletedPage._id, this.absoluteDeletesOnProcess);
-        messages.unshift(`Page "${deletedPage.name}" deleted (chunks: ${chunksToDelete.length})`);
-
-        compareResults.push({
-          result: DyNTS_DAI_CompareResult_Type.deleted,
-          newPage: null,
-          oldPage: deletedPage,
-          chunkCompareResults: [],
-          reports: messages,
-        });
-      }
-
-      return compareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processPages', error),
-        errorCode: 'DCH-CDP-PPS0',
-      });
-    }
-  }
-
-  async processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      let existingPage: T_Page;
-      if (page._id) {
-        existingPage = await this.findData({
-          _id: page._id,
-        } as DyFM_DBFilterSimple<T_Page>);
-      } else {
-        if (!page.externalPageId) {
-          throw new DyFM_Error({
-            ...this.getDefaultErrorSettings('processPage', new Error('externalPageId is required')),
-            errorCode: 'DCH-CDP-PP1',
-          });
-        }
-
-        existingPage = await this.findData({
-          /* documentId: page.documentId, */
-          externalPageId: page.externalPageId,
-        } as DyFM_DBFilterSimple<T_Page>);
-      }
-
-      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
-      
-      if (existingPage) {
-        // Compare first to determine if we need to save
-        pageCompareResult = await this.comparePage(page, existingPage);
-        
-        // Always process chunks after page comparison, regardless of page equality
-        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-          this.getDocChunkDataService();
-
-        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processPage(
-          existingPage,
-        );
-
-        pageCompareResult.chunkCompareResults = chunksCompareResults;
-
-        // Recalculate result based on both page metadata and chunk changes
-        if (
-          pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal && 
-          chunksCompareResults.some(r => r.result !== DyNTS_DAI_CompareResult_Type.equal)
-        ) {
-          pageCompareResult.result = DyNTS_DAI_CompareResult_Type.modified;
-          pageCompareResult.reports.push('Chunks modified');
-        }
-
-        if (pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
-          // No changes detected, reuse existing page without saving
-          pageCompareResult.newPage = existingPage;
-          pageCompareResult.oldPage = existingPage;
-        } else {
-          // Changes detected, save the updated page
-          const pageDataToSave: T_Page = { ...existingPage };
-          DyFM_Object.cleanAssign(pageDataToSave, page);
-          delete pageDataToSave.chunks;
-
-          const savedPage: T_Page = await this.saveData(pageDataToSave, true);
-
-          page.chunks.forEach((chunk: T_Chunk) => {
-            chunk.documentId = savedPage.documentId;
-            chunk.pageId = savedPage._id;
-          });
-          page._id = savedPage._id;
-          pageCompareResult.newPage = page;
-        }
-      } else {
-        // New page, save it first to get ID
-        const pageDataToSave: T_Page = {
-          ...page,
-        };
-        delete pageDataToSave.chunks;
-
-        const savedPage: T_Page = await this.saveData(pageDataToSave, true);
-
-        page.chunks.forEach((chunk: T_Chunk) => {
-          chunk.documentId = savedPage.documentId;
-          chunk.pageId = savedPage._id;
-        });
-        page._id = savedPage._id;
-
-        pageCompareResult = {
-          result: DyNTS_DAI_CompareResult_Type.new,
-          newPage: page,
-          oldPage: null,  
-          chunkCompareResults: [],
-          reports: [
-            `New page: ${page.name}`,
-            `${page.chunks.length} new chunks in ${page.name}`,
-          ],
-        };
-
-        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-          this.getDocChunkDataService();
-
-        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
-          savedPage,
-        );
-
-        pageCompareResult.chunkCompareResults = chunksCompareResults;
-      }
-
-      return pageCompareResult;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processPage', error),
-        errorCode: 'DCH-CDP-PP0',
-      });
-    }
-  }
-
-  async processNewPages(pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const pageCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.processNewPage(page);
-        pageCompareResults.push(pageCompareResult);
-      }
-
-      return pageCompareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processNewPages', error),
-        errorCode: 'DCH-CDP-PNPS0',
-      });
-    }
-  }
-
-  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      // save the page to ensure its id
-      const savedPage: T_Page = await this.saveData(page, true);
-
-      // create chunks
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
-        savedPage,
-      );
-      /* const chunks: T_Chunk[] = await docChunk_DS.processNewChunks(page); */
-
-      /* page.chunks.forEach((chunk: T_Chunk) => {
-        chunk.documentId = savedPage.documentId;
-        chunk.pageId = savedPage._id;
-      });
-      page._id = savedPage._id; */
-
-      const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
-        result: DyNTS_DAI_CompareResult_Type.new,
-        newPage: page,
-        oldPage: null,
-        chunkCompareResults: chunksCompareResults,
-        reports: [
-          `New page: ${page.name}`,
-          `${page.chunks.length} new chunks in ${page.name}`,
-        ],
-      };
-
-      return pageCompareResult;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processNewPage', error),
-        errorCode: 'DCH-CDP-PNP0',
-      });
-    }
-  }
-
-  /** 
-   * this is a deep compare, that will update chunks if needed 
-   * */
-  async comparePage(newPage: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      if (!oldPage) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('comparePage', new Error('oldPage is required')),
-          errorCode: 'DCH-CDP-CP1',
-        });
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${newPage.name}`);
-      }
-
-      let pageCompareResult: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
-      const messages: string[] = [];
-      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
-
-      // Compare all relevant fields for comprehensive change detection
-      if (newPage.name !== oldPage.name) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page name modified`);
-      }
-
-      if (newPage.content !== oldPage.content) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page content modified`);
-      }
-
-      // Compare allFlaggedParents arrays
-      const allFlaggedParentsEqual = this.compareStringArrays(
-        newPage.allFlaggedParents || [], 
-        oldPage.allFlaggedParents || []
-      );
-      if (!allFlaggedParentsEqual) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page allFlaggedParents modified`);
-      }
-
-      // Compare allFlaggedParentsMerged
-      if (newPage.allFlaggedParentsMerged !== oldPage.allFlaggedParentsMerged) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page allFlaggedParentsMerged modified`);
-      }
-
-      // Compare filePath
-      if (newPage.filePath !== oldPage.filePath) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page filePath modified`);
-      }
-
-      // Compare parentedPath arrays
-      const parentedPathEqual = this.compareStringArrays(
-        newPage.parentedPath || [], 
-        oldPage.parentedPath || []
-      );
-      if (!parentedPathEqual) {
-        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Page parentedPath modified`);
-      }
-
-      const result: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
-        result: pageCompareResult,
-        newPage: newPage,
-        oldPage: oldPage,
-        chunkCompareResults: chunkCompareResults,
-        reports: messages,
-      };
-      
-
-      return result;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePage', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  }
-
-  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
-    if (arr1.length !== arr2.length) {
-      return false;
-    }
-    return arr1.every((item, index) => item === arr2[index]);
-  }
-
-  /* async comparePage_deep(page: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
-      }
-
-      const pageCompareResult_flat: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = 
-        await this.comparePage(page, oldPage);
-      const messages: string[] = pageCompareResult_flat.reports;
-
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
-        page.chunks, oldPage.chunks);
-
-
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePage_deep', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  } */
-
-
-
-//  async comparePage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
-//    try {
-//      if (this.debugLog) {
-//        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
-//      }
-//
-//      if (!page?.allFlaggedParentsMerged) {
-//        DyFM_Log.T_error(
-//          `❌ ${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`, JSON.stringify(page, null, 2)
-//        );
-//
-//        throw new DyFM_Error({
-//          ...this.getDefaultErrorSettings(
-//            'comparePage', 
-//            new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`),
-//          ),
-//          errorCode: 'DCH-CSB-CP0',
-//        });
-//      }
-//
-//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-//        this.getDocChunkDataService();
-//
-//      const oldChunks: T_Chunk[] = await docChunk_DS.findDataList({
-//        path: page.allFlaggedParentsMerged,
-//      } as DyFM_DBFilterSimple<T_Chunk>);
-//
-//      const compareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
-//        page.chunks,
-//        oldChunks
-//      );
-//
-//      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
-//        compareResults: compareResults,
-//        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
-//        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
-//        modified: compareResults.some(
-//          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
-//        ),
-//      }); */
-//
-//      if (
-//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)
-//      ) {
-//        if (this.debugLog) {
-//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is equal: ${page.name}`);
-//        }
-//
-//        return {
-//          result: DyNTS_DAI_CompareResult_Type.equal,
-//          newPage: page,
-//          oldPageContent: page.content,
-//          chunkCompareResults: compareResults,
-//          newChunks: page.chunks,
-//          oldChunks: oldChunks,
-//        };
-//      }
-//
-//      if (
-//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.new)
-//      ) {
-//        if (this.debugLog) {
-//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is new: ${page.name}`);
-//        }
-//
-//        return {
-//          result: DyNTS_DAI_CompareResult_Type.new,
-//          newPage: page,
-//          oldPageContent: '',
-//          chunkCompareResults: compareResults,
-//          newChunks: page.chunks,
-//          oldChunks: oldChunks,
-//        };
-//      }
-//
-//      if (this.debugLog) {
-//        DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is modified: ${page.name}`);
-//      }
-//
-//      return {
-//        result: DyNTS_DAI_CompareResult_Type.modified,
-//        newPage: page,
-//        oldPageContent: DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(
-//          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
-//          this.issuer
-//        ),
-//        chunkCompareResults: compareResults,
-//        newChunks: page.chunks,
-//        oldChunks: oldChunks,
-//      };
-//    } catch (error) {
-//      throw new DyFM_Error({
-//        ...this.getDefaultErrorSettings('comparePage', error),
-//        errorCode: 'DCH-CDP-CP0',
-//      });
-//    }
-//  }
-
-  
-//  async updatePage(
-//    pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>, 
-//    issuer: string
-//  ): Promise<void> {
-//    try {
-//      // save chunks 
-//      // (only new and modified pages 
-//      //   (but all the page element should be updated 
-//      //     (delete extra elements)))
-//
-//      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
-//        // delete extra chunks
-//        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
-//
-//          if (i < pageCompareResult.newChunks.length) {
-//            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
-//          } else {
-//            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
-//              DyFM_Error.logSimple(`❌ ${this.dataParams.dataName} Chunk not deleted`, error);
-//            });
-//          }
-//        }
-//      }
-//
-//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-//        this.getDocChunkDataService(issuer);
-//
-//      await DyFM_Array.asyncForEachAllAtOnce(
-//        pageCompareResult.newChunks,
-//        async (chunk: T_Chunk) => {
-//          if (
-//            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_DAI_CompareResult_Type.equal
-//          ) {
-//            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
-//            chunk.na */
-//
-//            chunk.path = chunk.allFlaggedParents.join('/');
-//
-//            if (this.debugLog) {
-//              DyFM_Log.H_info(`✅ saving ${this.dataParams.dataName} Chunk: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`, {
-//                chunk: chunk,
-//              });
-//
-//              /* DyFM_Log.H_warn(`STOP`);
-//              await DyFM_Async.wait(10_000 * minute); */
-//            }
-//
-//            await docChunk_DS.saveData(chunk, false, true);
-//            DyFM_Log.info(`✅ ${this.dataParams.dataName} Chunk saved: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`)
-//          }
-//        }
-//      );
-//
-//      DyFM_Log.success(`✅ ${this.dataParams.dataName} Chunks updated(SAVED) for page: ${pageCompareResult.newPage.name}`);
-//    } catch (error) {
-//      throw new DyFM_Error({
-//        ...this.getDefaultErrorSettings('processPage', error),
-//        errorCode: 'DCH-CDP-PP0',
-//      });
-//    }
-//  }
-
-  /* async comparePages(pages: T_Page[], oldPages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
-    try {
-      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
-
-      for (const page of pages) {
-        const compareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.comparePage(page);
-        compareResults.push(compareResult);
-      }
-
-      return compareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePages', error),
-        errorCode: 'DCH-CDP-CP0',
-      });
-    }
-  } */
-}
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Object } from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
+import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
+import { DyNTS_DAI_DocChunking_Util } from '../../../document-ai/_collections/dai-chunking.util';
+import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_DocPage, DyNTS_DAI_docPage_dataParams } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../../document-ai/_models/data-models/dai-document.data-model';
+
+import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
+import { DyNTS_OAI_DocumentDataService } from './oai-document.data-service';
+import { DyNTS_DAI_CompareResult_Type } from '../../../document-ai/_enums/dai-compare-result-type.enum';
+
+
+export class DyNTS_OAI_DocPageDataService<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
+  T_ExternalDocumentId = string,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
+> extends DyNTS_DataService<T_Page> {
+
+  debugLog: boolean = true;
+  absoluteDeletesOnProcess: boolean = true;
+
+  constructor(
+    issuer: string,
+    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
+    data?: T_Page,
+    dataParams: DyFM_DataModel_Params<T_Page> = DyNTS_DAI_docPage_dataParams as DyFM_DataModel_Params<T_Page>,
+  ) {
+    super(
+      data,
+      dataParams,
+      issuer,
+    );
+  }
+
+  getDocumentDataService(): DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  async processPages(documentId: string, pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        page.documentId = documentId;
+        compareResults.push(await this.processPage(page));
+      }
+
+      // All pages that are not in the list, and are in the database saved to the same document
+      const deletedPages: T_Page[] = await this.findDataList({
+        documentId: documentId,
+        externalPageId: {
+          $nin: pages.map(page => page.externalPageId),
+        },
+        /* _id: {
+          $nin: pages.map(page => page._id),
+        }, */
+      } as DyFM_DBFilter<T_Page>);
+
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      for (const deletedPage of deletedPages) {
+        const messages: string[] = [];
+        
+        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
+          documentId: documentId,
+          pageId: deletedPage._id,
+        } as DyFM_DBFilterSimple<T_Chunk>);
+
+        for (const chunk of chunksToDelete) {
+          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
+        }
+        messages.push(`${chunksToDelete.length} chunks deleted`);
+
+        await this.deleteData(deletedPage._id, this.absoluteDeletesOnProcess);
+        messages.unshift(`Page "${deletedPage.name}" deleted (chunks: ${chunksToDelete.length})`);
+
+        compareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.deleted,
+          newPage: null,
+          oldPage: deletedPage,
+          chunkCompareResults: [],
+          reports: messages,
+        });
+      }
+
+      return compareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPages', error),
+        errorCode: 'DCH-CDP-PPS0',
+      });
+    }
+  }
+
+  async processPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      let existingPage: T_Page;
+      if (page._id) {
+        existingPage = await this.findData({
+          _id: page._id,
+        } as DyFM_DBFilterSimple<T_Page>);
+      } else {
+        if (!page.externalPageId) {
+          throw new DyFM_Error({
+            ...this.getDefaultErrorSettings('processPage', new Error('externalPageId is required')),
+            errorCode: 'DCH-CDP-PP1',
+          });
+        }
+
+        existingPage = await this.findData({
+          /* documentId: page.documentId, */
+          externalPageId: page.externalPageId,
+        } as DyFM_DBFilterSimple<T_Page>);
+      }
+
+      let pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>;
+      
+      if (existingPage) {
+        // Compare first to determine if we need to save
+        pageCompareResult = await this.comparePage(page, existingPage);
+        
+        // Always process chunks after page comparison, regardless of page equality
+        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+          this.getDocChunkDataService();
+
+        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processPage(
+          existingPage,
+        );
+
+        pageCompareResult.chunkCompareResults = chunksCompareResults;
+
+        // Recalculate result based on both page metadata and chunk changes
+        if (
+          pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal && 
+          chunksCompareResults.some(r => r.result !== DyNTS_DAI_CompareResult_Type.equal)
+        ) {
+          pageCompareResult.result = DyNTS_DAI_CompareResult_Type.modified;
+          pageCompareResult.reports.push('Chunks modified');
+        }
+
+        if (pageCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+          // No changes detected, reuse existing page without saving
+          pageCompareResult.newPage = existingPage;
+          pageCompareResult.oldPage = existingPage;
+        } else {
+          // Changes detected, save the updated page
+          const pageDataToSave: T_Page = { ...existingPage };
+          DyFM_Object.cleanAssign(pageDataToSave, page);
+          delete pageDataToSave.chunks;
+
+          const savedPage: T_Page = await this.saveData(pageDataToSave, true);
+
+          page.chunks.forEach((chunk: T_Chunk) => {
+            chunk.documentId = savedPage.documentId;
+            chunk.pageId = savedPage._id;
+          });
+          page._id = savedPage._id;
+          pageCompareResult.newPage = page;
+        }
+      } else {
+        // New page, save it first to get ID
+        const pageDataToSave: T_Page = {
+          ...page,
+        };
+        delete pageDataToSave.chunks;
+
+        const savedPage: T_Page = await this.saveData(pageDataToSave, true);
+
+        page.chunks.forEach((chunk: T_Chunk) => {
+          chunk.documentId = savedPage.documentId;
+          chunk.pageId = savedPage._id;
+        });
+        page._id = savedPage._id;
+
+        pageCompareResult = {
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newPage: page,
+          oldPage: null,  
+          chunkCompareResults: [],
+          reports: [
+            `New page: ${page.name}`,
+            `${page.chunks.length} new chunks in ${page.name}`,
+          ],
+        };
+
+        const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+          this.getDocChunkDataService();
+
+        const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
+          savedPage,
+        );
+
+        pageCompareResult.chunkCompareResults = chunksCompareResults;
+      }
+
+      return pageCompareResult;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPage', error),
+        errorCode: 'DCH-CDP-PP0',
+      });
+    }
+  }
+
+  async processNewPages(pages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const pageCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.processNewPage(page);
+        pageCompareResults.push(pageCompareResult);
+      }
+
+      return pageCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewPages', error),
+        errorCode: 'DCH-CDP-PNPS0',
+      });
+    }
+  }
+
+  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      // save the page to ensure its id
+      const savedPage: T_Page = await this.saveData(page, true);
+
+      // create chunks
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.processNewPage(
+        savedPage,
+      );
+      /* const chunks: T_Chunk[] = await docChunk_DS.processNewChunks(page); */
+
+      /* page.chunks.forEach((chunk: T_Chunk) => {
+        chunk.documentId = savedPage.documentId;
+        chunk.pageId = savedPage._id;
+      });
+      page._id = savedPage._id; */
+
+      const pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
+        result: DyNTS_DAI_CompareResult_Type.new,
+        newPage: page,
+        oldPage: null,
+        chunkCompareResults: chunksCompareResults,
+        reports: [
+          `New page: ${page.name}`,
+          `${page.chunks.length} new chunks in ${page.name}`,
+        ],
+      };
+
+      return pageCompareResult;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewPage', error),
+        errorCode: 'DCH-CDP-PNP0',
+      });
+    }
+  }
+
+  /** 
+   * this is a deep compare, that will update chunks if needed 
+   * */
+  async comparePage(newPage: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      if (!oldPage) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('comparePage', new Error('oldPage is required')),
+          errorCode: 'DCH-CDP-CP1',
+        });
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${newPage.name}`);
+      }
+
+      let pageCompareResult: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
+      const messages: string[] = [];
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+
+      // Compare all relevant fields for comprehensive change detection
+      if (newPage.name !== oldPage.name) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page name modified`);
+      }
+
+      if (newPage.content !== oldPage.content) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page content modified`);
+      }
+
+      // Compare allFlaggedParents arrays
+      const allFlaggedParentsEqual = this.compareStringArrays(
+        newPage.allFlaggedParents || [], 
+        oldPage.allFlaggedParents || []
+      );
+      if (!allFlaggedParentsEqual) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page allFlaggedParents modified`);
+      }
+
+      // Compare allFlaggedParentsMerged
+      if (newPage.allFlaggedParentsMerged !== oldPage.allFlaggedParentsMerged) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page allFlaggedParentsMerged modified`);
+      }
+
+      // Compare filePath
+      if (newPage.filePath !== oldPage.filePath) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page filePath modified`);
+      }
+
+      // Compare parentedPath arrays
+      const parentedPathEqual = this.compareStringArrays(
+        newPage.parentedPath || [], 
+        oldPage.parentedPath || []
+      );
+      if (!parentedPathEqual) {
+        pageCompareResult = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Page parentedPath modified`);
+      }
+
+      const result: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = {
+        result: pageCompareResult,
+        newPage: newPage,
+        oldPage: oldPage,
+        chunkCompareResults: chunkCompareResults,
+        reports: messages,
+      };
+      
+
+      return result;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePage', error),
+        errorCode: 'DCH-CDP-CP0',
+      });
+    }
+  }
+
+  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
+    if (arr1.length !== arr2.length) {
+      return false;
+    }
+    return arr1.every((item, index) => item === arr2[index]);
+  }
+
+  /* async comparePage_deep(page: T_Page, oldPage: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+    try {
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
+      }
+
+      const pageCompareResult_flat: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = 
+        await this.comparePage(page, oldPage);
+      const messages: string[] = pageCompareResult_flat.reports;
+
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      const chunksCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
+        page.chunks, oldPage.chunks);
+
+
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePage_deep', error),
+        errorCode: 'DCH-CDP-CP0',
+      });
+    }
+  } */
+
+
+
+//  async comparePage(page: T_Page): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>> {
+//    try {
+//      if (this.debugLog) {
+//        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
+//      }
+//
+//      if (!page?.allFlaggedParentsMerged) {
+//        DyFM_Log.T_error(
+//          `❌ ${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`, JSON.stringify(page, null, 2)
+//        );
+//
+//        throw new DyFM_Error({
+//          ...this.getDefaultErrorSettings(
+//            'comparePage', 
+//            new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`),
+//          ),
+//          errorCode: 'DCH-CSB-CP0',
+//        });
+//      }
+//
+//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+//        this.getDocChunkDataService();
+//
+//      const oldChunks: T_Chunk[] = await docChunk_DS.findDataList({
+//        path: page.allFlaggedParentsMerged,
+//      } as DyFM_DBFilterSimple<T_Chunk>);
+//
+//      const compareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = await docChunk_DS.compareChunks(
+//        page.chunks,
+//        oldChunks
+//      );
+//
+//      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
+//        compareResults: compareResults,
+//        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
+//        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
+//        modified: compareResults.some(
+//          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
+//        ),
+//      }); */
+//
+//      if (
+//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)
+//      ) {
+//        if (this.debugLog) {
+//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is equal: ${page.name}`);
+//        }
+//
+//        return {
+//          result: DyNTS_DAI_CompareResult_Type.equal,
+//          newPage: page,
+//          oldPageContent: page.content,
+//          chunkCompareResults: compareResults,
+//          newChunks: page.chunks,
+//          oldChunks: oldChunks,
+//        };
+//      }
+//
+//      if (
+//        compareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.new)
+//      ) {
+//        if (this.debugLog) {
+//          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is new: ${page.name}`);
+//        }
+//
+//        return {
+//          result: DyNTS_DAI_CompareResult_Type.new,
+//          newPage: page,
+//          oldPageContent: '',
+//          chunkCompareResults: compareResults,
+//          newChunks: page.chunks,
+//          oldChunks: oldChunks,
+//        };
+//      }
+//
+//      if (this.debugLog) {
+//        DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is modified: ${page.name}`);
+//      }
+//
+//      return {
+//        result: DyNTS_DAI_CompareResult_Type.modified,
+//        newPage: page,
+//        oldPageContent: DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(
+//          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
+//          this.issuer
+//        ),
+//        chunkCompareResults: compareResults,
+//        newChunks: page.chunks,
+//        oldChunks: oldChunks,
+//      };
+//    } catch (error) {
+//      throw new DyFM_Error({
+//        ...this.getDefaultErrorSettings('comparePage', error),
+//        errorCode: 'DCH-CDP-CP0',
+//      });
+//    }
+//  }
+
+  
+//  async updatePage(
+//    pageCompareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>, 
+//    issuer: string
+//  ): Promise<void> {
+//    try {
+//      // save chunks 
+//      // (only new and modified pages 
+//      //   (but all the page element should be updated 
+//      //     (delete extra elements)))
+//
+//      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
+//        // delete extra chunks
+//        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
+//
+//          if (i < pageCompareResult.newChunks.length) {
+//            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
+//          } else {
+//            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
+//              DyFM_Error.logSimple(`❌ ${this.dataParams.dataName} Chunk not deleted`, error);
+//            });
+//          }
+//        }
+//      }
+//
+//      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+//        this.getDocChunkDataService(issuer);
+//
+//      await DyFM_Array.asyncForEachAllAtOnce(
+//        pageCompareResult.newChunks,
+//        async (chunk: T_Chunk) => {
+//          if (
+//            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_DAI_CompareResult_Type.equal
+//          ) {
+//            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
+//            chunk.na */
+//
+//            chunk.path = chunk.allFlaggedParents.join('/');
+//
+//            if (this.debugLog) {
+//              DyFM_Log.H_info(`✅ saving ${this.dataParams.dataName} Chunk: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`, {
+//                chunk: chunk,
+//              });
+//
+//              /* DyFM_Log.H_warn(`STOP`);
+//              await DyFM_Async.wait(10_000 * minute); */
+//            }
+//
+//            await docChunk_DS.saveData(chunk, false, true);
+//            DyFM_Log.info(`✅ ${this.dataParams.dataName} Chunk saved: ${pageCompareResult.newPage.name}; ${chunk.chunkIndex}`)
+//          }
+//        }
+//      );
+//
+//      DyFM_Log.success(`✅ ${this.dataParams.dataName} Chunks updated(SAVED) for page: ${pageCompareResult.newPage.name}`);
+//    } catch (error) {
+//      throw new DyFM_Error({
+//        ...this.getDefaultErrorSettings('processPage', error),
+//        errorCode: 'DCH-CDP-PP0',
+//      });
+//    }
+//  }
+
+  /* async comparePages(pages: T_Page[], oldPages: T_Page[]): Promise<DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[]> {
+    try {
+      const compareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = [];
+
+      for (const page of pages) {
+        const compareResult: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page> = await this.comparePage(page);
+        compareResults.push(compareResult);
+      }
+
+      return compareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePages', error),
+        errorCode: 'DCH-CDP-CP0',
+      });
+    }
+  } */
+}