@futdevpro/nts-dynamo 1.15.151 → 1.15.154

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-document.data-service.ts

@@ -1,435 +1,435 @@
-import { DyFM_Array, DyFM_Async, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Object } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
-import { DyNTS_DAI_DocPage } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
-import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../../document-ai/_models/data-models/dai-document.data-model';
-import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
-import { DyNTS_OAI_DocPageDataService } from './oai-doc-page.data-service';
-import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
-import { DyNTS_DAI_DocumentCompareResult } from '../../../document-ai/_models/interfaces/dai-document-compare-result.interface';
-import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
-import { DyNTS_DAI_CompareResult_Type } from '../../../document-ai/_enums/dai-compare-result-type.enum';
-
-
-export class DyNTS_OAI_DocumentDataService<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
-  T_ExternalDocumentId = string,
-  T_ExternalMetadata = any,
-  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata>,
-> extends DyNTS_DataService<T_Document> {
-
-  debugLog: boolean = true;
-  absoluteDeletesOnProcess: boolean = true;
-    
-  constructor(
-    issuer: string,
-    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
-    data?: T_Document,
-    dataParams: DyFM_DataModel_Params<T_Document> = DyNTS_DAI_document_dataParams as DyFM_DataModel_Params<T_Document>,
-  ) {
-    super(
-      data,
-      dataParams,
-      issuer,
-    );
-  }
-
-  getDocPageDataService(): DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
-    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.docChunkDataServiceOpenAiSettings,
-    );
-  }
-
-  async processDocuments(documents: T_Document[]): Promise<
-    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[]
-  > {
-    try {
-      if (!documents || documents.length === 0) {
-        return [];
-      }
-
-      const compareResults: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[] = [];
-
-      await DyFM_Array.asyncForEachAllAtOnce(
-        documents, 
-        async (document: T_Document) => {
-          compareResults.push(await this.processDocument(document));
-        }
-      );
-
-      /* const deletedDocuments: T_Document[] = await this.findDataList({
-        _id: {
-          $nin: documents.map(document => document._id),
-        },
-      } as any as DyFM_DBFilter<T_Document>); */
-      const deletedDocuments: T_Document[] = await this.findDataList({
-        externalDocumentId: {
-          $nin: documents.map(document => document.externalDocumentId),
-        },
-      } as any as DyFM_DBFilter<T_Document>);
-
-      const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocPageDataService();
-
-      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocChunkDataService();
-
-      for (const deletedDocument of deletedDocuments) {
-        const messages: string[] = [];
-
-        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
-          documentId: deletedDocument._id,
-        } as DyFM_DBFilter<T_Chunk>);
-
-        for (const chunk of chunksToDelete) {
-          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
-        }
-        messages.push(`${chunksToDelete.length} chunks deleted`);
-
-        const pagesToDelete: T_Page[] = await docPage_DS.findDataList({
-          documentId: deletedDocument._id,
-        } as DyFM_DBFilter<T_Page>);
-
-        for (const page of pagesToDelete) {
-          await docPage_DS.deleteData(page._id, this.absoluteDeletesOnProcess);
-        }
-        messages.unshift(`${pagesToDelete.length} pages deleted`);
-
-        await this.deleteData(deletedDocument._id, this.absoluteDeletesOnProcess);
-        messages.unshift(`Document "${deletedDocument?.name}" deleted`);
-
-        compareResults.push({
-          result: DyNTS_DAI_CompareResult_Type.deleted,
-          newDocument: null,
-          oldDocument: deletedDocument,
-          pageCompareResults: [],
-          reports: messages,
-        });
-      }
-
-      for (const result of compareResults) {
-        // report the result
-        await this.reportDocumentProcessResult(result);
-      }
-
-      return compareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processDocuments', error),
-      });
-    }
-  }
-
-  async processDocument(document: T_Document): Promise<
-    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
-  > {
-    try {
-      let existingDocument: T_Document; 
-
-      if (document._id) {
-        existingDocument = await this.findData({
-          _id: document._id,
-        } as DyFM_DBFilterSimple<T_Document>);
-      } else {
-        if (!document.externalDocumentId) {
-          throw new DyFM_Error({
-            ...this.getDefaultErrorSettings('processDocument', new Error('externalDocumentId is required')),
-            errorCode: 'OAI-DSB-PD0',
-          });
-        }
-
-        existingDocument = await this.findData({
-          externalDocumentId: document.externalDocumentId,
-        });
-      }
-
-      let documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>;
-      
-      if (existingDocument) {
-        // Set the existing document's ID to the new document and its pages before comparison
-        document._id = existingDocument._id;
-        document.pages.forEach((page: T_Page) => {
-          page.documentId = existingDocument._id;
-        });
-
-        // Compare first to determine if we need to save
-        documentCompareResult = await this.compareDocument_deep(document, existingDocument);
-        
-        if (documentCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
-          // No changes detected, reuse existing document without saving
-          documentCompareResult.newDocument = existingDocument;
-          documentCompareResult.oldDocument = existingDocument;
-        } else {
-          // Changes detected, save the updated document
-          const docDataToSave: T_Document = { ...existingDocument };
-          DyFM_Object.cleanAssign(docDataToSave, document);
-          delete docDataToSave.pages;
-
-          const savedDocument: T_Document = await this.saveData(docDataToSave, true);
-
-          // set the id to the pages and the document
-          document.pages.forEach((page: T_Page) => {
-            page.documentId = savedDocument._id;
-          });
-          document._id = savedDocument._id;
-          documentCompareResult.newDocument = document;
-        }
-      } else {
-        // New document, save it first to get ID
-        const docDataToSave: T_Document = {
-          ...document,
-        };
-        delete docDataToSave.pages;
-
-        const savedDocument: T_Document = await this.saveData(docDataToSave, true);
-
-        // set the id to the pages and the document
-        document.pages.forEach((page: T_Page) => {
-          page.documentId = savedDocument._id;
-        });
-        document._id = savedDocument._id;
-
-        const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-          this.getDocPageDataService();
-
-        const pagesCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = await docPage_DS.processNewPages(
-          document.pages,
-        );
-
-        documentCompareResult = {
-          result: DyNTS_DAI_CompareResult_Type.new,
-          newDocument: document,
-          oldDocument: null,
-          pageCompareResults: pagesCompareResults,
-          reports: [
-            `New document: "${document?.name}"`,
-            `${document?.pages?.length ?? 0} new pages in "${document?.name}"`,
-          ],
-        };
-      }
-
-      return documentCompareResult;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processDocument', error),
-      });
-    }
-  }
-
-  async compareDocument_deep(newDocument: T_Document, oldDocument: T_Document): Promise<
-    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
-  > {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing "${this.dataParams.dataName}" document: "${newDocument?.name}"`);
-      }
-
-      if (!oldDocument) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('compareDocument_deep', new Error('oldDocument is required')),
-          errorCode: 'OAI-DSB-CD1',
-        });
-      }
-
-      const documentCompareResult_flat: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = 
-        await this.compareDocument_flat(newDocument, oldDocument);
-      const messages: string[] = documentCompareResult_flat.reports;
-
-      const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
-        this.getDocPageDataService();
-
-      const pagesCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = await docPage_DS.processPages(
-        newDocument._id,
-        newDocument.pages,
-      );
-
-      // determine the result based on the pages
-      let pagesResult: DyNTS_DAI_CompareResult_Type;
-      if (pagesCompareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)) {
-        pagesResult = DyNTS_DAI_CompareResult_Type.equal;
-        messages.push(`All pages are equal`);
-      } else if (
-        pagesCompareResults.some(
-          result => result.result === DyNTS_DAI_CompareResult_Type.new || 
-                    result.result === DyNTS_DAI_CompareResult_Type.modified ||
-                    result.result === DyNTS_DAI_CompareResult_Type.deleted
-        )
-      ) {
-        pagesResult = DyNTS_DAI_CompareResult_Type.modified;
-
-        const modifiedPages: T_Page[] = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.modified
-        ).map(result => result.newPage);
-        const deletedPages: T_Page[] = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.deleted
-        ).map(result => result.oldPage);
-        const newPages: T_Page[] = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.new
-        ).map(result => result.newPage);
-        /* const modifiedPagesCount: number = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.modified
-        ).length;
-        const deletedPagesCount: number = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.deleted
-        ).length;
-        const newPagesCount: number = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.new
-        ).length; */
-
-        if (modifiedPages.length > 0) {
-          messages.push(`${modifiedPages.length} modified pages: ${modifiedPages.map(page => `"${page.name}"`).join(', ')}`);
-        }
-        if (newPages.length > 0) {
-          messages.push(`${newPages.length} new pages: ${newPages.map(page => `"${page.name}"`).join(', ')}`);
-        }
-        if (deletedPages.length > 0) {
-          messages.push(`${deletedPages.length} deleted pages: ${deletedPages.map(page => `"${page.name}"`).join(', ')}`);
-        }
-      } else {
-        pagesResult = DyNTS_DAI_CompareResult_Type.new;
-        const newPagesCount: number = pagesCompareResults.filter(
-          result => result.result === DyNTS_DAI_CompareResult_Type.new
-        ).length;
-        messages.push(`${newPagesCount} new pages`);        
-      }
-
-      // determine the result based on both the document and the pages
-      let documentResult: DyNTS_DAI_CompareResult_Type;
-      if (documentCompareResult_flat.result === DyNTS_DAI_CompareResult_Type.new) {
-        documentResult = DyNTS_DAI_CompareResult_Type.new;
-      } else if (documentCompareResult_flat.result === DyNTS_DAI_CompareResult_Type.modified) {
-        documentResult = DyNTS_DAI_CompareResult_Type.modified;
-      } else {
-        if (pagesResult === DyNTS_DAI_CompareResult_Type.equal) {
-          documentResult = DyNTS_DAI_CompareResult_Type.equal;
-        } {
-          documentResult = DyNTS_DAI_CompareResult_Type.modified;
-        }
-      }
-      
-      const result: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = {
-        ...documentCompareResult_flat,
-        result: documentResult,
-
-        newDocument: newDocument,
-        oldDocument: oldDocument,
-
-        pageCompareResults: pagesCompareResults,
-      };
-
-      return result;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('compareDocument', error),
-      });
-    }
-  }
-
-  async compareDocument_flat(newDocument: T_Document, oldDocument: T_Document): Promise<
-    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
-  > {
-    try {
-      if (!oldDocument) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('compareDocument_flat', new Error('oldDocument is required')),
-          errorCode: 'OAI-DSB-CF1',
-        });
-      }
-
-      let result: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
-      const messages: string[] = [];
-
-      // Compare document name
-      if (newDocument.name !== oldDocument.name) {
-        result = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Document name changed from "${oldDocument.name}" to "${newDocument.name}"`);
-      }
-
-      // Compare externalDocumentId
-      if (newDocument.externalDocumentId !== oldDocument.externalDocumentId) {
-        result = DyNTS_DAI_CompareResult_Type.modified;
-        messages.push(`Document externalDocumentId changed`);
-      }
-
-      // Compare externalMetadata if present
-      if (newDocument.externalMetadata || oldDocument.externalMetadata) {
-        const newMetadataStr = newDocument.externalMetadata ? JSON.stringify(newDocument.externalMetadata) : '';
-        const oldMetadataStr = oldDocument.externalMetadata ? JSON.stringify(oldDocument.externalMetadata) : '';
-        
-        if (newMetadataStr !== oldMetadataStr) {
-          result = DyNTS_DAI_CompareResult_Type.modified;
-          messages.push(`Document externalMetadata changed`);
-        }
-      }
-
-      if (result === DyNTS_DAI_CompareResult_Type.equal) {
-        messages.push(`Document name or not changed: "${newDocument.name}"`);
-      }
-
-      const result_flat: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = {
-        result: result,
-        
-        newDocument: newDocument,
-        oldDocument: oldDocument,        
-
-        pageCompareResults: [],
-        reports: messages,
-      }
-
-      return result_flat;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('compareDocument_shallow', error),
-      });
-    }
-  }
-
-  async reportDocumentProcessResult(
-    documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>, 
-  ): Promise<void> {
-    try {
-      DyFM_Log.log(
-        `\n✅ Document processed: "${
-          documentCompareResult?.newDocument?.name ?? documentCompareResult?.oldDocument?.name ?? 'unknown'
-        }"`
-      );
-
-      switch (documentCompareResult.result) {
-        case DyNTS_DAI_CompareResult_Type.new:
-          DyFM_Log.info(`  ✨ New document`);
-          break;
-          
-        case DyNTS_DAI_CompareResult_Type.equal:
-          DyFM_Log.success(`  🟰 Equal document`);
-          break;
-
-        case DyNTS_DAI_CompareResult_Type.modified:
-          DyFM_Log.info(`  🔄 Modified document`);
-          break;
-
-        case DyNTS_DAI_CompareResult_Type.deleted:
-          DyFM_Log.warn(`  ❌ Deleted document`);
-          break;
-      }
-
-      for (const report of documentCompareResult.reports) {
-        DyFM_Log.log(`  - ${report}`);
-      }
-
-      DyFM_Log.log('');
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('reportDocumentProcessing', error),
-      });
-    }
-  }
-}
+import { DyFM_Array, DyFM_Async, DyFM_DataModel_Params, DyFM_DBFilter, DyFM_DBFilterExpressions, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Object } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
+import { DyNTS_DAI_DocChunk } from '../../../document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocPage } from '../../../document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_Document, DyNTS_DAI_document_dataParams } from '../../../document-ai/_models/data-models/dai-document.data-model';
+import { DyNTS_OAI_DocChunkDataService } from './oai-doc-chunk-data.service';
+import { DyNTS_OAI_DocPageDataService } from './oai-doc-page.data-service';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyNTS_DAI_DocumentCompareResult } from '../../../document-ai/_models/interfaces/dai-document-compare-result.interface';
+import { DyNTS_DAI_DocPageCompareResult } from '../../../document-ai/_models/interfaces/dai-doc-page-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../../document-ai/_enums/dai-compare-result-type.enum';
+
+
+export class DyNTS_OAI_DocumentDataService<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
+  T_ExternalDocumentId = string,
+  T_ExternalMetadata = any,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata> = DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata>,
+> extends DyNTS_DataService<T_Document> {
+
+  debugLog: boolean = true;
+  absoluteDeletesOnProcess: boolean = true;
+    
+  constructor(
+    issuer: string,
+    private docChunkDataServiceOpenAiSettings: DyFM_OAI_Settings,
+    data?: T_Document,
+    dataParams: DyFM_DataModel_Params<T_Document> = DyNTS_DAI_document_dataParams as DyFM_DataModel_Params<T_Document>,
+  ) {
+    super(
+      data,
+      dataParams,
+      issuer,
+    );
+  }
+
+  getDocPageDataService(): DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  getDocChunkDataService(): DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> {
+    return new DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.docChunkDataServiceOpenAiSettings,
+    );
+  }
+
+  async processDocuments(documents: T_Document[]): Promise<
+    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[]
+  > {
+    try {
+      if (!documents || documents.length === 0) {
+        return [];
+      }
+
+      const compareResults: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>[] = [];
+
+      await DyFM_Array.asyncForEachAllAtOnce(
+        documents, 
+        async (document: T_Document) => {
+          compareResults.push(await this.processDocument(document));
+        }
+      );
+
+      /* const deletedDocuments: T_Document[] = await this.findDataList({
+        _id: {
+          $nin: documents.map(document => document._id),
+        },
+      } as any as DyFM_DBFilter<T_Document>); */
+      const deletedDocuments: T_Document[] = await this.findDataList({
+        externalDocumentId: {
+          $nin: documents.map(document => document.externalDocumentId),
+        },
+      } as any as DyFM_DBFilter<T_Document>);
+
+      const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocPageDataService();
+
+      const docChunk_DS: DyNTS_OAI_DocChunkDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocChunkDataService();
+
+      for (const deletedDocument of deletedDocuments) {
+        const messages: string[] = [];
+
+        const chunksToDelete: T_Chunk[] = await docChunk_DS.findDataList({
+          documentId: deletedDocument._id,
+        } as DyFM_DBFilter<T_Chunk>);
+
+        for (const chunk of chunksToDelete) {
+          await docChunk_DS.deleteData(chunk._id, this.absoluteDeletesOnProcess);
+        }
+        messages.push(`${chunksToDelete.length} chunks deleted`);
+
+        const pagesToDelete: T_Page[] = await docPage_DS.findDataList({
+          documentId: deletedDocument._id,
+        } as DyFM_DBFilter<T_Page>);
+
+        for (const page of pagesToDelete) {
+          await docPage_DS.deleteData(page._id, this.absoluteDeletesOnProcess);
+        }
+        messages.unshift(`${pagesToDelete.length} pages deleted`);
+
+        await this.deleteData(deletedDocument._id, this.absoluteDeletesOnProcess);
+        messages.unshift(`Document "${deletedDocument?.name}" deleted`);
+
+        compareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.deleted,
+          newDocument: null,
+          oldDocument: deletedDocument,
+          pageCompareResults: [],
+          reports: messages,
+        });
+      }
+
+      for (const result of compareResults) {
+        // report the result
+        await this.reportDocumentProcessResult(result);
+      }
+
+      return compareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processDocuments', error),
+      });
+    }
+  }
+
+  async processDocument(document: T_Document): Promise<
+    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
+  > {
+    try {
+      let existingDocument: T_Document; 
+
+      if (document._id) {
+        existingDocument = await this.findData({
+          _id: document._id,
+        } as DyFM_DBFilterSimple<T_Document>);
+      } else {
+        if (!document.externalDocumentId) {
+          throw new DyFM_Error({
+            ...this.getDefaultErrorSettings('processDocument', new Error('externalDocumentId is required')),
+            errorCode: 'OAI-DSB-PD0',
+          });
+        }
+
+        existingDocument = await this.findData({
+          externalDocumentId: document.externalDocumentId,
+        });
+      }
+
+      let documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>;
+      
+      if (existingDocument) {
+        // Set the existing document's ID to the new document and its pages before comparison
+        document._id = existingDocument._id;
+        document.pages.forEach((page: T_Page) => {
+          page.documentId = existingDocument._id;
+        });
+
+        // Compare first to determine if we need to save
+        documentCompareResult = await this.compareDocument_deep(document, existingDocument);
+        
+        if (documentCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+          // No changes detected, reuse existing document without saving
+          documentCompareResult.newDocument = existingDocument;
+          documentCompareResult.oldDocument = existingDocument;
+        } else {
+          // Changes detected, save the updated document
+          const docDataToSave: T_Document = { ...existingDocument };
+          DyFM_Object.cleanAssign(docDataToSave, document);
+          delete docDataToSave.pages;
+
+          const savedDocument: T_Document = await this.saveData(docDataToSave, true);
+
+          // set the id to the pages and the document
+          document.pages.forEach((page: T_Page) => {
+            page.documentId = savedDocument._id;
+          });
+          document._id = savedDocument._id;
+          documentCompareResult.newDocument = document;
+        }
+      } else {
+        // New document, save it first to get ID
+        const docDataToSave: T_Document = {
+          ...document,
+        };
+        delete docDataToSave.pages;
+
+        const savedDocument: T_Document = await this.saveData(docDataToSave, true);
+
+        // set the id to the pages and the document
+        document.pages.forEach((page: T_Page) => {
+          page.documentId = savedDocument._id;
+        });
+        document._id = savedDocument._id;
+
+        const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+          this.getDocPageDataService();
+
+        const pagesCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = await docPage_DS.processNewPages(
+          document.pages,
+        );
+
+        documentCompareResult = {
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newDocument: document,
+          oldDocument: null,
+          pageCompareResults: pagesCompareResults,
+          reports: [
+            `New document: "${document?.name}"`,
+            `${document?.pages?.length ?? 0} new pages in "${document?.name}"`,
+          ],
+        };
+      }
+
+      return documentCompareResult;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processDocument', error),
+      });
+    }
+  }
+
+  async compareDocument_deep(newDocument: T_Document, oldDocument: T_Document): Promise<
+    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
+  > {
+    try {
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing "${this.dataParams.dataName}" document: "${newDocument?.name}"`);
+      }
+
+      if (!oldDocument) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('compareDocument_deep', new Error('oldDocument is required')),
+          errorCode: 'OAI-DSB-CD1',
+        });
+      }
+
+      const documentCompareResult_flat: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = 
+        await this.compareDocument_flat(newDocument, oldDocument);
+      const messages: string[] = documentCompareResult_flat.reports;
+
+      const docPage_DS: DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document> = 
+        this.getDocPageDataService();
+
+      const pagesCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[] = await docPage_DS.processPages(
+        newDocument._id,
+        newDocument.pages,
+      );
+
+      // determine the result based on the pages
+      let pagesResult: DyNTS_DAI_CompareResult_Type;
+      if (pagesCompareResults.every(result => result.result === DyNTS_DAI_CompareResult_Type.equal)) {
+        pagesResult = DyNTS_DAI_CompareResult_Type.equal;
+        messages.push(`All pages are equal`);
+      } else if (
+        pagesCompareResults.some(
+          result => result.result === DyNTS_DAI_CompareResult_Type.new || 
+                    result.result === DyNTS_DAI_CompareResult_Type.modified ||
+                    result.result === DyNTS_DAI_CompareResult_Type.deleted
+        )
+      ) {
+        pagesResult = DyNTS_DAI_CompareResult_Type.modified;
+
+        const modifiedPages: T_Page[] = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.modified
+        ).map(result => result.newPage);
+        const deletedPages: T_Page[] = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.deleted
+        ).map(result => result.oldPage);
+        const newPages: T_Page[] = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.new
+        ).map(result => result.newPage);
+        /* const modifiedPagesCount: number = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.modified
+        ).length;
+        const deletedPagesCount: number = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.deleted
+        ).length;
+        const newPagesCount: number = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.new
+        ).length; */
+
+        if (modifiedPages.length > 0) {
+          messages.push(`${modifiedPages.length} modified pages: ${modifiedPages.map(page => `"${page.name}"`).join(', ')}`);
+        }
+        if (newPages.length > 0) {
+          messages.push(`${newPages.length} new pages: ${newPages.map(page => `"${page.name}"`).join(', ')}`);
+        }
+        if (deletedPages.length > 0) {
+          messages.push(`${deletedPages.length} deleted pages: ${deletedPages.map(page => `"${page.name}"`).join(', ')}`);
+        }
+      } else {
+        pagesResult = DyNTS_DAI_CompareResult_Type.new;
+        const newPagesCount: number = pagesCompareResults.filter(
+          result => result.result === DyNTS_DAI_CompareResult_Type.new
+        ).length;
+        messages.push(`${newPagesCount} new pages`);        
+      }
+
+      // determine the result based on both the document and the pages
+      let documentResult: DyNTS_DAI_CompareResult_Type;
+      if (documentCompareResult_flat.result === DyNTS_DAI_CompareResult_Type.new) {
+        documentResult = DyNTS_DAI_CompareResult_Type.new;
+      } else if (documentCompareResult_flat.result === DyNTS_DAI_CompareResult_Type.modified) {
+        documentResult = DyNTS_DAI_CompareResult_Type.modified;
+      } else {
+        if (pagesResult === DyNTS_DAI_CompareResult_Type.equal) {
+          documentResult = DyNTS_DAI_CompareResult_Type.equal;
+        } {
+          documentResult = DyNTS_DAI_CompareResult_Type.modified;
+        }
+      }
+      
+      const result: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = {
+        ...documentCompareResult_flat,
+        result: documentResult,
+
+        newDocument: newDocument,
+        oldDocument: oldDocument,
+
+        pageCompareResults: pagesCompareResults,
+      };
+
+      return result;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareDocument', error),
+      });
+    }
+  }
+
+  async compareDocument_flat(newDocument: T_Document, oldDocument: T_Document): Promise<
+    DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>
+  > {
+    try {
+      if (!oldDocument) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('compareDocument_flat', new Error('oldDocument is required')),
+          errorCode: 'OAI-DSB-CF1',
+        });
+      }
+
+      let result: DyNTS_DAI_CompareResult_Type = DyNTS_DAI_CompareResult_Type.equal;
+      const messages: string[] = [];
+
+      // Compare document name
+      if (newDocument.name !== oldDocument.name) {
+        result = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Document name changed from "${oldDocument.name}" to "${newDocument.name}"`);
+      }
+
+      // Compare externalDocumentId
+      if (newDocument.externalDocumentId !== oldDocument.externalDocumentId) {
+        result = DyNTS_DAI_CompareResult_Type.modified;
+        messages.push(`Document externalDocumentId changed`);
+      }
+
+      // Compare externalMetadata if present
+      if (newDocument.externalMetadata || oldDocument.externalMetadata) {
+        const newMetadataStr = newDocument.externalMetadata ? JSON.stringify(newDocument.externalMetadata) : '';
+        const oldMetadataStr = oldDocument.externalMetadata ? JSON.stringify(oldDocument.externalMetadata) : '';
+        
+        if (newMetadataStr !== oldMetadataStr) {
+          result = DyNTS_DAI_CompareResult_Type.modified;
+          messages.push(`Document externalMetadata changed`);
+        }
+      }
+
+      if (result === DyNTS_DAI_CompareResult_Type.equal) {
+        messages.push(`Document name or not changed: "${newDocument.name}"`);
+      }
+
+      const result_flat: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document> = {
+        result: result,
+        
+        newDocument: newDocument,
+        oldDocument: oldDocument,        
+
+        pageCompareResults: [],
+        reports: messages,
+      }
+
+      return result_flat;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareDocument_shallow', error),
+      });
+    }
+  }
+
+  async reportDocumentProcessResult(
+    documentCompareResult: DyNTS_DAI_DocumentCompareResult<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata, T_Document>, 
+  ): Promise<void> {
+    try {
+      DyFM_Log.log(
+        `\n✅ Document processed: "${
+          documentCompareResult?.newDocument?.name ?? documentCompareResult?.oldDocument?.name ?? 'unknown'
+        }"`
+      );
+
+      switch (documentCompareResult.result) {
+        case DyNTS_DAI_CompareResult_Type.new:
+          DyFM_Log.info(`  ✨ New document`);
+          break;
+          
+        case DyNTS_DAI_CompareResult_Type.equal:
+          DyFM_Log.success(`  🟰 Equal document`);
+          break;
+
+        case DyNTS_DAI_CompareResult_Type.modified:
+          DyFM_Log.info(`  🔄 Modified document`);
+          break;
+
+        case DyNTS_DAI_CompareResult_Type.deleted:
+          DyFM_Log.warn(`  ❌ Deleted document`);
+          break;
+      }
+
+      for (const report of documentCompareResult.reports) {
+        DyFM_Log.log(`  - ${report}`);
+      }
+
+      DyFM_Log.log('');
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('reportDocumentProcessing', error),
+      });
+    }
+  }
+}