@futdevpro/nts-dynamo 1.15.38 → 1.15.40

package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.ts

@@ -1,276 +1,276 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { DyNTS_LVS_VectorDataService } from './lvs-local-vector-search.data-service';
-import { DyNTS_DAI_DocChunk, DyNTS_DAI_docChunk_dataParams } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model';
-import { DyNTS_DAI_DocPage } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model';
-import { DyNTS_DAI_Document } from '../../ai/_modules/document-ai/_models/data-models/dai-document.data-model';
-import { DyNTS_OAI_DocumentDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-document.data-service';
-import { DyNTS_OAI_DocPageDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service';
-import { DyNTS_DAI_DocChunkCompareResult } from '../../ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
-import { DyNTS_DAI_CompareResult_Type } from '../../ai/_modules/document-ai/_enums/dai-compare-result-type.enum';
-import { DyNTS_DAI_Document_Util } from '../../ai/_modules/document-ai/_collections/dai-document.util';
-import { DyNTS_DAI_DocChunking_Util } from '../../ai/_modules/document-ai/_collections/dai-chunking.util';
-
-/**
- * Local Doc Chunk Data Service
- * Extends {@link DyNTS_LVS_VectorDataService} to provide document chunking functionality
- * with local in-memory vector search instead of MongoDB vector search
- */
-export class DyNTS_LVS_DocChunkDataService<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, 
-  T_ExternalDocumentId = string,
-  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = 
-    DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
-> extends DyNTS_LVS_VectorDataService<T_Chunk> {
-
-  /* override debugLog: boolean = false; */
-
-  clickupWorkspaceId: string;
-
-  constructor(
-    issuer: string,
-    openAiSettings: DyFM_OAI_Settings,
-    dataParams: DyFM_DataModel_Params<T_Chunk> = 
-      DyNTS_DAI_docChunk_dataParams as DyFM_DataModel_Params<T_Chunk>,
-  ) {
-    super(
-      null, 
-      dataParams, 
-      openAiSettings,
-      issuer
-    );
-  }
-
-  getDocumentDataService(): DyNTS_OAI_DocumentDataService<
-    T_Chunk, 
-    T_Page, 
-    T_ExternalDocumentId, 
-    T_Document
-  > {
-    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.openAISettings,
-    );
-  }
-
-  getDocPageDataService(): DyNTS_OAI_DocPageDataService<
-    T_Chunk, 
-    T_Page, 
-    T_ExternalDocumentId, 
-    T_Document
-  > {
-    return new DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
-      this.issuer,
-      this.openAISettings,
-    );
-  }
-
-  async processPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
-    try {
-      const newChunks: T_Chunk[] = await this.chunkPage(page);
-
-      const oldChunks: T_Chunk[] = await this.findDataList({
-        /* pageId: page._id, */
-        externalPageId: page.externalPageId,
-      } as DyFM_DBFilterSimple<T_Chunk>);
-      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
-
-      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
-      for (let i = 0; i < newChunks.length; i++) {
-        const newChunk = newChunks[i];
-        newChunk.documentId = page.documentId;
-        newChunk.pageId = page._id;
-        newChunk.chunkIndex = i;
-
-        const oldChunk = oldChunks[i];
-        if (oldChunk) {
-          newChunk._id = oldChunk._id;
-          // Compare first to determine if we need to save
-          const chunkCompareResult: DyNTS_DAI_DocChunkCompareResult<T_Chunk> = 
-            await this.compareChunk(newChunk, oldChunk);
-          
-          if (chunkCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
-            // No changes detected, reuse existing chunk without saving
-            chunkCompareResult.newChunk = oldChunk;
-            chunkCompareResults.push(chunkCompareResult);
-          } else {
-            // Changes detected, save the updated chunk
-            const savedChunk: T_Chunk = await this.saveData(newChunk, true);
-            chunkCompareResult.newChunk = savedChunk;
-            chunkCompareResults.push(chunkCompareResult);
-          }
-        } else {
-          // New chunk, save it
-          const savedChunk: T_Chunk = await this.saveData(newChunk, true);
-          chunkCompareResults.push({
-            result: DyNTS_DAI_CompareResult_Type.new,
-            newChunk: savedChunk,
-            oldChunk: null,
-          });
-        }
-      }
-
-      for (let i = newChunks.length; i < oldChunks.length; i++) {
-        const oldChunk = oldChunks[i];
-        await this.deleteData(oldChunk._id);
-        chunkCompareResults.push({
-          result: DyNTS_DAI_CompareResult_Type.deleted,
-          newChunk: null,
-          oldChunk: oldChunk,
-        });
-      }
-
-      return chunkCompareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processPage', error),
-        errorCode: 'DCH-CSB-PP0',
-      });
-    }
-  }
-
-  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
-    try {
-      const chunks: T_Chunk[] = await this.chunkPage(page);
-
-      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
-      
-      for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
-        chunk.documentId = page.documentId;
-        chunk.pageId = page._id;
-        chunk.chunkIndex = i;
-
-        const savedChunk: T_Chunk = await this.saveData(chunk, true);
-
-        chunkCompareResults.push({
-          result: DyNTS_DAI_CompareResult_Type.new,
-          newChunk: savedChunk,
-          oldChunk: null,
-        });
-      }
-
-      return chunkCompareResults;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('processNewPage', error),
-        errorCode: 'DCH-CSB-PNP0',
-      });
-    }
-  }
-
-  async compareChunk(
-    newChunk: T_Chunk, 
-    oldChunk: T_Chunk
-  ): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>> {
-    try {
-      if (!oldChunk) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
-          errorCode: 'DCH-CSB-CC1',
-        });
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
-      }
-
-      // Compare all relevant fields for comprehensive change detection
-      const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
-      const pathEqual = oldChunk.path === newChunk.path;
-      const documentNameEqual = oldChunk.documentName === newChunk.documentName;
-      const pageNameEqual = oldChunk.pageName === newChunk.pageName;
-      const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
-      
-      // Compare allFlaggedParents arrays
-      const allFlaggedParentsEqual = this.compareStringArrays(
-        oldChunk.allFlaggedParents || [], 
-        newChunk.allFlaggedParents || []
-      );
-
-      if (
-        contentEqual && 
-        pathEqual && 
-        documentNameEqual && 
-        pageNameEqual && 
-        pageLinkEqual && 
-        allFlaggedParentsEqual
-      ) {
-        return {
-          result: DyNTS_DAI_CompareResult_Type.equal,
-          newChunk: newChunk,
-          oldChunk: oldChunk,
-        };
-      }
-
-      return {
-        result: DyNTS_DAI_CompareResult_Type.modified,
-        newChunk: newChunk,
-        oldChunk: oldChunk,
-      };
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('compareChunk', error),
-        errorCode: 'DCH-CSB-CC0',
-      });
-    }
-  }
-
-  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
-    if (arr1.length !== arr2.length) {
-      return false;
-    }
-    return arr1.every((item, index) => item === arr2[index]);
-  }
-  
-  async chunkPage(page: T_Page): Promise<T_Chunk[]> {
-    try {
-      page.chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(
-        page, 
-        (page: T_Page) => this.getPageLink(page), 
-        this.issuer
-      );
-
-      if (this.debugLog) {
-        DyFM_Log.H_info(
-          `Processing file: ${page.name}`, 
-          { 
-            'Page name': page.name, 
-            'Page filePath': page.filePath, 
-            'Page parentedPath': page.parentedPath, 
-            /* 'Page content': page.content,  */
-            'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
-            chunks: page.chunks.map(chunk => { 
-              return {
-                'Chunk name': chunk.documentName,
-                'Chunk path': chunk.path,
-                'Chunk index': chunk.chunkIndex,
-                /* 'Chunk content': chunk.chunkContent, */
-                'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
-              };
-            })
-          }
-        );
-      }
-
-      return page.chunks;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('chunkDocPages', error),
-        errorCode: 'DCH-CSB-CP0',
-      });
-    }
-  }
-
-  getPageLink(page: T_Page): string {
-    return DyNTS_DAI_Document_Util.getClickupPageLink(
-      page.name, 
-      this.clickupWorkspaceId, 
-      page.externalDocumentId, 
-      page.externalPageId
-    );
-  }
-}
-
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { DyNTS_LVS_VectorDataService } from './lvs-local-vector-search.data-service';
+import { DyNTS_DAI_DocChunk, DyNTS_DAI_docChunk_dataParams } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocPage } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_Document } from '../../ai/_modules/document-ai/_models/data-models/dai-document.data-model';
+import { DyNTS_OAI_DocumentDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-document.data-service';
+import { DyNTS_OAI_DocPageDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-doc-page.data-service';
+import { DyNTS_DAI_DocChunkCompareResult } from '../../ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../ai/_modules/document-ai/_enums/dai-compare-result-type.enum';
+import { DyNTS_DAI_Document_Util } from '../../ai/_modules/document-ai/_collections/dai-document.util';
+import { DyNTS_DAI_DocChunking_Util } from '../../ai/_modules/document-ai/_collections/dai-chunking.util';
+
+/**
+ * Local Doc Chunk Data Service
+ * Extends {@link DyNTS_LVS_VectorDataService} to provide document chunking functionality
+ * with local in-memory vector search instead of MongoDB vector search
+ */
+export class DyNTS_LVS_DocChunkDataService<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, 
+  T_ExternalDocumentId = string,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId> = 
+    DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId>,
+> extends DyNTS_LVS_VectorDataService<T_Chunk> {
+
+  /* override debugLog: boolean = false; */
+
+  clickupWorkspaceId: string;
+
+  constructor(
+    issuer: string,
+    openAiSettings: DyFM_OAI_Settings,
+    dataParams: DyFM_DataModel_Params<T_Chunk> = 
+      DyNTS_DAI_docChunk_dataParams as DyFM_DataModel_Params<T_Chunk>,
+  ) {
+    super(
+      null, 
+      dataParams, 
+      openAiSettings,
+      issuer
+    );
+  }
+
+  getDocumentDataService(): DyNTS_OAI_DocumentDataService<
+    T_Chunk, 
+    T_Page, 
+    T_ExternalDocumentId, 
+    T_Document
+  > {
+    return new DyNTS_OAI_DocumentDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.openAISettings,
+    );
+  }
+
+  getDocPageDataService(): DyNTS_OAI_DocPageDataService<
+    T_Chunk, 
+    T_Page, 
+    T_ExternalDocumentId, 
+    T_Document
+  > {
+    return new DyNTS_OAI_DocPageDataService<T_Chunk, T_Page, T_ExternalDocumentId, T_Document>(
+      this.issuer,
+      this.openAISettings,
+    );
+  }
+
+  async processPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
+    try {
+      const newChunks: T_Chunk[] = await this.chunkPage(page);
+
+      const oldChunks: T_Chunk[] = await this.findDataList({
+        /* pageId: page._id, */
+        externalPageId: page.externalPageId,
+      } as DyFM_DBFilterSimple<T_Chunk>);
+      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
+
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+      for (let i = 0; i < newChunks.length; i++) {
+        const newChunk = newChunks[i];
+        newChunk.documentId = page.documentId;
+        newChunk.pageId = page._id;
+        newChunk.chunkIndex = i;
+
+        const oldChunk = oldChunks[i];
+        if (oldChunk) {
+          newChunk._id = oldChunk._id;
+          // Compare first to determine if we need to save
+          const chunkCompareResult: DyNTS_DAI_DocChunkCompareResult<T_Chunk> = 
+            await this.compareChunk(newChunk, oldChunk);
+          
+          if (chunkCompareResult.result === DyNTS_DAI_CompareResult_Type.equal) {
+            // No changes detected, reuse existing chunk without saving
+            chunkCompareResult.newChunk = oldChunk;
+            chunkCompareResults.push(chunkCompareResult);
+          } else {
+            // Changes detected, save the updated chunk
+            const savedChunk: T_Chunk = await this.saveData(newChunk, true);
+            chunkCompareResult.newChunk = savedChunk;
+            chunkCompareResults.push(chunkCompareResult);
+          }
+        } else {
+          // New chunk, save it
+          const savedChunk: T_Chunk = await this.saveData(newChunk, true);
+          chunkCompareResults.push({
+            result: DyNTS_DAI_CompareResult_Type.new,
+            newChunk: savedChunk,
+            oldChunk: null,
+          });
+        }
+      }
+
+      for (let i = newChunks.length; i < oldChunks.length; i++) {
+        const oldChunk = oldChunks[i];
+        await this.deleteData(oldChunk._id);
+        chunkCompareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.deleted,
+          newChunk: null,
+          oldChunk: oldChunk,
+        });
+      }
+
+      return chunkCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processPage', error),
+        errorCode: 'DCH-CSB-PP0',
+      });
+    }
+  }
+
+  async processNewPage(page: T_Page): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>[]> {
+    try {
+      const chunks: T_Chunk[] = await this.chunkPage(page);
+
+      const chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[] = [];
+      
+      for (let i = 0; i < chunks.length; i++) {
+        const chunk = chunks[i];
+        chunk.documentId = page.documentId;
+        chunk.pageId = page._id;
+        chunk.chunkIndex = i;
+
+        const savedChunk: T_Chunk = await this.saveData(chunk, true);
+
+        chunkCompareResults.push({
+          result: DyNTS_DAI_CompareResult_Type.new,
+          newChunk: savedChunk,
+          oldChunk: null,
+        });
+      }
+
+      return chunkCompareResults;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('processNewPage', error),
+        errorCode: 'DCH-CSB-PNP0',
+      });
+    }
+  }
+
+  async compareChunk(
+    newChunk: T_Chunk, 
+    oldChunk: T_Chunk
+  ): Promise<DyNTS_DAI_DocChunkCompareResult<T_Chunk>> {
+    try {
+      if (!oldChunk) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('compareChunk', new Error('oldChunk is required')),
+          errorCode: 'DCH-CSB-CC1',
+        });
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
+      }
+
+      // Compare all relevant fields for comprehensive change detection
+      const contentEqual = oldChunk.chunkContent === newChunk.chunkContent;
+      const pathEqual = oldChunk.path === newChunk.path;
+      const documentNameEqual = oldChunk.documentName === newChunk.documentName;
+      const pageNameEqual = oldChunk.pageName === newChunk.pageName;
+      const pageLinkEqual = oldChunk.pageLink === newChunk.pageLink;
+      
+      // Compare allFlaggedParents arrays
+      const allFlaggedParentsEqual = this.compareStringArrays(
+        oldChunk.allFlaggedParents || [], 
+        newChunk.allFlaggedParents || []
+      );
+
+      if (
+        contentEqual && 
+        pathEqual && 
+        documentNameEqual && 
+        pageNameEqual && 
+        pageLinkEqual && 
+        allFlaggedParentsEqual
+      ) {
+        return {
+          result: DyNTS_DAI_CompareResult_Type.equal,
+          newChunk: newChunk,
+          oldChunk: oldChunk,
+        };
+      }
+
+      return {
+        result: DyNTS_DAI_CompareResult_Type.modified,
+        newChunk: newChunk,
+        oldChunk: oldChunk,
+      };
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareChunk', error),
+        errorCode: 'DCH-CSB-CC0',
+      });
+    }
+  }
+
+  private compareStringArrays(arr1: string[], arr2: string[]): boolean {
+    if (arr1.length !== arr2.length) {
+      return false;
+    }
+    return arr1.every((item, index) => item === arr2[index]);
+  }
+  
+  async chunkPage(page: T_Page): Promise<T_Chunk[]> {
+    try {
+      page.chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(
+        page, 
+        (page: T_Page) => this.getPageLink(page), 
+        this.issuer
+      );
+
+      if (this.debugLog) {
+        DyFM_Log.H_info(
+          `Processing file: ${page.name}`, 
+          { 
+            'Page name': page.name, 
+            'Page filePath': page.filePath, 
+            'Page parentedPath': page.parentedPath, 
+            /* 'Page content': page.content,  */
+            'Page parentFlags': '\n' + page.allFlaggedParents.join('\n') + '\n',
+            chunks: page.chunks.map(chunk => { 
+              return {
+                'Chunk name': chunk.documentName,
+                'Chunk path': chunk.path,
+                'Chunk index': chunk.chunkIndex,
+                /* 'Chunk content': chunk.chunkContent, */
+                'Chunk parentFlags': '\n' + chunk.allFlaggedParents.join('\n') + '\n',
+              };
+            })
+          }
+        );
+      }
+
+      return page.chunks;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('chunkDocPages', error),
+        errorCode: 'DCH-CSB-CP0',
+      });
+    }
+  }
+
+  getPageLink(page: T_Page): string {
+    return DyNTS_DAI_Document_Util.getClickupPageLink(
+      page.name, 
+      this.clickupWorkspaceId, 
+      page.externalDocumentId, 
+      page.externalPageId
+    );
+  }
+}
+