@futdevpro/nts-dynamo 1.12.15 → 1.14.1

package/src/_modules/open-ai/_services/oai-chunk.service-base.ts

@@ -1,280 +0,0 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata, DyFM_Async } from '@futdevpro/fsm-dynamo';
-import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
-import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
-import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
-import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
-import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
-import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
-import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
-import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
-
-
-export class DyNTS_OAI_Chunk_DataServiceBase<
-  T_Chunk extends DyNTS_OAI_Chunk = DyNTS_OAI_Chunk,
-  T_Page extends DyNTS_OAI_DocumentPage<T_Chunk> = DyNTS_OAI_DocumentPage<T_Chunk>, 
-> extends DyNTS_OAI_VectorDataService<T_Chunk> {
-
-  override debugLog: boolean = true;
-
-  constructor(
-    dataParams: DyFM_DataModel_Params<T_Chunk>,
-    openAiSettings: DyFM_OpenAI_Settings,
-    issuer: string
-  ) {
-    super(
-      null, 
-      dataParams, 
-      openAiSettings,
-      issuer
-    );
-  }
-
-  async updatePage(
-    pageCompareResult: DyNTS_OAI_PageCompareResult<T_Chunk, T_Page>, 
-    issuer: string
-  ): Promise<void> {
-    try {
-      // save chunks 
-      // (only new and modified pages 
-      //   (but all the page element should be updated 
-      //     (delete extra elements)))
-
-      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
-        // delete extra chunks
-        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
-
-          if (i < pageCompareResult.newChunks.length) {
-            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
-          } else {
-            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
-              DyFM_Error.logSimple(`❌ ${this.dataParams.dataName} Chunk not deleted`, error);
-            });
-          }
-        }
-      }
-
-      await DyFM_Array.asyncForEachAllAtOnce(
-        pageCompareResult.newChunks,
-        async (chunk) => {
-          if (
-            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_OAI_CompareResult_Type.equal
-          ) {
-            /* chunk.allFlaggedParents = chunk.allFlaggedParents.map(parent => parent.trim())
-            chunk.na */
-
-            chunk.path = chunk.allFlaggedParents.join('/');
-
-            if (this.debugLog) {
-              DyFM_Log.H_info(`✅ saving ${this.dataParams.dataName} Chunk: ${pageCompareResult.subjectPage.name}; ${chunk.chunkIndex}`, {
-                chunk: chunk,
-              });
-
-              /* DyFM_Log.H_warn(`STOP`);
-              await DyFM_Async.wait(10_000 * minute); */
-            }
-
-            await this.saveData(chunk, false, true);
-            DyFM_Log.info(`✅ ${this.dataParams.dataName} Chunk saved: ${pageCompareResult.subjectPage.name}; ${chunk.chunkIndex}`)
-          }
-        }
-      );
-
-      DyFM_Log.success(`✅ ${this.dataParams.dataName} Chunks updated(SAVED) for page: ${pageCompareResult.subjectPage.name}`);
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('updatePage', error),
-        errorCode: 'DCH-CSB-UP0',
-      });
-    }
-  }
-
-  async comparePage(page: T_Page, issuer: string): Promise<DyNTS_OAI_PageCompareResult<T_Chunk, T_Page>> {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} page: ${page.name}`);
-      }
-
-      if (!page?.allFlaggedParentsMerged) {
-        DyFM_Log.T_error(
-          `❌ ${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`, JSON.stringify(page, null, 2)
-        );
-
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'comparePage', 
-            new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set (${page.name})`),
-          ),
-          errorCode: 'DCH-CSB-CP0',
-        });
-      }
-
-      const oldChunks = await this.findDataList({
-        path: page.allFlaggedParentsMerged,
-      } as DyFM_DBFilterSimple<T_Chunk>);
-
-      const compareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[] = await this.compareChunks(
-        page.chunks,
-        oldChunks
-      );
-
-      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
-        compareResults: compareResults,
-        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
-        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
-        modified: compareResults.some(
-          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
-        ),
-      }); */
-
-      if (
-        compareResults.every(result => result.result === DyNTS_OAI_CompareResult_Type.equal)
-      ) {
-        if (this.debugLog) {
-          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is equal: ${page.name}`);
-        }
-
-        return {
-          result: DyNTS_OAI_CompareResult_Type.equal,
-          subjectPage: page,
-          oldPageContent: page.content,
-          chunkCompareResults: compareResults,
-          newChunks: page.chunks,
-          oldChunks: oldChunks,
-        };
-      }
-
-      if (
-        compareResults.every(result => result.result === DyNTS_OAI_CompareResult_Type.new)
-      ) {
-        if (this.debugLog) {
-          DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is new: ${page.name}`);
-        }
-
-        return {
-          result: DyNTS_OAI_CompareResult_Type.new,
-          subjectPage: page,
-          oldPageContent: '',
-          chunkCompareResults: compareResults,
-          newChunks: page.chunks,
-          oldChunks: oldChunks,
-        };
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.info(`✅ ${this.dataParams.dataName} Page is modified: ${page.name}`);
-      }
-
-      return {
-        result: DyNTS_OAI_CompareResult_Type.modified,
-        subjectPage: page,
-        oldPageContent: DyNTS_OAI_Chunking_Util.reassembleChunksToContent(
-          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
-          issuer
-        ),
-        chunkCompareResults: compareResults,
-        newChunks: page.chunks,
-        oldChunks: oldChunks,
-      };
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('comparePage', error),
-        errorCode: 'DCH-CSB-CP0',
-      });
-    }
-  }
-
-  async compareChunks(
-    chunks: T_Chunk[], 
-    oldChunks?: T_Chunk[]
-  ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>[]> {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunks: ${chunks.length}`);
-      }
-
-      if (chunks.length === 0) {
-        return [];
-      }
-
-      if (chunks.some(chunk => !chunk.path)) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'compareChunks', new Error(`${this.dataParams.dataName} allFlaggedParentsMerged is not set`),
-          ),
-          errorCode: 'DCH-CSB-CC0',
-        });
-      }
-
-      oldChunks ??= await this.findDataList({
-        path: chunks[0].path,
-      } as DyFM_DBFilterSimple<T_Chunk>);
-      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
-
-      return await DyFM_Array.asyncForEachAllAtOnce(
-        chunks,
-        async (chunk) => {
-          return await this.compareChunk(chunk, oldChunks[chunk.chunkIndex]);
-        }
-      )
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('compareChunks', error),
-        errorCode: 'DCH-CSB-CC0',
-      });
-    }
-  }
-
-  async compareChunk(
-    newChunk: T_Chunk, 
-    oldChunk?: T_Chunk
-  ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>> {
-    try {
-      if (this.debugLog) {
-        DyFM_Log.info(`🔍 Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`);
-      }
-
-      oldChunk ??= await this.findData({
-        path: newChunk.path,
-        chunkIndex: newChunk.chunkIndex,
-      } as DyFM_DBFilterSimple<T_Chunk>);
-
-      /* DyFM_Log.H_info(`✅ Comparing ${this.dataParams.dataName} chunk: ${newChunk.chunkIndex}`, {
-        newChunk: newChunk,
-        oldChunk: oldChunk,
-        new: !oldChunk,
-        equal: oldChunk?.chunkParentedContent === newChunk.chunkParentedContent,
-        modified: oldChunk && oldChunk.chunkParentedContent !== newChunk.chunkParentedContent,
-      }); */
-
-      if (!oldChunk) {
-        // itt kéne vektor search-el utána keresni, 
-        // hogy a page link-je megtalálható-e (avagy csak átnevezésre került-e)
-        // hogy az adott tartalom, nem-e csak átnevezésre, szétbontásra vagy összevonásra került-e
-        return {
-          result: DyNTS_OAI_CompareResult_Type.new,
-          oldChunk: null,
-        };
-      }
-
-      if (oldChunk.chunkContent === newChunk.chunkContent) {
-        return {
-          result: DyNTS_OAI_CompareResult_Type.equal,
-          oldChunk: oldChunk,
-        };
-      }
-
-      return {
-        result: DyNTS_OAI_CompareResult_Type.modified,
-        oldChunk: oldChunk,
-      };
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('compareChunk', error),
-        errorCode: 'DCH-CSB-CC1',
-      });
-    }
-
-    
-  }
-  
-}

package/src/_modules/open-ai/index.ts

@@ -1,55 +0,0 @@
-
-
-
-
-
-
-
-
-/* try {
-  require.resolve('open-ai');
-} catch {
-  console.log(
-    '❌ OpenAI package not found, please install it with: pnpm add open-ai'
-  );
-} */
-
-//
-// OpenAI module
-//
-// This module is created to simplify the usage of OpenAI API
-// It creates universal solutions for the most common use cases
-//
-
-
-export * from '@futdevpro/fsm-dynamo/open-ai';
-
-// COLLECTIONS
-export * from './_collections/oai-chunking.util';
-export * from './_collections/oai-document.util';
-export * from './_collections/oai-global-settings.const';
-export * from './_collections/oai-llm-predefined-requests.conts';
-export * from './_collections/oai-llm-predefined-requests-hu.conts';
-
-// ENUMS
-export * from './_enums/oai-compare-result-type.enum';
-export * from './_enums/oai-gpt-message-role.enum';
-
-// MODELS
-export * from './_models/oai-doc-chunk.data-model';
-// models/INTERFACES
-export * from './_models/interfaces/oai-chunk-compare-result.interface';
-export * from './_models/interfaces/oai-page-compare-result.interface';
-export * from './_models/interfaces/oai-document-page.interface';
-export * from './_models/interfaces/oai-global-settings.interface';
-export * from './_models/interfaces/oai-gpt-message.interface';
-export * from './_models/interfaces/oai-llm-predefined-requests.interface';
-export * from './_models/interfaces/oai-page-compare-result.interface';
-
-// SERVICES
-export * from './_services/oai-chunk.service-base';
-export * from './_services/oai-embedding.control-service';
-export * from './_services/oai-llm-chat.service-base';
-export * from './_services/oai-llm.service-base';
-export * from './_services/oai-vector-data.service';
-