@futdevpro/nts-dynamo 1.11.2 → 1.11.4

package/src/_modules/open-ai/_services/oai-chunk.service-base.ts

@@ -1,36 +1,34 @@
-import { DyFM_Array, DyFM_DataModel_Params, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
-import { DyNTS_OAI_DocChunk, DyNTS_OAI_docChunk_dataParams } from '../_models/oai-doc-chunk.data-model';
-import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_DataProperties, DyFM_DBFilterSimple, DyFM_Error, DyFM_Log, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
 import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
-import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
-import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
-import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
-import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
+import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
 import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
-import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+import { DyNTS_OAI_Chunk } from '../_models/oai-doc-chunk.data-model';
+import { DyNTS_OAI_ChunkCompareResult, DyNTS_OAI_CompareResult_Type, DyNTS_OAI_DocumentPage } from '../../discord-assistant';
+import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
 
 
-export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyNTS_OAI_DocChunk> {
+export class DyNTS_OAI_Chunk_DataServiceBase<
+  T_Page extends DyNTS_OAI_DocumentPage<T_Chunk>, 
+  T_Chunk extends DyNTS_OAI_Chunk
+> extends DyNTS_OAI_VectorDataService<T_Chunk> {
 
-  constructor(issuer: string) {
+  constructor(
+    dataParams: DyFM_DataModel_Params<T_Chunk>,
+    openAiSettings: DyFM_OpenAI_Settings,
+    issuer: string
+  ) {
     super(
       null, 
-      DyNTS_OAI_docChunk_dataParams, 
-      new DyFM_OpenAI_Settings({
-        openAIConfig: {
-          apiKey: DyNTS_global_settings.env_settings.openAi.apiKey,
-          organization: DyNTS_global_settings.env_settings.openAi.organization,
-          project: DyNTS_global_settings.env_settings.openAi.project,
-        },
-        defaultSettings: {
-          useModel: DyFM_OpenAIModel.textEmbedding_3Large,
-        },
-      }),
+      dataParams, 
+      openAiSettings,
       issuer
     );
   }
 
-  async updatePage(pageCompareResult: DyNTS_OAI_PageCompareResult, issuer: string): Promise<void> {
+  async updatePage(
+    pageCompareResult: DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>, 
+    issuer: string
+  ): Promise<void> {
     try {
       // save chunks 
       // (only new and modified pages 
@@ -71,14 +69,13 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
     }
   }
 
-  async comparePage(page: DyNTS_OAI_DocumentPage, issuer: string): Promise<DyNTS_OAI_PageCompareResult> {
+  async comparePage(page: T_Page, issuer: string): Promise<DyNTS_OAI_PageCompareResult<T_Page, T_Chunk>> {
     try {
       const oldChunks = await this.findDataList({
-        documentId: `${page.docId}`,
-        documentPageId: page._id,
-      });
+        allFlaggedParentsMerged: page.allFlaggedParentsMerged,
+      } as DyFM_DBFilterSimple<T_Chunk>);
 
-      const compareResults: DyNTS_OAI_ChunkCompareResult[] = await this.compareChunks(
+      const compareResults: DyNTS_OAI_ChunkCompareResult<T_Chunk>[] = await this.compareChunks(
         page.chunks,
         oldChunks
       );
@@ -138,18 +135,17 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
   }
 
   async compareChunks(
-    chunks: DyNTS_OAI_DocChunk[], 
-    oldChunks?: DyNTS_OAI_DocChunk[]
-  ): Promise<DyNTS_OAI_ChunkCompareResult[]> {
+    chunks: T_Chunk[], 
+    oldChunks?: T_Chunk[]
+  ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>[]> {
     try {
       if (chunks.length === 0) {
         return [];
       }
 
       oldChunks ??= await this.findDataList({
-        documentId: chunks[0].documentId,
-        documentPageId: chunks[0].documentPageId,
-      });
+        allFlaggedParentsMerged: chunks[0].allFlaggedParentsMerged,
+      } as DyFM_DBFilterSimple<T_Chunk>);
       oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
 
       return await DyFM_Array.asyncForEachAllAtOnce(
@@ -167,15 +163,14 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
   }
 
   async compareChunk(
-    newChunk: DyNTS_OAI_DocChunk, 
-    oldChunk?: DyNTS_OAI_DocChunk
-  ): Promise<DyNTS_OAI_ChunkCompareResult> {
+    newChunk: T_Chunk, 
+    oldChunk?: T_Chunk
+  ): Promise<DyNTS_OAI_ChunkCompareResult<T_Chunk>> {
     try {
       oldChunk ??= await this.findData({
-        documentId: newChunk.documentId,
-        documentPageId: newChunk.documentPageId,
+        allFlaggedParentsMerged: newChunk.allFlaggedParentsMerged,
         chunkIndex: newChunk.chunkIndex,
-      });
+      } as DyFM_DBFilterSimple<T_Chunk>);
 
       /* DyFM_Log.H_info(`✅ Comparing chunk: ${newChunk.chunkIndex}`, {
         newChunk: newChunk,
@@ -209,6 +204,8 @@ export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyN
         errorCode: 'DCH-CSB-CC1',
       });
     }
+
+    
   }
   
 }

package/src/_modules/open-ai/_services/oai-vector-data.service.ts

@@ -7,7 +7,7 @@ import {
   DyFM_Metadata,
   DyFM_sameObjects
 } from '@futdevpro/fsm-dynamo';
-import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
+import { DyFM_OpenAI_Settings, DyFM_OpenAIEmbeddingModel, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
 
 import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
 import { DyNTS_DataService } from '../../../_services/base/data.service';
@@ -58,6 +58,7 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
    * (properties that are marked with vectorizeFrom in other properties)
    */
   readonly vectorizeProperties: DyFM_DataProperty_Params<any>[];
+  readonly vectorizedProperties: DyFM_DataProperty_Params<any>[];
   /* readonly vectorizeProperties: { from: string, to: string }[]; */
 
   embedding_CS: DyNTS_OAI_Embedding_ControlService;
@@ -81,11 +82,25 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
     issuer: string
   ) {
     super(data, dataParams, issuer);
+
+    this.vectorizedProperties = Object.values(dataParams.properties).filter(
+      (property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom
+    );
+
+    this.vectorizeProperties = Object.keys(dataParams.properties).filter(
+      (key: string): boolean => this.vectorizedProperties.some(
+        (property: DyFM_DataProperty_Params<any>) => property.vectorizedFrom.includes(key)
+      )
+    ).map(
+      (key: string): DyFM_DataProperty_Params<any> => dataParams.properties[key] as DyFM_DataProperty_Params<any>
+    );
     
-    const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
-      (key: string): boolean => (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom &&
+    /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
+      (key: string): boolean => (
+        dataParams.properties[key] as DyFM_DataProperty_Params<any>
+      ).vectorizedFrom &&
         (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
-    );
+    ); */
 
     if (!openAISettings.openAIConfig.apiKey) {
       throw new DyFM_Error({
@@ -96,7 +111,8 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
       });
     }
 
-    const model = openAISettings.defaultSettings.useModel ?? DyFM_OpenAIModel.textEmbedding_3Large;
+    const model: DyFM_OpenAIEmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OpenAIEmbeddingModel ?? 
+      DyFM_OpenAIModel.textEmbedding_3Large;
 
     /* if (
       !DyFM_allOpenAIEmbeddingModels.includes(model)
@@ -109,7 +125,29 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
       });
     } */
 
-    if (!vectorizePropertyKeys.length) {
+    if (!this.vectorizedProperties.length) {
+      throw new Error(
+        'vectorizedProperties is missing! ' +
+        `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
+          (key: string): boolean => Boolean(
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
+          )
+        ).map(key => `"${key}"`).join(', ')}`
+      );
+    }
+
+    if (!this.vectorizeProperties.length) {
+      throw new Error(
+        'vectorizeProperties is missing! ' +
+        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
+          (key: string): boolean => Boolean(
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
+          )
+        ).map(key => `"${key}"`).join(', ')}`
+      );
+    }    
+
+    /* if (!vectorizePropertyKeys.length) {
       throw new Error(
         'vectorizedPropertyKeys is missing! ' +
         `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
@@ -118,16 +156,20 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
           )
         ).map(key => `"${key}"`).join(', ')}`
       );
-    }
+    } */
 
-    vectorizePropertyKeys.forEach((key: string): void => {
+    /* vectorizePropertyKeys.forEach((key: string): void => {
       dataParams.properties[key].embeddingModel ??= model;
+    }); */
+
+    this.vectorizeProperties.forEach((property: DyFM_DataProperty_Params<any>): void => {
+      property.embeddingModel ??= model;
     });
 
-    this.vectorizeProperties = vectorizePropertyKeys.map(
+    /* this.vectorizeProperties = vectorizePropertyKeys.map(
       (key: string): DyFM_DataProperty_Params<any> => 
         dataParams.properties[key] as DyFM_DataProperty_Params<any>
-    );
+    ); */
   }
 
   /**
@@ -140,43 +182,22 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
    * 
    * @returns The vectorized data
    */
-  async vectorizeDataProperties(data?: T, alwaysVectorize?: boolean): Promise<T> {
+  async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
     try {
       let existingData: T;
 
-      if (!alwaysVectorize) {
-        existingData = await this.getDataById(data._id, true, true);
+      if (!forceVectorize) {
+        const existingData: T = await this.getDataById(newData._id, true, true);
       }
 
       await DyFM_Array.asyncForEachAllAtOnce(
-        this.vectorizeProperties,
+        this.vectorizedProperties,
         async (property: DyFM_DataProperty_Params<any>) => {
-          if (
-            data[property.vectorizedFrom] &&
-              (!data[property.key]?.length || alwaysVectorize || property.vectorizeAlways ||
-                !DyFM_sameObjects(existingData[property.key], data[property.key])
-            )
-          ) {
-            if (!property.embeddingModel) {
-              throw new DyFM_Error({
-                ...this.getDefaultErrorSettings(
-                  'vectorizeDataProperties', 
-                  new Error(`Property "${property.key}" has no embedding model!`)
-                ),
-
-                errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
-              });
-            }
-
-            data[property.key] = await this.vectorize(
-              data[property.vectorizedFrom],
-              property.embeddingModel
-            );
-          }
+          await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
         }
       );
 
-      return data;
+      return newData;
     } catch (error) {
       throw new DyFM_Error({
         ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
@@ -184,6 +205,72 @@ export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_
     }
   }
 
+  private async checkAndVectorizeSingleProperty(
+    vectorizedProperty: DyFM_DataProperty_Params<any>,
+    newData: T,
+    existingData: T,
+    forceVectorize?: boolean
+  ): Promise<void> {
+    const newValue: string = await this.getVectorizingDataForVectorizedProperty(
+      vectorizedProperty,
+      newData
+    );
+    const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
+      vectorizedProperty,
+      existingData
+    );
+
+    const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
+    const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
+    const newDataIsNewChanged: boolean = 
+      (!existingValue && vectorizingPropertyHaveValue) ||
+      !DyFM_sameObjects(existingValue, newValue);
+
+    if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
+      if (!vectorizedProperty.embeddingModel) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorizeDataProperties', 
+            new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
+          ),
+
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
+        });
+      }
+
+      newData[vectorizedProperty.key] = await this.vectorize(
+        newValue,
+        vectorizedProperty.embeddingModel
+      );
+    }
+  }
+
+  private async getVectorizingDataForVectorizedProperty(
+    vectorizedProperty: DyFM_DataProperty_Params<any>,
+    data: T,
+    /* alwaysVectorize?: boolean */
+  ): Promise<string> {
+    if (vectorizedProperty.vectorizedFrom.length === 1) {
+      const value: unknown = data[vectorizedProperty.vectorizedFrom[0]];
+
+      if (typeof value === 'string') {
+        return value;
+      } else if (Array.isArray(value) || typeof value === 'object') {
+        return JSON.stringify(value);
+      } else {
+        return `${value}`;
+      }
+    } else {
+      const object: any = {};
+
+      vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
+        object[key] = data[key];
+      });
+
+      return JSON.stringify(object);
+    }
+  }
+
   /**
    * Vectorizes the input
    * 

package/src/_modules/open-ai/index.ts

@@ -33,6 +33,7 @@ export * from './_enums/oai-gpt-message-role.enum';
 export * from './_models/oai-doc-chunk.data-model';
 // models/INTERFACES
 export * from './_models/interfaces/oai-chunk-compare-result.interface';
+export * from './_models/interfaces/oai-page-compare-result.interface';
 export * from './_models/interfaces/oai-document-page.interface';
 export * from './_models/interfaces/oai-global-settings.interface';
 export * from './_models/interfaces/oai-gpt-message.interface';