@futdevpro/nts-dynamo 1.15.157 → 1.15.163

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts

@@ -1,631 +0,0 @@
-import { DyFM_Log } from '@futdevpro/fsm-dynamo';
-
-import {
-  DyFM_Array,
-  DyFM_DataModel_Params,
-  DyFM_DataProperty_Params,
-  DyFM_DBFilter,
-  DyFM_Error,
-  DyFM_Metadata,
-  DyFM_Object,
-} from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { DyNTS_global_settings } from '../../../../../../_collections/global-settings.const';
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_OAI_Embedding_ControlService } from '../oai-embedding.control-service';
-import { DyNTS_OAI_global_settings } from '../../_collections/oai-global-settings.const';
-import { CreateEmbeddingResponse } from 'openai/resources';
-
-/**
- * Vector Data Service that is connected to the relevant DBServices  
- * extends {@link DyNTS_DataService}
- * 
- * For the vectorization, you need to setup the index in the DB to something like this:
- * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
- * ```json
- * {
- *   "fields": [
- *     {
- *       "type": "vector",
- *       "path": "contentVectorized",
- *       "numDimensions": 3072,
- *       "similarity": "cosine"
- *     }
- *   ]
- * }
- * ```
- * 
- * @example
- * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
- *   constructor(
- *     set: {
- *       data?: UserMatchStatistics,
- *       openAISettings?: DyFM_OAI_Settings,
- *       issuer: string,
- *     }
- *   ) {
- *     super(
- *       new UserMatchStatistics(set?.data),
- *       userMatchStatisticsModelParams,
- *       set?.openAISettings ?? defaultOpenAISettings,
- *       set.issuer,
- *     );
- *   }
- * }
- */
-export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DataService<T> {
-
-  /**
-   * Properties that are vectorized
-   * (properties that are marked with vectorizeFrom in other properties)
-   */
-  readonly vectorizeProperties: DyFM_DataProperty_Params<any, T>[];
-  readonly vectorizedProperties: DyFM_DataProperty_Params<any, T>[];
-  /* readonly vectorizeProperties: { from: string, to: string }[]; */
-
-  debugLog: boolean = false;
-  swallowVectorizedProperties: boolean = true;
-
-  embedding_CS: DyNTS_OAI_Embedding_ControlService;
-
-  tokenSpent: number = 0;
-
-  constructor(
-    /**
-     * Initial data, this will be used by functions on default
-     */
-    data: T,
-    /**
-     * DB data prams will be used to connect to usable dbService on GlobalService
-     */
-    dataParams: DyFM_DataModel_Params<T>,
-    /**
-     * OpenAI settings
-     */
-    public readonly openAISettings: DyFM_OAI_Settings,
-    /**
-     * Initial set for issuer to be able to follow the issuer's activity
-     */
-    issuer: string
-  ) {
-    super(data, dataParams, issuer);
-
-    this.vectorizedProperties = Object.values(dataParams.properties).filter(
-      (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom
-    );
-
-    this.vectorizeProperties = Object.keys(dataParams.properties).filter(
-      (key: string): boolean => this.vectorizedProperties.some(
-        (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom.includes(key)
-      )
-    ).map(
-      (key: string): DyFM_DataProperty_Params<any, T> => dataParams.properties[key] as DyFM_DataProperty_Params<any, T>
-    );
-    
-    /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
-      (key: string): boolean => (
-        dataParams.properties[key] as DyFM_DataProperty_Params<any>
-      ).vectorizedFrom &&
-        (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
-    ); */
-
-    if (!openAISettings.config.apiKey) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'constructor', 
-          new Error('openAISettings.config.apiKey is required')
-        ),
-      });
-    }
-
-    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
-      DyFM_OAI_Model.textEmbedding_3Large;
-
-    /* if (
-      !DyFM_allOpenAIEmbeddingModels.includes(model)
-    ) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'constructor', 
-          new Error(`useModel "${model}" is not an embedding model!`)
-        ),
-      });
-    } */
-
-    if (!this.vectorizedProperties.length) {
-      throw new Error(
-        'vectorizedProperties is missing! ' +
-        `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    }
-
-    if (!this.vectorizeProperties.length) {
-      throw new Error(
-        'vectorizeProperties is missing! ' +
-        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    }    
-
-    /* if (!vectorizePropertyKeys.length) {
-      throw new Error(
-        'vectorizedPropertyKeys is missing! ' +
-        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    } */
-
-    /* vectorizePropertyKeys.forEach((key: string): void => {
-      dataParams.properties[key].embeddingModel ??= model;
-    }); */
-
-    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-      property.embeddingModel ??= model;
-    });
-
-    /* this.vectorizeProperties = vectorizePropertyKeys.map(
-      (key: string): DyFM_DataProperty_Params<any> => 
-        dataParams.properties[key] as DyFM_DataProperty_Params<any>
-    ); */
-  }
-
-  /**
-   * Vectorizes the properties of the data 
-   * IF they are not vectorized yet OR alwaysVectorize is true
-   * (That are marked with vectorizeFrom if the specific property is not defined)
-   * 
-   * @example
-   * const data = await vectorizeDataProperties(data);
-   * 
-   * @returns The vectorized data
-   */
-  async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
-    try {
-      let existingData: T;
-
-      if (!forceVectorize && newData?._id) {
-        existingData = await this.getDataById(newData._id, true, true);
-      }
-
-      await DyFM_Array.asyncForEachAllAtOnce(
-        this.vectorizedProperties,
-        async (property: DyFM_DataProperty_Params<any, T>) => {
-          await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
-        }
-      );
-
-      return newData;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
-      });
-    }
-  }
-
-  private async checkAndVectorizeSingleProperty(
-    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
-    newData: T,
-    existingData: T,
-    forceVectorize?: boolean
-  ): Promise<void> {
-    const newValue: string = await this.getVectorizingDataForVectorizedProperty(
-      vectorizedProperty,
-      newData
-    );
-    const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
-      vectorizedProperty,
-      existingData
-    );
-
-    const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
-    const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
-    const newDataIsNewChanged: boolean = 
-      (!existingValue && vectorizingPropertyHaveValue) ||
-      !DyFM_Object.sameObjects(existingValue, newValue);
-
-    if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
-      if (!vectorizedProperty.embeddingModel) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorizeDataProperties', 
-            new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
-          ),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.log(
-          `vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
-          (newData as any)?.name ? `: "${(newData as any)?.name}"` : ''
-        );
-      }
-
-      newData[vectorizedProperty.key] = await this.vectorize(
-        newValue,
-        vectorizedProperty.embeddingModel
-      );
-    } else if (existingData?.[vectorizedProperty.key] && !newData?.[vectorizedProperty.key]) {
-      newData[vectorizedProperty.key] = existingData[vectorizedProperty.key];
-    }
-  }
-
-  private async getVectorizingDataForVectorizedProperty(
-    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
-    data: T,
-    /* alwaysVectorize?: boolean */
-  ): Promise<string> {
-    if (vectorizedProperty.vectorizedFrom.length === 1) {
-      const value: unknown = data?.[vectorizedProperty.vectorizedFrom[0]];
-
-      if (value === undefined) {
-        return undefined;
-      } else if (typeof value === 'string') {
-        return value;
-      } else if (Array.isArray(value) || typeof value === 'object') {
-        return JSON.stringify(value);
-      } else {
-        return `${value}`;
-      }
-    } else {
-      const object: any = {};
-
-      vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
-        if (data?.[key] !== undefined) {
-          object[key] = data[key];
-        }
-      });
-
-      return JSON.stringify(object);
-    }
-  }
-
-  /**
-   * Vectorizes the input
-   * 
-   * @example
-   * const vector = await vectorize(input);
-   */
-  async vectorize(
-    input: string, 
-    useModel: DyFM_OAI_Model | string = this.openAISettings.defaultSettings.useModel
-  ): Promise<number[]> {
-    try {
-      if (!input) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('vectorize', new Error('input is required')),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V1`,
-        });
-      }
-
-      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
-        this.openAISettings
-      );
-      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
-        text: input,
-        model: useModel,
-        fullResponse: true,
-        issuer: this.issuer,
-      }) as CreateEmbeddingResponse;
-
-      this.tokenSpent += response.usage.total_tokens;
-
-      return response.data[0].embedding;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorize', error),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V0`,
-      });
-    }
-  }
-
-  /**
-   * Vectorizes the input and searches for the most similar data
-   * (the default limit is 3, and the default numberOfCandidates is 100)
-   * 
-   * @example
-   * const result = await vectorSearch(input, searchInKey);
-   */
-  async vectorSearch(
-    set: {
-      input: string, 
-      /** this should be the vectorized property key */
-      searchInKey: string, 
-      limit?: number, 
-      /**
-       * Number of candidates that are used to find the best match
-       */
-      numberOfCandidates?: number,
-      filterBy?: DyFM_DBFilter<T>,
-    },
-  ): Promise<any> {
-    try {
-      if (!set.input) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('vectorSearch', new Error('input is required')),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS1`,
-        });
-      }
-
-      set.limit ??= 3;
-      set.numberOfCandidates ??= 100;
-
-      const { input, searchInKey, limit, numberOfCandidates, filterBy } = set;
-
-      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
-        this.openAISettings
-      );
-
-      const property: DyFM_DataProperty_Params<any, T> = this.dataParams.properties[searchInKey];
-
-      if (!property) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch', 
-            new Error(
-              `Property "${searchInKey}" not found! ` +
-              `while searching "${this.dataParams.dataName}" ` +
-              `(The searchable properties are: ${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
-            )
-          ),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS2`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      if (!property.vectorizeUseIndex) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch', 
-            new Error(`Property "${searchInKey}" has no vectorizeUseIndex!`)
-          ),
-        });
-      }
-
-      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
-        text: input,
-        model: property.embeddingModel,
-        fullResponse: true,
-        issuer: this.issuer,
-      }) as CreateEmbeddingResponse;
-
-      this.tokenSpent += response.usage.total_tokens;
-
-      const embeddedInput: number[] = response.data[0].embedding;
-      /* const vectorSearchFilter =  */
-      const vectorSearchFilter: any = {
-        $vectorSearch: {
-          /** this is the index name that is created in the DB (in the Atlas)
-           *  also the same that is used in the {@link DyFM_DataProperty_Params} */
-          index: property.vectorizeUseIndex,
-          queryVector: embeddedInput,
-          /** this is the property key that is vectorized and used to search */
-          path: searchInKey,
-          numCandidates: numberOfCandidates,
-          limit: limit,
-        }
-      };
-
-      if (filterBy) {
-        vectorSearchFilter.filter = filterBy;
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.log(`vectorSearch...`);
-      }
-      
-      const result = await this.dataDBService.aggregate([ vectorSearchFilter ]);
-
-      if (this.debugLog) {
-        DyFM_Log.log(`vectorSearch: ${result.length} results`);
-      }
-
-      return result;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorSearch', error),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS0`,
-      });
-    }
-  }
-
-  /**
-   * VectorDataService overrides the saveData function to vectorize the data properties
-   * 
-   * {@link DyNTS_DataService.saveData}
-   */
-  override async saveData(
-    data?: T, 
-    dontSetToService?: boolean, 
-    forceVectorize?: boolean
-  ): Promise<T> {
-    try {
-      data = this.ensureData(data);
-        
-      data = await this.vectorizeDataProperties(data, forceVectorize);
-      
-      data = await super.saveData(data, dontSetToService);
-
-      if (this.swallowVectorizedProperties) {
-        this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-          delete data[property.key];
-        });
-      }
-
-      DyFM_Object.cleanObject(data);
-
-      return data;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'saveData',
-          error
-        ),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-SD0`,
-      });
-    }
-  }
-
-  override getProvidedData(data: T): T {
-    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-      delete data[property.key];
-    });
-
-    return data;
-  }
-}
-
-
-
-
-//import {
-//  DyFM_DataModel_Params,
-//  DyFM_DataProperty_Params,
-//  DyFM_Metadata,
-//} from '@futdevpro/fsm-dynamo';
-//import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-//
-//import { DyNTS_DAI_VectorDataService } from '../../_modules/document-ai/_services/dai-vector-data.service-base';
-//import { DyNTS_OAI_Embedding_Service } from './oai-embedding.service';
-//import { DyNTS_global_settings } from '../../../../_collections/global-settings.const';
-//import { DyNTS_OAI_global_settings } from '../_collections/oai-global-settings.const';
-//
-///**
-// * OpenAI implementation of the generic Vector Data Service
-// * Extends the generic DAI VectorDataService with OpenAI-specific functionality
-// */
-//export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DAI_VectorDataService<T> {
-//
-//  debugLog: boolean = true;
-//  swallowVectorizedProperties: boolean = true;
-//
-//  constructor(
-//    /** Initial data */
-//    data: T,
-//    /** DB data params */
-//    dataParams: DyFM_DataModel_Params<T>,
-//    /** OpenAI settings */
-//    public readonly openAISettings: DyFM_OAI_Settings,
-//    /** Issuer */
-//    issuer: string
-//  ) {
-//    // Create OpenAI embedding service
-//    const embeddingService = new DyNTS_OAI_Embedding_Service(openAISettings);
-//    
-//    super(data, dataParams, embeddingService, issuer);
-//
-//    if (!openAISettings.openAIConfig.apiKey) {
-//      throw new Error('openAIConfig.apiKey is required');
-//    }
-//
-//    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
-//      DyFM_OAI_Model.textEmbedding_3Large;
-//
-//    // Configure vectorized properties with OpenAI-specific settings
-//    this.configureVectorizedProperties(model);
-//  }
-//
-//  /**
-//   * Configure vectorized properties with OpenAI-specific settings
-//   */
-//  private configureVectorizedProperties(model: DyFM_OAI_EmbeddingModel): void {
-//    // Update vectorized properties with OpenAI model info
-//    this.vectorizedProperties.forEach(property => {
-//      if (property.vectorizedFrom) {
-//        // Set the embedding model for this property
-//        (property as any).embeddingModel = model;
-//      }
-//    });
-//  }
-//
-//  /**
-//   * Perform vector search with OpenAI-specific implementation
-//   */
-//  protected override async performVectorSearch(
-//    query: string,
-//    model: string,
-//    issuer: string,
-//    options: any
-//  ): Promise<T[]> {
-//    try {
-//      // Create embedding for the query
-//      const queryEmbedding = await this.embeddingService.createEmbedding(query, model, issuer);
-//      
-//      // Get all data
-//      const allData = await this.getAllData();
-//      
-//      // Calculate similarities (simplified implementation)
-//      const results = allData.map(item => {
-//        const itemEmbedding = (item as any).chunkParentedContentVectorized;
-//        if (!itemEmbedding) return { item, similarity: 0 };
-//        
-//        const similarity = this.calculateCosineSimilarity(queryEmbedding, itemEmbedding);
-//        return { item, similarity };
-//      })
-//      .filter(result => result.similarity >= (options.minSimilarity || 0))
-//      .sort((a, b) => b.similarity - a.similarity)
-//      .slice(0, options.limit || 10)
-//      .map(result => result.item);
-//
-//      return results;
-//    } catch (error) {
-//      throw new Error(`Vector search failed: ${error.message}`);
-//    }
-//  }
-//
-//  /**
-//   * Calculate cosine similarity between two vectors
-//   */
-//  private calculateCosineSimilarity(vecA: number[], vecB: number[]): number {
-//    if (vecA.length !== vecB.length) return 0;
-//    
-//    let dotProduct = 0;
-//    let normA = 0;
-//    let normB = 0;
-//    
-//    for (let i = 0; i < vecA.length; i++) {
-//      dotProduct += vecA[i] * vecB[i];
-//      normA += vecA[i] * vecA[i];
-//      normB += vecB[i] * vecB[i];
-//    }
-//    
-//    if (normA === 0 || normB === 0) return 0;
-//    
-//    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-//  }
-//
-//  /**
-//   * Get default error settings
-//   */
-//  protected getDefaultErrorSettings(
-//    method: string,
-//    error: any,
-//    issuer?: string
-//  ): any {
-//    return {
-//      method,
-//      error,
-//      issuer: issuer || this.issuer,
-//      userMsg: this.defaultErrorUserMsg,
-//    };
-//  }
-//}