@futdevpro/nts-dynamo 1.15.84 → 1.15.86

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.ts

@@ -1,631 +1,631 @@
-import { DyFM_Log } from '@futdevpro/fsm-dynamo';
-
-import {
-  DyFM_Array,
-  DyFM_DataModel_Params,
-  DyFM_DataProperty_Params,
-  DyFM_DBFilter,
-  DyFM_Error,
-  DyFM_Metadata,
-  DyFM_Object,
-} from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { DyNTS_global_settings } from '../../../../../../_collections/global-settings.const';
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_OAI_Embedding_ControlService } from '../oai-embedding.control-service';
-import { DyNTS_OAI_global_settings } from '../../_collections/oai-global-settings.const';
-import { CreateEmbeddingResponse } from 'openai/resources';
-
-/**
- * Vector Data Service that is connected to the relevant DBServices  
- * extends {@link DyNTS_DataService}
- * 
- * For the vectorization, you need to setup the index in the DB to something like this:
- * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
- * ```json
- * {
- *   "fields": [
- *     {
- *       "type": "vector",
- *       "path": "contentVectorized",
- *       "numDimensions": 3072,
- *       "similarity": "cosine"
- *     }
- *   ]
- * }
- * ```
- * 
- * @example
- * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
- *   constructor(
- *     set: {
- *       data?: UserMatchStatistics,
- *       openAISettings?: DyFM_OAI_Settings,
- *       issuer: string,
- *     }
- *   ) {
- *     super(
- *       new UserMatchStatistics(set?.data),
- *       userMatchStatisticsModelParams,
- *       set?.openAISettings ?? defaultOpenAISettings,
- *       set.issuer,
- *     );
- *   }
- * }
- */
-export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DataService<T> {
-
-  /**
-   * Properties that are vectorized
-   * (properties that are marked with vectorizeFrom in other properties)
-   */
-  readonly vectorizeProperties: DyFM_DataProperty_Params<any, T>[];
-  readonly vectorizedProperties: DyFM_DataProperty_Params<any, T>[];
-  /* readonly vectorizeProperties: { from: string, to: string }[]; */
-
-  debugLog: boolean = false;
-  swallowVectorizedProperties: boolean = true;
-
-  embedding_CS: DyNTS_OAI_Embedding_ControlService;
-
-  tokenSpent: number = 0;
-
-  constructor(
-    /**
-     * Initial data, this will be used by functions on default
-     */
-    data: T,
-    /**
-     * DB data prams will be used to connect to usable dbService on GlobalService
-     */
-    dataParams: DyFM_DataModel_Params<T>,
-    /**
-     * OpenAI settings
-     */
-    public readonly openAISettings: DyFM_OAI_Settings,
-    /**
-     * Initial set for issuer to be able to follow the issuer's activity
-     */
-    issuer: string
-  ) {
-    super(data, dataParams, issuer);
-
-    this.vectorizedProperties = Object.values(dataParams.properties).filter(
-      (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom
-    );
-
-    this.vectorizeProperties = Object.keys(dataParams.properties).filter(
-      (key: string): boolean => this.vectorizedProperties.some(
-        (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom.includes(key)
-      )
-    ).map(
-      (key: string): DyFM_DataProperty_Params<any, T> => dataParams.properties[key] as DyFM_DataProperty_Params<any, T>
-    );
-    
-    /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
-      (key: string): boolean => (
-        dataParams.properties[key] as DyFM_DataProperty_Params<any>
-      ).vectorizedFrom &&
-        (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
-    ); */
-
-    if (!openAISettings.config.apiKey) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'constructor', 
-          new Error('openAISettings.config.apiKey is required')
-        ),
-      });
-    }
-
-    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
-      DyFM_OAI_Model.textEmbedding_3Large;
-
-    /* if (
-      !DyFM_allOpenAIEmbeddingModels.includes(model)
-    ) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'constructor', 
-          new Error(`useModel "${model}" is not an embedding model!`)
-        ),
-      });
-    } */
-
-    if (!this.vectorizedProperties.length) {
-      throw new Error(
-        'vectorizedProperties is missing! ' +
-        `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    }
-
-    if (!this.vectorizeProperties.length) {
-      throw new Error(
-        'vectorizeProperties is missing! ' +
-        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    }    
-
-    /* if (!vectorizePropertyKeys.length) {
-      throw new Error(
-        'vectorizedPropertyKeys is missing! ' +
-        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
-          (key: string): boolean => Boolean(
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
-          )
-        ).map(key => `"${key}"`).join(', ')}`
-      );
-    } */
-
-    /* vectorizePropertyKeys.forEach((key: string): void => {
-      dataParams.properties[key].embeddingModel ??= model;
-    }); */
-
-    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-      property.embeddingModel ??= model;
-    });
-
-    /* this.vectorizeProperties = vectorizePropertyKeys.map(
-      (key: string): DyFM_DataProperty_Params<any> => 
-        dataParams.properties[key] as DyFM_DataProperty_Params<any>
-    ); */
-  }
-
-  /**
-   * Vectorizes the properties of the data 
-   * IF they are not vectorized yet OR alwaysVectorize is true
-   * (That are marked with vectorizeFrom if the specific property is not defined)
-   * 
-   * @example
-   * const data = await vectorizeDataProperties(data);
-   * 
-   * @returns The vectorized data
-   */
-  async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
-    try {
-      let existingData: T;
-
-      if (!forceVectorize && newData?._id) {
-        existingData = await this.getDataById(newData._id, true, true);
-      }
-
-      await DyFM_Array.asyncForEachAllAtOnce(
-        this.vectorizedProperties,
-        async (property: DyFM_DataProperty_Params<any, T>) => {
-          await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
-        }
-      );
-
-      return newData;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
-      });
-    }
-  }
-
-  private async checkAndVectorizeSingleProperty(
-    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
-    newData: T,
-    existingData: T,
-    forceVectorize?: boolean
-  ): Promise<void> {
-    const newValue: string = await this.getVectorizingDataForVectorizedProperty(
-      vectorizedProperty,
-      newData
-    );
-    const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
-      vectorizedProperty,
-      existingData
-    );
-
-    const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
-    const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
-    const newDataIsNewChanged: boolean = 
-      (!existingValue && vectorizingPropertyHaveValue) ||
-      !DyFM_Object.sameObjects(existingValue, newValue);
-
-    if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
-      if (!vectorizedProperty.embeddingModel) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorizeDataProperties', 
-            new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
-          ),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.log(
-          `vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
-          (newData as any)?.name ? `: "${(newData as any)?.name}"` : ''
-        );
-      }
-
-      newData[vectorizedProperty.key] = await this.vectorize(
-        newValue,
-        vectorizedProperty.embeddingModel
-      );
-    } else if (existingData?.[vectorizedProperty.key] && !newData?.[vectorizedProperty.key]) {
-      newData[vectorizedProperty.key] = existingData[vectorizedProperty.key];
-    }
-  }
-
-  private async getVectorizingDataForVectorizedProperty(
-    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
-    data: T,
-    /* alwaysVectorize?: boolean */
-  ): Promise<string> {
-    if (vectorizedProperty.vectorizedFrom.length === 1) {
-      const value: unknown = data?.[vectorizedProperty.vectorizedFrom[0]];
-
-      if (value === undefined) {
-        return undefined;
-      } else if (typeof value === 'string') {
-        return value;
-      } else if (Array.isArray(value) || typeof value === 'object') {
-        return JSON.stringify(value);
-      } else {
-        return `${value}`;
-      }
-    } else {
-      const object: any = {};
-
-      vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
-        if (data?.[key] !== undefined) {
-          object[key] = data[key];
-        }
-      });
-
-      return JSON.stringify(object);
-    }
-  }
-
-  /**
-   * Vectorizes the input
-   * 
-   * @example
-   * const vector = await vectorize(input);
-   */
-  async vectorize(
-    input: string, 
-    useModel: DyFM_OAI_Model | string = this.openAISettings.defaultSettings.useModel
-  ): Promise<number[]> {
-    try {
-      if (!input) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('vectorize', new Error('input is required')),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V1`,
-        });
-      }
-
-      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
-        this.openAISettings
-      );
-      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
-        text: input,
-        model: useModel,
-        fullResponse: true,
-        issuer: this.issuer,
-      }) as CreateEmbeddingResponse;
-
-      this.tokenSpent += response.usage.total_tokens;
-
-      return response.data[0].embedding;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorize', error),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V0`,
-      });
-    }
-  }
-
-  /**
-   * Vectorizes the input and searches for the most similar data
-   * (the default limit is 3, and the default numberOfCandidates is 100)
-   * 
-   * @example
-   * const result = await vectorSearch(input, searchInKey);
-   */
-  async vectorSearch(
-    set: {
-      input: string, 
-      /** this should be the vectorized property key */
-      searchInKey: string, 
-      limit?: number, 
-      /**
-       * Number of candidates that are used to find the best match
-       */
-      numberOfCandidates?: number,
-      filterBy?: DyFM_DBFilter<T>,
-    },
-  ): Promise<any> {
-    try {
-      if (!set.input) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings('vectorSearch', new Error('input is required')),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS1`,
-        });
-      }
-
-      set.limit ??= 3;
-      set.numberOfCandidates ??= 100;
-
-      const { input, searchInKey, limit, numberOfCandidates, filterBy } = set;
-
-      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
-        this.openAISettings
-      );
-
-      const property: DyFM_DataProperty_Params<any, T> = this.dataParams.properties[searchInKey];
-
-      if (!property) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch', 
-            new Error(
-              `Property "${searchInKey}" not found! ` +
-              `while searching "${this.dataParams.dataName}" ` +
-              `(The searchable properties are: ${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
-            )
-          ),
-
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS2`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      if (!property.vectorizeUseIndex) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch', 
-            new Error(`Property "${searchInKey}" has no vectorizeUseIndex!`)
-          ),
-        });
-      }
-
-      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
-        text: input,
-        model: property.embeddingModel,
-        fullResponse: true,
-        issuer: this.issuer,
-      }) as CreateEmbeddingResponse;
-
-      this.tokenSpent += response.usage.total_tokens;
-
-      const embeddedInput: number[] = response.data[0].embedding;
-      /* const vectorSearchFilter =  */
-      const vectorSearchFilter: any = {
-        $vectorSearch: {
-          /** this is the index name that is created in the DB (in the Atlas)
-           *  also the same that is used in the {@link DyFM_DataProperty_Params} */
-          index: property.vectorizeUseIndex,
-          queryVector: embeddedInput,
-          /** this is the property key that is vectorized and used to search */
-          path: searchInKey,
-          numCandidates: numberOfCandidates,
-          limit: limit,
-        }
-      };
-
-      if (filterBy) {
-        vectorSearchFilter.filter = filterBy;
-      }
-
-      if (this.debugLog) {
-        DyFM_Log.log(`vectorSearch...`);
-      }
-      
-      const result = await this.dataDBService.aggregate([ vectorSearchFilter ]);
-
-      if (this.debugLog) {
-        DyFM_Log.log(`vectorSearch: ${result.length} results`);
-      }
-
-      return result;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorSearch', error),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS0`,
-      });
-    }
-  }
-
-  /**
-   * VectorDataService overrides the saveData function to vectorize the data properties
-   * 
-   * {@link DyNTS_DataService.saveData}
-   */
-  override async saveData(
-    data?: T, 
-    dontSetToService?: boolean, 
-    forceVectorize?: boolean
-  ): Promise<T> {
-    try {
-      data = this.ensureData(data);
-        
-      data = await this.vectorizeDataProperties(data, forceVectorize);
-      
-      data = await super.saveData(data, dontSetToService);
-
-      if (this.swallowVectorizedProperties) {
-        this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-          delete data[property.key];
-        });
-      }
-
-      DyFM_Object.cleanObject(data);
-
-      return data;
-    } catch (error) {
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings(
-          'saveData',
-          error
-        ),
-
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-SD0`,
-      });
-    }
-  }
-
-  override getProvidedData(data: T): T {
-    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
-      delete data[property.key];
-    });
-
-    return data;
-  }
-}
-
-
-
-
-//import {
-//  DyFM_DataModel_Params,
-//  DyFM_DataProperty_Params,
-//  DyFM_Metadata,
-//} from '@futdevpro/fsm-dynamo';
-//import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-//
-//import { DyNTS_DAI_VectorDataService } from '../../_modules/document-ai/_services/dai-vector-data.service-base';
-//import { DyNTS_OAI_Embedding_Service } from './oai-embedding.service';
-//import { DyNTS_global_settings } from '../../../../_collections/global-settings.const';
-//import { DyNTS_OAI_global_settings } from '../_collections/oai-global-settings.const';
-//
-///**
-// * OpenAI implementation of the generic Vector Data Service
-// * Extends the generic DAI VectorDataService with OpenAI-specific functionality
-// */
-//export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DAI_VectorDataService<T> {
-//
-//  debugLog: boolean = true;
-//  swallowVectorizedProperties: boolean = true;
-//
-//  constructor(
-//    /** Initial data */
-//    data: T,
-//    /** DB data params */
-//    dataParams: DyFM_DataModel_Params<T>,
-//    /** OpenAI settings */
-//    public readonly openAISettings: DyFM_OAI_Settings,
-//    /** Issuer */
-//    issuer: string
-//  ) {
-//    // Create OpenAI embedding service
-//    const embeddingService = new DyNTS_OAI_Embedding_Service(openAISettings);
-//    
-//    super(data, dataParams, embeddingService, issuer);
-//
-//    if (!openAISettings.openAIConfig.apiKey) {
-//      throw new Error('openAIConfig.apiKey is required');
-//    }
-//
-//    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
-//      DyFM_OAI_Model.textEmbedding_3Large;
-//
-//    // Configure vectorized properties with OpenAI-specific settings
-//    this.configureVectorizedProperties(model);
-//  }
-//
-//  /**
-//   * Configure vectorized properties with OpenAI-specific settings
-//   */
-//  private configureVectorizedProperties(model: DyFM_OAI_EmbeddingModel): void {
-//    // Update vectorized properties with OpenAI model info
-//    this.vectorizedProperties.forEach(property => {
-//      if (property.vectorizedFrom) {
-//        // Set the embedding model for this property
-//        (property as any).embeddingModel = model;
-//      }
-//    });
-//  }
-//
-//  /**
-//   * Perform vector search with OpenAI-specific implementation
-//   */
-//  protected override async performVectorSearch(
-//    query: string,
-//    model: string,
-//    issuer: string,
-//    options: any
-//  ): Promise<T[]> {
-//    try {
-//      // Create embedding for the query
-//      const queryEmbedding = await this.embeddingService.createEmbedding(query, model, issuer);
-//      
-//      // Get all data
-//      const allData = await this.getAllData();
-//      
-//      // Calculate similarities (simplified implementation)
-//      const results = allData.map(item => {
-//        const itemEmbedding = (item as any).chunkParentedContentVectorized;
-//        if (!itemEmbedding) return { item, similarity: 0 };
-//        
-//        const similarity = this.calculateCosineSimilarity(queryEmbedding, itemEmbedding);
-//        return { item, similarity };
-//      })
-//      .filter(result => result.similarity >= (options.minSimilarity || 0))
-//      .sort((a, b) => b.similarity - a.similarity)
-//      .slice(0, options.limit || 10)
-//      .map(result => result.item);
-//
-//      return results;
-//    } catch (error) {
-//      throw new Error(`Vector search failed: ${error.message}`);
-//    }
-//  }
-//
-//  /**
-//   * Calculate cosine similarity between two vectors
-//   */
-//  private calculateCosineSimilarity(vecA: number[], vecB: number[]): number {
-//    if (vecA.length !== vecB.length) return 0;
-//    
-//    let dotProduct = 0;
-//    let normA = 0;
-//    let normB = 0;
-//    
-//    for (let i = 0; i < vecA.length; i++) {
-//      dotProduct += vecA[i] * vecB[i];
-//      normA += vecA[i] * vecA[i];
-//      normB += vecB[i] * vecB[i];
-//    }
-//    
-//    if (normA === 0 || normB === 0) return 0;
-//    
-//    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-//  }
-//
-//  /**
-//   * Get default error settings
-//   */
-//  protected getDefaultErrorSettings(
-//    method: string,
-//    error: any,
-//    issuer?: string
-//  ): any {
-//    return {
-//      method,
-//      error,
-//      issuer: issuer || this.issuer,
-//      userMsg: this.defaultErrorUserMsg,
-//    };
-//  }
+import { DyFM_Log } from '@futdevpro/fsm-dynamo';
+
+import {
+  DyFM_Array,
+  DyFM_DataModel_Params,
+  DyFM_DataProperty_Params,
+  DyFM_DBFilter,
+  DyFM_Error,
+  DyFM_Metadata,
+  DyFM_Object,
+} from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { DyNTS_global_settings } from '../../../../../../_collections/global-settings.const';
+import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
+import { DyNTS_OAI_Embedding_ControlService } from '../oai-embedding.control-service';
+import { DyNTS_OAI_global_settings } from '../../_collections/oai-global-settings.const';
+import { CreateEmbeddingResponse } from 'openai/resources';
+
+/**
+ * Vector Data Service that is connected to the relevant DBServices  
+ * extends {@link DyNTS_DataService}
+ * 
+ * For the vectorization, you need to setup the index in the DB to something like this:
+ * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
+ * ```json
+ * {
+ *   "fields": [
+ *     {
+ *       "type": "vector",
+ *       "path": "contentVectorized",
+ *       "numDimensions": 3072,
+ *       "similarity": "cosine"
+ *     }
+ *   ]
+ * }
+ * ```
+ * 
+ * @example
+ * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
+ *   constructor(
+ *     set: {
+ *       data?: UserMatchStatistics,
+ *       openAISettings?: DyFM_OAI_Settings,
+ *       issuer: string,
+ *     }
+ *   ) {
+ *     super(
+ *       new UserMatchStatistics(set?.data),
+ *       userMatchStatisticsModelParams,
+ *       set?.openAISettings ?? defaultOpenAISettings,
+ *       set.issuer,
+ *     );
+ *   }
+ * }
+ */
+export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DataService<T> {
+
+  /**
+   * Properties that are vectorized
+   * (properties that are marked with vectorizeFrom in other properties)
+   */
+  readonly vectorizeProperties: DyFM_DataProperty_Params<any, T>[];
+  readonly vectorizedProperties: DyFM_DataProperty_Params<any, T>[];
+  /* readonly vectorizeProperties: { from: string, to: string }[]; */
+
+  debugLog: boolean = false;
+  swallowVectorizedProperties: boolean = true;
+
+  embedding_CS: DyNTS_OAI_Embedding_ControlService;
+
+  tokenSpent: number = 0;
+
+  constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data: T,
+    /**
+     * DB data prams will be used to connect to usable dbService on GlobalService
+     */
+    dataParams: DyFM_DataModel_Params<T>,
+    /**
+     * OpenAI settings
+     */
+    public readonly openAISettings: DyFM_OAI_Settings,
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer: string
+  ) {
+    super(data, dataParams, issuer);
+
+    this.vectorizedProperties = Object.values(dataParams.properties).filter(
+      (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom
+    );
+
+    this.vectorizeProperties = Object.keys(dataParams.properties).filter(
+      (key: string): boolean => this.vectorizedProperties.some(
+        (property: DyFM_DataProperty_Params<any, T>) => property.vectorizedFrom.includes(key)
+      )
+    ).map(
+      (key: string): DyFM_DataProperty_Params<any, T> => dataParams.properties[key] as DyFM_DataProperty_Params<any, T>
+    );
+    
+    /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
+      (key: string): boolean => (
+        dataParams.properties[key] as DyFM_DataProperty_Params<any>
+      ).vectorizedFrom &&
+        (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
+    ); */
+
+    if (!openAISettings.config.apiKey) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings(
+          'constructor', 
+          new Error('openAISettings.config.apiKey is required')
+        ),
+      });
+    }
+
+    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
+      DyFM_OAI_Model.textEmbedding_3Large;
+
+    /* if (
+      !DyFM_allOpenAIEmbeddingModels.includes(model)
+    ) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings(
+          'constructor', 
+          new Error(`useModel "${model}" is not an embedding model!`)
+        ),
+      });
+    } */
+
+    if (!this.vectorizedProperties.length) {
+      throw new Error(
+        'vectorizedProperties is missing! ' +
+        `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter(
+          (key: string): boolean => Boolean(
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
+          )
+        ).map(key => `"${key}"`).join(', ')}`
+      );
+    }
+
+    if (!this.vectorizeProperties.length) {
+      throw new Error(
+        'vectorizeProperties is missing! ' +
+        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
+          (key: string): boolean => Boolean(
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any, T>).vectorizedFrom
+          )
+        ).map(key => `"${key}"`).join(', ')}`
+      );
+    }    
+
+    /* if (!vectorizePropertyKeys.length) {
+      throw new Error(
+        'vectorizedPropertyKeys is missing! ' +
+        `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
+          (key: string): boolean => Boolean(
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
+          )
+        ).map(key => `"${key}"`).join(', ')}`
+      );
+    } */
+
+    /* vectorizePropertyKeys.forEach((key: string): void => {
+      dataParams.properties[key].embeddingModel ??= model;
+    }); */
+
+    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
+      property.embeddingModel ??= model;
+    });
+
+    /* this.vectorizeProperties = vectorizePropertyKeys.map(
+      (key: string): DyFM_DataProperty_Params<any> => 
+        dataParams.properties[key] as DyFM_DataProperty_Params<any>
+    ); */
+  }
+
+  /**
+   * Vectorizes the properties of the data 
+   * IF they are not vectorized yet OR alwaysVectorize is true
+   * (That are marked with vectorizeFrom if the specific property is not defined)
+   * 
+   * @example
+   * const data = await vectorizeDataProperties(data);
+   * 
+   * @returns The vectorized data
+   */
+  async vectorizeDataProperties(newData?: T, forceVectorize?: boolean): Promise<T> {
+    try {
+      let existingData: T;
+
+      if (!forceVectorize && newData?._id) {
+        existingData = await this.getDataById(newData._id, true, true);
+      }
+
+      await DyFM_Array.asyncForEachAllAtOnce(
+        this.vectorizedProperties,
+        async (property: DyFM_DataProperty_Params<any, T>) => {
+          await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
+        }
+      );
+
+      return newData;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
+      });
+    }
+  }
+
+  private async checkAndVectorizeSingleProperty(
+    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
+    newData: T,
+    existingData: T,
+    forceVectorize?: boolean
+  ): Promise<void> {
+    const newValue: string = await this.getVectorizingDataForVectorizedProperty(
+      vectorizedProperty,
+      newData
+    );
+    const existingValue: string = await this.getVectorizingDataForVectorizedProperty(
+      vectorizedProperty,
+      existingData
+    );
+
+    const vectorizedFrom: string = vectorizedProperty.vectorizedFrom[0];
+    const vectorizingPropertyHaveValue: boolean = Boolean(newData[vectorizedFrom]);
+    const newDataIsNewChanged: boolean = 
+      (!existingValue && vectorizingPropertyHaveValue) ||
+      !DyFM_Object.sameObjects(existingValue, newValue);
+
+    if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
+      if (!vectorizedProperty.embeddingModel) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorizeDataProperties', 
+            new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)
+          ),
+
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
+          (newData as any)?.name ? `: "${(newData as any)?.name}"` : ''
+        );
+      }
+
+      newData[vectorizedProperty.key] = await this.vectorize(
+        newValue,
+        vectorizedProperty.embeddingModel
+      );
+    } else if (existingData?.[vectorizedProperty.key] && !newData?.[vectorizedProperty.key]) {
+      newData[vectorizedProperty.key] = existingData[vectorizedProperty.key];
+    }
+  }
+
+  private async getVectorizingDataForVectorizedProperty(
+    vectorizedProperty: DyFM_DataProperty_Params<any, T>,
+    data: T,
+    /* alwaysVectorize?: boolean */
+  ): Promise<string> {
+    if (vectorizedProperty.vectorizedFrom.length === 1) {
+      const value: unknown = data?.[vectorizedProperty.vectorizedFrom[0]];
+
+      if (value === undefined) {
+        return undefined;
+      } else if (typeof value === 'string') {
+        return value;
+      } else if (Array.isArray(value) || typeof value === 'object') {
+        return JSON.stringify(value);
+      } else {
+        return `${value}`;
+      }
+    } else {
+      const object: any = {};
+
+      vectorizedProperty.vectorizedFrom.forEach((key: string): void => {
+        if (data?.[key] !== undefined) {
+          object[key] = data[key];
+        }
+      });
+
+      return JSON.stringify(object);
+    }
+  }
+
+  /**
+   * Vectorizes the input
+   * 
+   * @example
+   * const vector = await vectorize(input);
+   */
+  async vectorize(
+    input: string, 
+    useModel: DyFM_OAI_Model | string = this.openAISettings.defaultSettings.useModel
+  ): Promise<number[]> {
+    try {
+      if (!input) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('vectorize', new Error('input is required')),
+
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V1`,
+        });
+      }
+
+      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
+        this.openAISettings
+      );
+      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
+        text: input,
+        model: useModel,
+        fullResponse: true,
+        issuer: this.issuer,
+      }) as CreateEmbeddingResponse;
+
+      this.tokenSpent += response.usage.total_tokens;
+
+      return response.data[0].embedding;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorize', error),
+
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V0`,
+      });
+    }
+  }
+
+  /**
+   * Vectorizes the input and searches for the most similar data
+   * (the default limit is 3, and the default numberOfCandidates is 100)
+   * 
+   * @example
+   * const result = await vectorSearch(input, searchInKey);
+   */
+  async vectorSearch(
+    set: {
+      input: string, 
+      /** this should be the vectorized property key */
+      searchInKey: string, 
+      limit?: number, 
+      /**
+       * Number of candidates that are used to find the best match
+       */
+      numberOfCandidates?: number,
+      filterBy?: DyFM_DBFilter<T>,
+    },
+  ): Promise<any> {
+    try {
+      if (!set.input) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings('vectorSearch', new Error('input is required')),
+
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS1`,
+        });
+      }
+
+      set.limit ??= 3;
+      set.numberOfCandidates ??= 100;
+
+      const { input, searchInKey, limit, numberOfCandidates, filterBy } = set;
+
+      this.embedding_CS ??= new DyNTS_OAI_Embedding_ControlService(
+        this.openAISettings
+      );
+
+      const property: DyFM_DataProperty_Params<any, T> = this.dataParams.properties[searchInKey];
+
+      if (!property) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch', 
+            new Error(
+              `Property "${searchInKey}" not found! ` +
+              `while searching "${this.dataParams.dataName}" ` +
+              `(The searchable properties are: ${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS2`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      if (!property.vectorizeUseIndex) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch', 
+            new Error(`Property "${searchInKey}" has no vectorizeUseIndex!`)
+          ),
+        });
+      }
+
+      const response: CreateEmbeddingResponse = await this.embedding_CS.createEmbedding({
+        text: input,
+        model: property.embeddingModel,
+        fullResponse: true,
+        issuer: this.issuer,
+      }) as CreateEmbeddingResponse;
+
+      this.tokenSpent += response.usage.total_tokens;
+
+      const embeddedInput: number[] = response.data[0].embedding;
+      /* const vectorSearchFilter =  */
+      const vectorSearchFilter: any = {
+        $vectorSearch: {
+          /** this is the index name that is created in the DB (in the Atlas)
+           *  also the same that is used in the {@link DyFM_DataProperty_Params} */
+          index: property.vectorizeUseIndex,
+          queryVector: embeddedInput,
+          /** this is the property key that is vectorized and used to search */
+          path: searchInKey,
+          numCandidates: numberOfCandidates,
+          limit: limit,
+        }
+      };
+
+      if (filterBy) {
+        vectorSearchFilter.filter = filterBy;
+      }
+
+      if (this.debugLog) {
+        DyFM_Log.log(`vectorSearch...`);
+      }
+      
+      const result = await this.dataDBService.aggregate([ vectorSearchFilter ]);
+
+      if (this.debugLog) {
+        DyFM_Log.log(`vectorSearch: ${result.length} results`);
+      }
+
+      return result;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorSearch', error),
+
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS0`,
+      });
+    }
+  }
+
+  /**
+   * VectorDataService overrides the saveData function to vectorize the data properties
+   * 
+   * {@link DyNTS_DataService.saveData}
+   */
+  override async saveData(
+    data?: T, 
+    dontSetToService?: boolean, 
+    forceVectorize?: boolean
+  ): Promise<T> {
+    try {
+      data = this.ensureData(data);
+        
+      data = await this.vectorizeDataProperties(data, forceVectorize);
+      
+      data = await super.saveData(data, dontSetToService);
+
+      if (this.swallowVectorizedProperties) {
+        this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
+          delete data[property.key];
+        });
+      }
+
+      DyFM_Object.cleanObject(data);
+
+      return data;
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings(
+          'saveData',
+          error
+        ),
+
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-SD0`,
+      });
+    }
+  }
+
+  override getProvidedData(data: T): T {
+    this.vectorizedProperties.forEach((property: DyFM_DataProperty_Params<any, T>): void => {
+      delete data[property.key];
+    });
+
+    return data;
+  }
+}
+
+
+
+
+//import {
+//  DyFM_DataModel_Params,
+//  DyFM_DataProperty_Params,
+//  DyFM_Metadata,
+//} from '@futdevpro/fsm-dynamo';
+//import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+//
+//import { DyNTS_DAI_VectorDataService } from '../../_modules/document-ai/_services/dai-vector-data.service-base';
+//import { DyNTS_OAI_Embedding_Service } from './oai-embedding.service';
+//import { DyNTS_global_settings } from '../../../../_collections/global-settings.const';
+//import { DyNTS_OAI_global_settings } from '../_collections/oai-global-settings.const';
+//
+///**
+// * OpenAI implementation of the generic Vector Data Service
+// * Extends the generic DAI VectorDataService with OpenAI-specific functionality
+// */
+//export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DAI_VectorDataService<T> {
+//
+//  debugLog: boolean = true;
+//  swallowVectorizedProperties: boolean = true;
+//
+//  constructor(
+//    /** Initial data */
+//    data: T,
+//    /** DB data params */
+//    dataParams: DyFM_DataModel_Params<T>,
+//    /** OpenAI settings */
+//    public readonly openAISettings: DyFM_OAI_Settings,
+//    /** Issuer */
+//    issuer: string
+//  ) {
+//    // Create OpenAI embedding service
+//    const embeddingService = new DyNTS_OAI_Embedding_Service(openAISettings);
+//    
+//    super(data, dataParams, embeddingService, issuer);
+//
+//    if (!openAISettings.openAIConfig.apiKey) {
+//      throw new Error('openAIConfig.apiKey is required');
+//    }
+//
+//    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
+//      DyFM_OAI_Model.textEmbedding_3Large;
+//
+//    // Configure vectorized properties with OpenAI-specific settings
+//    this.configureVectorizedProperties(model);
+//  }
+//
+//  /**
+//   * Configure vectorized properties with OpenAI-specific settings
+//   */
+//  private configureVectorizedProperties(model: DyFM_OAI_EmbeddingModel): void {
+//    // Update vectorized properties with OpenAI model info
+//    this.vectorizedProperties.forEach(property => {
+//      if (property.vectorizedFrom) {
+//        // Set the embedding model for this property
+//        (property as any).embeddingModel = model;
+//      }
+//    });
+//  }
+//
+//  /**
+//   * Perform vector search with OpenAI-specific implementation
+//   */
+//  protected override async performVectorSearch(
+//    query: string,
+//    model: string,
+//    issuer: string,
+//    options: any
+//  ): Promise<T[]> {
+//    try {
+//      // Create embedding for the query
+//      const queryEmbedding = await this.embeddingService.createEmbedding(query, model, issuer);
+//      
+//      // Get all data
+//      const allData = await this.getAllData();
+//      
+//      // Calculate similarities (simplified implementation)
+//      const results = allData.map(item => {
+//        const itemEmbedding = (item as any).chunkParentedContentVectorized;
+//        if (!itemEmbedding) return { item, similarity: 0 };
+//        
+//        const similarity = this.calculateCosineSimilarity(queryEmbedding, itemEmbedding);
+//        return { item, similarity };
+//      })
+//      .filter(result => result.similarity >= (options.minSimilarity || 0))
+//      .sort((a, b) => b.similarity - a.similarity)
+//      .slice(0, options.limit || 10)
+//      .map(result => result.item);
+//
+//      return results;
+//    } catch (error) {
+//      throw new Error(`Vector search failed: ${error.message}`);
+//    }
+//  }
+//
+//  /**
+//   * Calculate cosine similarity between two vectors
+//   */
+//  private calculateCosineSimilarity(vecA: number[], vecB: number[]): number {
+//    if (vecA.length !== vecB.length) return 0;
+//    
+//    let dotProduct = 0;
+//    let normA = 0;
+//    let normB = 0;
+//    
+//    for (let i = 0; i < vecA.length; i++) {
+//      dotProduct += vecA[i] * vecB[i];
+//      normA += vecA[i] * vecA[i];
+//      normB += vecB[i] * vecB[i];
+//    }
+//    
+//    if (normA === 0 || normB === 0) return 0;
+//    
+//    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+//  }
+//
+//  /**
+//   * Get default error settings
+//   */
+//  protected getDefaultErrorSettings(
+//    method: string,
+//    error: any,
+//    issuer?: string
+//  ): any {
+//    return {
+//      method,
+//      error,
+//      issuer: issuer || this.issuer,
+//      userMsg: this.defaultErrorUserMsg,
+//    };
+//  }
 //}