@futdevpro/nts-dynamo 1.15.156 → 1.15.160

package/build/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.js

@@ -1,481 +1,481 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DyNTS_OAI_VectorDataService = void 0;
-const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
-const fsm_dynamo_2 = require("@futdevpro/fsm-dynamo");
-const open_ai_1 = require("@futdevpro/fsm-dynamo/ai/open-ai");
-const global_settings_const_1 = require("../../../../../../_collections/global-settings.const");
-const data_service_1 = require("../../../../../../_services/base/data.service");
-const oai_embedding_control_service_1 = require("../oai-embedding.control-service");
-/**
- * Vector Data Service that is connected to the relevant DBServices
- * extends {@link DyNTS_DataService}
- *
- * For the vectorization, you need to setup the index in the DB to something like this:
- * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
- * ```json
- * {
- *   "fields": [
- *     {
- *       "type": "vector",
- *       "path": "contentVectorized",
- *       "numDimensions": 3072,
- *       "similarity": "cosine"
- *     }
- *   ]
- * }
- * ```
- *
- * @example
- * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
- *   constructor(
- *     set: {
- *       data?: UserMatchStatistics,
- *       openAISettings?: DyFM_OAI_Settings,
- *       issuer: string,
- *     }
- *   ) {
- *     super(
- *       new UserMatchStatistics(set?.data),
- *       userMatchStatisticsModelParams,
- *       set?.openAISettings ?? defaultOpenAISettings,
- *       set.issuer,
- *     );
- *   }
- * }
- */
-class DyNTS_OAI_VectorDataService extends data_service_1.DyNTS_DataService {
-    openAISettings;
-    /**
-     * Properties that are vectorized
-     * (properties that are marked with vectorizeFrom in other properties)
-     */
-    vectorizeProperties;
-    vectorizedProperties;
-    /* readonly vectorizeProperties: { from: string, to: string }[]; */
-    debugLog = false;
-    swallowVectorizedProperties = true;
-    embedding_CS;
-    tokenSpent = 0;
-    constructor(
-    /**
-     * Initial data, this will be used by functions on default
-     */
-    data, 
-    /**
-     * DB data prams will be used to connect to usable dbService on GlobalService
-     */
-    dataParams, 
-    /**
-     * OpenAI settings
-     */
-    openAISettings, 
-    /**
-     * Initial set for issuer to be able to follow the issuer's activity
-     */
-    issuer) {
-        super(data, dataParams, issuer);
-        this.openAISettings = openAISettings;
-        this.vectorizedProperties = Object.values(dataParams.properties).filter((property) => property.vectorizedFrom);
-        this.vectorizeProperties = Object.keys(dataParams.properties).filter((key) => this.vectorizedProperties.some((property) => property.vectorizedFrom.includes(key))).map((key) => dataParams.properties[key]);
-        /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
-          (key: string): boolean => (
-            dataParams.properties[key] as DyFM_DataProperty_Params<any>
-          ).vectorizedFrom &&
-            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
-        ); */
-        if (!openAISettings.config.apiKey) {
-            throw new fsm_dynamo_2.DyFM_Error({
-                ...this.getDefaultErrorSettings('constructor', new Error('openAISettings.config.apiKey is required')),
-            });
-        }
-        const model = openAISettings.defaultSettings.useModel ??
-            open_ai_1.DyFM_OAI_Model.textEmbedding_3Large;
-        /* if (
-          !DyFM_allOpenAIEmbeddingModels.includes(model)
-        ) {
-          throw new DyFM_Error({
-            ...this.getDefaultErrorSettings(
-              'constructor',
-              new Error(`useModel "${model}" is not an embedding model!`)
-            ),
-          });
-        } */
-        if (!this.vectorizedProperties.length) {
-            throw new Error('vectorizedProperties is missing! ' +
-                `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter((key) => Boolean(dataParams.properties[key].vectorizedFrom)).map(key => `"${key}"`).join(', ')}`);
-        }
-        if (!this.vectorizeProperties.length) {
-            throw new Error('vectorizeProperties is missing! ' +
-                `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter((key) => Boolean(dataParams.properties[key].vectorizedFrom)).map(key => `"${key}"`).join(', ')}`);
-        }
-        /* if (!vectorizePropertyKeys.length) {
-          throw new Error(
-            'vectorizedPropertyKeys is missing! ' +
-            `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
-              (key: string): boolean => Boolean(
-                (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
-              )
-            ).map(key => `"${key}"`).join(', ')}`
-          );
-        } */
-        /* vectorizePropertyKeys.forEach((key: string): void => {
-          dataParams.properties[key].embeddingModel ??= model;
-        }); */
-        this.vectorizedProperties.forEach((property) => {
-            property.embeddingModel ??= model;
-        });
-        /* this.vectorizeProperties = vectorizePropertyKeys.map(
-          (key: string): DyFM_DataProperty_Params<any> =>
-            dataParams.properties[key] as DyFM_DataProperty_Params<any>
-        ); */
-    }
-    /**
-     * Vectorizes the properties of the data
-     * IF they are not vectorized yet OR alwaysVectorize is true
-     * (That are marked with vectorizeFrom if the specific property is not defined)
-     *
-     * @example
-     * const data = await vectorizeDataProperties(data);
-     *
-     * @returns The vectorized data
-     */
-    async vectorizeDataProperties(newData, forceVectorize) {
-        try {
-            let existingData;
-            if (!forceVectorize && newData?._id) {
-                existingData = await this.getDataById(newData._id, true, true);
-            }
-            await fsm_dynamo_2.DyFM_Array.asyncForEachAllAtOnce(this.vectorizedProperties, async (property) => {
-                await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
-            });
-            return newData;
-        }
-        catch (error) {
-            throw new fsm_dynamo_2.DyFM_Error({
-                ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
-            });
-        }
-    }
-    async checkAndVectorizeSingleProperty(vectorizedProperty, newData, existingData, forceVectorize) {
-        const newValue = await this.getVectorizingDataForVectorizedProperty(vectorizedProperty, newData);
-        const existingValue = await this.getVectorizingDataForVectorizedProperty(vectorizedProperty, existingData);
-        const vectorizedFrom = vectorizedProperty.vectorizedFrom[0];
-        const vectorizingPropertyHaveValue = Boolean(newData[vectorizedFrom]);
-        const newDataIsNewChanged = (!existingValue && vectorizingPropertyHaveValue) ||
-            !fsm_dynamo_2.DyFM_Object.sameObjects(existingValue, newValue);
-        if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
-            if (!vectorizedProperty.embeddingModel) {
-                throw new fsm_dynamo_2.DyFM_Error({
-                    ...this.getDefaultErrorSettings('vectorizeDataProperties', new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)),
-                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
-                    __localStack: this.dataParams.stackLocation,
-                });
-            }
-            if (this.debugLog) {
-                fsm_dynamo_1.DyFM_Log.log(`vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
-                    newData?.name ? `: "${newData?.name}"` : '');
-            }
-            newData[vectorizedProperty.key] = await this.vectorize(newValue, vectorizedProperty.embeddingModel);
-        }
-        else if (existingData?.[vectorizedProperty.key] && !newData?.[vectorizedProperty.key]) {
-            newData[vectorizedProperty.key] = existingData[vectorizedProperty.key];
-        }
-    }
-    async getVectorizingDataForVectorizedProperty(vectorizedProperty, data) {
-        if (vectorizedProperty.vectorizedFrom.length === 1) {
-            const value = data?.[vectorizedProperty.vectorizedFrom[0]];
-            if (value === undefined) {
-                return undefined;
-            }
-            else if (typeof value === 'string') {
-                return value;
-            }
-            else if (Array.isArray(value) || typeof value === 'object') {
-                return JSON.stringify(value);
-            }
-            else {
-                return `${value}`;
-            }
-        }
-        else {
-            const object = {};
-            vectorizedProperty.vectorizedFrom.forEach((key) => {
-                if (data?.[key] !== undefined) {
-                    object[key] = data[key];
-                }
-            });
-            return JSON.stringify(object);
-        }
-    }
-    /**
-     * Vectorizes the input
-     *
-     * @example
-     * const vector = await vectorize(input);
-     */
-    async vectorize(input, useModel = this.openAISettings.defaultSettings.useModel) {
-        try {
-            if (!input) {
-                throw new fsm_dynamo_2.DyFM_Error({
-                    ...this.getDefaultErrorSettings('vectorize', new Error('input is required')),
-                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V1`,
-                });
-            }
-            this.embedding_CS ??= new oai_embedding_control_service_1.DyNTS_OAI_Embedding_ControlService(this.openAISettings);
-            const response = await this.embedding_CS.createEmbedding({
-                text: input,
-                model: useModel,
-                fullResponse: true,
-                issuer: this.issuer,
-            });
-            this.tokenSpent += response.usage.total_tokens;
-            return response.data[0].embedding;
-        }
-        catch (error) {
-            throw new fsm_dynamo_2.DyFM_Error({
-                ...this.getDefaultErrorSettings('vectorize', error),
-                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V0`,
-            });
-        }
-    }
-    /**
-     * Vectorizes the input and searches for the most similar data
-     * (the default limit is 3, and the default numberOfCandidates is 100)
-     *
-     * @example
-     * const result = await vectorSearch(input, searchInKey);
-     */
-    async vectorSearch(set) {
-        try {
-            if (!set.input) {
-                throw new fsm_dynamo_2.DyFM_Error({
-                    ...this.getDefaultErrorSettings('vectorSearch', new Error('input is required')),
-                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS1`,
-                });
-            }
-            set.limit ??= 3;
-            set.numberOfCandidates ??= 100;
-            const { input, searchInKey, limit, numberOfCandidates, filterBy } = set;
-            this.embedding_CS ??= new oai_embedding_control_service_1.DyNTS_OAI_Embedding_ControlService(this.openAISettings);
-            const property = this.dataParams.properties[searchInKey];
-            if (!property) {
-                throw new fsm_dynamo_2.DyFM_Error({
-                    ...this.getDefaultErrorSettings('vectorSearch', new Error(`Property "${searchInKey}" not found! ` +
-                        `while searching "${this.dataParams.dataName}" ` +
-                        `(The searchable properties are: ${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`)),
-                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS2`,
-                    __localStack: this.dataParams.stackLocation,
-                });
-            }
-            if (!property.vectorizeUseIndex) {
-                throw new fsm_dynamo_2.DyFM_Error({
-                    ...this.getDefaultErrorSettings('vectorSearch', new Error(`Property "${searchInKey}" has no vectorizeUseIndex!`)),
-                });
-            }
-            const response = await this.embedding_CS.createEmbedding({
-                text: input,
-                model: property.embeddingModel,
-                fullResponse: true,
-                issuer: this.issuer,
-            });
-            this.tokenSpent += response.usage.total_tokens;
-            const embeddedInput = response.data[0].embedding;
-            /* const vectorSearchFilter =  */
-            const vectorSearchFilter = {
-                $vectorSearch: {
-                    /** this is the index name that is created in the DB (in the Atlas)
-                     *  also the same that is used in the {@link DyFM_DataProperty_Params} */
-                    index: property.vectorizeUseIndex,
-                    queryVector: embeddedInput,
-                    /** this is the property key that is vectorized and used to search */
-                    path: searchInKey,
-                    numCandidates: numberOfCandidates,
-                    limit: limit,
-                }
-            };
-            if (filterBy) {
-                vectorSearchFilter.filter = filterBy;
-            }
-            if (this.debugLog) {
-                fsm_dynamo_1.DyFM_Log.log(`vectorSearch...`);
-            }
-            const result = await this.dataDBService.aggregate([vectorSearchFilter]);
-            if (this.debugLog) {
-                fsm_dynamo_1.DyFM_Log.log(`vectorSearch: ${result.length} results`);
-            }
-            return result;
-        }
-        catch (error) {
-            throw new fsm_dynamo_2.DyFM_Error({
-                ...this.getDefaultErrorSettings('vectorSearch', error),
-                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS0`,
-            });
-        }
-    }
-    /**
-     * VectorDataService overrides the saveData function to vectorize the data properties
-     *
-     * {@link DyNTS_DataService.saveData}
-     */
-    async saveData(data, dontSetToService, forceVectorize) {
-        try {
-            data = this.ensureData(data);
-            data = await this.vectorizeDataProperties(data, forceVectorize);
-            data = await super.saveData(data, dontSetToService);
-            if (this.swallowVectorizedProperties) {
-                this.vectorizedProperties.forEach((property) => {
-                    delete data[property.key];
-                });
-            }
-            fsm_dynamo_2.DyFM_Object.cleanObject(data);
-            return data;
-        }
-        catch (error) {
-            throw new fsm_dynamo_2.DyFM_Error({
-                ...this.getDefaultErrorSettings('saveData', error),
-                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-SD0`,
-            });
-        }
-    }
-    getProvidedData(data) {
-        this.vectorizedProperties.forEach((property) => {
-            delete data[property.key];
-        });
-        return data;
-    }
-}
-exports.DyNTS_OAI_VectorDataService = DyNTS_OAI_VectorDataService;
-//import {
-//  DyFM_DataModel_Params,
-//  DyFM_DataProperty_Params,
-//  DyFM_Metadata,
-//} from '@futdevpro/fsm-dynamo';
-//import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-//
-//import { DyNTS_DAI_VectorDataService } from '../../_modules/document-ai/_services/dai-vector-data.service-base';
-//import { DyNTS_OAI_Embedding_Service } from './oai-embedding.service';
-//import { DyNTS_global_settings } from '../../../../_collections/global-settings.const';
-//import { DyNTS_OAI_global_settings } from '../_collections/oai-global-settings.const';
-//
-///**
-// * OpenAI implementation of the generic Vector Data Service
-// * Extends the generic DAI VectorDataService with OpenAI-specific functionality
-// */
-//export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DAI_VectorDataService<T> {
-//
-//  debugLog: boolean = true;
-//  swallowVectorizedProperties: boolean = true;
-//
-//  constructor(
-//    /** Initial data */
-//    data: T,
-//    /** DB data params */
-//    dataParams: DyFM_DataModel_Params<T>,
-//    /** OpenAI settings */
-//    public readonly openAISettings: DyFM_OAI_Settings,
-//    /** Issuer */
-//    issuer: string
-//  ) {
-//    // Create OpenAI embedding service
-//    const embeddingService = new DyNTS_OAI_Embedding_Service(openAISettings);
-//    
-//    super(data, dataParams, embeddingService, issuer);
-//
-//    if (!openAISettings.openAIConfig.apiKey) {
-//      throw new Error('openAIConfig.apiKey is required');
-//    }
-//
-//    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
-//      DyFM_OAI_Model.textEmbedding_3Large;
-//
-//    // Configure vectorized properties with OpenAI-specific settings
-//    this.configureVectorizedProperties(model);
-//  }
-//
-//  /**
-//   * Configure vectorized properties with OpenAI-specific settings
-//   */
-//  private configureVectorizedProperties(model: DyFM_OAI_EmbeddingModel): void {
-//    // Update vectorized properties with OpenAI model info
-//    this.vectorizedProperties.forEach(property => {
-//      if (property.vectorizedFrom) {
-//        // Set the embedding model for this property
-//        (property as any).embeddingModel = model;
-//      }
-//    });
-//  }
-//
-//  /**
-//   * Perform vector search with OpenAI-specific implementation
-//   */
-//  protected override async performVectorSearch(
-//    query: string,
-//    model: string,
-//    issuer: string,
-//    options: any
-//  ): Promise<T[]> {
-//    try {
-//      // Create embedding for the query
-//      const queryEmbedding = await this.embeddingService.createEmbedding(query, model, issuer);
-//      
-//      // Get all data
-//      const allData = await this.getAllData();
-//      
-//      // Calculate similarities (simplified implementation)
-//      const results = allData.map(item => {
-//        const itemEmbedding = (item as any).chunkParentedContentVectorized;
-//        if (!itemEmbedding) return { item, similarity: 0 };
-//        
-//        const similarity = this.calculateCosineSimilarity(queryEmbedding, itemEmbedding);
-//        return { item, similarity };
-//      })
-//      .filter(result => result.similarity >= (options.minSimilarity || 0))
-//      .sort((a, b) => b.similarity - a.similarity)
-//      .slice(0, options.limit || 10)
-//      .map(result => result.item);
-//
-//      return results;
-//    } catch (error) {
-//      throw new Error(`Vector search failed: ${error.message}`);
-//    }
-//  }
-//
-//  /**
-//   * Calculate cosine similarity between two vectors
-//   */
-//  private calculateCosineSimilarity(vecA: number[], vecB: number[]): number {
-//    if (vecA.length !== vecB.length) return 0;
-//    
-//    let dotProduct = 0;
-//    let normA = 0;
-//    let normB = 0;
-//    
-//    for (let i = 0; i < vecA.length; i++) {
-//      dotProduct += vecA[i] * vecB[i];
-//      normA += vecA[i] * vecA[i];
-//      normB += vecB[i] * vecB[i];
-//    }
-//    
-//    if (normA === 0 || normB === 0) return 0;
-//    
-//    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-//  }
-//
-//  /**
-//   * Get default error settings
-//   */
-//  protected getDefaultErrorSettings(
-//    method: string,
-//    error: any,
-//    issuer?: string
-//  ): any {
-//    return {
-//      method,
-//      error,
-//      issuer: issuer || this.issuer,
-//      userMsg: this.defaultErrorUserMsg,
-//    };
-//  }
-//}
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DyNTS_OAI_VectorDataService = void 0;
+const fsm_dynamo_1 = require("@futdevpro/fsm-dynamo");
+const fsm_dynamo_2 = require("@futdevpro/fsm-dynamo");
+const open_ai_1 = require("@futdevpro/fsm-dynamo/ai/open-ai");
+const global_settings_const_1 = require("../../../../../../_collections/global-settings.const");
+const data_service_1 = require("../../../../../../_services/base/data.service");
+const oai_embedding_control_service_1 = require("../oai-embedding.control-service");
+/**
+ * Vector Data Service that is connected to the relevant DBServices
+ * extends {@link DyNTS_DataService}
+ *
+ * For the vectorization, you need to setup the index in the DB to something like this:
+ * (the path declared here must be the same as the vectorizeUseIndex in the {@link DyFM_DataProperty_Params})
+ * ```json
+ * {
+ *   "fields": [
+ *     {
+ *       "type": "vector",
+ *       "path": "contentVectorized",
+ *       "numDimensions": 3072,
+ *       "similarity": "cosine"
+ *     }
+ *   ]
+ * }
+ * ```
+ *
+ * @example
+ * export class UserMatchStatisticsService extends DyNTS_DataService<UserMatchStatistics> {
+ *   constructor(
+ *     set: {
+ *       data?: UserMatchStatistics,
+ *       openAISettings?: DyFM_OAI_Settings,
+ *       issuer: string,
+ *     }
+ *   ) {
+ *     super(
+ *       new UserMatchStatistics(set?.data),
+ *       userMatchStatisticsModelParams,
+ *       set?.openAISettings ?? defaultOpenAISettings,
+ *       set.issuer,
+ *     );
+ *   }
+ * }
+ */
+class DyNTS_OAI_VectorDataService extends data_service_1.DyNTS_DataService {
+    openAISettings;
+    /**
+     * Properties that are vectorized
+     * (properties that are marked with vectorizeFrom in other properties)
+     */
+    vectorizeProperties;
+    vectorizedProperties;
+    /* readonly vectorizeProperties: { from: string, to: string }[]; */
+    debugLog = false;
+    swallowVectorizedProperties = true;
+    embedding_CS;
+    tokenSpent = 0;
+    constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data, 
+    /**
+     * DB data prams will be used to connect to usable dbService on GlobalService
+     */
+    dataParams, 
+    /**
+     * OpenAI settings
+     */
+    openAISettings, 
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer) {
+        super(data, dataParams, issuer);
+        this.openAISettings = openAISettings;
+        this.vectorizedProperties = Object.values(dataParams.properties).filter((property) => property.vectorizedFrom);
+        this.vectorizeProperties = Object.keys(dataParams.properties).filter((key) => this.vectorizedProperties.some((property) => property.vectorizedFrom.includes(key))).map((key) => dataParams.properties[key]);
+        /* const vectorizePropertyKeys: string[] = Object.keys(dataParams.properties).filter(
+          (key: string): boolean => (
+            dataParams.properties[key] as DyFM_DataProperty_Params<any>
+          ).vectorizedFrom &&
+            (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom !== key
+        ); */
+        if (!openAISettings.config.apiKey) {
+            throw new fsm_dynamo_2.DyFM_Error({
+                ...this.getDefaultErrorSettings('constructor', new Error('openAISettings.config.apiKey is required')),
+            });
+        }
+        const model = openAISettings.defaultSettings.useModel ??
+            open_ai_1.DyFM_OAI_Model.textEmbedding_3Large;
+        /* if (
+          !DyFM_allOpenAIEmbeddingModels.includes(model)
+        ) {
+          throw new DyFM_Error({
+            ...this.getDefaultErrorSettings(
+              'constructor',
+              new Error(`useModel "${model}" is not an embedding model!`)
+            ),
+          });
+        } */
+        if (!this.vectorizedProperties.length) {
+            throw new Error('vectorizedProperties is missing! ' +
+                `\nkeys with vectorizedFrom: ${Object.keys(dataParams.properties).filter((key) => Boolean(dataParams.properties[key].vectorizedFrom)).map(key => `"${key}"`).join(', ')}`);
+        }
+        if (!this.vectorizeProperties.length) {
+            throw new Error('vectorizeProperties is missing! ' +
+                `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter((key) => Boolean(dataParams.properties[key].vectorizedFrom)).map(key => `"${key}"`).join(', ')}`);
+        }
+        /* if (!vectorizePropertyKeys.length) {
+          throw new Error(
+            'vectorizedPropertyKeys is missing! ' +
+            `\nkeys with vectorizeFrom: ${Object.keys(dataParams.properties).filter(
+              (key: string): boolean => Boolean(
+                (dataParams.properties[key] as DyFM_DataProperty_Params<any>).vectorizedFrom
+              )
+            ).map(key => `"${key}"`).join(', ')}`
+          );
+        } */
+        /* vectorizePropertyKeys.forEach((key: string): void => {
+          dataParams.properties[key].embeddingModel ??= model;
+        }); */
+        this.vectorizedProperties.forEach((property) => {
+            property.embeddingModel ??= model;
+        });
+        /* this.vectorizeProperties = vectorizePropertyKeys.map(
+          (key: string): DyFM_DataProperty_Params<any> =>
+            dataParams.properties[key] as DyFM_DataProperty_Params<any>
+        ); */
+    }
+    /**
+     * Vectorizes the properties of the data
+     * IF they are not vectorized yet OR alwaysVectorize is true
+     * (That are marked with vectorizeFrom if the specific property is not defined)
+     *
+     * @example
+     * const data = await vectorizeDataProperties(data);
+     *
+     * @returns The vectorized data
+     */
+    async vectorizeDataProperties(newData, forceVectorize) {
+        try {
+            let existingData;
+            if (!forceVectorize && newData?._id) {
+                existingData = await this.getDataById(newData._id, true, true);
+            }
+            await fsm_dynamo_2.DyFM_Array.asyncForEachAllAtOnce(this.vectorizedProperties, async (property) => {
+                await this.checkAndVectorizeSingleProperty(property, newData, existingData, forceVectorize);
+            });
+            return newData;
+        }
+        catch (error) {
+            throw new fsm_dynamo_2.DyFM_Error({
+                ...this.getDefaultErrorSettings('vectorizeDataProperties', error),
+            });
+        }
+    }
+    async checkAndVectorizeSingleProperty(vectorizedProperty, newData, existingData, forceVectorize) {
+        const newValue = await this.getVectorizingDataForVectorizedProperty(vectorizedProperty, newData);
+        const existingValue = await this.getVectorizingDataForVectorizedProperty(vectorizedProperty, existingData);
+        const vectorizedFrom = vectorizedProperty.vectorizedFrom[0];
+        const vectorizingPropertyHaveValue = Boolean(newData[vectorizedFrom]);
+        const newDataIsNewChanged = (!existingValue && vectorizingPropertyHaveValue) ||
+            !fsm_dynamo_2.DyFM_Object.sameObjects(existingValue, newValue);
+        if (forceVectorize || (vectorizingPropertyHaveValue && newDataIsNewChanged)) {
+            if (!vectorizedProperty.embeddingModel) {
+                throw new fsm_dynamo_2.DyFM_Error({
+                    ...this.getDefaultErrorSettings('vectorizeDataProperties', new Error(`Property "${vectorizedProperty.key}" has no embedding model!`)),
+                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VDB1`,
+                    __localStack: this.dataParams.stackLocation,
+                });
+            }
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.log(`vectorizing "${vectorizedProperty.key}" in new "${this.dataParams.dataName}"` +
+                    newData?.name ? `: "${newData?.name}"` : '');
+            }
+            newData[vectorizedProperty.key] = await this.vectorize(newValue, vectorizedProperty.embeddingModel);
+        }
+        else if (existingData?.[vectorizedProperty.key] && !newData?.[vectorizedProperty.key]) {
+            newData[vectorizedProperty.key] = existingData[vectorizedProperty.key];
+        }
+    }
+    async getVectorizingDataForVectorizedProperty(vectorizedProperty, data) {
+        if (vectorizedProperty.vectorizedFrom.length === 1) {
+            const value = data?.[vectorizedProperty.vectorizedFrom[0]];
+            if (value === undefined) {
+                return undefined;
+            }
+            else if (typeof value === 'string') {
+                return value;
+            }
+            else if (Array.isArray(value) || typeof value === 'object') {
+                return JSON.stringify(value);
+            }
+            else {
+                return `${value}`;
+            }
+        }
+        else {
+            const object = {};
+            vectorizedProperty.vectorizedFrom.forEach((key) => {
+                if (data?.[key] !== undefined) {
+                    object[key] = data[key];
+                }
+            });
+            return JSON.stringify(object);
+        }
+    }
+    /**
+     * Vectorizes the input
+     *
+     * @example
+     * const vector = await vectorize(input);
+     */
+    async vectorize(input, useModel = this.openAISettings.defaultSettings.useModel) {
+        try {
+            if (!input) {
+                throw new fsm_dynamo_2.DyFM_Error({
+                    ...this.getDefaultErrorSettings('vectorize', new Error('input is required')),
+                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V1`,
+                });
+            }
+            this.embedding_CS ??= new oai_embedding_control_service_1.DyNTS_OAI_Embedding_ControlService(this.openAISettings);
+            const response = await this.embedding_CS.createEmbedding({
+                text: input,
+                model: useModel,
+                fullResponse: true,
+                issuer: this.issuer,
+            });
+            this.tokenSpent += response.usage.total_tokens;
+            return response.data[0].embedding;
+        }
+        catch (error) {
+            throw new fsm_dynamo_2.DyFM_Error({
+                ...this.getDefaultErrorSettings('vectorize', error),
+                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-V0`,
+            });
+        }
+    }
+    /**
+     * Vectorizes the input and searches for the most similar data
+     * (the default limit is 3, and the default numberOfCandidates is 100)
+     *
+     * @example
+     * const result = await vectorSearch(input, searchInKey);
+     */
+    async vectorSearch(set) {
+        try {
+            if (!set.input) {
+                throw new fsm_dynamo_2.DyFM_Error({
+                    ...this.getDefaultErrorSettings('vectorSearch', new Error('input is required')),
+                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS1`,
+                });
+            }
+            set.limit ??= 3;
+            set.numberOfCandidates ??= 100;
+            const { input, searchInKey, limit, numberOfCandidates, filterBy } = set;
+            this.embedding_CS ??= new oai_embedding_control_service_1.DyNTS_OAI_Embedding_ControlService(this.openAISettings);
+            const property = this.dataParams.properties[searchInKey];
+            if (!property) {
+                throw new fsm_dynamo_2.DyFM_Error({
+                    ...this.getDefaultErrorSettings('vectorSearch', new Error(`Property "${searchInKey}" not found! ` +
+                        `while searching "${this.dataParams.dataName}" ` +
+                        `(The searchable properties are: ${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`)),
+                    errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS2`,
+                    __localStack: this.dataParams.stackLocation,
+                });
+            }
+            if (!property.vectorizeUseIndex) {
+                throw new fsm_dynamo_2.DyFM_Error({
+                    ...this.getDefaultErrorSettings('vectorSearch', new Error(`Property "${searchInKey}" has no vectorizeUseIndex!`)),
+                });
+            }
+            const response = await this.embedding_CS.createEmbedding({
+                text: input,
+                model: property.embeddingModel,
+                fullResponse: true,
+                issuer: this.issuer,
+            });
+            this.tokenSpent += response.usage.total_tokens;
+            const embeddedInput = response.data[0].embedding;
+            /* const vectorSearchFilter =  */
+            const vectorSearchFilter = {
+                $vectorSearch: {
+                    /** this is the index name that is created in the DB (in the Atlas)
+                     *  also the same that is used in the {@link DyFM_DataProperty_Params} */
+                    index: property.vectorizeUseIndex,
+                    queryVector: embeddedInput,
+                    /** this is the property key that is vectorized and used to search */
+                    path: searchInKey,
+                    numCandidates: numberOfCandidates,
+                    limit: limit,
+                }
+            };
+            if (filterBy) {
+                vectorSearchFilter.filter = filterBy;
+            }
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.log(`vectorSearch...`);
+            }
+            const result = await this.dataDBService.aggregate([vectorSearchFilter]);
+            if (this.debugLog) {
+                fsm_dynamo_1.DyFM_Log.log(`vectorSearch: ${result.length} results`);
+            }
+            return result;
+        }
+        catch (error) {
+            throw new fsm_dynamo_2.DyFM_Error({
+                ...this.getDefaultErrorSettings('vectorSearch', error),
+                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-VS0`,
+            });
+        }
+    }
+    /**
+     * VectorDataService overrides the saveData function to vectorize the data properties
+     *
+     * {@link DyNTS_DataService.saveData}
+     */
+    async saveData(data, dontSetToService, forceVectorize) {
+        try {
+            data = this.ensureData(data);
+            data = await this.vectorizeDataProperties(data, forceVectorize);
+            data = await super.saveData(data, dontSetToService);
+            if (this.swallowVectorizedProperties) {
+                this.vectorizedProperties.forEach((property) => {
+                    delete data[property.key];
+                });
+            }
+            fsm_dynamo_2.DyFM_Object.cleanObject(data);
+            return data;
+        }
+        catch (error) {
+            throw new fsm_dynamo_2.DyFM_Error({
+                ...this.getDefaultErrorSettings('saveData', error),
+                errorCode: `${global_settings_const_1.DyNTS_global_settings.systemShortCodeName}|DyNTS-VDB-SD0`,
+            });
+        }
+    }
+    getProvidedData(data) {
+        this.vectorizedProperties.forEach((property) => {
+            delete data[property.key];
+        });
+        return data;
+    }
+}
+exports.DyNTS_OAI_VectorDataService = DyNTS_OAI_VectorDataService;
+//import {
+//  DyFM_DataModel_Params,
+//  DyFM_DataProperty_Params,
+//  DyFM_Metadata,
+//} from '@futdevpro/fsm-dynamo';
+//import { DyFM_OAI_Settings, DyFM_OAI_EmbeddingModel, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+//
+//import { DyNTS_DAI_VectorDataService } from '../../_modules/document-ai/_services/dai-vector-data.service-base';
+//import { DyNTS_OAI_Embedding_Service } from './oai-embedding.service';
+//import { DyNTS_global_settings } from '../../../../_collections/global-settings.const';
+//import { DyNTS_OAI_global_settings } from '../_collections/oai-global-settings.const';
+//
+///**
+// * OpenAI implementation of the generic Vector Data Service
+// * Extends the generic DAI VectorDataService with OpenAI-specific functionality
+// */
+//export class DyNTS_OAI_VectorDataService<T extends DyFM_Metadata> extends DyNTS_DAI_VectorDataService<T> {
+//
+//  debugLog: boolean = true;
+//  swallowVectorizedProperties: boolean = true;
+//
+//  constructor(
+//    /** Initial data */
+//    data: T,
+//    /** DB data params */
+//    dataParams: DyFM_DataModel_Params<T>,
+//    /** OpenAI settings */
+//    public readonly openAISettings: DyFM_OAI_Settings,
+//    /** Issuer */
+//    issuer: string
+//  ) {
+//    // Create OpenAI embedding service
+//    const embeddingService = new DyNTS_OAI_Embedding_Service(openAISettings);
+//    
+//    super(data, dataParams, embeddingService, issuer);
+//
+//    if (!openAISettings.openAIConfig.apiKey) {
+//      throw new Error('openAIConfig.apiKey is required');
+//    }
+//
+//    const model: DyFM_OAI_EmbeddingModel = openAISettings.defaultSettings.useModel as DyFM_OAI_EmbeddingModel ?? 
+//      DyFM_OAI_Model.textEmbedding_3Large;
+//
+//    // Configure vectorized properties with OpenAI-specific settings
+//    this.configureVectorizedProperties(model);
+//  }
+//
+//  /**
+//   * Configure vectorized properties with OpenAI-specific settings
+//   */
+//  private configureVectorizedProperties(model: DyFM_OAI_EmbeddingModel): void {
+//    // Update vectorized properties with OpenAI model info
+//    this.vectorizedProperties.forEach(property => {
+//      if (property.vectorizedFrom) {
+//        // Set the embedding model for this property
+//        (property as any).embeddingModel = model;
+//      }
+//    });
+//  }
+//
+//  /**
+//   * Perform vector search with OpenAI-specific implementation
+//   */
+//  protected override async performVectorSearch(
+//    query: string,
+//    model: string,
+//    issuer: string,
+//    options: any
+//  ): Promise<T[]> {
+//    try {
+//      // Create embedding for the query
+//      const queryEmbedding = await this.embeddingService.createEmbedding(query, model, issuer);
+//      
+//      // Get all data
+//      const allData = await this.getAllData();
+//      
+//      // Calculate similarities (simplified implementation)
+//      const results = allData.map(item => {
+//        const itemEmbedding = (item as any).chunkParentedContentVectorized;
+//        if (!itemEmbedding) return { item, similarity: 0 };
+//        
+//        const similarity = this.calculateCosineSimilarity(queryEmbedding, itemEmbedding);
+//        return { item, similarity };
+//      })
+//      .filter(result => result.similarity >= (options.minSimilarity || 0))
+//      .sort((a, b) => b.similarity - a.similarity)
+//      .slice(0, options.limit || 10)
+//      .map(result => result.item);
+//
+//      return results;
+//    } catch (error) {
+//      throw new Error(`Vector search failed: ${error.message}`);
+//    }
+//  }
+//
+//  /**
+//   * Calculate cosine similarity between two vectors
+//   */
+//  private calculateCosineSimilarity(vecA: number[], vecB: number[]): number {
+//    if (vecA.length !== vecB.length) return 0;
+//    
+//    let dotProduct = 0;
+//    let normA = 0;
+//    let normB = 0;
+//    
+//    for (let i = 0; i < vecA.length; i++) {
+//      dotProduct += vecA[i] * vecB[i];
+//      normA += vecA[i] * vecA[i];
+//      normB += vecB[i] * vecB[i];
+//    }
+//    
+//    if (normA === 0 || normB === 0) return 0;
+//    
+//    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+//  }
+//
+//  /**
+//   * Get default error settings
+//   */
+//  protected getDefaultErrorSettings(
+//    method: string,
+//    error: any,
+//    issuer?: string
+//  ): any {
+//    return {
+//      method,
+//      error,
+//      issuer: issuer || this.issuer,
+//      userMsg: this.defaultErrorUserMsg,
+//    };
+//  }
+//}
 //# sourceMappingURL=oai-vector-data.service.js.map