@futdevpro/nts-dynamo 1.15.11 → 1.15.14

package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts

@@ -1,330 +1,330 @@
-import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
-import {
-  DyFM_DataModel_Params,
-  DyFM_DataProperty_Params,
-  DyFM_DBFilter,
-  DyFM_Metadata,
-} from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
-
-import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
-import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
-import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
-import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
-import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
-
-/**
- * Local Vector Data Service
- * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
- * instead of MongoDB vector search
- * 
- * This service loads data from the database, builds an in-memory vector pool,
- * and performs brute-force vector similarity search using cosine similarity or L2 distance
- * 
- * @example
- * ```typescript
- * const service = new DyNTS_OAI_LocalVectorDataService(
- *   data,
- *   dataParams,
- *   openAISettings,
- *   issuer
- * );
- * 
- * const results = await service.vectorSearch({
- *   input: 'search query',
- *   searchInKey: 'contentVectorized',
- *   limit: 10,
- *   filterBy: { category: 'docs' }
- * });
- * ```
- */
-export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
-
-  /**
-   * Default search mode
-   * Cosine similarity is typically preferred for text embeddings
-   */
-  defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
-
-  /**
-   * Whether to use L2 normalization when storing vectors
-   * Normalized vectors are more efficient for cosine similarity calculations
-   */
-  useL2Normalization: boolean = true;
-
-  /**
-   * Instance-based vector pool utility
-   * Minden service instance saját vektor pool-t tart fenn
-   */
-  private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
-
-  constructor(
-    /**
-     * Initial data, this will be used by functions on default
-     */
-    data: T,
-    /**
-     * DB data params will be used to connect to usable dbService on GlobalService
-     */
-    dataParams: DyFM_DataModel_Params<T>,
-    /**
-     * OpenAI settings
-     */
-    openAISettings: DyFM_OAI_Settings,
-    /**
-     * Initial set for issuer to be able to follow the issuer's activity
-     */
-    issuer: string
-  ) {
-    super(data, dataParams, openAISettings, issuer);
-  }
-
-  /**
-   * Overrides the parent vectorSearch method to perform local in-memory vector search
-   * 
-   * Process:
-   * 1. Load data from DB using filterBy (if provided) or getAll()
-   * 2. Extract vectorized properties from loaded data
-   * 3. Build in-memory vector pool
-   * 4. Vectorize input query
-   * 5. Perform local vector search
-   * 6. Map search results back to full data objects
-   * 
-   * @param set Search parameters
-   * @returns Array of data objects sorted by similarity score
-   */
-  override async vectorSearch(
-    set: {
-      input: string;
-      /** this should be the vectorized property key */
-      searchInKey: string;
-      limit?: number;
-      /**
-       * Number of candidates that are used to find the best match
-       * (Not used in local search, but kept for compatibility)
-       */
-      numberOfCandidates?: number;
-      /**
-       * Filter to narrow down candidates from DB before vector search
-       */
-      filterBy?: DyFM_DBFilter<T>;
-      /**
-       * Search mode (cosine similarity or L2 distance)
-       * Defaults to this.defaultSearchMode
-       */
-      searchMode?: LVS_Search_Mode;
-    },
-  ): Promise<T[]> {
-    try {
-      if (!set.input) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch',
-            new Error('input is required')
-          ),
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
-        });
-      }
-
-      set.limit ??= 3;
-      const { input, searchInKey, limit, filterBy, searchMode } = set;
-
-      // Validáljuk, hogy a searchInKey létezik-e
-      const property: DyFM_DataProperty_Params<any, T> = 
-        this.dataParams.properties[searchInKey];
-
-      if (!property) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch',
-            new Error(
-              `Property "${searchInKey}" not found! ` +
-              `while searching "${this.dataParams.dataName}" ` +
-              `(The searchable properties are: ` +
-              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
-            )
-          ),
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      // Ellenőrizzük, hogy a property vectorized-e
-      if (!this.vectorizedProperties.some(
-        (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
-      )) {
-        throw new DyFM_Error({
-          ...this.getDefaultErrorSettings(
-            'vectorSearch',
-            new Error(
-              `Property "${searchInKey}" is not a vectorized property! ` +
-              `(The vectorized properties are: ` +
-              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
-            )
-          ),
-          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
-          __localStack: this.dataParams.stackLocation,
-        });
-      }
-
-      // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
-      let dataList: T[];
-      if (filterBy) {
-        // Ha van filterBy, akkor először szűrünk a DB-ben
-        if (this.debugLog) {
-          DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
-        }
-        dataList = await this.findDataList(filterBy, true);
-      } else {
-        // Ha nincs filterBy, akkor minden adatot betöltünk
-        if (this.debugLog) {
-          DyFM_Log.log(`Local vector search: loading all data...`);
-        }
-        dataList = await this.getAll(true);
-      }
-
-      if (dataList.length === 0) {
-        if (this.debugLog) {
-          DyFM_Log.log(`Local vector search: no data found`);
-        }
-        return [];
-      }
-
-      // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
-      if (this.debugLog) {
-        DyFM_Log.log(
-          `Local vector search: building vector pool from ${dataList.length} items...`
-        );
-      }
-
-      // Töröljük a korábbi pool-t
-      this.vectorPool.clearPool();
-
-      // Hozzáadjuk a vektorokat a pool-hoz
-      const dataMap: Map<string, T> = new Map<string, T>();
-      for (const dataItem of dataList) {
-        if (!dataItem._id) {
-          // Ha nincs _id, akkor kihagyjuk
-          if (this.debugLog) {
-            DyFM_Log.warn(
-              `Local vector search: skipping item without _id`
-            );
-          }
-          continue;
-        }
-
-        const vectorizedValue: number[] | undefined = 
-          dataItem[searchInKey] as number[] | undefined;
-
-        if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
-          // Ha nincs vectorized érték, akkor kihagyjuk
-          if (this.debugLog) {
-            DyFM_Log.warn(
-              `Local vector search: skipping item "${dataItem._id}" ` +
-              `without vectorized value for "${searchInKey}"`
-            );
-          }
-          continue;
-        }
-
-        // Hozzáadjuk a vektort a pool-hoz
-        this.vectorPool.addVector(dataItem._id, vectorizedValue);
-        dataMap.set(dataItem._id, dataItem);
-      }
-
-      if (dataMap.size === 0) {
-        if (this.debugLog) {
-          DyFM_Log.log(`Local vector search: no valid vectors found`);
-        }
-        return [];
-      }
-
-      // 3. Vectorizáljuk a query input-ot
-      if (this.debugLog) {
-        DyFM_Log.log(`Local vector search: vectorizing query input...`);
-      }
-
-      const queryVector: number[] = await this.vectorize(
-        input,
-        property.embeddingModel
-      );
-
-      // 4. Végrehajtjuk a local vector search-t
-      const mode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
-
-      if (this.debugLog) {
-        DyFM_Log.log(
-          `Local vector search: searching with mode "${mode}", limit: ${limit}...`
-        );
-      }
-
-      const searchResults: LVS_SearchResult[] = this.vectorPool.search(
-        queryVector,
-        limit,
-        mode
-      );
-
-      if (this.debugLog) {
-        DyFM_Log.log(
-          `Local vector search: found ${searchResults.length} results`
-        );
-      }
-
-      // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
-      const results: T[] = [];
-      for (const searchResult of searchResults) {
-        const dataItem: T | undefined = dataMap.get(searchResult.id);
-        if (dataItem) {
-          results.push(dataItem);
-        }
-      }
-
-      // Töröljük a pool-t a memória felszabadítása érdekében
-      this.vectorPool.clearPool();
-
-      return results;
-    } catch (error) {
-      // Töröljük a pool-t hiba esetén is
-      this.vectorPool.clearPool();
-
-      throw new DyFM_Error({
-        ...this.getDefaultErrorSettings('vectorSearch', error),
-        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
-      });
-    }
-  }
-}
-
-/**
- * @example
- * ```typescript
- * // Create a local vector data service
- * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
- *   new CodeChunk(),
- *   codeChunk_dataParams,
- *   openAISettings,
- *   'example-issuer'
- * );
- * 
- * // Perform vector search with filterBy
- * const results = await localVectorService.vectorSearch({
- *   input: 'How to implement authentication?',
- *   searchInKey: 'chunkParentedContentVectorized',
- *   limit: 5,
- *   filterBy: {
- *     // Filter by category before vector search
- *     category: 'documentation'
- *   },
- *   searchMode: LVS_Search_Mode.cosineSimilarity
- * });
- * 
- * // Results are sorted by similarity score (highest first for cosine)
- * console.log(`Found ${results.length} similar chunks`);
- * 
- * // Access the results
- * for (const result of results) {
- *   console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
- * }
- * ```
- */
-
+import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
+import {
+  DyFM_DataModel_Params,
+  DyFM_DataProperty_Params,
+  DyFM_DBFilter,
+  DyFM_Metadata,
+} from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
+import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
+import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
+import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
+import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+
+/**
+ * Local Vector Data Service
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
+ * instead of MongoDB vector search
+ * 
+ * This service loads data from the database, builds an in-memory vector pool,
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
+ * 
+ * @example
+ * ```typescript
+ * const service = new DyNTS_OAI_LocalVectorDataService(
+ *   data,
+ *   dataParams,
+ *   openAISettings,
+ *   issuer
+ * );
+ * 
+ * const results = await service.vectorSearch({
+ *   input: 'search query',
+ *   searchInKey: 'contentVectorized',
+ *   limit: 10,
+ *   filterBy: { category: 'docs' }
+ * });
+ * ```
+ */
+export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
+
+  /**
+   * Default search mode
+   * Cosine similarity is typically preferred for text embeddings
+   */
+  defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
+
+  /**
+   * Whether to use L2 normalization when storing vectors
+   * Normalized vectors are more efficient for cosine similarity calculations
+   */
+  useL2Normalization: boolean = true;
+
+  /**
+   * Instance-based vector pool utility
+   * Minden service instance saját vektor pool-t tart fenn
+   */
+  private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
+
+  constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data: T,
+    /**
+     * DB data params will be used to connect to usable dbService on GlobalService
+     */
+    dataParams: DyFM_DataModel_Params<T>,
+    /**
+     * OpenAI settings
+     */
+    openAISettings: DyFM_OAI_Settings,
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer: string
+  ) {
+    super(data, dataParams, openAISettings, issuer);
+  }
+
+  /**
+   * Overrides the parent vectorSearch method to perform local in-memory vector search
+   * 
+   * Process:
+   * 1. Load data from DB using filterBy (if provided) or getAll()
+   * 2. Extract vectorized properties from loaded data
+   * 3. Build in-memory vector pool
+   * 4. Vectorize input query
+   * 5. Perform local vector search
+   * 6. Map search results back to full data objects
+   * 
+   * @param set Search parameters
+   * @returns Array of data objects sorted by similarity score
+   */
+  override async vectorSearch(
+    set: {
+      input: string;
+      /** this should be the vectorized property key */
+      searchInKey: string;
+      limit?: number;
+      /**
+       * Number of candidates that are used to find the best match
+       * (Not used in local search, but kept for compatibility)
+       */
+      numberOfCandidates?: number;
+      /**
+       * Filter to narrow down candidates from DB before vector search
+       */
+      filterBy?: DyFM_DBFilter<T>;
+      /**
+       * Search mode (cosine similarity or L2 distance)
+       * Defaults to this.defaultSearchMode
+       */
+      searchMode?: LVS_Search_Mode;
+    },
+  ): Promise<T[]> {
+    try {
+      if (!set.input) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error('input is required')
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
+        });
+      }
+
+      set.limit ??= 3;
+      const { input, searchInKey, limit, filterBy, searchMode } = set;
+
+      // Validáljuk, hogy a searchInKey létezik-e
+      const property: DyFM_DataProperty_Params<any, T> = 
+        this.dataParams.properties[searchInKey];
+
+      if (!property) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" not found! ` +
+              `while searching "${this.dataParams.dataName}" ` +
+              `(The searchable properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // Ellenőrizzük, hogy a property vectorized-e
+      if (!this.vectorizedProperties.some(
+        (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
+      )) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" is not a vectorized property! ` +
+              `(The vectorized properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
+      let dataList: T[];
+      if (filterBy) {
+        // Ha van filterBy, akkor először szűrünk a DB-ben
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
+        }
+        dataList = await this.findDataList(filterBy, true);
+      } else {
+        // Ha nincs filterBy, akkor minden adatot betöltünk
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: loading all data...`);
+        }
+        dataList = await this.getAll(true);
+      }
+
+      if (dataList.length === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no data found`);
+        }
+        return [];
+      }
+
+      // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: building vector pool from ${dataList.length} items...`
+        );
+      }
+
+      // Töröljük a korábbi pool-t
+      this.vectorPool.clearPool();
+
+      // Hozzáadjuk a vektorokat a pool-hoz
+      const dataMap: Map<string, T> = new Map<string, T>();
+      for (const dataItem of dataList) {
+        if (!dataItem._id) {
+          // Ha nincs _id, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item without _id`
+            );
+          }
+          continue;
+        }
+
+        const vectorizedValue: number[] | undefined = 
+          dataItem[searchInKey] as number[] | undefined;
+
+        if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
+          // Ha nincs vectorized érték, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item "${dataItem._id}" ` +
+              `without vectorized value for "${searchInKey}"`
+            );
+          }
+          continue;
+        }
+
+        // Hozzáadjuk a vektort a pool-hoz
+        this.vectorPool.addVector(dataItem._id, vectorizedValue);
+        dataMap.set(dataItem._id, dataItem);
+      }
+
+      if (dataMap.size === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no valid vectors found`);
+        }
+        return [];
+      }
+
+      // 3. Vectorizáljuk a query input-ot
+      if (this.debugLog) {
+        DyFM_Log.log(`Local vector search: vectorizing query input...`);
+      }
+
+      const queryVector: number[] = await this.vectorize(
+        input,
+        property.embeddingModel
+      );
+
+      // 4. Végrehajtjuk a local vector search-t
+      const mode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: searching with mode "${mode}", limit: ${limit}...`
+        );
+      }
+
+      const searchResults: LVS_SearchResult[] = this.vectorPool.search(
+        queryVector,
+        limit,
+        mode
+      );
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: found ${searchResults.length} results`
+        );
+      }
+
+      // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
+      const results: T[] = [];
+      for (const searchResult of searchResults) {
+        const dataItem: T | undefined = dataMap.get(searchResult.id);
+        if (dataItem) {
+          results.push(dataItem);
+        }
+      }
+
+      // Töröljük a pool-t a memória felszabadítása érdekében
+      this.vectorPool.clearPool();
+
+      return results;
+    } catch (error) {
+      // Töröljük a pool-t hiba esetén is
+      this.vectorPool.clearPool();
+
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorSearch', error),
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
+      });
+    }
+  }
+}
+
+/**
+ * @example
+ * ```typescript
+ * // Create a local vector data service
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
+ *   new CodeChunk(),
+ *   codeChunk_dataParams,
+ *   openAISettings,
+ *   'example-issuer'
+ * );
+ * 
+ * // Perform vector search with filterBy
+ * const results = await localVectorService.vectorSearch({
+ *   input: 'How to implement authentication?',
+ *   searchInKey: 'chunkParentedContentVectorized',
+ *   limit: 5,
+ *   filterBy: {
+ *     // Filter by category before vector search
+ *     category: 'documentation'
+ *   },
+ *   searchMode: LVS_Search_Mode.cosineSimilarity
+ * });
+ * 
+ * // Results are sorted by similarity score (highest first for cosine)
+ * console.log(`Found ${results.length} similar chunks`);
+ * 
+ * // Access the results
+ * for (const result of results) {
+ *   console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
+ * }
+ * ```
+ */
+