@futdevpro/nts-dynamo 1.14.64 → 1.14.67

package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts

@@ -0,0 +1,330 @@
+import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
+import {
+  DyFM_DataModel_Params,
+  DyFM_DataProperty_Params,
+  DyFM_DBFilter,
+  DyFM_Metadata,
+} from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
+import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
+import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
+import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
+import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+
+/**
+ * Local Vector Data Service
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
+ * instead of MongoDB vector search
+ * 
+ * This service loads data from the database, builds an in-memory vector pool,
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
+ * 
+ * @example
+ * ```typescript
+ * const service = new DyNTS_OAI_LocalVectorDataService(
+ *   data,
+ *   dataParams,
+ *   openAISettings,
+ *   issuer
+ * );
+ * 
+ * const results = await service.vectorSearch({
+ *   input: 'search query',
+ *   searchInKey: 'contentVectorized',
+ *   limit: 10,
+ *   filterBy: { category: 'docs' }
+ * });
+ * ```
+ */
+export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
+
+  /**
+   * Default search mode
+   * Cosine similarity is typically preferred for text embeddings
+   */
+  defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
+
+  /**
+   * Whether to use L2 normalization when storing vectors
+   * Normalized vectors are more efficient for cosine similarity calculations
+   */
+  useL2Normalization: boolean = true;
+
+  /**
+   * Instance-based vector pool utility
+   * Minden service instance saját vektor pool-t tart fenn
+   */
+  private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
+
+  constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data: T,
+    /**
+     * DB data params will be used to connect to usable dbService on GlobalService
+     */
+    dataParams: DyFM_DataModel_Params<T>,
+    /**
+     * OpenAI settings
+     */
+    openAISettings: DyFM_OAI_Settings,
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer: string
+  ) {
+    super(data, dataParams, openAISettings, issuer);
+  }
+
+  /**
+   * Overrides the parent vectorSearch method to perform local in-memory vector search
+   * 
+   * Process:
+   * 1. Load data from DB using filterBy (if provided) or getAll()
+   * 2. Extract vectorized properties from loaded data
+   * 3. Build in-memory vector pool
+   * 4. Vectorize input query
+   * 5. Perform local vector search
+   * 6. Map search results back to full data objects
+   * 
+   * @param set Search parameters
+   * @returns Array of data objects sorted by similarity score
+   */
+  override async vectorSearch(
+    set: {
+      input: string;
+      /** this should be the vectorized property key */
+      searchInKey: string;
+      limit?: number;
+      /**
+       * Number of candidates that are used to find the best match
+       * (Not used in local search, but kept for compatibility)
+       */
+      numberOfCandidates?: number;
+      /**
+       * Filter to narrow down candidates from DB before vector search
+       */
+      filterBy?: DyFM_DBFilter<T>;
+      /**
+       * Search mode (cosine similarity or L2 distance)
+       * Defaults to this.defaultSearchMode
+       */
+      searchMode?: LVS_Search_Mode;
+    },
+  ): Promise<T[]> {
+    try {
+      if (!set.input) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error('input is required')
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
+        });
+      }
+
+      set.limit ??= 3;
+      const { input, searchInKey, limit, filterBy, searchMode } = set;
+
+      // Validáljuk, hogy a searchInKey létezik-e
+      const property: DyFM_DataProperty_Params<any, T> = 
+        this.dataParams.properties[searchInKey];
+
+      if (!property) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" not found! ` +
+              `while searching "${this.dataParams.dataName}" ` +
+              `(The searchable properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // Ellenőrizzük, hogy a property vectorized-e
+      if (!this.vectorizedProperties.some(
+        (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
+      )) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" is not a vectorized property! ` +
+              `(The vectorized properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
+      let dataList: T[];
+      if (filterBy) {
+        // Ha van filterBy, akkor először szűrünk a DB-ben
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
+        }
+        dataList = await this.findDataList(filterBy, true);
+      } else {
+        // Ha nincs filterBy, akkor minden adatot betöltünk
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: loading all data...`);
+        }
+        dataList = await this.getAll(true);
+      }
+
+      if (dataList.length === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no data found`);
+        }
+        return [];
+      }
+
+      // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: building vector pool from ${dataList.length} items...`
+        );
+      }
+
+      // Töröljük a korábbi pool-t
+      this.vectorPool.clearPool();
+
+      // Hozzáadjuk a vektorokat a pool-hoz
+      const dataMap: Map<string, T> = new Map<string, T>();
+      for (const dataItem of dataList) {
+        if (!dataItem._id) {
+          // Ha nincs _id, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item without _id`
+            );
+          }
+          continue;
+        }
+
+        const vectorizedValue: number[] | undefined = 
+          dataItem[searchInKey] as number[] | undefined;
+
+        if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
+          // Ha nincs vectorized érték, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item "${dataItem._id}" ` +
+              `without vectorized value for "${searchInKey}"`
+            );
+          }
+          continue;
+        }
+
+        // Hozzáadjuk a vektort a pool-hoz
+        this.vectorPool.addVector(dataItem._id, vectorizedValue);
+        dataMap.set(dataItem._id, dataItem);
+      }
+
+      if (dataMap.size === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no valid vectors found`);
+        }
+        return [];
+      }
+
+      // 3. Vectorizáljuk a query input-ot
+      if (this.debugLog) {
+        DyFM_Log.log(`Local vector search: vectorizing query input...`);
+      }
+
+      const queryVector: number[] = await this.vectorize(
+        input,
+        property.embeddingModel
+      );
+
+      // 4. Végrehajtjuk a local vector search-t
+      const mode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: searching with mode "${mode}", limit: ${limit}...`
+        );
+      }
+
+      const searchResults: LVS_SearchResult[] = this.vectorPool.search(
+        queryVector,
+        limit,
+        mode
+      );
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: found ${searchResults.length} results`
+        );
+      }
+
+      // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
+      const results: T[] = [];
+      for (const searchResult of searchResults) {
+        const dataItem: T | undefined = dataMap.get(searchResult.id);
+        if (dataItem) {
+          results.push(dataItem);
+        }
+      }
+
+      // Töröljük a pool-t a memória felszabadítása érdekében
+      this.vectorPool.clearPool();
+
+      return results;
+    } catch (error) {
+      // Töröljük a pool-t hiba esetén is
+      this.vectorPool.clearPool();
+
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorSearch', error),
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
+      });
+    }
+  }
+}
+
+/**
+ * @example
+ * ```typescript
+ * // Create a local vector data service
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
+ *   new CodeChunk(),
+ *   codeChunk_dataParams,
+ *   openAISettings,
+ *   'example-issuer'
+ * );
+ * 
+ * // Perform vector search with filterBy
+ * const results = await localVectorService.vectorSearch({
+ *   input: 'How to implement authentication?',
+ *   searchInKey: 'chunkParentedContentVectorized',
+ *   limit: 5,
+ *   filterBy: {
+ *     // Filter by category before vector search
+ *     category: 'documentation'
+ *   },
+ *   searchMode: LVS_Search_Mode.cosineSimilarity
+ * });
+ * 
+ * // Results are sorted by similarity score (highest first for cosine)
+ * console.log(`Found ${results.length} similar chunks`);
+ * 
+ * // Access the results
+ * for (const result of results) {
+ *   console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
+ * }
+ * ```
+ */
+