@futdevpro/nts-dynamo 1.14.63 → 1.14.65

package/src/_modules/local-vector-search/_services/lvs-local-vector-search.data-service.ts

@@ -0,0 +1,330 @@
+import { DyFM_Log, DyFM_Error } from '@futdevpro/fsm-dynamo';
+import {
+  DyFM_DataModel_Params,
+  DyFM_DataProperty_Params,
+  DyFM_DBFilter,
+  DyFM_Metadata,
+} from '@futdevpro/fsm-dynamo';
+import { DyFM_OAI_Settings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
+import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
+import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
+import { DyNTS_OAI_VectorDataService } from '../../ai/_modules/open-ai/_services/data-services/oai-vector-data.service';
+import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+
+/**
+ * Local Vector Data Service
+ * Extends {@link DyNTS_OAI_VectorDataService} to perform in-memory vector similarity search
+ * instead of MongoDB vector search
+ * 
+ * This service loads data from the database, builds an in-memory vector pool,
+ * and performs brute-force vector similarity search using cosine similarity or L2 distance
+ * 
+ * @example
+ * ```typescript
+ * const service = new DyNTS_OAI_LocalVectorDataService(
+ *   data,
+ *   dataParams,
+ *   openAISettings,
+ *   issuer
+ * );
+ * 
+ * const results = await service.vectorSearch({
+ *   input: 'search query',
+ *   searchInKey: 'contentVectorized',
+ *   limit: 10,
+ *   filterBy: { category: 'docs' }
+ * });
+ * ```
+ */
+export class DyNTS_LVS_VectorDataService<T extends DyFM_Metadata> extends DyNTS_OAI_VectorDataService<T> {
+
+  /**
+   * Default search mode
+   * Cosine similarity is typically preferred for text embeddings
+   */
+  defaultSearchMode: LVS_Search_Mode = LVS_Search_Mode.cosineSimilarity;
+
+  /**
+   * Whether to use L2 normalization when storing vectors
+   * Normalized vectors are more efficient for cosine similarity calculations
+   */
+  useL2Normalization: boolean = true;
+
+  /**
+   * Instance-based vector pool utility
+   * Minden service instance saját vektor pool-t tart fenn
+   */
+  private vectorPool: LVS_VectorPool_ControlService = new LVS_VectorPool_ControlService();
+
+  constructor(
+    /**
+     * Initial data, this will be used by functions on default
+     */
+    data: T,
+    /**
+     * DB data params will be used to connect to usable dbService on GlobalService
+     */
+    dataParams: DyFM_DataModel_Params<T>,
+    /**
+     * OpenAI settings
+     */
+    openAISettings: DyFM_OAI_Settings,
+    /**
+     * Initial set for issuer to be able to follow the issuer's activity
+     */
+    issuer: string
+  ) {
+    super(data, dataParams, openAISettings, issuer);
+  }
+
+  /**
+   * Overrides the parent vectorSearch method to perform local in-memory vector search
+   * 
+   * Process:
+   * 1. Load data from DB using filterBy (if provided) or getAll()
+   * 2. Extract vectorized properties from loaded data
+   * 3. Build in-memory vector pool
+   * 4. Vectorize input query
+   * 5. Perform local vector search
+   * 6. Map search results back to full data objects
+   * 
+   * @param set Search parameters
+   * @returns Array of data objects sorted by similarity score
+   */
+  override async vectorSearch(
+    set: {
+      input: string;
+      /** this should be the vectorized property key */
+      searchInKey: string;
+      limit?: number;
+      /**
+       * Number of candidates that are used to find the best match
+       * (Not used in local search, but kept for compatibility)
+       */
+      numberOfCandidates?: number;
+      /**
+       * Filter to narrow down candidates from DB before vector search
+       */
+      filterBy?: DyFM_DBFilter<T>;
+      /**
+       * Search mode (cosine similarity or L2 distance)
+       * Defaults to this.defaultSearchMode
+       */
+      searchMode?: LVS_Search_Mode;
+    },
+  ): Promise<T[]> {
+    try {
+      if (!set.input) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error('input is required')
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS1`,
+        });
+      }
+
+      set.limit ??= 3;
+      const { input, searchInKey, limit, filterBy, searchMode } = set;
+
+      // Validáljuk, hogy a searchInKey létezik-e
+      const property: DyFM_DataProperty_Params<any, T> = 
+        this.dataParams.properties[searchInKey];
+
+      if (!property) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" not found! ` +
+              `while searching "${this.dataParams.dataName}" ` +
+              `(The searchable properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS2`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // Ellenőrizzük, hogy a property vectorized-e
+      if (!this.vectorizedProperties.some(
+        (p: DyFM_DataProperty_Params<any, T>) => p.key === searchInKey
+      )) {
+        throw new DyFM_Error({
+          ...this.getDefaultErrorSettings(
+            'vectorSearch',
+            new Error(
+              `Property "${searchInKey}" is not a vectorized property! ` +
+              `(The vectorized properties are: ` +
+              `${this.vectorizedProperties.map(p => `"${p.key}"`).join(', ')})`
+            )
+          ),
+          errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS3`,
+          __localStack: this.dataParams.stackLocation,
+        });
+      }
+
+      // 1. Betöltjük az adatokat a DB-ből (filterBy-val szűrve, ha van)
+      let dataList: T[];
+      if (filterBy) {
+        // Ha van filterBy, akkor először szűrünk a DB-ben
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: filtering data with filterBy...`);
+        }
+        dataList = await this.findDataList(filterBy, true);
+      } else {
+        // Ha nincs filterBy, akkor minden adatot betöltünk
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: loading all data...`);
+        }
+        dataList = await this.getAll(true);
+      }
+
+      if (dataList.length === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no data found`);
+        }
+        return [];
+      }
+
+      // 2. Kinyerjük a vectorized property-ket és építjük a vector pool-t
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: building vector pool from ${dataList.length} items...`
+        );
+      }
+
+      // Töröljük a korábbi pool-t
+      this.vectorPool.clearPool();
+
+      // Hozzáadjuk a vektorokat a pool-hoz
+      const dataMap: Map<string, T> = new Map<string, T>();
+      for (const dataItem of dataList) {
+        if (!dataItem._id) {
+          // Ha nincs _id, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item without _id`
+            );
+          }
+          continue;
+        }
+
+        const vectorizedValue: number[] | undefined = 
+          dataItem[searchInKey] as number[] | undefined;
+
+        if (!vectorizedValue || !Array.isArray(vectorizedValue)) {
+          // Ha nincs vectorized érték, akkor kihagyjuk
+          if (this.debugLog) {
+            DyFM_Log.warn(
+              `Local vector search: skipping item "${dataItem._id}" ` +
+              `without vectorized value for "${searchInKey}"`
+            );
+          }
+          continue;
+        }
+
+        // Hozzáadjuk a vektort a pool-hoz
+        this.vectorPool.addVector(dataItem._id, vectorizedValue);
+        dataMap.set(dataItem._id, dataItem);
+      }
+
+      if (dataMap.size === 0) {
+        if (this.debugLog) {
+          DyFM_Log.log(`Local vector search: no valid vectors found`);
+        }
+        return [];
+      }
+
+      // 3. Vectorizáljuk a query input-ot
+      if (this.debugLog) {
+        DyFM_Log.log(`Local vector search: vectorizing query input...`);
+      }
+
+      const queryVector: number[] = await this.vectorize(
+        input,
+        property.embeddingModel
+      );
+
+      // 4. Végrehajtjuk a local vector search-t
+      const mode: LVS_Search_Mode = searchMode ?? this.defaultSearchMode;
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: searching with mode "${mode}", limit: ${limit}...`
+        );
+      }
+
+      const searchResults: LVS_SearchResult[] = this.vectorPool.search(
+        queryVector,
+        limit,
+        mode
+      );
+
+      if (this.debugLog) {
+        DyFM_Log.log(
+          `Local vector search: found ${searchResults.length} results`
+        );
+      }
+
+      // 5. Map-eljük vissza a search results-ot a teljes data objektumokra
+      const results: T[] = [];
+      for (const searchResult of searchResults) {
+        const dataItem: T | undefined = dataMap.get(searchResult.id);
+        if (dataItem) {
+          results.push(dataItem);
+        }
+      }
+
+      // Töröljük a pool-t a memória felszabadítása érdekében
+      this.vectorPool.clearPool();
+
+      return results;
+    } catch (error) {
+      // Töröljük a pool-t hiba esetén is
+      this.vectorPool.clearPool();
+
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('vectorSearch', error),
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-LVS-VS0`,
+      });
+    }
+  }
+}
+
+/**
+ * @example
+ * ```typescript
+ * // Create a local vector data service
+ * const localVectorService = new DyNTS_OAI_LocalVectorDataService<CodeChunk>(
+ *   new CodeChunk(),
+ *   codeChunk_dataParams,
+ *   openAISettings,
+ *   'example-issuer'
+ * );
+ * 
+ * // Perform vector search with filterBy
+ * const results = await localVectorService.vectorSearch({
+ *   input: 'How to implement authentication?',
+ *   searchInKey: 'chunkParentedContentVectorized',
+ *   limit: 5,
+ *   filterBy: {
+ *     // Filter by category before vector search
+ *     category: 'documentation'
+ *   },
+ *   searchMode: LVS_Search_Mode.cosineSimilarity
+ * });
+ * 
+ * // Results are sorted by similarity score (highest first for cosine)
+ * console.log(`Found ${results.length} similar chunks`);
+ * 
+ * // Access the results
+ * for (const result of results) {
+ *   console.log(`Chunk ID: ${result._id}, Content: ${result.chunkContent}`);
+ * }
+ * ```
+ */
+

package/src/_modules/local-vector-search/_services/lvs-vector-pool.control-service.ts

@@ -0,0 +1,220 @@
+import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
+import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
+
+/**
+ * Vektor pool kezelő utility class
+ * Instance-based class, amely in-memory vektor pool-t kezel és vektor hasonlósági számításokat végez
+ * Minden instance saját vektor pool-t tart fenn
+ */
+export class LVS_VectorPool_ControlService {
+  /**
+   * In-memory vektor pool tárolás
+   * Key: vektor ID, Value: vektor értékek
+   */
+  private vectorPool: Map<string, number[]> = new Map<string, number[]>();
+
+  /**
+   * Normalizált vektor pool tárolás (L2 normalizált másolatok)
+   * Key: vektor ID, Value: L2 normalizált vektor értékek
+   */
+  private normalizedVectorPool: Map<string, number[]> = new Map<string, number[]>();
+
+  /**
+   * Hozzáad egy vektort a pool-hoz
+   * Opcionálisan L2 normalizált másolatot is tárol
+   */
+  addVector(id: string, vector: number[]): void {
+    if (!id) {
+      throw new Error('Vector ID is required');
+    }
+
+    if (!Array.isArray(vector) || vector.length === 0) {
+      throw new Error('Vector must be a non-empty array');
+    }
+
+    // Validáljuk, hogy a vektor számokból áll
+    for (let i: number = 0; i < vector.length; i++) {
+      if (typeof vector[i] !== 'number' || !isFinite(vector[i])) {
+        throw new Error(`Vector must contain only finite numbers at index ${i}`);
+      }
+    }
+
+    // Tároljuk az eredeti vektort
+    this.vectorPool.set(id, vector);
+
+    // L2 normalizált másolatot is tárolunk a cosine similarity számításokhoz
+    const normalized: number[] = LVS_VectorPool_ControlService.l2Normalize(vector);
+    this.normalizedVectorPool.set(id, normalized);
+  }
+
+  /**
+   * Eltávolít egy vektort a pool-ból
+   */
+  removeVector(id: string): void {
+    this.vectorPool.delete(id);
+    this.normalizedVectorPool.delete(id);
+  }
+
+  /**
+   * Frissít egy vektort a pool-ban
+   */
+  updateVector(id: string, vector: number[]): void {
+    if (!this.vectorPool.has(id)) {
+      throw new Error(`Vector with ID "${id}" does not exist`);
+    }
+
+    this.addVector(id, vector);
+  }
+
+  /**
+   * Törli az összes vektort a pool-ból
+   */
+  clearPool(): void {
+    this.vectorPool.clear();
+    this.normalizedVectorPool.clear();
+  }
+
+  /**
+   * Visszaadja az összes vektort a pool-ból
+   * Debugging és export céljára
+   */
+  getAll(): Map<string, number[]> {
+    return new Map<string, number[]>(this.vectorPool);
+  }
+
+  /**
+   * L2 normalizálás egy vektorra
+   * A vektor hosszát 1-re normalizálja
+   * Statikus metódus, mivel nincs szüksége instance state-re
+   */
+  private static l2Normalize(vector: number[]): number[] {
+    const magnitude: number = Math.sqrt(
+      vector.reduce((sum: number, value: number) => sum + value * value, 0)
+    );
+
+    if (magnitude === 0) {
+      // Ha a vektor nullvektor, akkor nullvektort adunk vissza
+      return new Array<number>(vector.length).fill(0);
+    }
+
+    return vector.map((value: number) => value / magnitude);
+  }
+
+  /**
+   * Cosine similarity számítás két vektor között
+   * Normalizált vektorokat használ a számításhoz
+   * Eredmény: 0 és 1 közötti érték, ahol 1 a legnagyobb hasonlóság
+   * Statikus metódus, mivel nincs szüksége instance state-re
+   */
+  static cosineSimilarity(a: number[], b: number[]): number {
+    if (a.length !== b.length) {
+      throw new Error(
+        `Vectors must have the same dimension. Got ${a.length} and ${b.length}`
+      );
+    }
+
+    // Normalizáljuk a vektorokat
+    const normalizedA: number[] = this.l2Normalize(a);
+    const normalizedB: number[] = this.l2Normalize(b);
+
+    // Dot product számítása
+    let dotProduct: number = 0;
+    for (let i: number = 0; i < normalizedA.length; i++) {
+      dotProduct += normalizedA[i] * normalizedB[i];
+    }
+
+    // Cosine similarity = dot product of normalized vectors
+    // Mivel a vektorok normalizáltak, ez egyenlő a cosine similarity-vel
+    return dotProduct;
+  }
+
+  /**
+   * L2 distance (Euklideszi távolság) számítás két vektor között
+   * Eredmény: 0 vagy pozitív érték, ahol 0 a legkisebb távolság
+   * Statikus metódus, mivel nincs szüksége instance state-re
+   */
+  static l2Distance(a: number[], b: number[]): number {
+    if (a.length !== b.length) {
+      throw new Error(
+        `Vectors must have the same dimension. Got ${a.length} and ${b.length}`
+      );
+    }
+
+    // L2 distance = sqrt(sum((a[i] - b[i])^2))
+    let sumSquaredDiff: number = 0;
+    for (let i: number = 0; i < a.length; i++) {
+      const diff: number = a[i] - b[i];
+      sumSquaredDiff += diff * diff;
+    }
+
+    return Math.sqrt(sumSquaredDiff);
+  }
+
+  /**
+   * Vektor keresés a pool-ban
+   * Brute-force megközelítés (lineáris keresés)
+   * 
+   * @param query A keresési query vektor
+   * @param k A visszaadandó találatok száma (top-K)
+   * @param mode A keresési mód (cosine similarity vagy L2 distance)
+   * @returns A top-K találatok ID-val és score-rel, megfelelő sorrendben
+   */
+  search(
+    query: number[],
+    k: number,
+    mode: LVS_Search_Mode
+  ): LVS_SearchResult[] {
+    if (!Array.isArray(query) || query.length === 0) {
+      throw new Error('Query vector must be a non-empty array');
+    }
+
+    if (k <= 0) {
+      throw new Error('k must be a positive number');
+    }
+
+    if (this.vectorPool.size === 0) {
+      return [];
+    }
+
+    // Validáljuk, hogy minden vektor ugyanolyan dimenziójú-e
+    const firstVector: number[] | undefined = Array.from(this.vectorPool.values())[0];
+    if (firstVector && firstVector.length !== query.length) {
+      throw new Error(
+        `Query vector dimension (${query.length}) does not match pool vector ` +
+        `dimension (${firstVector.length})`
+      );
+    }
+
+    // Számoljuk ki a hasonlóságot/távolságot minden vektorra
+    const results: LVS_SearchResult[] = [];
+
+    for (const [id, vector] of this.vectorPool.entries()) {
+      let score: number;
+
+      if (mode === LVS_Search_Mode.cosineSimilarity) {
+        // Cosine similarity: nagyobb érték = jobb hasonlóság
+        score = LVS_VectorPool_ControlService.cosineSimilarity(query, vector);
+      } else if (mode === LVS_Search_Mode.l2Distance) {
+        // L2 distance: kisebb érték = jobb hasonlóság
+        score = LVS_VectorPool_ControlService.l2Distance(query, vector);
+      } else {
+        throw new Error(`Unknown search mode: ${mode}`);
+      }
+
+      results.push({ id, score });
+    }
+
+    // Rendezzük az eredményeket
+    if (mode === LVS_Search_Mode.cosineSimilarity) {
+      // Cosine similarity: csökkenő sorrend (legnagyobb hasonlóság először)
+      results.sort((a: LVS_SearchResult, b: LVS_SearchResult) => b.score - a.score);
+    } else {
+      // L2 distance: növekvő sorrend (legkisebb távolság először)
+      results.sort((a: LVS_SearchResult, b: LVS_SearchResult) => a.score - b.score);
+    }
+
+    // Visszaadjuk a top-K találatot
+    return results.slice(0, k);
+  }
+}
+

package/src/_modules/local-vector-search/index.ts

@@ -0,0 +1,12 @@
+
+
+// ENUMS
+export * from './_enums/lvs-search-mode.enum';
+
+// INTERFACES
+export * from './_models/lvs-search-result.interface';
+
+// SERVICES
+export * from './_services/lvs-doc-chunk-data.service';
+export * from './_services/lvs-local-vector-search.data-service';
+export * from './_services/lvs-vector-pool.control-service';

package/src/_services/base/data.service.ts

@@ -17,6 +17,7 @@ import {
   DyFM_Log,
   DyFM_Metadata,
   DyFM_NestPropertySearch,
+  DyFM_Object,
   DyFM_RangeValue,
   DyFM_SearchQuery,
   DyFM_SearchResult,
@@ -792,7 +793,7 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
     if (!data) {
       throw new DyFM_Error({ 
         ...this._getDefaultErrorSettings(
-          'saveData', 
+          'ensureData', 
           new Error(`no data to save! (${this.dataParams.dataName})`)
         ),
         
@@ -803,6 +804,44 @@ export class DyNTS_DataService<T extends DyFM_Metadata> {
     return data;
   }
 
+  async patchData(data?: T): Promise<T> {
+    try {
+      data = this.ensureData(data);
+
+      if (!data._id) {
+        throw new DyFM_Error({ 
+          ...this._getDefaultErrorSettings(
+            'patchData',
+            new Error(`no ID to patch data! (${this.dataParams.dataName})`)
+          ),
+        });
+      }
+
+      const dataExists = await this.getDataById(data._id, true);
+
+      if (!dataExists) {
+        throw new DyFM_Error({ 
+          ...this._getDefaultErrorSettings(
+            'patchData',
+            new Error(`data not found! (${this.dataParams.dataName})`)
+          ),
+        });
+      }
+
+      DyFM_Object.cleanAssign(dataExists, data);
+      
+      await this.validateForSave(dataExists);
+
+      return await this.dataDBService.modifyData(dataExists, this.issuer);  
+    } catch (error) {
+      throw new DyFM_Error({ 
+        ...this._getDefaultErrorSettings('patchData', error),
+        
+        errorCode: `${DyNTS_global_settings.systemShortCodeName}|DyNTS-DS0-PD0`,
+      });
+    }
+  }
+
   /**
    * modifies data if the data have ID and already exists in the DB,   
    * creates new if the ID is not present or cant find in DB,