@futdevpro/nts-dynamo 1.15.156 → 1.15.160

package/src/_modules/local-vector-search/_services/lvs-persistent-vector-pool.control-service.ts

@@ -1,186 +0,0 @@
-
-import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
-
-import { LVS_Search_Mode } from '../_enums/lvs-search-mode.enum';
-import { LVS_SearchResult } from '../_models/lvs-search-result.interface';
-import { DyNTS_LVS_VectorPersist } from '../_models/data-models/lvs-vector-persist.data-model';
-import { DyNTS_LVS_VectorPersist_DataService } from './lvs-vector-persist.data-service';
-import { LVS_VectorPool_ControlService } from './lvs-vector-pool.control-service';
-
-/**
- * `DyNTS_LVS_PersistentVectorPool_ControlService` (BFR-AM-001) — egy `LVS_VectorPool_ControlService` in-memory pool +
- * MongoDB-perzisztencia **kompozíciója**. A FAM (fdp-agent-memory) `FAM_VectorSearch_ControlService`
- * persist+hydrate workaround-ját generalizálja bedrock-szintre: a vektorok a SAJÁT Mongo-ban
- * (`dynts_lvs_vector`) élnek, boot-kor a memória-pool-ba hidratálódnak — **NEM** MongoDB Atlas
- * Vector Search.
- *
- * **Miért KOMPOZÍCIÓ (NEM a pool-class módosítása):** a `LVS_VectorPool_ControlService`-t egy másik,
- * párhuzamos WIP (hybrid cosine+BM25) is érinti; a pool-class változtatása merge-konfliktust okozna.
- * Ez a wrapper a pool-t VÁLTOZATLANUL hagyja (delegál: `addVector`/`removeVector`/`updateVector`/
- * `search`), és a perzisztencia-mellékhatást a wrapper-metódusokban végzi. Egy wrapper-instance egy
- * `collectionKey` (logikai pool); több pool = több wrapper.
- *
- * **FIGYELEM (memory: dynts_dataservice_eager_resolve):** a `DyNTS_DataService` base-ctor EAGER
- * `getDBService`-t hív, ezért a wrapper NEM tart élő data-service-mezőt — minden DB-művelet előtt lazy
- * `new DyNTS_LVS_VectorPersist_DataService(...)` (a `getPersistService` ezt adja).
- */
-export class DyNTS_LVS_PersistentVectorPool_ControlService {
-
-  /** A wrapped in-memory pool (a Dynamo LVS engine; VÁLTOZATLAN — kompozíció, nem módosítás). */
-  private readonly pool: LVS_VectorPool_ControlService;
-
-  /** A pool-particionálás kulcsa (a Mongo-rekordok `collectionKey`-e + a hidratálás-szűrő). */
-  private readonly collectionKey: string;
-
-  /** A perzisztencia-műveletek issuer-e (audit). */
-  private readonly issuer: string;
-
-  constructor(set: {
-    collectionKey: string,
-    issuer: string,
-    pool?: LVS_VectorPool_ControlService,
-  }) {
-    this.collectionKey = set.collectionKey;
-    this.issuer = set.issuer;
-    // Saját pool, ha a hívó nem ad be egyet (a wrapper nem módosítja a pool-osztályt — kompozíció).
-    this.pool = set.pool ?? new LVS_VectorPool_ControlService();
-  }
-
-  /** A wrapped in-memory pool (közvetlen olvasáshoz — pl. más search-mód, getAll). */
-  getPool(): LVS_VectorPool_ControlService {
-    return this.pool;
-  }
-
-  /**
-   * Egy vektor felvétele a memória-pool-ba ÉS tartós upsert-je Mongo-ba (persist-on-write, FAM-minta).
-   * A pool `addVector`-ja upsert-szemantikájú (azonos kulcson felülír), a Mongo-oldal is `(collectionKey,
-   * vectorId)`-upsert. A Mongo-írás MELLÉKHATÁS a memória-művelet után (a memória a forrás, a Mongo a tár).
-   */
-  async addVector(vectorId: string, vector: number[], metadata?: unknown): Promise<void> {
-    this.pool.addVector(vectorId, vector);
-    await this.getPersistService().upsertVector({
-      collectionKey: this.collectionKey,
-      vectorId: vectorId,
-      embedding: vector,
-      metadata: metadata,
-    });
-  }
-
-  /**
-   * Egy MEGLÉVŐ vektor frissítése a memória-pool-ban + tartós upsert Mongo-ba. A pool `updateVector`-ja
-   * dob, ha a kulcs nem létezik (a pool-szerződés változatlan) — a Mongo-írás csak siker után fut.
-   */
-  async updateVector(vectorId: string, vector: number[], metadata?: unknown): Promise<void> {
-    this.pool.updateVector(vectorId, vector);
-    await this.getPersistService().upsertVector({
-      collectionKey: this.collectionKey,
-      vectorId: vectorId,
-      embedding: vector,
-      metadata: metadata,
-    });
-  }
-
-  /** Egy vektor eltávolítása a memória-pool-ból ÉS a tartós tárból (`(collectionKey, vectorId)`). */
-  async removeVector(vectorId: string): Promise<void> {
-    this.pool.removeVector(vectorId);
-    await this.getPersistService().removeVector(this.collectionKey, vectorId);
-  }
-
-  /** Keresés a memória-pool-on (a wrapped pool `search`-jét delegálja — VÁLTOZATLAN engine). */
-  search(query: number[], k: number, mode: LVS_Search_Mode): LVS_SearchResult[] {
-    return this.pool.search(query, k, mode);
-  }
-
-  /**
-   * BOOT-HIDRATÁLÁS (FAM-minta, BFR-AM-001): a `collectionKey` ÖSSZES perzistált vektorát betölti a
-   * MEMÓRIA-pool-ba (`pool.addVector`), így a szerver-restart utáni in-memory index újraépül. A pool-t
-   * előbb üríti (idempotens hidratálás). A hiányos (üres `vectorId`/`embedding`) rekordokat átugorja.
-   * Visszaadja a betöltött vektorok számát. **NEM** Atlas — saját Mongo + in-memory pool.
-   */
-  async hydrateFromMongo(set?: {
-    /** Cursor-batch méret (default 500) — a heap-tranziens felső korlátja egy batch. */
-    batchSize?: number,
-    /** Opcionális felső korlát a pool-méretre; elérésekor a hidratálás áll + WARN (no-silent-cap). */
-    maxVectors?: number,
-  }): Promise<number> {
-    // BFR-AM-013 (hydration-paging): a korábbi impl a TELJES collection-t egyetlen
-    // `findByCollectionKey`-jel (Mongoose-doc-tömb) töltötte memóriába → 32k×2560-dim vektornál
-    // RUNTIME-IGAZOLT 93% heap-spike (OOM-közeli). Mostantól LEAN + projektált (vectorId+embedding,
-    // a content-szöveg NÉLKÜL) CURSOR-batch — a tranziens felső korlátja EGY batch.
-    this.pool.clearPool();
-
-    let loaded: number = 0;
-    let skipped: number = 0;
-    let truncated: boolean = false;
-
-    await this.getPersistService().findLeanBatchedByCollectionKey(this.collectionKey, {
-      batchSize: set?.batchSize,
-      handler: (batch: Partial<DyNTS_LVS_VectorPersist>[]): void => {
-        for (const record of batch) {
-          if (truncated) {
-            return;
-          }
-          if (set?.maxVectors !== undefined && loaded >= set.maxVectors) {
-            truncated = true;
-            DyFM_Log.warn(
-              `[DyNTS LVS hydrate] '${this.collectionKey}': maxVectors (${set.maxVectors}) elérve — ` +
-              'a hidratálás CSONKOLVA (no-silent-cap; a maradék vektor NEM került a poolba).',
-            );
-            return;
-          }
-          if (!record.vectorId || !record.embedding?.length) {
-            continue;
-          }
-          try {
-            this.pool.addVector(record.vectorId, record.embedding);
-            loaded++;
-          } catch (error) {
-            // BFR-AM-013 (NaN/±Inf-guard a hidratálásban): EGY hibás (nem-véges elemű) perzistált
-            // vektor NEM boríthatja a teljes boot-hidratálást — skip + warn (nem néma).
-            skipped++;
-            DyFM_Log.warn(
-              `[DyNTS LVS hydrate] '${this.collectionKey}': vektor kihagyva ("${record.vectorId}") — ` +
-              `${DyFM_Error.getErrorMessage(error)}`,
-            );
-          }
-        }
-      },
-    });
-
-    DyFM_Log.log(
-      `[DyNTS LVS hydrate] '${this.collectionKey}' pool: ${loaded} vektor betöltve az in-memory pool-ba` +
-      (skipped ? ` (${skipped} hibás vektor kihagyva)` : '') +
-      (truncated ? ' (CSONKOLT — maxVectors)' : '') + '.',
-    );
-
-    return loaded;
-  }
-
-  /**
-   * Perzistált rekordok közvetlen hidratálása a memória-pool-ba (Mongo-olvasás NÉLKÜL) — ha a hívó már
-   * kezében van a rekord-halmaz (pl. batch-boot). A pool-t előbb üríti (idempotens). Visszaadja a
-   * betöltött vektorok számát.
-   */
-  hydrate(records: DyNTS_LVS_VectorPersist[]): number {
-    this.pool.clearPool();
-    let loaded: number = 0;
-
-    for (const record of records) {
-      if (!record.vectorId || !record.embedding?.length) {
-        continue;
-      }
-      this.pool.addVector(record.vectorId, record.embedding);
-      loaded++;
-    }
-
-    return loaded;
-  }
-
-  /**
-   * Lazy persist-data-service (memory: dynts_dataservice_eager_resolve — a base-ctor eager
-   * `getDBService`-e miatt NEM tartható élő mezőként).
-   */
-  private getPersistService(): DyNTS_LVS_VectorPersist_DataService {
-    return new DyNTS_LVS_VectorPersist_DataService({ issuer: this.issuer });
-  }
-}

package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.spec.ts

@@ -1,167 +0,0 @@
-
-import { DyFM_EnvironmentFlag } from '@futdevpro/fsm-dynamo';
-
-import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
-import { DyNTS_GlobalService } from '../../../_services/core/global.service';
-import { DyNTS_LVS_VectorPersist } from '../_models/data-models/lvs-vector-persist.data-model';
-import { DyNTS_LVS_VectorPersist_DataService } from './lvs-vector-persist.data-service';
-
-/**
- * BFR-AM-001 — a `DyNTS_LVS_VectorPersist_DataService` spec-jei. A base-ctor eager `getDBService`-jét
- * stub-oljuk (memory: dynts_dataservice_eager_resolve), így NEM kell élő Mongo. Az upsert atomikus
- * `$set`-jét (Mixed `metadata` — memory: mongoose_mixed_atomic_write) + a dimenzió-tárolást bizonyítja.
- */
-describe('| DyNTS_LVS_VectorPersist_DataService', () => {
-
-  let mockDBService: jasmine.SpyObj<{ find: () => Promise<unknown[]>; findOne: () => Promise<unknown> }>;
-
-  beforeAll(() => {
-    if (!DyNTS_global_settings.systemShortCodeName) {
-      (DyNTS_global_settings as { systemShortCodeName?: string }).systemShortCodeName = 'TEST';
-    }
-    if (!DyNTS_global_settings.env_settings) {
-      (DyNTS_global_settings as { env_settings?: unknown }).env_settings = {
-        environment: DyFM_EnvironmentFlag.local,
-      };
-    }
-  });
-
-  beforeEach(() => {
-    mockDBService = jasmine.createSpyObj('DyNTS_DBService', ['find', 'findOne']);
-    mockDBService.find.and.returnValue(Promise.resolve([]));
-    mockDBService.findOne.and.returnValue(Promise.resolve(null));
-    spyOn(DyNTS_GlobalService, 'getDBService').and.returnValue(mockDBService as never);
-  });
-
-  describe('| constructor', () => {
-
-    it('| should create the service with a data model', () => {
-      const data: DyNTS_LVS_VectorPersist = new DyNTS_LVS_VectorPersist({
-        collectionKey: 'pool-a',
-        vectorId: 'v1',
-        embedding: [1, 2, 3],
-      });
-
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ data: data, issuer: 'issuer-123' });
-
-      expect(service).toBeInstanceOf(DyNTS_LVS_VectorPersist_DataService);
-      expect(service.data).toBeDefined();
-      expect(service.data.vectorId).toBe('v1');
-      expect(service.data.collectionKey).toBe('pool-a');
-    });
-
-    it('| should create the service without data', () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      expect(service).toBeInstanceOf(DyNTS_LVS_VectorPersist_DataService);
-      expect(service.data).toBeInstanceOf(DyNTS_LVS_VectorPersist);
-    });
-  });
-
-  describe('| findByCollectionKey', () => {
-
-    it('| should query the persisted vectors for a collectionKey', async () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      const records: DyNTS_LVS_VectorPersist[] = [
-        new DyNTS_LVS_VectorPersist({ collectionKey: 'pool-a', vectorId: 'v1', embedding: [1, 0] }),
-      ];
-      const findSpy = spyOn(service, 'findDataList').and.returnValue(Promise.resolve(records));
-
-      const result: DyNTS_LVS_VectorPersist[] = await service.findByCollectionKey('pool-a');
-
-      expect(findSpy).toHaveBeenCalledTimes(1);
-      const filterArg = findSpy.calls.mostRecent().args[0] as { collectionKey: string };
-      expect(filterArg.collectionKey).toBe('pool-a');
-      expect(result).toBe(records);
-    });
-  });
-
-  describe('| upsertVector — atomic $set + dimension store', () => {
-
-    it('| should issue an updateOne $set (with dimensions) for an EXISTING record', async () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      const existing: DyNTS_LVS_VectorPersist = new DyNTS_LVS_VectorPersist({
-        collectionKey: 'pool-a', vectorId: 'v1', embedding: [1, 2, 3],
-      });
-      existing._id = 'rec-1';
-
-      spyOn(service, 'findData').and.returnValue(Promise.resolve(existing));
-      const updateSpy = spyOn(service, 'updateData').and.returnValue(Promise.resolve());
-      const saveSpy = spyOn(service, 'saveData').and.returnValue(Promise.resolve(existing));
-
-      await service.upsertVector({
-        collectionKey: 'pool-a', vectorId: 'v1', embedding: [4, 5, 6, 7], metadata: { tag: 'x' },
-      });
-
-      expect(saveSpy).not.toHaveBeenCalled();
-      expect(updateSpy).toHaveBeenCalledTimes(1);
-      const arg = updateSpy.calls.mostRecent().args[0] as {
-        filterBy: { _id: string };
-        update: { $set: { embedding: number[]; dimensions: number; metadata: unknown } };
-      };
-      expect(arg.filterBy._id).toBe('rec-1');
-      expect(arg.update.$set.embedding).toEqual([4, 5, 6, 7]);
-      expect(arg.update.$set.dimensions).toBe(4);
-      expect(arg.update.$set.metadata).toEqual({ tag: 'x' });
-    });
-
-    it('| should saveData a NEW record (with stored dimensions) when none exists', async () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      spyOn(service, 'findData').and.returnValue(Promise.resolve(null));
-      const updateSpy = spyOn(service, 'updateData').and.returnValue(Promise.resolve());
-      const saveSpy = spyOn(service, 'saveData')
-        .and.callFake((data?: DyNTS_LVS_VectorPersist) =>
-          Promise.resolve(data as DyNTS_LVS_VectorPersist));
-
-      await service.upsertVector({ collectionKey: 'pool-a', vectorId: 'v2', embedding: [0.1, 0.2] });
-
-      expect(updateSpy).not.toHaveBeenCalled();
-      expect(saveSpy).toHaveBeenCalledTimes(1);
-      const written = saveSpy.calls.mostRecent().args[0] as DyNTS_LVS_VectorPersist;
-      expect(written.vectorId).toBe('v2');
-      expect(written.collectionKey).toBe('pool-a');
-      expect(written.embedding).toEqual([0.1, 0.2]);
-      expect(written.dimensions).toBe(2);
-    });
-  });
-
-  describe('| removeVector', () => {
-
-    it('| should absolute-delete the matching record', async () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      const existing: DyNTS_LVS_VectorPersist = new DyNTS_LVS_VectorPersist({
-        collectionKey: 'pool-a', vectorId: 'v1', embedding: [1, 0],
-      });
-      existing._id = 'rec-1';
-
-      spyOn(service, 'findData').and.returnValue(Promise.resolve(existing));
-      const deleteSpy = spyOn(service, 'deleteData').and.returnValue(Promise.resolve());
-
-      await service.removeVector('pool-a', 'v1');
-
-      expect(deleteSpy).toHaveBeenCalledWith('rec-1', true);
-    });
-
-    it('| should be a no-op when no matching record exists', async () => {
-      const service: DyNTS_LVS_VectorPersist_DataService =
-        new DyNTS_LVS_VectorPersist_DataService({ issuer: 'issuer-123' });
-
-      spyOn(service, 'findData').and.returnValue(Promise.resolve(null));
-      const deleteSpy = spyOn(service, 'deleteData').and.returnValue(Promise.resolve());
-
-      await service.removeVector('pool-a', 'missing');
-
-      expect(deleteSpy).not.toHaveBeenCalled();
-    });
-  });
-});

package/src/_modules/local-vector-search/_services/lvs-vector-persist.data-service.ts

@@ -1,130 +0,0 @@
-
-import { DyFM_DBFilter, DyFM_DBFilterSimple } from '@futdevpro/fsm-dynamo';
-
-import { DyNTS_DataService } from '../../../_services/base/data.service';
-import {
-  DyNTS_LVS_VectorPersist,
-  DyNTS_lvsVectorPersist_dataParams
-} from '../_models/data-models/lvs-vector-persist.data-model';
-
-/**
- * `DyNTS_LVS_VectorPersist_DataService` (BFR-AM-001) — a `dynts_lvs_vector` collection CRUD-rétege; a
- * `LVS_VectorPool_ControlService` in-memory pool-jának tartós Mongo-háttértára. `extends
- * DyNTS_DataService`; a nts-konvenció szerinti `{ data?, issuer }` constructor.
- *
- * **FIGYELEM (memory: dynts_dataservice_eager_resolve):** a `DyNTS_DataService` base-ctor EAGER
- * `getDBService`-t hív → a hívó NEM tarthat élő data-service-példányt mezőként; minden művelet előtt
- * lazy `new DyNTS_LVS_VectorPersist_DataService(...)` kell (a `DyNTS_LVS_PersistentVectorPool_ControlService` így jár el).
- *
- * **Atomikus írás (memory: mongoose_mixed_atomic_write):** az upsert a `(collectionKey, vectorId)`
- * logikai-kulcson MEGLÉVŐ rekordot `updateOne({...},{$set:{embedding,dimensions,metadata}})`-tel
- * frissíti (a Mixed `metadata` `findOne→mutate→save` SILENT-DROP-ja ellen); ÚJ rekordot `saveData`-val ír.
- */
-export class DyNTS_LVS_VectorPersist_DataService extends DyNTS_DataService<DyNTS_LVS_VectorPersist> {
-
-  constructor(
-    set: {
-      data?: DyNTS_LVS_VectorPersist,
-      issuer: string,
-    },
-  ) {
-    super(
-      set.data instanceof DyNTS_LVS_VectorPersist
-        ? set.data
-        : new DyNTS_LVS_VectorPersist(set.data),
-      DyNTS_lvsVectorPersist_dataParams,
-      set.issuer,
-    );
-  }
-
-  /**
-   * Egy (logikai) pool ÖSSZES perzistált, nem-soft-delete-elt vektor-rekordja — a boot-hidratáláshoz
-   * (a `DyNTS_LVS_PersistentVectorPool_ControlService.hydrateFromMongo` ezt tölti az in-memory pool-ba).
-   */
-  async findByCollectionKey(collectionKey: string): Promise<DyNTS_LVS_VectorPersist[]> {
-    return this.findDataList(
-      { collectionKey: collectionKey },
-      true,
-    );
-  }
-
-  /**
-   * BFR-AM-013 (hydration-paging): LEAN + projektált (CSAK `vectorId` + `embedding` — a content-szöveg
-   * és metadata NÉLKÜL) cursor-batch scan a boot-hidratáláshoz. A heap-tranziens felső korlátja EGY batch
-   * (a teljes result-set + Mongoose-doc-boxing kimarad). Soft-delete-elt rekordok kihagyva.
-   */
-  async findLeanBatchedByCollectionKey(
-    collectionKey: string,
-    set: {
-      batchSize?: number,
-      handler: (batch: Partial<DyNTS_LVS_VectorPersist>[]) => Promise<void> | void,
-    }
-  ): Promise<number> {
-    return this.dataDBService.findLeanBatched(
-      { collectionKey: collectionKey, _deleted: undefined },
-      {
-        projection: { vectorId: 1, embedding: 1 },
-        batchSize: set.batchSize,
-        handler: set.handler,
-      }
-    );
-  }
-
-  /**
-   * Egy vektor ATOMIKUS upsert-je a `(collectionKey, vectorId)` logikai-kulcson. MEGLÉVŐ rekordnál
-   * `$set` (a Mixed `metadata` silent-drop ellen — memory: mongoose_mixed_atomic_write); ÚJ rekordnál
-   * `saveData`. A dimenzió az `embedding.length`-ből rögzül.
-   */
-  async upsertVector(set: {
-    collectionKey: string,
-    vectorId: string,
-    embedding: number[],
-    metadata?: unknown,
-  }): Promise<void> {
-    const existing: DyNTS_LVS_VectorPersist = await this.findData(
-      { collectionKey: set.collectionKey, vectorId: set.vectorId },
-      true,
-    );
-
-    if (existing?._id) {
-      await this.updateData({
-        filterBy: { _id: existing._id },
-        update: {
-          $set: {
-            embedding: set.embedding,
-            dimensions: set.embedding.length,
-            metadata: set.metadata,
-          },
-        },
-      });
-
-      return;
-    }
-
-    await this.saveData(
-      new DyNTS_LVS_VectorPersist({
-        collectionKey: set.collectionKey,
-        vectorId: set.vectorId,
-        embedding: set.embedding,
-        dimensions: set.embedding.length,
-        metadata: set.metadata,
-      }),
-      true,
-    );
-  }
-
-  /**
-   * Egy vektor-rekord végleges törlése a `(collectionKey, vectorId)` logikai-kulcson (a pool
-   * `removeVector`-ja után — a tartós tár is törli). `addArchive` nincs → `absolute` true-val töröl.
-   */
-  async removeVector(collectionKey: string, vectorId: string): Promise<void> {
-    const existing: DyNTS_LVS_VectorPersist = await this.findData(
-      { collectionKey: collectionKey, vectorId: vectorId },
-      true,
-    );
-
-    if (existing?._id) {
-      await this.deleteData(existing._id, true);
-    }
-  }
-}