@futdevpro/nts-dynamo 1.15.157 → 1.15.160

package/src/_modules/ai/_modules/open-ai/_services/data-services/oai-vector-data.service.spec.ts

@@ -1,550 +0,0 @@
-
-import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
-import { DyFM_Metadata, DyFM_DataModel_Params, DyFM_DataProperty_Params, DyFM_BasicProperty_Type, DyFM_Error, DyFM_Object, DyFM_EnvironmentFlag } from '@futdevpro/fsm-dynamo';
-import { DyFM_OAI_Settings, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-import { DyNTS_OAI_Embedding_ControlService } from '../oai-embedding.control-service';
-import { DyNTS_DataService } from '../../../../../../_services/base/data.service';
-import { DyNTS_GlobalService } from '../../../../../../_services/core/global.service';
-import { CreateEmbeddingResponse } from 'openai/resources';
-import { DyNTS_global_settings } from '../../../../../../_collections/global-settings.const';
-
-class TestMetadata extends DyFM_Metadata {
-  name: string;
-  content: string;
-  contentVectorized?: number[];
-}
-
-const testDataParams = new DyFM_DataModel_Params<TestMetadata>({
-  dataName: 'test_data',
-  properties: {
-    name: {
-      key: 'name',
-      type: DyFM_BasicProperty_Type.string,
-      required: true,
-    },
-    content: {
-      key: 'content',
-      type: DyFM_BasicProperty_Type.string,
-    },
-    contentVectorized: {
-      key: 'contentVectorized',
-      type: 'number[]',  // Must be number[] for vectorized property
-      vectorizedFrom: ['content'],
-      embeddingModel: DyFM_OAI_Model.textEmbedding_3Small,
-      vectorizeUseIndex: 'content_vector_index',
-    },
-  },
-});
-
-describe('| DyNTS_OAI_VectorDataService', () => {
-  let service: DyNTS_OAI_VectorDataService<TestMetadata>;
-  let mockEmbedding_CS: jasmine.SpyObj<DyNTS_OAI_Embedding_ControlService>;
-  let mockDBService: jasmine.SpyObj<any>;
-  let testData: TestMetadata;
-
-  const mockOpenAISettings: DyFM_OAI_Settings = {
-    config: {
-      apiKey: 'test-api-key',
-      organization: 'test-org',
-    },
-    defaultSettings: {
-      useModel: DyFM_OAI_Model.textEmbedding_3Small,
-    },
-  };
-
-  beforeEach(() => {
-    DyNTS_global_settings.env_settings = {
-      environment: DyFM_EnvironmentFlag.local,
-      mongoUri: 'mongodb://localhost:27017/test',
-    } as typeof DyNTS_global_settings.env_settings;
-
-    testData = Object.assign(new TestMetadata(), {
-      _id: 'test-id-123',
-      content: 'Test content',
-    });
-
-    mockDBService = jasmine.createSpyObj('DyNTS_DBService', [
-      'getAll', 'getDataById', 'aggregate', 'createData', 'updateOne', 'find', 'findOne',
-      'getDataListByIds', 'getDataByDependencyId', 'getDataListByDependencyId',
-      'getDataListByDependencyIds', 'modifyData', 'markDeletedById', 'trueDeleteDataById',
-      'trueDeleteAllData', 'restoreDeletedById',
-    ]);
-
-    mockEmbedding_CS = jasmine.createSpyObj('DyNTS_OAI_Embedding_ControlService', ['createEmbedding']);
-
-    spyOn(DyNTS_GlobalService, 'getDBService').and.returnValue(mockDBService as never);
-    spyOn(DyNTS_GlobalService, 'getDBServiceByKey').and.returnValue(mockDBService as never);
-
-    spyOn(DyNTS_DataService.prototype, 'constructor' as never).and.callThrough();
-    spyOn(DyNTS_DataService.prototype, 'getDataById').and.returnValue(Promise.resolve(testData));
-    spyOn(DyNTS_DataService.prototype, 'saveData').and.returnValue(Promise.resolve(testData));
-    spyOn(DyNTS_DataService.prototype as unknown as { ensureData: (d?: unknown) => TestMetadata }, 'ensureData').and.returnValue(testData);
-
-    service = new DyNTS_OAI_VectorDataService<TestMetadata>(
-      testData,
-      testDataParams,
-      mockOpenAISettings,
-      'test-issuer'
-    );
-
-    (service as { dataDBService: typeof mockDBService }).dataDBService = mockDBService;
-    (service as unknown as { embedding_CS: typeof mockEmbedding_CS }).embedding_CS = mockEmbedding_CS;
-  });
-
-  describe('| constructor', () => {
-    it('| should initialize with data, params, and settings', () => {
-      expect(service.data).toBe(testData);
-      expect(service.dataParams).toBe(testDataParams);
-      expect((service as any).openAISettings).toEqual(mockOpenAISettings);
-      expect(service.issuer).toBe('test-issuer');
-    });
-
-    it('| should identify vectorized properties', () => {
-      expect((service as any).vectorizedProperties).toBeDefined();
-      expect((service as any).vectorizedProperties.length).toBeGreaterThan(0);
-    });
-
-    it('| should identify vectorize properties', () => {
-      expect((service as any).vectorizeProperties).toBeDefined();
-      expect((service as any).vectorizeProperties.length).toBeGreaterThan(0);
-    });
-
-    it('| should throw error if apiKey not provided', () => {
-      const invalidSettings: DyFM_OAI_Settings = {
-        config: {
-          apiKey: '',
-        },
-        defaultSettings: {
-          useModel: DyFM_OAI_Model.textEmbedding_3Small,
-        },
-      };
-
-      expect(() => {
-        new DyNTS_OAI_VectorDataService<TestMetadata>(
-          testData,
-          testDataParams,
-          invalidSettings,
-          'test-issuer'
-        );
-      }).toThrow();
-    });
-
-    it('| should throw error if vectorizedProperties missing', () => {
-      const paramsWithoutVectorized = new DyFM_DataModel_Params<TestMetadata>({
-        dataName: 'test_data',
-        properties: {
-          name: {
-            key: 'name',
-            type: DyFM_BasicProperty_Type.string,
-          },
-          content: {
-            key: 'content',
-            type: DyFM_BasicProperty_Type.string,
-          },
-        },
-      });
-
-      expect(() => {
-        new DyNTS_OAI_VectorDataService<TestMetadata>(
-          testData,
-          paramsWithoutVectorized,
-          mockOpenAISettings,
-          'test-issuer'
-        );
-      }).toThrow();
-    });
-  });
-
-  describe('| vectorize', () => {
-    it('| should create embedding for input', async () => {
-      const mockResponse: CreateEmbeddingResponse = {
-        data: [
-          {
-            embedding: [0.1, 0.2, 0.3],
-            index: 0,
-            object: 'embedding',
-          },
-        ],
-        model: 'text-embedding-3-small',
-        object: 'list',
-        usage: {
-          prompt_tokens: 5,
-          total_tokens: 5,
-        },
-      };
-
-      mockEmbedding_CS.createEmbedding.and.returnValue(Promise.resolve(mockResponse));
-
-      const result = await service.vectorize('Test input');
-
-      expect(result).toEqual([0.1, 0.2, 0.3]);
-      expect(mockEmbedding_CS.createEmbedding).toHaveBeenCalledWith({
-        text: 'Test input',
-        model: DyFM_OAI_Model.textEmbedding_3Small,
-        fullResponse: true,
-        issuer: 'test-issuer',
-      });
-      expect((service as any).tokenSpent).toBe(5);
-    });
-
-    it('| should throw error if input is empty', async () => {
-      try {
-        await service.vectorize('');
-        fail('Should have thrown an error');
-      } catch (error) {
-        expect(error).toBeInstanceOf(DyFM_Error);
-        expect((error as DyFM_Error)._errorCode).toContain('DyNTS-VDB-V1');
-      }
-    });
-
-    it('| should handle embedding errors', async () => {
-      const error = new Error('Embedding failed');
-      mockEmbedding_CS.createEmbedding.and.returnValue(Promise.reject(error));
-
-      try {
-        await service.vectorize('Test input');
-        fail('Should have thrown an error');
-      } catch (err) {
-        expect(err).toBeInstanceOf(DyFM_Error);
-        expect((err as DyFM_Error)._errorCode).toContain('DyNTS-VDB-V0');
-      }
-    });
-  });
-
-  describe('| vectorizeDataProperties', () => {
-    it('| should vectorize properties when data is new', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-      });
-
-      (DyNTS_DataService.prototype.getDataById as jasmine.Spy).and.returnValue(Promise.resolve(null));
-      spyOn(service, 'vectorize').and.returnValue(Promise.resolve([0.1, 0.2, 0.3]));
-
-      const result = await service.vectorizeDataProperties(newData);
-
-      expect(service.vectorize).toHaveBeenCalled();
-      expect(result.contentVectorized).toEqual([0.1, 0.2, 0.3]);
-    });
-
-    it('| should not vectorize if data unchanged', async () => {
-      const existingData: TestMetadata = Object.assign(new TestMetadata(), {
-        _id: 'test-id-123',
-        content: 'Test content',
-        contentVectorized: [0.1, 0.2, 0.3],
-      });
-
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        _id: 'test-id-123',
-        content: 'Test content',
-      });
-
-      (DyNTS_DataService.prototype.getDataById as jasmine.Spy).and.returnValue(Promise.resolve(existingData));
-      spyOn(service, 'vectorize').and.returnValue(Promise.resolve([0.4, 0.5, 0.6]));
-
-      const result = await service.vectorizeDataProperties(newData);
-
-      expect(service.vectorize).not.toHaveBeenCalled();
-      expect(result.contentVectorized).toEqual([0.1, 0.2, 0.3]);
-    });
-
-    it('| should vectorize if forceVectorize is true', async () => {
-      const existingData: TestMetadata = Object.assign(new TestMetadata(), {
-        _id: 'test-id-123',
-        content: 'Test content',
-        contentVectorized: [0.1, 0.2, 0.3],
-      });
-
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        _id: 'test-id-123',
-        content: 'Test content',
-      });
-
-      (DyNTS_DataService.prototype.getDataById as jasmine.Spy).and.returnValue(Promise.resolve(existingData));
-      spyOn(service, 'vectorize').and.returnValue(Promise.resolve([0.4, 0.5, 0.6]));
-
-      const result = await service.vectorizeDataProperties(newData, true);
-
-      expect(service.vectorize).toHaveBeenCalled();
-      expect(result.contentVectorized).toEqual([0.4, 0.5, 0.6]);
-    });
-
-    it('| should throw error if embedding model not set', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-      });
-
-      const paramsWithoutModel = new DyFM_DataModel_Params<TestMetadata>({
-        dataName: 'test_data',
-        properties: {
-          name: {
-            key: 'name',
-            type: DyFM_BasicProperty_Type.string,
-          },
-          content: {
-            key: 'content',
-            type: DyFM_BasicProperty_Type.string,
-          },
-          contentVectorized: {
-            key: 'contentVectorized',
-            type: DyFM_BasicProperty_Type.array,
-            vectorizedFrom: ['content'],
-            // embeddingModel not set
-          },
-        },
-      });
-
-      const serviceWithoutModel = new DyNTS_OAI_VectorDataService<TestMetadata>(
-        testData,
-        paramsWithoutModel,
-        mockOpenAISettings,
-        'test-issuer'
-      );
-
-      (serviceWithoutModel as any).vectorizedProperties[0].embeddingModel = undefined;
-      (DyNTS_DataService.prototype.getDataById as jasmine.Spy).and.returnValue(Promise.resolve(null));
-
-      try {
-        await serviceWithoutModel.vectorizeDataProperties(newData);
-        fail('Should have thrown an error');
-      } catch (error) {
-        expect(error).toBeInstanceOf(DyFM_Error);
-        expect((error as DyFM_Error)._errorCode).toContain('DyNTS-VDB-VDB1');
-      }
-    });
-  });
-
-  describe('| vectorSearch', () => {
-    it('| should perform vector search', async () => {
-      const mockResponse: CreateEmbeddingResponse = {
-        data: [
-          {
-            embedding: [0.1, 0.2, 0.3],
-            index: 0,
-            object: 'embedding',
-          },
-        ],
-        model: 'text-embedding-3-small',
-        object: 'list',
-        usage: {
-          prompt_tokens: 5,
-          total_tokens: 5,
-        },
-      };
-
-      mockEmbedding_CS.createEmbedding.and.returnValue(Promise.resolve(mockResponse));
-      mockDBService.aggregate.and.returnValue(Promise.resolve([testData]));
-
-      const result = await service.vectorSearch({
-        input: 'Test search',
-        searchInKey: 'contentVectorized',
-      });
-
-      expect(mockEmbedding_CS.createEmbedding).toHaveBeenCalled();
-      expect(mockDBService.aggregate).toHaveBeenCalled();
-      expect(result).toEqual([testData]);
-    });
-
-    it('| should use default limit and numberOfCandidates', async () => {
-      const mockResponse: CreateEmbeddingResponse = {
-        data: [
-          {
-            embedding: [0.1, 0.2, 0.3],
-            index: 0,
-            object: 'embedding',
-          },
-        ],
-        model: 'text-embedding-3-small',
-        object: 'list',
-        usage: {
-          prompt_tokens: 5,
-          total_tokens: 5,
-        },
-      };
-
-      mockEmbedding_CS.createEmbedding.and.returnValue(Promise.resolve(mockResponse));
-      mockDBService.aggregate.and.returnValue(Promise.resolve([]));
-
-      await service.vectorSearch({
-        input: 'Test search',
-        searchInKey: 'contentVectorized',
-      });
-
-      const aggregateCall = mockDBService.aggregate.calls.mostRecent().args[0][0];
-      expect(aggregateCall.$vectorSearch.limit).toBe(3);
-      expect(aggregateCall.$vectorSearch.numCandidates).toBe(100);
-    });
-
-    it('| should throw error if input is empty', async () => {
-      try {
-        await service.vectorSearch({
-          input: '',
-          searchInKey: 'contentVectorized',
-        });
-        fail('Should have thrown an error');
-      } catch (error) {
-        expect(error).toBeInstanceOf(DyFM_Error);
-        expect((error as DyFM_Error)._errorCode).toContain('DyNTS-VDB-VS1');
-      }
-    });
-
-    it('| should throw error if property not found', async () => {
-      try {
-        await service.vectorSearch({
-          input: 'Test search',
-          searchInKey: 'non-existent',
-        });
-        fail('Should have thrown an error');
-      } catch (error) {
-        expect(error).toBeInstanceOf(DyFM_Error);
-        expect((error as DyFM_Error)._errorCode).toContain('DyNTS-VDB-VS2');
-      }
-    });
-
-    it('| should throw error if property has no vectorizeUseIndex', async () => {
-      const paramsWithoutIndex = new DyFM_DataModel_Params<TestMetadata>({
-        dataName: 'test_data',
-        properties: {
-          name: {
-            key: 'name',
-            type: DyFM_BasicProperty_Type.string,
-          },
-          content: {
-            key: 'content',
-            type: DyFM_BasicProperty_Type.string,
-          },
-          contentVectorized: {
-            key: 'contentVectorized',
-            type: DyFM_BasicProperty_Type.array,
-            vectorizedFrom: ['content'],
-            embeddingModel: DyFM_OAI_Model.textEmbedding_3Small,
-            // vectorizeUseIndex not set
-          },
-        },
-      });
-
-      const serviceWithoutIndex = new DyNTS_OAI_VectorDataService<TestMetadata>(
-        testData,
-        paramsWithoutIndex,
-        mockOpenAISettings,
-        'test-issuer'
-      );
-
-      (serviceWithoutIndex as any).dataDBService = mockDBService;
-      (serviceWithoutIndex as any).embedding_CS = mockEmbedding_CS;
-
-      try {
-        await serviceWithoutIndex.vectorSearch({
-          input: 'Test search',
-          searchInKey: 'contentVectorized',
-        });
-        fail('Should have thrown an error');
-      } catch (error) {
-        expect(error).toBeInstanceOf(DyFM_Error);
-      }
-    });
-  });
-
-  describe('| saveData', () => {
-    it('| should vectorize properties before saving', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-      });
-
-      (DyNTS_DataService.prototype as unknown as { ensureData: jasmine.Spy }).ensureData.and.returnValue(newData);
-      spyOn(service, 'vectorizeDataProperties').and.returnValue(Promise.resolve(newData));
-      (DyNTS_DataService.prototype.saveData as jasmine.Spy).and.returnValue(Promise.resolve(newData));
-
-      const result = await service.saveData(newData);
-
-      expect(service.vectorizeDataProperties).toHaveBeenCalledWith(newData, undefined);
-      expect(result).toBeDefined();
-    });
-
-    it('| should remove vectorized properties if swallowVectorizedProperties is true', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-        contentVectorized: [0.1, 0.2, 0.3],
-      });
-
-      (service as any).swallowVectorizedProperties = true;
-      spyOn(service, 'vectorizeDataProperties').and.returnValue(Promise.resolve(newData));
-      (DyNTS_DataService.prototype.saveData as jasmine.Spy).and.returnValue(Promise.resolve(newData));
-      spyOn(DyFM_Object, 'cleanObject');
-
-      const result = await service.saveData(newData);
-
-      expect(result.contentVectorized).toBeUndefined();
-      expect(DyFM_Object.cleanObject).toHaveBeenCalled();
-    });
-
-    it('| should keep vectorized properties if swallowVectorizedProperties is false', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-        contentVectorized: [0.1, 0.2, 0.3],
-      });
-
-      (service as any).swallowVectorizedProperties = false;
-      spyOn(service, 'vectorizeDataProperties').and.returnValue(Promise.resolve(newData));
-      (DyNTS_DataService.prototype.saveData as jasmine.Spy).and.returnValue(Promise.resolve(newData));
-
-      const result = await service.saveData(newData);
-
-      expect(result.contentVectorized).toBeDefined();
-    });
-
-    it('| should handle save errors', async () => {
-      const newData: TestMetadata = Object.assign(new TestMetadata(), {
-        name: 'New Name',
-        content: 'New content',
-      });
-
-      const error = new Error('Save failed');
-      spyOn(service, 'vectorizeDataProperties').and.returnValue(Promise.resolve(newData));
-      (DyNTS_DataService.prototype.saveData as jasmine.Spy).and.returnValue(Promise.reject(error));
-
-      try {
-        await service.saveData(newData);
-        fail('Should have thrown an error');
-      } catch (err) {
-        expect(err).toBeInstanceOf(DyFM_Error);
-        expect((err as DyFM_Error)._errorCode).toContain('DyNTS-VDB-SD0');
-      }
-    });
-  });
-
-  describe('| properties', () => {
-    it('| should have tokenSpent property', () => {
-      expect((service as any).tokenSpent).toBe(0);
-    });
-
-    it('| should track token usage', async () => {
-      const mockResponse: CreateEmbeddingResponse = {
-        data: [
-          {
-            embedding: [0.1, 0.2, 0.3],
-            index: 0,
-            object: 'embedding',
-          },
-        ],
-        model: 'text-embedding-3-small',
-        object: 'list',
-        usage: {
-          prompt_tokens: 10,
-          total_tokens: 10,
-        },
-      };
-
-      mockEmbedding_CS.createEmbedding.and.returnValue(Promise.resolve(mockResponse));
-
-      await service.vectorize('Test input');
-
-      expect((service as any).tokenSpent).toBe(10);
-    });
-  });
-});
-