@futdevpro/nts-dynamo 1.15.1 → 1.15.3

package/src/_modules/local-vector-search/_services/lvs-doc-chunk-data.service.spec.ts

@@ -0,0 +1,418 @@
+
+import { DyNTS_LVS_DocChunkDataService } from './lvs-doc-chunk-data.service';
+import { DyFM_OAI_Settings, DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyNTS_DAI_DocChunk, DyNTS_DAI_docChunk_dataParams } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocPage } from '../../ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from '../../ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../ai/_modules/document-ai/_enums/dai-compare-result-type.enum';
+import { DyFM_Error, DyFM_EnvironmentFlag } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunking_Util } from '../../ai/_modules/document-ai/_collections/dai-chunking.util';
+import { DyNTS_DAI_Document_Util } from '../../ai/_modules/document-ai/_collections/dai-document.util';
+import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+import { DyNTS_GlobalService } from '../../../_services/core/global.service';
+
+describe('| DyNTS_LVS_DocChunkDataService', () => {
+  let service: DyNTS_LVS_DocChunkDataService;
+  let mockOpenAISettings: DyFM_OAI_Settings;
+  let mockDBService: jasmine.SpyObj<{
+    find: (filter: unknown) => Promise<unknown[]>;
+    findOne: (filter: unknown) => Promise<unknown>;
+    getDataById: (id: string) => Promise<unknown>;
+    getAll: () => Promise<unknown[]>;
+    createData: (data: unknown, issuer: string) => Promise<unknown>;
+    modifyData: (data: unknown, issuer: string, dontUpdateModified?: boolean) => Promise<unknown>;
+    markDeletedById: (id: string, issuer: string) => Promise<void>;
+  }>;
+
+  beforeEach(() => {
+    DyNTS_global_settings.env_settings = {
+      environment: DyFM_EnvironmentFlag.local,
+      mongoUri: 'mongodb://localhost:27017/test',
+    } as typeof DyNTS_global_settings.env_settings;
+
+    mockDBService = jasmine.createSpyObj('DyNTS_DBService', [
+      'find',
+      'findOne',
+      'getDataById',
+      'getAll',
+      'getDataListByIds',
+      'getDataByDependencyId',
+      'getDataListByDependencyId',
+      'getDataListByDependencyIds',
+      'createData',
+      'modifyData',
+      'updateOne',
+      'markDeletedById',
+      'trueDeleteDataById',
+      'trueDeleteAllData',
+      'restoreDeletedById',
+    ]);
+    spyOn(DyNTS_GlobalService, 'getDBService').and.returnValue(mockDBService as never);
+    spyOn(DyNTS_GlobalService, 'getDBServiceByKey').and.returnValue(mockDBService as never);
+
+    mockOpenAISettings = {
+      config: {
+        apiKey: 'test-api-key',
+        organization: 'test-org',
+      },
+      defaultSettings: {
+        useModel: DyFM_OAI_Model.textEmbedding_3Small,
+      },
+    };
+
+    service = new DyNTS_LVS_DocChunkDataService(
+      'test-issuer',
+      mockOpenAISettings
+    );
+    service.clickupWorkspaceId = 'workspace-123';
+  });
+
+  describe('| constructor', () => {
+    it('| should initialize with issuer and settings', () => {
+      expect((service as any).issuer).toBe('test-issuer');
+      expect((service as any).openAISettings).toEqual(mockOpenAISettings);
+    });
+  });
+
+  describe('| getDocumentDataService', () => {
+    it('| should return document data service', () => {
+      const result = service.getDocumentDataService();
+
+      expect(result).toBeDefined();
+      expect((result as any).issuer).toBe('test-issuer');
+    });
+  });
+
+  describe('| getDocPageDataService', () => {
+    it('| should return doc page data service', () => {
+      const result = service.getDocPageDataService();
+
+      expect(result).toBeDefined();
+      expect((result as any).issuer).toBe('test-issuer');
+    });
+  });
+
+  describe('| getPageLink', () => {
+    it('| should return page link', () => {
+      const mockPage = {
+        name: 'Test Page',
+        externalDocumentId: 'doc-123',
+        externalPageId: 'page-456',
+      } as any;
+
+      spyOn(DyNTS_DAI_Document_Util, 'getClickupPageLink').and.returnValue('[Test Page](link)');
+
+      const result = service.getPageLink(mockPage);
+
+      expect(result).toBe('[Test Page](link)');
+      expect(DyNTS_DAI_Document_Util.getClickupPageLink).toHaveBeenCalledWith(
+        'Test Page',
+        'workspace-123',
+        'doc-123',
+        'page-456'
+      );
+    });
+  });
+
+  describe('| chunkPage', () => {
+    it('| should chunk page content', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+        externalDocumentId: 'doc-123',
+        externalPageId: 'page-456',
+      } as any;
+
+      spyOn(DyNTS_DAI_DocChunking_Util, 'chunkMdContent').and.returnValue(Promise.resolve([
+        {
+          _id: 'chunk-1',
+          chunkContent: '# Test\n\nContent here',
+          chunkIndex: 0,
+        } as any,
+      ]));
+
+      const result = await service.chunkPage(mockPage);
+
+      expect(result).toBeDefined();
+      expect(result.length).toBeGreaterThan(0);
+      expect(DyNTS_DAI_DocChunking_Util.chunkMdContent).toHaveBeenCalled();
+    });
+
+    it('| should handle chunking errors', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+      } as any;
+
+      const error = new Error('Chunking failed');
+      spyOn(DyNTS_DAI_DocChunking_Util, 'chunkMdContent').and.returnValue(Promise.reject(error));
+
+      try {
+        await service.chunkPage(mockPage);
+        fail('Should have thrown an error');
+      } catch (err) {
+        expect(err).toBeInstanceOf(DyFM_Error);
+        expect((err as DyFM_Error)._errorCode).toContain('DCH-CSB-CP0');
+      }
+    });
+  });
+
+  describe('| compareChunk', () => {
+    it('| should return equal when chunks are identical', async () => {
+      const oldChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Test content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1', 'flag2'],
+      } as any;
+
+      const newChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Test content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1', 'flag2'],
+      } as any;
+
+      const result = await service.compareChunk(newChunk, oldChunk);
+
+      expect(result.result).toBe(DyNTS_DAI_CompareResult_Type.equal);
+      expect(result.newChunk).toBe(newChunk);
+      expect(result.oldChunk).toBe(oldChunk);
+    });
+
+    it('| should return modified when chunks differ', async () => {
+      const oldChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Old content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1'],
+      } as any;
+
+      const newChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'New content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1'],
+      } as any;
+
+      const result = await service.compareChunk(newChunk, oldChunk);
+
+      expect(result.result).toBe(DyNTS_DAI_CompareResult_Type.modified);
+      expect(result.newChunk).toBe(newChunk);
+      expect(result.oldChunk).toBe(oldChunk);
+    });
+
+    it('| should throw error if oldChunk is missing', async () => {
+      const newChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'New content',
+      } as any;
+
+      try {
+        await service.compareChunk(newChunk, null as any);
+        fail('Should have thrown an error');
+      } catch (err) {
+        expect(err).toBeInstanceOf(DyFM_Error);
+        expect((err as DyFM_Error)._errorCode).toContain('DCH-CSB-CC1');
+      }
+    });
+
+    it('| should detect changes in allFlaggedParents', async () => {
+      const oldChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Test content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1'],
+      } as any;
+
+      const newChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Test content',
+        path: 'test/path',
+        documentName: 'Test Doc',
+        pageName: 'Test Page',
+        pageLink: '[Test Page](link)',
+        allFlaggedParents: ['flag1', 'flag2'],
+      } as any;
+
+      const result = await service.compareChunk(newChunk, oldChunk);
+
+      expect(result.result).toBe(DyNTS_DAI_CompareResult_Type.modified);
+    });
+  });
+
+  describe('| processNewPage', () => {
+    it('| should process new page and return compare results', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+        documentId: 'doc-123',
+        externalDocumentId: 'ext-doc-123',
+        externalPageId: 'ext-page-123',
+      } as any;
+
+      const mockChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Test content',
+        chunkIndex: 0,
+      } as any;
+
+      spyOn(service, 'chunkPage').and.returnValue(Promise.resolve([mockChunk]));
+      spyOn(service, 'saveData').and.returnValue(Promise.resolve(mockChunk));
+
+      const result = await service.processNewPage(mockPage);
+
+      expect(result).toBeDefined();
+      expect(result.length).toBe(1);
+      expect(result[0].result).toBe(DyNTS_DAI_CompareResult_Type.new);
+      expect(result[0].newChunk).toBe(mockChunk);
+      expect(result[0].oldChunk).toBeNull();
+    });
+
+    it('| should handle processing errors', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+      } as any;
+
+      const error = new Error('Processing failed');
+      spyOn(service, 'chunkPage').and.returnValue(Promise.reject(error));
+
+      try {
+        await service.processNewPage(mockPage);
+        fail('Should have thrown an error');
+      } catch (err) {
+        expect(err).toBeInstanceOf(DyFM_Error);
+        expect((err as DyFM_Error)._errorCode).toContain('DCH-CSB-PNP0');
+      }
+    });
+  });
+
+  describe('| processPage', () => {
+    it('| should process page and return compare results', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+        documentId: 'doc-123',
+        externalDocumentId: 'ext-doc-123',
+        externalPageId: 'ext-page-123',
+      } as any;
+
+      const mockNewChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'New content',
+        chunkIndex: 0,
+      } as any;
+
+      const mockOldChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Old content',
+        chunkIndex: 0,
+      } as any;
+
+      spyOn(service, 'chunkPage').and.returnValue(Promise.resolve([mockNewChunk]));
+      spyOn(service, 'findDataList').and.returnValue(Promise.resolve([mockOldChunk]));
+      spyOn(service, 'compareChunk').and.returnValue(Promise.resolve({
+        result: DyNTS_DAI_CompareResult_Type.modified,
+        newChunk: mockNewChunk,
+        oldChunk: mockOldChunk,
+      }));
+      spyOn(service, 'saveData').and.returnValue(Promise.resolve(mockNewChunk));
+
+      const result = await service.processPage(mockPage);
+
+      expect(result).toBeDefined();
+      expect(result.length).toBeGreaterThan(0);
+    });
+
+    it('| should handle processing errors', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+      } as any;
+
+      const error = new Error('Processing failed');
+      spyOn(service, 'chunkPage').and.returnValue(Promise.reject(error));
+
+      try {
+        await service.processPage(mockPage);
+        fail('Should have thrown an error');
+      } catch (err) {
+        expect(err).toBeInstanceOf(DyFM_Error);
+        expect((err as DyFM_Error)._errorCode).toContain('DCH-CSB-PP0');
+      }
+    });
+
+    it('| should delete old chunks when new page has fewer chunks', async () => {
+      const mockPage: DyNTS_DAI_DocPage = {
+        _id: 'page-123',
+        name: 'Test Page',
+        content: '# Test\n\nContent here',
+        chunks: [],
+        documentId: 'doc-123',
+        externalDocumentId: 'ext-doc-123',
+        externalPageId: 'ext-page-123',
+      } as any;
+
+      const mockNewChunk: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'New content',
+        chunkIndex: 0,
+      } as any;
+
+      const mockOldChunk1: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-1',
+        chunkContent: 'Old content',
+        chunkIndex: 0,
+      } as any;
+
+      const mockOldChunk2: DyNTS_DAI_DocChunk = {
+        _id: 'chunk-2',
+        chunkContent: 'Old content 2',
+        chunkIndex: 1,
+      } as any;
+
+      spyOn(service, 'chunkPage').and.returnValue(Promise.resolve([mockNewChunk]));
+      spyOn(service, 'findDataList').and.returnValue(Promise.resolve([mockOldChunk1, mockOldChunk2]));
+      spyOn(service, 'compareChunk').and.returnValue(Promise.resolve({
+        result: DyNTS_DAI_CompareResult_Type.equal,
+        newChunk: mockNewChunk,
+        oldChunk: mockOldChunk1,
+      }));
+      spyOn(service, 'deleteData').and.returnValue(Promise.resolve());
+
+      const result = await service.processPage(mockPage);
+
+      expect(service.deleteData).toHaveBeenCalledWith('chunk-2');
+      expect(result.some(r => r.result === DyNTS_DAI_CompareResult_Type.deleted)).toBeTrue();
+    });
+  });
+});
+