@futdevpro/nts-dynamo 1.15.74 → 1.15.78

package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts

@@ -1,242 +1,242 @@
-
-import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_DocChunking_Util } from './dai-chunking.util';
-import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
-import { DyNTS_DAI_DocChunk } from '../_models/data-models/dai-doc-chunk.data-model';
-import { DyNTS_global_settings } from '../../../../../_collections/global-settings.const';
-
-describe('| DyNTS_DAI_DocChunking_Util', () => {
-  describe('| chunkMdContent', () => {
-    it('| should throw error when page documentId is not set', async () => {
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: 'Test content',
-        documentId: '',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      await expectAsync(
-        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
-      ).toBeRejected();
-    });
-
-    it('| should throw error when page _id is not set', async () => {
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: 'Test content',
-        documentId: 'doc-123',
-        _id: '',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      await expectAsync(
-        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
-      ).toBeRejected();
-    });
-
-    it('| should create chunks from markdown content', async () => {
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: '# Header 1\n\nThis is some content.\n\n## Header 2\n\nMore content here.',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
-
-      expect(chunks.length).toBeGreaterThan(0);
-      expect(chunks[0]).toBeInstanceOf(DyNTS_DAI_DocChunk);
-      expect(chunks[0].documentId).toBe('doc-123');
-      expect(chunks[0].pageId).toBe('page-123');
-      expect(chunks[0].chunkContent).toBeDefined();
-    });
-
-    it('| should handle empty content', async () => {
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: '',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
-
-      expect(chunks.length).toBe(0);
-    });
-
-    it('| should log warning for level 5+ headers', async () => {
-      const logSpy = spyOn(DyFM_Log, 'warn');
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: '##### Level 5 Header\n\nContent',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
-
-      expect(logSpy).toHaveBeenCalled();
-    });
-
-    it('| should respect maxChunkCount limit', async () => {
-      const logSpy = spyOn(DyFM_Log, 'warn');
-      const longContent = '# Header\n\n' + 'Content. '.repeat(10000);
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: longContent,
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
-
-      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
-
-      expect(chunks.length).toBeLessThanOrEqual(DyNTS_global_settings.docChunking.maxChunkCount);
-      if (chunks.length >= DyNTS_global_settings.docChunking.maxChunkCount) {
-        expect(logSpy).toHaveBeenCalled();
-      }
-    });
-  });
-
-  describe('| reassembleChunksToContent', () => {
-    it('| should reassemble chunks to content', () => {
-      const chunks = [
-        new DyNTS_DAI_DocChunk({
-          documentId: 'doc-123',
-          pageId: 'page-123',
-          filePathParents: [],
-          chunkHeadParents: [],
-          allFlaggedParents: [],
-          path: '',
-          documentName: 'Test Document',
-          pageName: 'Test Page',
-          pageLink: '',
-          chunkContent: 'First chunk content',
-          chunkIndex: 0,
-          _id: 'chunk-1',
-        } as any),
-        new DyNTS_DAI_DocChunk({
-          documentId: 'doc-123',
-          pageId: 'page-123',
-          filePathParents: [],
-          chunkHeadParents: [],
-          allFlaggedParents: [],
-          path: '',
-          documentName: 'Test Document',
-          pageName: 'Test Page',
-          pageLink: '',
-          chunkContent: 'Second chunk content',
-          chunkIndex: 1,
-          _id: 'chunk-2',
-        } as any),
-      ];
-
-      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
-
-      expect(result).toContain('First chunk content');
-      expect(result).toContain('Second chunk content');
-    });
-
-    it('| should handle empty chunks array', () => {
-      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent([], 'issuer-123');
-
-      expect(result).toBe('');
-    });
-
-    it('| should concatenate chunks in order', () => {
-      const chunks = [
-        new DyNTS_DAI_DocChunk({
-          documentId: 'doc-123',
-          pageId: 'page-123',
-          filePathParents: [],
-          chunkHeadParents: [],
-          allFlaggedParents: [],
-          path: '',
-          documentName: 'Test Document',
-          pageName: 'Test Page',
-          pageLink: '',
-          chunkContent: 'First',
-          chunkIndex: 0,
-          _id: 'chunk-1',
-        } as any),
-        new DyNTS_DAI_DocChunk({
-          documentId: 'doc-123',
-          pageId: 'page-123',
-          filePathParents: [],
-          chunkHeadParents: [],
-          allFlaggedParents: [],
-          path: '',
-          documentName: 'Test Document',
-          pageName: 'Test Page',
-          pageLink: '',
-          chunkContent: 'Second',
-          chunkIndex: 1,
-          _id: 'chunk-2',
-        } as any),
-      ];
-
-      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
-
-      expect(result).toBe('FirstSecond');
-    });
-
-    it('| should throw error when chunk is null', () => {
-      const chunks = [
-        new DyNTS_DAI_DocChunk({
-          documentId: 'doc-123',
-          pageId: 'page-123',
-          filePathParents: [],
-          chunkHeadParents: [],
-          allFlaggedParents: [],
-          path: '',
-          documentName: 'Test Document',
-          pageName: 'Test Page',
-          pageLink: '',
-          chunkContent: 'First',
-          chunkIndex: 0,
-          _id: 'chunk-1',
-        } as any),
-        null as any,
-      ];
-
-      expect(() => {
-        DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
-      }).toThrow();
-    });
-  });
-});
-
+
+import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunking_Util } from './dai-chunking.util';
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_DocChunk } from '../_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_global_settings } from '../../../../../_collections/global-settings.const';
+
+describe('| DyNTS_DAI_DocChunking_Util', () => {
+  describe('| chunkMdContent', () => {
+    it('| should throw error when page documentId is not set', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: 'Test content',
+        documentId: '',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await expectAsync(
+        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
+      ).toBeRejected();
+    });
+
+    it('| should throw error when page _id is not set', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: 'Test content',
+        documentId: 'doc-123',
+        _id: '',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await expectAsync(
+        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
+      ).toBeRejected();
+    });
+
+    it('| should create chunks from markdown content', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '# Header 1\n\nThis is some content.\n\n## Header 2\n\nMore content here.',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks[0]).toBeInstanceOf(DyNTS_DAI_DocChunk);
+      expect(chunks[0].documentId).toBe('doc-123');
+      expect(chunks[0].pageId).toBe('page-123');
+      expect(chunks[0].chunkContent).toBeDefined();
+    });
+
+    it('| should handle empty content', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBe(0);
+    });
+
+    it('| should log warning for level 5+ headers', async () => {
+      const logSpy = spyOn(DyFM_Log, 'warn');
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '##### Level 5 Header\n\nContent',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(logSpy).toHaveBeenCalled();
+    });
+
+    it('| should respect maxChunkCount limit', async () => {
+      const logSpy = spyOn(DyFM_Log, 'warn');
+      const longContent = '# Header\n\n' + 'Content. '.repeat(10000);
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: longContent,
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBeLessThanOrEqual(DyNTS_global_settings.docChunking.maxChunkCount);
+      if (chunks.length >= DyNTS_global_settings.docChunking.maxChunkCount) {
+        expect(logSpy).toHaveBeenCalled();
+      }
+    });
+  });
+
+  describe('| reassembleChunksToContent', () => {
+    it('| should reassemble chunks to content', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First chunk content',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'Second chunk content',
+          chunkIndex: 1,
+          _id: 'chunk-2',
+        } as any),
+      ];
+
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+
+      expect(result).toContain('First chunk content');
+      expect(result).toContain('Second chunk content');
+    });
+
+    it('| should handle empty chunks array', () => {
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent([], 'issuer-123');
+
+      expect(result).toBe('');
+    });
+
+    it('| should concatenate chunks in order', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'Second',
+          chunkIndex: 1,
+          _id: 'chunk-2',
+        } as any),
+      ];
+
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+
+      expect(result).toBe('FirstSecond');
+    });
+
+    it('| should throw error when chunk is null', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        null as any,
+      ];
+
+      expect(() => {
+        DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+      }).toThrow();
+    });
+  });
+});
+