@futdevpro/nts-dynamo 1.15.1 → 1.15.3

package/src/_modules/ai/_modules/document-ai/_collections/dai-chunking.util.spec.ts

@@ -0,0 +1,242 @@
+
+import { DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunking_Util } from './dai-chunking.util';
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_DocChunk } from '../_models/data-models/dai-doc-chunk.data-model';
+import { DyNTS_global_settings } from '../../../../../_collections/global-settings.const';
+
+describe('| DyNTS_DAI_DocChunking_Util', () => {
+  describe('| chunkMdContent', () => {
+    it('| should throw error when page documentId is not set', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: 'Test content',
+        documentId: '',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await expectAsync(
+        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
+      ).toBeRejected();
+    });
+
+    it('| should throw error when page _id is not set', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: 'Test content',
+        documentId: 'doc-123',
+        _id: '',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await expectAsync(
+        DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123')
+      ).toBeRejected();
+    });
+
+    it('| should create chunks from markdown content', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '# Header 1\n\nThis is some content.\n\n## Header 2\n\nMore content here.',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks[0]).toBeInstanceOf(DyNTS_DAI_DocChunk);
+      expect(chunks[0].documentId).toBe('doc-123');
+      expect(chunks[0].pageId).toBe('page-123');
+      expect(chunks[0].chunkContent).toBeDefined();
+    });
+
+    it('| should handle empty content', async () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBe(0);
+    });
+
+    it('| should log warning for level 5+ headers', async () => {
+      const logSpy = spyOn(DyFM_Log, 'warn');
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '##### Level 5 Header\n\nContent',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(logSpy).toHaveBeenCalled();
+    });
+
+    it('| should respect maxChunkCount limit', async () => {
+      const logSpy = spyOn(DyFM_Log, 'warn');
+      const longContent = '# Header\n\n' + 'Content. '.repeat(10000);
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: longContent,
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const getPageLink = (page: any, issuer: string) => 'http://example.com/page';
+
+      const chunks = await DyNTS_DAI_DocChunking_Util.chunkMdContent(page, getPageLink, 'issuer-123');
+
+      expect(chunks.length).toBeLessThanOrEqual(DyNTS_global_settings.docChunking.maxChunkCount);
+      if (chunks.length >= DyNTS_global_settings.docChunking.maxChunkCount) {
+        expect(logSpy).toHaveBeenCalled();
+      }
+    });
+  });
+
+  describe('| reassembleChunksToContent', () => {
+    it('| should reassemble chunks to content', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First chunk content',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'Second chunk content',
+          chunkIndex: 1,
+          _id: 'chunk-2',
+        } as any),
+      ];
+
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+
+      expect(result).toContain('First chunk content');
+      expect(result).toContain('Second chunk content');
+    });
+
+    it('| should handle empty chunks array', () => {
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent([], 'issuer-123');
+
+      expect(result).toBe('');
+    });
+
+    it('| should concatenate chunks in order', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'Second',
+          chunkIndex: 1,
+          _id: 'chunk-2',
+        } as any),
+      ];
+
+      const result = DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+
+      expect(result).toBe('FirstSecond');
+    });
+
+    it('| should throw error when chunk is null', () => {
+      const chunks = [
+        new DyNTS_DAI_DocChunk({
+          documentId: 'doc-123',
+          pageId: 'page-123',
+          filePathParents: [],
+          chunkHeadParents: [],
+          allFlaggedParents: [],
+          path: '',
+          documentName: 'Test Document',
+          pageName: 'Test Page',
+          pageLink: '',
+          chunkContent: 'First',
+          chunkIndex: 0,
+          _id: 'chunk-1',
+        } as any),
+        null as any,
+      ];
+
+      expect(() => {
+        DyNTS_DAI_DocChunking_Util.reassembleChunksToContent(chunks, 'issuer-123');
+      }).toThrow();
+    });
+  });
+});
+

package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts

@@ -0,0 +1,209 @@
+
+import { DyNTS_DAI_Document_Util } from './dai-document.util';
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
+import { DyFM_Log } from '@futdevpro/fsm-dynamo';
+
+describe('| DyNTS_DAI_Document_Util', () => {
+  describe('| getClickupPageLink', () => {
+    it('| should generate ClickUp page link', () => {
+      const result = DyNTS_DAI_Document_Util.getClickupPageLink(
+        'Test Page',
+        'workspace-123',
+        'doc-456',
+        'page-789'
+      );
+
+      expect(result).toContain('Test Page');
+      expect(result).toContain('https://app.clickup.com/workspace-123/v/dc/doc-456/page-789');
+      expect(result).toMatch(/\[Test Page\]\(https:\/\/app\.clickup\.com\/workspace-123\/v\/dc\/doc-456\/page-789\)/);
+    });
+  });
+
+  describe('| validateDocumentPage', () => {
+    it('| should return true for valid page', () => {
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: 'Test content',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
+
+      expect(result).toBe(true);
+    });
+
+    it('| should return false when name is missing', () => {
+      const logSpy = spyOn(DyFM_Log, 'error');
+      const page = new DyNTS_DAI_DocPage({
+        name: '',
+        content: 'Test content',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
+
+      expect(result).toBe(false);
+      expect(logSpy).toHaveBeenCalled();
+    });
+
+    it('| should return false when content is missing', () => {
+      const logSpy = spyOn(DyFM_Log, 'error');
+      const page = new DyNTS_DAI_DocPage({
+        name: 'Test Page',
+        content: '',
+        documentId: 'doc-123',
+        _id: 'page-123',
+        parentedPath: [],
+        allFlaggedParents: [],
+        allFlaggedParentsMerged: '',
+        filePath: '',
+      });
+
+      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
+
+      expect(result).toBe(false);
+      expect(logSpy).toHaveBeenCalled();
+    });
+  });
+
+  describe('| normalizeContent', () => {
+    it('| should normalize line endings from CRLF to LF', () => {
+      const content = 'Line 1\r\nLine 2\r\nLine 3';
+      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
+
+      expect(result).toBe('Line 1\nLine 2\nLine 3');
+    });
+
+    it('| should normalize line endings from CR to LF', () => {
+      const content = 'Line 1\rLine 2\rLine 3';
+      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
+
+      expect(result).toBe('Line 1\nLine 2\nLine 3');
+    });
+
+    it('| should trim leading and trailing whitespace', () => {
+      const content = '  Line 1\nLine 2  ';
+      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
+
+      expect(result).toBe('Line 1\nLine 2');
+    });
+
+    it('| should handle mixed line endings', () => {
+      const content = 'Line 1\r\nLine 2\rLine 3\nLine 4';
+      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
+
+      expect(result).toBe('Line 1\nLine 2\nLine 3\nLine 4');
+    });
+  });
+
+  describe('| extractMetadata', () => {
+    it('| should extract title from first header', () => {
+      const content = '# Title\n\nContent here';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      expect(result.title).toBe('Title');
+    });
+
+    it('| should extract word count', () => {
+      const content = 'This is a test content with multiple words';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      // Content.split(/\s+/) returns 8 words: "This", "is", "a", "test", "content", "with", "multiple", "words"
+      expect(result.wordCount).toBe(8);
+    });
+
+    it('| should extract character count', () => {
+      const content = 'Test content';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      expect(result.characterCount).toBe(content.length);
+    });
+
+    it('| should extract header count', () => {
+      const content = '# Header 1\n## Header 2\n### Header 3';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      expect(result.headerCount).toBe(3);
+    });
+
+    it('| should return 0 header count when no headers', () => {
+      const content = 'Just plain text without headers';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      expect(result.headerCount).toBe(0);
+    });
+
+    it('| should extract all metadata fields', () => {
+      const content = '# Document Title\n\nThis is the content of the document.';
+      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
+
+      expect(result.title).toBe('Document Title');
+      expect(result.wordCount).toBeGreaterThan(0);
+      expect(result.characterCount).toBeGreaterThan(0);
+      expect(result.headerCount).toBe(1);
+    });
+  });
+
+  describe('| isEmpty', () => {
+    it('| should return true for empty string', () => {
+      expect(DyNTS_DAI_Document_Util.isEmpty('')).toBe(true);
+    });
+
+    it('| should return true for whitespace only', () => {
+      expect(DyNTS_DAI_Document_Util.isEmpty('   \n\t  ')).toBe(true);
+    });
+
+    it('| should return false for content with text', () => {
+      expect(DyNTS_DAI_Document_Util.isEmpty('Some content')).toBe(false);
+    });
+
+    it('| should return true for null or undefined', () => {
+      expect(DyNTS_DAI_Document_Util.isEmpty(null as any)).toBe(true);
+      expect(DyNTS_DAI_Document_Util.isEmpty(undefined as any)).toBe(true);
+    });
+  });
+
+  describe('| getPreview', () => {
+    it('| should return full content when shorter than maxLength', () => {
+      const content = 'Short content';
+      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
+
+      expect(result).toBe('Short content');
+    });
+
+    it('| should truncate content when longer than maxLength', () => {
+      const content = 'A'.repeat(300);
+      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
+
+      expect(result.length).toBe(203); // 200 + '...'
+      expect(result.endsWith('...')).toBe(true);
+    });
+
+    it('| should use default maxLength of 200', () => {
+      const content = 'A'.repeat(250);
+      const result = DyNTS_DAI_Document_Util.getPreview(content);
+
+      expect(result.length).toBe(203); // 200 + '...'
+      expect(result.endsWith('...')).toBe(true);
+    });
+
+    it('| should handle exact maxLength', () => {
+      const content = 'A'.repeat(200);
+      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
+
+      expect(result).toBe(content);
+      expect(result.endsWith('...')).toBe(false);
+    });
+  });
+});
+