@futdevpro/nts-dynamo 1.15.157 → 1.15.160

package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.spec.ts

@@ -1,209 +0,0 @@
-
-import { DyNTS_DAI_Document_Util } from './dai-document.util';
-import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
-import { DyFM_Log } from '@futdevpro/fsm-dynamo';
-
-describe('| DyNTS_DAI_Document_Util', () => {
-  describe('| getClickupPageLink', () => {
-    it('| should generate ClickUp page link', () => {
-      const result = DyNTS_DAI_Document_Util.getClickupPageLink(
-        'Test Page',
-        'workspace-123',
-        'doc-456',
-        'page-789'
-      );
-
-      expect(result).toContain('Test Page');
-      expect(result).toContain('https://app.clickup.com/workspace-123/v/dc/doc-456/page-789');
-      expect(result).toMatch(/\[Test Page\]\(https:\/\/app\.clickup\.com\/workspace-123\/v\/dc\/doc-456\/page-789\)/);
-    });
-  });
-
-  describe('| validateDocumentPage', () => {
-    it('| should return true for valid page', () => {
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: 'Test content',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
-
-      expect(result).toBe(true);
-    });
-
-    it('| should return false when name is missing', () => {
-      const logSpy = spyOn(DyFM_Log, 'error');
-      const page = new DyNTS_DAI_DocPage({
-        name: '',
-        content: 'Test content',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
-
-      expect(result).toBe(false);
-      expect(logSpy).toHaveBeenCalled();
-    });
-
-    it('| should return false when content is missing', () => {
-      const logSpy = spyOn(DyFM_Log, 'error');
-      const page = new DyNTS_DAI_DocPage({
-        name: 'Test Page',
-        content: '',
-        documentId: 'doc-123',
-        _id: 'page-123',
-        parentedPath: [],
-        allFlaggedParents: [],
-        allFlaggedParentsMerged: '',
-        filePath: '',
-      });
-
-      const result = DyNTS_DAI_Document_Util.validateDocumentPage(page);
-
-      expect(result).toBe(false);
-      expect(logSpy).toHaveBeenCalled();
-    });
-  });
-
-  describe('| normalizeContent', () => {
-    it('| should normalize line endings from CRLF to LF', () => {
-      const content = 'Line 1\r\nLine 2\r\nLine 3';
-      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
-
-      expect(result).toBe('Line 1\nLine 2\nLine 3');
-    });
-
-    it('| should normalize line endings from CR to LF', () => {
-      const content = 'Line 1\rLine 2\rLine 3';
-      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
-
-      expect(result).toBe('Line 1\nLine 2\nLine 3');
-    });
-
-    it('| should trim leading and trailing whitespace', () => {
-      const content = '  Line 1\nLine 2  ';
-      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
-
-      expect(result).toBe('Line 1\nLine 2');
-    });
-
-    it('| should handle mixed line endings', () => {
-      const content = 'Line 1\r\nLine 2\rLine 3\nLine 4';
-      const result = DyNTS_DAI_Document_Util.normalizeContent(content);
-
-      expect(result).toBe('Line 1\nLine 2\nLine 3\nLine 4');
-    });
-  });
-
-  describe('| extractMetadata', () => {
-    it('| should extract title from first header', () => {
-      const content = '# Title\n\nContent here';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      expect(result.title).toBe('Title');
-    });
-
-    it('| should extract word count', () => {
-      const content = 'This is a test content with multiple words';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      // Content.split(/\s+/) returns 8 words: "This", "is", "a", "test", "content", "with", "multiple", "words"
-      expect(result.wordCount).toBe(8);
-    });
-
-    it('| should extract character count', () => {
-      const content = 'Test content';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      expect(result.characterCount).toBe(content.length);
-    });
-
-    it('| should extract header count', () => {
-      const content = '# Header 1\n## Header 2\n### Header 3';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      expect(result.headerCount).toBe(3);
-    });
-
-    it('| should return 0 header count when no headers', () => {
-      const content = 'Just plain text without headers';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      expect(result.headerCount).toBe(0);
-    });
-
-    it('| should extract all metadata fields', () => {
-      const content = '# Document Title\n\nThis is the content of the document.';
-      const result = DyNTS_DAI_Document_Util.extractMetadata(content);
-
-      expect(result.title).toBe('Document Title');
-      expect(result.wordCount).toBeGreaterThan(0);
-      expect(result.characterCount).toBeGreaterThan(0);
-      expect(result.headerCount).toBe(1);
-    });
-  });
-
-  describe('| isEmpty', () => {
-    it('| should return true for empty string', () => {
-      expect(DyNTS_DAI_Document_Util.isEmpty('')).toBe(true);
-    });
-
-    it('| should return true for whitespace only', () => {
-      expect(DyNTS_DAI_Document_Util.isEmpty('   \n\t  ')).toBe(true);
-    });
-
-    it('| should return false for content with text', () => {
-      expect(DyNTS_DAI_Document_Util.isEmpty('Some content')).toBe(false);
-    });
-
-    it('| should return true for null or undefined', () => {
-      expect(DyNTS_DAI_Document_Util.isEmpty(null as any)).toBe(true);
-      expect(DyNTS_DAI_Document_Util.isEmpty(undefined as any)).toBe(true);
-    });
-  });
-
-  describe('| getPreview', () => {
-    it('| should return full content when shorter than maxLength', () => {
-      const content = 'Short content';
-      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
-
-      expect(result).toBe('Short content');
-    });
-
-    it('| should truncate content when longer than maxLength', () => {
-      const content = 'A'.repeat(300);
-      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
-
-      expect(result.length).toBe(203); // 200 + '...'
-      expect(result.endsWith('...')).toBe(true);
-    });
-
-    it('| should use default maxLength of 200', () => {
-      const content = 'A'.repeat(250);
-      const result = DyNTS_DAI_Document_Util.getPreview(content);
-
-      expect(result.length).toBe(203); // 200 + '...'
-      expect(result.endsWith('...')).toBe(true);
-    });
-
-    it('| should handle exact maxLength', () => {
-      const content = 'A'.repeat(200);
-      const result = DyNTS_DAI_Document_Util.getPreview(content, 200);
-
-      expect(result).toBe(content);
-      expect(result.endsWith('...')).toBe(false);
-    });
-  });
-});
-

package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts

@@ -1,85 +0,0 @@
-import { DyFM_Log } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
-
-/**
- * Generic document utility functions
- * Provider-agnostic document operations
- */
-export class DyNTS_DAI_Document_Util {
-
-  static getClickupPageLink(pageName: string, workspaceId: string, documentId: string, pageId: string): string {
-         // https://app.clickup.com/9012753333/v/dc/8ck7cxn-692/8ck7cxn-1052
-    const link: string = `https://app.clickup.com/${workspaceId}/v/dc/${documentId}/${pageId}`;
-    return `[${pageName}](${link})`;
-  }
-  
-  /**
-   * Validate document page structure
-   */
-  static validateDocumentPage(page: DyNTS_DAI_DocPage): boolean {
-    if (!page.name) {
-      DyFM_Log.error('Document page validation failed: missing name');
-      return false;
-    }
-
-    if (!page.content) {
-      DyFM_Log.error('Document page validation failed: missing content');
-      return false;
-    }
-
-    return true;
-  }
-
-  /**
-   * Normalize document page content
-   */
-  static normalizeContent(content: string): string {
-    return content
-      .replace(/\r\n/g, '\n')  // Normalize line endings
-      .replace(/\r/g, '\n')    // Handle old Mac line endings
-      .trim();                 // Remove leading/trailing whitespace
-  }
-
-  /**
-   * Extract metadata from document content
-   */
-  static extractMetadata(content: string): Record<string, any> {
-    const metadata: Record<string, any> = {};
-    
-    // Extract title from first header
-    const titleMatch: RegExpMatchArray | null = content.match(/^#\s+(.+)$/m);
-    if (titleMatch) {
-      metadata.title = titleMatch[1].trim();
-    }
-
-    // Extract word count
-    metadata.wordCount = content.split(/\s+/).length;
-
-    // Extract character count
-    metadata.characterCount = content.length;
-
-    // Extract header count
-    const headerMatches = content.match(/^#{1,6}\s+.+$/gm);
-    metadata.headerCount = headerMatches ? headerMatches.length : 0;
-
-    return metadata;
-  }
-
-  /**
-   * Check if content is empty or whitespace only
-   */
-  static isEmpty(content: string): boolean {
-    return !content || content.trim().length === 0;
-  }
-
-  /**
-   * Get content preview (first N characters)
-   */
-  static getPreview(content: string, maxLength: number = 200): string {
-    if (content.length <= maxLength) {
-      return content;
-    }
-    
-    return content.substring(0, maxLength) + '...';
-  }
-}

package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts

@@ -1,8 +0,0 @@
-
-
-export enum DyNTS_DAI_CompareResult_Type {
-  equal = 'equal',
-  modified = 'modified',
-  new = 'new',
-  deleted = 'deleted',
-}

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts

@@ -1,146 +0,0 @@
-import { DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
-import { DyNTS_DAI_docPage_dataParams } from './dai-doc-page.data-model';
-
-/**
- * Generic chunk model for document processing
- * Provider-agnostic version of document chunks
- */
-export class DyNTS_DAI_DocChunk extends DyFM_Metadata {
-  documentId: string;
-  /** 
-   * this is the document id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalDocumentId?: string;
-  pageId: string;
-  /** 
-   * this is the page id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalPageId?: string;
-
-  /** Path to the page where the chunk is located ending with the page name */
-  filePathParents: string[] = [];
-  /** Headers that are parents of the chunk (all headers, all parent pages, and document name) */
-  chunkHeadParents: string[] = [];
-
-  /** 
-   * All parents of the chunk, flagged, from the highest level to the lowest local level
-   * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc. 
-   */
-  allFlaggedParents: string[] = [];
-  /** 
-   * allFlaggedParentsMerged
-   * All parents of the chunk, flagged, merged into a with '/' as separator
-   * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
-   */
-  path: string;
-  /** 
-   * The page name with the full path
-   * System/Project/Side/Module/Document/Page
-   * */
-  /* pageNameWithFullPath: string; */
-
-  /** The document name */
-  documentName: string;
-  pageName: string;
-  pageLink: string;
-  /** Index of the chunk in the page */
-  chunkIndex: number;
-
-  /* chunkOriginalContent: string; */
-
-  /**
-   * chunk Parented Content
-   */
-  chunkContent: string;
-  chunkParentedContentVectorized?: number[];
-  
-  constructor(
-    set?: DyNTS_DAI_DocChunk    
-  ) {
-    super(set);
-    
-    if (set) {
-      Object.assign(this, set);
-    }
-  }
-}
-
-export const DyNTS_DAI_docChunk_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocChunk> = 
-  new DyFM_DataModel_Params<DyNTS_DAI_DocChunk>({
-    dataName: 'doc_chunk',
-    properties: {
-      externalDocumentId: {
-        type: 'string', 
-        required: true, index: true,
-      },
-      documentId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
-      },
-    
-      externalPageId: {
-        type: 'string',
-        required: true, index: true,
-      },
-      pageId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_docPage_dataParams.dataName,
-      },
-      
-      filePathParents: {
-        type: 'string[]', 
-        required: true,
-      },
-      chunkHeadParents: {
-        type: 'string[]', 
-      },
-    
-      allFlaggedParents: {
-        type: 'string[]', 
-      },
-      path: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-    
-      documentName: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-      pageName: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-      pageLink: {
-        type: 'string',
-      },
-      chunkIndex: {
-        type: 'number', 
-        index: true,
-        required: true,
-      },
-    
-      chunkContent: {
-        type: 'string',
-        required: true,
-      },
-
-      chunkParentedContentVectorized: {
-        type: 'number[]',
-        required: true,
-        embeddingModel: DyFM_OAI_Model.textEmbedding_3Large,
-        // Generic embedding configuration - will be set by provider
-        vectorizedFrom: [ 'path', 'chunkContent' ],
-        vectorizeUseIndex: 'chunkParentedContentVectorized',
-      },
-    }
-  });

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts

@@ -1,163 +0,0 @@
-import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_DocChunk } from './dai-doc-chunk.data-model';
-import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
-
-
-
-export class DyNTS_DAI_DocPage<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_ExternalMetadata = any,
-> extends DyFM_Metadata {
-  documentId: string;
-  /** 
-   * this is the document id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalDocumentId?: string;
-  /** this is the page id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalPageId?: string;
-
-  name: string;
-  
-  allFlaggedParents: string[];
-  allFlaggedParentsMerged: string;
-  
-  filePath: string;
-  /** Path to the page in the document, starting with the document name */
-  parentedPath: string[];
-
-  /** runtime only */
-  chunks?: T_Chunk[] = [];
-  content: string;
-
-  /** runtime only */
-  pages?: DyNTS_DAI_DocPage<T_Chunk>[] = [];
-
-  externalMetadata?: T_ExternalMetadata;
-
-  // TODO: add last x edit history as git-like changes
-
-  constructor(
-    set?: DyNTS_DAI_DocPage<T_Chunk, T_ExternalMetadata>
-  ) {
-    super(set);
-
-    if (set) {
-      Object.assign(this, set);
-    }
-  }
-}
-
-/**
- * Data parameters for the DyNTS_DAI_DocPage data model
- * 
- * For enabling the vector search, you need to set the following in the database:
- * In the dataName, "doc_page" table, add the following searchIndex at indexes:
- * with name "chunkParentedContentVectorized":
- * ```json
- * {
- *   "fields": [
- *     {
- *       "type": "vector",
- *       "path": "chunkParentedContentVectorized",
- *       "numDimensions": 3072,
- *       "similarity": "cosine"
- *     }
- *   ]
- * }
- * ```
- */
-export const DyNTS_DAI_docPage_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocPage> = 
-  new DyFM_DataModel_Params<DyNTS_DAI_DocPage>({
-    dataName: 'doc_page',
-    properties: {
-      externalDocumentId: {
-        type: 'string',
-        required: true, index: true,
-      },
-      documentId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
-      },
-      externalPageId: {
-        type: 'string',
-        required: true, index: true, unique: true,
-      },
-
-      name: {
-        type: 'string',
-        required: true,
-      },
-
-      allFlaggedParents: {
-        type: 'string[]',
-        required: true,
-      },
-      allFlaggedParentsMerged: {
-        type: 'string',
-        required: true, 
-      },
-
-      filePath: {
-        type: 'string',
-        required: true,
-      },
-      parentedPath: {
-        type: 'string[]',
-        required: true,
-      },
-      
-      chunks: {
-        type: 'DyNTS_DAI_DocChunk[]',
-        forbidden: true,
-      },
-      content: {
-        type: 'string',
-      },
-
-      externalMetadata: {
-        type: 'T_ExternalMetadata',
-      },
-    },
-  });
-
-
-
-
-
-
-///**
-// * Generic document page interface
-// * Provider-agnostic document page structure
-// */
-//export interface DyNTS_DAI_DocPage<T_Chunk = any> {
-//  /** Unique identifier for the page */
-//  id?: string;
-//  /** Name of the page */
-//  name: string;
-//  /** Content of the page */
-//  content: string;
-//  /** Link to the page */
-//  link?: string;
-//  /** Path to the page */
-//  path?: string;
-//  /** Parent path elements */
-//  parentedPath?: string[];
-//  /** All flagged parents merged */
-//  allFlaggedParentsMerged?: string;
-//  /** All flagged parents as array */
-//  allFlaggedParents?: string[];
-//  /** Chunks contained in this page */
-//  chunks?: T_Chunk[];
-//  /** Metadata about the page */
-//  metadata?: {
-//    createdAt?: Date;
-//    updatedAt?: Date;
-//    size?: number;
-//    [key: string]: any;
-//  };
-//}
-//