@futdevpro/nts-dynamo 1.12.15 → 1.14.2

package/src/_modules/ai/_modules/document-ai/_collections/dai-document.util.ts

@@ -0,0 +1,85 @@
+import { DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocPage } from '../_models/data-models/dai-doc-page.data-model';
+
+/**
+ * Generic document utility functions
+ * Provider-agnostic document operations
+ */
+export class DyNTS_DAI_Document_Util {
+
+  static getClickupPageLink(pageName: string, workspaceId: string, documentId: string, pageId: string): string {
+         // https://app.clickup.com/9012753333/v/dc/8ck7cxn-692/8ck7cxn-1052
+    const link: string = `https://app.clickup.com/${workspaceId}/v/dc/${documentId}/${pageId}`;
+    return `[${pageName}](${link})`;
+  }
+  
+  /**
+   * Validate document page structure
+   */
+  static validateDocumentPage(page: DyNTS_DAI_DocPage): boolean {
+    if (!page.name) {
+      DyFM_Log.error('Document page validation failed: missing name');
+      return false;
+    }
+
+    if (!page.content) {
+      DyFM_Log.error('Document page validation failed: missing content');
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Normalize document page content
+   */
+  static normalizeContent(content: string): string {
+    return content
+      .replace(/\r\n/g, '\n')  // Normalize line endings
+      .replace(/\r/g, '\n')    // Handle old Mac line endings
+      .trim();                 // Remove leading/trailing whitespace
+  }
+
+  /**
+   * Extract metadata from document content
+   */
+  static extractMetadata(content: string): Record<string, any> {
+    const metadata: Record<string, any> = {};
+    
+    // Extract title from first header
+    const titleMatch: RegExpMatchArray | null = content.match(/^#\s+(.+)$/m);
+    if (titleMatch) {
+      metadata.title = titleMatch[1].trim();
+    }
+
+    // Extract word count
+    metadata.wordCount = content.split(/\s+/).length;
+
+    // Extract character count
+    metadata.characterCount = content.length;
+
+    // Extract header count
+    const headerMatches = content.match(/^#{1,6}\s+.+$/gm);
+    metadata.headerCount = headerMatches ? headerMatches.length : 0;
+
+    return metadata;
+  }
+
+  /**
+   * Check if content is empty or whitespace only
+   */
+  static isEmpty(content: string): boolean {
+    return !content || content.trim().length === 0;
+  }
+
+  /**
+   * Get content preview (first N characters)
+   */
+  static getPreview(content: string, maxLength: number = 200): string {
+    if (content.length <= maxLength) {
+      return content;
+    }
+    
+    return content.substring(0, maxLength) + '...';
+  }
+}

package/src/_modules/ai/_modules/document-ai/_enums/dai-compare-result-type.enum.ts

@@ -0,0 +1,8 @@
+
+
+export enum DyNTS_DAI_CompareResult_Type {
+  equal = 'equal',
+  modified = 'modified',
+  new = 'new',
+  deleted = 'deleted',
+}

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts

@@ -0,0 +1,144 @@
+import { DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
+import { DyNTS_DAI_docPage_dataParams } from './dai-doc-page.data-model';
+
+/**
+ * Generic chunk model for document processing
+ * Provider-agnostic version of document chunks
+ */
+export class DyNTS_DAI_DocChunk extends DyFM_Metadata {
+  documentId: string;
+  /** 
+   * this is the document id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalDocumentId?: string;
+  pageId: string;
+  /** 
+   * this is the page id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalPageId?: string;
+
+  /** Path to the page where the chunk is located ending with the page name */
+  filePathParents: string[] = [];
+  /** Headers that are parents of the chunk (all headers, all parent pages, and document name) */
+  chunkHeadParents: string[] = [];
+
+  /** 
+   * All parents of the chunk, flagged, from the highest level to the lowest local level
+   * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc. 
+   */
+  allFlaggedParents: string[] = [];
+  /** 
+   * allFlaggedParentsMerged
+   * All parents of the chunk, flagged, merged into a with '/' as separator
+   * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
+   */
+  path: string;
+  /** 
+   * The page name with the full path
+   * System/Project/Side/Module/Document/Page
+   * */
+  /* pageNameWithFullPath: string; */
+
+  /** The document name */
+  documentName: string;
+  pageName: string;
+  pageLink: string;
+  /** Index of the chunk in the page */
+  chunkIndex: number;
+
+  /* chunkOriginalContent: string; */
+
+  /**
+   * chunk Parented Content
+   */
+  chunkContent: string;
+  chunkParentedContentVectorized?: number[];
+  
+  constructor(
+    set?: DyNTS_DAI_DocChunk    
+  ) {
+    super(set);
+    
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+export const DyNTS_DAI_docChunk_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocChunk> = 
+  new DyFM_DataModel_Params<DyNTS_DAI_DocChunk>({
+    dataName: 'doc_chunk',
+    properties: {
+      externalDocumentId: {
+        type: 'string',
+      },
+      documentId: {
+        type: 'string',
+        required: true, index: true,
+        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
+      },
+    
+      externalPageId: {
+        type: 'string',
+      },
+      pageId: {
+        type: 'string',
+        required: true, index: true,
+        dependencyDataName: DyNTS_DAI_docPage_dataParams.dataName,
+      },
+      
+      filePathParents: {
+        type: 'string[]', 
+        required: true,
+      },
+      chunkHeadParents: {
+        type: 'string[]', 
+      },
+    
+      allFlaggedParents: {
+        type: 'string[]', 
+      },
+      path: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+    
+      documentName: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+      pageName: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+      pageLink: {
+        type: 'string',
+      },
+      chunkIndex: {
+        type: 'number', 
+        index: true,
+        required: true,
+      },
+    
+      chunkContent: {
+        type: 'string',
+        required: true,
+      },
+      chunkParentedContentVectorized: {
+        type: 'number[]',
+        required: true,
+        embeddingModel: DyFM_OAI_Model.textEmbedding_3Large,
+        // Generic embedding configuration - will be set by provider
+        vectorizedFrom: [ 'path', 'chunkContent' ],
+        vectorizeUseIndex: 'chunkParentedContentVectorized',
+        vectorizeAlways: true,
+      },
+    }
+  });

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts

@@ -0,0 +1,136 @@
+import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunk } from './dai-doc-chunk.data-model';
+import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
+
+
+
+export class DyNTS_DAI_DocPage<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_ExternalMetadata = any,
+> extends DyFM_Metadata {
+  documentId: string;
+  /** 
+   * this is the document id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalDocumentId?: string;
+  /** this is the page id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalPageId?: string;
+
+  name: string;
+  
+  allFlaggedParents: string[];
+  allFlaggedParentsMerged: string;
+  
+  filePath: string;
+  /** Path to the page in the document, starting with the document name */
+  parentedPath: string[];
+
+  /** runtime only */
+  chunks?: T_Chunk[] = [];
+  content: string;
+
+  /** runtime only */
+  pages?: DyNTS_DAI_DocPage<T_Chunk>[] = [];
+
+  externalMetadata?: T_ExternalMetadata;
+
+  constructor(
+    set?: DyNTS_DAI_DocPage<T_Chunk, T_ExternalMetadata>
+  ) {
+    super(set);
+
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+export const DyNTS_DAI_docPage_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocPage> = {
+  dataName: 'doc_page',
+  properties: {
+    externalDocumentId: {
+      type: 'string',
+    },
+    documentId: {
+      type: 'string',
+      required: true, index: true,
+      dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
+    },
+    externalPageId: {
+      type: 'string',
+    },
+
+    name: {
+      type: 'string',
+      required: true,
+    },
+
+    allFlaggedParents: {
+      type: 'string[]',
+      required: true,
+    },
+    allFlaggedParentsMerged: {
+      type: 'string',
+      required: true,
+    },
+
+    filePath: {
+      type: 'string',
+      required: true,
+    },
+    parentedPath: {
+      type: 'string[]',
+      required: true,
+    },
+    
+    content: {
+      type: 'string',
+      required: true,
+    },
+
+    externalMetadata: {
+      type: 'T_ExternalMetadata',
+    },
+  },
+};
+
+
+
+
+
+
+///**
+// * Generic document page interface
+// * Provider-agnostic document page structure
+// */
+//export interface DyNTS_DAI_DocPage<T_Chunk = any> {
+//  /** Unique identifier for the page */
+//  id?: string;
+//  /** Name of the page */
+//  name: string;
+//  /** Content of the page */
+//  content: string;
+//  /** Link to the page */
+//  link?: string;
+//  /** Path to the page */
+//  path?: string;
+//  /** Parent path elements */
+//  parentedPath?: string[];
+//  /** All flagged parents merged */
+//  allFlaggedParentsMerged?: string;
+//  /** All flagged parents as array */
+//  allFlaggedParents?: string[];
+//  /** Chunks contained in this page */
+//  chunks?: T_Chunk[];
+//  /** Metadata about the page */
+//  metadata?: {
+//    createdAt?: Date;
+//    updatedAt?: Date;
+//    size?: number;
+//    [key: string]: any;
+//  };
+//}
+//

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-document.data-model.ts

@@ -0,0 +1,93 @@
+import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunk } from './dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocPage } from './dai-doc-page.data-model';
+
+
+
+export class DyNTS_DAI_Document<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>,
+  T_ExternalDocumentId = string, // | Record<string, string | number>,
+  T_ExternalMetadata = any,
+> extends DyFM_Metadata {
+  /** 
+   * this is the document id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalDocumentId?: T_ExternalDocumentId; // | Record<string, string | number>;
+
+  name: string;
+
+  /** runtime only */
+  pages?: T_Page[] = [];
+
+  externalMetadata?: T_ExternalMetadata;
+
+  constructor(
+    set?: DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata>
+  ) {
+    super(set);
+
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+export const DyNTS_DAI_document_dataParams: DyFM_DataModel_Params<DyNTS_DAI_Document> = 
+  new DyFM_DataModel_Params<DyNTS_DAI_Document>({
+    dataName: 'document',
+    properties: {
+      externalDocumentId: {
+        type: 'string',
+        required: true,
+      },
+
+      name: {
+        type: 'string',
+        required: true,
+      },
+
+      externalMetadata: {
+        type: 'T_ExternalMetadata',
+      },
+    },
+  });
+
+
+
+
+
+
+///**
+// * Generic document page interface
+// * Provider-agnostic document page structure
+// */
+//export interface DyNTS_DAI_DocPage<T_Chunk = any> {
+//  /** Unique identifier for the page */
+//  id?: string;
+//  /** Name of the page */
+//  name: string;
+//  /** Content of the page */
+//  content: string;
+//  /** Link to the page */
+//  link?: string;
+//  /** Path to the page */
+//  path?: string;
+//  /** Parent path elements */
+//  parentedPath?: string[];
+//  /** All flagged parents merged */
+//  allFlaggedParentsMerged?: string;
+//  /** All flagged parents as array */
+//  allFlaggedParents?: string[];
+//  /** Chunks contained in this page */
+//  chunks?: T_Chunk[];
+//  /** Metadata about the page */
+//  metadata?: {
+//    createdAt?: Date;
+//    updatedAt?: Date;
+//    size?: number;
+//    [key: string]: any;
+//  };
+//}
+//

package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-chunk-compare-result.interface.ts

@@ -0,0 +1,18 @@
+
+
+import { DyNTS_DAI_CompareResult_Type } from '../../_enums/dai-compare-result-type.enum';
+import { DyNTS_DAI_DocChunk } from '../data-models/dai-doc-chunk.data-model';
+
+
+export interface DyNTS_DAI_DocChunkCompareResult<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk
+> {
+  result: DyNTS_DAI_CompareResult_Type;
+  newChunk: T_Chunk;
+  oldChunk?: T_Chunk;
+
+  reports?: string[];
+}
+
+
+

package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-doc-page-compare-result.interface.ts

@@ -0,0 +1,19 @@
+import { DyNTS_DAI_DocChunk } from '../data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from './dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../_enums/dai-compare-result-type.enum';
+import { DyNTS_DAI_DocPage } from '../data-models/dai-doc-page.data-model';
+
+export interface DyNTS_DAI_DocPageCompareResult<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, 
+> {
+  result: DyNTS_DAI_CompareResult_Type;
+  newPage: T_Page;
+  oldPage?: T_Page;
+  
+  /* newChunks: T_Chunk[];
+  oldChunks: T_Chunk[]; */
+  chunkCompareResults: DyNTS_DAI_DocChunkCompareResult<T_Chunk>[];
+
+  reports?: string[];
+}

package/src/_modules/ai/_modules/document-ai/_models/interfaces/dai-document-compare-result.interface.ts

@@ -0,0 +1,25 @@
+import { DyNTS_DAI_DocChunk } from '../data-models/dai-doc-chunk.data-model';
+import { DyNTS_DAI_DocChunkCompareResult } from './dai-doc-chunk-compare-result.interface';
+import { DyNTS_DAI_CompareResult_Type } from '../../_enums/dai-compare-result-type.enum';
+import { DyNTS_DAI_DocPage } from '../data-models/dai-doc-page.data-model';
+import { DyNTS_DAI_Document } from '../data-models/dai-document.data-model';
+import { DyNTS_DAI_DocPageCompareResult } from './dai-doc-page-compare-result.interface';
+
+export interface DyNTS_DAI_DocumentCompareResult<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_Page extends DyNTS_DAI_DocPage<T_Chunk> = DyNTS_DAI_DocPage<T_Chunk>, 
+  T_ExternalDocumentId = string,
+  T_ExternalMetadata = any,
+  T_Document extends DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata> = 
+    DyNTS_DAI_Document<T_Chunk, T_Page, T_ExternalDocumentId, T_ExternalMetadata>,
+> {
+  result: DyNTS_DAI_CompareResult_Type;
+  newDocument: T_Document;
+  oldDocument?: T_Document;
+
+  /* newPages: T_Page[];
+  oldPages?: T_Page[]; */
+  pageCompareResults: DyNTS_DAI_DocPageCompareResult<T_Chunk, T_Page>[];
+
+  reports: string[];
+}

package/src/_modules/ai/_modules/document-ai/index.ts

@@ -0,0 +1,28 @@
+// Re-export from FSM
+export * from '@futdevpro/fsm-dynamo/ai/document-ai';
+
+// COLLECTIONS
+export * from './_collections/dai-chunking.util';
+export * from './_collections/dai-document.util';
+
+// ENUMS
+//export * from './_enums/dai-validation-error.enum';
+export * from './_enums/dai-compare-result-type.enum';
+
+// MODELS
+// models/DATA-MODELS
+export * from './_models/data-models/dai-doc-chunk.data-model';
+export * from './_models/data-models/dai-document.data-model';
+export * from './_models/data-models/dai-doc-page.data-model';
+
+// models/INTERFACES
+export * from './_models/interfaces/dai-doc-chunk-compare-result.interface';
+export * from './_models/interfaces/dai-doc-page-compare-result.interface';
+export * from './_models/interfaces/dai-document-compare-result.interface';
+
+//export * from './_models/dai-embedding-info.interface';
+//export * from './_models/dai-embedding-validation.interface';
+
+// SERVICES
+//export * from './_services/dai-vector-data.service-base';
+/* export * from './_services/dai-doc-chunk.service-base'; */

package/src/_modules/{open-ai → ai/_modules/open-ai}/_collections/oai-global-settings.const.ts

@@ -1,9 +1,9 @@
-import { DyFM_GPTCall_Settings } from '@futdevpro/fsm-dynamo/open-ai';
+import { DyFM_OAI_CallSettings } from '@futdevpro/fsm-dynamo/ai/open-ai';
 import { DyNTS_OAI_Global_Settings } from '../_models/interfaces/oai-global-settings.interface';
 
 
 
 export const DyNTS_OAI_global_settings: DyNTS_OAI_Global_Settings = {
-  defaultSettings: new DyFM_GPTCall_Settings(),
+  defaultSettings: new DyFM_OAI_CallSettings(),
   maxMessagesStringLength: 120_000,
 }

package/src/_modules/ai/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts

@@ -0,0 +1,8 @@
+import { DyFM_OAI_CallSettings } from '@futdevpro/fsm-dynamo/ai/open-ai';
+
+
+
+export interface DyNTS_OAI_Global_Settings {
+  defaultSettings: DyFM_OAI_CallSettings;
+  maxMessagesStringLength: number;
+}