@futdevpro/nts-dynamo 1.15.13 → 1.15.15

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-chunk.data-model.ts

@@ -1,146 +1,146 @@
-import { DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
-import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
-import { DyNTS_DAI_docPage_dataParams } from './dai-doc-page.data-model';
-
-/**
- * Generic chunk model for document processing
- * Provider-agnostic version of document chunks
- */
-export class DyNTS_DAI_DocChunk extends DyFM_Metadata {
-  documentId: string;
-  /** 
-   * this is the document id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalDocumentId?: string;
-  pageId: string;
-  /** 
-   * this is the page id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalPageId?: string;
-
-  /** Path to the page where the chunk is located ending with the page name */
-  filePathParents: string[] = [];
-  /** Headers that are parents of the chunk (all headers, all parent pages, and document name) */
-  chunkHeadParents: string[] = [];
-
-  /** 
-   * All parents of the chunk, flagged, from the highest level to the lowest local level
-   * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc. 
-   */
-  allFlaggedParents: string[] = [];
-  /** 
-   * allFlaggedParentsMerged
-   * All parents of the chunk, flagged, merged into a with '/' as separator
-   * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
-   */
-  path: string;
-  /** 
-   * The page name with the full path
-   * System/Project/Side/Module/Document/Page
-   * */
-  /* pageNameWithFullPath: string; */
-
-  /** The document name */
-  documentName: string;
-  pageName: string;
-  pageLink: string;
-  /** Index of the chunk in the page */
-  chunkIndex: number;
-
-  /* chunkOriginalContent: string; */
-
-  /**
-   * chunk Parented Content
-   */
-  chunkContent: string;
-  chunkParentedContentVectorized?: number[];
-  
-  constructor(
-    set?: DyNTS_DAI_DocChunk    
-  ) {
-    super(set);
-    
-    if (set) {
-      Object.assign(this, set);
-    }
-  }
-}
-
-export const DyNTS_DAI_docChunk_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocChunk> = 
-  new DyFM_DataModel_Params<DyNTS_DAI_DocChunk>({
-    dataName: 'doc_chunk',
-    properties: {
-      externalDocumentId: {
-        type: 'string', 
-        required: true, index: true,
-      },
-      documentId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
-      },
-    
-      externalPageId: {
-        type: 'string',
-        required: true, index: true,
-      },
-      pageId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_docPage_dataParams.dataName,
-      },
-      
-      filePathParents: {
-        type: 'string[]', 
-        required: true,
-      },
-      chunkHeadParents: {
-        type: 'string[]', 
-      },
-    
-      allFlaggedParents: {
-        type: 'string[]', 
-      },
-      path: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-    
-      documentName: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-      pageName: {
-        type: 'string', 
-        index: true,
-        required: true,
-      },
-      pageLink: {
-        type: 'string',
-      },
-      chunkIndex: {
-        type: 'number', 
-        index: true,
-        required: true,
-      },
-    
-      chunkContent: {
-        type: 'string',
-        required: true,
-      },
-
-      chunkParentedContentVectorized: {
-        type: 'number[]',
-        required: true,
-        embeddingModel: DyFM_OAI_Model.textEmbedding_3Large,
-        // Generic embedding configuration - will be set by provider
-        vectorizedFrom: [ 'path', 'chunkContent' ],
-        vectorizeUseIndex: 'chunkParentedContentVectorized',
-      },
-    }
-  });
+import { DyFM_OAI_Model } from '@futdevpro/fsm-dynamo/ai/open-ai';
+import { DyFM_DataModel_Params, DyFM_DataProperties, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
+import { DyNTS_DAI_docPage_dataParams } from './dai-doc-page.data-model';
+
+/**
+ * Generic chunk model for document processing
+ * Provider-agnostic version of document chunks
+ */
+export class DyNTS_DAI_DocChunk extends DyFM_Metadata {
+  documentId: string;
+  /** 
+   * this is the document id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalDocumentId?: string;
+  pageId: string;
+  /** 
+   * this is the page id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalPageId?: string;
+
+  /** Path to the page where the chunk is located ending with the page name */
+  filePathParents: string[] = [];
+  /** Headers that are parents of the chunk (all headers, all parent pages, and document name) */
+  chunkHeadParents: string[] = [];
+
+  /** 
+   * All parents of the chunk, flagged, from the highest level to the lowest local level
+   * like: System:FDP, Project:Organizer, Side:Backend, Module:Chat, etc. 
+   */
+  allFlaggedParents: string[] = [];
+  /** 
+   * allFlaggedParentsMerged
+   * All parents of the chunk, flagged, merged into a with '/' as separator
+   * like: System:FDP/Project:Organizer/Side:Backend/Module:Notes/Document:Note Data Model
+   */
+  path: string;
+  /** 
+   * The page name with the full path
+   * System/Project/Side/Module/Document/Page
+   * */
+  /* pageNameWithFullPath: string; */
+
+  /** The document name */
+  documentName: string;
+  pageName: string;
+  pageLink: string;
+  /** Index of the chunk in the page */
+  chunkIndex: number;
+
+  /* chunkOriginalContent: string; */
+
+  /**
+   * chunk Parented Content
+   */
+  chunkContent: string;
+  chunkParentedContentVectorized?: number[];
+  
+  constructor(
+    set?: DyNTS_DAI_DocChunk    
+  ) {
+    super(set);
+    
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+export const DyNTS_DAI_docChunk_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocChunk> = 
+  new DyFM_DataModel_Params<DyNTS_DAI_DocChunk>({
+    dataName: 'doc_chunk',
+    properties: {
+      externalDocumentId: {
+        type: 'string', 
+        required: true, index: true,
+      },
+      documentId: {
+        type: 'string',
+        required: true, index: true,
+        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
+      },
+    
+      externalPageId: {
+        type: 'string',
+        required: true, index: true,
+      },
+      pageId: {
+        type: 'string',
+        required: true, index: true,
+        dependencyDataName: DyNTS_DAI_docPage_dataParams.dataName,
+      },
+      
+      filePathParents: {
+        type: 'string[]', 
+        required: true,
+      },
+      chunkHeadParents: {
+        type: 'string[]', 
+      },
+    
+      allFlaggedParents: {
+        type: 'string[]', 
+      },
+      path: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+    
+      documentName: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+      pageName: {
+        type: 'string', 
+        index: true,
+        required: true,
+      },
+      pageLink: {
+        type: 'string',
+      },
+      chunkIndex: {
+        type: 'number', 
+        index: true,
+        required: true,
+      },
+    
+      chunkContent: {
+        type: 'string',
+        required: true,
+      },
+
+      chunkParentedContentVectorized: {
+        type: 'number[]',
+        required: true,
+        embeddingModel: DyFM_OAI_Model.textEmbedding_3Large,
+        // Generic embedding configuration - will be set by provider
+        vectorizedFrom: [ 'path', 'chunkContent' ],
+        vectorizeUseIndex: 'chunkParentedContentVectorized',
+      },
+    }
+  });

package/src/_modules/ai/_modules/document-ai/_models/data-models/dai-doc-page.data-model.ts

@@ -1,163 +1,163 @@
-import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
-import { DyNTS_DAI_DocChunk } from './dai-doc-chunk.data-model';
-import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
-
-
-
-export class DyNTS_DAI_DocPage<
-  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
-  T_ExternalMetadata = any,
-> extends DyFM_Metadata {
-  documentId: string;
-  /** 
-   * this is the document id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalDocumentId?: string;
-  /** this is the page id in the outside source 
-   * eg.: the id in Clickup, Confluence, etc.
-   * */
-  externalPageId?: string;
-
-  name: string;
-  
-  allFlaggedParents: string[];
-  allFlaggedParentsMerged: string;
-  
-  filePath: string;
-  /** Path to the page in the document, starting with the document name */
-  parentedPath: string[];
-
-  /** runtime only */
-  chunks?: T_Chunk[] = [];
-  content: string;
-
-  /** runtime only */
-  pages?: DyNTS_DAI_DocPage<T_Chunk>[] = [];
-
-  externalMetadata?: T_ExternalMetadata;
-
-  // TODO: add last x edit history as git-like changes
-
-  constructor(
-    set?: DyNTS_DAI_DocPage<T_Chunk, T_ExternalMetadata>
-  ) {
-    super(set);
-
-    if (set) {
-      Object.assign(this, set);
-    }
-  }
-}
-
-/**
- * Data parameters for the DyNTS_DAI_DocPage data model
- * 
- * For enabling the vector search, you need to set the following in the database:
- * In the dataName, "doc_page" table, add the following searchIndex at indexes:
- * with name "chunkParentedContentVectorized":
- * ```json
- * {
- *   "fields": [
- *     {
- *       "type": "vector",
- *       "path": "chunkParentedContentVectorized",
- *       "numDimensions": 3072,
- *       "similarity": "cosine"
- *     }
- *   ]
- * }
- * ```
- */
-export const DyNTS_DAI_docPage_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocPage> = 
-  new DyFM_DataModel_Params<DyNTS_DAI_DocPage>({
-    dataName: 'doc_page',
-    properties: {
-      externalDocumentId: {
-        type: 'string',
-        required: true, index: true,
-      },
-      documentId: {
-        type: 'string',
-        required: true, index: true,
-        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
-      },
-      externalPageId: {
-        type: 'string',
-        required: true, index: true, unique: true,
-      },
-
-      name: {
-        type: 'string',
-        required: true,
-      },
-
-      allFlaggedParents: {
-        type: 'string[]',
-        required: true,
-      },
-      allFlaggedParentsMerged: {
-        type: 'string',
-        required: true, 
-      },
-
-      filePath: {
-        type: 'string',
-        required: true,
-      },
-      parentedPath: {
-        type: 'string[]',
-        required: true,
-      },
-      
-      chunks: {
-        type: 'DyNTS_DAI_DocChunk[]',
-        forbidden: true,
-      },
-      content: {
-        type: 'string',
-      },
-
-      externalMetadata: {
-        type: 'T_ExternalMetadata',
-      },
-    },
-  });
-
-
-
-
-
-
-///**
-// * Generic document page interface
-// * Provider-agnostic document page structure
-// */
-//export interface DyNTS_DAI_DocPage<T_Chunk = any> {
-//  /** Unique identifier for the page */
-//  id?: string;
-//  /** Name of the page */
-//  name: string;
-//  /** Content of the page */
-//  content: string;
-//  /** Link to the page */
-//  link?: string;
-//  /** Path to the page */
-//  path?: string;
-//  /** Parent path elements */
-//  parentedPath?: string[];
-//  /** All flagged parents merged */
-//  allFlaggedParentsMerged?: string;
-//  /** All flagged parents as array */
-//  allFlaggedParents?: string[];
-//  /** Chunks contained in this page */
-//  chunks?: T_Chunk[];
-//  /** Metadata about the page */
-//  metadata?: {
-//    createdAt?: Date;
-//    updatedAt?: Date;
-//    size?: number;
-//    [key: string]: any;
-//  };
-//}
+import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_DAI_DocChunk } from './dai-doc-chunk.data-model';
+import { DyNTS_DAI_document_dataParams } from './dai-document.data-model';
+
+
+
+export class DyNTS_DAI_DocPage<
+  T_Chunk extends DyNTS_DAI_DocChunk = DyNTS_DAI_DocChunk,
+  T_ExternalMetadata = any,
+> extends DyFM_Metadata {
+  documentId: string;
+  /** 
+   * this is the document id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalDocumentId?: string;
+  /** this is the page id in the outside source 
+   * eg.: the id in Clickup, Confluence, etc.
+   * */
+  externalPageId?: string;
+
+  name: string;
+  
+  allFlaggedParents: string[];
+  allFlaggedParentsMerged: string;
+  
+  filePath: string;
+  /** Path to the page in the document, starting with the document name */
+  parentedPath: string[];
+
+  /** runtime only */
+  chunks?: T_Chunk[] = [];
+  content: string;
+
+  /** runtime only */
+  pages?: DyNTS_DAI_DocPage<T_Chunk>[] = [];
+
+  externalMetadata?: T_ExternalMetadata;
+
+  // TODO: add last x edit history as git-like changes
+
+  constructor(
+    set?: DyNTS_DAI_DocPage<T_Chunk, T_ExternalMetadata>
+  ) {
+    super(set);
+
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+/**
+ * Data parameters for the DyNTS_DAI_DocPage data model
+ * 
+ * For enabling the vector search, you need to set the following in the database:
+ * In the dataName, "doc_page" table, add the following searchIndex at indexes:
+ * with name "chunkParentedContentVectorized":
+ * ```json
+ * {
+ *   "fields": [
+ *     {
+ *       "type": "vector",
+ *       "path": "chunkParentedContentVectorized",
+ *       "numDimensions": 3072,
+ *       "similarity": "cosine"
+ *     }
+ *   ]
+ * }
+ * ```
+ */
+export const DyNTS_DAI_docPage_dataParams: DyFM_DataModel_Params<DyNTS_DAI_DocPage> = 
+  new DyFM_DataModel_Params<DyNTS_DAI_DocPage>({
+    dataName: 'doc_page',
+    properties: {
+      externalDocumentId: {
+        type: 'string',
+        required: true, index: true,
+      },
+      documentId: {
+        type: 'string',
+        required: true, index: true,
+        dependencyDataName: DyNTS_DAI_document_dataParams.dataName,
+      },
+      externalPageId: {
+        type: 'string',
+        required: true, index: true, unique: true,
+      },
+
+      name: {
+        type: 'string',
+        required: true,
+      },
+
+      allFlaggedParents: {
+        type: 'string[]',
+        required: true,
+      },
+      allFlaggedParentsMerged: {
+        type: 'string',
+        required: true, 
+      },
+
+      filePath: {
+        type: 'string',
+        required: true,
+      },
+      parentedPath: {
+        type: 'string[]',
+        required: true,
+      },
+      
+      chunks: {
+        type: 'DyNTS_DAI_DocChunk[]',
+        forbidden: true,
+      },
+      content: {
+        type: 'string',
+      },
+
+      externalMetadata: {
+        type: 'T_ExternalMetadata',
+      },
+    },
+  });
+
+
+
+
+
+
+///**
+// * Generic document page interface
+// * Provider-agnostic document page structure
+// */
+//export interface DyNTS_DAI_DocPage<T_Chunk = any> {
+//  /** Unique identifier for the page */
+//  id?: string;
+//  /** Name of the page */
+//  name: string;
+//  /** Content of the page */
+//  content: string;
+//  /** Link to the page */
+//  link?: string;
+//  /** Path to the page */
+//  path?: string;
+//  /** Parent path elements */
+//  parentedPath?: string[];
+//  /** All flagged parents merged */
+//  allFlaggedParentsMerged?: string;
+//  /** All flagged parents as array */
+//  allFlaggedParents?: string[];
+//  /** Chunks contained in this page */
+//  chunks?: T_Chunk[];
+//  /** Metadata about the page */
+//  metadata?: {
+//    createdAt?: Date;
+//    updatedAt?: Date;
+//    size?: number;
+//    [key: string]: any;
+//  };
+//}
 //