@futdevpro/nts-dynamo 1.10.77 → 1.11.1

package/src/_modules/open-ai/_enums/oai-gpt-message-role.enum.ts

@@ -0,0 +1,45 @@
+
+
+
+export enum DyNTS_OAI_GPT_Message_Role {
+  
+  /**
+   * Developer-provided instructions that the model should follow, regardless of
+   * messages sent by the user. With o1 models and newer, use `developer` messages
+   * for this purpose instead.
+   */
+  system = 'system',
+  
+  /**
+   * Messages sent by an end user, containing prompts or additional context
+   * information.
+   */
+  user = 'user',
+  
+  /**
+   * Messages sent by the model in response to user messages.
+   */
+  assistant = 'assistant',
+
+  /**
+   * Developer-provided instructions that the model should follow, regardless of
+   * messages sent by the user. With o1 models and newer, `developer` messages
+   * replace the previous `system` messages.
+   * 
+   * @deprecated
+   */
+  /* developer = 'developer', */
+
+  /** 
+   * ??? 
+   * @deprecated
+   */
+  /* tool = 'tool', */
+  
+  /**
+   * ???
+   * @deprecated
+   */
+  /* function = 'function', */
+}
+

package/src/_modules/open-ai/_models/interfaces/oai-chunk-compare-result.interface.ts

@@ -0,0 +1,10 @@
+
+
+import { DyNTS_OAI_CompareResult_Type } from '../../_enums/oai-compare-result-type.enum';
+import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
+
+
+export interface DyNTS_OAI_ChunkCompareResult {
+  result: DyNTS_OAI_CompareResult_Type;
+  oldChunk: DyNTS_OAI_DocChunk;
+}

package/src/_modules/open-ai/_models/interfaces/oai-document-page.interface.ts

@@ -0,0 +1,20 @@
+import { DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
+
+
+
+
+export interface DyNTS_OAI_DocumentPage extends DyFM_Metadata {
+  workspaceId: string;
+  docId: string;
+  parentPageId: string;
+
+  path: string[];
+  documentParent: string;
+
+  name: string;
+  content: string;
+  
+  pages: DyNTS_OAI_DocumentPage[];
+  chunks: DyNTS_OAI_DocChunk[];
+}

package/src/_modules/open-ai/_models/interfaces/oai-global-settings.interface.ts

@@ -0,0 +1,8 @@
+import { DyFM_GPTCall_Settings } from '@futdevpro/fsm-dynamo/open-ai';
+
+
+
+export interface DyNTS_OAI_Global_Settings {
+  defaultSettings: DyFM_GPTCall_Settings;
+  maxMessagesStringLength: number;
+}

package/src/_modules/open-ai/_models/interfaces/oai-gpt-message.interface.ts

@@ -0,0 +1,7 @@
+import { DyNTS_OAI_GPT_Message_Role } from '../../_enums/oai-gpt-message-role.enum';
+
+export interface DyNTS_OAI_GPT_Message {
+  role: DyNTS_OAI_GPT_Message_Role /* | string */;
+  content: string;
+}
+

package/src/_modules/open-ai/_models/interfaces/oai-llm-predefined-requests.interface.ts

@@ -0,0 +1,57 @@
+
+
+export interface DyNTS_OAI_LLM_Predefined_Requests {
+  defaultSystemPrompt: string;
+
+
+  yesNo: {
+    /**
+     * @default ' (Only answer with "Yes" or "No")'
+     */
+    request: string;
+    /** 
+     * This will be used to check if the answer is yes (using includes in the answer)
+     * @default 'YES'
+     */
+    upperCaseYes: string;
+  },
+  /**
+   * @default ' (Only answer with a number between 0 and 1, e.g.: 0.65)'
+   */
+  percentageOnly: string;
+  /**
+   * use {{DyNTS_selectOptions}} without spaces to insert the options
+   * @default ' (Only answer with one of the following: {{DyNTS_selectOptions}})'
+   * */
+  selectRequest: string;
+  /**
+   * use {{DyNTS_selectOptions}} without spaces to insert the options
+   * @default ' (Only answer with one or more of the following: {{DyNTS_selectOptions}})'
+   */
+  multiselect: string;
+  /**
+   * @default ' (The answer must be in JSON format)'
+   */
+  jsonRequest: string;
+  /**
+   * use {{DyNTS_jsonKeysDescription}} to insert the keys description
+   * @default ' (The answer must be in JSON format, {{DyNTS_jsonKeysDescription}})'
+   */
+  jsonRequestWithKeysDescription: string;
+  /**
+   * @default ' (The answer must be in JSON format, where the keys are: {{DyNTS_jsonKeys}})'
+   */
+  jsonRequestWithExactKeys: string;
+  /**
+   * @default ' (The answer must be in JSON list format)'
+   */
+  listRequest: string;
+
+  vectorSearch: {
+    systemPrompt: string;
+    getSearchQuestion: string;
+    resolveSearchResults: string;
+    doWeNeedVectorSearch: string;
+  }
+}
+

package/src/_modules/open-ai/_models/interfaces/oai-page-compare-result.interface.ts

@@ -0,0 +1,18 @@
+
+import { DyNTS_OAI_CompareResult_Type } from '../../_enums/oai-compare-result-type.enum';
+import { DyNTS_OAI_DocChunk } from '../oai-doc-chunk.data-model';
+import { DyNTS_OAI_ChunkCompareResult } from './oai-chunk-compare-result.interface';
+import { DyNTS_OAI_DocumentPage } from './oai-document-page.interface';
+
+
+
+
+export interface DyNTS_OAI_PageCompareResult {
+  result: DyNTS_OAI_CompareResult_Type;
+  subjectPage: DyNTS_OAI_DocumentPage;
+  oldPageContent: string;
+  chunkCompareResults: DyNTS_OAI_ChunkCompareResult[];
+
+  newChunks: DyNTS_OAI_DocChunk[];
+  oldChunks: DyNTS_OAI_DocChunk[];
+}

package/src/_modules/open-ai/_models/oai-doc-chunk.data-model.ts

@@ -0,0 +1,92 @@
+
+
+
+import { DyFM_DataModel_Params, DyFM_Metadata } from '@futdevpro/fsm-dynamo';
+import { DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
+
+
+
+export class DyNTS_OAI_DocChunk extends DyFM_Metadata {
+  workspaceId?: string;
+  documentId?: string;
+  documentPageId?: string;
+  mainDocumentName?: string;
+  documentName?: string;
+  documentPath?: string[];
+
+  chunkIndex?: number;
+  /** parent headers */
+  chunkPath?: string[];
+
+  chunkOriginalContent?: string;
+
+  chunkParentedContent?: string;
+  chunkParentedContentVectorized?: number[] = [];
+
+  constructor(
+    set?: DyNTS_OAI_DocChunk,
+  ) {
+    super(set);
+
+    if (set) {
+      Object.assign(this, set);
+    }
+  }
+}
+
+export const DyNTS_OAI_docChunk_dataParams = new DyFM_DataModel_Params<DyNTS_OAI_DocChunk>({
+  dataName: 'oai_doc_chunk',
+  properties: {
+    workspaceId: { 
+      type: 'string', index: true,
+      required: true, 
+    },
+    documentId: { 
+      type: 'string', index: true,
+      required: true, 
+    },
+    documentPageId: { 
+      type: 'string', index: true,
+      required: true, 
+    },
+    mainDocumentName: { 
+      type: 'string',
+      required: true, 
+    },
+    documentName: { 
+      type: 'string',
+      required: true, 
+    },
+    documentPath: { 
+      type: 'string[]',
+      required: true, 
+    },
+
+    chunkIndex: { 
+      type: 'number', index: true,
+      required: true, 
+    },
+    chunkPath: { 
+      type: 'string[]',
+      required: true, 
+    },
+
+    chunkOriginalContent: { 
+      type: 'string',
+      required: true,
+    },
+
+    chunkParentedContent: { 
+      type: 'string', index: true,
+      required: true,
+    },
+    chunkParentedContentVectorized: { 
+      type: 'number[]', 
+      required: true, index: true,
+      embeddingModel: DyFM_OpenAIModel.textEmbedding_3Large,
+      vectorizedFrom: 'chunkParentedContent',
+      vectorizeUseIndex: 'chunkParentedContentVectorized',
+      vectorizeAlways: true,
+    },
+  },
+});

package/src/_modules/open-ai/_services/oai-chunk.service-base.ts

@@ -0,0 +1,214 @@
+import { DyFM_Array, DyFM_DataModel_Params, DyFM_Error, DyFM_Log } from '@futdevpro/fsm-dynamo';
+import { DyNTS_OAI_DocChunk, DyNTS_OAI_docChunk_dataParams } from '../_models/oai-doc-chunk.data-model';
+import { DyNTS_OAI_VectorDataService } from './oai-vector-data.service';
+import { DyFM_OpenAI_Settings, DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
+import { DyNTS_OAI_PageCompareResult } from '../_models/interfaces/oai-page-compare-result.interface';
+import { DyNTS_OAI_CompareResult_Type } from '../_enums/oai-compare-result-type.enum';
+import { DyNTS_OAI_DocumentPage } from '../_models/interfaces/oai-document-page.interface';
+import { DyNTS_OAI_ChunkCompareResult } from '../_models/interfaces/oai-chunk-compare-result.interface';
+import { DyNTS_OAI_Chunking_Util } from '../_collections/oai-chunking.util';
+import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
+
+
+export class DyNTS_OAI_Chunk_ServiceBase extends DyNTS_OAI_VectorDataService<DyNTS_OAI_DocChunk> {
+
+  constructor(issuer: string) {
+    super(
+      null, 
+      DyNTS_OAI_docChunk_dataParams, 
+      new DyFM_OpenAI_Settings({
+        openAIConfig: {
+          apiKey: DyNTS_global_settings.env_settings.openAi.apiKey,
+          organization: DyNTS_global_settings.env_settings.openAi.organization,
+          project: DyNTS_global_settings.env_settings.openAi.project,
+        },
+        defaultSettings: {
+          useModel: DyFM_OpenAIModel.textEmbedding_3Large,
+        },
+      }),
+      issuer
+    );
+  }
+
+  async updatePage(pageCompareResult: DyNTS_OAI_PageCompareResult, issuer: string): Promise<void> {
+    try {
+      // save chunks 
+      // (only new and modified pages 
+      //   (but all the page element should be updated 
+      //     (delete extra elements)))
+
+      if (pageCompareResult.oldChunks.length > pageCompareResult.newChunks.length) {
+        // delete extra chunks
+        for (let i = 0; i < pageCompareResult.oldChunks.length; i++) {
+          if (i < pageCompareResult.newChunks.length) {
+            pageCompareResult.newChunks[i]._id = pageCompareResult.oldChunks[i]._id;
+          } else {
+            await this.deleteData(pageCompareResult.oldChunks[i]._id, true).catch(error => {
+              DyFM_Error.logSimple('❌ Chunk not deleted', error);
+            });
+          }
+        }
+      }
+
+      await DyFM_Array.asyncForEachAllAtOnce(
+        pageCompareResult.newChunks,
+        async (chunk) => {
+          if (
+            pageCompareResult.chunkCompareResults[chunk.chunkIndex].result !== DyNTS_OAI_CompareResult_Type.equal
+          ) {
+            await this.saveData(chunk, false, true);
+            DyFM_Log.info(`✅ Chunk saved: ${pageCompareResult.subjectPage.name}; ${chunk.chunkIndex}`)
+          }
+        }
+      );
+
+      DyFM_Log.success(`✅ Chunks updated(SAVED) for page: ${pageCompareResult.subjectPage.name}`);
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('updatePage', error),
+        errorCode: 'DCH-CSB-UP0',
+      });
+    }
+  }
+
+  async comparePage(page: DyNTS_OAI_DocumentPage, issuer: string): Promise<DyNTS_OAI_PageCompareResult> {
+    try {
+      const oldChunks = await this.findDataList({
+        documentId: `${page.docId}`,
+        documentPageId: page._id,
+      });
+
+      const compareResults: DyNTS_OAI_ChunkCompareResult[] = await this.compareChunks(
+        page.chunks,
+        oldChunks
+      );
+
+      /* DyFM_Log.H_info(`✅ Modified page: ${page.name}`, {
+        compareResults: compareResults,
+        equal: compareResults.every(result => result.result === DCh_CompareResult_Type.equal),
+        new: compareResults.every(result => result.result === DCh_CompareResult_Type.new),
+        modified: compareResults.some(
+          result => ![ DCh_CompareResult_Type.equal, DCh_CompareResult_Type.new ].includes(result.result)
+        ),
+      }); */
+
+      if (
+        compareResults.every(result => result.result === DyNTS_OAI_CompareResult_Type.equal)
+      ) {
+        return {
+          result: DyNTS_OAI_CompareResult_Type.equal,
+          subjectPage: page,
+          oldPageContent: page.content,
+          chunkCompareResults: compareResults,
+          newChunks: page.chunks,
+          oldChunks: oldChunks,
+        };
+      }
+
+      if (
+        compareResults.every(result => result.result === DyNTS_OAI_CompareResult_Type.new)
+      ) {
+        return {
+          result: DyNTS_OAI_CompareResult_Type.new,
+          subjectPage: page,
+          oldPageContent: '',
+          chunkCompareResults: compareResults,
+          newChunks: page.chunks,
+          oldChunks: oldChunks,
+        };
+      }
+
+      return {
+        result: DyNTS_OAI_CompareResult_Type.modified,
+        subjectPage: page,
+        oldPageContent: DyNTS_OAI_Chunking_Util.reassembleChunksToContent(
+          compareResults.filter(result => result.oldChunk).map(result => result.oldChunk),
+          issuer
+        ),
+        chunkCompareResults: compareResults,
+        newChunks: page.chunks,
+        oldChunks: oldChunks,
+      };
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('comparePage', error),
+        errorCode: 'DCH-CSB-CP0',
+      });
+    }
+  }
+
+  async compareChunks(
+    chunks: DyNTS_OAI_DocChunk[], 
+    oldChunks?: DyNTS_OAI_DocChunk[]
+  ): Promise<DyNTS_OAI_ChunkCompareResult[]> {
+    try {
+      if (chunks.length === 0) {
+        return [];
+      }
+
+      oldChunks ??= await this.findDataList({
+        documentId: chunks[0].documentId,
+        documentPageId: chunks[0].documentPageId,
+      });
+      oldChunks.sort((a, b) => a.chunkIndex - b.chunkIndex);
+
+      return await DyFM_Array.asyncForEachAllAtOnce(
+        chunks,
+        async (chunk) => {
+          return await this.compareChunk(chunk, oldChunks[chunk.chunkIndex]);
+        }
+      )
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareChunks', error),
+        errorCode: 'DCH-CSB-CC0',
+      });
+    }
+  }
+
+  async compareChunk(
+    newChunk: DyNTS_OAI_DocChunk, 
+    oldChunk?: DyNTS_OAI_DocChunk
+  ): Promise<DyNTS_OAI_ChunkCompareResult> {
+    try {
+      oldChunk ??= await this.findData({
+        documentId: newChunk.documentId,
+        documentPageId: newChunk.documentPageId,
+        chunkIndex: newChunk.chunkIndex,
+      });
+
+      /* DyFM_Log.H_info(`✅ Comparing chunk: ${newChunk.chunkIndex}`, {
+        newChunk: newChunk,
+        oldChunk: oldChunk,
+        new: !oldChunk,
+        equal: oldChunk?.chunkParentedContent === newChunk.chunkParentedContent,
+        modified: oldChunk && oldChunk.chunkParentedContent !== newChunk.chunkParentedContent,
+      }); */
+
+      if (!oldChunk) {
+        return {
+          result: DyNTS_OAI_CompareResult_Type.new,
+          oldChunk: null,
+        };
+      }
+
+      if (oldChunk.chunkParentedContent === newChunk.chunkParentedContent) {
+        return {
+          result: DyNTS_OAI_CompareResult_Type.equal,
+          oldChunk: oldChunk,
+        };
+      }
+
+      return {
+        result: DyNTS_OAI_CompareResult_Type.modified,
+        oldChunk: oldChunk,
+      };
+    } catch (error) {
+      throw new DyFM_Error({
+        ...this.getDefaultErrorSettings('compareChunk', error),
+        errorCode: 'DCH-CSB-CC1',
+      });
+    }
+  }
+  
+}

package/src/_modules/open-ai/_services/{embedding.control-service.ts → oai-embedding.control-service.ts}

@@ -2,10 +2,10 @@
 import { DyFM_OpenAIModel } from '@futdevpro/fsm-dynamo/open-ai';
 import { DyFM_Error } from '@futdevpro/fsm-dynamo';
 
-import { DyNTS_OpenAI_ServiceBase } from './open-ai.service-base';
+import { DyNTS_OAI_LLM_ServiceBase } from './oai-llm.service-base';
 import { DyNTS_global_settings } from '../../../_collections/global-settings.const';
 
-export class Embedding_ControlService extends DyNTS_OpenAI_ServiceBase {
+export class DyNTS_OAI_Embedding_ControlService extends DyNTS_OAI_LLM_ServiceBase {
 
   async createEmbedding(
     text: string,