@aj-archipelago/cortex 1.1.20 → 1.1.22

package/pathways/translate_gpt4.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpt4',
+    enableDuplicateRequests: false,
+
+}

package/pathways/translate_gpt4_turbo.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpt4-turbo',
+    enableDuplicateRequests: false,
+
+}

package/pathways/translate_turbo.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpturbo',
+    enableDuplicateRequests: false,
+
+}

package/pathways/vision.js

@@ -1,18 +1,20 @@
 import { Prompt } from '../server/prompt.js';
 
 export default {
-    prompt: [
-        new Prompt({ messages: [
-            "{{chatHistory}}",
-        ]}),
-    ],
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Jarvis Vision, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your primary expertise is image analysis. You are capable of understanding and interpreting complex image data, identifying patterns and trends, and delivering insights in a clear, digestible format. You know the current date and time - it is {{now}}."},
+                "{{chatHistory}}",
+            ]}),
+        ],
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
     },
     max_tokens: 1024,
-    model: 'oai-gpt4-vision',
-    tokenRatio: 0.96,
+    model: 'oai-gpt4o',
     useInputChunking: false,
     enableDuplicateRequests: false,
+    timeout: 600,
 }

package/server/modelExecutor.js

@@ -23,6 +23,7 @@ import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
 import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
+import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -57,6 +58,9 @@ class ModelExecutor {
             case 'OPENAI-WHISPER':
                 plugin = new OpenAIWhisperPlugin(pathway, model);
                 break;
+            case 'NEURALSPACE':
+                plugin = new NeuralSpacePlugin(pathway, model);
+                break;
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -25,8 +25,10 @@ class PathwayResolver {
         this.useInputChunking = pathway.useInputChunking;
         this.chunkMaxTokenLength = 0;
         this.warnings = [];
+        this.errors = [];
         this.requestId = uuidv4();
         this.responseParser = new PathwayResponseParser(pathway);
+        this.tool = null;
         this.modelName = [
             pathway.model,
             args?.model,
@@ -178,6 +180,15 @@ class PathwayResolver {
         }
     }
 
+    mergeResults(mergeData) {
+        if (mergeData) {
+            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
+            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
+            this.errors = [...this.errors, ...(mergeData.errors || [])];
+            this.tool = mergeData.tool || this.tool;
+        }
+    }
+
     async resolve(args) {
         // Either we're dealing with an async request, stream, or regular request
         if (args.async || args.stream) {
@@ -244,6 +255,12 @@ class PathwayResolver {
         logger.warn(warning);
     }
 
+    // Add an error and log it
+    logError(error) {
+        this.errors.push(error);
+        logger.error(error);
+    }
+
     // Here we choose how to handle long input - either summarize or chunk
     processInputText(text) {
         let chunkTokenLength = 0;
@@ -416,7 +433,8 @@ class PathwayResolver {
             }
         }
 
-        if (prompt.saveResultTo) {
+        // save the result to the context if requested and no errors
+        if (prompt.saveResultTo && this.errors.length === 0) {
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;

package/server/plugins/azureCognitivePlugin.js

@@ -34,7 +34,7 @@ class AzureCognitivePlugin extends ModelPlugin {
     async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, cortexRequest) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
-        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo } = combinedParameters;
+        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo, chatId } = combinedParameters;
         const data = {};
 
         if (mode == 'delete') {
@@ -46,6 +46,10 @@ class AzureCognitivePlugin extends ModelPlugin {
                 searchQuery += ` AND docId:'${docId}'`;
             }
 
+            if (chatId) {
+                searchQuery += ` AND chatId:'${chatId}'`;
+            }
+
             cortexRequest.url = searchUrl;
             cortexRequest.data =
             { search: searchQuery,  
@@ -75,6 +79,7 @@ class AzureCognitivePlugin extends ModelPlugin {
                 content: text,
                 owner: savedContextId,
                 docId: docId || uuidv4(),
+                chatId: chatId,
                 createdAt: new Date().toISOString()
             }
 
@@ -116,6 +121,10 @@ class AzureCognitivePlugin extends ModelPlugin {
         if (indexName == 'indexcortex') { //if private, filter by owner via contextId //privateData && 
             data.filter && (data.filter = data.filter + ' and ');
             data.filter = `owner eq '${savedContextId}'`;
+
+            if(chatId){
+                data.filter += ` and chatId eq '${chatId}'`;
+            }
         }
 
         return { data };

package/server/plugins/claude3VertexPlugin.js

@@ -138,7 +138,8 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
     modifiedMessages = combinedMessages;
 
-    // Claude vertex requires an even number of messages
+    // Claude vertex requires an odd number of messages
+    // for proper conversation turn-taking
     if (modifiedMessages.length % 2 === 0) {
       modifiedMessages = modifiedMessages.slice(1);
     }

package/server/plugins/gemini15ChatPlugin.js

@@ -76,7 +76,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -135,7 +135,13 @@ class Gemini15ChatPlugin extends ModelPlugin {
         if (data && data.contents && Array.isArray(data.contents)) {
             dataToMerge = data.contents;
         } else if (data && data.candidates && Array.isArray(data.candidates)) {
-            return data.candidates[0].content.parts[0].text;
+            const { content, finishReason, safetyRatings } = data.candidates[0];
+            if (finishReason === 'STOP') {
+                return content?.parts?.[0]?.text ?? '';
+            } else {
+                const returnString = `Response was not completed.  Finish reason: ${finishReason}, Safety ratings: ${JSON.stringify(safetyRatings, null, 2)}`;
+                throw new Error(returnString);
+            }
         } else if (Array.isArray(data)) {
             dataToMerge = data;
         } else {
@@ -154,7 +160,6 @@ class Gemini15ChatPlugin extends ModelPlugin {
         cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
         cortexRequest.params = {}; // query params
         cortexRequest.stream = stream;
-        cortexRequest.stream = stream;
         cortexRequest.urlSuffix = cortexRequest.stream ? ':streamGenerateContent?alt=sse' : ':generateContent';
 
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');

package/server/plugins/gemini15VisionPlugin.js

@@ -1,6 +1,5 @@
 import Gemini15ChatPlugin from './gemini15ChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class Gemini15VisionPlugin extends Gemini15ChatPlugin {
 
@@ -51,8 +50,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${inputPart}`);
+                        // this space intentionally left blank
                     }
+
                     return { text: inputPart };
                 };
     
@@ -83,7 +83,7 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -96,6 +96,22 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
         };
     }
 
+    async execute(text, parameters, prompt, cortexRequest) {
+        let result = null;
+        try {
+            result = await super.execute(text, parameters, prompt, cortexRequest);
+        } catch (e) {
+            const { data } = e;
+            if (data && data.error) {
+                if (data.error.code === 400 && data.error.message === 'Precondition check failed.') {
+                    throw new Error('One or more of the included files is too large to process. Please try again with a smaller file.');
+                }
+            throw e;
+            }
+        }
+        return result; 
+    }
+
 }
 
 export default Gemini15VisionPlugin;

package/server/plugins/geminiChatPlugin.js

@@ -81,7 +81,7 @@ class GeminiChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }

package/server/plugins/geminiVisionPlugin.js

@@ -1,6 +1,5 @@
 import GeminiChatPlugin from './geminiChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class GeminiVisionPlugin extends GeminiChatPlugin {
 
@@ -55,7 +54,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                        // this space intentionally left blank
                     }
                     return { text: partString };
                 };
@@ -87,7 +86,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }

package/server/plugins/neuralSpacePlugin.js

@@ -0,0 +1,252 @@
+import ModelPlugin from "./modelPlugin.js";
+import fs from "fs";
+import FormData from "form-data";
+import logger from "../../lib/logger.js";
+import {
+  alignSubtitles,
+  deleteTempPath,
+  downloadFile,
+  getMediaChunks,
+} from "../../lib/util.js";
+import CortexRequest from "../../lib/cortexRequest.js";
+import { publishRequestProgress } from "../../lib/redisSubscription.js";
+
+const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
+
+function convertToSrt(timestamps) {
+  let srt = "";
+  for (let i = 0; i < timestamps.length; i++) {
+    const _start = timestamps[i].start ?? timestamps[i].startTime;
+    const _end = timestamps[i].end ?? timestamps[i].endTime;
+    const _text = timestamps[i].word ?? timestamps[i].text;
+    const start = new Date(_start * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    const end = new Date(_end * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    srt += `${i + 1}\n${start} --> ${end}\n${_text}\n\n`;
+  }
+  return srt;
+}
+
+function convertToVtt(timestamps) {
+  let vtt = "WEBVTT\n\n";
+  for (let i = 0; i < timestamps.length; i++) {
+    const _start = timestamps[i].start ?? timestamps[i].startTime;
+    const _end = timestamps[i].end ?? timestamps[i].endTime;
+    const _text = timestamps[i].word ?? timestamps[i].text;
+    const start = new Date(_start * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    const end = new Date(_end * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    vtt += `${start} --> ${end}\n${_text}\n\n`;
+  }
+  return vtt;
+}
+
+class NeuralSpacePlugin extends ModelPlugin {
+  constructor(pathway, model) {
+    super(pathway, model);
+    this.pathwayResolver = null;
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const { responseFormat, file, language, wordTimestamped, maxLineWidth } =
+      parameters;
+
+    let chunks = [];
+    let offsets = [];
+
+    try {
+      const { pathwayResolver } = cortexRequest;
+
+      const { requestId } = pathwayResolver;
+
+      const mediaChunks = await getMediaChunks(file, requestId);
+
+      if (!mediaChunks || !mediaChunks.length) {
+        throw new Error(
+          `Error in getting chunks from media helper for file ${file}`
+        );
+      }
+
+      const uris = mediaChunks.map((chunk) => chunk?.uri || chunk);
+      offsets = mediaChunks.map(
+        (chunk, index) => chunk?.offset || index * OFFSET_CHUNK
+      );
+
+      let totalCount = uris.length * 2; // [download, request] jobs per chunk
+      let completedCount = 0;
+
+      const sendProgress = () => {
+        completedCount++;
+        if (completedCount >= totalCount) return;
+
+        const progress = completedCount / totalCount;
+        logger.info(`Progress for ${requestId}: ${progress}`);
+
+        publishRequestProgress({
+          requestId,
+          progress,
+          data: null,
+        });
+      };
+
+      for (let i = 0; i < uris.length; i++) {
+        const uri = uris[i];
+        try {
+          const chunk = await downloadFile(uri);
+          chunks.push(chunk);
+          sendProgress();
+        } catch (err) {
+          logger.error(`Error downloading chunk: ${err}`);
+          throw err;
+        }
+      }
+
+      const jobs = [];
+
+      for (const chunk of chunks) {
+        const cortexRequest = new CortexRequest({ pathwayResolver });
+        cortexRequest.url = this.requestUrl();
+
+        const formData = new FormData();
+        formData.append("files", fs.createReadStream(chunk));
+        const configObj = {
+          file_transcription: {
+            mode: "advanced",
+          },
+        };
+
+        //phrase/segment level
+        if ((responseFormat && !wordTimestamped) || maxLineWidth) {
+          configObj.speaker_diarization = {
+            // mode: "speakers",
+            // num_speakers: numSpeakers,
+            // overrides: {
+            //   clustering: {
+            //     threshold: clusteringThreshold,
+            //   },
+            // },
+          };
+
+          configObj.subtitles_guidelines = {
+            line_count: 1
+          };
+        }
+
+        if (maxLineWidth) {
+          configObj.subtitles_guidelines = {
+            character_count: maxLineWidth,
+          };
+        }
+
+        if (language) {
+          configObj.file_transcription.language_id = language;
+        }
+        formData.append("config", JSON.stringify(configObj));
+
+        cortexRequest.data = formData;
+        cortexRequest.params = {};
+        cortexRequest.headers = {
+          ...cortexRequest.headers,
+          ...formData.getHeaders(),
+        };
+
+        const result = await this.executeRequest(cortexRequest);
+
+        const jobId = result?.data?.jobId;
+        if (!jobId) {
+          logger.error(`Error in creating job: ${JSON.stringify(result)}`);
+          return;
+        }
+        logger.info(`Job created successfully with ID: ${jobId}`);
+        jobs.push(jobId);
+      }
+
+      return await this.checkJobStatus(
+        jobs,
+        pathwayResolver,
+        sendProgress,
+        responseFormat,
+        offsets
+      );
+    } catch (error) {
+      logger.error(`Error occurred while executing: ${error}`);
+      throw error;
+    } finally {
+      for (const chunk of chunks) {
+        try {
+          await deleteTempPath(chunk);
+        } catch (error) {
+          // Ignore error
+          logger.error(`Error deleting temp file: ${error}`);
+        }
+      }
+    }
+  }
+
+  async checkJobStatus(
+    jobs,
+    pathwayResolver,
+    sendProgress,
+    responseFormat,
+    offsets
+  ) {
+    const textResults = [];
+    const timestampResults = [];
+    for (let i = 0; i < jobs.length; i++) {
+      const jobId = jobs[i];
+      const result = await this.getJobStatus(jobId, pathwayResolver);
+      const text = result.data.result.transcription.channels[0].transcript;
+      textResults.push(text);
+      timestampResults.push(
+        result.data.result.transcription?.segments?.length > 0 ? 
+        result.data.result.transcription.segments :
+        result.data.result.transcription.channels[0].timestamps
+      );
+      sendProgress();
+    }
+
+    if (responseFormat) {
+      const output = timestampResults.map((t) =>
+        responseFormat === "srt" ? convertToSrt(t) : convertToVtt(t)
+      );
+      return alignSubtitles(output, responseFormat, offsets);
+    }
+
+    return textResults.join(" ").trim();
+  }
+
+  async getJobStatus(jobId, pathwayResolver) {
+    const cortexRequest = new CortexRequest({ pathwayResolver });
+    cortexRequest.url = `${this.requestUrl()}/${jobId}`;
+    cortexRequest.method = "GET";
+    const result = await this.executeRequest(cortexRequest);
+
+    const status = result?.data?.status;
+    if (!status) {
+      throw new Error(`Error in getting job status: ${result}`);
+    }
+
+    if (status === "Completed") {
+      return result;
+    }
+
+    if (status === "Failed") {
+      throw new Error(`Job failed with error: ${result.data.error}`);
+    } else {
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      return this.getJobStatus(jobId, pathwayResolver);
+    }
+  }
+}
+
+export default NeuralSpacePlugin;

package/server/plugins/openAiVisionPlugin.js

@@ -1,27 +1,38 @@
 import OpenAIChatPlugin from './openAiChatPlugin.js';
 
+function safeJsonParse(content) {
+    try {
+        const parsedContent = JSON.parse(content);
+        return (typeof parsedContent === 'object' && parsedContent !== null) ? parsedContent : content;
+    } catch (e) {
+        return content;
+    }
+}
 
 class OpenAIVisionPlugin extends OpenAIChatPlugin {
-
+    
     tryParseMessages(messages) {
-        //check if elements of messages strings are JSON, if valid JSON parse them to obj
-        messages.map(message => {
+        return messages.map(message => {
             try {
-                // message.content can be array or string
                 if (typeof message.content === 'string') {
-                    message.content = JSON.parse(message.content);
-                } else if (Array.isArray(message.content)) {
+                    message.content = safeJsonParse(message.content);
+                }
+                if (Array.isArray(message.content)) {
                     message.content = message.content.map(item => {
-                        const parsedItem = JSON.parse(item);
-                        const { type, text, image_url, url } = parsedItem;
-                        return { type, text, image_url: url || image_url};
+                        if (typeof item === 'string') {
+                            return { type: 'text', text: item };
+                        } else {
+                            const parsedItem = safeJsonParse(item);
+                            const { type, text, image_url, url } = parsedItem;
+                            return { type, text, image_url: url || image_url };
+                        }
                     });
-                }     
+                }
             } catch (e) {
                 return message;
             }
+            return message;
         });
-        return messages;
     }
 
     getRequestParameters(text, parameters, prompt) {
@@ -29,8 +40,16 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
 
         this.tryParseMessages(requestParameters.messages);
 
-        if(this.promptParameters.max_tokens) {
-            requestParameters.max_tokens = this.promptParameters.max_tokens;
+        const modelMaxReturnTokens = this.getModelMaxReturnTokens();
+        const maxTokensPrompt = this.promptParameters.max_tokens;
+        const maxTokensModel = this.getModelMaxTokenLength() * (1 - this.getPromptTokenRatio());
+
+        const maxTokens = maxTokensPrompt || maxTokensModel;
+
+        requestParameters.max_tokens = maxTokens ? Math.min(maxTokens, modelMaxReturnTokens) : modelMaxReturnTokens;
+
+        if (this.promptParameters.json) {
+            //requestParameters.response_format = { type: "json_object", }
         }
 
         return requestParameters;