@aj-archipelago/cortex 1.1.20 → 1.1.21

package/server/plugins/neuralSpacePlugin.js

@@ -0,0 +1,252 @@
+import ModelPlugin from "./modelPlugin.js";
+import fs from "fs";
+import FormData from "form-data";
+import logger from "../../lib/logger.js";
+import {
+  alignSubtitles,
+  deleteTempPath,
+  downloadFile,
+  getMediaChunks,
+} from "../../lib/util.js";
+import CortexRequest from "../../lib/cortexRequest.js";
+import { publishRequestProgress } from "../../lib/redisSubscription.js";
+
+const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
+
+function convertToSrt(timestamps) {
+  let srt = "";
+  for (let i = 0; i < timestamps.length; i++) {
+    const _start = timestamps[i].start ?? timestamps[i].startTime;
+    const _end = timestamps[i].end ?? timestamps[i].endTime;
+    const _text = timestamps[i].word ?? timestamps[i].text;
+    const start = new Date(_start * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    const end = new Date(_end * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    srt += `${i + 1}\n${start} --> ${end}\n${_text}\n\n`;
+  }
+  return srt;
+}
+
+function convertToVtt(timestamps) {
+  let vtt = "WEBVTT\n\n";
+  for (let i = 0; i < timestamps.length; i++) {
+    const _start = timestamps[i].start ?? timestamps[i].startTime;
+    const _end = timestamps[i].end ?? timestamps[i].endTime;
+    const _text = timestamps[i].word ?? timestamps[i].text;
+    const start = new Date(_start * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    const end = new Date(_end * 1000)
+      .toISOString()
+      .slice(11, -1)
+      .replace(".", ",");
+    vtt += `${start} --> ${end}\n${_text}\n\n`;
+  }
+  return vtt;
+}
+
+class NeuralSpacePlugin extends ModelPlugin {
+  constructor(pathway, model) {
+    super(pathway, model);
+    this.pathwayResolver = null;
+  }
+
+  async execute(text, parameters, prompt, cortexRequest) {
+    const { responseFormat, file, language, wordTimestamped, maxLineWidth } =
+      parameters;
+
+    let chunks = [];
+    let offsets = [];
+
+    try {
+      const { pathwayResolver } = cortexRequest;
+
+      const { requestId } = pathwayResolver;
+
+      const mediaChunks = await getMediaChunks(file, requestId);
+
+      if (!mediaChunks || !mediaChunks.length) {
+        throw new Error(
+          `Error in getting chunks from media helper for file ${file}`
+        );
+      }
+
+      const uris = mediaChunks.map((chunk) => chunk?.uri || chunk);
+      offsets = mediaChunks.map(
+        (chunk, index) => chunk?.offset || index * OFFSET_CHUNK
+      );
+
+      let totalCount = uris.length * 2; // [download, request] jobs per chunk
+      let completedCount = 0;
+
+      const sendProgress = () => {
+        completedCount++;
+        if (completedCount >= totalCount) return;
+
+        const progress = completedCount / totalCount;
+        logger.info(`Progress for ${requestId}: ${progress}`);
+
+        publishRequestProgress({
+          requestId,
+          progress,
+          data: null,
+        });
+      };
+
+      for (let i = 0; i < uris.length; i++) {
+        const uri = uris[i];
+        try {
+          const chunk = await downloadFile(uri);
+          chunks.push(chunk);
+          sendProgress();
+        } catch (err) {
+          logger.error(`Error downloading chunk: ${err}`);
+          throw err;
+        }
+      }
+
+      const jobs = [];
+
+      for (const chunk of chunks) {
+        const cortexRequest = new CortexRequest({ pathwayResolver });
+        cortexRequest.url = this.requestUrl();
+
+        const formData = new FormData();
+        formData.append("files", fs.createReadStream(chunk));
+        const configObj = {
+          file_transcription: {
+            mode: "advanced",
+          },
+        };
+
+        //phrase/segment level
+        if ((responseFormat && !wordTimestamped) || maxLineWidth) {
+          configObj.speaker_diarization = {
+            // mode: "speakers",
+            // num_speakers: numSpeakers,
+            // overrides: {
+            //   clustering: {
+            //     threshold: clusteringThreshold,
+            //   },
+            // },
+          };
+
+          configObj.subtitles_guidelines = {
+            line_count: 1
+          };
+        }
+
+        if (maxLineWidth) {
+          configObj.subtitles_guidelines = {
+            character_count: maxLineWidth,
+          };
+        }
+
+        if (language) {
+          configObj.file_transcription.language_id = language;
+        }
+        formData.append("config", JSON.stringify(configObj));
+
+        cortexRequest.data = formData;
+        cortexRequest.params = {};
+        cortexRequest.headers = {
+          ...cortexRequest.headers,
+          ...formData.getHeaders(),
+        };
+
+        const result = await this.executeRequest(cortexRequest);
+
+        const jobId = result?.data?.jobId;
+        if (!jobId) {
+          logger.error(`Error in creating job: ${JSON.stringify(result)}`);
+          return;
+        }
+        logger.info(`Job created successfully with ID: ${jobId}`);
+        jobs.push(jobId);
+      }
+
+      return await this.checkJobStatus(
+        jobs,
+        pathwayResolver,
+        sendProgress,
+        responseFormat,
+        offsets
+      );
+    } catch (error) {
+      logger.error(`Error occurred while executing: ${error}`);
+      throw error;
+    } finally {
+      for (const chunk of chunks) {
+        try {
+          await deleteTempPath(chunk);
+        } catch (error) {
+          // Ignore error
+          logger.error(`Error deleting temp file: ${error}`);
+        }
+      }
+    }
+  }
+
+  async checkJobStatus(
+    jobs,
+    pathwayResolver,
+    sendProgress,
+    responseFormat,
+    offsets
+  ) {
+    const textResults = [];
+    const timestampResults = [];
+    for (let i = 0; i < jobs.length; i++) {
+      const jobId = jobs[i];
+      const result = await this.getJobStatus(jobId, pathwayResolver);
+      const text = result.data.result.transcription.channels[0].transcript;
+      textResults.push(text);
+      timestampResults.push(
+        result.data.result.transcription?.segments?.length > 0 ? 
+        result.data.result.transcription.segments :
+        result.data.result.transcription.channels[0].timestamps
+      );
+      sendProgress();
+    }
+
+    if (responseFormat) {
+      const output = timestampResults.map((t) =>
+        responseFormat === "srt" ? convertToSrt(t) : convertToVtt(t)
+      );
+      return alignSubtitles(output, responseFormat, offsets);
+    }
+
+    return textResults.join(" ").trim();
+  }
+
+  async getJobStatus(jobId, pathwayResolver) {
+    const cortexRequest = new CortexRequest({ pathwayResolver });
+    cortexRequest.url = `${this.requestUrl()}/${jobId}`;
+    cortexRequest.method = "GET";
+    const result = await this.executeRequest(cortexRequest);
+
+    const status = result?.data?.status;
+    if (!status) {
+      throw new Error(`Error in getting job status: ${result}`);
+    }
+
+    if (status === "Completed") {
+      return result;
+    }
+
+    if (status === "Failed") {
+      throw new Error(`Job failed with error: ${result.data.error}`);
+    } else {
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      return this.getJobStatus(jobId, pathwayResolver);
+    }
+  }
+}
+
+export default NeuralSpacePlugin;

package/server/plugins/openAiVisionPlugin.js

@@ -4,17 +4,20 @@ import OpenAIChatPlugin from './openAiChatPlugin.js';
 class OpenAIVisionPlugin extends OpenAIChatPlugin {
 
     tryParseMessages(messages) {
-        //check if elements of messages strings are JSON, if valid JSON parse them to obj
         messages.map(message => {
             try {
-                // message.content can be array or string
                 if (typeof message.content === 'string') {
                     message.content = JSON.parse(message.content);
-                } else if (Array.isArray(message.content)) {
+                }
+                if (Array.isArray(message.content)) {
                     message.content = message.content.map(item => {
-                        const parsedItem = JSON.parse(item);
-                        const { type, text, image_url, url } = parsedItem;
-                        return { type, text, image_url: url || image_url};
+                        if (typeof item === 'string') {
+                            return { type: 'text', text: item };
+                        } else {
+                            const parsedItem = JSON.parse(item);
+                            const { type, text, image_url, url } = parsedItem;
+                            return { type, text, image_url: url || image_url };
+                        }
                     });
                 }     
             } catch (e) {
@@ -29,8 +32,16 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
 
         this.tryParseMessages(requestParameters.messages);
 
-        if(this.promptParameters.max_tokens) {
-            requestParameters.max_tokens = this.promptParameters.max_tokens;
+        const modelMaxReturnTokens = this.getModelMaxReturnTokens();
+        const maxTokensPrompt = this.promptParameters.max_tokens;
+        const maxTokensModel = this.getModelMaxTokenLength() * (1 - this.getPromptTokenRatio());
+
+        const maxTokens = maxTokensPrompt || maxTokensModel;
+
+        requestParameters.max_tokens = maxTokens ? Math.min(maxTokens, modelMaxReturnTokens) : modelMaxReturnTokens;
+
+        if (this.promptParameters.json) {
+            //requestParameters.response_format = { type: "json_object", }
         }
 
         return requestParameters;

package/server/plugins/openAiWhisperPlugin.js

@@ -1,24 +1,13 @@
 // openAiWhisperPlugin.js
 import ModelPlugin from './modelPlugin.js';
 import { config } from '../../config.js';
-import subsrt from 'subsrt';
 import FormData from 'form-data';
 import fs from 'fs';
-import { axios } from '../../lib/requestExecutor.js';
-import stream from 'stream';
-import os from 'os';
-import path from 'path';
-import http from 'http';
-import https from 'https';
-import { URL } from 'url';
-import { v4 as uuidv4 } from 'uuid';
-import { promisify } from 'util';
 import { publishRequestProgress } from '../../lib/redisSubscription.js';
 import logger from '../../lib/logger.js';
 import CortexRequest from '../../lib/cortexRequest.js';
-const pipeline = promisify(stream.pipeline);
+import { downloadFile, deleteTempPath, convertSrtToText, alignSubtitles, getMediaChunks, markCompletedForCleanUp  } from '../../lib/util.js';
 
-const API_URL = config.get('whisperMediaApiUrl');
 const WHISPER_TS_API_URL  = config.get('whisperTSApiUrl');
 if(WHISPER_TS_API_URL){
     logger.info(`WHISPER API URL using ${WHISPER_TS_API_URL}`);
@@ -28,147 +17,11 @@ if(WHISPER_TS_API_URL){
 
 const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
-async function deleteTempPath(path) {
-    try {
-        if (!path) {
-            logger.warn('Temporary path is not defined.');
-            return;
-        }
-        if (!fs.existsSync(path)) {
-            logger.warn(`Temporary path ${path} does not exist.`);
-            return;
-        }
-        const stats = fs.statSync(path);
-        if (stats.isFile()) {
-            fs.unlinkSync(path);
-            logger.info(`Temporary file ${path} deleted successfully.`);
-        } else if (stats.isDirectory()) {
-            fs.rmSync(path, { recursive: true });
-            logger.info(`Temporary folder ${path} and its contents deleted successfully.`);
-        }
-    } catch (err) {
-        logger.error(`Error occurred while deleting the temporary path: ${err}`);
-    }
-}
-
-function generateUniqueFilename(extension) {
-    return `${uuidv4()}.${extension}`;
-}
-
-const downloadFile = async (fileUrl) => {
-    const fileExtension = path.extname(fileUrl).slice(1);
-    const uniqueFilename = generateUniqueFilename(fileExtension);
-    const tempDir = os.tmpdir();
-    const localFilePath = `${tempDir}/${uniqueFilename}`;
-
-    // eslint-disable-next-line no-async-promise-executor
-    return new Promise(async (resolve, reject) => {
-        try {
-            const parsedUrl = new URL(fileUrl);
-            const protocol = parsedUrl.protocol === 'https:' ? https : http;
-
-            const response = await new Promise((resolve, reject) => {
-                protocol.get(parsedUrl, (res) => {
-                    if (res.statusCode === 200) {
-                        resolve(res);
-                    } else {
-                        reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
-                    }
-                }).on('error', reject);
-            });
-
-            await pipeline(response, fs.createWriteStream(localFilePath));
-            logger.info(`Downloaded file to ${localFilePath}`);
-            resolve(localFilePath);
-        } catch (error) {
-            fs.unlink(localFilePath, () => {
-                reject(error);
-            });
-            //throw error;
-        }
-    });
-};
-
-// convert srt format to text
-function convertToText(str) {
-    return str
-      .split('\n')
-      .filter(line => !line.match(/^\d+$/) && !line.match(/^\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}$/) && line !== '')
-      .join(' ');
-}
-
-function alignSubtitles(subtitles, format, offsets) {
-    const result = [];
-
-    function preprocessStr(str) {
-        try{
-            if(!str) return '';
-            return str.trim().replace(/(\n\n)(?!\n)/g, '\n\n\n');
-        }catch(e){
-            logger.error(`An error occurred in content text preprocessing: ${e}`);
-            return '';
-        }
-    }
-
-    function shiftSubtitles(subtitle, shiftOffset) {
-        const captions = subsrt.parse(preprocessStr(subtitle));
-        const resynced = subsrt.resync(captions, { offset: shiftOffset });
-        return resynced;
-    }
-
-    for (let i = 0; i < subtitles.length; i++) {
-        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
-    }
-    
-    try {
-        //if content has needed html style tags, keep them
-        for(const obj of result) {
-            if(obj && obj.content){ 
-                obj.text = obj.content;
-            }
-        }
-    } catch (error) {
-        logger.error(`An error occurred in content text parsing: ${error}`);
-    }
-    
-    return subsrt.build(result, { format: format === 'vtt' ? 'vtt' : 'srt' });
-}
-
-
 class OpenAIWhisperPlugin extends ModelPlugin {
     constructor(pathway, model) {
         super(pathway, model);
     }
 
-    async getMediaChunks(file, requestId) {
-        try {
-            if (API_URL) {
-                //call helper api and get list of file uris
-                const res = await axios.get(API_URL, { params: { uri: file, requestId } });
-                return res.data;
-            } else {
-                logger.info(`No API_URL set, returning file as chunk`);
-                return [file];
-            }
-        } catch (err) {
-            logger.error(`Error getting media chunks list from api: ${err}`);
-            throw err;
-        }
-    }
-
-    async markCompletedForCleanUp(requestId) {
-        try {
-            if (API_URL) {
-                //call helper api to mark processing as completed
-                const res = await axios.delete(API_URL, { params: { requestId } });
-                logger.info(`Marked request ${requestId} as completed:`, res.data);
-                return res.data;
-            }
-        } catch (err) {
-            logger.error(`Error marking request ${requestId} as completed: ${err}`);
-        }
-    }
-
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, cortexRequest) {
         const { pathwayResolver } = cortexRequest;
@@ -252,7 +105,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
             if(!wordTimestamped && !responseFormat){ 
                 //if no response format, convert to text
-                return convertToText(res);
+                return convertSrtToText(res);
             }
             return res;
         }
@@ -324,7 +177,7 @@ let offsets = [];
 let uris = []
 
 try {
-    const mediaChunks = await this.getMediaChunks(file, requestId);
+    const mediaChunks = await getMediaChunks(file, requestId);
     
     if (!mediaChunks || !mediaChunks.length) {
         throw new Error(`Error in getting chunks from media helper for file ${file}`);
@@ -363,14 +216,14 @@ try {
                     } 
                 }
 
-                await this.markCompletedForCleanUp(requestId);
+                await markCompletedForCleanUp(requestId);
 
                 //check cleanup for whisper temp uploaded files url
                 const regex = /whispertempfiles\/([a-z0-9-]+)/;
                 const match = file.match(regex);
                 if (match && match[1]) {
                     const extractedValue = match[1];
-                    await this.markCompletedForCleanUp(extractedValue);
+                    await markCompletedForCleanUp(extractedValue);
                     logger.info(`Cleaned temp whisper file ${file} with request id ${extractedValue}`);
                 }
 

package/server/plugins/palmChatPlugin.js

@@ -100,7 +100,7 @@ class PalmChatPlugin extends ModelPlugin {
             throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens.  The model will not be called.`);
         }
     
-        // Ensure there are an even number of messages (PaLM requires an even number of messages)
+        // Ensure there are an odd number of messages for turn taking
         if (requestMessages.length % 2 === 0) {
             requestMessages = requestMessages.slice(1);
         }

package/server/resolver.js

@@ -1,6 +1,5 @@
 import { fulfillWithTimeout } from '../lib/promiser.js';
 import { PathwayResolver } from './pathwayResolver.js';
-import logger from '../lib/logger.js';
 
 // This resolver uses standard parameters required by Apollo server:
 // (parent, args, contextValue, info)
@@ -22,16 +21,16 @@ const rootResolver = async (parent, args, contextValue, info) => {
     try {
         result = await fulfillWithTimeout(pathway.resolver(parent, args, contextValue, info), pathway.timeout);
     } catch (error) {
-        logger.error(`Request failed with error: ${error}`);
+        pathwayResolver.logError(error);
         result = error.message || error.toString();
     }
     
-    const { warnings, previousResult, savedContextId, tool } = pathwayResolver;
+    const { warnings, errors, previousResult, savedContextId, tool } = pathwayResolver;
     
     // Add request parameters back as debug
     const debug = pathwayResolver.prompts.map(prompt => prompt.debugInfo || '').join('\n').trim();
     
-    return { debug, result, warnings, previousResult, tool, contextId: savedContextId }
+    return { debug, result, warnings, errors, previousResult, tool, contextId: savedContextId }
 }
 
 // This resolver is used by the root resolver to process the request

package/server/typeDef.js

@@ -50,6 +50,7 @@ const typeDef = (pathway) => {
         result: ${resultStr}
         previousResult: String
         warnings: [String]
+        errors: [String]
         contextId: String
         tool: String
 }`;