@aj-archipelago/cortex 1.1.20 → 1.1.21

package/config.js

@@ -111,6 +111,13 @@ var config = convict({
                     "model": "whisper-1"
                 },
             },
+            "neuralspace": {
+                "type": "NEURALSPACE",
+                "url": "https://voice.neuralspace.ai/api/v2/jobs",
+                "headers": {
+                    "Authorization": "{{NEURALSPACE_API_KEY}}",
+                },
+            },
             "azure-cognitive": {
                 "type": "AZURE-COGNITIVE",
                 "url": "{{{AZURE_COGNITIVE_API_URL}}}",
@@ -222,6 +229,11 @@ var config = convict({
         default: 0,
         env: 'SUBSCRIPTION_KEEP_ALIVE'
     },
+    neuralSpaceApiKey: {
+        format: String,
+        default: null,
+        env: 'NEURALSPACE_API_KEY'
+    },
 });
 
 // Read in environment variables and set up service configuration

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -76,6 +76,33 @@ if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) {
 
 const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
 
+
+async function gcsUrlExists(url, defaultReturn = true) {
+    try {
+        if(!url) {
+            return defaultReturn; // Cannot check return
+        }
+        if (!gcs) {
+            console.warn('GCS environment variables are not set. Unable to check if URL exists in GCS.');
+            return defaultReturn; // Cannot check return 
+        }
+
+        const urlParts = url.replace('gs://', '').split('/');
+        const bucketName = urlParts[0];
+        const fileName = urlParts.slice(1).join('/');
+
+        const bucket = gcs.bucket(bucketName);
+        const file = bucket.file(fileName);
+
+        const [exists] = await file.exists();
+        
+        return exists;
+    } catch (error) {
+        console.error('Error checking if GCS URL exists:', error);
+        return false;
+    }
+}
+
 const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
@@ -101,7 +128,7 @@ const getBlobClient = async () => {
 async function saveFileToBlob(chunkPath, requestId) {
   const { containerClient } = await getBlobClient();
   // Use the filename with a UUID as the blob name
-  const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+  const blobName = `${requestId}/${uuidv4()}_${encodeURIComponent(path.basename(chunkPath))}`;
 
   // Create a read stream for the chunk file
   const fileStream = fs.createReadStream(chunkPath);
@@ -174,7 +201,7 @@ async function uploadBlob(
           const localPath = join(publicFolder, requestId);
           fs.mkdirSync(localPath, { recursive: true });
 
-          const filename = `${uuidv4()}_${info.filename}`;
+          const filename = encodeURIComponent(`${uuidv4()}_${info.filename}`);
           const destinationPath = `${localPath}/${filename}`;
 
           await pipeline(file, fs.createWriteStream(destinationPath));
@@ -188,7 +215,7 @@ async function uploadBlob(
 
           resolve(body); // Resolve the promise
         } else {
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
           const { containerClient } = await getBlobClient();
 
           const contentType = mime.lookup(filename);  // content type based on file extension
@@ -217,7 +244,7 @@ async function uploadBlob(
 
         if (useGoogle && useGoogle !== "false") {
           const { url } = body;
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
           const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(filename);
           const writeStream = gcsFile.createWriteStream();
 
@@ -348,4 +375,4 @@ async function cleanupGCS(urls=null) {
   return cleanedURLs;
 }
 
-export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS };
+export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists };

package/helper-apps/cortex-file-handler/index.js

@@ -1,5 +1,5 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
@@ -122,14 +122,15 @@ async function main(context, req) {
         context.log(`Checking hash: ${hash}`);
         const result = await getFileStoreMap(hash);
 
-        const exists = await urlExists(result?.url);
+        if(result){
+            const exists = await urlExists(result?.url);
+            const gcsExists = await gcsUrlExists(result?.gcs);
 
-        if(!exists){
-            await removeFromFileStoreMap(hash);
-            return;
-        }
+            if(!exists || !gcsExists){
+                await removeFromFileStoreMap(hash);
+                return;
+            }
 
-        if(result){
             context.log(`Hash exists: ${hash}`);
             //update redis timestamp with current time
             await setFileStoreMap(hash, result);

package/lib/pathwayTools.js

@@ -3,7 +3,7 @@ import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
 
 // callPathway - call a pathway from another pathway
-const callPathway = async (pathwayName, inArgs) => {
+const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
 
     // Clone the args object to avoid modifying the original
     const args = JSON.parse(JSON.stringify(inArgs));
@@ -15,6 +15,12 @@ const callPathway = async (pathwayName, inArgs) => {
     const requestState = {};
     const parent = {};
     const data = await pathway.rootResolver(parent, args, { config, pathway, requestState } );
+    
+    // Merge the results into the pathwayResolver if it was provided
+    if (pathwayResolver) {
+        pathwayResolver.mergeResults(data);
+    }
+
     return data?.result;
 };
 

package/lib/requestExecutor.js

@@ -184,6 +184,7 @@ setInterval(() => {
 }, 30000); // Log rates every 30 seconds
 
 const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
+    //logger.warn(`Requesting ${url} with data: ${JSON.stringify(data)}`);
     const callId = endpoint?.monitor?.startCall();
     let response;
     try {
@@ -356,14 +357,13 @@ const executeRequest = async (cortexRequest) => {
         if (cached) {
             logger.info(`<<< [${requestId}] served with cached response.`);
         }
-        if (error && error.length > 0) {
-            const lastError = error[error.length - 1];
-            return { error: lastError.toJSON() ?? lastError ?? error };
+        if (error) {
+            throw { error: error.toJSON() ?? error };
         }
         return { data, duration };
     } catch (error) {
         logger.error(`Error in request: ${error.message || error}`);
-        return { error: error };
+        throw error;
     } 
 }
 

package/lib/util.js

@@ -1,3 +1,22 @@
+import logger from "./logger.js";
+import stream from 'stream';
+import subsrt from 'subsrt';
+import os from 'os';
+import path from 'path';
+import http from 'http';
+import https from 'https';
+import { URL } from 'url';
+import { v4 as uuidv4 } from 'uuid';
+import { promisify } from 'util';
+import { axios } from './requestExecutor.js';
+import { config } from '../config.js';
+import fs from 'fs';
+
+const pipeline = promisify(stream.pipeline);
+const MEDIA_API_URL = config.get('whisperMediaApiUrl');
+
+
+
 function convertToSingleContentChatHistory(chatHistory){
     for(let i=0; i<chatHistory.length; i++){
         //if isarray make it single string
@@ -29,8 +48,151 @@ function chatArgsHasImageUrl(args){
     return chatArgsHasType(args, 'image_url');
 }
 
+
+async function deleteTempPath(path) {
+    try {
+        if (!path) {
+            logger.warn('Temporary path is not defined.');
+            return;
+        }
+        if (!fs.existsSync(path)) {
+            logger.warn(`Temporary path ${path} does not exist.`);
+            return;
+        }
+        const stats = fs.statSync(path);
+        if (stats.isFile()) {
+            fs.unlinkSync(path);
+            logger.info(`Temporary file ${path} deleted successfully.`);
+        } else if (stats.isDirectory()) {
+            fs.rmSync(path, { recursive: true });
+            logger.info(`Temporary folder ${path} and its contents deleted successfully.`);
+        }
+    } catch (err) {
+        logger.error(`Error occurred while deleting the temporary path: ${err}`);
+    }
+}
+
+function generateUniqueFilename(extension) {
+    return `${uuidv4()}.${extension}`;
+}
+
+const downloadFile = async (fileUrl) => {
+    const fileExtension = path.extname(fileUrl).slice(1);
+    const uniqueFilename = generateUniqueFilename(fileExtension);
+    const tempDir = os.tmpdir();
+    const localFilePath = `${tempDir}/${uniqueFilename}`;
+
+    // eslint-disable-next-line no-async-promise-executor
+    return new Promise(async (resolve, reject) => {
+        try {
+            const parsedUrl = new URL(fileUrl);
+            const protocol = parsedUrl.protocol === 'https:' ? https : http;
+
+            const response = await new Promise((resolve, reject) => {
+                protocol.get(parsedUrl, (res) => {
+                    if (res.statusCode === 200) {
+                        resolve(res);
+                    } else {
+                        reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
+                    }
+                }).on('error', reject);
+            });
+
+            await pipeline(response, fs.createWriteStream(localFilePath));
+            logger.info(`Downloaded file to ${localFilePath}`);
+            resolve(localFilePath);
+        } catch (error) {
+            fs.unlink(localFilePath, () => {
+                reject(error);
+            });
+            //throw error;
+        }
+    });
+};
+
+// convert srt format to text
+function convertSrtToText(str) {
+    return str
+      .split('\n')
+      .filter(line => !line.match(/^\d+$/) && !line.match(/^\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}$/) && line !== '')
+      .join(' ');
+}
+
+function alignSubtitles(subtitles, format, offsets) {
+    const result = [];
+
+    function preprocessStr(str) {
+        try{
+            if(!str) return '';
+            return str.trim().replace(/(\n\n)(?!\n)/g, '\n\n\n');
+        }catch(e){
+            logger.error(`An error occurred in content text preprocessing: ${e}`);
+            return '';
+        }
+    }
+
+    function shiftSubtitles(subtitle, shiftOffset) {
+        const captions = subsrt.parse(preprocessStr(subtitle));
+        const resynced = subsrt.resync(captions, { offset: shiftOffset });
+        return resynced;
+    }
+
+    for (let i = 0; i < subtitles.length; i++) {
+        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
+    }
+    
+    try {
+        //if content has needed html style tags, keep them
+        for(const obj of result) {
+            if(obj && obj.content){ 
+                obj.text = obj.content;
+            }
+        }
+    } catch (error) {
+        logger.error(`An error occurred in content text parsing: ${error}`);
+    }
+    
+    return subsrt.build(result, { format: format === 'vtt' ? 'vtt' : 'srt' });
+}
+
+
+async function getMediaChunks(file, requestId) {
+    try {
+        if (MEDIA_API_URL) {
+            //call helper api and get list of file uris
+            const res = await axios.get(MEDIA_API_URL, { params: { uri: file, requestId } });
+            return res.data;
+        } else {
+            logger.info(`No API_URL set, returning file as chunk`);
+            return [file];
+        }
+    } catch (err) {
+        logger.error(`Error getting media chunks list from api: ${err}`);
+        throw err;
+    }
+}
+
+async function markCompletedForCleanUp(requestId) {
+    try {
+        if (MEDIA_API_URL) {
+            //call helper api to mark processing as completed
+            const res = await axios.delete(MEDIA_API_URL, { params: { requestId } });
+            logger.info(`Marked request ${requestId} as completed:`, res.data);
+            return res.data;
+        }
+    } catch (err) {
+        logger.error(`Error marking request ${requestId} as completed: ${err}`);
+    }
+}
+
 export { 
     convertToSingleContentChatHistory,
     chatArgsHasImageUrl, 
-    chatArgsHasType 
+    chatArgsHasType,
+    deleteTempPath,
+    downloadFile,
+    convertSrtToText,
+    alignSubtitles,
+    getMediaChunks,
+    markCompletedForCleanUp
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.20",
+  "version": "1.1.21",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/index.js

@@ -16,6 +16,7 @@ import test_cohere_summarize from './test_cohere_summarize.js';
 import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
+import transcribe_neuralspace from './transcribe_neuralspace.js';
 import translate from './translate.js';
 import embeddings from './embeddings.js';
 import vision from './vision.js';
@@ -42,6 +43,7 @@ export {
     test_langchain,
     test_palm_chat,
     transcribe,
+    transcribe_neuralspace,
     translate,
     vision,
 };

package/pathways/transcribe_neuralspace.js

@@ -0,0 +1,18 @@
+export default {
+    prompt: `{{text}}`,
+    model: `neuralspace`,
+    inputParameters: {
+        file: ``,
+        language: ``,
+        responseFormat: `text`,
+        wordTimestamped: false,
+        highlightWords: false,
+        maxLineWidth: 0,
+        maxLineCount: 0,
+        maxWordsPerLine: 0,
+    },
+    timeout: 3600, // in seconds
+    enableDuplicateRequests: false,
+};
+
+

package/server/modelExecutor.js

@@ -23,6 +23,7 @@ import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
 import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
+import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -57,6 +58,9 @@ class ModelExecutor {
             case 'OPENAI-WHISPER':
                 plugin = new OpenAIWhisperPlugin(pathway, model);
                 break;
+            case 'NEURALSPACE':
+                plugin = new NeuralSpacePlugin(pathway, model);
+                break;
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -25,8 +25,10 @@ class PathwayResolver {
         this.useInputChunking = pathway.useInputChunking;
         this.chunkMaxTokenLength = 0;
         this.warnings = [];
+        this.errors = [];
         this.requestId = uuidv4();
         this.responseParser = new PathwayResponseParser(pathway);
+        this.tool = null;
         this.modelName = [
             pathway.model,
             args?.model,
@@ -178,6 +180,15 @@ class PathwayResolver {
         }
     }
 
+    mergeResults(mergeData) {
+        if (mergeData) {
+            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
+            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
+            this.errors = [...this.errors, ...(mergeData.errors || [])];
+            this.tool = mergeData.tool || this.tool;
+        }
+    }
+
     async resolve(args) {
         // Either we're dealing with an async request, stream, or regular request
         if (args.async || args.stream) {
@@ -244,6 +255,12 @@ class PathwayResolver {
         logger.warn(warning);
     }
 
+    // Add an error and log it
+    logError(error) {
+        this.errors.push(error);
+        logger.error(error);
+    }
+
     // Here we choose how to handle long input - either summarize or chunk
     processInputText(text) {
         let chunkTokenLength = 0;
@@ -416,7 +433,8 @@ class PathwayResolver {
             }
         }
 
-        if (prompt.saveResultTo) {
+        // save the result to the context if requested and no errors
+        if (prompt.saveResultTo && this.errors.length === 0) {
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;

package/server/plugins/claude3VertexPlugin.js

@@ -138,7 +138,8 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
     modifiedMessages = combinedMessages;
 
-    // Claude vertex requires an even number of messages
+    // Claude vertex requires an odd number of messages
+    // for proper conversation turn-taking
     if (modifiedMessages.length % 2 === 0) {
       modifiedMessages = modifiedMessages.slice(1);
     }

package/server/plugins/gemini15ChatPlugin.js

@@ -76,7 +76,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -135,7 +135,13 @@ class Gemini15ChatPlugin extends ModelPlugin {
         if (data && data.contents && Array.isArray(data.contents)) {
             dataToMerge = data.contents;
         } else if (data && data.candidates && Array.isArray(data.candidates)) {
-            return data.candidates[0].content.parts[0].text;
+            const { content, finishReason, safetyRatings } = data.candidates[0];
+            if (finishReason === 'STOP') {
+                return content?.parts?.[0]?.text ?? '';
+            } else {
+                const returnString = `Response was not completed.  Finish reason: ${finishReason}, Safety ratings: ${JSON.stringify(safetyRatings, null, 2)}`;
+                throw new Error(returnString);
+            }
         } else if (Array.isArray(data)) {
             dataToMerge = data;
         } else {
@@ -154,7 +160,6 @@ class Gemini15ChatPlugin extends ModelPlugin {
         cortexRequest.data = { ...(cortexRequest.data || {}), ...requestParameters };
         cortexRequest.params = {}; // query params
         cortexRequest.stream = stream;
-        cortexRequest.stream = stream;
         cortexRequest.urlSuffix = cortexRequest.stream ? ':streamGenerateContent?alt=sse' : ':generateContent';
 
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');

package/server/plugins/gemini15VisionPlugin.js

@@ -1,6 +1,5 @@
 import Gemini15ChatPlugin from './gemini15ChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class Gemini15VisionPlugin extends Gemini15ChatPlugin {
 
@@ -51,8 +50,9 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${inputPart}`);
+                        // this space intentionally left blank
                     }
+
                     return { text: inputPart };
                 };
     
@@ -83,7 +83,7 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }
@@ -96,6 +96,22 @@ class Gemini15VisionPlugin extends Gemini15ChatPlugin {
         };
     }
 
+    async execute(text, parameters, prompt, cortexRequest) {
+        let result = null;
+        try {
+            result = await super.execute(text, parameters, prompt, cortexRequest);
+        } catch (e) {
+            const { data } = e;
+            if (data && data.error) {
+                if (data.error.code === 400 && data.error.message === 'Precondition check failed.') {
+                    throw new Error('One or more of the included files is too large to process. Please try again with a smaller file.');
+                }
+            throw e;
+            }
+        }
+        return result; 
+    }
+
 }
 
 export default Gemini15VisionPlugin;

package/server/plugins/geminiChatPlugin.js

@@ -81,7 +81,7 @@ class GeminiChatPlugin extends ModelPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }

package/server/plugins/geminiVisionPlugin.js

@@ -1,6 +1,5 @@
 import GeminiChatPlugin from './geminiChatPlugin.js';
 import mime from 'mime-types';
-import logger from '../../lib/logger.js';
 
 class GeminiVisionPlugin extends GeminiChatPlugin {
 
@@ -55,7 +54,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
                             }
                         }
                     } catch (e) {
-                        logger.warn(`Unable to parse part - including as string: ${partString}`);
+                        // this space intentionally left blank
                     }
                     return { text: partString };
                 };
@@ -87,7 +86,7 @@ class GeminiVisionPlugin extends GeminiChatPlugin {
             });
         }
     
-        // Gemini requires an even number of messages
+        // Gemini requires an odd number of messages
         if (modifiedMessages.length % 2 === 0) {
             modifiedMessages = modifiedMessages.slice(1);
         }