@aj-archipelago/cortex 1.1.19 → 1.1.21

package/config/default.example.json

@@ -1,5 +1,5 @@
 {
-    "defaultModelName": "oai-td3",
+    "defaultModelName": "oai-gpturbo",
     "models": {
         "azure-translate": {
             "type": "AZURE-TRANSLATE",
@@ -12,19 +12,6 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
-        "oai-td3": {
-            "type": "OPENAI-COMPLETION",
-            "url": "https://api.openai.com/v1/completions",
-            "headers": {
-                "Authorization": "Bearer {{OPENAI_API_KEY}}",
-                "Content-Type": "application/json"
-            },
-            "params": {
-                "model": "text-davinci-003"
-            },
-            "requestsPerSecond": 10,
-            "maxTokenLength": 4096
-        },
         "oai-gpturbo": {
             "type": "OPENAI-CHAT",
             "url": "https://api.openai.com/v1/chat/completions",

package/config.js

@@ -111,6 +111,13 @@ var config = convict({
                     "model": "whisper-1"
                 },
             },
+            "neuralspace": {
+                "type": "NEURALSPACE",
+                "url": "https://voice.neuralspace.ai/api/v2/jobs",
+                "headers": {
+                    "Authorization": "{{NEURALSPACE_API_KEY}}",
+                },
+            },
             "azure-cognitive": {
                 "type": "AZURE-COGNITIVE",
                 "url": "{{{AZURE_COGNITIVE_API_URL}}}",
@@ -222,6 +229,11 @@ var config = convict({
         default: 0,
         env: 'SUBSCRIPTION_KEEP_ALIVE'
     },
+    neuralSpaceApiKey: {
+        format: String,
+        default: null,
+        env: 'NEURALSPACE_API_KEY'
+    },
 });
 
 // Read in environment variables and set up service configuration

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -11,6 +11,7 @@ import { join } from "path";
 import { Storage } from "@google-cloud/storage";
 import axios from "axios";
 import { publicFolder, port, ipAddress } from "./start.js";
+import mime from "mime-types";
 
 const IMAGE_EXTENSIONS = [
   ".jpg",
@@ -21,6 +22,7 @@ const IMAGE_EXTENSIONS = [
   ".webp",
   ".tiff",
   ".svg",
+  ".pdf"
 ];
 
 const VIDEO_EXTENSIONS = [
@@ -74,7 +76,34 @@ if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) {
 
 const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
 
-const getBlobClient = () => {
+
+async function gcsUrlExists(url, defaultReturn = true) {
+    try {
+        if(!url) {
+            return defaultReturn; // Cannot check return
+        }
+        if (!gcs) {
+            console.warn('GCS environment variables are not set. Unable to check if URL exists in GCS.');
+            return defaultReturn; // Cannot check return 
+        }
+
+        const urlParts = url.replace('gs://', '').split('/');
+        const bucketName = urlParts[0];
+        const fileName = urlParts.slice(1).join('/');
+
+        const bucket = gcs.bucket(bucketName);
+        const file = bucket.file(fileName);
+
+        const [exists] = await file.exists();
+        
+        return exists;
+    } catch (error) {
+        console.error('Error checking if GCS URL exists:', error);
+        return false;
+    }
+}
+
+const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
   if (!connectionString || !containerName) {
@@ -83,17 +112,23 @@ const getBlobClient = () => {
     );
   }
 
-  const blobServiceClient =
-    BlobServiceClient.fromConnectionString(connectionString);
+  const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+
+  const serviceProperties = await blobServiceClient.getProperties();
+  if(!serviceProperties.defaultServiceVersion) {
+    serviceProperties.defaultServiceVersion = '2020-02-10';
+    await blobServiceClient.setProperties(serviceProperties);
+  }
+
   const containerClient = blobServiceClient.getContainerClient(containerName);
 
   return { blobServiceClient, containerClient };
 };
 
 async function saveFileToBlob(chunkPath, requestId) {
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
   // Use the filename with a UUID as the blob name
-  const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+  const blobName = `${requestId}/${uuidv4()}_${encodeURIComponent(path.basename(chunkPath))}`;
 
   // Create a read stream for the chunk file
   const fileStream = fs.createReadStream(chunkPath);
@@ -110,7 +145,7 @@ async function saveFileToBlob(chunkPath, requestId) {
 //deletes blob that has the requestId
 async function deleteBlob(requestId) {
   if (!requestId) throw new Error("Missing requestId parameter");
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
   // List the blobs in the container with the specified prefix
   const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
 
@@ -166,7 +201,7 @@ async function uploadBlob(
           const localPath = join(publicFolder, requestId);
           fs.mkdirSync(localPath, { recursive: true });
 
-          const filename = `${uuidv4()}_${info.filename}`;
+          const filename = encodeURIComponent(`${uuidv4()}_${info.filename}`);
           const destinationPath = `${localPath}/${filename}`;
 
           await pipeline(file, fs.createWriteStream(destinationPath));
@@ -180,15 +215,21 @@ async function uploadBlob(
 
           resolve(body); // Resolve the promise
         } else {
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
-          const { containerClient } = getBlobClient();
+          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
+          const { containerClient } = await getBlobClient();
+
+          const contentType = mime.lookup(filename);  // content type based on file extension
+          const options = {};
+          if (contentType) {
+            options.blobHTTPHeaders = { blobContentType: contentType };
+          }
 
           const blockBlobClient = containerClient.getBlockBlobClient(filename);
 
           const passThroughStream = new PassThrough();
           file.pipe(passThroughStream);
 
-          await blockBlobClient.uploadStream(passThroughStream);
+          await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
 
           const message = `File '${filename}' uploaded successfully.`;
           const url = blockBlobClient.url;
@@ -203,7 +244,7 @@ async function uploadBlob(
 
         if (useGoogle && useGoogle !== "false") {
           const { url } = body;
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
           const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(filename);
           const writeStream = gcsFile.createWriteStream();
 
@@ -250,7 +291,7 @@ async function uploadBlob(
 
 // Function to delete files that haven't been used in more than a month
 async function cleanup(urls=null) {
-  const { containerClient } = getBlobClient();
+  const { containerClient } = await getBlobClient();
 
   if(!urls) {
     const xMonthAgo = new Date();
@@ -334,4 +375,4 @@ async function cleanupGCS(urls=null) {
   return cleanedURLs;
 }
 
-export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS };
+export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists };

package/helper-apps/cortex-file-handler/index.js

@@ -1,5 +1,5 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
@@ -122,14 +122,15 @@ async function main(context, req) {
         context.log(`Checking hash: ${hash}`);
         const result = await getFileStoreMap(hash);
 
-        const exists = await urlExists(result?.url);
+        if(result){
+            const exists = await urlExists(result?.url);
+            const gcsExists = await gcsUrlExists(result?.gcs);
 
-        if(!exists){
-            await removeFromFileStoreMap(hash);
-            return;
-        }
+            if(!exists || !gcsExists){
+                await removeFromFileStoreMap(hash);
+                return;
+            }
 
-        if(result){
             context.log(`Hash exists: ${hash}`);
             //update redis timestamp with current time
             await setFileStoreMap(hash, result);

package/helper-apps/cortex-file-handler/package.json

@@ -5,6 +5,7 @@
   "type": "module",
   "scripts": {
     "start": "node start.js",
+    "dev": "node -r dotenv/config start.js",
     "test": "echo \"No tests yet...\""
   },
   "dependencies": {

package/lib/pathwayTools.js

@@ -3,7 +3,11 @@ import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
 
 // callPathway - call a pathway from another pathway
-const callPathway = async (pathwayName, args) => {
+const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
+
+    // Clone the args object to avoid modifying the original
+    const args = JSON.parse(JSON.stringify(inArgs));
+    
     const pathway = config.get(`pathways.${pathwayName}`);
     if (!pathway) {
         throw new Error(`Pathway ${pathwayName} not found`);
@@ -11,6 +15,12 @@ const callPathway = async (pathwayName, args) => {
     const requestState = {};
     const parent = {};
     const data = await pathway.rootResolver(parent, args, { config, pathway, requestState } );
+    
+    // Merge the results into the pathwayResolver if it was provided
+    if (pathwayResolver) {
+        pathwayResolver.mergeResults(data);
+    }
+
     return data?.result;
 };
 

package/lib/requestExecutor.js

@@ -184,6 +184,7 @@ setInterval(() => {
 }, 30000); // Log rates every 30 seconds
 
 const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
+    //logger.warn(`Requesting ${url} with data: ${JSON.stringify(data)}`);
     const callId = endpoint?.monitor?.startCall();
     let response;
     try {
@@ -356,14 +357,13 @@ const executeRequest = async (cortexRequest) => {
         if (cached) {
             logger.info(`<<< [${requestId}] served with cached response.`);
         }
-        if (error && error.length > 0) {
-            const lastError = error[error.length - 1];
-            return { error: lastError.toJSON() ?? lastError ?? error };
+        if (error) {
+            throw { error: error.toJSON() ?? error };
         }
         return { data, duration };
     } catch (error) {
         logger.error(`Error in request: ${error.message || error}`);
-        return { error: error };
+        throw error;
     } 
 }
 

package/lib/util.js

@@ -1,3 +1,22 @@
+import logger from "./logger.js";
+import stream from 'stream';
+import subsrt from 'subsrt';
+import os from 'os';
+import path from 'path';
+import http from 'http';
+import https from 'https';
+import { URL } from 'url';
+import { v4 as uuidv4 } from 'uuid';
+import { promisify } from 'util';
+import { axios } from './requestExecutor.js';
+import { config } from '../config.js';
+import fs from 'fs';
+
+const pipeline = promisify(stream.pipeline);
+const MEDIA_API_URL = config.get('whisperMediaApiUrl');
+
+
+
 function convertToSingleContentChatHistory(chatHistory){
     for(let i=0; i<chatHistory.length; i++){
         //if isarray make it single string
@@ -29,8 +48,151 @@ function chatArgsHasImageUrl(args){
     return chatArgsHasType(args, 'image_url');
 }
 
+
+async function deleteTempPath(path) {
+    try {
+        if (!path) {
+            logger.warn('Temporary path is not defined.');
+            return;
+        }
+        if (!fs.existsSync(path)) {
+            logger.warn(`Temporary path ${path} does not exist.`);
+            return;
+        }
+        const stats = fs.statSync(path);
+        if (stats.isFile()) {
+            fs.unlinkSync(path);
+            logger.info(`Temporary file ${path} deleted successfully.`);
+        } else if (stats.isDirectory()) {
+            fs.rmSync(path, { recursive: true });
+            logger.info(`Temporary folder ${path} and its contents deleted successfully.`);
+        }
+    } catch (err) {
+        logger.error(`Error occurred while deleting the temporary path: ${err}`);
+    }
+}
+
+function generateUniqueFilename(extension) {
+    return `${uuidv4()}.${extension}`;
+}
+
+const downloadFile = async (fileUrl) => {
+    const fileExtension = path.extname(fileUrl).slice(1);
+    const uniqueFilename = generateUniqueFilename(fileExtension);
+    const tempDir = os.tmpdir();
+    const localFilePath = `${tempDir}/${uniqueFilename}`;
+
+    // eslint-disable-next-line no-async-promise-executor
+    return new Promise(async (resolve, reject) => {
+        try {
+            const parsedUrl = new URL(fileUrl);
+            const protocol = parsedUrl.protocol === 'https:' ? https : http;
+
+            const response = await new Promise((resolve, reject) => {
+                protocol.get(parsedUrl, (res) => {
+                    if (res.statusCode === 200) {
+                        resolve(res);
+                    } else {
+                        reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
+                    }
+                }).on('error', reject);
+            });
+
+            await pipeline(response, fs.createWriteStream(localFilePath));
+            logger.info(`Downloaded file to ${localFilePath}`);
+            resolve(localFilePath);
+        } catch (error) {
+            fs.unlink(localFilePath, () => {
+                reject(error);
+            });
+            //throw error;
+        }
+    });
+};
+
+// convert srt format to text
+function convertSrtToText(str) {
+    return str
+      .split('\n')
+      .filter(line => !line.match(/^\d+$/) && !line.match(/^\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}$/) && line !== '')
+      .join(' ');
+}
+
+function alignSubtitles(subtitles, format, offsets) {
+    const result = [];
+
+    function preprocessStr(str) {
+        try{
+            if(!str) return '';
+            return str.trim().replace(/(\n\n)(?!\n)/g, '\n\n\n');
+        }catch(e){
+            logger.error(`An error occurred in content text preprocessing: ${e}`);
+            return '';
+        }
+    }
+
+    function shiftSubtitles(subtitle, shiftOffset) {
+        const captions = subsrt.parse(preprocessStr(subtitle));
+        const resynced = subsrt.resync(captions, { offset: shiftOffset });
+        return resynced;
+    }
+
+    for (let i = 0; i < subtitles.length; i++) {
+        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
+    }
+    
+    try {
+        //if content has needed html style tags, keep them
+        for(const obj of result) {
+            if(obj && obj.content){ 
+                obj.text = obj.content;
+            }
+        }
+    } catch (error) {
+        logger.error(`An error occurred in content text parsing: ${error}`);
+    }
+    
+    return subsrt.build(result, { format: format === 'vtt' ? 'vtt' : 'srt' });
+}
+
+
+async function getMediaChunks(file, requestId) {
+    try {
+        if (MEDIA_API_URL) {
+            //call helper api and get list of file uris
+            const res = await axios.get(MEDIA_API_URL, { params: { uri: file, requestId } });
+            return res.data;
+        } else {
+            logger.info(`No API_URL set, returning file as chunk`);
+            return [file];
+        }
+    } catch (err) {
+        logger.error(`Error getting media chunks list from api: ${err}`);
+        throw err;
+    }
+}
+
+async function markCompletedForCleanUp(requestId) {
+    try {
+        if (MEDIA_API_URL) {
+            //call helper api to mark processing as completed
+            const res = await axios.delete(MEDIA_API_URL, { params: { requestId } });
+            logger.info(`Marked request ${requestId} as completed:`, res.data);
+            return res.data;
+        }
+    } catch (err) {
+        logger.error(`Error marking request ${requestId} as completed: ${err}`);
+    }
+}
+
 export { 
     convertToSingleContentChatHistory,
     chatArgsHasImageUrl, 
-    chatArgsHasType 
+    chatArgsHasType,
+    deleteTempPath,
+    downloadFile,
+    convertSrtToText,
+    alignSubtitles,
+    getMediaChunks,
+    markCompletedForCleanUp
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.19",
+  "version": "1.1.21",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/index.js

@@ -16,6 +16,7 @@ import test_cohere_summarize from './test_cohere_summarize.js';
 import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
+import transcribe_neuralspace from './transcribe_neuralspace.js';
 import translate from './translate.js';
 import embeddings from './embeddings.js';
 import vision from './vision.js';
@@ -42,6 +43,7 @@ export {
     test_langchain,
     test_palm_chat,
     transcribe,
+    transcribe_neuralspace,
     translate,
     vision,
 };

package/pathways/transcribe_neuralspace.js

@@ -0,0 +1,18 @@
+export default {
+    prompt: `{{text}}`,
+    model: `neuralspace`,
+    inputParameters: {
+        file: ``,
+        language: ``,
+        responseFormat: `text`,
+        wordTimestamped: false,
+        highlightWords: false,
+        maxLineWidth: 0,
+        maxLineCount: 0,
+        maxWordsPerLine: 0,
+    },
+    timeout: 3600, // in seconds
+    enableDuplicateRequests: false,
+};
+
+

package/server/modelExecutor.js

@@ -23,6 +23,7 @@ import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
 import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
+import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -57,6 +58,9 @@ class ModelExecutor {
             case 'OPENAI-WHISPER':
                 plugin = new OpenAIWhisperPlugin(pathway, model);
                 break;
+            case 'NEURALSPACE':
+                plugin = new NeuralSpacePlugin(pathway, model);
+                break;
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -25,8 +25,10 @@ class PathwayResolver {
         this.useInputChunking = pathway.useInputChunking;
         this.chunkMaxTokenLength = 0;
         this.warnings = [];
+        this.errors = [];
         this.requestId = uuidv4();
         this.responseParser = new PathwayResponseParser(pathway);
+        this.tool = null;
         this.modelName = [
             pathway.model,
             args?.model,
@@ -178,6 +180,15 @@ class PathwayResolver {
         }
     }
 
+    mergeResults(mergeData) {
+        if (mergeData) {
+            this.previousResult = mergeData.previousResult ? mergeData.previousResult : this.previousResult;
+            this.warnings = [...this.warnings, ...(mergeData.warnings || [])];
+            this.errors = [...this.errors, ...(mergeData.errors || [])];
+            this.tool = mergeData.tool || this.tool;
+        }
+    }
+
     async resolve(args) {
         // Either we're dealing with an async request, stream, or regular request
         if (args.async || args.stream) {
@@ -244,6 +255,12 @@ class PathwayResolver {
         logger.warn(warning);
     }
 
+    // Add an error and log it
+    logError(error) {
+        this.errors.push(error);
+        logger.error(error);
+    }
+
     // Here we choose how to handle long input - either summarize or chunk
     processInputText(text) {
         let chunkTokenLength = 0;
@@ -416,7 +433,8 @@ class PathwayResolver {
             }
         }
 
-        if (prompt.saveResultTo) {
+        // save the result to the context if requested and no errors
+        if (prompt.saveResultTo && this.errors.length === 0) {
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;