@aj-archipelago/cortex 1.1.20 → 1.1.22

package/config/default.example.json

@@ -12,6 +12,62 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
+
+        "gemini-pro-chat": {
+            "type": "GEMINI-CHAT",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-pro:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768,
+            "maxReturnTokens": 8192,
+            "supportsStreaming": true
+        },
+        "gemini-pro-vision": {
+            "type": "GEMINI-VISION",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-pro-vision:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "gemini-pro-15-vision": {
+            "type": "GEMINI-VISION",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-1.5-pro-preview-0215:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 1048576,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "claude-3-haiku-vertex": {
+            "type": "CLAUDE-3-VERTEX",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/anthropic/models/claude-3-haiku@20240307",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 200000,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "claude-35-sonnet-vertex": {
+            "type": "CLAUDE-3-VERTEX",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet@20240229",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 200000,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
         "oai-gpturbo": {
             "type": "OPENAI-CHAT",
             "url": "https://api.openai.com/v1/chat/completions",
@@ -38,6 +94,34 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 8192
         },
+        "oai-gpt4-32": {
+            "type": "OPENAI-CHAT",
+            "url": "https://api.openai.com/v1/chat/completions",
+            "headers": {
+                "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                "Content-Type": "application/json"
+            },
+            "params": {
+                "model": "gpt-4-32"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768
+        },
+        "oai-gpt4o": {
+            "type": "OPENAI-VISION",
+            "url": "https://api.openai.com/v1/chat/completions",
+            "headers": {
+                "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                "Content-Type": "application/json"
+            },
+            "params": {
+                "model": "gpt-4o"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 131072,
+            "maxReturnTokens": 4096,
+            "supportsStreaming": true
+        },
         "palm-text": {
             "type": "PALM-COMPLETION",
             "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/text-bison@001:predict",

package/config.js

@@ -111,6 +111,13 @@ var config = convict({
                     "model": "whisper-1"
                 },
             },
+            "neuralspace": {
+                "type": "NEURALSPACE",
+                "url": "https://voice.neuralspace.ai/api/v2/jobs",
+                "headers": {
+                    "Authorization": "{{NEURALSPACE_API_KEY}}",
+                },
+            },
             "azure-cognitive": {
                 "type": "AZURE-COGNITIVE",
                 "url": "{{{AZURE_COGNITIVE_API_URL}}}",
@@ -132,7 +139,7 @@ var config = convict({
                 },
                 "maxTokenLength": 8192,
             },
-            "oai-gpt4-vision": {
+            "oai-gpt4o": {
                 "type": "OPENAI-VISION",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
@@ -140,10 +147,11 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
                 "params": {
-                    "model": "gpt-4-vision-preview"
+                    "model": "gpt-4o"
                 },
-                "requestsPerSecond": 1,
-                "maxTokenLength": 128000,
+                "requestsPerSecond": 50,
+                "maxTokenLength": 131072,
+                "maxReturnTokens": 4096,
                 "supportsStreaming": true
             },
             "azure-bing": {
@@ -222,6 +230,11 @@ var config = convict({
         default: 0,
         env: 'SUBSCRIPTION_KEEP_ALIVE'
     },
+    neuralSpaceApiKey: {
+        format: String,
+        default: null,
+        env: 'NEURALSPACE_API_KEY'
+    },
 });
 
 // Read in environment variables and set up service configuration

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -36,6 +36,15 @@ const VIDEO_EXTENSIONS = [
   ".mkv",
 ];
 
+const AUDIO_EXTENSIONS = [
+  ".mp3",
+  ".wav",
+  ".ogg",
+  ".flac",
+  ".aac",
+  ".aiff",
+];
+
 function isBase64(str) {
   try {
     return btoa(atob(str)) == str;
@@ -76,6 +85,33 @@ if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) {
 
 const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
 
+
+async function gcsUrlExists(url, defaultReturn = true) {
+    try {
+        if(!url) {
+            return defaultReturn; // Cannot check return
+        }
+        if (!gcs) {
+            console.warn('GCS environment variables are not set. Unable to check if URL exists in GCS.');
+            return defaultReturn; // Cannot check return 
+        }
+
+        const urlParts = url.replace('gs://', '').split('/');
+        const bucketName = urlParts[0];
+        const fileName = urlParts.slice(1).join('/');
+
+        const bucket = gcs.bucket(bucketName);
+        const file = bucket.file(fileName);
+
+        const [exists] = await file.exists();
+        
+        return exists;
+    } catch (error) {
+        console.error('Error checking if GCS URL exists:', error);
+        return false;
+    }
+}
+
 const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
@@ -101,7 +137,7 @@ const getBlobClient = async () => {
 async function saveFileToBlob(chunkPath, requestId) {
   const { containerClient } = await getBlobClient();
   // Use the filename with a UUID as the blob name
-  const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+  const blobName = `${requestId}/${uuidv4()}_${encodeURIComponent(path.basename(chunkPath))}`;
 
   // Create a read stream for the chunk file
   const fileStream = fs.createReadStream(chunkPath);
@@ -135,131 +171,139 @@ async function deleteBlob(requestId) {
   return result;
 }
 
-async function uploadBlob(
-  context,
-  req,
-  saveToLocal = false,
-  useGoogle = false
-) {
+async function uploadBlob(context, req, saveToLocal = false, useGoogle = false, filePath=null) {
   return new Promise((resolve, reject) => {
     try {
-      const busboy = Busboy({ headers: req.headers });
       let requestId = uuidv4();
       let body = {};
 
-      busboy.on("field", (fieldname, value) => {
-        if (fieldname === "requestId") {
-          requestId = value;
-        } else if (fieldname === "useGoogle") {
-          useGoogle = value;
-        }
-      });
-
-      busboy.on("file", async (fieldname, file, info) => {
-        //do not use google if file is not image or video
-        const ext = path.extname(info.filename).toLowerCase();
-        const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext);
-        if(!canUseGoogle) {
-          useGoogle = false;
-        }
+      // If filePath is given, we are dealing with local file and not form-data
+      if (filePath) {
+        const file = fs.createReadStream(filePath);
+        const filename = path.basename(filePath);
+        uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve)
+      } else {
+        // Otherwise, continue working with form-data
+        const busboy = Busboy({ headers: req.headers });
+      
+        busboy.on("field", (fieldname, value) => {
+          if (fieldname === "requestId") {
+            requestId = value;
+          } else if (fieldname === "useGoogle") {
+            useGoogle = value;
+          }
+        });
+
+        busboy.on("file", async (fieldname, file, filename) => {
+          uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename?.filename || filename, resolve)
+        });
+
+        busboy.on("error", (error) => {
+          context.log.error("Error processing file upload:", error);
+          context.res = {
+            status: 500,
+            body: "Error processing file upload.",
+          };
+          reject(error); // Reject the promise
+        });
+
+        req.pipe(busboy);
+      }
+    } catch (error) {
+      context.log.error("Error processing file upload:", error);
+      context.res = {
+        status: 500,
+        body: "Error processing file upload.",
+      };
+      reject(error); // Reject the promise
+    }
+  });
+}
 
-        //check if useGoogle is set but no gcs and warn
-        if(useGoogle && useGoogle !== "false" && !gcs) {
-          context.log.warn("Google Cloud Storage is not initialized reverting google upload ");
-          useGoogle = false;
-        }
+async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve) {
+  // do not use Google if the file is not an image or video
+  const ext = path.extname(filename).toLowerCase();
+  const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext) || AUDIO_EXTENSIONS.includes(ext);
+  if (!canUseGoogle) {
+    useGoogle = false;
+  }
 
-        if (saveToLocal) {
-          // Create the target folder if it doesn't exist
-          const localPath = join(publicFolder, requestId);
-          fs.mkdirSync(localPath, { recursive: true });
+  // check if useGoogle is set but no gcs and warn
+  if (useGoogle && useGoogle !== "false" && !gcs) {
+    context.log.warn("Google Cloud Storage is not initialized reverting google upload ");
+    useGoogle = false;
+  }
 
-          const filename = `${uuidv4()}_${info.filename}`;
-          const destinationPath = `${localPath}/${filename}`;
+  const encodedFilename = encodeURIComponent(`${requestId || uuidv4()}_${filename}`);
 
-          await pipeline(file, fs.createWriteStream(destinationPath));
 
-          const message = `File '${filename}' saved to folder successfully.`;
-          context.log(message);
+  if (saveToLocal) {
+    // create the target folder if it doesn't exist
+    const localPath = join(publicFolder, requestId);
+    fs.mkdirSync(localPath, { recursive: true });
 
-          const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
+    const destinationPath = `${localPath}/${encodedFilename}`;
 
-          body = { message, url };
+    await pipeline(file, fs.createWriteStream(destinationPath));
 
-          resolve(body); // Resolve the promise
-        } else {
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
-          const { containerClient } = await getBlobClient();
+    const message = `File '${encodedFilename}' saved to folder successfully.`;
+    context.log(message);
 
-          const contentType = mime.lookup(filename);  // content type based on file extension
-          const options = {};
-          if (contentType) {
-            options.blobHTTPHeaders = { blobContentType: contentType };
-          }
+    const url = `http://${ipAddress}:${port}/files/${requestId}/${encodedFilename}`;
 
-          const blockBlobClient = containerClient.getBlockBlobClient(filename);
+    body = { message, url };
 
-          const passThroughStream = new PassThrough();
-          file.pipe(passThroughStream);
+    resolve(body); // Resolve the promise
+  } else {
+    const { containerClient } = await getBlobClient();
 
-          await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
+    const contentType = mime.lookup(encodedFilename);  // content type based on file extension
+    const options = {};
+    if (contentType) {
+      options.blobHTTPHeaders = { blobContentType: contentType };
+    }
 
-          const message = `File '${filename}' uploaded successfully.`;
-          const url = blockBlobClient.url;
-          context.log(message);
-          body = { message, url };
-        }
+    const blockBlobClient = containerClient.getBlockBlobClient(encodedFilename);
 
-        context.res = {
-          status: 200,
-          body,
-        };
+    const passThroughStream = new PassThrough();
+    file.pipe(passThroughStream);
 
-        if (useGoogle && useGoogle !== "false") {
-          const { url } = body;
-          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
-          const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(filename);
-          const writeStream = gcsFile.createWriteStream();
+    await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
 
-          const response = await axios({
-            method: "get",
-            url: url,
-            responseType: "stream",
-          });
+    const message = `File '${encodedFilename}' uploaded successfully.`;
+    const url = blockBlobClient.url;
+    context.log(message);
+    body = { message, url };
+  }
 
-          // Pipe the Axios response stream directly into the GCS Write Stream
-          response.data.pipe(writeStream);
+  context.res = {
+    status: 200,
+    body,
+  };
 
-          await new Promise((resolve, reject) => {
-            writeStream.on("finish", resolve);
-            writeStream.on("error", reject);
-          });
+  if (useGoogle && useGoogle !== "false") {
+    const { url } = body;
+    const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(encodedFilename);
+    const writeStream = gcsFile.createWriteStream();
 
-          body.gcs = `gs://${GCS_BUCKETNAME}/${filename}`;
-        }
+    const response = await axios({
+      method: "get",
+      url: url,
+      responseType: "stream",
+    });
 
-        resolve(body); // Resolve the promise
-      });
+    // pipe the Axios response stream directly into the GCS Write Stream
+    response.data.pipe(writeStream);
 
-      busboy.on("error", (error) => {
-        context.log.error("Error processing file upload:", error);
-        context.res = {
-          status: 500,
-          body: "Error processing file upload.",
-        };
-        reject(error); // Reject the promise
-      });
+    await new Promise((resolve, reject) => {
+      writeStream.on("finish", resolve);
+      writeStream.on("error", reject);
+    });
 
-      req.pipe(busboy);
-    } catch (error) {
-      context.log.error("Error processing file upload:", error);
-      context.res = {
-        status: 500,
-        body: "Error processing file upload.",
-      };
-      reject(error); // Reject the promise
-    }
-  });
+    body.gcs = `gs://${GCS_BUCKETNAME}/${encodedFilename}`;
+  }
+  
+  resolve(body); // Resolve the promise
 }
 
 // Function to delete files that haven't been used in more than a month
@@ -348,4 +392,4 @@ async function cleanupGCS(urls=null) {
   return cleanedURLs;
 }
 
-export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS };
+export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists };

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -112,16 +112,20 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
     }
 }
 
-const ytdlDownload = async (url, filename) => {
+const ytdlDownload = async (url, filename, video = false) => {
     return new Promise((resolve, reject) => {
-        const video = ytdl(url, { quality: 'highestaudio' });
+        const videoOptions = video 
+            ? { filter: 'audioandvideo' }  // audio and video
+            : { quality: 'highestaudio' }; // audio only
+
+        const videoStream = ytdl(url, videoOptions);
         let lastLoggedTime = Date.now();
 
-        video.on('error', (error) => {
+        videoStream.on('error', (error) => {
             reject(error);
         });
 
-        video.on('progress', (chunkLength, downloaded, total) => {
+        videoStream.on('progress', (chunkLength, downloaded, total) => {
             const currentTime = Date.now();
             if (currentTime - lastLoggedTime >= 2000) { // Log every 2 seconds
                 const percent = downloaded / total;
@@ -130,7 +134,7 @@ const ytdlDownload = async (url, filename) => {
             }
         });
 
-        video.pipe(fs.createWriteStream(filename))
+        videoStream.pipe(fs.createWriteStream(filename))
             .on('finish', () => {
                 resolve();
             })
@@ -140,10 +144,11 @@ const ytdlDownload = async (url, filename) => {
     });
 };
 
-const processYoutubeUrl = async (url) => {
+async function processYoutubeUrl(url, video=false) {
     try {
-        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
-        await ytdlDownload(url, outputFileName);
+        const outputFormat = video ? '.mp4' : '.mp3';
+        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}${outputFormat}`);
+        await ytdlDownload(url, outputFileName, video);
         return outputFileName;
     } catch (e) {
         console.log(e);

package/helper-apps/cortex-file-handler/index.js

@@ -1,5 +1,5 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS, gcsUrlExists } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
@@ -10,6 +10,10 @@ import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
 import http from 'http';
 import https from 'https';
+import axios from "axios";
+import { pipeline } from "stream";
+import { promisify } from "util";
+const pipelineUtility = promisify(pipeline); // To pipe streams using async/await
 
 const DOC_EXTENSIONS =  [".txt", ".json", ".csv", ".md", ".xml", ".js", ".html", ".css", '.pdf', '.docx', '.xlsx', '.csv'];
 
@@ -116,20 +120,62 @@ async function main(context, req) {
         return;
     }
 
-    const { uri, requestId, save, hash, checkHash } = req.body?.params || req.query;
+    const { uri, requestId, save, hash, checkHash, fetch, load, restore } = req.body?.params || req.query;
+
+    const filepond = fetch || restore || load;
+    if (req.method.toLowerCase() === `get` && filepond) {
+        context.log(`Remote file: ${filepond}`);
+        // Check if file already exists (using hash as the key)
+        const exists = await getFileStoreMap(filepond);
+        if(exists){
+            context.res = {
+                status: 200,
+                body: exists // existing file URL
+            };
+            return;
+        }
+
+        // Check if it's a youtube url
+        let youtubeDownloadedFile = null; 
+        if(isValidYoutubeUrl(filepond)){
+            youtubeDownloadedFile = await processYoutubeUrl(filepond, true);
+        }
+        const filename = path.join(os.tmpdir(), path.basename(youtubeDownloadedFile || filepond));
+        // Download the remote file to a local/temporary location keep name & ext
+        if(!youtubeDownloadedFile){
+            const response = await axios.get(filepond, { responseType: "stream" });
+            await pipelineUtility(response.data, fs.createWriteStream(filename));
+        }
+
+        
+        const res = await uploadBlob(context, null, !useAzure, true, filename); 
+        context.log(`File uploaded: ${JSON.stringify(res)}`);
+
+        //Update Redis (using hash as the key)
+        await setFileStoreMap(filepond, res);
+
+        // Return the file URL
+        context.res = {
+            status: 200,
+            body: res,
+        };
+
+        return;
+    }
 
     if(hash && checkHash){ //check if hash exists
         context.log(`Checking hash: ${hash}`);
         const result = await getFileStoreMap(hash);
 
-        const exists = await urlExists(result?.url);
+        if(result){
+            const exists = await urlExists(result?.url);
+            const gcsExists = await gcsUrlExists(result?.gcs);
 
-        if(!exists){
-            await removeFromFileStoreMap(hash);
-            return;
-        }
+            if(!exists || !gcsExists){
+                await removeFromFileStoreMap(hash);
+                return;
+            }
 
-        if(result){
             context.log(`Hash exists: ${hash}`);
             //update redis timestamp with current time
             await setFileStoreMap(hash, result);
@@ -228,7 +274,8 @@ async function main(context, req) {
 
             if (isYoutubeUrl) {
                 // totalCount += 1; // extra 1 step for youtube download
-                file = await processYoutubeUrl(file);
+                const processAsVideo = req.body?.params?.processAsVideo || req.query?.processAsVideo;
+                file = await processYoutubeUrl(file, processAsVideo);
             }
 
             const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);

package/lib/pathwayTools.js

@@ -3,7 +3,7 @@ import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
 
 // callPathway - call a pathway from another pathway
-const callPathway = async (pathwayName, inArgs) => {
+const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
 
     // Clone the args object to avoid modifying the original
     const args = JSON.parse(JSON.stringify(inArgs));
@@ -15,6 +15,12 @@ const callPathway = async (pathwayName, inArgs) => {
     const requestState = {};
     const parent = {};
     const data = await pathway.rootResolver(parent, args, { config, pathway, requestState } );
+    
+    // Merge the results into the pathwayResolver if it was provided
+    if (pathwayResolver) {
+        pathwayResolver.mergeResults(data);
+    }
+
     return data?.result;
 };
 

package/lib/requestExecutor.js

@@ -184,6 +184,7 @@ setInterval(() => {
 }, 30000); // Log rates every 30 seconds
 
 const requestWithMonitor = async (endpoint, url, data, axiosConfigObj) => {
+    //logger.warn(`Requesting ${url} with data: ${JSON.stringify(data)}`);
     const callId = endpoint?.monitor?.startCall();
     let response;
     try {
@@ -356,14 +357,13 @@ const executeRequest = async (cortexRequest) => {
         if (cached) {
             logger.info(`<<< [${requestId}] served with cached response.`);
         }
-        if (error && error.length > 0) {
-            const lastError = error[error.length - 1];
-            return { error: lastError.toJSON() ?? lastError ?? error };
+        if (error) {
+            throw { error: error.toJSON() ?? error };
         }
         return { data, duration };
     } catch (error) {
         logger.error(`Error in request: ${error.message || error}`);
-        return { error: error };
+        throw error;
     } 
 }