@aj-archipelago/cortex 1.0.24 → 1.0.25

package/config.js

@@ -9,6 +9,11 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // Schema for config
 var config = convict({
+    cortexId: {
+        format: String,
+        default: 'local',
+        env: 'CORTEX_ID'
+    },
     basePathwayPath: {
         format: String,
         default: path.join(__dirname, 'pathways', 'basePathway.js'),
@@ -109,6 +114,20 @@ var config = convict({
                 },
                 "maxTokenLength": 8192,
             },
+            "oai-gpt4-vision": {
+                "type": "OPENAI-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-4-vision-preview"
+                },
+                "requestsPerSecond": 1,
+                "maxTokenLength": 128000,
+                "supportsStreaming": true
+            },
         },
         env: 'CORTEX_MODELS'
     },

package/helper_apps/{MediaFileChunker → CortexFileHandler}/blobHandler.js

@@ -145,6 +145,32 @@ async function uploadBlob(context, req, saveToLocal = false) {
     });
 }
 
+// Function to delete files that haven't been used in more than a month
+async function cleanup() {
+    const { containerClient } = getBlobClient();
+  
+    // List all the blobs in the container
+    const blobs = containerClient.listBlobsFlat();
+  
+    // Calculate the date that is x month ago
+    const xMonthAgo = new Date();
+    xMonthAgo.setMonth(xMonthAgo.getMonth() - 1);
+  
+    // Iterate through the blobs
+    for await (const blob of blobs) {
+      // Get the last modified date of the blob
+      const lastModified = blob.properties.lastModified;
+  
+      // Compare the last modified date with one month ago
+      if (lastModified < xMonthAgo) {
+        // Delete the blob
+        const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
+        await blockBlobClient.delete();
+        console.log(`Cleaned blob: ${blob.name}`);
+      }
+    }
+}
+
 export {
-    saveFileToBlob, deleteBlob, uploadBlob
+    saveFileToBlob, deleteBlob, uploadBlob, cleanup
 }

package/helper_apps/{MediaFileChunker → CortexFileHandler}/fileChunker.js

@@ -3,7 +3,6 @@ import path from 'path';
 import ffmpeg from 'fluent-ffmpeg';
 import { v4 as uuidv4 } from 'uuid';
 import os from 'os';
-import ytdl from 'ytdl-core';
 import { promisify } from 'util';
 import axios from 'axios';
 import { ensureEncoded } from './helper.js';

package/helper_apps/{MediaFileChunker → CortexFileHandler}/index.js

@@ -1,8 +1,8 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup } from './blobHandler.js';
 import { publishRequestProgress } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
-import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
+import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
 import { documentToText, easyChunker } from './docHelper.js';
 import path from 'path';
 import os from 'os';
@@ -15,9 +15,29 @@ const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
 console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
 
 
+let isCleanupRunning = false;
+async function cleanupInactive(useAzure) {
+    try {
+        if (isCleanupRunning) { return; } //no need to cleanup every call
+        isCleanupRunning = true;
+        if (useAzure) {
+            await cleanup();
+        } else {
+            await cleanupLocal();
+        }
+    } catch (error) {
+        console.log('Error occurred during cleanup:', error);
+    } finally{
+        isCleanupRunning = false;
+    }
+}
+
+  
 async function main(context, req) {
     context.log('Starting req processing..');
 
+    cleanupInactive(useAzure); //trigger & no need to wait for it
+
     // Clean up blob when request delete which means processing marked completed
     if (req.method.toLowerCase() === `delete`) {
         const { requestId } = req.query;

package/helper_apps/{MediaFileChunker → CortexFileHandler}/localFileHandler.js

@@ -4,7 +4,6 @@ import { v4 as uuidv4 } from 'uuid';
 
 import { publicFolder, port, ipAddress } from "./start.js";
 
-
 async function moveFileToPublicFolder(chunkPath, requestId) {
     // Use the filename with a UUID as the blob name
     const filename = `${requestId}/${uuidv4()}_${basename(chunkPath)}`;
@@ -30,7 +29,44 @@ async function deleteFolder(requestId) {
     console.log(`Cleaned folder: ${targetFolder}`);
 }
 
+async function cleanupLocal() {
+  try {
+    // Read the directory
+    const items = await fs.readdir(publicFolder);
+
+    // Calculate the date that is x months ago
+    const monthsAgo = new Date();
+    monthsAgo.setMonth(monthsAgo.getMonth() - 1);
+
+    // Iterate through the items
+    for (const item of items) {
+      const itemPath = join(publicFolder, item);
+
+      // Get the stats of the item
+      const stats = await fs.stat(itemPath);
+
+      // Check if the item is a file or a directory
+      const isDirectory = stats.isDirectory();
+
+      // Compare the last modified date with three months ago
+      if (stats.mtime < monthsAgo) {
+        if (isDirectory) {
+          // If it's a directory, delete it recursively
+          await fs.rm(itemPath, { recursive: true });
+          console.log(`Cleaned directory: ${item}`);
+        } else {
+          // If it's a file, delete it
+          await fs.unlink(itemPath);
+          console.log(`Cleaned file: ${item}`);
+        }
+      }
+    }
+  } catch (error) {
+    console.error(`Error cleaning up files: ${error}`);
+  }
+}
+
 
 export {
-    moveFileToPublicFolder, deleteFolder
+    moveFileToPublicFolder, deleteFolder, cleanupLocal
 };

package/helper_apps/{MediaFileChunker → CortexFileHandler}/package-lock.json

@@ -21,8 +21,7 @@
         "pdfjs-dist": "^3.9.179",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "xlsx": "^0.18.5",
-        "ytdl-core": "git+ssh://git@github.com:khlevon/node-ytdl-core.git#v4.11.4-patch.2"
+        "xlsx": "^0.18.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -2545,19 +2544,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
-    },
-    "node_modules/ytdl-core": {
-      "version": "0.0.0-development",
-      "resolved": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "license": "MIT",
-      "dependencies": {
-        "m3u8stream": "^0.8.6",
-        "miniget": "^4.2.2",
-        "sax": "^1.1.3"
-      },
-      "engines": {
-        "node": ">=12"
-      }
     }
   },
   "dependencies": {
@@ -4452,15 +4438,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
-    },
-    "ytdl-core": {
-      "version": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "from": "ytdl-core@git+ssh://git@github.com:khlevon/node-ytdl-core.git#v4.11.4-patch.2",
-      "requires": {
-        "m3u8stream": "^0.8.6",
-        "miniget": "^4.2.2",
-        "sax": "^1.1.3"
-      }
     }
   }
 }

package/lib/keyValueStorageClient.js

@@ -2,10 +2,7 @@ import Keyv from 'keyv';
 import { config } from '../config.js';
 
 const storageConnectionString = config.get('storageConnectionString');
-
-if (!config.get('storageConnectionString')) {
-    console.log('No storageConnectionString specified. Please set the storageConnectionString or STORAGE_CONNECTION_STRING environment variable if you need caching or stored context.')
-}
+const cortexId = config.get('cortexId');
 
 // Create a keyv client to store data
 const keyValueStorageClient = new Keyv(storageConnectionString, {
@@ -13,7 +10,7 @@ const keyValueStorageClient = new Keyv(storageConnectionString, {
     abortConnect: false,
     serialize: JSON.stringify,
     deserialize: JSON.parse,
-    namespace: 'cortex-context'
+    namespace: `${cortexId}-cortex-context`
 });
 
 // Set values to keyv

package/lib/redisSubscription.js

@@ -3,48 +3,60 @@ import { config } from '../config.js';
 import pubsub from '../server/pubsub.js';
 
 const connectionString = config.get('storageConnectionString');
-const client = new Redis(connectionString);
-
 const channel = 'requestProgress';
+let client;
 
-client.on('error', (error) => {
-    console.error(`Redis client error: ${error}`);
-});
-
-client.on('connect', () => {
-    client.subscribe(channel, (error) => {
-        if (error) {
-            console.error(`Error subscribing to channel ${channel}: ${error}`);
-        } else {
-            console.log(`Subscribed to channel ${channel}`);
-        }
-    });
-});
-
-client.on('message', (channel, message) => {
-    if (channel === 'requestProgress') {
-        console.log(`Received message from ${channel}: ${message}`);
-        let parsedMessage;
-
-        try {
-            parsedMessage = JSON.parse(message);
-        } catch (error) {
-            parsedMessage = message;
-        }
-
-        handleMessage(parsedMessage);
-    }
-});
-
-const handleMessage = (data) => {
-    // Process the received data
-    console.log('Processing data:', data);
+if (connectionString) {
+    console.log(`Using Redis subscription for channel ${channel}`);
     try {
-        pubsub.publish('REQUEST_PROGRESS', { requestProgress: data });
+        client = connectionString && new Redis(connectionString);
     } catch (error) {
-        console.error(`Error publishing data to pubsub: ${error}`);
+        console.error('Redis connection error: ', error);
+    }
+
+    if (client) {
+        const channel = 'requestProgress';
+
+        client.on('error', (error) => {
+            console.error(`Redis client error: ${error}`);
+        });
+
+        client.on('connect', () => {
+            client.subscribe(channel, (error) => {
+                if (error) {
+                    console.error(`Error subscribing to channel ${channel}: ${error}`);
+                } else {
+                    console.log(`Subscribed to channel ${channel}`);
+                }
+            });
+        });
+
+        client.on('message', (channel, message) => {
+            if (channel === 'requestProgress') {
+                console.log(`Received message from ${channel}: ${message}`);
+                let parsedMessage;
+
+                try {
+                    parsedMessage = JSON.parse(message);
+                } catch (error) {
+                    parsedMessage = message;
+                }
+
+                handleMessage(parsedMessage);
+            }
+        });
+
+        const handleMessage = (data) => {
+            // Process the received data
+            console.log('Processing data:', data);
+            try {
+                pubsub.publish('REQUEST_PROGRESS', { requestProgress: data });
+            } catch (error) {
+                console.error(`Error publishing data to pubsub: ${error}`);
+            }
+        };
     }
-};
+}
 
 export {
     client as subscriptionClient,

package/lib/request.js

@@ -3,37 +3,71 @@ import RequestMonitor from './requestMonitor.js';
 import { config } from '../config.js';
 import axios from 'axios';
 import { setupCache } from 'axios-cache-interceptor';
+import Redis from 'ioredis';
 
-let cortexAxios = axios;
+const connectionString = config.get('storageConnectionString');
 
-if (config.get('enableCache')) {
-    // Setup cache
-    cortexAxios = setupCache(axios, {
-        // enable cache for all requests by default
-        methods: ['get', 'post', 'put', 'delete', 'patch'],
-        interpretHeader: false,
-        ttl: 1000 * 60 * 60 * 24 * 7, // 7 days
-    }); 
+if (!connectionString) {
+    console.log('No STORAGE_CONNECTION_STRING found in environment. Redis features (caching, pubsub, clustered limiters) disabled.')
+} else {
+    console.log('Using Redis connection specified in STORAGE_CONNECTION_STRING.');
+}   
+
+let client;
+
+if (connectionString) {
+    try {
+        client = new Redis(connectionString);
+    } catch (error) {
+        console.error('Redis connection error: ', error);
+    }
 }
 
+const cortexId = config.get('cortexId');
+const connection = client && new Bottleneck.IORedisConnection({ client: client });
+
 const limiters = {};
 const monitors = {};
 
 const buildLimiters = (config) => {
-    console.log('Building limiters...');
+    console.log(`Building ${connection ? 'Redis clustered' : 'local'} model rate limiters for ${cortexId}...`);
     for (const [name, model] of Object.entries(config.get('models'))) {
         const rps = model.requestsPerSecond ?? 100;
-        limiters[name] = new Bottleneck({
+        let limiterOptions = {
             minTime: 1000 / rps,
             maxConcurrent: rps,
             reservoir: rps,      // Number of tokens available initially
             reservoirRefreshAmount: rps,     // Number of tokens added per interval
             reservoirRefreshInterval: 1000, // Interval in milliseconds
+        };
+
+        // If Redis connection exists, add id and connection to enable clustering
+        if (connection) {
+            limiterOptions.id = `${cortexId}-${name}-limiter`; // Unique id for each limiter
+            limiterOptions.connection = connection;  // Shared Redis connection
+            limiterOptions.clearDatastore = true;    // Clear Redis datastore on startup
+        }
+
+        limiters[name] = new Bottleneck(limiterOptions);
+        limiters[name].on('error', (err) => {
+            console.error(`Limiter error for ${cortexId}-${name}:`, err);
         });
         monitors[name] = new RequestMonitor();
     }
 }
 
+let cortexAxios = axios;
+
+if (config.get('enableCache')) {
+    // Setup cache
+    cortexAxios = setupCache(axios, {
+        // enable cache for all requests by default
+        methods: ['get', 'post', 'put', 'delete', 'patch'],
+        interpretHeader: false,
+        ttl: 1000 * 60 * 60 * 24 * 7, // 7 days
+    }); 
+}
+
 setInterval(() => {
     const monitorKeys = Object.keys(monitors);
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.24",
+  "version": "1.0.25",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/index.js

@@ -18,6 +18,7 @@ import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
 import translate from './translate.js';
 import embeddings from './embeddings.js';
+import vision from './vision.js';
 
 export {
     edit,
@@ -39,5 +40,6 @@ export {
     test_langchain,
     test_palm_chat,
     transcribe,
-    translate
+    translate,
+    vision,
 };

package/pathways/transcribe.js

@@ -6,6 +6,10 @@ export default {
         language: ``,
         responseFormat: `text`,
         wordTimestamped: false,
+        highlightWords: false,
+        maxLineWidth: 0,
+        maxLineCount: 0,
+        maxWordsPerLine: 0,
     },
     timeout: 3600, // in seconds
     enableDuplicateRequests: false,

package/pathways/vision.js

@@ -0,0 +1,18 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: [
+        new Prompt({ messages: [
+            "{{chatHistory}}",
+        ]}),
+    ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+    },
+    max_tokens: 1024,
+    model: 'oai-gpt4-vision',
+    tokenRatio: 0.96,
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+}

package/server/chunker.js

@@ -11,12 +11,52 @@ const getLastNToken = (text, maxTokenLen) => {
 }
 
 const getFirstNToken = (text, maxTokenLen) => {
-    const encoded = encode(text);
-    if (encoded.length > maxTokenLen) {
-        text = decode(encoded.slice(0, maxTokenLen + 1));
-        text = text.slice(0,text.search(/\s[^\s]*$/)); // skip potential partial word
-    }
-    return text;
+  if (Array.isArray(text)) {
+    return getFirstNTokenArray(text, maxTokenLen);
+  } else {
+    return getFirstNTokenSingle(text, maxTokenLen);
+  }
+}
+
+const getFirstNTokenSingle = (text, maxTokenLen) => {
+  const encoded = encode(text);
+  if (encoded.length > maxTokenLen) {
+      text = decode(encoded.slice(0, maxTokenLen + 1));
+      text = text.slice(0,text.search(/\s[^\s]*$/)); // skip potential partial word
+  }
+  return text;
+}
+
+
+function getFirstNTokenArray(content, tokensToKeep) {
+  let totalTokens = 0;
+  let result = [];
+
+  for (let i = content.length - 1; i >= 0; i--) {
+      const message = content[i];
+      const messageTokens = encode(message).length;
+
+      if (totalTokens + messageTokens <= tokensToKeep) {
+          totalTokens += messageTokens;
+          result.unshift(message); // Add message to the start
+      } else {
+          try{
+            const messageObj = JSON.parse(message);
+            if(messageObj.type === "image_url"){
+              break;
+            }
+          }catch(e){
+            // ignore
+          }
+
+          const remainingTokens = tokensToKeep - totalTokens;
+          const truncatedMessage = getFirstNToken(message, remainingTokens);
+          result.unshift(truncatedMessage); // Add truncated message to the start
+          break;
+      }
+  }
+
+  return result;
 }
 
 const determineTextFormat = (text) => {

package/server/graphql.js

@@ -11,6 +11,7 @@ import { useServer } from 'graphql-ws/lib/use/ws';
 import express from 'express';
 import http from 'http';
 import Keyv from 'keyv';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import cors from 'cors';
 import { KeyvAdapter } from '@apollo/utils.keyvadapter';
 import responseCachePlugin from '@apollo/server-plugin-response-cache';
@@ -40,6 +41,7 @@ const getPlugins = (config) => {
         // TODO: custom cache key:
         // https://www.apollographql.com/docs/apollo-server/performance/cache-backends#implementing-your-own-cache-backend
         plugins.push(responseCachePlugin({ cache }));
+        console.log('Using Redis for GraphQL cache');
     }
 
     return { plugins, cache };
@@ -167,6 +169,11 @@ const build = async (config) => {
         ]),
     });
 
+    // Healthcheck endpoint is valid regardless of auth
+    app.get('/healthcheck', (req, res) => {
+        res.status(200).send('OK');
+    });
+
     // If CORTEX_API_KEY is set, we roll our own auth middleware - usually not used if you're being fronted by a proxy
     const cortexApiKey = config.get('cortexApiKey');
     if (cortexApiKey) {
@@ -202,7 +209,7 @@ const build = async (config) => {
                 next();
             }
         });
-    };
+    }
 
     // Parse the body for REST endpoints
     app.use(express.json());

package/server/pathwayPrompter.js

@@ -14,6 +14,7 @@ import AzureCognitivePlugin from './plugins/azureCognitivePlugin.js';
 import OpenAiEmbeddingsPlugin from './plugins/openAiEmbeddingsPlugin.js';
 import OpenAIImagePlugin from './plugins/openAiImagePlugin.js';
 import OpenAIDallE3Plugin from './plugins/openAiDallE3Plugin.js';
+import OpenAIVisionPlugin from './plugins/openAiVisionPlugin.js';
 
 class PathwayPrompter {
     constructor(config, pathway, modelName, model) {
@@ -66,6 +67,9 @@ class PathwayPrompter {
             case 'COHERE-SUMMARIZE':
                 plugin = new CohereSummarizePlugin(config, pathway, modelName, model);
                 break;
+            case 'OPENAI-VISION':
+                plugin = new OpenAIVisionPlugin(config, pathway, modelName, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/server/pathwayResolver.js

@@ -95,6 +95,8 @@ class PathwayResolver {
                 try {
                     const incomingMessage = responseData;
 
+                    let messageBuffer = '';
+
                     const processData = (data) => {
                         try {
                             //console.log(`\n\nReceived stream data for requestId ${this.requestId}`, data.toString());
@@ -108,27 +110,27 @@ class PathwayResolver {
                                 // skip empty events
                                 if (!(event.trim() === '')) {
                                     //console.log(`Processing stream event for requestId ${this.requestId}`, event);
-
-                                    let message = event.replace(/^data: /, '');
+                                    messageBuffer += event.replace(/^data: /, '');
 
                                     const requestProgress = {
                                         requestId: this.requestId,
-                                        data: message,
+                                        data: messageBuffer,
                                     }
 
                                     // check for end of stream or in-stream errors
-                                    if (message.trim() === '[DONE]') {
+                                    if (messageBuffer.trim() === '[DONE]') {
                                         requestProgress.progress = 1;
                                     } else {
                                         let parsedMessage;
                                         try {
-                                            parsedMessage = JSON.parse(message);
+                                            parsedMessage = JSON.parse(messageBuffer);
+                                            messageBuffer = '';
                                         } catch (error) {
-                                            console.error('Could not JSON parse stream message', message, error);
+                                            // incomplete stream message, try to buffer more data
                                             return;
                                         }
 
-                                        const streamError = parsedMessage.error || parsedMessage?.choices?.[0]?.delta?.content?.error || parsedMessage?.choices?.[0]?.text?.error;
+                                        const streamError = parsedMessage?.error || parsedMessage?.choices?.[0]?.delta?.content?.error || parsedMessage?.choices?.[0]?.text?.error;
                                         if (streamError) {
                                             streamErrorOccurred = true;
                                             console.error(`Stream error: ${streamError.message}`);
@@ -143,7 +145,7 @@ class PathwayResolver {
                                             requestProgress: requestProgress
                                         });
                                     } catch (error) {
-                                        console.error('Could not publish the stream message', message, error);
+                                        console.error('Could not publish the stream message', messageBuffer, error);
                                     }
                                 }
                             }

package/server/plugins/modelPlugin.js

@@ -74,13 +74,16 @@ class ModelPlugin {
                 const otherMessageTokens = totalTokenLength - currentTokenLength;
                 const tokensToKeep = targetTokenLength - (otherMessageTokens + emptyContentLength);
 
-                if (tokensToKeep <= 0) {
+                if (tokensToKeep <= 0 || Array.isArray(message?.content)) {  
                     // If the message needs to be empty to make the target, remove it entirely
                     totalTokenLength -= currentTokenLength;
                     tokenLengths.splice(index, 1);
+                    if(tokenLengths.length == 0){
+                        throw new Error(`Unable to process your request as your single message content is too long. Please try again with a shorter message.`);
+                    }
                 } else {
                     // Otherwise, update the message and token length
-                    const truncatedContent = getFirstNToken(message.content, tokensToKeep);
+                    const truncatedContent = getFirstNToken(message?.content ?? message, tokensToKeep);
                     const truncatedMessage = { ...message, content: truncatedContent };
 
                     tokenLengths[index] = {