@aj-archipelago/cortex 1.0.24 → 1.1.0-beta.0

package/.eslintrc

@@ -16,7 +16,7 @@
     ],
     "rules": {
         "import/no-unresolved": "error",
-        "import/no-extraneous-dependencies": ["error", {"devDependencies": true}],
+        "import/no-extraneous-dependencies": ["error", {"devDependencies": true, "dependencies": true}],
         "no-unused-vars": ["error", { "argsIgnorePattern": "^_" }]
     },
     "settings": {

package/config.js

@@ -4,11 +4,22 @@ import HandleBars from './lib/handleBars.js';
 import fs from 'fs';
 import { fileURLToPath, pathToFileURL } from 'url';
 import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
+import logger from './lib/logger.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // Schema for config
 var config = convict({
+    env: {
+        format: String,
+        default: 'development',
+        env: 'NODE_ENV'
+    },
+    cortexId: {
+        format: String,
+        default: 'local',
+        env: 'CORTEX_ID'
+    },
     basePathwayPath: {
         format: String,
         default: path.join(__dirname, 'pathways', 'basePathway.js'),
@@ -109,6 +120,20 @@ var config = convict({
                 },
                 "maxTokenLength": 8192,
             },
+            "oai-gpt4-vision": {
+                "type": "OPENAI-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-4-vision-preview"
+                },
+                "requestsPerSecond": 1,
+                "maxTokenLength": 128000,
+                "supportsStreaming": true
+            },
         },
         env: 'CORTEX_MODELS'
     },
@@ -176,14 +201,16 @@ const configFile = config.get('cortexConfigFile');
 
 // Load config file
 if (configFile && fs.existsSync(configFile)) {
-    console.log('Loading config from', configFile);
+    logger.info(`Loading config from ${configFile}`);
     config.loadFile(configFile);
 } else {
     const openaiApiKey = config.get('openaiApiKey');
     if (!openaiApiKey) {
-        throw console.log('No config file or api key specified. Please set the OPENAI_API_KEY to use OAI or use CORTEX_CONFIG_FILE environment variable to point at the Cortex configuration for your project.');
+        const errorString = 'No config file or api key specified. Please set the OPENAI_API_KEY to use OAI or use CORTEX_CONFIG_FILE environment variable to point at the Cortex configuration for your project.';
+        logger.error(errorString);
+        throw new Error(errorString);
     } else {
-        console.log(`Using default model with OPENAI_API_KEY environment variable`)
+        logger.info(`Using default model with OPENAI_API_KEY environment variable`)
     }
 }
 
@@ -204,12 +231,12 @@ const buildPathways = async (config) => {
     const basePathway = await import(basePathwayURL).then(module => module.default);
 
     // Load core pathways, default from the Cortex package
-    console.log('Loading core pathways from', corePathwaysPath)
+    logger.info(`Loading core pathways from ${corePathwaysPath}`)
     let loadedPathways = await import(`${corePathwaysURL}/index.js`).then(module => module);
 
     // Load custom pathways and override core pathways if same
     if (pathwaysPath && fs.existsSync(pathwaysPath)) {
-        console.log('Loading custom pathways from', pathwaysPath)
+        logger.info(`Loading custom pathways from ${pathwaysPath}`)
         const customPathways = await import(`${pathwaysURL}/index.js`).then(module => module);
         loadedPathways = { ...loadedPathways, ...customPathways };
     }
@@ -244,12 +271,14 @@ const buildModels = (config) => {
 
     // Check that models are specified, Cortex cannot run without a model
     if (Object.keys(config.get('models')).length <= 0) {
-        throw console.log('No models specified! Please set the models in your config file or via CORTEX_MODELS environment variable to point at the models for your project.');
+        const errorString = 'No models specified! Please set the models in your config file or via CORTEX_MODELS environment variable to point at the models for your project.';
+        logger.error(errorString);
+        throw new Error(errorString);
     }
 
     // Set default model name to the first model in the config in case no default is specified
     if (!config.get('defaultModelName')) {
-        console.log('No default model specified, using first model as default.');
+        logger.warn('No default model specified, using first model as default.');
         config.load({ defaultModelName: Object.keys(config.get('models'))[0] });
     }
 

package/helper_apps/{MediaFileChunker → CortexFileHandler}/blobHandler.js

@@ -145,6 +145,32 @@ async function uploadBlob(context, req, saveToLocal = false) {
     });
 }
 
+// Function to delete files that haven't been used in more than a month
+async function cleanup() {
+    const { containerClient } = getBlobClient();
+  
+    // List all the blobs in the container
+    const blobs = containerClient.listBlobsFlat();
+  
+    // Calculate the date that is x month ago
+    const xMonthAgo = new Date();
+    xMonthAgo.setMonth(xMonthAgo.getMonth() - 1);
+  
+    // Iterate through the blobs
+    for await (const blob of blobs) {
+      // Get the last modified date of the blob
+      const lastModified = blob.properties.lastModified;
+  
+      // Compare the last modified date with one month ago
+      if (lastModified < xMonthAgo) {
+        // Delete the blob
+        const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
+        await blockBlobClient.delete();
+        console.log(`Cleaned blob: ${blob.name}`);
+      }
+    }
+}
+
 export {
-    saveFileToBlob, deleteBlob, uploadBlob
+    saveFileToBlob, deleteBlob, uploadBlob, cleanup
 }

package/helper_apps/{MediaFileChunker → CortexFileHandler}/fileChunker.js

@@ -3,7 +3,6 @@ import path from 'path';
 import ffmpeg from 'fluent-ffmpeg';
 import { v4 as uuidv4 } from 'uuid';
 import os from 'os';
-import ytdl from 'ytdl-core';
 import { promisify } from 'util';
 import axios from 'axios';
 import { ensureEncoded } from './helper.js';

package/helper_apps/{MediaFileChunker → CortexFileHandler}/index.js

@@ -1,8 +1,8 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup } from './blobHandler.js';
 import { publishRequestProgress } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
-import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
+import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
 import { documentToText, easyChunker } from './docHelper.js';
 import path from 'path';
 import os from 'os';
@@ -15,9 +15,29 @@ const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
 console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
 
 
+let isCleanupRunning = false;
+async function cleanupInactive(useAzure) {
+    try {
+        if (isCleanupRunning) { return; } //no need to cleanup every call
+        isCleanupRunning = true;
+        if (useAzure) {
+            await cleanup();
+        } else {
+            await cleanupLocal();
+        }
+    } catch (error) {
+        console.log('Error occurred during cleanup:', error);
+    } finally{
+        isCleanupRunning = false;
+    }
+}
+
+  
 async function main(context, req) {
     context.log('Starting req processing..');
 
+    cleanupInactive(useAzure); //trigger & no need to wait for it
+
     // Clean up blob when request delete which means processing marked completed
     if (req.method.toLowerCase() === `delete`) {
         const { requestId } = req.query;

package/helper_apps/{MediaFileChunker → CortexFileHandler}/localFileHandler.js

@@ -4,7 +4,6 @@ import { v4 as uuidv4 } from 'uuid';
 
 import { publicFolder, port, ipAddress } from "./start.js";
 
-
 async function moveFileToPublicFolder(chunkPath, requestId) {
     // Use the filename with a UUID as the blob name
     const filename = `${requestId}/${uuidv4()}_${basename(chunkPath)}`;
@@ -30,7 +29,44 @@ async function deleteFolder(requestId) {
     console.log(`Cleaned folder: ${targetFolder}`);
 }
 
+async function cleanupLocal() {
+  try {
+    // Read the directory
+    const items = await fs.readdir(publicFolder);
+
+    // Calculate the date that is x months ago
+    const monthsAgo = new Date();
+    monthsAgo.setMonth(monthsAgo.getMonth() - 1);
+
+    // Iterate through the items
+    for (const item of items) {
+      const itemPath = join(publicFolder, item);
+
+      // Get the stats of the item
+      const stats = await fs.stat(itemPath);
+
+      // Check if the item is a file or a directory
+      const isDirectory = stats.isDirectory();
+
+      // Compare the last modified date with three months ago
+      if (stats.mtime < monthsAgo) {
+        if (isDirectory) {
+          // If it's a directory, delete it recursively
+          await fs.rm(itemPath, { recursive: true });
+          console.log(`Cleaned directory: ${item}`);
+        } else {
+          // If it's a file, delete it
+          await fs.unlink(itemPath);
+          console.log(`Cleaned file: ${item}`);
+        }
+      }
+    }
+  } catch (error) {
+    console.error(`Error cleaning up files: ${error}`);
+  }
+}
+
 
 export {
-    moveFileToPublicFolder, deleteFolder
+    moveFileToPublicFolder, deleteFolder, cleanupLocal
 };

package/helper_apps/{MediaFileChunker → CortexFileHandler}/package-lock.json

@@ -21,8 +21,7 @@
         "pdfjs-dist": "^3.9.179",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "xlsx": "^0.18.5",
-        "ytdl-core": "git+ssh://git@github.com:khlevon/node-ytdl-core.git#v4.11.4-patch.2"
+        "xlsx": "^0.18.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -2545,19 +2544,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
-    },
-    "node_modules/ytdl-core": {
-      "version": "0.0.0-development",
-      "resolved": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "license": "MIT",
-      "dependencies": {
-        "m3u8stream": "^0.8.6",
-        "miniget": "^4.2.2",
-        "sax": "^1.1.3"
-      },
-      "engines": {
-        "node": ">=12"
-      }
     }
   },
   "dependencies": {
@@ -4452,15 +4438,6 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
-    },
-    "ytdl-core": {
-      "version": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "from": "ytdl-core@git+ssh://git@github.com:khlevon/node-ytdl-core.git#v4.11.4-patch.2",
-      "requires": {
-        "m3u8stream": "^0.8.6",
-        "miniget": "^4.2.2",
-        "sax": "^1.1.3"
-      }
     }
   }
 }

package/lib/keyValueStorageClient.js

@@ -2,10 +2,7 @@ import Keyv from 'keyv';
 import { config } from '../config.js';
 
 const storageConnectionString = config.get('storageConnectionString');
-
-if (!config.get('storageConnectionString')) {
-    console.log('No storageConnectionString specified. Please set the storageConnectionString or STORAGE_CONNECTION_STRING environment variable if you need caching or stored context.')
-}
+const cortexId = config.get('cortexId');
 
 // Create a keyv client to store data
 const keyValueStorageClient = new Keyv(storageConnectionString, {
@@ -13,7 +10,7 @@ const keyValueStorageClient = new Keyv(storageConnectionString, {
     abortConnect: false,
     serialize: JSON.stringify,
     deserialize: JSON.parse,
-    namespace: 'cortex-context'
+    namespace: `${cortexId}-cortex-context`
 });
 
 // Set values to keyv

package/lib/logger.js

@@ -0,0 +1,29 @@
+// logger.js
+import winston from 'winston';
+
+const format = winston.format.combine(
+    //winston.format.timestamp(),
+    winston.format.colorize({ all: true }),
+    winston.format.simple()
+);
+
+const transports = [
+    new winston.transports.Console({ format })
+];
+
+const logger = winston.createLogger({
+    level: process.env.NODE_ENV === 'production' ? 'info' : 'debug',
+    format: format,
+    transports: transports
+});
+
+winston.addColors({
+    debug: 'green',
+    verbose: 'blue',
+    http: 'gray',
+    info: 'cyan',
+    warn: 'yellow',
+    error: 'red'
+});
+
+export default logger;

package/lib/redisSubscription.js

@@ -1,51 +1,135 @@
 import Redis from 'ioredis';
 import { config } from '../config.js';
 import pubsub from '../server/pubsub.js';
+import { requestState } from '../server/requestState.js';
+import logger from '../lib/logger.js';
 
 const connectionString = config.get('storageConnectionString');
-const client = new Redis(connectionString);
+const channels = ['requestProgress', 'requestProgressSubscriptions'];
+let client;
 
-const channel = 'requestProgress';
+if (connectionString) {
+    logger.info(`Using Redis subscription for channel(s) ${channels.join(', ')}`);
+    try {
+        client = connectionString && new Redis(connectionString);
+    } catch (error) {
+        logger.error(`Redis connection error: ${JSON.stringify(error)}`);
+    }
 
-client.on('error', (error) => {
-    console.error(`Redis client error: ${error}`);
-});
+    if (client) {
 
-client.on('connect', () => {
-    client.subscribe(channel, (error) => {
-        if (error) {
-            console.error(`Error subscribing to channel ${channel}: ${error}`);
-        } else {
-            console.log(`Subscribed to channel ${channel}`);
-        }
-    });
-});
+        client.on('error', (error) => {
+            logger.error(`Redis client error: ${JSON.stringify(error)}`);
+        });
+
+        client.on('connect', () => {
+            client.subscribe('requestProgress', (error) => {
+                if (error) {
+                    logger.error(`Error subscribing to redis channel requestProgress: ${JSON.stringify(error)}`);
+                } else {
+                    logger.info(`Subscribed to channel requestProgress`);
+                }
+            });
+            client.subscribe('requestProgressSubscriptions', (error) => {
+                if (error) {
+                    logger.error(`Error subscribing to redis channel requestProgressSubscriptions: ${JSON.stringify(error)}`);
+                } else {
+                    logger.info(`Subscribed to channel requestProgressSubscriptions`);
+                }
+            });
+        });
+
+        client.on('message', (channel, message) => {
+            if (channel === 'requestProgress') {
+                logger.debug(`Received message from ${channel}: ${message}`);
+                let parsedMessage;
+
+                try {
+                    parsedMessage = JSON.parse(message);
+                } catch (error) {
+                    parsedMessage = message;
+                }
+
+                pubsubHandleMessage(parsedMessage);
+            } else {
+                if (channel === 'requestProgressSubscriptions') {
+                    logger.debug(`Received message from ${channel}: ${message}`);
+                    let parsedMessage;
+
+                    try {
+                        parsedMessage = JSON.parse(message);
+                    } catch (error) {
+                        parsedMessage = message;
+                    }
+
+                    handleSubscription(parsedMessage);
+                }
+            }
+        });
+    }
+}
 
-client.on('message', (channel, message) => {
-    if (channel === 'requestProgress') {
-        console.log(`Received message from ${channel}: ${message}`);
-        let parsedMessage;
 
+let publisherClient;
+
+if (connectionString) {
+    logger.info(`Using Redis publish for channel(s) ${channels.join(', ')}`);
+    publisherClient = Redis.createClient(connectionString);
+} else {
+    logger.info(`Using pubsub publish for channel ${channels[0]}`);
+}
+
+async function publishRequestProgress(data) {
+    if (publisherClient) {
         try {
-            parsedMessage = JSON.parse(message);
+            const message = JSON.stringify(data);
+            logger.debug(`Publishing message ${message} to channel ${channels[0]}`);
+            await publisherClient.publish(channels[0], message);
         } catch (error) {
-            parsedMessage = message;
+            logger.error(`Error publishing message: ${JSON.stringify(error)}`);
         }
+    } else {
+        pubsubHandleMessage(data);
+    }
+}
 
-        handleMessage(parsedMessage);
+async function publishRequestProgressSubscription(data) {
+    if (publisherClient) {
+        try {
+            const message = JSON.stringify(data);
+            logger.debug(`Publishing message ${message} to channel ${channels[1]}`);
+            await publisherClient.publish(channels[1], message);
+        } catch (error) {
+            logger.error(`Error publishing message: ${JSON.stringify(error)}`);
+        }
+    } else {
+        handleSubscription(data);
     }
-});
+}
 
-const handleMessage = (data) => {
-    // Process the received data
-    console.log('Processing data:', data);
+function pubsubHandleMessage(data){
+    const message = JSON.stringify(data);
+    logger.debug(`Publishing message to pubsub: ${message}`);
     try {
         pubsub.publish('REQUEST_PROGRESS', { requestProgress: data });
     } catch (error) {
-        console.error(`Error publishing data to pubsub: ${error}`);
+        logger.error(`Error publishing data to pubsub: ${JSON.stringify(error)}`);
     }
-};
+}
+
+function handleSubscription(data){
+    const requestIds = data;
+    for (const requestId of requestIds) {
+        if (requestState[requestId] && !requestState[requestId].started) {
+            requestState[requestId].started = true;
+            logger.info(`Subscription starting async requestProgress, requestId: ${requestId}`);
+            const { resolver, args } = requestState[requestId];
+            resolver(args);
+        }
+    }
+}
+
 
 export {
-    client as subscriptionClient,
+    client as subscriptionClient, publishRequestProgress, publishRequestProgressSubscription
 };