@aj-archipelago/cortex 1.1.18 → 1.1.20

package/config/default.example.json

@@ -1,5 +1,5 @@
 {
-    "defaultModelName": "oai-td3",
+    "defaultModelName": "oai-gpturbo",
     "models": {
         "azure-translate": {
             "type": "AZURE-TRANSLATE",
@@ -12,19 +12,6 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
-        "oai-td3": {
-            "type": "OPENAI-COMPLETION",
-            "url": "https://api.openai.com/v1/completions",
-            "headers": {
-                "Authorization": "Bearer {{OPENAI_API_KEY}}",
-                "Content-Type": "application/json"
-            },
-            "params": {
-                "model": "text-davinci-003"
-            },
-            "requestsPerSecond": 10,
-            "maxTokenLength": 4096
-        },
         "oai-gpturbo": {
             "type": "OPENAI-CHAT",
             "url": "https://api.openai.com/v1/chat/completions",

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -1,176 +1,351 @@
-import fs from 'fs';
-import path from 'path';
-import { BlobServiceClient } from '@azure/storage-blob';
-import { v4 as uuidv4 } from 'uuid';
-import Busboy from 'busboy';
-import { PassThrough } from 'stream';
-import { pipeline as _pipeline } from 'stream';
-import { promisify } from 'util';
+import fs from "fs";
+import path from "path";
+import { BlobServiceClient } from "@azure/storage-blob";
+import { v4 as uuidv4 } from "uuid";
+import Busboy from "busboy";
+import { PassThrough } from "stream";
+import { pipeline as _pipeline } from "stream";
+import { promisify } from "util";
 const pipeline = promisify(_pipeline);
-import { join } from 'path';
+import { join } from "path";
+import { Storage } from "@google-cloud/storage";
+import axios from "axios";
+import { publicFolder, port, ipAddress } from "./start.js";
+import mime from "mime-types";
+
+const IMAGE_EXTENSIONS = [
+  ".jpg",
+  ".jpeg",
+  ".png",
+  ".gif",
+  ".bmp",
+  ".webp",
+  ".tiff",
+  ".svg",
+  ".pdf"
+];
+
+const VIDEO_EXTENSIONS = [
+  ".mp4",
+  ".webm",
+  ".ogg",
+  ".mov",
+  ".avi",
+  ".flv",
+  ".wmv",
+  ".mkv",
+];
+
+function isBase64(str) {
+  try {
+    return btoa(atob(str)) == str;
+  } catch (err) {
+    return false;
+  }
+}
 
+const GCP_SERVICE_ACCOUNT_KEY =
+  process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 ||
+  process.env.GCP_SERVICE_ACCOUNT_KEY ||
+  "{}";
+const GCP_SERVICE_ACCOUNT = isBase64(GCP_SERVICE_ACCOUNT_KEY)
+  ? JSON.parse(Buffer.from(GCP_SERVICE_ACCOUNT_KEY, "base64").toString())
+  : JSON.parse(GCP_SERVICE_ACCOUNT_KEY);
+const { project_id: GCP_PROJECT_ID } = GCP_SERVICE_ACCOUNT;
+
+let gcs;
+if (!GCP_PROJECT_ID || !GCP_SERVICE_ACCOUNT) {
+  console.warn(
+    "Google Cloud Project ID or Service Account details are missing"
+  );
+} else {
+  try {
+    gcs = new Storage({
+      projectId: GCP_PROJECT_ID,
+      credentials: GCP_SERVICE_ACCOUNT,
+    });
 
-import { publicFolder, port, ipAddress } from "./start.js";
+    // Rest of your Google Cloud operations using gcs object
+  } catch (error) {
+    console.error(
+      "Provided Google Cloud Service Account details are invalid: ",
+      error
+    );
+  }
+}
 
-const getBlobClient = () => {
-    const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
-    const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
-    if (!connectionString || !containerName) {
-        throw new Error('Missing Azure Storage connection string or container name environment variable');
-    }
+const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles";
 
-    const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
-    const containerClient = blobServiceClient.getContainerClient(containerName);
+const getBlobClient = async () => {
+  const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+  const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
+  if (!connectionString || !containerName) {
+    throw new Error(
+      "Missing Azure Storage connection string or container name environment variable"
+    );
+  }
 
-    return { blobServiceClient, containerClient };
-}
+  const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+
+  const serviceProperties = await blobServiceClient.getProperties();
+  if(!serviceProperties.defaultServiceVersion) {
+    serviceProperties.defaultServiceVersion = '2020-02-10';
+    await blobServiceClient.setProperties(serviceProperties);
+  }
+
+  const containerClient = blobServiceClient.getContainerClient(containerName);
+
+  return { blobServiceClient, containerClient };
+};
 
 async function saveFileToBlob(chunkPath, requestId) {
-    const { containerClient } = getBlobClient();
-    // Use the filename with a UUID as the blob name
-    const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+  const { containerClient } = await getBlobClient();
+  // Use the filename with a UUID as the blob name
+  const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
 
-    // Create a read stream for the chunk file
-    const fileStream = fs.createReadStream(chunkPath);
+  // Create a read stream for the chunk file
+  const fileStream = fs.createReadStream(chunkPath);
 
-    // Upload the chunk to Azure Blob Storage using the stream
-    const blockBlobClient = containerClient.getBlockBlobClient(blobName);
-    await blockBlobClient.uploadStream(fileStream);
+  // Upload the chunk to Azure Blob Storage using the stream
+  const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+  await blockBlobClient.uploadStream(fileStream);
 
-    // Return the full URI of the uploaded blob
-    const blobUrl = blockBlobClient.url;
-    return blobUrl;
+  // Return the full URI of the uploaded blob
+  const blobUrl = blockBlobClient.url;
+  return blobUrl;
 }
 
 //deletes blob that has the requestId
 async function deleteBlob(requestId) {
-    if (!requestId) throw new Error('Missing requestId parameter');
-    const { containerClient } = getBlobClient();
-    // List the blobs in the container with the specified prefix
-    const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
+  if (!requestId) throw new Error("Missing requestId parameter");
+  const { containerClient } = await getBlobClient();
+  // List the blobs in the container with the specified prefix
+  const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
+
+  const result = [];
+  // Iterate through the blobs
+  for await (const blob of blobs) {
+    // Delete the matching blob
+    const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
+    await blockBlobClient.delete();
+    console.log(`Cleaned blob: ${blob.name}`);
+    result.push(blob.name);
+  }
+
+  return result;
+}
 
-    const result = []
-    // Iterate through the blobs
-    for await (const blob of blobs) {
-        // Delete the matching blob
-        const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
-        await blockBlobClient.delete();
-        console.log(`Cleaned blob: ${blob.name}`);
-        result.push(blob.name);
-    }
+async function uploadBlob(
+  context,
+  req,
+  saveToLocal = false,
+  useGoogle = false
+) {
+  return new Promise((resolve, reject) => {
+    try {
+      const busboy = Busboy({ headers: req.headers });
+      let requestId = uuidv4();
+      let body = {};
+
+      busboy.on("field", (fieldname, value) => {
+        if (fieldname === "requestId") {
+          requestId = value;
+        } else if (fieldname === "useGoogle") {
+          useGoogle = value;
+        }
+      });
+
+      busboy.on("file", async (fieldname, file, info) => {
+        //do not use google if file is not image or video
+        const ext = path.extname(info.filename).toLowerCase();
+        const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext);
+        if(!canUseGoogle) {
+          useGoogle = false;
+        }
 
-    return result
-}
+        //check if useGoogle is set but no gcs and warn
+        if(useGoogle && useGoogle !== "false" && !gcs) {
+          context.log.warn("Google Cloud Storage is not initialized reverting google upload ");
+          useGoogle = false;
+        }
+
+        if (saveToLocal) {
+          // Create the target folder if it doesn't exist
+          const localPath = join(publicFolder, requestId);
+          fs.mkdirSync(localPath, { recursive: true });
+
+          const filename = `${uuidv4()}_${info.filename}`;
+          const destinationPath = `${localPath}/${filename}`;
+
+          await pipeline(file, fs.createWriteStream(destinationPath));
+
+          const message = `File '${filename}' saved to folder successfully.`;
+          context.log(message);
+
+          const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
+
+          body = { message, url };
+
+          resolve(body); // Resolve the promise
+        } else {
+          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+          const { containerClient } = await getBlobClient();
+
+          const contentType = mime.lookup(filename);  // content type based on file extension
+          const options = {};
+          if (contentType) {
+            options.blobHTTPHeaders = { blobContentType: contentType };
+          }
 
-async function uploadBlob(context, req, saveToLocal = false) {
-    return new Promise((resolve, reject) => {
-        try {
-            const busboy = Busboy({ headers: req.headers });
-            let requestId = uuidv4();
-
-            busboy.on('field', (fieldname, value) => {
-                if (fieldname === 'requestId') {
-                    requestId = value;
-                }
-            });
-
-            busboy.on('file', async (fieldname, file, info) => {
-                if (saveToLocal) {
-                    // Create the target folder if it doesn't exist
-                    const localPath = join(publicFolder, requestId);
-                    fs.mkdirSync(localPath, { recursive: true });
-
-                    const filename = `${uuidv4()}_${info.filename}`;
-                    const destinationPath = `${localPath}/${filename}`;
-
-                    await pipeline(file, fs.createWriteStream(destinationPath));
-
-                    const message = `File '${filename}' saved to folder successfully.`;
-                    context.log(message);
-
-                    const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
-
-                    const body = { message, url };
-
-                    context.res = {
-                        status: 200,
-                        body,
-                    };
-
-
-                    resolve(body); // Resolve the promise
-                } else {
-                    const { containerClient } = getBlobClient();
-                    const filename = `${requestId}/${uuidv4()}_${info.filename}`;
-
-                    const blockBlobClient = containerClient.getBlockBlobClient(filename);
-
-                    const passThroughStream = new PassThrough();
-                    file.pipe(passThroughStream);
-
-                    await blockBlobClient.uploadStream(passThroughStream);
-
-                    const message = `File '${filename}' uploaded successfully.`;
-                    const url = blockBlobClient.url;
-                    context.log(message);
-                    const body = { message, url };
-
-                    context.res = {
-                        status: 200,
-                        body,
-                    };
-
-                    resolve(body); // Resolve the promise
-                }
-            });
-
-            busboy.on('error', (error) => {
-                context.log.error('Error processing file upload:', error);
-                context.res = {
-                    status: 500,
-                    body: 'Error processing file upload.',
-                };
-                reject(error); // Reject the promise
-            });
-
-            req.pipe(busboy);
-        } catch (error) {
-            context.log.error('Error processing file upload:', error);
-            context.res = {
-                status: 500,
-                body: 'Error processing file upload.',
-            };
-            reject(error); // Reject the promise
+          const blockBlobClient = containerClient.getBlockBlobClient(filename);
+
+          const passThroughStream = new PassThrough();
+          file.pipe(passThroughStream);
+
+          await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
+
+          const message = `File '${filename}' uploaded successfully.`;
+          const url = blockBlobClient.url;
+          context.log(message);
+          body = { message, url };
         }
-    });
+
+        context.res = {
+          status: 200,
+          body,
+        };
+
+        if (useGoogle && useGoogle !== "false") {
+          const { url } = body;
+          const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+          const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(filename);
+          const writeStream = gcsFile.createWriteStream();
+
+          const response = await axios({
+            method: "get",
+            url: url,
+            responseType: "stream",
+          });
+
+          // Pipe the Axios response stream directly into the GCS Write Stream
+          response.data.pipe(writeStream);
+
+          await new Promise((resolve, reject) => {
+            writeStream.on("finish", resolve);
+            writeStream.on("error", reject);
+          });
+
+          body.gcs = `gs://${GCS_BUCKETNAME}/${filename}`;
+        }
+
+        resolve(body); // Resolve the promise
+      });
+
+      busboy.on("error", (error) => {
+        context.log.error("Error processing file upload:", error);
+        context.res = {
+          status: 500,
+          body: "Error processing file upload.",
+        };
+        reject(error); // Reject the promise
+      });
+
+      req.pipe(busboy);
+    } catch (error) {
+      context.log.error("Error processing file upload:", error);
+      context.res = {
+        status: 500,
+        body: "Error processing file upload.",
+      };
+      reject(error); // Reject the promise
+    }
+  });
 }
 
 // Function to delete files that haven't been used in more than a month
-async function cleanup() {
-    const { containerClient } = getBlobClient();
-  
-    // List all the blobs in the container
-    const blobs = containerClient.listBlobsFlat();
-  
-    // Calculate the date that is x month ago
+async function cleanup(urls=null) {
+  const { containerClient } = await getBlobClient();
+
+  if(!urls) {
     const xMonthAgo = new Date();
     xMonthAgo.setMonth(xMonthAgo.getMonth() - 1);
-  
-    // Iterate through the blobs
+
+    const blobs = containerClient.listBlobsFlat();
+    const cleanedURLs = [];
+    
     for await (const blob of blobs) {
-      // Get the last modified date of the blob
       const lastModified = blob.properties.lastModified;
-  
-      // Compare the last modified date with one month ago
       if (lastModified < xMonthAgo) {
-        // Delete the blob
         const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
         await blockBlobClient.delete();
         console.log(`Cleaned blob: ${blob.name}`);
+        cleanedURLs.push(blob.name);
+      }
+    }
+    
+    return cleanedURLs;
+  }else{
+    // Delete the blobs with the specified URLs 
+    const cleanedURLs = [];
+    for(const url of urls) {
+      // Remove the base url to get the blob name
+      const blobName = url.replace(containerClient.url, '');
+      const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+      await blockBlobClient.delete();
+      console.log(`Cleaned blob: ${blobName}`);
+      cleanedURLs.push(blobName);
+    }
+    return cleanedURLs;
+  }
+}
+
+async function cleanupGCS(urls=null) {
+  const bucket = gcs.bucket(GCS_BUCKETNAME);
+  const directories = new Set();
+  const cleanedURLs = [];
+
+  if(!urls){
+    const daysN = 30;
+    const thirtyDaysAgo = new Date(Date.now() - daysN * 24 * 60 * 60 * 1000);
+    const [files] = await bucket.getFiles();
+
+    for (const file of files) {
+      const [metadata] = await file.getMetadata();
+      const directoryPath = path.dirname(file.name);
+      directories.add(directoryPath);
+      if (metadata.updated) {
+        const updatedTime = new Date(metadata.updated);
+        if (updatedTime.getTime() < thirtyDaysAgo.getTime()) {
+          console.log(`Cleaning file: ${file.name}`);
+          await file.delete();
+          cleanedURLs.push(file.name);
+        }
+      }
+    }
+  }else{
+    try {
+      for(const url of urls) {
+        const filename = path.join(url.split('/').slice(3).join('/'));
+        const file = bucket.file(filename);
+        const directoryPath = path.dirname(file.name);
+        directories.add(directoryPath);
+        await file.delete();
+        cleanedURLs.push(url);
       }
+    }catch(error){
+      console.error(`Error cleaning up files: ${error}`);
     }
+  }
+
+  for (const directory of directories) {
+    const [files] = await bucket.getFiles({ prefix: directory });
+    if (files.length === 0) {
+      console.log(`Deleting empty directory: ${directory}`);
+      await bucket.deleteFiles({ prefix: directory });
+    }
+  }
+
+  return cleanedURLs;
 }
 
-export {
-    saveFileToBlob, deleteBlob, uploadBlob, cleanup
-}
+export { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS };

package/helper-apps/cortex-file-handler/index.js

@@ -1,6 +1,6 @@
 import { downloadFile, processYoutubeUrl, splitMediaFile } from './fileChunker.js';
-import { saveFileToBlob, deleteBlob, uploadBlob, cleanup } from './blobHandler.js';
-import { publishRequestProgress } from './redis.js';
+import { saveFileToBlob, deleteBlob, uploadBlob, cleanup, cleanupGCS } from './blobHandler.js';
+import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
 import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
 import { documentToText, easyChunker } from './docHelper.js';
@@ -8,6 +8,8 @@ import path from 'path';
 import os from 'os';
 import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
+import http from 'http';
+import https from 'https';
 
 const DOC_EXTENSIONS =  [".txt", ".json", ".csv", ".md", ".xml", ".js", ".html", ".css", '.pdf', '.docx', '.xlsx', '.csv'];
 
@@ -16,15 +18,58 @@ console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
 
 
 let isCleanupRunning = false;
-async function cleanupInactive(useAzure) {
+async function cleanupInactive() {
     try {
         if (isCleanupRunning) { return; } //no need to cleanup every call
         isCleanupRunning = true;
-        if (useAzure) {
-            await cleanup();
-        } else {
-            await cleanupLocal();
+        const cleaned = await cleanupRedisFileStoreMap();
+
+        const cleanedAzure = [];
+        const cleanedLocal = [];
+        const cleanedGCS = [];
+
+        for(const key in cleaned){
+            const item = cleaned[key];
+            const {url,gcs} = item;
+            if(url){
+                if(url.includes('.blob.core.windows.net/')){ 
+                    cleanedAzure.push(url);
+                }else if(url.startsWith('gs://')){
+                    cleanedGCS.push(url);
+                }else{
+                    cleanedLocal.push(url);
+                }
+            }
+
+            if(item && item.gcs){
+                cleanedGCS.push(gcs);
+            }
+        }
+        
+        try {
+            if (cleanedAzure && cleanedAzure.length > 0) {
+                await cleanup(cleanedAzure);
+            }
+        } catch (error) {
+            console.log('Error occurred during azure cleanup:', error);
+        }
+
+        try {
+            if (cleanedLocal && cleanedLocal.length > 0) {
+                await cleanupLocal(cleanedLocal);
+            }
+        }catch(err){
+            console.log('Error occurred during local cleanup:', err);
+        }
+
+        try{
+            if(cleanedGCS && cleanedGCS.length > 0){
+                await cleanupGCS(cleanedGCS);
+            }
+        }catch(err){
+            console.log('Error occurred during GCS cleanup:', err);
         }
+         
     } catch (error) {
         console.log('Error occurred during cleanup:', error);
     } finally{
@@ -32,11 +77,27 @@ async function cleanupInactive(useAzure) {
     }
 }
 
+async function urlExists(url) {
+  if(!url) return false;
+  const httpModule = url.startsWith('https') ? https : http;
+  
+  return new Promise((resolve) => {
+    httpModule
+      .get(url, function (response) {
+        // Check if the response status is OK
+        resolve(response.statusCode === 200);
+      })
+      .on('error', function () {
+        resolve(false);
+      });
+  });
+}
+
   
 async function main(context, req) {
     context.log('Starting req processing..');
 
-    cleanupInactive(useAzure); //trigger & no need to wait for it
+    cleanupInactive(); //trigger & no need to wait for it
 
     // Clean up blob when request delete which means processing marked completed
     if (req.method.toLowerCase() === `delete`) {
@@ -55,13 +116,40 @@ async function main(context, req) {
         return;
     }
 
+    const { uri, requestId, save, hash, checkHash } = req.body?.params || req.query;
+
+    if(hash && checkHash){ //check if hash exists
+        context.log(`Checking hash: ${hash}`);
+        const result = await getFileStoreMap(hash);
+
+        const exists = await urlExists(result?.url);
+
+        if(!exists){
+            await removeFromFileStoreMap(hash);
+            return;
+        }
+
+        if(result){
+            context.log(`Hash exists: ${hash}`);
+            //update redis timestamp with current time
+            await setFileStoreMap(hash, result);
+        }
+        context.res = {
+            body: result
+        };
+        return;
+    }
+
     if (req.method.toLowerCase() === `post`) {
-        const { url } = await uploadBlob(context, req, !useAzure);
+        const { useGoogle } = req.body?.params || req.query;
+        const { url } = await uploadBlob(context, req, !useAzure, useGoogle);
         context.log(`File url: ${url}`);
+        if(hash && context?.res?.body){ //save hash after upload
+            await setFileStoreMap(hash, context.res.body);
+        }
         return
     }
 
-    const { uri, requestId, save } = req.body?.params || req.query;
     if (!uri || !requestId) {
         context.res = {
             status: 400,