@aj-archipelago/cortex 1.3.8 → 1.3.9

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -191,7 +191,7 @@ async function deleteBlob(requestId) {
   return result;
 }
 
-async function uploadBlob(context, req, saveToLocal = false, useGoogle = false, filePath=null) {
+async function uploadBlob(context, req, saveToLocal = false, useGoogle = false, filePath=null, hash=null) {
   return new Promise((resolve, reject) => {
     try {
       let requestId = uuidv4();
@@ -201,7 +201,7 @@ async function uploadBlob(context, req, saveToLocal = false, useGoogle = false,
       if (filePath) {
         const file = fs.createReadStream(filePath);
         const filename = path.basename(filePath);
-        uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve)
+        uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve, hash);
       } else {
         // Otherwise, continue working with form-data
         const busboy = Busboy({ headers: req.headers });
@@ -215,7 +215,7 @@ async function uploadBlob(context, req, saveToLocal = false, useGoogle = false,
         });
 
         busboy.on("file", async (fieldname, file, filename) => {
-          uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename?.filename || filename, resolve)
+          uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename?.filename || filename, resolve, hash);
         });
 
         busboy.on("error", (error) => {
@@ -240,7 +240,7 @@ async function uploadBlob(context, req, saveToLocal = false, useGoogle = false,
   });
 }
 
-async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve) {
+async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve, hash=null) {
   // do not use Google if the file is not an image or video
   const ext = path.extname(filename).toLowerCase();
   const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext) || AUDIO_EXTENSIONS.includes(ext);
@@ -306,16 +306,29 @@ async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file
     const { url } = body;
     const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(encodedFilename);
     const writeStream = gcsFile.createWriteStream();
-
+    
     const response = await axios({
       method: "get",
       url: url,
       responseType: "stream",
     });
-
-    // pipe the Axios response stream directly into the GCS Write Stream
+    
+    // // Get the total file size from the response headers
+    // const totalSize = Number(response.headers["content-length"]);
+    // let downloadedSize = 0;
+    
+    // // Listen to the 'data' event to track the progress
+    // response.data.on("data", (chunk) => {
+    //   downloadedSize += chunk.length;
+    
+    //   // Calculate and display the progress
+    //   const progress = (downloadedSize / totalSize) * 100;
+    //   console.log(`Progress gsc of ${encodedFilename}: ${progress.toFixed(2)}%`);
+    // });
+    
+    // Pipe the Axios response stream directly into the GCS Write Stream
     response.data.pipe(writeStream);
-
+    
     await new Promise((resolve, reject) => {
       writeStream.on("finish", resolve);
       writeStream.on("error", reject);
@@ -324,6 +337,12 @@ async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file
     body.gcs = `gs://${GCS_BUCKETNAME}/${encodedFilename}`;
   }
   
+  if(!body.filename) {
+    body.filename = filename;
+  }
+  if(hash && !body.hash) {
+    body.hash = hash;
+  }
   resolve(body); // Resolve the promise
 }
 

package/helper-apps/cortex-file-handler/index.js

@@ -120,7 +120,7 @@ async function main(context, req) {
         return;
     }
 
-    const { uri, requestId, save, hash, checkHash, fetch, load, restore } = req.body?.params || req.query;
+    const { uri, requestId, save, hash, checkHash, clearHash, fetch, load, restore } = req.body?.params || req.query;
 
     const filepond = fetch || restore || load;
     if (req.method.toLowerCase() === `get` && filepond) {
@@ -163,6 +163,24 @@ async function main(context, req) {
         return;
     }
 
+    if(hash && clearHash){
+        try {
+            const hashValue = await getFileStoreMap(hash);
+            await removeFromFileStoreMap(hash);
+            context.res = {
+                status: 200,
+                body: hashValue ? `Hash ${hash} removed` : `Hash ${hash} not found`
+            };
+        } catch (error) {
+            context.res = {
+                status: 500,
+                body: `Error occurred during hash cleanup: ${error}`
+            };
+            console.log('Error occurred during hash cleanup:', error);
+        }
+        return
+    }
+
     if(hash && checkHash){ //check if hash exists
         context.log(`Checking hash: ${hash}`);
         const result = await getFileStoreMap(hash);
@@ -188,7 +206,7 @@ async function main(context, req) {
 
     if (req.method.toLowerCase() === `post`) {
         const { useGoogle } = req.body?.params || req.query;
-        const { url } = await uploadBlob(context, req, !useAzure, useGoogle);
+        const { url } = await uploadBlob(context, req, !useAzure, useGoogle, null, hash);
         context.log(`File url: ${url}`);
         if(hash && context?.res?.body){ //save hash after upload
             await setFileStoreMap(hash, context.res.body);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.8",
+  "version": "1.3.9",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/azureVideoTranslatePlugin.js

@@ -81,17 +81,6 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
                 throw new Error(this.jsonBuffer);
             }
 
-            if (isValidJSON(this.jsonBuffer)) {
-                const parsedData = JSON.parse(this.jsonBuffer);
-                if (parsedData.progress !== undefined) {
-                    publishRequestProgress({
-                        requestId: this.requestId,
-                        progress: parsedData.progress,
-                        info: this.jsonBuffer
-                    });
-                }
-            }
-
             onData(this.jsonBuffer);
             this.jsonBuffer = '';
             this.jsonDepth = 0;
@@ -118,11 +107,34 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
                 let finalJson = '';
                 this.handleStream(response.data,
                     (data) => {
-                        publishRequestProgress({
-                            requestId: this.requestId,
-                            info: data
-                        });
+                        let sent = false;
+                        if (isValidJSON(data)) {
+                            const parsedData = JSON.parse(data);
+                            if (parsedData.progress !== undefined) {
+                                let timeInfo = '';
+                                if (parsedData.estimated_time_remaining && parsedData.elapsed_time) {
+                                    const minutes = Math.ceil(parsedData.estimated_time_remaining / 60);
+                                    timeInfo = minutes <= 2 
+                                        ? `Should be done soon (${parsedData.elapsed_time} elapsed)`
+                                        : `Estimated ${minutes} minutes remaining`;
+                                }
+
+                                publishRequestProgress({
+                                    requestId: this.requestId,
+                                    progress: parsedData.progress,
+                                    info: timeInfo
+                                });
+                                sent = true;
+                            }
+                        }
+                        if (!sent) {
+                            publishRequestProgress({
+                                requestId: this.requestId,
+                                info: data
+                            });
+                        }
                         logger.debug('Data:', data);
+                        
                         // Extract JSON content if message contains targetLocales
                         const jsonMatch = data.match(/{[\s\S]*"targetLocales"[\s\S]*}/);
                         if (jsonMatch) {