@aj-archipelago/cortex 1.3.12 → 1.3.14

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -11,6 +11,7 @@ import { join } from "path";
 import { Storage } from "@google-cloud/storage";
 import axios from "axios";
 import { publicFolder, port, ipAddress } from "./start.js";
+// eslint-disable-next-line import/no-extraneous-dependencies
 import mime from "mime-types";
 
 function isBase64(str) {
@@ -65,6 +66,19 @@ async function gcsUrlExists(url, defaultReturn = false) {
         const bucketName = urlParts[0];
         const fileName = urlParts.slice(1).join('/');
 
+        if (process.env.STORAGE_EMULATOR_HOST) {
+            try {
+                const response = await axios.get(
+                    `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucketName}/o/${encodeURIComponent(fileName)}`,
+                    { validateStatus: status => status === 200 || status === 404 }
+                );
+                return response.status === 200;
+            } catch (error) {
+                console.error('Error checking emulator file:', error);
+                return false;
+            }
+        }
+
         const bucket = gcs.bucket(bucketName);
         const file = bucket.file(fileName);
 
@@ -77,7 +91,7 @@ async function gcsUrlExists(url, defaultReturn = false) {
     }
 }
 
-const getBlobClient = async () => {
+export const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = AZURE_STORAGE_CONTAINER_NAME;
   if (!connectionString || !containerName) {
@@ -159,102 +173,104 @@ async function deleteBlob(requestId) {
   return result;
 }
 
-async function uploadBlob(context, req, saveToLocal = false, filePath=null, hash=null) {
-  return new Promise(async (resolve, reject) => {
-    try {
-      let requestId = uuidv4();
-      let body = {};
-
-      // If filePath is given, we are dealing with local file and not form-data
-      if (filePath) {
-        const file = fs.createReadStream(filePath);
-        const filename = path.basename(filePath);
-        try {
-          const result = await uploadFile(context, requestId, body, saveToLocal, file, filename, resolve, hash);
-          resolve(result);
-        } catch (error) {
-          const err = new Error("Error processing file upload.");
-          err.status = 500;
-          throw err;
-        }
-      } else {
-        // Otherwise, continue working with form-data
-        const busboy = Busboy({ headers: req.headers });
-        let hasFile = false;
-        let errorOccurred = false;
-      
-        busboy.on("field", (fieldname, value) => {
-          if (fieldname === "requestId") {
-            requestId = value;
+function uploadBlob(context, req, saveToLocal = false, filePath=null, hash=null) {
+  return new Promise((resolve, reject) => {
+    (async () => {
+      try {
+        let requestId = uuidv4();
+        let body = {};
+
+        // If filePath is given, we are dealing with local file and not form-data
+        if (filePath) {
+          const file = fs.createReadStream(filePath);
+          const filename = path.basename(filePath);
+          try {
+            const result = await uploadFile(context, requestId, body, saveToLocal, file, filename, resolve, hash);
+            resolve(result);
+          } catch (error) {
+            const err = new Error("Error processing file upload.");
+            err.status = 500;
+            throw err;
           }
-        });
+        } else {
+          // Otherwise, continue working with form-data
+          const busboy = Busboy({ headers: req.headers });
+          let hasFile = false;
+          let errorOccurred = false;
+        
+          busboy.on("field", (fieldname, value) => {
+            if (fieldname === "requestId") {
+              requestId = value;
+            }
+          });
 
-        busboy.on("file", async (fieldname, file, filename) => {
-          if (errorOccurred) return;
-          hasFile = true;
-          uploadFile(context, requestId, body, saveToLocal, file, filename?.filename || filename, resolve, hash).catch(error => {
+          busboy.on("file", async (fieldname, file, filename) => {
+            if (errorOccurred) return;
+            hasFile = true;
+            uploadFile(context, requestId, body, saveToLocal, file, filename?.filename || filename, resolve, hash).catch(_error => {
+              if (errorOccurred) return;
+              errorOccurred = true;
+              const err = new Error("Error processing file upload.");
+              err.status = 500;
+              reject(err);
+            });
+          });
+
+          busboy.on("error", (_error) => {
             if (errorOccurred) return;
             errorOccurred = true;
-            const err = new Error("Error processing file upload.");
-            err.status = 500;
+            const err = new Error("No file provided in request");
+            err.status = 400;
             reject(err);
           });
-        });
-
-        busboy.on("error", (error) => {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
-        });
-
-        busboy.on("finish", () => {
-          if (errorOccurred) return;
-          if (!hasFile) {
+
+          busboy.on("finish", () => {
+            if (errorOccurred) return;
+            if (!hasFile) {
+              errorOccurred = true;
+              const err = new Error("No file provided in request");
+              err.status = 400;
+              reject(err);
+            }
+          });
+
+          // Handle errors from piping the request
+          req.on('error', (error) => {
+            if (errorOccurred) return;
             errorOccurred = true;
+            // Only log unexpected errors
+            if (error.message !== "No file provided in request") {
+              context.log("Error in request stream:", error);
+            }
             const err = new Error("No file provided in request");
             err.status = 400;
             reject(err);
-          }
-        });
-
-        // Handle errors from piping the request
-        req.on('error', (error) => {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          // Only log unexpected errors
-          if (error.message !== "No file provided in request") {
-            context.log("Error in request stream:", error);
-          }
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
-        });
+          });
 
-        try {
-          req.pipe(busboy);
-        } catch (error) {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          // Only log unexpected errors
-          if (error.message !== "No file provided in request") {
-            context.log("Error piping request to busboy:", error);
+          try {
+            req.pipe(busboy);
+          } catch (error) {
+            if (errorOccurred) return;
+            errorOccurred = true;
+            // Only log unexpected errors
+            if (error.message !== "No file provided in request") {
+              context.log("Error piping request to busboy:", error);
+            }
+            const err = new Error("No file provided in request");
+            err.status = 400;
+            reject(err);
           }
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
         }
+      } catch (error) {
+        // Only log unexpected errors
+        if (error.message !== "No file provided in request") {
+          context.log("Error processing file upload:", error);
+        }
+        const err = new Error(error.message || "Error processing file upload.");
+        err.status = error.status || 500;
+        reject(err);
       }
-    } catch (error) {
-      // Only log unexpected errors
-      if (error.message !== "No file provided in request") {
-        context.log("Error processing file upload:", error);
-      }
-      const err = new Error(error.message || "Error processing file upload.");
-      err.status = error.status || 500;
-      reject(err);
-    }
+    })();
   });
 }
 
@@ -511,7 +527,16 @@ async function ensureGCSUpload(context, existingFile) {
   if (!existingFile.gcs && gcs) {
     context.log(`GCS file was missing - uploading.`);
     const encodedFilename = path.basename(existingFile.url.split('?')[0]);
-    existingFile.gcs = await uploadToGCS(context, existingFile.url, encodedFilename);
+    
+    // Download the file from Azure/local storage
+    const response = await axios({
+      method: 'get',
+      url: existingFile.url,
+      responseType: 'stream'
+    });
+    
+    // Upload the file stream to GCS
+    existingFile.gcs = await uploadToGCS(context, response.data, encodedFilename);
   }
   return existingFile;
 }

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -18,8 +18,7 @@ const tempDirectories = new Map(); // dir -> { createdAt, requestId }
 
 // Temp directory cleanup
 async function cleanupTempDirectories() {
-    const tempDir = os.tmpdir();
-    
+   
     for (const [dir, info] of tempDirectories) {
         try {
             // Cleanup directories older than 1 hour

package/helper-apps/cortex-file-handler/helper.js

@@ -1,6 +1,8 @@
 import fs from 'fs';
-import { ACCEPTED_MIME_TYPES } from './constants.js';
+import { ACCEPTED_MIME_TYPES, isAcceptedMimeType } from './constants.js';
 import path from 'path';
+import http from 'http';
+import https from 'https';
 
 export async function deleteTempPath(path) {
     try {
@@ -65,3 +67,45 @@ export function ensureEncoded(url) {
         return url;
     }
 }
+
+export async function urlExists(url) {
+    if(!url) return false;
+    
+    try {
+        // Basic URL validation
+        const urlObj = new URL(url);
+        if (!['http:', 'https:'].includes(urlObj.protocol)) {
+            throw new Error('Invalid protocol - only HTTP and HTTPS are supported');
+        }
+
+        const httpModule = urlObj.protocol === 'https:' ? https : http;
+        
+        return new Promise((resolve) => {
+            const request = httpModule.request(url, { method: 'HEAD' }, function(response) {
+                if (response.statusCode >= 200 && response.statusCode < 400) {
+                    const contentType = response.headers['content-type'];
+                    const cleanContentType = contentType ? contentType.split(';')[0].trim() : '';
+                    // Check if the content type is one we accept
+                    if (cleanContentType && isAcceptedMimeType(cleanContentType)) {
+                        resolve({ valid: true, contentType: cleanContentType });
+                    } else {
+                        console.log(`Unsupported content type: ${contentType}`);
+                        resolve({ valid: false });
+                    }
+                } else {
+                    resolve({ valid: false });
+                }
+            });
+            
+            request.on('error', function(err) {
+                console.error('URL validation error:', err.message);
+                resolve({ valid: false });
+            });
+            
+            request.end();
+        });
+    } catch (error) {
+        console.error('URL validation error:', error.message);
+        return { valid: false };
+    }
+}

package/helper-apps/cortex-file-handler/index.js

@@ -1,20 +1,19 @@
 import { downloadFile, splitMediaFile } from './fileChunker.js';
 import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
-import { ensureEncoded, ensureFileExtension } from './helper.js';
+import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
 import { documentToText, easyChunker } from './docHelper.js';
-import { DOC_EXTENSIONS, isAcceptedMimeType } from './constants.js';
+import { DOC_EXTENSIONS } from './constants.js';
 import path from 'path';
 import os from 'os';
 import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
-import http from 'http';
-import https from 'https';
 
 const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
+const useGCS = process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY ? true : false;
 
-console.log(`Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${gcs ? ' and Google Cloud Storage' : ''}`);
+console.log(`Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${useGCS ? ' and Google Cloud Storage' : ''}`);
 
 let isCleanupRunning = false;
 async function cleanupInactive(context) {
@@ -76,48 +75,6 @@ async function cleanupInactive(context) {
     }
 }
 
-async function urlExists(url) {
-  if(!url) return false;
-  
-  try {
-    // Basic URL validation
-    const urlObj = new URL(url);
-    if (!['http:', 'https:'].includes(urlObj.protocol)) {
-      throw new Error('Invalid protocol - only HTTP and HTTPS are supported');
-    }
-
-    const httpModule = urlObj.protocol === 'https:' ? https : http;
-    
-    return new Promise((resolve) => {
-      const request = httpModule.request(url, { method: 'HEAD' }, function(response) {
-        if (response.statusCode >= 200 && response.statusCode < 400) {
-          const contentType = response.headers['content-type'];
-          const cleanContentType = contentType ? contentType.split(';')[0].trim() : '';
-          // Check if the content type is one we accept
-          if (cleanContentType && isAcceptedMimeType(cleanContentType)) {
-            resolve({ valid: true, contentType: cleanContentType });
-          } else {
-            console.log(`Unsupported content type: ${contentType}`);
-            resolve({ valid: false });
-          }
-        } else {
-          resolve({ valid: false });
-        }
-      });
-      
-      request.on('error', function(err) {
-        console.error('URL validation error:', err.message);
-        resolve({ valid: false });
-      });
-      
-      request.end();
-    });
-  } catch (error) {
-    console.error('URL validation error:', error.message);
-    return { valid: false };
-  }
-}
-
 async function CortexFileHandler(context, req) {
     const { uri, requestId, save, hash, checkHash, clearHash, fetch, load, restore } = req.body?.params || req.query;
     const operation = save ? 'save' : 
@@ -268,24 +225,55 @@ async function CortexFileHandler(context, req) {
 
         if(hashResult){
             context.log(`File exists in map: ${hash}`);
-            const exists = await urlExists(hashResult?.url);
+            
+            // Check primary storage (Azure/Local) first
+            const primaryExists = await urlExists(hashResult?.url);
+            const gcsExists = gcs ? await gcsUrlExists(hashResult?.gcs) : false;
 
-            if(!exists.valid){
-                context.log(`File is not in storage. Removing from map: ${hash}`);
+            // If neither storage has the file, remove from map and return not found
+            if (!primaryExists.valid && !gcsExists) {
+                context.log(`File not found in any storage. Removing from map: ${hash}`);
                 await removeFromFileStoreMap(hash);
                 context.res = {
                     status: 404,
-                    body: `Hash ${hash} not found`
+                    body: `Hash ${hash} not found in storage`
                 };
                 return;
             }
 
-            if (gcs) {
-                const gcsExists = await gcsUrlExists(hashResult?.gcs);
-                if(!gcsExists){
-                    hashResult = await ensureGCSUpload(context, hashResult);
+            // If primary is missing but GCS exists, restore from GCS
+            if (!primaryExists.valid && gcsExists) {
+                context.log(`Primary storage file missing, restoring from GCS: ${hash}`);
+                try {
+                    const res = await CortexFileHandler(context, {
+                        method: 'GET',
+                        body: { params: { fetch: hashResult.gcs } }
+                    });
+                    if (res?.body?.url) {
+                        hashResult.url = res.body.url;
+                    }
+                } catch (error) {
+                    console.error('Error restoring from GCS:', error);
                 }
             }
+            // If GCS is missing but primary exists, restore to GCS
+            else if (primaryExists.valid && gcs && !gcsExists) {
+                context.log(`GCS file missing, restoring from primary: ${hash}`);
+                const { gcs: _, ...fileInfo } = hashResult; // eslint-disable-line no-unused-vars
+                hashResult = await ensureGCSUpload(context, fileInfo);
+            }
+
+            // Final check to ensure we have at least one valid storage location
+            const finalPrimaryCheck = await urlExists(hashResult?.url);
+            if (!finalPrimaryCheck.valid && !await gcsUrlExists(hashResult?.gcs)) {
+                context.log(`Failed to restore file. Removing from map: ${hash}`);
+                await removeFromFileStoreMap(hash);
+                context.res = {
+                    status: 404,
+                    body: `Hash ${hash} not found and restoration failed`
+                };
+                return;
+            }
 
             //update redis timestamp with current time
             await setFileStoreMap(hash, hashResult);

package/helper-apps/cortex-file-handler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {
@@ -43,6 +43,7 @@
     "timeout": "1m",
     "nodeArguments": [
       "--experimental-modules"
-    ]
+    ],
+    "serial": true
   }
 }

package/helper-apps/cortex-file-handler/scripts/test-azure.sh

@@ -18,7 +18,7 @@ node scripts/setup-azure-container.js
 
 # Run the tests
 echo "Running tests..."
-node -r dotenv/config node_modules/ava/entrypoints/cli.mjs
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs "$@"
 
 # Store test result
 TEST_RESULT=$?

package/helper-apps/cortex-file-handler/start.js

@@ -3,6 +3,7 @@ import express from "express";
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import cors from 'cors';
+import { readFileSync } from 'fs';
 
 import { publicIpv4 } from 'public-ip';
 const ipAddress = await publicIpv4();
@@ -11,10 +12,22 @@ const app = express();
 const port = process.env.PORT || 7071;
 const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
 
+// Get version from package.json
+const packageJson = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'package.json'), 'utf8'));
+const version = packageJson.version;
+
 app.use(cors());
 // Serve static files from the public folder
 app.use('/files', express.static(publicFolder));
 
+// Health check endpoint
+app.get('/health', (req, res) => {
+    res.status(200).json({
+        status: 'healthy',
+        version: version
+    });
+});
+
 // New primary endpoint
 app.all('/api/CortexFileHandler', async (req, res) => {
     const context = { req, res, log: console.log }
@@ -44,7 +57,7 @@ app.all('/api/MediaFileChunker', async (req, res) => {
 });
 
 app.listen(port, () => {
-    console.log(`Cortex File Handler running on port ${port} (includes legacy MediaFileChunker endpoint)`);
+    console.log(`Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`);
 });
 
 export { port, publicFolder, ipAddress };

package/helper-apps/cortex-file-handler/tests/blobHandler.test.js

@@ -0,0 +1,292 @@
+import test from 'ava';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { uploadBlob, ensureGCSUpload, gcsUrlExists, deleteGCS, getBlobClient } from '../blobHandler.js';
+import axios from 'axios';
+import CortexFileHandler from '../index.js';
+import { setFileStoreMap } from '../redis.js';
+import { urlExists } from '../helper.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Helper function to determine if GCS is configured
+function isGCSConfigured() {
+    return process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY;
+}
+
+// Helper function to check file size in GCS
+async function getGCSFileSize(gcsUrl) {
+    if (!isGCSConfigured()) return null;
+    try {
+        const bucket = gcsUrl.split('/')[2];
+        const filename = gcsUrl.split('/').slice(3).join('/');
+
+        if (process.env.STORAGE_EMULATOR_HOST) {
+            const response = await axios.get(
+                `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucket}/o/${encodeURIComponent(filename)}`,
+                { validateStatus: status => status === 200 || status === 404 }
+            );
+            if (response.status === 200) {
+                return parseInt(response.data.size);
+            }
+            return null;
+        }
+    } catch (error) {
+        return null;
+    }
+}
+
+// Helper function to check file size in Azure/HTTP
+async function getHttpFileSize(url) {
+    try {
+        const response = await axios.head(url);
+        const contentLength = response.headers['content-length'];
+        return contentLength ? parseInt(contentLength) : null;
+    } catch (error) {
+        console.error('Error getting HTTP file size:', error);
+        return null;
+    }
+}
+
+test('test GCS backup during initial upload', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // Upload the file - should go to both Azure/local and GCS
+        const context = { log: console.log };
+        const result = await uploadBlob(context, null, false, testFile);
+        
+        // Verify we got both URLs
+        t.truthy(result.url, 'Should have primary storage URL');
+        t.truthy(result.gcs, 'Should have GCS backup URL');
+        
+        // Verify GCS file exists
+        const gcsExists = await gcsUrlExists(result.gcs);
+        t.true(gcsExists, 'File should exist in GCS');
+        
+        // Verify file content size in GCS
+        const gcsSize = await getGCSFileSize(result.gcs);
+        t.is(gcsSize, testContent.length, 'GCS file size should match original');
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test GCS backup restoration when missing', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000); // Create a decent sized file
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const result = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(result.gcs, 'Should have GCS backup URL after initial upload');
+        
+        // Delete the GCS file
+        const gcsFileName = result.gcs.replace('gs://cortextempfiles/', '');
+        await deleteGCS(gcsFileName);
+        
+        // Verify file is gone
+        const existsAfterDelete = await gcsUrlExists(result.gcs);
+        t.false(existsAfterDelete, 'File should not exist in GCS after deletion');
+        
+        // Remove GCS URL to simulate missing backup
+        const { gcs: _, ...fileInfo } = result; // eslint-disable-line no-unused-vars
+        
+        // Try to ensure GCS backup
+        const updatedResult = await ensureGCSUpload(context, fileInfo);
+        
+        // Verify GCS URL was added
+        t.truthy(updatedResult.gcs, 'Should have GCS backup URL after ensure');
+        
+        // Verify GCS file exists
+        const gcsExists = await gcsUrlExists(updatedResult.gcs);
+        t.true(gcsExists, 'File should exist in GCS after ensure');
+        
+        // Verify file content size in GCS
+        const gcsSize = await getGCSFileSize(updatedResult.gcs);
+        t.is(gcsSize, testContent.length, 'GCS file size should match original after ensure');
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test primary storage restoration from GCS backup', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000);
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const initialResult = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(initialResult.url, 'Should have primary storage URL');
+        t.truthy(initialResult.gcs, 'Should have GCS backup URL');
+        
+        // Store the hash and simulate a missing primary file by requesting with a bad URL
+        const hash = 'test_primary_restore';
+        const modifiedResult = {
+            ...initialResult,
+            url: initialResult.url.replace('.blob.core.windows.net', '.invalid.url')
+        };
+        
+        // Set up Redis state with the bad URL
+        await setFileStoreMap(hash, modifiedResult);
+        
+        // Set up request for the handler
+        const mockReq = {
+            method: 'GET',
+            body: { params: { hash, checkHash: true } }
+        };
+        
+        // Set up context for the handler
+        const handlerContext = { 
+            log: console.log,
+            res: null
+        };
+        
+        // Call the handler which should restore from GCS
+        await CortexFileHandler(handlerContext, mockReq);
+        
+        // Verify we got a valid response
+        t.is(handlerContext.res.status, 200, 'Should get successful response');
+        t.truthy(handlerContext.res.body.url, 'Should have restored primary URL');
+        t.truthy(handlerContext.res.body.gcs, 'Should still have GCS URL');
+        
+        // Verify the restored URL is accessible
+        const { valid } = await urlExists(handlerContext.res.body.url);
+        t.true(valid, 'Restored URL should be accessible');
+
+        // Verify file sizes match in both storages
+        const gcsSize = await getGCSFileSize(handlerContext.res.body.gcs);
+        const azureSize = await getHttpFileSize(handlerContext.res.body.url);
+        t.is(azureSize, testContent.length, 'Azure file size should match original');
+        t.is(gcsSize, azureSize, 'Azure and GCS file sizes should match');
+        
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+});
+
+test('test hash check returns 404 when both storages are empty', async t => {
+    if (!isGCSConfigured()) {
+        t.pass('Skipping test - GCS not configured');
+        return;
+    }
+
+    // Create a test file with known content
+    const testContent = 'Hello World!'.repeat(1000);
+    const testFile = path.join(__dirname, 'test.txt');
+    fs.writeFileSync(testFile, testContent);
+    
+    try {
+        // First upload normally
+        const context = { log: console.log };
+        const initialResult = await uploadBlob(context, null, false, testFile);
+        
+        // Verify initial upload worked
+        t.truthy(initialResult.url, 'Should have primary storage URL');
+        t.truthy(initialResult.gcs, 'Should have GCS backup URL');
+        
+        // Store the hash
+        const hash = 'test_both_missing';
+        await setFileStoreMap(hash, initialResult);
+        
+        // Verify both files exist initially
+        const initialPrimaryCheck = await urlExists(initialResult.url);
+        const initialGcsCheck = await gcsUrlExists(initialResult.gcs);
+        t.true(initialPrimaryCheck.valid, 'Primary file should exist initially');
+        t.true(initialGcsCheck, 'GCS file should exist initially');
+
+        // Delete from Azure/primary storage
+        const azureUrl = new URL(initialResult.url);
+        console.log('Azure URL:', initialResult.url);
+        // Get the path without query parameters and decode it
+        const fullPath = decodeURIComponent(azureUrl.pathname);
+        console.log('Full path:', fullPath);
+        // Get the request ID and filename from the path
+        const pathParts = fullPath.split('/');
+        const blobName = pathParts[pathParts.length - 1];
+        console.log('Attempting to delete Azure blob:', blobName);
+        
+        // Delete the blob using the correct container name
+        const { containerClient } = await getBlobClient();
+        const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+        await blockBlobClient.delete();
+        console.log('Azure deletion completed');
+
+        // Add a small delay to ensure deletion is complete
+        await new Promise(resolve => setTimeout(resolve, 1000));
+
+        // Delete from GCS
+        const gcsFileName = initialResult.gcs.replace('gs://cortextempfiles/', '');
+        console.log('Attempting to delete GCS file:', gcsFileName);
+        await deleteGCS(gcsFileName);
+        console.log('GCS deletion completed');
+        
+        // Verify both files are gone
+        const primaryExists = await urlExists(initialResult.url);
+        console.log('Primary exists after deletion:', primaryExists.valid);
+        const gcsExists = await gcsUrlExists(initialResult.gcs);
+        console.log('GCS exists after deletion:', gcsExists);
+        t.false(primaryExists.valid, 'Primary file should be deleted');
+        t.false(gcsExists, 'GCS file should be deleted');
+        
+        // Try to get the file via hash - should fail
+        const handlerContext = { 
+            log: console.log,
+            res: null
+        };
+        
+        await CortexFileHandler(handlerContext, {
+            method: 'GET',
+            body: { params: { hash, checkHash: true } }
+        });
+        
+        // Verify we got a 404 response
+        t.is(handlerContext.res.status, 404, 'Should get 404 when both files are missing');
+        t.true(handlerContext.res.body.includes('not found in storage'), 'Should indicate files are missing in storage');
+        
+    } finally {
+        // Cleanup
+        if (fs.existsSync(testFile)) {
+            fs.unlinkSync(testFile);
+        }
+    }
+}); 

package/helper-apps/cortex-file-handler/tests/fileChunker.test.js

@@ -5,7 +5,6 @@ import fs from 'fs/promises';
 import { existsSync } from 'fs';
 import { splitMediaFile, downloadFile } from '../fileChunker.js';
 import nock from 'nock';
-import path from 'path';
 import os from 'os';
 import { execSync } from 'child_process';
 import { performance } from 'perf_hooks';
@@ -94,7 +93,7 @@ test.before(async t => {
     }
 });
 
-// Cleanup: Remove test files and worker pool
+// Cleanup: Remove test files
 test.after.always(async t => {
     // Clean up test files
     if (t.context.testDir) {
@@ -108,16 +107,6 @@ test.after.always(async t => {
 
     // Clean up nock
     nock.cleanAll();
-    
-    // Clean up worker pool if it exists
-    if (typeof workerPool !== 'undefined' && Array.isArray(workerPool)) {
-        try {
-            await Promise.all(workerPool.map(worker => worker.terminate()));
-            console.log('Worker pool cleaned up successfully');
-        } catch (error) {
-            console.error('Error cleaning up worker pool:', error);
-        }
-    }
 });
 
 // Test successful chunking of a short file
@@ -250,7 +239,7 @@ function formatDuration(ms) {
 test('performance test - 1 hour file', async t => {
     const start = performance.now();
     
-    const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(t.context.testFile1h);
+    const { chunkPromises, uniqueOutputPath } = await splitMediaFile(t.context.testFile1h);
     
     // Wait for all chunks to complete
     const chunkPaths = await Promise.all(chunkPromises);
@@ -274,7 +263,7 @@ test('performance test - 1 hour file', async t => {
 test('performance test - 4 hour file', async t => {
     const start = performance.now();
     
-    const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(t.context.testFile4h);
+    const { chunkPromises, uniqueOutputPath } = await splitMediaFile(t.context.testFile4h);
     
     // Wait for all chunks to complete
     const chunkPaths = await Promise.all(chunkPromises);

package/helper-apps/cortex-file-handler/tests/start.test.js

@@ -1,5 +1,7 @@
+/* eslint-disable no-unused-vars */
 import test from 'ava';
 import axios from 'axios';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import FormData from 'form-data';
 import { port, publicFolder, ipAddress } from '../start.js';
 import { v4 as uuidv4 } from 'uuid';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.12",
+  "version": "1.3.14",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/replicateApiPlugin.js

@@ -1,6 +1,7 @@
 // replicateApiPlugin.js
 import ModelPlugin from "./modelPlugin.js";
 import logger from "../../lib/logger.js";
+import axios from "axios";
 
 class ReplicateApiPlugin extends ModelPlugin {
   constructor(pathway, model) {
@@ -106,10 +107,61 @@ class ReplicateApiPlugin extends ModelPlugin {
     cortexRequest.data = requestParameters;
     cortexRequest.params = requestParameters.params;
 
-    return this.executeRequest(cortexRequest);
+    // Make initial request to start prediction
+    const stringifiedResponse = await this.executeRequest(cortexRequest);
+    const parsedResponse = JSON.parse(stringifiedResponse);
+
+    // If we got a completed response, return it
+    if (parsedResponse?.status === "succeeded") {
+      return stringifiedResponse;
+    }
+    
+    logger.info("Replicate API returned a non-completed response.");
+
+    if (!parsedResponse?.id) {
+      throw new Error("No prediction ID returned from Replicate API");
+    }
+
+    // Get the prediction ID and polling URL
+    const predictionId = parsedResponse.id;
+    const pollUrl = parsedResponse.urls?.get;
+
+    if (!pollUrl) {
+      throw new Error("No polling URL returned from Replicate API");
+    }
+
+    // Poll for results
+    const maxAttempts = 60; // 5 minutes with 5 second intervals
+    const pollInterval = 5000;
+    
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      try {
+        const pollResponse = await axios.get(pollUrl, {
+          headers: cortexRequest.headers
+        });
+
+        logger.info("Polling Replicate API - attempt " + attempt);
+        const status = pollResponse.data?.status;
+        
+        if (status === "succeeded") {
+          logger.info("Replicate API returned a completed response after polling");
+          return JSON.stringify(pollResponse.data);
+        } else if (status === "failed" || status === "canceled") {
+          throw new Error(`Prediction ${status}: ${pollResponse.data?.error || "Unknown error"}`);
+        }
+
+        // Wait before next poll
+        await new Promise(resolve => setTimeout(resolve, pollInterval));
+      } catch (error) {
+        logger.error(`Error polling prediction ${predictionId}: ${error.message}`);
+        throw error;
+      }
+    }
+
+    throw new Error(`Prediction ${predictionId} timed out after ${maxAttempts * pollInterval / 1000} seconds`);
   }
 
-  // Parse the response from the Replicate API
+  // Stringify the response from the Replicate API
   parseResponse(data) {
     if (data.data) {
       return JSON.stringify(data.data);