@aj-archipelago/cortex 1.3.12 → 1.3.15

package/README.md

@@ -131,6 +131,16 @@ Cortex pathway prompt enhancements include:
 ### Pathway Development
 To add a new pathway to Cortex, you create a new JavaScript file and define the prompts, properties, and functions that implement the desired functionality. Cortex provides defaults for almost everything, so in the simplest case a pathway can really just consist of a string prompt like the spelling example above. You can then save this file in the `pathways` directory in your Cortex project and it will be picked up and made available as a GraphQL query.
 
+### Specifying a Model
+When determining which model to use for a pathway, Cortex follows this order of precedence:
+
+1. `pathway.model` - The model specified directly in the pathway definition
+2. `args.model` - The model passed in the request arguments
+3. `pathway.inputParameters.model` - The model specified in the pathway's input parameters
+4. `config.get('defaultModelName')` - The default model specified in the configuration
+
+The first valid model found in this order will be used. If none of these models are found in the configured endpoints, Cortex will log a warning and use the default model defined in the configuration.
+
 ### Prompt
 When you define a new pathway, you need to at least specify a prompt that will be passed to the model for processing. In the simplest case, a prompt is really just a string, but the prompt is polymorphic - it can be a string or an object that contains information for the model API that you wish to call. Prompts can also be an array of strings or an array of objects for sequential operations. In this way Cortex aims to support the most simple to advanced prompting scenarios.
 
@@ -577,7 +587,7 @@ The following properties can be configured through environment variables or the
 - `subscriptionKeepAlive`: Keep-alive time for subscriptions in seconds. Default is 0.
 
 API-specific configuration:
-- `azureVideoTranslationApiUrl`: URL for Azure video translation API. Default is 'http://127.0.0.1:5005'.
+- `azureVideoTranslationApiKey`: API key for Azure video translation API. Default is null.
 - `dalleImageApiUrl`: URL for DALL-E image API. Default is 'null'.
 - `neuralSpaceApiKey`: API key for NeuralSpace services. Default is null.
 - `whisperMediaApiUrl`: URL for Whisper media API. Default is 'null'.

package/config.js

@@ -271,18 +271,19 @@ var config = convict({
             },
             "azure-video-translate": {
                 "type": "AZURE-VIDEO-TRANSLATE",
+                "url": "https://eastus.api.cognitive.microsoft.com/videotranslation",
                 "headers": {
                     "Content-Type": "application/json"
                 },
-                "supportsStreaming": true,
             }
         },
         env: 'CORTEX_MODELS'
     },
-    azureVideoTranslationApiUrl: {
+    azureVideoTranslationApiKey: {
         format: String,
-        default: 'http://127.0.0.1:5005',
-        env: 'AZURE_VIDEO_TRANSLATION_API_URL'
+        default: null,
+        env: 'AZURE_VIDEO_TRANSLATION_API_KEY',
+        sensitive: true
     },
     openaiApiKey: {
         format: String,

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -11,6 +11,7 @@ import { join } from "path";
 import { Storage } from "@google-cloud/storage";
 import axios from "axios";
 import { publicFolder, port, ipAddress } from "./start.js";
+// eslint-disable-next-line import/no-extraneous-dependencies
 import mime from "mime-types";
 
 function isBase64(str) {
@@ -65,6 +66,19 @@ async function gcsUrlExists(url, defaultReturn = false) {
         const bucketName = urlParts[0];
         const fileName = urlParts.slice(1).join('/');
 
+        if (process.env.STORAGE_EMULATOR_HOST) {
+            try {
+                const response = await axios.get(
+                    `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${bucketName}/o/${encodeURIComponent(fileName)}`,
+                    { validateStatus: status => status === 200 || status === 404 }
+                );
+                return response.status === 200;
+            } catch (error) {
+                console.error('Error checking emulator file:', error);
+                return false;
+            }
+        }
+
         const bucket = gcs.bucket(bucketName);
         const file = bucket.file(fileName);
 
@@ -77,7 +91,7 @@ async function gcsUrlExists(url, defaultReturn = false) {
     }
 }
 
-const getBlobClient = async () => {
+export const getBlobClient = async () => {
   const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
   const containerName = AZURE_STORAGE_CONTAINER_NAME;
   if (!connectionString || !containerName) {
@@ -159,102 +173,104 @@ async function deleteBlob(requestId) {
   return result;
 }
 
-async function uploadBlob(context, req, saveToLocal = false, filePath=null, hash=null) {
-  return new Promise(async (resolve, reject) => {
-    try {
-      let requestId = uuidv4();
-      let body = {};
-
-      // If filePath is given, we are dealing with local file and not form-data
-      if (filePath) {
-        const file = fs.createReadStream(filePath);
-        const filename = path.basename(filePath);
-        try {
-          const result = await uploadFile(context, requestId, body, saveToLocal, file, filename, resolve, hash);
-          resolve(result);
-        } catch (error) {
-          const err = new Error("Error processing file upload.");
-          err.status = 500;
-          throw err;
-        }
-      } else {
-        // Otherwise, continue working with form-data
-        const busboy = Busboy({ headers: req.headers });
-        let hasFile = false;
-        let errorOccurred = false;
-      
-        busboy.on("field", (fieldname, value) => {
-          if (fieldname === "requestId") {
-            requestId = value;
+function uploadBlob(context, req, saveToLocal = false, filePath=null, hash=null) {
+  return new Promise((resolve, reject) => {
+    (async () => {
+      try {
+        let requestId = uuidv4();
+        let body = {};
+
+        // If filePath is given, we are dealing with local file and not form-data
+        if (filePath) {
+          const file = fs.createReadStream(filePath);
+          const filename = path.basename(filePath);
+          try {
+            const result = await uploadFile(context, requestId, body, saveToLocal, file, filename, resolve, hash);
+            resolve(result);
+          } catch (error) {
+            const err = new Error("Error processing file upload.");
+            err.status = 500;
+            throw err;
           }
-        });
+        } else {
+          // Otherwise, continue working with form-data
+          const busboy = Busboy({ headers: req.headers });
+          let hasFile = false;
+          let errorOccurred = false;
+        
+          busboy.on("field", (fieldname, value) => {
+            if (fieldname === "requestId") {
+              requestId = value;
+            }
+          });
 
-        busboy.on("file", async (fieldname, file, filename) => {
-          if (errorOccurred) return;
-          hasFile = true;
-          uploadFile(context, requestId, body, saveToLocal, file, filename?.filename || filename, resolve, hash).catch(error => {
+          busboy.on("file", async (fieldname, file, filename) => {
+            if (errorOccurred) return;
+            hasFile = true;
+            uploadFile(context, requestId, body, saveToLocal, file, filename?.filename || filename, resolve, hash).catch(_error => {
+              if (errorOccurred) return;
+              errorOccurred = true;
+              const err = new Error("Error processing file upload.");
+              err.status = 500;
+              reject(err);
+            });
+          });
+
+          busboy.on("error", (_error) => {
             if (errorOccurred) return;
             errorOccurred = true;
-            const err = new Error("Error processing file upload.");
-            err.status = 500;
+            const err = new Error("No file provided in request");
+            err.status = 400;
             reject(err);
           });
-        });
-
-        busboy.on("error", (error) => {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
-        });
-
-        busboy.on("finish", () => {
-          if (errorOccurred) return;
-          if (!hasFile) {
+
+          busboy.on("finish", () => {
+            if (errorOccurred) return;
+            if (!hasFile) {
+              errorOccurred = true;
+              const err = new Error("No file provided in request");
+              err.status = 400;
+              reject(err);
+            }
+          });
+
+          // Handle errors from piping the request
+          req.on('error', (error) => {
+            if (errorOccurred) return;
             errorOccurred = true;
+            // Only log unexpected errors
+            if (error.message !== "No file provided in request") {
+              context.log("Error in request stream:", error);
+            }
             const err = new Error("No file provided in request");
             err.status = 400;
             reject(err);
-          }
-        });
-
-        // Handle errors from piping the request
-        req.on('error', (error) => {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          // Only log unexpected errors
-          if (error.message !== "No file provided in request") {
-            context.log("Error in request stream:", error);
-          }
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
-        });
+          });
 
-        try {
-          req.pipe(busboy);
-        } catch (error) {
-          if (errorOccurred) return;
-          errorOccurred = true;
-          // Only log unexpected errors
-          if (error.message !== "No file provided in request") {
-            context.log("Error piping request to busboy:", error);
+          try {
+            req.pipe(busboy);
+          } catch (error) {
+            if (errorOccurred) return;
+            errorOccurred = true;
+            // Only log unexpected errors
+            if (error.message !== "No file provided in request") {
+              context.log("Error piping request to busboy:", error);
+            }
+            const err = new Error("No file provided in request");
+            err.status = 400;
+            reject(err);
           }
-          const err = new Error("No file provided in request");
-          err.status = 400;
-          reject(err);
         }
+      } catch (error) {
+        // Only log unexpected errors
+        if (error.message !== "No file provided in request") {
+          context.log("Error processing file upload:", error);
+        }
+        const err = new Error(error.message || "Error processing file upload.");
+        err.status = error.status || 500;
+        reject(err);
       }
-    } catch (error) {
-      // Only log unexpected errors
-      if (error.message !== "No file provided in request") {
-        context.log("Error processing file upload:", error);
-      }
-      const err = new Error(error.message || "Error processing file upload.");
-      err.status = error.status || 500;
-      reject(err);
-    }
+    })();
   });
 }
 
@@ -511,7 +527,16 @@ async function ensureGCSUpload(context, existingFile) {
   if (!existingFile.gcs && gcs) {
     context.log(`GCS file was missing - uploading.`);
     const encodedFilename = path.basename(existingFile.url.split('?')[0]);
-    existingFile.gcs = await uploadToGCS(context, existingFile.url, encodedFilename);
+    
+    // Download the file from Azure/local storage
+    const response = await axios({
+      method: 'get',
+      url: existingFile.url,
+      responseType: 'stream'
+    });
+    
+    // Upload the file stream to GCS
+    existingFile.gcs = await uploadToGCS(context, response.data, encodedFilename);
   }
   return existingFile;
 }

package/helper-apps/cortex-file-handler/constants.js

@@ -68,6 +68,7 @@ export const ACCEPTED_MIME_TYPES = {
     'image/webp': ['.webp'],
     'image/heic': ['.heic'],
     'image/heif': ['.heif'],
+    'application/octet-stream': ['.jpg', '.jpeg', '.png', '.webp', '.heic', '.heif'],
     'application/pdf': ['.pdf'],
     
     // Audio types

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -18,8 +18,7 @@ const tempDirectories = new Map(); // dir -> { createdAt, requestId }
 
 // Temp directory cleanup
 async function cleanupTempDirectories() {
-    const tempDir = os.tmpdir();
-    
+   
     for (const [dir, info] of tempDirectories) {
         try {
             // Cleanup directories older than 1 hour

package/helper-apps/cortex-file-handler/helper.js

@@ -1,6 +1,8 @@
 import fs from 'fs';
-import { ACCEPTED_MIME_TYPES } from './constants.js';
+import { ACCEPTED_MIME_TYPES, isAcceptedMimeType } from './constants.js';
 import path from 'path';
+import http from 'http';
+import https from 'https';
 
 export async function deleteTempPath(path) {
     try {
@@ -65,3 +67,45 @@ export function ensureEncoded(url) {
         return url;
     }
 }
+
+export async function urlExists(url) {
+    if(!url) return false;
+    
+    try {
+        // Basic URL validation
+        const urlObj = new URL(url);
+        if (!['http:', 'https:'].includes(urlObj.protocol)) {
+            throw new Error('Invalid protocol - only HTTP and HTTPS are supported');
+        }
+
+        const httpModule = urlObj.protocol === 'https:' ? https : http;
+        
+        return new Promise((resolve) => {
+            const request = httpModule.request(url, { method: 'HEAD' }, function(response) {
+                if (response.statusCode >= 200 && response.statusCode < 400) {
+                    const contentType = response.headers['content-type'];
+                    const cleanContentType = contentType ? contentType.split(';')[0].trim() : '';
+                    // Check if the content type is one we accept
+                    if (cleanContentType && isAcceptedMimeType(cleanContentType)) {
+                        resolve({ valid: true, contentType: cleanContentType });
+                    } else {
+                        console.log(`Unsupported content type: ${contentType}`);
+                        resolve({ valid: false });
+                    }
+                } else {
+                    resolve({ valid: false });
+                }
+            });
+            
+            request.on('error', function(err) {
+                console.error('URL validation error:', err.message);
+                resolve({ valid: false });
+            });
+            
+            request.end();
+        });
+    } catch (error) {
+        console.error('URL validation error:', error.message);
+        return { valid: false };
+    }
+}

package/helper-apps/cortex-file-handler/index.js

@@ -1,20 +1,19 @@
 import { downloadFile, splitMediaFile } from './fileChunker.js';
 import { saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, cleanup, cleanupGCS, gcsUrlExists, ensureGCSUpload, gcs, AZURE_STORAGE_CONTAINER_NAME } from './blobHandler.js';
 import { cleanupRedisFileStoreMap, getFileStoreMap, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap } from './redis.js';
-import { ensureEncoded, ensureFileExtension } from './helper.js';
+import { ensureEncoded, ensureFileExtension, urlExists } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder, cleanupLocal } from './localFileHandler.js';
 import { documentToText, easyChunker } from './docHelper.js';
-import { DOC_EXTENSIONS, isAcceptedMimeType } from './constants.js';
+import { DOC_EXTENSIONS } from './constants.js';
 import path from 'path';
 import os from 'os';
 import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
-import http from 'http';
-import https from 'https';
 
 const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
+const useGCS = process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || process.env.GCP_SERVICE_ACCOUNT_KEY ? true : false;
 
-console.log(`Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${gcs ? ' and Google Cloud Storage' : ''}`);
+console.log(`Storage configuration - ${useAzure ? 'Azure' : 'Local'} Storage${useGCS ? ' and Google Cloud Storage' : ''}`);
 
 let isCleanupRunning = false;
 async function cleanupInactive(context) {
@@ -76,48 +75,6 @@ async function cleanupInactive(context) {
     }
 }
 
-async function urlExists(url) {
-  if(!url) return false;
-  
-  try {
-    // Basic URL validation
-    const urlObj = new URL(url);
-    if (!['http:', 'https:'].includes(urlObj.protocol)) {
-      throw new Error('Invalid protocol - only HTTP and HTTPS are supported');
-    }
-
-    const httpModule = urlObj.protocol === 'https:' ? https : http;
-    
-    return new Promise((resolve) => {
-      const request = httpModule.request(url, { method: 'HEAD' }, function(response) {
-        if (response.statusCode >= 200 && response.statusCode < 400) {
-          const contentType = response.headers['content-type'];
-          const cleanContentType = contentType ? contentType.split(';')[0].trim() : '';
-          // Check if the content type is one we accept
-          if (cleanContentType && isAcceptedMimeType(cleanContentType)) {
-            resolve({ valid: true, contentType: cleanContentType });
-          } else {
-            console.log(`Unsupported content type: ${contentType}`);
-            resolve({ valid: false });
-          }
-        } else {
-          resolve({ valid: false });
-        }
-      });
-      
-      request.on('error', function(err) {
-        console.error('URL validation error:', err.message);
-        resolve({ valid: false });
-      });
-      
-      request.end();
-    });
-  } catch (error) {
-    console.error('URL validation error:', error.message);
-    return { valid: false };
-  }
-}
-
 async function CortexFileHandler(context, req) {
     const { uri, requestId, save, hash, checkHash, clearHash, fetch, load, restore } = req.body?.params || req.query;
     const operation = save ? 'save' : 
@@ -268,24 +225,55 @@ async function CortexFileHandler(context, req) {
 
         if(hashResult){
             context.log(`File exists in map: ${hash}`);
-            const exists = await urlExists(hashResult?.url);
+            
+            // Check primary storage (Azure/Local) first
+            const primaryExists = await urlExists(hashResult?.url);
+            const gcsExists = gcs ? await gcsUrlExists(hashResult?.gcs) : false;
 
-            if(!exists.valid){
-                context.log(`File is not in storage. Removing from map: ${hash}`);
+            // If neither storage has the file, remove from map and return not found
+            if (!primaryExists.valid && !gcsExists) {
+                context.log(`File not found in any storage. Removing from map: ${hash}`);
                 await removeFromFileStoreMap(hash);
                 context.res = {
                     status: 404,
-                    body: `Hash ${hash} not found`
+                    body: `Hash ${hash} not found in storage`
                 };
                 return;
             }
 
-            if (gcs) {
-                const gcsExists = await gcsUrlExists(hashResult?.gcs);
-                if(!gcsExists){
-                    hashResult = await ensureGCSUpload(context, hashResult);
+            // If primary is missing but GCS exists, restore from GCS
+            if (!primaryExists.valid && gcsExists) {
+                context.log(`Primary storage file missing, restoring from GCS: ${hash}`);
+                try {
+                    const res = await CortexFileHandler(context, {
+                        method: 'GET',
+                        body: { params: { fetch: hashResult.gcs } }
+                    });
+                    if (res?.body?.url) {
+                        hashResult.url = res.body.url;
+                    }
+                } catch (error) {
+                    console.error('Error restoring from GCS:', error);
                 }
             }
+            // If GCS is missing but primary exists, restore to GCS
+            else if (primaryExists.valid && gcs && !gcsExists) {
+                context.log(`GCS file missing, restoring from primary: ${hash}`);
+                const { gcs: _, ...fileInfo } = hashResult; // eslint-disable-line no-unused-vars
+                hashResult = await ensureGCSUpload(context, fileInfo);
+            }
+
+            // Final check to ensure we have at least one valid storage location
+            const finalPrimaryCheck = await urlExists(hashResult?.url);
+            if (!finalPrimaryCheck.valid && !await gcsUrlExists(hashResult?.gcs)) {
+                context.log(`Failed to restore file. Removing from map: ${hash}`);
+                await removeFromFileStoreMap(hash);
+                context.res = {
+                    status: 404,
+                    body: `Hash ${hash} not found and restoration failed`
+                };
+                return;
+            }
 
             //update redis timestamp with current time
             await setFileStoreMap(hash, hashResult);

package/helper-apps/cortex-file-handler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {
@@ -43,6 +43,7 @@
     "timeout": "1m",
     "nodeArguments": [
       "--experimental-modules"
-    ]
+    ],
+    "serial": true
   }
 }

package/helper-apps/cortex-file-handler/scripts/test-azure.sh

@@ -18,7 +18,7 @@ node scripts/setup-azure-container.js
 
 # Run the tests
 echo "Running tests..."
-node -r dotenv/config node_modules/ava/entrypoints/cli.mjs
+node -r dotenv/config node_modules/ava/entrypoints/cli.mjs "$@"
 
 # Store test result
 TEST_RESULT=$?

package/helper-apps/cortex-file-handler/start.js

@@ -3,6 +3,7 @@ import express from "express";
 import { fileURLToPath } from 'url';
 import { dirname, join } from 'path';
 import cors from 'cors';
+import { readFileSync } from 'fs';
 
 import { publicIpv4 } from 'public-ip';
 const ipAddress = await publicIpv4();
@@ -11,10 +12,22 @@ const app = express();
 const port = process.env.PORT || 7071;
 const publicFolder = join(dirname(fileURLToPath(import.meta.url)), 'files');
 
+// Get version from package.json
+const packageJson = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), 'package.json'), 'utf8'));
+const version = packageJson.version;
+
 app.use(cors());
 // Serve static files from the public folder
 app.use('/files', express.static(publicFolder));
 
+// Health check endpoint
+app.get('/health', (req, res) => {
+    res.status(200).json({
+        status: 'healthy',
+        version: version
+    });
+});
+
 // New primary endpoint
 app.all('/api/CortexFileHandler', async (req, res) => {
     const context = { req, res, log: console.log }
@@ -44,7 +57,7 @@ app.all('/api/MediaFileChunker', async (req, res) => {
 });
 
 app.listen(port, () => {
-    console.log(`Cortex File Handler running on port ${port} (includes legacy MediaFileChunker endpoint)`);
+    console.log(`Cortex File Handler v${version} running on port ${port} (includes legacy MediaFileChunker endpoint)`);
 });
 
 export { port, publicFolder, ipAddress };