@aj-archipelago/cortex 1.3.11 → 1.3.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.11",
+  "version": "1.3.14",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/system/entity/sys_entity_continue.js

@@ -34,7 +34,7 @@ export default {
 
             const newArgs = {
                 ...args,
-                chatHistory: args.chatHistory.slice(-6)
+                chatHistory: args.chatHistory.slice(-20)
             };
 
             if (generatorPathway === 'sys_generator_document') {

package/pathways/system/entity/sys_entity_start.js

@@ -179,6 +179,7 @@ export default {
                     case "video":
                     case "audio":
                     case "pdf":
+                    case "text":
                         if (visionContentPresent) {
                             toolCallbackName = 'sys_generator_video_vision';
                             toolCallbackMessage = toolMessage;

package/pathways/system/entity/sys_generator_video_vision.js

@@ -4,7 +4,7 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou are the part of {{aiName}} that can view, hear, and understand media files of all sorts (images, videos, audio, pdfs, etc.) - you provide the capability to view and analyze media files that the user provides.\nMany of your subsystems cannot independently view or analyze media files, so make sure that you describe the details of what you see in the media files in your response so you can refer to the descriptions later. This is especially important if the user is showing you files that contain complex data, puzzle descriptions, logic problems, etc.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\nYou are the part of {{aiName}} that can view, hear, and understand files of all sorts (images, videos, audio, pdfs, text, etc.) - you provide the capability to view and analyze files that the user provides.\nMany of your subsystems cannot independently view or analyze files, so make sure that you describe the details of what you see in the files in your response so you can refer to the descriptions later. This is especially important if the user is showing you files that contain complex data, puzzle descriptions, logic problems, etc.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\nThe user has provided you with one or more files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.\n{{renderTemplate AI_DATETIME}}`},
                 "{{chatHistory}}",
             ]}),
         ],
@@ -19,6 +19,7 @@ export default {
     useInputChunking: false,
     enableDuplicateRequests: false,
     timeout: 600,
+    
     executePathway: async ({args, runAllPrompts, resolver}) => {
         const result = await runAllPrompts({ ...args });
         resolver.tool = JSON.stringify({ toolUsed: "vision" });

package/pathways/system/entity/sys_router_tool.js

@@ -33,13 +33,15 @@ Available tools and their specific use cases:
 
 8. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
 
-9. PDF: Use specifically for processing and answering questions about PDF file content.
+9. PDF: Use specifically for analyzing and answering questions about PDF file content. Use this tool any time the user is asking you questions about a PDF file.
 
-10. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
+10. Text: Use specifically for analyzing and answering questions about text file content. Use this tool any time the user is asking you questions about a text file.
 
-11. Video: Use for processing and answering questions about video or audio file content.
+11. Vision: Use specifically for analyzing and answering questions about image files (jpg, gif, bmp, png, etc). Use this tool any time the user is asking you questions about an uploaded image file.
 
-12. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
+12. Video: Use specifically for analyzing and answering questions about video or audio file content. Use this tool any time the user is asking you questions about an uploaded video or audio file.
+
+13. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
 
 Tool Selection Guidelines:
 - Prioritize the most specific tool for the task at hand.

package/server/plugins/openAiWhisperPlugin.js

@@ -64,7 +64,6 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         const processTS = async (uri) => {
-            const cortexRequest = new CortexRequest({ pathwayResolver });
 
             const tsparams = { fileurl:uri };
             const { language } = parameters;
@@ -81,8 +80,14 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 }
             }
 
+            const cortexRequest = new CortexRequest({ pathwayResolver });
             cortexRequest.url = WHISPER_TS_API_URL;
             cortexRequest.data = tsparams;
+            const whisperInitCallback = (requestInstance) => {
+                requestInstance.url = WHISPER_TS_API_URL;
+                requestInstance.data = tsparams;
+            };
+            cortexRequest.initCallback = whisperInitCallback;
 
             const MAX_RETRIES = 3;
             let attempt = 0;
@@ -91,8 +96,8 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 sendProgress(true, true);
                 try {
                     res = await this.executeRequest(cortexRequest);
-                    if(res.statusCode && res.statusCode >= 400){
-                        throw new Error(res.message || 'An error occurred.');
+                    if(res?.statusCode && res?.statusCode >= 400){
+                        throw new Error(res?.message || 'An error occurred.');
                     }
                     break;
                 }
@@ -102,7 +107,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 }
             }
 
-            if (res.statusCode && res.statusCode >= 400) {
+            if (res?.statusCode && res?.statusCode >= 400) {
                 throw new Error(res.message || 'An error occurred.');
             }
 
@@ -221,15 +226,6 @@ try {
 
                 await markCompletedForCleanUp(requestId);
 
-                //check cleanup for whisper temp uploaded files url
-                const regex = /whispertempfiles\/([a-z0-9-]+)/;
-                const match = file.match(regex);
-                if (match && match[1]) {
-                    const extractedValue = match[1];
-                    await markCompletedForCleanUp(extractedValue);
-                    logger.info(`Cleaned temp whisper file ${file} with request id ${extractedValue}`);
-                }
-
             } catch (error) {
                 logger.error(`An error occurred while deleting: ${error}`);
             }

package/server/plugins/replicateApiPlugin.js

@@ -1,6 +1,7 @@
 // replicateApiPlugin.js
 import ModelPlugin from "./modelPlugin.js";
 import logger from "../../lib/logger.js";
+import axios from "axios";
 
 class ReplicateApiPlugin extends ModelPlugin {
   constructor(pathway, model) {
@@ -106,10 +107,61 @@ class ReplicateApiPlugin extends ModelPlugin {
     cortexRequest.data = requestParameters;
     cortexRequest.params = requestParameters.params;
 
-    return this.executeRequest(cortexRequest);
+    // Make initial request to start prediction
+    const stringifiedResponse = await this.executeRequest(cortexRequest);
+    const parsedResponse = JSON.parse(stringifiedResponse);
+
+    // If we got a completed response, return it
+    if (parsedResponse?.status === "succeeded") {
+      return stringifiedResponse;
+    }
+    
+    logger.info("Replicate API returned a non-completed response.");
+
+    if (!parsedResponse?.id) {
+      throw new Error("No prediction ID returned from Replicate API");
+    }
+
+    // Get the prediction ID and polling URL
+    const predictionId = parsedResponse.id;
+    const pollUrl = parsedResponse.urls?.get;
+
+    if (!pollUrl) {
+      throw new Error("No polling URL returned from Replicate API");
+    }
+
+    // Poll for results
+    const maxAttempts = 60; // 5 minutes with 5 second intervals
+    const pollInterval = 5000;
+    
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      try {
+        const pollResponse = await axios.get(pollUrl, {
+          headers: cortexRequest.headers
+        });
+
+        logger.info("Polling Replicate API - attempt " + attempt);
+        const status = pollResponse.data?.status;
+        
+        if (status === "succeeded") {
+          logger.info("Replicate API returned a completed response after polling");
+          return JSON.stringify(pollResponse.data);
+        } else if (status === "failed" || status === "canceled") {
+          throw new Error(`Prediction ${status}: ${pollResponse.data?.error || "Unknown error"}`);
+        }
+
+        // Wait before next poll
+        await new Promise(resolve => setTimeout(resolve, pollInterval));
+      } catch (error) {
+        logger.error(`Error polling prediction ${predictionId}: ${error.message}`);
+        throw error;
+      }
+    }
+
+    throw new Error(`Prediction ${predictionId} timed out after ${maxAttempts * pollInterval / 1000} seconds`);
   }
 
-  // Parse the response from the Replicate API
+  // Stringify the response from the Replicate API
   parseResponse(data) {
     if (data.data) {
       return JSON.stringify(data.data);