@aj-archipelago/cortex 1.2.0 → 1.3.0

package/pathways/{sys_openai_chat_gpt4.js → system/rest_streaming/sys_openai_chat_gpt4.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4.js
 // override handler for gpt-4
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_32.js → system/rest_streaming/sys_openai_chat_gpt4_32.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_32.js
 // override handler for gpt-4-32
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_gpt4_turbo.js → system/rest_streaming/sys_openai_chat_gpt4_turbo.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_gpt4_turbo.js
 // override handler for gpt-4-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_parse_numbered_object_list.js → system/sys_parse_numbered_object_list.js}

@@ -1,10 +1,10 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [
         new Prompt({
             messages: [
-                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If a field value is numeric, it should be returned as a number in the JSON object. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
                 { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
             ]
         })

package/pathways/{sys_repair_json.js → system/sys_repair_json.js}

@@ -1,4 +1,4 @@
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude35_sonnet.js → system/workspaces/run_claude35_sonnet.js}

@@ -1,6 +1,6 @@
 
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_claude3_haiku.js → system/workspaces/run_claude3_haiku.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt35turbo.js → system/workspaces/run_gpt35turbo.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from "../server/prompt.js" 
+import { Prompt } from "../../../server/prompt.js" 
 
 export default {
     prompt: [

package/pathways/{run_gpt4.js → system/workspaces/run_gpt4.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/pathways/{run_gpt4_32.js → system/workspaces/run_gpt4_32.js}

@@ -1,5 +1,5 @@
 // Import required modules
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: [

package/server/parser.js

@@ -40,7 +40,12 @@ const isNumberedList = (data) => {
 
 async function parseJson(str) {
     try {
-        JSON.parse(str); // Validate JSON
+        // check for the common error case that the JSON is surrounded by markdown
+        const match = str.match(/```\s*(?:json)?(.*?)```/s);
+        if (match) {
+            str = match[1].trim();
+        }
+        JSON.parse(str);
         return str;
     } catch (error) {
         try {

package/server/pathwayResolver.js

@@ -221,15 +221,59 @@ class PathwayResolver {
         // Get saved context from contextId or change contextId if needed
         const { contextId } = args;
         this.savedContextId = contextId ? contextId : uuidv4();
-        this.savedContext = contextId ? (getv && (await getv(contextId)) || {}) : {};
-
-        // Save the context before processing the request
-        const savedContextStr = JSON.stringify(this.savedContext);
+        
+        const loadMemory = async () => {
+            // Load initial values
+            this.savedContext = (getv && await getv(contextId)) || {};
+            this.memorySelf = (getv && await getv(`${contextId}-memorySelf`)) || "";
+            this.memoryDirectives = (getv && await getv(`${contextId}-memoryDirectives`)) || "";
+            this.memoryTopics = (getv && await getv(`${contextId}-memoryTopics`)) || "";
+            this.memoryUser = (getv && await getv(`${contextId}-memoryUser`)) || "";
+
+            // Store initial state for comparison
+            this.initialState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+        };
+
+        const saveChangedMemory = async () => {
+            this.savedContextId = this.savedContextId || uuidv4();
+            
+            const currentState = {
+                savedContext: this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser
+            };
+
+            if (currentState.savedContext !== this.initialState.savedContext) {
+                setv && await setv(this.savedContextId, this.savedContext);
+            }
+            if (currentState.memorySelf !== this.initialState.memorySelf) {
+                setv && await setv(`${this.savedContextId}-memorySelf`, this.memorySelf);
+            }
+            if (currentState.memoryDirectives !== this.initialState.memoryDirectives) {
+                setv && await setv(`${this.savedContextId}-memoryDirectives`, this.memoryDirectives);
+            }
+            if (currentState.memoryTopics !== this.initialState.memoryTopics) {
+                setv && await setv(`${this.savedContextId}-memoryTopics`, this.memoryTopics);
+            }
+            if (currentState.memoryUser !== this.initialState.memoryUser) {
+                setv && await setv(`${this.savedContextId}-memoryUser`, this.memoryUser);
+            }
+        };
 
         const MAX_RETRIES = 3;
         let data = null;
         
         for (let retries = 0; retries < MAX_RETRIES; retries++) {
+            await loadMemory(); // Reset memory state on each retry
+            
             data = await this.processRequest(args);
             if (!data) {
                 break;
@@ -241,13 +285,10 @@ class PathwayResolver {
             }
 
             logger.warn(`Bad pathway result - retrying pathway. Attempt ${retries + 1} of ${MAX_RETRIES}`);
-            this.savedContext = JSON.parse(savedContextStr);
         }
 
-        // Update saved context if it has changed, generating a new contextId if necessary
-        if (savedContextStr !== JSON.stringify(this.savedContext)) {
-            this.savedContextId = this.savedContextId || uuidv4();
-            setv && setv(this.savedContextId, this.savedContext);
+        if (data !== null) {
+            await saveChangedMemory();
         }
 
         return data;
@@ -419,7 +460,14 @@ class PathwayResolver {
 
         // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
         if (!/^\s*$/.test(text) || parameters?.file || parameters?.inputVector || this?.modelName.includes('cognitive')) {
-            result = await this.modelExecutor.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+            result = await this.modelExecutor.execute(text, { 
+                ...parameters, 
+                ...this.savedContext,
+                memorySelf: this.memorySelf,
+                memoryDirectives: this.memoryDirectives,
+                memoryTopics: this.memoryTopics,
+                memoryUser: this.memoryUser 
+            }, prompt, this);
         } else {
             result = text;
         }
@@ -439,6 +487,10 @@ class PathwayResolver {
 
         // save the result to the context if requested and no errors
         if (prompt.saveResultTo && this.errors.length === 0) {
+            // Update memory property if it matches a known type
+            if (["memorySelf", "memoryUser", "memoryDirectives", "memoryTopics"].includes(prompt.saveResultTo)) {
+                this[prompt.saveResultTo] = result;
+            }
             this.savedContext[prompt.saveResultTo] = result;
         }
         return result;

package/server/plugins/azureCognitivePlugin.js

@@ -55,7 +55,7 @@ class AzureCognitivePlugin extends ModelPlugin {
             { search: searchQuery,  
                 "searchMode": "all",
                 "queryType": "full",
-                select: 'id', top: TOP 
+                select: 'id', top: TOP, skip: 0
             };
 
             const docsToDelete = JSON.parse(await this.executeRequest(cortexRequest));
@@ -115,6 +115,19 @@ class AzureCognitivePlugin extends ModelPlugin {
             ];
         } else {
             data.search = modelPromptText;
+            data.top = parameters.top || 50;
+            data.skip = 0;
+            if (parameters.titleOnly) {
+                switch(indexName){
+                    case 'indexcortex':
+                    case 'indexwires':
+                        data.select = 'title,id';
+                        break;
+                    default:
+                        data.select = 'title,id,url';
+                        break;
+                }
+            }
         }
 
         filter && (data.filter = filter);

package/server/plugins/claude3VertexPlugin.js

@@ -1,9 +1,10 @@
 import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
 import logger from "../../lib/logger.js";
+import axios from 'axios';
 
 const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
-async function convertContentItem(item) {
 
+async function convertContentItem(item, maxImageSize) {
   let imageUrl = "";
 
   try {
@@ -27,6 +28,14 @@ async function convertContentItem(item) {
             try {
               const urlData = imageUrl.startsWith("data:") ? imageUrl : await fetchImageAsDataURL(imageUrl);
               if (!urlData) { return null; }
+              
+              // Check base64 size
+              const base64Size = (urlData.length * 3) / 4;
+              if (base64Size > maxImageSize) {
+                logger.warn(`Image size ${base64Size} bytes exceeds maximum allowed size ${maxImageSize} - skipping image content.`);
+                return null;
+              }
+              
               const [, mimeType = "image/jpeg"] = urlData.match(/data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,.*/) || [];
               const base64Image = urlData.split(",")[1];
 
@@ -60,25 +69,26 @@ async function convertContentItem(item) {
 // Fetch image and convert to base 64 data URL
 async function fetchImageAsDataURL(imageUrl) {
   try {
-    const response = await fetch(imageUrl, { method: 'HEAD' });
-
-    if (!response.ok) {
-      throw new Error(`HTTP error! status: ${response.status}`);
-    }
+    // First check headers
+    const headResponse = await axios.head(imageUrl, {
+      timeout: 30000, // 30 second timeout
+      maxRedirects: 5
+    });
 
-    const contentType = response.headers.get('content-type');
+    const contentType = headResponse.headers['content-type'];
     if (!contentType || !allowedMIMETypes.includes(contentType)) {
       logger.warn(`Unsupported image type: ${contentType} - skipping image content.`);
       return null;
     }
 
-    const dataResponse = await fetch(imageUrl);
-    if (!dataResponse.ok) {
-      throw new Error(`HTTP error! status: ${dataResponse.status}`);
-    }
+    // Then get the actual image data
+    const dataResponse = await axios.get(imageUrl, {
+      timeout: 30000,
+      responseType: 'arraybuffer',
+      maxRedirects: 5
+    });
 
-    const buffer = await dataResponse.arrayBuffer();
-    const base64Image = Buffer.from(buffer).toString("base64");
+    const base64Image = Buffer.from(dataResponse.data).toString('base64');
     return `data:${contentType};base64,${base64Image}`;
   }
   catch (e) {
@@ -151,7 +161,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const claude3Messages = await Promise.all(
       finalMessages.map(async (message) => {
         const contentArray = Array.isArray(message.content) ? message.content : [message.content];
-        const claude3Content = await Promise.all(contentArray.map(convertContentItem));
+        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize())));
         return {
           role: message.role,
           content: claude3Content.filter(Boolean),
@@ -301,7 +311,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
 
   shortenContent(content, maxWords = 40) {
     const words = content.split(" ");
-    if (words.length <= maxWords) {
+    if (words.length <= maxWords || logger.level === 'debug') {
       return content;
     }
     return words.slice(0, maxWords / 2).join(" ") +

package/server/plugins/gemini15ChatPlugin.js

@@ -200,7 +200,7 @@ class Gemini15ChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);

package/server/plugins/geminiChatPlugin.js

@@ -195,7 +195,7 @@ class GeminiChatPlugin extends ModelPlugin {
         } else if (Array.isArray(responseData)) {
             const { mergedResult, safetyRatings } = mergeResults(responseData);
             if (safetyRatings?.length) {
-                logger.warn(`!!! response was blocked because the input or response potentially violates policies`);
+                logger.warn(`response was blocked because the input or response potentially violates policies`);
                 logger.verbose(`Safety Ratings: ${JSON.stringify(safetyRatings, null, 2)}`);
             }
             const { length, units } = this.getLength(mergedResult);

package/server/plugins/modelPlugin.js

@@ -9,6 +9,7 @@ import { config } from '../../config.js';
 const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_MAX_RETURN_TOKENS = 256;
 const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+const DEFAULT_MAX_IMAGE_SIZE = 20 * 1024 * 1024; // 20MB default
 
 class ModelPlugin {
     constructor(pathway, model) {
@@ -249,7 +250,12 @@ class ModelPlugin {
         let length = 0;
         let units = isProd ? 'characters' : 'tokens';
         if (data) {
-           length = isProd ? data.length : encode(data).length;
+            if (isProd || data.length > 5000) {
+                length = data.length;
+                units = 'characters';
+            } else {
+                length = encode(data).length;
+            }
         }
         return {length, units};
     }
@@ -341,6 +347,9 @@ class ModelPlugin {
         return requestProgress;
     }
 
+    getModelMaxImageSize() {
+        return (this.promptParameters.maxImageSize ?? this.model.maxImageSize ?? DEFAULT_MAX_IMAGE_SIZE);
+    }
 
 }
 

package/server/plugins/openAiChatPlugin.js

@@ -115,9 +115,10 @@ class OpenAIChatPlugin extends ModelPlugin {
                 const content = message.content === undefined ? JSON.stringify(message) : (Array.isArray(message.content) ? message.content.map(item => JSON.stringify(item)).join(', ') : message.content);
                 const words = content.split(" ");
                 const { length, units } = this.getLength(content);
-                const preview = words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${preview}"`);
+                
+                const displayContent = logger.level === 'debug' ? content : (words.length < 41 ? content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" "));
+
+                logger.verbose(`message ${index + 1}: role: ${message.role}, ${units}: ${length}, content: "${displayContent}"`);
                 totalLength += length;
                 totalUnits = units;
             });

package/server/plugins/openAiDallE3Plugin.js

@@ -52,14 +52,20 @@ class OpenAIDallE3Plugin extends ModelPlugin {
 
         requestPromise
         .then((response) => handleResponse(response))
-        .catch((error) => handleResponse(error));
+        .catch((error) => handleResponse(error, true));
 
-        function handleResponse(response) {
+        function handleResponse(response, isError = false) {
             let status = "succeeded";
-            let data = JSON.stringify(response);
-            if (response.data.error) {
+            let data;
+
+            if (isError) {
+                status = "failed";
+                data = JSON.stringify({ error: response.message || response });
+            } else if (response.data?.error) {
                 status = "failed";
                 data = JSON.stringify(response.data);
+            } else {
+                data = JSON.stringify(response);
             }
 
             const requestProgress = {
@@ -80,12 +86,14 @@ class OpenAIDallE3Plugin extends ModelPlugin {
             let progress =
                 requestDurationEstimator.calculatePercentComplete(callid);
 
+            if (typeof progress === 'number' && !isNaN(progress) && progress >= 0 && progress <= 1) {
                 await publishRequestProgress({
                     requestId,
                     status: "pending",
                     progress,
                     data,
                 });
+            }
 
             if (state.status !== "pending") {
                 break;

package/server/plugins/openAiVisionPlugin.js

@@ -31,8 +31,7 @@ class OpenAIVisionPlugin extends OpenAIChatPlugin {
                         }
 
                         if (typeof parsedItem === 'object' && parsedItem !== null && parsedItem.type === 'image_url') {
-                            parsedItem.image_url.url = parsedItem.url || parsedItem.image_url.url;
-                            return parsedItem;
+                            return {type: parsedItem.type, image_url: {url: parsedItem.url || parsedItem.image_url.url}};
                         }
                         
                         return parsedItem;

package/server/plugins/replicateApiPlugin.js

@@ -16,19 +16,53 @@ class ReplicateApiPlugin extends ModelPlugin {
       prompt,
     );
 
-    const requestParameters = {
-      input: {
-        aspect_ratio: "1:1",
-        output_format: "webp",
-        output_quality: 80,
-        prompt: modelPromptText,
-        //prompt_upsampling: false,
-        //safety_tolerance: 5,
-        go_fast: true,
-        megapixels: "1",
-        num_outputs: combinedParameters.numberResults,
-      },
+    const isValidSchnellAspectRatio = (ratio) => {
+      const validRatios = [
+        '1:1', '16:9', '21:9', '3:2', '2:3', '4:5',
+        '5:4', '3:4', '4:3', '9:16', '9:21'
+      ];
+      
+      return validRatios.includes(ratio);
     };
+    
+    let requestParameters = {};
+
+    switch (combinedParameters.model) {
+      case "replicate-flux-11-pro":
+        requestParameters = {
+          input: {
+            aspect_ratio: combinedParameters.aspectRatio || "1:1",
+            output_format: combinedParameters.outputFormat || "webp",
+            output_quality: combinedParameters.outputQuality || 80,
+            prompt: modelPromptText,
+            prompt_upsampling: combinedParameters.promptUpsampling || false,
+            safety_tolerance: combinedParameters.safety_tolerance || 3,
+            go_fast: true,
+            megapixels: "1",
+            width: combinedParameters.width,
+            height: combinedParameters.height,
+            size: combinedParameters.size || "1024x1024",
+            style: combinedParameters.style || "realistic_image",
+          },
+        };
+        break;
+      case "replicate-flux-1-schnell":
+        requestParameters = {
+          input: {
+            aspect_ratio: isValidSchnellAspectRatio(combinedParameters.aspectRatio) ? combinedParameters.aspectRatio : "1:1",
+            output_format: combinedParameters.outputFormat || "webp",
+            output_quality: combinedParameters.outputQuality || 80,
+            prompt: modelPromptText,
+            go_fast: true,
+            megapixels: "1",
+            num_outputs: combinedParameters.numberResults,
+            num_inference_steps: combinedParameters.steps || 4,
+            disable_safety_checker: true,
+          },
+        };
+        break;
+
+    }
 
     return requestParameters;
   }

package/tests/multimodal_conversion.test.js

@@ -112,11 +112,10 @@ test('Cortex special properties conversion', async (t) => {
         ]}
     ];
 
-    const parsedOpenAI = openai.tryParseMessages(cortexMessages);
-    const { system: claudeSystem, modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(parsedOpenAI);
+    const { system: claudeSystem, modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(cortexMessages);
 
-    const { modifiedMessages: geminiMessages } = gemini.convertMessagesToGemini(parsedOpenAI);
-    const { modifiedMessages: geminiMessages15, system: geminiSystem15 } = gemini15.convertMessagesToGemini(parsedOpenAI);
+    const { modifiedMessages: geminiMessages } = gemini.convertMessagesToGemini(cortexMessages);
+    const { modifiedMessages: geminiMessages15, system: geminiSystem15 } = gemini15.convertMessagesToGemini(cortexMessages);
 
     // Check Claude conversion
     t.true(claudeMessages[0].content[1].source.data.startsWith('/9j/4AAQ'));
@@ -142,10 +141,9 @@ test('Mixed content types conversion', async (t) => {
         ]}
     ];
 
-    const parsedOpenAI = openai.tryParseMessages(mixedMessages);
-    const { system: claudeSystem, modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(parsedOpenAI);
-    const { modifiedMessages } = gemini.convertMessagesToGemini(parsedOpenAI);
-    const { modifiedMessages: modifiedMessages15, system: system15 } = gemini15.convertMessagesToGemini(parsedOpenAI);
+    const { system: claudeSystem, modifiedMessages: claudeMessages } = await claude.convertMessagesToClaudeVertex(mixedMessages);
+    const { modifiedMessages } = gemini.convertMessagesToGemini(mixedMessages);
+    const { modifiedMessages: modifiedMessages15, system: system15 } = gemini15.convertMessagesToGemini(mixedMessages);
 
     // Check Claude conversion
     t.is(claudeMessages.length, 3);