@aj-archipelago/cortex 1.3.27 → 1.3.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.27",
+  "version": "1.3.29",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/translate_subtitle.js

@@ -2,7 +2,7 @@ import { parse, build } from "@aj-archipelago/subvibe";
 import logger from "../lib/logger.js";
 import { callPathway } from "../lib/pathwayTools.js";
 
-function splitIntoOverlappingChunks(captions, chunkSize = 20, overlap = 3) {
+export function splitIntoOverlappingChunks(captions, chunkSize = 20, overlap = 3) {
   const chunks = [];
   for (let i = 0; i < captions.length; i += (chunkSize - overlap)) {
     const end = Math.min(i + chunkSize, captions.length);
@@ -17,26 +17,51 @@ function splitIntoOverlappingChunks(captions, chunkSize = 20, overlap = 3) {
   return chunks;
 }
 
-function selectBestTranslation(translations, startIndex, endIndex) {
-  // If we only have one translation for this caption, use it
-  if (translations.length === 1) return translations[0];
+export function selectBestTranslation(translations, startIndex, endIndex) {
+  try {
+    if (!translations || !Array.isArray(translations)) {
+      logger.warn(`Invalid translations input: ${JSON.stringify(translations)}`);
+      return null;
+    }
+    
+    if (translations.length === 0) {
+      logger.warn(`No translations available for selection`);
+      return null;
+    }
+    
+    // If we only have one translation for this caption, use it
+    if (translations.length === 1) return translations[0];
 
-  // For multiple translations, prefer the one from the middle of its chunk
-  // This helps avoid edge effects in translation
-  return translations.reduce((best, current) => {
-    const currentDistance = Math.min(
-      Math.abs(current.chunkStart - startIndex),
-      Math.abs(current.chunkEnd - endIndex)
-    );
-    const bestDistance = Math.min(
-      Math.abs(best.chunkStart - startIndex),
-      Math.abs(best.chunkEnd - endIndex)
-    );
-    return currentDistance < bestDistance ? current : best;
-  });
+    // Use the first translation as a starting point
+    const first = translations[0];
+    
+    // For multiple translations, prefer the one whose identifier is closest to the middle
+    // of the requested range
+    const targetValue = (Number(startIndex) + Number(endIndex)) / 2;
+    
+    return translations.reduce((best, current) => {
+      try {
+        // Use identifier for comparison if available, otherwise use index
+        const currentValue = Number(current.identifier !== undefined ? current.identifier : current.index || 0);
+        const bestValue = Number(best.identifier !== undefined ? best.identifier : best.index || 0);
+        
+        const currentDistance = Math.abs(currentValue - targetValue);
+        const bestDistance = Math.abs(bestValue - targetValue);
+        
+        return currentDistance < bestDistance ? current : best;
+      } catch (err) {
+        logger.warn(`Error comparing translations: ${err.message}`);
+        return best; // Fallback to existing best on error
+      }
+    }, first);
+  } catch (err) {
+    logger.error(`Error in selectBestTranslation: ${err.message}`);
+    // Return the first translation if available, otherwise null
+    return translations && translations.length ? translations[0] : null;
+  }
 }
 
-async function translateChunk(chunk, args, maxRetries = 3) {
+export async function translateChunk(chunk, args, maxRetries = 3) {
   const chunkText = build(chunk.captions, { format: args.format, preserveIndexes: true });
   
   for (let attempt = 0; attempt < maxRetries; attempt++) {
@@ -82,7 +107,7 @@ export default {
   timeout: 3600,
   executePathway: async ({args}) => {
     try {
-      const { text, format = 'srt' } = args;
+      const { text, format = 'vtt' } = args;
       const parsed = parse(text, { format, preserveIndexes: true });
       const captions = parsed.cues;
   
@@ -101,16 +126,18 @@ export default {
       // Create a map of caption index to all its translations
       const translationMap = new Map();
       translatedChunks.flat().forEach(caption => {
-        if (!translationMap.has(caption.index)) {
-          translationMap.set(caption.index, []);
+        const identifier = caption.identifier || caption.index;
+        if (!translationMap.has(identifier)) {
+          translationMap.set(identifier, []);
         }
-        translationMap.get(caption.index).push(caption);
+        translationMap.get(identifier).push(caption);
       });
       
       // Select best translation for each caption
       const finalCaptions = captions.map(caption => {
-        const translations = translationMap.get(caption.index) || [caption];
-        const bestTranslation = selectBestTranslation(translations, caption.index, caption.index);
+        const identifier = caption.identifier || caption.index;
+        const translations = translationMap.get(identifier) || [caption];
+        const bestTranslation = selectBestTranslation(translations, identifier, identifier);
         const text = bestTranslation?.text || caption?.text;
         return { ...caption, text };
       });

package/server/parser.js

@@ -16,7 +16,7 @@ const parseNumberedList = (str) => {
 async function parseNumberedObjectList(text, format) {
     const parsedList = await callPathway('sys_parse_numbered_object_list', { text, format });
     try {
-        return JSON.parse(parsedList);
+        return JSON.parse(parsedList) || [];
     } catch (error) {
         logger.warn(`Failed to parse numbered object list: ${error.message}`);
         return [];

package/server/plugins/openAiWhisperPlugin.js

@@ -72,13 +72,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             if(maxLineWidth) tsparams.max_line_width = maxLineWidth;
             if(maxLineCount) tsparams.max_line_count = maxLineCount;
             if(maxWordsPerLine) tsparams.max_words_per_line = maxWordsPerLine;
-            if(wordTimestamped!=null) {
-                if(!wordTimestamped) {
-                    tsparams.word_timestamps = "False";
-                }else{
-                    tsparams.word_timestamps = wordTimestamped;    
-                }
-            }
+            tsparams.word_timestamps = !wordTimestamped ? "False" : wordTimestamped;
 
             const cortexRequest = new CortexRequest({ pathwayResolver });
             cortexRequest.url = WHISPER_TS_API_URL;
@@ -157,7 +151,8 @@ async function processURI(uri) {
 
     const intervalId = setInterval(() => sendProgress(true), 3000);
 
-    const useTS = WHISPER_TS_API_URL && (wordTimestamped || highlightWords);
+    //const useTS = WHISPER_TS_API_URL && (wordTimestamped || highlightWords); // use TS API only for word timestamped
+    const useTS = !!WHISPER_TS_API_URL; // use TS API always if URL is set
 
     if (useTS) {
         _promise = processTS;

package/server/rest.js

@@ -48,7 +48,7 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
             return value.map(msg => ({
                 ...msg,
                 content: Array.isArray(msg.content) ? 
-                    JSON.stringify(msg.content) : 
+                    msg.content.map(item => JSON.stringify(item)) : 
                     msg.content
             }));
         } else {

package/tests/claude3VertexPlugin.test.js

@@ -212,3 +212,57 @@ test('convertMessagesToClaudeVertex user message with no content', async (t) =>
   t.deepEqual(output, { system: '', modifiedMessages: [] });
 });
 
+test('convertMessagesToClaudeVertex with multi-part content array', async (t) => {
+  const plugin = new Claude3VertexPlugin(pathway, model);
+  
+  // Test with multi-part content array
+  const multiPartContent = [
+    {
+      type: 'text', 
+      text: 'Hello world'
+    },
+    {
+      type: 'text', 
+      text: 'Hello2 world2'
+    },
+    {
+      type: 'image_url', 
+      image_url: 'https://static.toiimg.com/thumb/msid-102827471,width-1280,height-720,resizemode-4/102827471.jpg'
+    }
+  ];
+  
+  const messages = [
+    { role: 'system', content: 'System message' },
+    { role: 'user', content: multiPartContent }
+  ];
+  
+  const output = await plugin.convertMessagesToClaudeVertex(messages);
+  
+  // Verify system message is preserved
+  t.is(output.system, 'System message');
+  
+  // Verify the user message role is preserved
+  t.is(output.modifiedMessages[0].role, 'user');
+  
+  // Verify the content array has the correct number of items
+  // We expect 3 items: 2 text items and 1 image item
+  t.is(output.modifiedMessages[0].content.length, 3);
+  
+  // Verify the text content items
+  t.is(output.modifiedMessages[0].content[0].type, 'text');
+  t.is(output.modifiedMessages[0].content[0].text, 'Hello world');
+  
+  t.is(output.modifiedMessages[0].content[1].type, 'text');
+  t.is(output.modifiedMessages[0].content[1].text, 'Hello2 world2');
+  
+  // Verify the image content item
+  t.is(output.modifiedMessages[0].content[2].type, 'image');
+  t.is(output.modifiedMessages[0].content[2].source.type, 'base64');
+  t.is(output.modifiedMessages[0].content[2].source.media_type, 'image/jpeg');
+  
+  // Check if the base64 data looks reasonable
+  const base64Data = output.modifiedMessages[0].content[2].source.data;
+  const base64Regex = /^[A-Za-z0-9+/]+={0,2}$/;
+  t.true(base64Data.length > 100); // Check if the data is sufficiently long
+  t.true(base64Regex.test(base64Data)); // Check if the data matches the base64 regex
+});

package/tests/openai_api.test.js

@@ -496,4 +496,129 @@ test('POST /chat/completions should return complete responses from gpt-4o', asyn
   const content = response.body.choices[0].message.content;
   t.regex(content, /END_MARKER_XYZ$/);
 }); 
+
+test('POST /chat/completions should handle array content properly', async (t) => {
+  // This test verifies the functionality in server/rest.js where array content is JSON stringified
+  // Specifically testing: content: Array.isArray(msg.content) ? msg.content.map(item => JSON.stringify(item)) : msg.content
+  
+  // Create a request with MultiMessage array content
+  const testContent = [
+    {
+      type: 'text', 
+      text: 'Hello world'
+    },
+    {
+      type: 'text', 
+      text: 'Hello2 world2'
+    },
+    {
+      type: 'image', 
+      url: 'https://example.com/test.jpg'
+    }
+  ];
+  
+  try {
+    // First, check if the API server is running and get available models
+    let modelToUse = '*'; // Default fallback model
+    try {
+      const modelsResponse = await got(`${API_BASE}/models`, { responseType: 'json' });
+      if (modelsResponse.body && modelsResponse.body.data && modelsResponse.body.data.length > 0) {
+        const models = modelsResponse.body.data.map(model => model.id);
+        
+        // Priority 1: Find sonnet with highest version (e.g., claude-3.7-sonnet)
+        const sonnetVersions = models
+          .filter(id => id.includes('-sonnet') && id.startsWith('claude-'))
+          .sort((a, b) => {
+            // Extract version numbers and compare
+            const versionA = a.match(/claude-(\d+\.\d+)-sonnet/);
+            const versionB = b.match(/claude-(\d+\.\d+)-sonnet/);
+            if (versionA && versionB) {
+              return parseFloat(versionB[1]) - parseFloat(versionA[1]); // Descending order
+            }
+            return 0;
+          });
+        
+        if (sonnetVersions.length > 0) {
+          modelToUse = sonnetVersions[0]; // Use highest version sonnet
+        } else {
+          // Priority 2: Any model ending with -sonnet
+          const anySonnet = models.find(id => id.endsWith('-sonnet'));
+          if (anySonnet) {
+            modelToUse = anySonnet;
+          } else {
+            // Priority 3: Any model starting with claude-
+            const anyClaude = models.find(id => id.startsWith('claude-'));
+            if (anyClaude) {
+              modelToUse = anyClaude;
+            } else {
+              // Fallback: Just use the first available model
+              modelToUse = models[0];
+            }
+          }
+        }
+        
+        t.log(`Using model: ${modelToUse}`);
+      }
+    } catch (modelError) {
+      t.log('Could not get available models, using default model');
+    }
+    
+    // Make a direct HTTP request to the REST API
+    const response = await axios.post(`${API_BASE}/chat/completions`, {
+      model: modelToUse,
+      messages: [
+        {
+          role: 'user',
+          content: testContent
+        }
+      ]
+    });
+
+    t.log('Response:', response.data.choices[0].message);
+
+    const message = response.data.choices[0].message;
+
+    //message should not have anything similar to:
+    //Execution failed for sys_claude_37_sonnet: HTTP error: 400 Bad Request
+    //HTTP error:
+    t.falsy(message.content.startsWith('HTTP error:'));
+    //400 Bad Request
+    t.falsy(message.content.startsWith('400 Bad Request'));
+    //Execution failed
+    t.falsy(message.content.startsWith('Execution failed'));
+    //Invalid JSON
+    t.falsy(message.content.startsWith('Invalid JSON'));
+
+    
+    // If the request succeeds, it means the array content was properly processed
+    // If the JSON.stringify was not applied correctly, the request would fail
+    t.truthy(response.data);
+    t.pass('REST API successfully processed array content');
+  } catch (error) {
+    // If there's a connection error (e.g., API not running), we'll skip this test
+    if (error.code === 'ECONNREFUSED') {
+      t.pass('Skipping test - REST API not available');
+    } else {
+      // Check if the error response contains useful information
+      if (error.response) {
+        // We got a response from the server, but with an error status
+        t.log('Server responded with:', error.response.data);
+        
+        // Skip the test if the server is running but no pathway is configured to handle the request
+        if (error.response.status === 404 && 
+            error.response.data.error && 
+            error.response.data.error.includes('not found')) {
+          t.pass('Skipping test - No suitable pathway configured for this API endpoint');
+        } else {
+          t.fail(`API request failed with status ${error.response.status}: ${error.response.statusText}`);
+        }
+      } else {
+        // No response received
+        t.fail(`API request failed: ${error.message}`);
+      }
+    }
+  }
+});  
+  
+