@aj-archipelago/cortex 1.3.7 → 1.3.9

package/pathways/translate_subtitle.js

@@ -1,164 +1,291 @@
-import subsrt from "subsrt";
 import logger from "../lib/logger.js";
 import { callPathway } from "../lib/pathwayTools.js";
-import { publishRequestProgress } from "../lib/redisSubscription.js";
 
-function preprocessStr(str) {
+function preprocessStr(str, format) {
   try {
     if (!str) return "";
-    return (
-      str
-        .replace(/\r\n?/g, "\n")
-        .replace(/\n+/g, "\n")
-        .replace(/(\d+)\n(\d{2}:\d{2}:\d{2},\d{3})/g, "\n\n$1\n$2")
-        .trim() + "\n\n"
-    );
+    let content = str
+      // Normalize line endings
+      .replace(/\r\n?/g, "\n")
+      // Remove WEBVTT header for processing
+      .replace(/^WEBVTT\n\n/, '');
+
+    // For SRT, convert commas to dots in timestamps
+    if (format === 'srt') {
+      content = content.replace(/(\d{2}:\d{2}:\d{2}),(\d{3})/g, "$1.$2");
+    }
+
+    return content
+      // Ensure each subtitle block is properly separated
+      .split(/\n\s*\n/)
+      .map(block => block.trim())
+      .filter(block => {
+        // Match both numeric indices (SRT) and optional caption identifiers (VTT)
+        const firstLine = block.split('\n')[0];
+        return block && (
+          /^\d+$/.test(firstLine) || // SRT style
+          /^\d{2}:\d{2}/.test(firstLine) || // VTT style without identifier
+          /^[^\n]+\n\d{2}:\d{2}/.test(block) // VTT style with identifier
+        );
+      })
+      .join("\n\n")
+      + "\n\n";
   } catch (e) {
     logger.error(`An error occurred in content text preprocessing: ${e}`);
     return "";
   }
 }
 
-async function processBatch(batch, args) {
-  const batchText = batch
-    .map((caption, index) => `LINE#${index + 1}: ${caption.content}`)
-    .join("\n");
+function timeToMs(timeStr) {
+  const [time, ms] = timeStr.split(/[.,]/);
+  const [hours, minutes, seconds] = time.split(':').map(Number);
+  return (hours * 3600 + minutes * 60 + seconds) * 1000 + parseInt(ms);
+}
 
-  const translatedText = await callPathway("translate_subtitle_helper", {
-    ...args,
-    text: batchText,
-    async: false,
-  });
+function msToTimestamp(ms, format) {
+  const date = new Date(ms);
+  const timestamp = date.toISOString().slice(11, 23);
+  return format === 'srt' ? timestamp.replace('.', ',') : timestamp;
+}
 
-  // Remove LINE# and LINE() labels
-  const translatedLines = translatedText.split("\n");
-    translatedLines.forEach((line, i) => {
-    translatedLines[i] = line.replace(/^LINE#\d+:\s*/, "").trim();
-  });
-  //make sure translatedLines.length===batch.length
-  if (translatedLines.length < batch.length) {
-    const emptyLines = Array(batch.length - translatedLines.length).fill("-");
-    translatedLines.push(...emptyLines);
-  } else if (translatedLines.length > batch.length) {
-    //first remove the empty lines
-    translatedLines.splice(0, translatedLines.length, ...translatedLines.filter(line => line.trim() !== ""));
-
-    if(translatedLines.length>batch.length) {
-        //merge extra lines to end
-        const lastLine = translatedLines[batch.length - 1];
-        const mergedLines = translatedLines.slice(batch.length);
-        mergedLines.unshift(lastLine);
-        translatedLines.splice(batch.length - 1, translatedLines.length - batch.length + 1, mergedLines.join(" "));
-    }else {
-        const emptyLines = Array(batch.length - translatedLines.length).fill("-");
-        translatedLines.push(...emptyLines);
+function parseSubtitles(content, format) {
+  const blocks = content.split(/\n\s*\n/).filter(block => block.trim());
+  const captions = [];
+  
+  for (const block of blocks) {
+    const lines = block.split('\n');
+    if (lines.length < 2) continue;
+    
+    let index, timelineIndex;
+    if (format === 'srt') {
+      // SRT format: numeric index required
+      if (!/^\d+$/.test(lines[0])) continue;
+      index = parseInt(lines[0]);
+      timelineIndex = 1;
+    } else {
+      // VTT format: optional identifier
+      timelineIndex = /^\d{2}:\d{2}/.test(lines[0]) ? 0 : 1;
+      index = timelineIndex === 0 ? captions.length + 1 : lines[0];
     }
+    
+    const timeMatch = lines[timelineIndex].match(/^(\d{2}:\d{2}:\d{2}[.,]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[.,]\d{3})/);
+    if (!timeMatch) continue;
+    
+    const startTime = timeMatch[1].replace(',', '.');
+    const endTime = timeMatch[2].replace(',', '.');
+    const content = lines.slice(timelineIndex + 1).join('\n');
+    
+    captions.push({
+      type: "caption",
+      index: typeof index === 'number' ? index : captions.length + 1,
+      identifier: typeof index === 'string' ? index : null,
+      start: timeToMs(startTime),
+      end: timeToMs(endTime),
+      duration: timeToMs(endTime) - timeToMs(startTime),
+      content: content,
+      text: content
+    });
   }
+  
+  return captions;
+}
 
+function splitIntoOverlappingChunks(captions, chunkSize = 20, overlap = 3) {
+  const chunks = [];
+  for (let i = 0; i < captions.length; i += (chunkSize - overlap)) {
+    const end = Math.min(i + chunkSize, captions.length);
+    const chunk = captions.slice(i, end);
+    chunks.push({
+      captions: chunk,
+      startIndex: i,
+      endIndex: end - 1,
+      isOverlap: i > 0 || end < captions.length
+    });
+  }
+  return chunks;
+}
 
-  // Handle last empty line
-    if (translatedLines[translatedLines.length - 1].trim() === "") {
-        let lastNonEmptyIndex = translatedLines.length - 2;
-        while (lastNonEmptyIndex >= 0 && translatedLines[lastNonEmptyIndex].trim() === "") {
-            lastNonEmptyIndex--;
-        }
-        if (lastNonEmptyIndex >= 0) {
-            translatedLines[translatedLines.length - 1] = translatedLines[lastNonEmptyIndex];
-            translatedLines[lastNonEmptyIndex] = "";
-        }
-    }
-
+function selectBestTranslation(translations, startIndex, endIndex) {
+  // If we only have one translation for this caption, use it
+  if (translations.length === 1) return translations[0];
 
-  return batch.map((caption, i) => ({
-    ...caption,
-    content: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
-    text: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
-  }));
+  // For multiple translations, prefer the one from the middle of its chunk
+  // This helps avoid edge effects in translation
+  return translations.reduce((best, current) => {
+    const currentDistance = Math.min(
+      Math.abs(current.chunkStart - startIndex),
+      Math.abs(current.chunkEnd - endIndex)
+    );
+    const bestDistance = Math.min(
+      Math.abs(best.chunkStart - startIndex),
+      Math.abs(best.chunkEnd - endIndex)
+    );
+    return currentDistance < bestDistance ? current : best;
+  });
 }
 
-async function myResolver(args, requestId) {
-  try {
-    const { text, format } = args;
-    const captions = subsrt.parse(preprocessStr(text), {
-      format: format,
-      verbose: true,
-      eol: "\n",
-    });
-    const maxLineCount = 100;
-    const maxWordCount = 300;
-    let translatedCaptions = [];
-    let currentBatch = [];
-    let currentWordCount = 0;
-
-    const totalCount = captions.length;
-    let completedCount = 0;
-
-    const sendProgress = () => {
-      if (completedCount >= totalCount) return;
-      if(!requestId) {
-        logger.warn(`No requestId found for progress update`);
-        return;
-      }
+function validateFinalOutput(result, originalText, format) {
+  // Basic structure validation
+  if (!result || !result.trim()) {
+    logger.error("Empty or whitespace-only result");
+    return false;
+  }
+  
+  // Check for VTT header if needed
+  if (format === 'vtt' && !result.startsWith('WEBVTT\n\n')) {
+    logger.error("Missing WEBVTT header");
+    return false;
+  }
+  
+  // Check for timestamp format
+  const timestampPattern = format === 'srt' 
+    ? /\d{2}:\d{2}:\d{2},\d{3}\s*-->\s*\d{2}:\d{2}:\d{2},\d{3}/
+    : /\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}/;
+  
+  const hasTimestamps = timestampPattern.test(result);
+  if (!hasTimestamps) {
+    logger.error(`No valid ${format.toUpperCase()} timestamps found in result`);
+    return false;
+  }
 
-      const progress = completedCount / totalCount;
-      logger.info(`Progress for ${requestId}: ${progress}`);
+  // Check overall length ratio
+  if (result.length < originalText.length * 0.5) {
+    logger.error(`Result length (${result.length}) is less than 50% of original length (${originalText.length})`);
+    return false;
+  }
 
-      publishRequestProgress({
-        requestId,
-        progress,
-        data: null,
-      });
-    };
-
-    for (let i = 0; i < captions.length; i++) {
-      const caption = captions[i];
-      const captionWordCount = caption.content.split(/\s+/).length;
-      if (
-        (currentWordCount + captionWordCount > maxWordCount ||
-          currentBatch.length >= maxLineCount) &&
-        currentBatch.length > 0
-      ) {
-        completedCount=i;
-        sendProgress();
-        const translatedBatch = await processBatch(
-          currentBatch,
-          args,
-        );
-        translatedCaptions = translatedCaptions.concat(translatedBatch);
-        currentBatch = [];
-        currentWordCount = 0;
+  // Validate subtitle block structure
+  const blocks = result.split(/\n\s*\n/).filter(block => block.trim());
+  
+  // Skip WEBVTT header for VTT format
+  const startIndex = format === 'vtt' && blocks[0].trim() === 'WEBVTT' ? 1 : 0;
+  
+  for (let i = startIndex; i < blocks.length; i++) {
+    const block = blocks[i];
+    const lines = block.trim().split('\n');
+    
+    if (lines.length < 2) {
+      logger.error(`Block ${i + 1} has insufficient lines (${lines.length}):\n${block}`);
+      return false;
+    }
+    
+    // Find the timestamp line
+    let timestampLineIndex = -1;
+    for (let j = 0; j < lines.length; j++) {
+      if (timestampPattern.test(lines[j])) {
+        timestampLineIndex = j;
+        break;
       }
-      currentBatch.push(caption);
-      currentWordCount += captionWordCount;
     }
-
-    if (currentBatch.length > 0) {
-      const translatedBatch = await processBatch(
-        currentBatch,
-        args,
-      );
-      translatedCaptions = translatedCaptions.concat(translatedBatch);
+    
+    if (timestampLineIndex === -1) {
+      logger.error(`Block ${i + 1} has no valid timestamp line:\n${block}`);
+      return false;
     }
+    
+    // Check that we have content after the timestamp
+    if (timestampLineIndex === lines.length - 1) {
+      logger.error(`Block ${i + 1} has no content after timestamp:\n${block}`);
+      return false;
+    }
+    
+    // Log the content for inspection
+    logger.debug(`Block ${i + 1} content:\n${lines.slice(timestampLineIndex + 1).join('\n')}`);
+  }
 
-    return (
-      subsrt
-        .build(translatedCaptions, {
-          format: format === "vtt" ? "vtt" : "srt",
-          eol: "\n",
-        })
-        .trim() + "\n"
-    );
-  } catch (e) {
-    logger.warn(
-      `${e} - could be that there are no subtitles, so attempting block translation.`
-    );
+  return true;
+}
+
+async function translateChunk(chunk, args, maxRetries = 3) {
+  const format = args.format || 'srt';
+  const chunkText = chunk.captions
+    .map(c => {
+      const startTime = msToTimestamp(c.start, format);
+      const endTime = msToTimestamp(c.end, format);
+      const index = format === 'srt' || !c.identifier ? c.index : c.identifier;
+      return `${index}\n${startTime} --> ${endTime}\n${c.content}`;
+    })
+    .join('\n\n');
+  
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
-      return await callPathway("translate_gpt4_omni", {...args, async: false});
+      const translated = await callPathway("translate_subtitle_helper", {
+        ...args,
+        text: chunkText,
+        async: false,
+      });
+
+      // Basic validation - just check for SUBTITLES tags and some content
+      const match = translated.match(/<SUBTITLES>([\s\S]*)<\/SUBTITLES>/);
+      if (!match || !match[1].trim()) {
+        logger.warn(`Attempt ${attempt + 1}: Invalid translation format`);
+        continue;
+      }
+      
+      const content = match[1].trim();
+      const blocks = content.split(/\n\s*\n/);
+      
+      // Check if any blocks are empty or invalid
+      let hasEmptyBlocks = false;
+      const processedBlocks = chunk.captions.map((caption, index) => {
+        const block = blocks[index];
+        if (!block) {
+          logger.warn(`Attempt ${attempt + 1}: Empty block for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        const lines = block.split('\n');
+        if (lines.length < 3) {
+          logger.warn(`Attempt ${attempt + 1}: Invalid block structure for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        const content = lines.slice(2).join('\n').trim();
+        if (!content) {
+          logger.warn(`Attempt ${attempt + 1}: Empty content for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        return {
+          ...caption,
+          content: content,
+          text: content,
+          chunkStart: chunk.startIndex,
+          chunkEnd: chunk.endIndex
+        };
+      });
+      
+      // If no empty blocks, return the processed blocks
+      if (!hasEmptyBlocks) {
+        return processedBlocks;
+      }
+      
+      // If this was the last attempt and we still have empty blocks,
+      // return what we have but keep original content for empty blocks
+      if (attempt === maxRetries - 1) {
+        logger.warn(`Failed to get valid translations for all blocks after ${maxRetries} attempts`);
+        return chunk.captions.map((caption, index) => {
+          return processedBlocks[index] || {
+            ...caption,
+            chunkStart: chunk.startIndex,
+            chunkEnd: chunk.endIndex
+          };
+        });
+      }
+      
+      // Otherwise, try again
+      logger.info(`Retrying chunk due to empty blocks (attempt ${attempt + 1}/${maxRetries})`);
+      
     } catch (e) {
-      logger.error(`An error occurred in subtitle translation: ${e}`);
-      return "";
+      logger.error(`Error translating chunk ${chunk.startIndex}-${chunk.endIndex} (attempt ${attempt + 1}): ${e}`);
+      if (attempt === maxRetries - 1) throw e;
     }
   }
+  
+  throw new Error(`Failed to translate chunk ${chunk.startIndex}-${chunk.endIndex} after ${maxRetries} attempts`);
 }
 
 export default {
@@ -173,9 +300,73 @@ export default {
   model: "oai-gpt4o",
   enableDuplicateRequests: false,
   timeout: 3600,
-  executePathway: async (executePathwayArgs) => {
-    const { args } = executePathwayArgs;
-    const requestId = executePathwayArgs?.resolver?.requestId;
-    return await myResolver(args, requestId);
+  executePathway: async ({args}) => {
+    try {
+      const { text, format = 'srt' } = args;
+      const preprocessedText = preprocessStr(text, format);
+      const captions = parseSubtitles(preprocessedText, format);
+  
+      if (!captions || captions.length === 0) {
+        throw new Error("No captions found in input");
+      }
+  
+      // Split into overlapping chunks
+      const chunks = splitIntoOverlappingChunks(captions);
+      logger.info(`Split subtitles into ${chunks.length} overlapping chunks`);
+      
+      // Translate all chunks in parallel
+      const chunkPromises = chunks.map(chunk => translateChunk(chunk, args));
+      const translatedChunks = await Promise.all(chunkPromises);
+      
+      // Create a map of caption index to all its translations
+      const translationMap = new Map();
+      translatedChunks.flat().forEach(caption => {
+        if (!translationMap.has(caption.index)) {
+          translationMap.set(caption.index, []);
+        }
+        translationMap.get(caption.index).push(caption);
+      });
+      
+      // Select best translation for each caption
+      const finalCaptions = captions.map(caption => {
+        const translations = translationMap.get(caption.index) || [caption];
+        return selectBestTranslation(translations, caption.index, caption.index);
+      });
+      
+      // Format the output
+      let result = finalCaptions
+        .map(caption => {
+          const startTime = msToTimestamp(caption.start, format);
+          const endTime = msToTimestamp(caption.end, format);
+          // Only include index/identifier if it was in the original
+          const hasIdentifier = caption.identifier !== null || format === 'srt';
+          const index = format === 'srt' || !caption.identifier ? caption.index : caption.identifier;
+          return hasIdentifier ? 
+            `${index}\n${startTime} --> ${endTime}\n${caption.content}` :
+            `${startTime} --> ${endTime}\n${caption.content}`;
+        })
+        .join('\n\n')
+        .trim();
+
+      // Add final newline only if input had one
+      if (text.endsWith('\n')) {
+        result += '\n';
+      }
+
+      // Add WEBVTT header for VTT format
+      if (format === 'vtt') {
+        result = 'WEBVTT\n\n' + result;
+      }
+
+      // Validate final output
+      if (!validateFinalOutput(result, text, format)) {
+        throw new Error("Final subtitle reconstruction failed validation");
+      }
+
+      return result;
+    } catch (e) {
+      logger.error(`Subtitle translation failed: ${e}`);
+      throw e;
+    }
   },
 };

package/pathways/translate_subtitle_helper.js

@@ -7,25 +7,13 @@ export default {
             {
                 role: "system",
                 content: 
-`Expert translator: Convert ALL text to {{to}}. Unbreakable rules:
-
-1. Translate EVERY SINGLE LINE. Zero exceptions.
-2. Output MUST have EXACTLY the same line count as input.
-3. One input line = One output line. Always.
-4. Only translations. Nothing extra.
-5. Non-translatable stays unchanged.
-6. Keep all formatting and characters.
-7. Prefix: "LINE#lineNumber:".
-8. Untranslatable: Copy as-is with prefix.
-9. Internal checks: Verify line count and content after each line.
-10. Final verification: Recount, check numbering, confirm content, cross-check with input.
-
-Translate ALL lines. Constant vigilance. Exhaustive final cross-check.`
+`You are an expert subtitle translator. You will be given a block of subtitles and asked to translate them into {{to}}.
+You must maintain the original format (caption numbers and timestamps) exactly and make the content fit as naturally as possible.
+Output only the translated subtitles in a <SUBTITLES> tag with no other text or commentary.`
             },
             {
                 role: "user",
-                // content: `"PreviousLines":\n{{{prevLine}}}\n\n"CurrentLines":\n{{{text}}}\n"NextLines":\n{{{nextLine}}}\n\n`,
-                content: `{{{text}}}`,
+                content: `<SUBTITLES>\n{{{text}}}\n</SUBTITLES>`,
             },
             ],
         }),

package/server/plugins/azureVideoTranslatePlugin.js

@@ -81,17 +81,6 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
                 throw new Error(this.jsonBuffer);
             }
 
-            if (isValidJSON(this.jsonBuffer)) {
-                const parsedData = JSON.parse(this.jsonBuffer);
-                if (parsedData.progress !== undefined) {
-                    publishRequestProgress({
-                        requestId: this.requestId,
-                        progress: parsedData.progress,
-                        info: this.jsonBuffer
-                    });
-                }
-            }
-
             onData(this.jsonBuffer);
             this.jsonBuffer = '';
             this.jsonDepth = 0;
@@ -118,11 +107,34 @@ class AzureVideoTranslatePlugin extends ModelPlugin {
                 let finalJson = '';
                 this.handleStream(response.data,
                     (data) => {
-                        publishRequestProgress({
-                            requestId: this.requestId,
-                            info: data
-                        });
+                        let sent = false;
+                        if (isValidJSON(data)) {
+                            const parsedData = JSON.parse(data);
+                            if (parsedData.progress !== undefined) {
+                                let timeInfo = '';
+                                if (parsedData.estimated_time_remaining && parsedData.elapsed_time) {
+                                    const minutes = Math.ceil(parsedData.estimated_time_remaining / 60);
+                                    timeInfo = minutes <= 2 
+                                        ? `Should be done soon (${parsedData.elapsed_time} elapsed)`
+                                        : `Estimated ${minutes} minutes remaining`;
+                                }
+
+                                publishRequestProgress({
+                                    requestId: this.requestId,
+                                    progress: parsedData.progress,
+                                    info: timeInfo
+                                });
+                                sent = true;
+                            }
+                        }
+                        if (!sent) {
+                            publishRequestProgress({
+                                requestId: this.requestId,
+                                info: data
+                            });
+                        }
                         logger.debug('Data:', data);
+                        
                         // Extract JSON content if message contains targetLocales
                         const jsonMatch = data.match(/{[\s\S]*"targetLocales"[\s\S]*}/);
                         if (jsonMatch) {

package/server/plugins/claude3VertexPlugin.js

@@ -2,9 +2,7 @@ import OpenAIVisionPlugin from "./openAiVisionPlugin.js";
 import logger from "../../lib/logger.js";
 import axios from 'axios';
 
-const allowedMIMETypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
-
-async function convertContentItem(item, maxImageSize) {
+async function convertContentItem(item, maxImageSize, plugin) {
   let imageUrl = "";
 
   try {
@@ -26,6 +24,12 @@ async function convertContentItem(item, maxImageSize) {
             }
 
             try {
+              // First validate the image URL
+              if (!await plugin.validateImageUrl(imageUrl)) {
+                return null;
+              }
+
+              // Then fetch and convert to base64 if needed
               const urlData = imageUrl.startsWith("data:") ? imageUrl : await fetchImageAsDataURL(imageUrl);
               if (!urlData) { return null; }
               
@@ -69,25 +73,14 @@ async function convertContentItem(item, maxImageSize) {
 // Fetch image and convert to base 64 data URL
 async function fetchImageAsDataURL(imageUrl) {
   try {
-    // First check headers
-    const headResponse = await axios.head(imageUrl, {
-      timeout: 30000, // 30 second timeout
-      maxRedirects: 5
-    });
-
-    const contentType = headResponse.headers['content-type'];
-    if (!contentType || !allowedMIMETypes.includes(contentType)) {
-      logger.warn(`Unsupported image type: ${contentType} - skipping image content.`);
-      return null;
-    }
-
-    // Then get the actual image data
+    // Get the actual image data
     const dataResponse = await axios.get(imageUrl, {
       timeout: 30000,
       responseType: 'arraybuffer',
       maxRedirects: 5
     });
 
+    const contentType = dataResponse.headers['content-type'];
     const base64Image = Buffer.from(dataResponse.data).toString('base64');
     return `data:${contentType};base64,${base64Image}`;
   }
@@ -161,7 +154,7 @@ class Claude3VertexPlugin extends OpenAIVisionPlugin {
     const claude3Messages = await Promise.all(
       finalMessages.map(async (message) => {
         const contentArray = Array.isArray(message.content) ? message.content : [message.content];
-        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize())));
+        const claude3Content = await Promise.all(contentArray.map(item => convertContentItem(item, this.getModelMaxImageSize(), this)));
         return {
           role: message.role,
           content: claude3Content.filter(Boolean),