@aj-archipelago/cortex 1.3.6 → 1.3.8

package/pathways/system/entity/sys_entity_start.js

@@ -79,24 +79,23 @@ export default {
             args.chatHistory = args.chatHistory.slice(-20);
         }
 
-        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0 });
-        if (memoryContext) {
-            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
-        }
-
         const pathwayResolver = resolver;
         const { anthropicModel, openAIModel } = pathwayResolver.pathway;
-
         const styleModel = args.aiStyle === "Anthropic" ? anthropicModel : openAIModel;
 
         // if the model has been overridden, make sure to use it
         if (pathwayResolver.modelName) {
             args.model = pathwayResolver.modelName;
         }
+
+        const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0, stream: false }, pathwayResolver);
+        if (memoryContext) {
+            args.chatHistory.splice(-1, 0, { role: 'assistant', content: memoryContext });
+        }
         
         let ackResponse = null;
         if (args.voiceResponse) {
-            ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false }, pathwayResolver);
+            ackResponse = await callPathway('sys_generator_ack', { ...args, stream: false });
             if (ackResponse && ackResponse !== "none") {
                 await say(pathwayResolver.requestId, ackResponse, 100);
                 args.chatHistory.push({ role: 'assistant', content: ackResponse });

package/pathways/system/entity/sys_generator_voice_sample.js

@@ -4,8 +4,8 @@ export default {
     prompt:
         [
             new Prompt({ messages: [
-                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}\nYour voice communication system needs some examples to train it to sound like you. Based on your perception of yourself from your memories and your unique voice, generate some sample dialogue for your voice communication system to use as a reference for your style and tone. It can be anything, but make sure to overindex on your personality and voice for good training data. Make sure to reference a greeting and a closing statement. Put it between <EXAMPLE_DIALOGUE> tags and don't generate any other commentary outside of the tags.`},
-                {"role": "user", "content": `Generate a sample dialogue for your voice communication system to use as a reference for your style and tone.`},
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}\nYour voice communication system needs some examples to train it to sound like you. Based on your unique voice and style, generate some sample dialogue for your voice communication system to use as a reference for your style and tone. It can be anything, but make sure to overindex on your personality for good training examples. Make sure to reference a greeting and a closing statement. Put it between <EXAMPLE_DIALOGUE> tags and don't generate any other commentary outside of the tags.`},
+                {"role": "user", "content": `Generate a sample dialogue for your voice communication system to use as a reference for representingyour style and tone.`},
             ]}),
         ],
     inputParameters: {

package/pathways/translate_gpt4_omni.js

@@ -0,0 +1,20 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text to translate in any language, assistant will create a translation of that text in {{to}}. All text that the user posts is to be translated - assistant must not respond to the user in any way and should produce only the translation with no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 1000,
+    model: 'oai-gpt4o',
+    enableDuplicateRequests: false,
+    useParallelChunkProcessing: true,
+
+}

package/pathways/translate_subtitle.js

@@ -1,164 +1,291 @@
-import subsrt from "subsrt";
 import logger from "../lib/logger.js";
 import { callPathway } from "../lib/pathwayTools.js";
-import { publishRequestProgress } from "../lib/redisSubscription.js";
 
-function preprocessStr(str) {
+function preprocessStr(str, format) {
   try {
     if (!str) return "";
-    return (
-      str
-        .replace(/\r\n?/g, "\n")
-        .replace(/\n+/g, "\n")
-        .replace(/(\d+)\n(\d{2}:\d{2}:\d{2},\d{3})/g, "\n\n$1\n$2")
-        .trim() + "\n\n"
-    );
+    let content = str
+      // Normalize line endings
+      .replace(/\r\n?/g, "\n")
+      // Remove WEBVTT header for processing
+      .replace(/^WEBVTT\n\n/, '');
+
+    // For SRT, convert commas to dots in timestamps
+    if (format === 'srt') {
+      content = content.replace(/(\d{2}:\d{2}:\d{2}),(\d{3})/g, "$1.$2");
+    }
+
+    return content
+      // Ensure each subtitle block is properly separated
+      .split(/\n\s*\n/)
+      .map(block => block.trim())
+      .filter(block => {
+        // Match both numeric indices (SRT) and optional caption identifiers (VTT)
+        const firstLine = block.split('\n')[0];
+        return block && (
+          /^\d+$/.test(firstLine) || // SRT style
+          /^\d{2}:\d{2}/.test(firstLine) || // VTT style without identifier
+          /^[^\n]+\n\d{2}:\d{2}/.test(block) // VTT style with identifier
+        );
+      })
+      .join("\n\n")
+      + "\n\n";
   } catch (e) {
     logger.error(`An error occurred in content text preprocessing: ${e}`);
     return "";
   }
 }
 
-async function processBatch(batch, args) {
-  const batchText = batch
-    .map((caption, index) => `LINE#${index + 1}: ${caption.content}`)
-    .join("\n");
+function timeToMs(timeStr) {
+  const [time, ms] = timeStr.split(/[.,]/);
+  const [hours, minutes, seconds] = time.split(':').map(Number);
+  return (hours * 3600 + minutes * 60 + seconds) * 1000 + parseInt(ms);
+}
 
-  const translatedText = await callPathway("translate_subtitle_helper", {
-    ...args,
-    text: batchText,
-    async: false,
-  });
+function msToTimestamp(ms, format) {
+  const date = new Date(ms);
+  const timestamp = date.toISOString().slice(11, 23);
+  return format === 'srt' ? timestamp.replace('.', ',') : timestamp;
+}
 
-  // Remove LINE# and LINE() labels
-  const translatedLines = translatedText.split("\n");
-    translatedLines.forEach((line, i) => {
-    translatedLines[i] = line.replace(/^LINE#\d+:\s*/, "").trim();
-  });
-  //make sure translatedLines.length===batch.length
-  if (translatedLines.length < batch.length) {
-    const emptyLines = Array(batch.length - translatedLines.length).fill("-");
-    translatedLines.push(...emptyLines);
-  } else if (translatedLines.length > batch.length) {
-    //first remove the empty lines
-    translatedLines.splice(0, translatedLines.length, ...translatedLines.filter(line => line.trim() !== ""));
-
-    if(translatedLines.length>batch.length) {
-        //merge extra lines to end
-        const lastLine = translatedLines[batch.length - 1];
-        const mergedLines = translatedLines.slice(batch.length);
-        mergedLines.unshift(lastLine);
-        translatedLines.splice(batch.length - 1, translatedLines.length - batch.length + 1, mergedLines.join(" "));
-    }else {
-        const emptyLines = Array(batch.length - translatedLines.length).fill("-");
-        translatedLines.push(...emptyLines);
+function parseSubtitles(content, format) {
+  const blocks = content.split(/\n\s*\n/).filter(block => block.trim());
+  const captions = [];
+  
+  for (const block of blocks) {
+    const lines = block.split('\n');
+    if (lines.length < 2) continue;
+    
+    let index, timelineIndex;
+    if (format === 'srt') {
+      // SRT format: numeric index required
+      if (!/^\d+$/.test(lines[0])) continue;
+      index = parseInt(lines[0]);
+      timelineIndex = 1;
+    } else {
+      // VTT format: optional identifier
+      timelineIndex = /^\d{2}:\d{2}/.test(lines[0]) ? 0 : 1;
+      index = timelineIndex === 0 ? captions.length + 1 : lines[0];
     }
+    
+    const timeMatch = lines[timelineIndex].match(/^(\d{2}:\d{2}:\d{2}[.,]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[.,]\d{3})/);
+    if (!timeMatch) continue;
+    
+    const startTime = timeMatch[1].replace(',', '.');
+    const endTime = timeMatch[2].replace(',', '.');
+    const content = lines.slice(timelineIndex + 1).join('\n');
+    
+    captions.push({
+      type: "caption",
+      index: typeof index === 'number' ? index : captions.length + 1,
+      identifier: typeof index === 'string' ? index : null,
+      start: timeToMs(startTime),
+      end: timeToMs(endTime),
+      duration: timeToMs(endTime) - timeToMs(startTime),
+      content: content,
+      text: content
+    });
   }
+  
+  return captions;
+}
 
+function splitIntoOverlappingChunks(captions, chunkSize = 20, overlap = 3) {
+  const chunks = [];
+  for (let i = 0; i < captions.length; i += (chunkSize - overlap)) {
+    const end = Math.min(i + chunkSize, captions.length);
+    const chunk = captions.slice(i, end);
+    chunks.push({
+      captions: chunk,
+      startIndex: i,
+      endIndex: end - 1,
+      isOverlap: i > 0 || end < captions.length
+    });
+  }
+  return chunks;
+}
 
-  // Handle last empty line
-    if (translatedLines[translatedLines.length - 1].trim() === "") {
-        let lastNonEmptyIndex = translatedLines.length - 2;
-        while (lastNonEmptyIndex >= 0 && translatedLines[lastNonEmptyIndex].trim() === "") {
-            lastNonEmptyIndex--;
-        }
-        if (lastNonEmptyIndex >= 0) {
-            translatedLines[translatedLines.length - 1] = translatedLines[lastNonEmptyIndex];
-            translatedLines[lastNonEmptyIndex] = "";
-        }
-    }
-
+function selectBestTranslation(translations, startIndex, endIndex) {
+  // If we only have one translation for this caption, use it
+  if (translations.length === 1) return translations[0];
 
-  return batch.map((caption, i) => ({
-    ...caption,
-    content: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
-    text: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
-  }));
+  // For multiple translations, prefer the one from the middle of its chunk
+  // This helps avoid edge effects in translation
+  return translations.reduce((best, current) => {
+    const currentDistance = Math.min(
+      Math.abs(current.chunkStart - startIndex),
+      Math.abs(current.chunkEnd - endIndex)
+    );
+    const bestDistance = Math.min(
+      Math.abs(best.chunkStart - startIndex),
+      Math.abs(best.chunkEnd - endIndex)
+    );
+    return currentDistance < bestDistance ? current : best;
+  });
 }
 
-async function myResolver(args, requestId) {
-  try {
-    const { text, format } = args;
-    const captions = subsrt.parse(preprocessStr(text), {
-      format: format,
-      verbose: true,
-      eol: "\n",
-    });
-    const maxLineCount = 100;
-    const maxWordCount = 300;
-    let translatedCaptions = [];
-    let currentBatch = [];
-    let currentWordCount = 0;
-
-    const totalCount = captions.length;
-    let completedCount = 0;
-
-    const sendProgress = () => {
-      if (completedCount >= totalCount) return;
-      if(!requestId) {
-        logger.warn(`No requestId found for progress update`);
-        return;
-      }
+function validateFinalOutput(result, originalText, format) {
+  // Basic structure validation
+  if (!result || !result.trim()) {
+    logger.error("Empty or whitespace-only result");
+    return false;
+  }
+  
+  // Check for VTT header if needed
+  if (format === 'vtt' && !result.startsWith('WEBVTT\n\n')) {
+    logger.error("Missing WEBVTT header");
+    return false;
+  }
+  
+  // Check for timestamp format
+  const timestampPattern = format === 'srt' 
+    ? /\d{2}:\d{2}:\d{2},\d{3}\s*-->\s*\d{2}:\d{2}:\d{2},\d{3}/
+    : /\d{2}:\d{2}:\d{2}\.\d{3}\s*-->\s*\d{2}:\d{2}:\d{2}\.\d{3}/;
+  
+  const hasTimestamps = timestampPattern.test(result);
+  if (!hasTimestamps) {
+    logger.error(`No valid ${format.toUpperCase()} timestamps found in result`);
+    return false;
+  }
 
-      const progress = completedCount / totalCount;
-      logger.info(`Progress for ${requestId}: ${progress}`);
+  // Check overall length ratio
+  if (result.length < originalText.length * 0.5) {
+    logger.error(`Result length (${result.length}) is less than 50% of original length (${originalText.length})`);
+    return false;
+  }
 
-      publishRequestProgress({
-        requestId,
-        progress,
-        data: null,
-      });
-    };
-
-    for (let i = 0; i < captions.length; i++) {
-      const caption = captions[i];
-      const captionWordCount = caption.content.split(/\s+/).length;
-      if (
-        (currentWordCount + captionWordCount > maxWordCount ||
-          currentBatch.length >= maxLineCount) &&
-        currentBatch.length > 0
-      ) {
-        completedCount=i;
-        sendProgress();
-        const translatedBatch = await processBatch(
-          currentBatch,
-          args,
-        );
-        translatedCaptions = translatedCaptions.concat(translatedBatch);
-        currentBatch = [];
-        currentWordCount = 0;
+  // Validate subtitle block structure
+  const blocks = result.split(/\n\s*\n/).filter(block => block.trim());
+  
+  // Skip WEBVTT header for VTT format
+  const startIndex = format === 'vtt' && blocks[0].trim() === 'WEBVTT' ? 1 : 0;
+  
+  for (let i = startIndex; i < blocks.length; i++) {
+    const block = blocks[i];
+    const lines = block.trim().split('\n');
+    
+    if (lines.length < 2) {
+      logger.error(`Block ${i + 1} has insufficient lines (${lines.length}):\n${block}`);
+      return false;
+    }
+    
+    // Find the timestamp line
+    let timestampLineIndex = -1;
+    for (let j = 0; j < lines.length; j++) {
+      if (timestampPattern.test(lines[j])) {
+        timestampLineIndex = j;
+        break;
       }
-      currentBatch.push(caption);
-      currentWordCount += captionWordCount;
     }
-
-    if (currentBatch.length > 0) {
-      const translatedBatch = await processBatch(
-        currentBatch,
-        args,
-      );
-      translatedCaptions = translatedCaptions.concat(translatedBatch);
+    
+    if (timestampLineIndex === -1) {
+      logger.error(`Block ${i + 1} has no valid timestamp line:\n${block}`);
+      return false;
     }
+    
+    // Check that we have content after the timestamp
+    if (timestampLineIndex === lines.length - 1) {
+      logger.error(`Block ${i + 1} has no content after timestamp:\n${block}`);
+      return false;
+    }
+    
+    // Log the content for inspection
+    logger.debug(`Block ${i + 1} content:\n${lines.slice(timestampLineIndex + 1).join('\n')}`);
+  }
 
-    return (
-      subsrt
-        .build(translatedCaptions, {
-          format: format === "vtt" ? "vtt" : "srt",
-          eol: "\n",
-        })
-        .trim() + "\n"
-    );
-  } catch (e) {
-    logger.warn(
-      `${e} - could be that there are no subtitles, so attempting block translation.`
-    );
+  return true;
+}
+
+async function translateChunk(chunk, args, maxRetries = 3) {
+  const format = args.format || 'srt';
+  const chunkText = chunk.captions
+    .map(c => {
+      const startTime = msToTimestamp(c.start, format);
+      const endTime = msToTimestamp(c.end, format);
+      const index = format === 'srt' || !c.identifier ? c.index : c.identifier;
+      return `${index}\n${startTime} --> ${endTime}\n${c.content}`;
+    })
+    .join('\n\n');
+  
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
     try {
-      return await callPathway("translate_gpt4_omni", {...args, async: false});
+      const translated = await callPathway("translate_subtitle_helper", {
+        ...args,
+        text: chunkText,
+        async: false,
+      });
+
+      // Basic validation - just check for SUBTITLES tags and some content
+      const match = translated.match(/<SUBTITLES>([\s\S]*)<\/SUBTITLES>/);
+      if (!match || !match[1].trim()) {
+        logger.warn(`Attempt ${attempt + 1}: Invalid translation format`);
+        continue;
+      }
+      
+      const content = match[1].trim();
+      const blocks = content.split(/\n\s*\n/);
+      
+      // Check if any blocks are empty or invalid
+      let hasEmptyBlocks = false;
+      const processedBlocks = chunk.captions.map((caption, index) => {
+        const block = blocks[index];
+        if (!block) {
+          logger.warn(`Attempt ${attempt + 1}: Empty block for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        const lines = block.split('\n');
+        if (lines.length < 3) {
+          logger.warn(`Attempt ${attempt + 1}: Invalid block structure for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        const content = lines.slice(2).join('\n').trim();
+        if (!content) {
+          logger.warn(`Attempt ${attempt + 1}: Empty content for caption ${caption.index}`);
+          hasEmptyBlocks = true;
+          return null;
+        }
+        
+        return {
+          ...caption,
+          content: content,
+          text: content,
+          chunkStart: chunk.startIndex,
+          chunkEnd: chunk.endIndex
+        };
+      });
+      
+      // If no empty blocks, return the processed blocks
+      if (!hasEmptyBlocks) {
+        return processedBlocks;
+      }
+      
+      // If this was the last attempt and we still have empty blocks,
+      // return what we have but keep original content for empty blocks
+      if (attempt === maxRetries - 1) {
+        logger.warn(`Failed to get valid translations for all blocks after ${maxRetries} attempts`);
+        return chunk.captions.map((caption, index) => {
+          return processedBlocks[index] || {
+            ...caption,
+            chunkStart: chunk.startIndex,
+            chunkEnd: chunk.endIndex
+          };
+        });
+      }
+      
+      // Otherwise, try again
+      logger.info(`Retrying chunk due to empty blocks (attempt ${attempt + 1}/${maxRetries})`);
+      
     } catch (e) {
-      logger.error(`An error occurred in subtitle translation: ${e}`);
-      return "";
+      logger.error(`Error translating chunk ${chunk.startIndex}-${chunk.endIndex} (attempt ${attempt + 1}): ${e}`);
+      if (attempt === maxRetries - 1) throw e;
     }
   }
+  
+  throw new Error(`Failed to translate chunk ${chunk.startIndex}-${chunk.endIndex} after ${maxRetries} attempts`);
 }
 
 export default {
@@ -173,9 +300,73 @@ export default {
   model: "oai-gpt4o",
   enableDuplicateRequests: false,
   timeout: 3600,
-  executePathway: async (executePathwayArgs) => {
-    const { args } = executePathwayArgs;
-    const requestId = executePathwayArgs?.resolver?.requestId;
-    return await myResolver(args, requestId);
+  executePathway: async ({args}) => {
+    try {
+      const { text, format = 'srt' } = args;
+      const preprocessedText = preprocessStr(text, format);
+      const captions = parseSubtitles(preprocessedText, format);
+  
+      if (!captions || captions.length === 0) {
+        throw new Error("No captions found in input");
+      }
+  
+      // Split into overlapping chunks
+      const chunks = splitIntoOverlappingChunks(captions);
+      logger.info(`Split subtitles into ${chunks.length} overlapping chunks`);
+      
+      // Translate all chunks in parallel
+      const chunkPromises = chunks.map(chunk => translateChunk(chunk, args));
+      const translatedChunks = await Promise.all(chunkPromises);
+      
+      // Create a map of caption index to all its translations
+      const translationMap = new Map();
+      translatedChunks.flat().forEach(caption => {
+        if (!translationMap.has(caption.index)) {
+          translationMap.set(caption.index, []);
+        }
+        translationMap.get(caption.index).push(caption);
+      });
+      
+      // Select best translation for each caption
+      const finalCaptions = captions.map(caption => {
+        const translations = translationMap.get(caption.index) || [caption];
+        return selectBestTranslation(translations, caption.index, caption.index);
+      });
+      
+      // Format the output
+      let result = finalCaptions
+        .map(caption => {
+          const startTime = msToTimestamp(caption.start, format);
+          const endTime = msToTimestamp(caption.end, format);
+          // Only include index/identifier if it was in the original
+          const hasIdentifier = caption.identifier !== null || format === 'srt';
+          const index = format === 'srt' || !caption.identifier ? caption.index : caption.identifier;
+          return hasIdentifier ? 
+            `${index}\n${startTime} --> ${endTime}\n${caption.content}` :
+            `${startTime} --> ${endTime}\n${caption.content}`;
+        })
+        .join('\n\n')
+        .trim();
+
+      // Add final newline only if input had one
+      if (text.endsWith('\n')) {
+        result += '\n';
+      }
+
+      // Add WEBVTT header for VTT format
+      if (format === 'vtt') {
+        result = 'WEBVTT\n\n' + result;
+      }
+
+      // Validate final output
+      if (!validateFinalOutput(result, text, format)) {
+        throw new Error("Final subtitle reconstruction failed validation");
+      }
+
+      return result;
+    } catch (e) {
+      logger.error(`Subtitle translation failed: ${e}`);
+      throw e;
+    }
   },
 };

package/pathways/translate_subtitle_helper.js

@@ -7,25 +7,13 @@ export default {
             {
                 role: "system",
                 content: 
-`Expert translator: Convert ALL text to {{to}}. Unbreakable rules:
-
-1. Translate EVERY SINGLE LINE. Zero exceptions.
-2. Output MUST have EXACTLY the same line count as input.
-3. One input line = One output line. Always.
-4. Only translations. Nothing extra.
-5. Non-translatable stays unchanged.
-6. Keep all formatting and characters.
-7. Prefix: "LINE#lineNumber:".
-8. Untranslatable: Copy as-is with prefix.
-9. Internal checks: Verify line count and content after each line.
-10. Final verification: Recount, check numbering, confirm content, cross-check with input.
-
-Translate ALL lines. Constant vigilance. Exhaustive final cross-check.`
+`You are an expert subtitle translator. You will be given a block of subtitles and asked to translate them into {{to}}.
+You must maintain the original format (caption numbers and timestamps) exactly and make the content fit as naturally as possible.
+Output only the translated subtitles in a <SUBTITLES> tag with no other text or commentary.`
             },
             {
                 role: "user",
-                // content: `"PreviousLines":\n{{{prevLine}}}\n\n"CurrentLines":\n{{{text}}}\n"NextLines":\n{{{nextLine}}}\n\n`,
-                content: `{{{text}}}`,
+                content: `<SUBTITLES>\n{{{text}}}\n</SUBTITLES>`,
             },
             ],
         }),

package/server/pathwayResolver.js

@@ -226,7 +226,7 @@ class PathwayResolver {
             try {
                 // Load saved context and core memory if it exists
                 const [savedContext, memorySelf, memoryDirectives, memoryTopics, memoryUser, memoryContext] = await Promise.all([
-                    (getv && getv(contextId)) || {},
+                    (getv && await getv(this.savedContextId)) || {},
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memorySelf', priority: 1}),
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryDirectives', priority: 1 }),
                     callPathway('sys_read_memory', { contextId: this.savedContextId, section: 'memoryTopics', priority: 0, numResults: 10 }),