@aj-archipelago/cortex 1.1.21 → 1.1.23

package/pathways/transcribe.js

@@ -1,6 +1,6 @@
 export default {
     prompt: `{{text}}`,
-    model: `oai-whisper`,
+    model: `azure-whisper`,
     inputParameters: {
         file: ``,
         language: ``,
@@ -13,6 +13,4 @@ export default {
     },
     timeout: 3600, // in seconds
     enableDuplicateRequests: false,
-};
-
-
+};

package/pathways/translate.js

@@ -1,21 +1,19 @@
-// translate.js
-// Translation module
-// This module exports a prompt that takes an input text and translates it from one language to another.
+import { Prompt } from '../server/prompt.js';
 
 export default {
-    // Set the temperature to 0 to favor more deterministic output when generating translations.
-    temperature: 0,
 
-    prompt: `Translate the following text to {{to}}:\n\nOriginal Language:\n{{{text}}}\n\n{{to}}:\n`,
-
-    // Define input parameters for the prompt, such as the target language for translation.
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
     inputParameters: {
         to: `Arabic`,
+        tokenRatio: 0.2,
     },
-
-    // Set the timeout for the translation process, in seconds.
-    timeout: 400,
     inputChunkSize: 500,
+    model: 'oai-gpt4o',
     enableDuplicateRequests: false,
-};
 
+}

package/pathways/translate_azure.js

@@ -0,0 +1,13 @@
+// Description: Translate a text from one language to another
+
+export default {
+    temperature: 0,
+    prompt: `{{{text}}}`,
+    inputParameters: {
+        to: `en`,
+        tokenRatio: 0.2,
+    },
+    //inputChunkSize: 500,
+    model: 'azure-translate',
+    timeout: 120,
+}

package/pathways/translate_context.js

@@ -0,0 +1,21 @@
+// Description: Translate a text from one language to another
+
+export default {
+    temperature: 0,
+    prompt:
+        [
+            // `{{{text}}}\n\nList all of the named entities in the above document in the original language:\n`,
+            //`{{{previousResult}}}\n\nTranslate this list to {{to}}:\n`,
+            //`{{{text}}}\nTranscribe the names of all people and places exactly from this document in the original language:\n`,
+            `{{{text}}}\nCopy the names of all people and places exactly from this document in the language above:\n`,
+            //`{{{previousResult}}}\n\nTranscribe exactly to {{to}}:\n`,
+            `Original Language:\n{{{previousResult}}}\n\n{{to}}:\n`,
+            //`Entities in the document:\n\n{{{previousResult}}}\n\nDocument:\n{{{text}}}\nTranslate the document to {{to}} and rewrite it to sound like a native {{to}} speaker:\n\n`
+            `Entities in the document:\n\n{{{previousResult}}}\n\nDocument:\n{{{text}}}\nRewrite the document in {{to}}. If the document is already in {{to}}, copy it exactly below:\n`
+        ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    timeout: 120,
+}

package/pathways/translate_gpt4.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpt4',
+    enableDuplicateRequests: false,
+
+}

package/pathways/translate_gpt4_turbo.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpt4-turbo',
+    enableDuplicateRequests: false,
+
+}

package/pathways/translate_subtitle.js

@@ -0,0 +1,201 @@
+import subsrt from "subsrt";
+import logger from "../lib/logger.js";
+import { callPathway } from "../lib/pathwayTools.js";
+
+function preprocessStr(str) {
+  try {
+    if (!str) return "";
+    return (
+      str
+        .replace(/\r\n?/g, "\n")
+        .replace(/\n+/g, "\n")
+        .replace(/(\d+)\n(\d{2}:\d{2}:\d{2},\d{3})/g, "\n\n$1\n$2")
+        .trim() + "\n\n"
+    );
+  } catch (e) {
+    logger.error(`An error occurred in content text preprocessing: ${e}`);
+    return "";
+  }
+}
+
+function getContextLines(captions, startIndex, direction, wordLimit = 100) {
+  let context = "";
+  let wordCount = 0;
+  let i = startIndex;
+
+  while (i >= 0 && i < captions.length && wordCount < wordLimit) {
+    const words = captions[i].content.split(/\s+/);
+    if (wordCount + words.length <= wordLimit) {
+      context =
+        direction === "prev"
+          ? captions[i].content + " " + context
+          : context + " " + captions[i].content;
+      wordCount += words.length;
+    } else {
+      const remainingWords = wordLimit - wordCount;
+      const partialContent =
+        direction === "prev"
+          ? words.slice(-remainingWords).join(" ")
+          : words.slice(0, remainingWords).join(" ");
+      context =
+        direction === "prev"
+          ? partialContent + " " + context
+          : context + " " + partialContent;
+      break;
+    }
+    i += direction === "prev" ? -1 : 1;
+  }
+
+  return context.trim();
+}
+
+async function processBatch(batch, args, captions, batchStartIndex) {
+  const batchText = batch
+    .map((caption, index) => `LINE#${index + 1}: ${caption.content}`)
+    .join("\n");
+  const prevLines = getContextLines(captions, batchStartIndex - 1, "prev");
+  const nextLines = getContextLines(
+    captions,
+    batchStartIndex + batch.length,
+    "next"
+  );
+
+  const translatedText = await callPathway("translate_subtitle_helper", {
+    ...args,
+    text: batchText,
+    prevLines,
+    nextLines,
+    async: false,
+  });
+
+  // Remove LINE# and LINE() labels
+    const translatedLines = translatedText.split("\n");
+    translatedLines.forEach((line, i) => {
+    translatedLines[i] = line.replace(/^LINE#\d+:\s*/, "").trim();
+    });
+  //make sure translatedLines.length===batch.length
+  if (translatedLines.length < batch.length) {
+    const emptyLines = Array(batch.length - translatedLines.length).fill("");
+    translatedLines.push(...emptyLines);
+  } else if (translatedLines.length > batch.length) {
+    //first remove the empty lines
+    translatedLines.splice(0, translatedLines.length, ...translatedLines.filter(line => line.trim() !== ""));
+
+    if(translatedLines.length>batch.length) {
+        //merge extra lines to end
+        const lastLine = translatedLines[batch.length - 1];
+        const mergedLines = translatedLines.slice(batch.length);
+        mergedLines.unshift(lastLine);
+        translatedLines.splice(batch.length - 1, translatedLines.length - batch.length + 1, mergedLines.join(" "));
+    }else {
+        const emptyLines = Array(batch.length - translatedLines.length).fill("");
+        translatedLines.push(...emptyLines);
+    }
+  }
+
+
+  // Handle last empty line
+    if (translatedLines[translatedLines.length - 1].trim() === "") {
+        let lastNonEmptyIndex = translatedLines.length - 2;
+        while (lastNonEmptyIndex >= 0 && translatedLines[lastNonEmptyIndex].trim() === "") {
+            lastNonEmptyIndex--;
+        }
+        if (lastNonEmptyIndex >= 0) {
+            translatedLines[translatedLines.length - 1] = translatedLines[lastNonEmptyIndex];
+            translatedLines[lastNonEmptyIndex] = "";
+        }
+    }
+
+
+  return batch.map((caption, i) => ({
+    ...caption,
+    content: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
+    text: translatedLines[i].replace(/^LINE\(\d+\):\s*/, "").trim(),
+  }));
+}
+
+async function myResolver(args) {
+  try {
+    const { text, format } = args;
+    const captions = subsrt.parse(preprocessStr(text), {
+      format: format,
+      verbose: true,
+      eol: "\n",
+    });
+    const maxLineCount = 100;
+    const maxWordCount = 1000;
+    let translatedCaptions = [];
+    let currentBatch = [];
+    let currentWordCount = 0;
+    let batchStartIndex = 0;
+
+    for (let i = 0; i < captions.length; i++) {
+      const caption = captions[i];
+      const captionWordCount = caption.content.split(/\s+/).length;
+      if (
+        (currentWordCount + captionWordCount > maxWordCount ||
+          currentBatch.length >= maxLineCount) &&
+        currentBatch.length > 0
+      ) {
+        const translatedBatch = await processBatch(
+          currentBatch,
+          args,
+          captions,
+          batchStartIndex
+        );
+        translatedCaptions = translatedCaptions.concat(translatedBatch);
+        currentBatch = [];
+        currentWordCount = 0;
+        batchStartIndex = i;
+      }
+      currentBatch.push(caption);
+      currentWordCount += captionWordCount;
+    }
+
+    if (currentBatch.length > 0) {
+      const translatedBatch = await processBatch(
+        currentBatch,
+        args,
+        captions,
+        batchStartIndex
+      );
+      translatedCaptions = translatedCaptions.concat(translatedBatch);
+    }
+
+    return (
+      subsrt
+        .build(translatedCaptions, {
+          format: format === "vtt" ? "vtt" : "srt",
+          eol: "\n",
+        })
+        .trim() + "\n"
+    );
+  } catch (e) {
+    logger.error(
+      `An error occurred in subtitle translation, trying direct translation next: ${e}`
+    );
+    try {
+      return await callPathway("translate_gpt4", {...args, async: false});
+    } catch (e) {
+      logger.error(`An error occurred in subtitle translation: ${e}`);
+      return "";
+    }
+  }
+}
+
+export default {
+  inputParameters: {
+    to: `Arabic`,
+    tokenRatio: 0.2,
+    format: `srt`,
+    prevLines: ``,
+    nextLines: ``,
+  },
+  inputChunkSize: 500,
+  model: "oai-gpt4o",
+  enableDuplicateRequests: false,
+  timeout: 3600,
+  executePathway: async ({ args }) => {
+    return await myResolver(args);
+  },
+};

package/pathways/translate_subtitle_helper.js

@@ -0,0 +1,31 @@
+import { Prompt } from '../server/prompt.js';
+
+
+export default {
+    prompt: [
+        new Prompt({
+            messages: [
+            {
+                role: "system",
+                content: 
+                `Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. User will most probably produce previous and next lines for context with "PreviousLines" and "NextLines" labels, and you are asked to translate current lines one by one in given sequence with "CurrentLines" label. CurrentLines might have numbered labels as LINE#{lineNo} e.g. LINE#1, LINE#2. If currentline is a word only translate that word. You must keep input and output number of lines same, so do not merge translation of lines, single line must always map to single line. Assistant's output translated number of lines must always be equal to the input number of currentlines. For output, Assistant will produce only the translated text, ignore all LINE#{lineNo} and "CurrentLines" labels, and give no additional notes or commentary.`,
+            },
+            {
+                role: "user",
+                content: `"PreviousLines":\n{{{prevLine}}}\n\n"CurrentLines":\n{{{text}}}\n"NextLines":\n{{{nextLine}}}\n\n`,
+            },
+            ],
+        }),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+        format: `srt`,
+        prevLine: ``,
+        nextLine: ``,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpt4o',
+    enableDuplicateRequests: false,
+
+}

package/pathways/translate_turbo.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+
+    prompt: [
+        new Prompt({ messages: [
+            {"role": "system", "content": "Assistant is a highly skilled multilingual translator for a prestigious news agency. When the user posts any text in any language, assistant will create a translation of that text in {{to}}. Assistant will produce only the translation and no additional notes or commentary."},
+            {"role": "user", "content": "{{{text}}}"}
+        ]}),
+    ],
+    inputParameters: {
+        to: `Arabic`,
+        tokenRatio: 0.2,
+    },
+    inputChunkSize: 500,
+    model: 'oai-gpturbo',
+    enableDuplicateRequests: false,
+
+}

package/pathways/vision.js

@@ -1,18 +1,20 @@
 import { Prompt } from '../server/prompt.js';
 
 export default {
-    prompt: [
-        new Prompt({ messages: [
-            "{{chatHistory}}",
-        ]}),
-    ],
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Jarvis Vision, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your primary expertise is image analysis. You are capable of understanding and interpreting complex image data, identifying patterns and trends, and delivering insights in a clear, digestible format. You know the current date and time - it is {{now}}."},
+                "{{chatHistory}}",
+            ]}),
+        ],
     inputParameters: {
         chatHistory: [{role: '', content: []}],
         contextId: ``,
     },
     max_tokens: 1024,
-    model: 'oai-gpt4-vision',
-    tokenRatio: 0.96,
+    model: 'oai-gpt4o',
     useInputChunking: false,
     enableDuplicateRequests: false,
+    timeout: 600,
 }

package/server/pathwayResolver.js

@@ -94,7 +94,7 @@ class PathwayResolver {
 
         // If the response is a string, it's a regular long running response
         if (args.async || typeof responseData === 'string') {
-            const { completedCount, totalCount } = requestState[this.requestId];
+            const { completedCount=1, totalCount=1 } = requestState[this.requestId];
             requestState[this.requestId].data = responseData;
             
             // some models don't support progress updates

package/server/plugins/azureCognitivePlugin.js

@@ -34,7 +34,7 @@ class AzureCognitivePlugin extends ModelPlugin {
     async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId, cortexRequest) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
-        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo } = combinedParameters;
+        const { inputVector, calculateInputVector, privateData, filter, docId, title, chunkNo, chatId } = combinedParameters;
         const data = {};
 
         if (mode == 'delete') {
@@ -46,6 +46,10 @@ class AzureCognitivePlugin extends ModelPlugin {
                 searchQuery += ` AND docId:'${docId}'`;
             }
 
+            if (chatId) {
+                searchQuery += ` AND chatId:'${chatId}'`;
+            }
+
             cortexRequest.url = searchUrl;
             cortexRequest.data =
             { search: searchQuery,  
@@ -75,6 +79,7 @@ class AzureCognitivePlugin extends ModelPlugin {
                 content: text,
                 owner: savedContextId,
                 docId: docId || uuidv4(),
+                chatId: chatId,
                 createdAt: new Date().toISOString()
             }
 
@@ -116,6 +121,10 @@ class AzureCognitivePlugin extends ModelPlugin {
         if (indexName == 'indexcortex') { //if private, filter by owner via contextId //privateData && 
             data.filter && (data.filter = data.filter + ' and ');
             data.filter = `owner eq '${savedContextId}'`;
+
+            if(chatId){
+                data.filter += ` and chatId eq '${chatId}'`;
+            }
         }
 
         return { data };

package/server/plugins/openAiVisionPlugin.js

@@ -1,30 +1,38 @@
 import OpenAIChatPlugin from './openAiChatPlugin.js';
 
+function safeJsonParse(content) {
+    try {
+        const parsedContent = JSON.parse(content);
+        return (typeof parsedContent === 'object' && parsedContent !== null) ? parsedContent : content;
+    } catch (e) {
+        return content;
+    }
+}
 
 class OpenAIVisionPlugin extends OpenAIChatPlugin {
-
+    
     tryParseMessages(messages) {
-        messages.map(message => {
+        return messages.map(message => {
             try {
                 if (typeof message.content === 'string') {
-                    message.content = JSON.parse(message.content);
+                    message.content = safeJsonParse(message.content);
                 }
                 if (Array.isArray(message.content)) {
                     message.content = message.content.map(item => {
                         if (typeof item === 'string') {
                             return { type: 'text', text: item };
                         } else {
-                            const parsedItem = JSON.parse(item);
+                            const parsedItem = safeJsonParse(item);
                             const { type, text, image_url, url } = parsedItem;
                             return { type, text, image_url: url || image_url };
                         }
                     });
-                }     
+                }
             } catch (e) {
                 return message;
             }
+            return message;
         });
-        return messages;
     }
 
     getRequestParameters(text, parameters, prompt) {

package/tests/main.test.js

@@ -4,6 +4,13 @@
 
 import test from 'ava';
 import serverFactory from '../index.js';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+import fs from 'fs';
+import path from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 
 let testServer;
 
@@ -85,8 +92,8 @@ test('validates edit endpoint', async (t) => {
 
 test('validates summary endpoint', async (t) => {
     const response = await testServer.executeOperation({
-        query: 'query summary($text: String!) { summary(text: $text) { result } }',
-        variables: { text: 'hello there my dear world!' },
+        query: 'query summary($text: String!, $targetLength: Int) { summary(text: $text, targetLength: $targetLength) { result } }',
+        variables: { text: 'Now is the time for all good men to come to the aid of their country.  We ride at dawn!', targetLength: 50 },
     });
 
     t.is(response.body?.singleResult?.errors, undefined);
@@ -348,3 +355,56 @@ test('test translate endpoint with huge arabic text english translation and chec
     // check return only contains non-Arabic characters
     t.notRegex(response.body?.singleResult?.data?.translate.result, /[ء-ي]/);
 });
+
+
+async function testTranslateSrt(t, text, language='English') {
+    const response = await testServer.executeOperation({
+        query: 'query translate_subtitle($text: String!, $to:String) { translate_subtitle(text: $text, to:$to) { result } }',
+        variables: {
+            to: language,
+            text
+         },
+    });
+
+    t.falsy(response.body?.singleResult?.errors);
+
+    const result = response.body?.singleResult?.data?.translate_subtitle?.result;
+    t.true(result?.length > text.length*0.5);
+
+    //check all timestamps are still there and not translated
+    const originalTimestamps = text.match(/\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}/g);
+    const translatedTimestamps = result.match(/\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}/g);
+    
+    t.deepEqual(originalTimestamps, translatedTimestamps, 'All timestamps should be present and unchanged');
+
+    const originalLineCount = text.split('\n').length;
+    const translatedLineCount = result.split('\n').length;
+    
+    t.is(originalLineCount, translatedLineCount, 'Total number of lines should be the same');
+}
+
+test('test translate_srt endpoint with simple srt', async t => {
+    const text = `1
+00:00:03,069 --> 00:00:04,771
+Who’s that?
+
+2
+00:00:04,771 --> 00:00:06,039
+Aseel.
+
+3
+00:00:06,039 --> 00:00:07,474
+Who is Aseel a mom to?
+
+4
+00:00:07,474 --> 00:00:09,376
+Aseel is mommy
+`;
+
+    await testTranslateSrt(t, text, 'Spanish');
+});
+
+test('test translate_srt endpoint with long srt file', async t => {
+    const text = fs.readFileSync(path.join(__dirname, 'sublong.srt'), 'utf8');
+    await testTranslateSrt(t, text, 'English');
+});