@aj-archipelago/cortex 1.1.36 → 1.2.0

package/config.js

@@ -155,6 +155,21 @@ var config = convict({
                 "maxReturnTokens": 4096,
                 "supportsStreaming": true
             },
+            "oai-gpt4o-mini": {
+                "type": "OPENAI-VISION",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "gpt-4o-mini"
+                },
+                "requestsPerSecond": 50,
+                "maxTokenLength": 131072,
+                "maxReturnTokens": 4096,
+                "supportsStreaming": true
+            },
             "oai-o1-mini": {
                 "type": "OPENAI-REASONING",
                 "url": "https://api.openai.com/v1/chat/completions",
@@ -202,9 +217,48 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
             },
+            "replicate-flux-11-pro": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-1.1-pro/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-flux-1-schnell": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-schnell/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "replicate-flux-1-dev": {
+                "type": "REPLICATE-API",
+                "url": "https://api.replicate.com/v1/models/black-forest-labs/flux-dev/predictions",
+                "headers": {
+                    "Prefer": "wait",
+                    "Authorization": "Token {{REPLICATE_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+            },
+            "azure-video-translate": {
+                "type": "AZURE-VIDEO-TRANSLATE",
+                "headers": {
+                    "Content-Type": "application/json"
+                },
+                "supportsStreaming": true,
+            }
         },
         env: 'CORTEX_MODELS'
     },
+    azureVideoTranslationApiUrl: {
+        format: String,
+        default: 'http://127.0.0.1:5005',
+        env: 'AZURE_VIDEO_TRANSLATION_API_URL'
+    },
     openaiApiKey: {
         format: String,
         default: null,
@@ -248,6 +302,12 @@ var config = convict({
         env: 'REDIS_ENCRYPTION_KEY',
         sensitive: true
     },
+    replicateApiKey: {
+        format: String,
+        default: null,
+        env: 'REPLICATE_API_KEY',
+        sensitive: true
+    },
     runwareAiApiKey: {
         format: String,
         default: null,

package/lib/pathwayManager.js

@@ -3,6 +3,7 @@ import { S3 } from '@aws-sdk/client-s3';
 import fs from 'fs';
 import path from 'path';
 import logger from './logger.js';
+import { Prompt } from '../server/prompt.js';
 
 class StorageStrategy {
   async load() { throw new Error('Not implemented'); }
@@ -285,6 +286,33 @@ class PathwayManager {
     await this.storage.save(pathways);
   }
 
+  /**
+   * Transforms the prompts in a pathway to include the system prompt.
+   * @param {Object} pathway - The pathway object to transform.
+   * @param {string[]} pathway.prompt - Array of user prompts.
+   * @param {string} pathway.systemPrompt - The system prompt to prepend to each user prompt.
+   * @returns {Object} A new pathway object with transformed prompts.
+   */
+  async transformPrompts(pathway) {
+    const { prompt, systemPrompt } = pathway;
+
+    const newPathway = { ...pathway };
+
+    // Transform each prompt in the array
+    newPathway.prompt = prompt.map(p => {
+      return new Prompt({
+        messages: [
+          // Prepend the system prompt as a system message
+          { "role": "system", "content": systemPrompt },
+          // Add the original prompt as a user message
+          { "role": "user", "content": p },
+        ]
+      })
+    });
+
+    return newPathway;
+  }
+
   async putPathway(name, pathway, userId, secret, displayName) {
     if (!userId || !secret) {
       throw new Error('Both userId and secret are mandatory for adding or updating a pathway');
@@ -329,6 +357,7 @@ class PathwayManager {
     
     input PathwayInput {
       prompt: [String!]!
+      systemPrompt: String
       inputParameters: JSONObject
       model: String
       enableCache: Boolean
@@ -413,7 +442,7 @@ class PathwayManager {
       throw new Error(`Pathway '${pathwayName}' not found for user '${userId}'`);
     }
 
-    return pathways[userId][pathwayName];
+    return this.transformPrompts(pathways[userId][pathwayName]);
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.36",
+  "version": "1.2.0",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/flux_image.js

@@ -1,8 +1,9 @@
 export default {
   prompt: ["{{text}}"],
-  model: "runware-flux-schnell",
+
   enableDuplicateRequests: false,
   inputParameters: {
+    model: "runware-flux-schnell",
     negativePrompt: "",
     width: 512,
     height: 512,

package/pathways/index.js

@@ -65,6 +65,8 @@ import sys_openai_chat_gpt4 from './sys_openai_chat_gpt4.js';
 import sys_openai_chat_gpt4_32 from './sys_openai_chat_gpt4_32.js';
 import sys_openai_chat_gpt4_turbo from './sys_openai_chat_gpt4_turbo.js';
 import sys_openai_completion from './sys_openai_completion.js';
+import sys_parse_numbered_object_list from './sys_parse_numbered_object_list.js';
+import sys_repair_json from './sys_repair_json.js';
 import tags from './tags.js';
 import taxonomy from './taxonomy.js';
 import timeline from './timeline.js';
@@ -133,7 +135,10 @@ export {
     sys_google_code_chat,
     sys_google_gemini_chat, sys_openai_chat, sys_openai_chat_16, sys_openai_chat_gpt4, sys_openai_chat_gpt4_32,
     sys_openai_completion,
-    sys_openai_chat_gpt4_turbo, tags,
+    sys_openai_chat_gpt4_turbo,
+    sys_parse_numbered_object_list,
+    sys_repair_json,
+    tags,
     taxonomy,
     timeline, topics, topics_sentiment, transcribe,
     transcribe_neuralspace,

package/pathways/sys_parse_numbered_object_list.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: [
+        new Prompt({
+            messages: [
+                { "role": "system", "content": "Assistant is a list parsing AI. When user posts text including a numbered list and a desired set of fields, assistant will carefully read the list and attempt to convert the list into a JSON object with the given fields. If there are extra fields, assistant will ignore them. If there are some missing fields, assistant will just skip the missing fields and return the rest. If the conversion is not at all possible, assistant will return an empty JSON array. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "user", "content": `Fields: {{{format}}}\nList: {{{text}}}`},
+            ]
+        })
+    ],
+    format: '',
+    model: 'oai-gpt4o',
+    temperature: 0.0,
+    enableCache: true,
+    enableDuplicateRequests: false,
+    json: true
+}
+

package/pathways/sys_repair_json.js

@@ -0,0 +1,17 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: [
+        new Prompt({
+            messages: [
+                { "role": "system", "content": "Assistant is a JSON repair assistant. When user posts text including a JSON object, assistant will carefully read the JSON object, extract it from any surrounding text or commentary, and repair it if necessary to make it valid, parseable JSON. If there is no JSON in the response, assistant will return an empty JSON object. Assistant will generate only the repaired JSON object in a directly parseable format with no markdown surrounding it and no other response or commentary." },
+                { "role": "user", "content": `{{{text}}}`},
+            ]
+        })
+    ],
+    model: 'oai-gpt4o-mini',
+    temperature: 0.0,
+    enableCache: true,
+    enableDuplicateRequests: false,
+}
+

package/server/chunker.js

@@ -27,7 +27,6 @@ const getFirstNTokenSingle = (text, maxTokenLen) => {
   return text;
 }
 
-
 function getFirstNTokenArray(content, tokensToKeep) {
   let totalTokens = 0;
   let result = [];
@@ -71,138 +70,182 @@ const determineTextFormat = (text) => {
 }
 
 const getSemanticChunks = (text, chunkSize, inputFormat = 'text') => {
-  const breakByRegex = (str, regex, preserveWhitespace = false) => {
-    const result = [];
-    let match;
-  
-    while ((match = regex.exec(str)) !== null) {
-      const value = str.slice(0, match.index);
-      result.push(value);
-  
-      if (preserveWhitespace || /\S/.test(match[0])) {
-        result.push(match[0]);
-      }
-  
-      str = str.slice(match.index + match[0].length);
-    }
-  
-    if (str) {
-      result.push(str);
-    }
-  
-    return result.filter(Boolean);
-  };
-
-  const breakByParagraphs = (str) => breakByRegex(str, /[\r\n]+/, true);
-  const breakBySentences = (str) => breakByRegex(str, /(?<=[.。؟！?!\n])\s+/, true);
-  const breakByWords = (str) => breakByRegex(str, /(\s,;:.+)/);
-
-  const breakByHtmlElements = (str) => {
-    const $ = cheerio.load(str, null, true);
-
-    // the .filter() call is important to get the text nodes
-    // https://stackoverflow.com/questions/54878673/cheerio-get-normal-text-nodes
-    let rootNodes = $('body').contents();
-
-    // create an array with the outerHTML of each node
-    const nodes = rootNodes.map((i, el) => $(el).prop('outerHTML') || $(el).text()).get();
+  if (!Number.isInteger(chunkSize) || chunkSize <= 0) {
+    throw new Error('Invalid chunkSize: must be a positive integer');
+  }
 
-    return nodes;
-};
+  if (inputFormat === 'html') {
+    return getHtmlChunks(text, chunkSize);
+  } else {
+    // Pre-calculate encoding ratio with a sample to avoid encoding entire text
+    const sampleSize = Math.min(500, text.length);
+    const sample = text.slice(0, sampleSize);
+    const sampleEncoded = encode(sample);
+    const avgCharsPerToken = sample.length / sampleEncoded.length;
+    const charChunkSize = Math.round(chunkSize * avgCharsPerToken);
+    return findChunks(text, charChunkSize, chunkSize);
+  }
+}
 
-  const createChunks = (tokens) => {
-    let chunks = [];
-    let currentChunk = '';
+const getHtmlChunks = (html, chunkSize) => {
+  const $ = cheerio.load(html, null, true);
+  const nodes = $('body').contents().map((_, el) => $.html(el)).get();
   
-    for (const token of tokens) {
-      const currentTokenLength = encode(currentChunk + token).length;
-      if (currentTokenLength <= chunkSize) {
-        currentChunk += token;
-      } else {
-        if (currentChunk) {
-          chunks.push(currentChunk);
-        }
-        currentChunk = token;
-      }
-    }
+  let chunks = [];
+  let currentChunk = '';
   
-    if (currentChunk) {
-      chunks.push(currentChunk);
+  for (const node of nodes) {
+    if (encode(node).length > chunkSize && node.startsWith('<') && node.endsWith('>')) {
+      throw new Error('The HTML contains elements that are larger than the chunk size. Please try again with HTML that has smaller elements.');
     }
-  
-    return chunks;
-  };
-
-  const combineChunks = (chunks) => {
-    let optimizedChunks = [];
-  
-    for (let i = 0; i < chunks.length; i++) {
-      if (i < chunks.length - 1) {
-        const combinedChunk = chunks[i] + chunks[i + 1];
-        const combinedLen = encode(combinedChunk).length;
-  
-        if (combinedLen <= chunkSize) {
-          optimizedChunks.push(combinedChunk);
-          i += 1;
-        } else {
-          optimizedChunks.push(chunks[i]);
-        }
+    
+    if (encode(currentChunk + node).length <= chunkSize) {
+      currentChunk += node;
+    } else {
+      if (currentChunk) {
+        chunks.push(currentChunk);
+        currentChunk = '';
+      }
+      if (encode(node).length > chunkSize) {
+        // If the node is larger than chunkSize, split it
+        const textChunks = getSemanticChunks(node, chunkSize, 'text');
+        chunks.push(...textChunks);
       } else {
-        optimizedChunks.push(chunks[i]);
+        currentChunk = node;
       }
     }
+  }
+  
+  if (currentChunk) {
+    chunks.push(currentChunk);
+  }
   
-    return optimizedChunks;
-  };
+  return chunks;
+};
+
+const findChunks = (text, chunkSize, maxTokenLen) => {
+  const chunks = [];
+  let startIndex = 0;
 
-  const breakText = (str) => {
-    const tokenLength = encode(str).length;
+  while (startIndex < text.length) {
+    let endIndex = Math.min(startIndex + chunkSize, text.length);
 
-    if (tokenLength <= chunkSize) {
-      return [str];
+    if (endIndex == text.length) {
+      chunks.push(text.slice(startIndex));
+      break;
     }
 
-    const breakers = [breakByParagraphs, breakBySentences, breakByWords];
-
-    for (let i = 0; i < breakers.length; i++) {
-      const tokens = breakers[i](str);
-      if (tokens.length > 1) {
-        let chunks = createChunks(tokens);
-        chunks = combineChunks(chunks);
-        const brokenChunks = chunks.flatMap(breakText);
-        if (brokenChunks.every(chunk => encode(chunk).length <= chunkSize)) {
-          return brokenChunks;
-        }
-      }
+    const searchWindow = text.slice(startIndex, endIndex);
+    
+    // Find semantic break point, minimum 1 character
+    let breakPoint = Math.max(findSemanticBreak(searchWindow), 1);
+    let chunk = searchWindow.slice(0, breakPoint);
+
+    // If chunk is too large, reduce size until it fits
+    while (encode(chunk).length > maxTokenLen && chunkSize > 1) {
+      // reduce chunk size by a proportional amount
+      const reductionFactor = maxTokenLen / encode(chunk).length;
+      chunkSize = Math.floor(chunkSize * reductionFactor);
+      endIndex = Math.min(chunkSize, searchWindow.length);
+      breakPoint = Math.max(findSemanticBreak(searchWindow.slice(0, endIndex)), 1);
+      chunk = searchWindow.slice(0, breakPoint);
     }
 
-    return createChunks([...str]); // Split by characters
-  };
+    // Force single character if still too large
+    if (encode(chunk).length > maxTokenLen) {
+      breakPoint = 1;
+      chunk = searchWindow.slice(0, 1);
+    }
 
-  if (inputFormat === 'html') {
-    const tokens = breakByHtmlElements(text);
-    let chunks = createChunks(tokens);
-    chunks = combineChunks(chunks);
+    chunks.push(chunk);
+    startIndex += breakPoint;
+  }
 
-    chunks = chunks.flatMap(chunk => {
-      if (determineTextFormat(chunk) === 'text') {
-        return getSemanticChunks(chunk, chunkSize);
-      } else {
-        return chunk;
-      }
-    });
+  return chunks;
+}
 
-    if (chunks.filter(c => determineTextFormat(c) === 'html').some(chunk => encode(chunk).length > chunkSize)) {
-      throw new Error('The HTML contains elements that are larger than the chunk size. Please try again with HTML that has smaller elements.');
+const findSemanticBreak = (text) => {
+  const findLastDelimiter = (text, delimiters) => {
+    let lastIndex = -1;
+    for (const delimiter of delimiters) {
+      const index = text.lastIndexOf(delimiter);
+      if (index > -1) {
+        const delimitedIndex = index + delimiter.length;
+        if (delimitedIndex > lastIndex) lastIndex = delimitedIndex;
+      }
     }
-
-    return chunks;
+    return lastIndex;
   }
-  else {
-      return breakText(text);
-  }
-}
 
+  let breakIndex;
+
+  // Look for paragraph break (including different newline styles)
+  const paragraphDelimiters = ['\n\n', '\r\n\r\n', '\r\r', '\n'];
+  breakIndex = findLastDelimiter(text, paragraphDelimiters);
+  if (breakIndex !== -1) return breakIndex;
+
+  // Look for sentence break
+  const sentenceDelimiters = [
+    // Latin/European
+    '.', '!', '?', 
+    // CJK
+    '。', '！', '？', '．', '…', 
+    // Arabic/Persian/Urdu
+    '؟', '۔', '.',
+    // Devanagari/Hindi
+    '।',
+    // Thai
+    '๏', 'ฯ',
+    // Armenian
+    '։',
+    // Ethiopian
+    '።'
+  ];
+  breakIndex = findLastDelimiter(text, sentenceDelimiters);
+  if (breakIndex !== -1) return breakIndex;
+
+  // Look for phrase break
+  const phraseDelimiters = [
+    // Latin/European
+    '-', ';', ':', ',',
+    // CJK
+    '、', '，', '；', '：', '─',
+    // Arabic/Persian/Urdu
+    '،', '؛', '٬',
+    // Devanagari/Hindi
+    '॥', ',',
+    // Thai
+    '๚', '、'
+  ];
+  breakIndex = findLastDelimiter(text, phraseDelimiters);
+  if (breakIndex !== -1) return breakIndex;
+
+  // Look for word break (Unicode whitespace)
+  const whitespaceDelimiters = [
+    ' ',    // Space
+    '\t',   // Tab
+    '\u00A0', // No-Break Space
+    '\u1680', // Ogham Space Mark
+    '\u2000', // En Quad
+    '\u2001', // Em Quad
+    '\u2002', // En Space
+    '\u2003', // Em Space
+    '\u2004', // Three-Per-Em Space
+    '\u2005', // Four-Per-Em Space
+    '\u2006', // Six-Per-Em Space
+    '\u2007', // Figure Space
+    '\u2008', // Punctuation Space
+    '\u2009', // Thin Space
+    '\u200A', // Hair Space
+    '\u202F', // Narrow No-Break Space
+    '\u205F', // Medium Mathematical Space
+    '\u3000'  // Ideographic Space
+  ];
+  breakIndex = findLastDelimiter(text, whitespaceDelimiters);
+  if (breakIndex !== -1) return breakIndex;
+  
+  return text.length - 1;
+};
 
 const semanticTruncate = (text, maxLength) => {
   if (text.length <= maxLength) {
@@ -224,4 +267,4 @@ const getSingleTokenChunks = (text) => {
 
 export {
     getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat, getSingleTokenChunks
-};
+};

package/server/modelExecutor.js

@@ -25,7 +25,9 @@ import Gemini15VisionPlugin from './plugins/gemini15VisionPlugin.js';
 import AzureBingPlugin from './plugins/azureBingPlugin.js';
 import Claude3VertexPlugin from './plugins/claude3VertexPlugin.js';
 import NeuralSpacePlugin from './plugins/neuralSpacePlugin.js';
-import RunwareAiPlugin from './plugins/runwareAIPlugin.js';
+import RunwareAiPlugin from './plugins/runwareAiPlugin.js';
+import ReplicateApiPlugin from './plugins/replicateApiPlugin.js';
+import AzureVideoTranslatePlugin from './plugins/azureVideoTranslatePlugin.js';
 
 class ModelExecutor {
     constructor(pathway, model) {
@@ -108,6 +110,12 @@ class ModelExecutor {
             case 'RUNWARE-AI':
                 plugin = new RunwareAiPlugin(pathway, model);
                 break;
+            case 'REPLICATE-API':
+                plugin = new ReplicateApiPlugin(pathway, model);
+                break;
+            case 'AZURE-VIDEO-TRANSLATE':
+                plugin = new AzureVideoTranslatePlugin(pathway, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/server/parser.js

@@ -1,4 +1,5 @@
 import logger from '../lib/logger.js';
+import { callPathway } from '../lib/pathwayTools.js';
 
 //simply trim and parse with given regex
 const regexParser = (text, regex) => {
@@ -12,26 +13,14 @@ const parseNumberedList = (str) => {
     return regexParser(str, /^\s*[\[\{\(]*\d+[\s.=\-:,;\]\)\}]/gm);
 }
 
-// parse a numbered object list text format into list of objects
-const parseNumberedObjectList = (text, format) => {
-    const fields = format.match(/\b(\w+)\b/g);
-    const values = parseNumberedList(text);
-
-    const result = [];
-    for (const value of values) {
-        try {
-            const splitted = regexParser(value, /[:-](.*)/);
-            const obj = {};
-            for (let i = 0; i < fields.length; i++) {
-                obj[fields[i]] = splitted[i];
-            }
-            result.push(obj);
-        } catch (e) {
-            logger.warn(`Failed to parse value in parseNumberedObjectList, value: ${value}, fields: ${fields}`);
-        }
+async function parseNumberedObjectList(text, format) {
+    const parsedList = await callPathway('sys_parse_numbered_object_list', { text, format });
+    try {
+        return JSON.parse(parsedList);
+    } catch (error) {
+        logger.warn(`Failed to parse numbered object list: ${error.message}`);
+        return [];
     }
-
-    return result;
 }
 
 // parse a comma-separated list text format into list
@@ -49,25 +38,18 @@ const isNumberedList = (data) => {
     return numberedListPattern.test(data.trim());
 }
 
-function parseJson(str) {
+async function parseJson(str) {
     try {
-      const start = Math.min(
-        str.indexOf('{') !== -1 ? str.indexOf('{') : Infinity,
-        str.indexOf('[') !== -1 ? str.indexOf('[') : Infinity
-      );
-
-      const end = Math.max(
-        str.lastIndexOf('}') !== -1 ? str.lastIndexOf('}') + 1 : 0,
-        str.lastIndexOf(']') !== -1 ? str.lastIndexOf(']') + 1 : 0
-      );
-  
-      const jsonStr = str.slice(start, end);
-      // eslint-disable-next-line no-unused-vars
-      const json = JSON.parse(jsonStr);
-      return jsonStr;
+        JSON.parse(str); // Validate JSON
+        return str;
     } catch (error) {
-      logger.warn(`Pathway requires JSON format result. Failed to parse JSON: ${error.message}`);
-      return null;
+        try {
+            const repairedJson = await callPathway('sys_repair_json', { text: str });
+            return JSON.parse(repairedJson) ? repairedJson : null;
+        } catch (repairError) {
+            logger.warn(`Failed to parse JSON: ${repairError.message}`);
+            return null;
+        }
     }
 }
 

package/server/pathwayResolver.js

@@ -235,7 +235,7 @@ class PathwayResolver {
                 break;
             }
 
-            data = this.responseParser.parse(data);
+            data = await this.responseParser.parse(data);
             if (data !== null) {
                 break;
             }

package/server/pathwayResponseParser.js

@@ -5,7 +5,7 @@ class PathwayResponseParser {
         this.pathway = pathway;
     }
 
-    parse(data) {
+    async parse(data) {
         if (this.pathway.parser) {
             return this.pathway.parser(data);
         }
@@ -13,7 +13,7 @@ class PathwayResponseParser {
         if (this.pathway.list) {
             if (isNumberedList(data)) {
                 if (this.pathway.format) {
-                    return parseNumberedObjectList(data, this.pathway.format);
+                    return await parseNumberedObjectList(data, this.pathway.format);
                 }
                 return parseNumberedList(data);
             } else if (isCommaSeparatedList(data)) {
@@ -23,7 +23,7 @@ class PathwayResponseParser {
         }
 
         if (this.pathway.json) {
-            return parseJson(data);
+            return await parseJson(data);
         }
 
         return data;