@aj-archipelago/cortex 1.0.9 → 1.0.11

package/config.js

@@ -9,42 +9,42 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // Schema for config
 var config = convict({
-    pathwaysPath: {
+    basePathwayPath: {
         format: String,
-        default: path.join(process.cwd(), '/pathways'),
-        env: 'CORTEX_PATHWAYS_PATH'
+        default: path.join(__dirname, 'pathways', 'basePathway.js'),
+        env: 'CORTEX_BASE_PATHWAY_PATH'
     },
     corePathwaysPath: {
         format: String,
         default: path.join(__dirname, 'pathways'),
         env: 'CORTEX_CORE_PATHWAYS_PATH'
     },
-    basePathwayPath: {
+    cortexApiKey: {
         format: String,
-        default: path.join(__dirname, 'pathways', 'basePathway.js'),
-        env: 'CORTEX_BASE_PATHWAY_PATH'
-    },
-    storageConnectionString: {
-        doc: 'Connection string used for access to Storage',
-        format: '*',
-        default: '',
-        sensitive: true,
-        env: 'STORAGE_CONNECTION_STRING'
+        default: null,
+        env: 'CORTEX_API_KEY',
+        sensitive: true
     },
-    PORT: {
-        format: 'port',
-        default: 4000,
-        env: 'CORTEX_PORT'
+    cortexConfigFile: {
+        format: String,
+        default: null,
+        env: 'CORTEX_CONFIG_FILE'
     },
-    pathways: {
-        format: Object,
-        default: {}
+    defaultModelName: {
+        format: String,
+        default: null,
+        env: 'DEFAULT_MODEL_NAME'
     },
     enableCache: {
         format: Boolean,
         default: true,
         env: 'CORTEX_ENABLE_CACHE'
     },
+    enableDuplicateRequests: {
+        format: Boolean,
+        default: true,
+        env: 'CORTEX_ENABLE_DUPLICATE_REQUESTS'
+    },
     enableGraphqlCache: {
         format: Boolean,
         default: false,
@@ -55,17 +55,12 @@ var config = convict({
         default: false,
         env: 'CORTEX_ENABLE_REST'
     },
-    cortexApiKey: {
+    gcpServiceAccountKey: {
         format: String,
         default: null,
-        env: 'CORTEX_API_KEY',
+        env: 'GCP_SERVICE_ACCOUNT_KEY',
         sensitive: true
     },
-    defaultModelName: {
-        format: String,
-        default: null,
-        env: 'DEFAULT_MODEL_NAME'
-    },
     models: {
         format: Object,
         default: {
@@ -80,7 +75,8 @@ var config = convict({
                     "model": "gpt-3.5-turbo"
                 },
                 "requestsPerSecond": 10,
-                "maxTokenLength": 8192
+                "maxTokenLength": 8192,
+                "supportsStreaming": true,
             },
             "oai-whisper": {
                 "type": "OPENAI-WHISPER",
@@ -95,11 +91,6 @@ var config = convict({
         },
         env: 'CORTEX_MODELS'
     },
-    openaiDefaultModel: {
-        format: String,
-        default: 'gpt-3.5-turbo',
-        env: 'OPENAI_DEFAULT_MODEL'
-    },
     openaiApiKey: {
         format: String,
         default: null,
@@ -111,10 +102,31 @@ var config = convict({
         default: 'https://api.openai.com/v1/completions',
         env: 'OPENAI_API_URL'
     },
-    cortexConfigFile: {
+    openaiDefaultModel: {
         format: String,
-        default: null,
-        env: 'CORTEX_CONFIG_FILE'
+        default: 'gpt-3.5-turbo',
+        env: 'OPENAI_DEFAULT_MODEL'
+    },
+    pathways: {
+        format: Object,
+        default: {}
+    },
+    pathwaysPath: {
+        format: String,
+        default: path.join(process.cwd(), '/pathways'),
+        env: 'CORTEX_PATHWAYS_PATH'
+    },
+    PORT: {
+        format: 'port',
+        default: 4000,
+        env: 'CORTEX_PORT'
+    },
+    storageConnectionString: {
+        doc: 'Connection string used for access to Storage',
+        format: '*',
+        default: '',
+        sensitive: true,
+        env: 'STORAGE_CONNECTION_STRING'
     },
     whisperMediaApiUrl: {
         format: String,
@@ -126,12 +138,6 @@ var config = convict({
         default: 'null',
         env: 'WHISPER_TS_API_URL'
     },
-    gcpServiceAccountKey: {
-        format: String,
-        default: null,
-        env: 'GCP_SERVICE_ACCOUNT_KEY',
-        sensitive: true
-    },
 });
 
 // Read in environment variables and set up service configuration
@@ -145,7 +151,7 @@ if (configFile && fs.existsSync(configFile)) {
     const openaiApiKey = config.get('openaiApiKey');
     if (!openaiApiKey) {
         throw console.log('No config file or api key specified. Please set the OPENAI_API_KEY to use OAI or use CORTEX_CONFIG_FILE environment variable to point at the Cortex configuration for your project.');
-    }else {
+    } else {
         console.log(`Using default model with OPENAI_API_KEY environment variable`)
     }
 }
@@ -196,7 +202,7 @@ const buildPathways = async (config) => {
 const buildModels = (config) => {
     const { models } = config.getProperties();
 
-    for (const [key, model] of Object.entries(models)) {        
+    for (const [key, model] of Object.entries(models)) {
         // Compile handlebars templates for models
         models[key] = JSON.parse(HandleBars.compile(JSON.stringify(model))({ ...config.getEnv(), ...config.getProperties() }))
     }
@@ -213,8 +219,8 @@ const buildModels = (config) => {
     // Set default model name to the first model in the config in case no default is specified
     if (!config.get('defaultModelName')) {
         console.log('No default model specified, using first model as default.');
-        config.load({ defaultModelName: Object.keys(config.get('models'))[0] }); 
-    } 
+        config.load({ defaultModelName: Object.keys(config.get('models'))[0] });
+    }
 
     return models;
 }

package/helper_apps/MediaFileChunker/package-lock.json

@@ -18,7 +18,7 @@
         "ioredis": "^5.3.1",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "ytdl-core": "github:khlevon/node-ytdl-core#v4.11.4-patch.2"
+        "ytdl-core": "^4.11.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -1740,9 +1740,9 @@
       }
     },
     "node_modules/ytdl-core": {
-      "version": "0.0.0-development",
-      "resolved": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "license": "MIT",
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
       "dependencies": {
         "m3u8stream": "^0.8.6",
         "miniget": "^4.2.2",
@@ -2962,8 +2962,9 @@
       "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
     },
     "ytdl-core": {
-      "version": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "from": "ytdl-core@github:khlevon/node-ytdl-core#v4.11.4-patch.2",
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
       "requires": {
         "m3u8stream": "^0.8.6",
         "miniget": "^4.2.2",

package/helper_apps/MediaFileChunker/package.json

@@ -18,6 +18,6 @@
     "ioredis": "^5.3.1",
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
-    "ytdl-core": "github:khlevon/node-ytdl-core#v4.11.4-patch.2"
+    "ytdl-core": "^4.11.5"
   }
 }

package/lib/request.js

@@ -62,48 +62,163 @@ const postWithMonitor = async (model, url, data, axiosConfigObj) => {
     return cortexAxios.post(url, data, axiosConfigObj);
 }
 
-const MAX_RETRY = 10;
-const postRequest = async ({ url, data, params, headers, cache }, model) => {
-    const errors = []
+const MAX_RETRY = 10; // retries for error handling
+const MAX_DUPLICATE_REQUESTS = 3; // duplicate requests to manage latency spikes
+const DUPLICATE_REQUEST_AFTER = 10; // 10 seconds
+
+const postRequest = async ({ url, data, params, headers, cache }, model, requestId, pathway) => {
+    let promises = [];
     for (let i = 0; i < MAX_RETRY; i++) {
+        const modelProperties = config.get('models')[model];
+        const enableDuplicateRequests = pathway.enableDuplicateRequests !== undefined ? pathway.enableDuplicateRequests : config.get('enableDuplicateRequests');
+        let maxDuplicateRequests = enableDuplicateRequests ? MAX_DUPLICATE_REQUESTS : 1;
+        let duplicateRequestAfter = (pathway.duplicateRequestAfter || DUPLICATE_REQUEST_AFTER) * 1000;
+
+        if (enableDuplicateRequests) {
+            //console.log(`>>> [${requestId}] Duplicate requests enabled after ${duplicateRequestAfter / 1000} seconds`);
+        }
+
+        const axiosConfigObj = { params, headers, cache };
+        const streamRequested = (params.stream || data.stream);
+        if (streamRequested && modelProperties.supportsStreaming) {
+            axiosConfigObj.responseType = 'stream';
+            promises.push(limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj)));
+        } else {
+            if (streamRequested) {
+                console.log(`>>> [${requestId}] ${model} does not support streaming - sending non-streaming request`);
+                axiosConfigObj.params.stream = false;
+                data.stream = false;
+            }
+            const controllers = Array.from({ length: maxDuplicateRequests }, () => new AbortController());
+            promises = controllers.map((controller, index) =>
+                new Promise((resolve, reject) => {
+                    const duplicateRequestTime = duplicateRequestAfter * Math.pow(2, index) - duplicateRequestAfter;
+                    const jitter = duplicateRequestTime * 0.2 * Math.random();
+                    const duplicateRequestTimeout = Math.max(0, duplicateRequestTime + jitter);
+                    setTimeout(async () => {
+                        try {
+                            if (!limiters[model]) {
+                                throw new Error(`No limiter for model ${model}!`);
+                            }
+                            const axiosConfigObj = { params, headers, cache };
+
+                            let response = null;
+
+                            if (!controller.signal?.aborted) {
+
+                                axiosConfigObj.signal = controller.signal;
+                                axiosConfigObj.headers['X-Cortex-Request-Index'] = index;
+
+                                if (index === 0) {
+                                    //console.log(`>>> [${requestId}] sending request to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`);
+                                } else {
+                                    if (modelProperties.supportsStreaming) {
+                                        axiosConfigObj.responseType = 'stream';
+                                        axiosConfigObj.cache = false;
+                                    }
+                                    const logMessage = `>>> [${requestId}] taking too long - sending duplicate request ${index} to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`;
+                                    const header = '>'.repeat(logMessage.length);
+                                    console.log(`\n${header}\n${logMessage}`);
+                                }
+
+                                response = await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
+
+                                if (!controller.signal?.aborted) {
+
+                                    //console.log(`<<< [${requestId}] received response for request ${index}`);
+
+                                    if (axiosConfigObj.responseType === 'stream') {
+                                        // Buffering and collecting the stream data
+                                        console.log(`<<< [${requestId}] buffering streaming response for request ${index}`);
+                                        response = await new Promise((resolve, reject) => {
+                                            let responseData = '';
+                                            response.data.on('data', (chunk) => {
+                                                responseData += chunk;
+                                                //console.log(`<<< [${requestId}] received chunk for request ${index}`);
+                                            });
+                                            response.data.on('end', () => {
+                                                response.data = JSON.parse(responseData);
+                                                resolve(response);
+                                            });
+                                            response.data.on('error', (error) => {
+                                                reject(error);
+                                            });
+                                        });
+                                    }
+                                }
+                            }
+
+                            resolve(response);
+
+                        } catch (error) {
+                            if (error.name === 'AbortError' || error.name === 'CanceledError') {
+                                //console.log(`XXX [${requestId}] request ${index} was cancelled`);
+                                reject(error);
+                            } else {
+                                console.log(`!!! [${requestId}] request ${index} failed with error: ${error?.response?.data?.error?.message || error}`);
+                                reject(error);
+                            }
+                        } finally {
+                            controllers.forEach(controller => controller.abort());
+                        }
+                    }, duplicateRequestTimeout);
+                })
+            );
+        }
+
         try {
-            if (i > 0) {
-                console.log(`Retrying request #retry ${i}: ${JSON.stringify(data)}...`);
-                await new Promise(r => setTimeout(r, 200 * Math.pow(2, i))); // exponential backoff
-            }           
-            if (!limiters[model]) {
-                throw new Error(`No limiter for model ${model}!`);
+            const response = await Promise.race(promises);
+
+            if (response.status === 200) {
+                return response;
+            } else {
+                throw new Error(`Received error response: ${response.status}`);
             }
-            const axiosConfigObj = { params, headers, cache };
-            if (params.stream || data.stream) {
-                axiosConfigObj.responseType = 'stream';
-            }           
-            return await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
-        } catch (e) {
-            console.error(`Failed request with data ${JSON.stringify(data)}: ${e} - ${e.response?.data?.error?.type || 'error'}: ${e.response?.data?.error?.message}`);
-            if (e.response?.status && e.response?.status === 429) {
-                monitors[model].incrementError429Count();
+        } catch (error) {
+            //console.error(`!!! [${requestId}] failed request with data ${JSON.stringify(data)}: ${error}`);
+            if (error.response) {
+                const status = error.response.status;
+                if ((status === 429) || (status >= 500 && status < 600)) {
+                    if (status === 429) {
+                        monitors[model].incrementError429Count();
+                    }
+                    console.log(`>>> [${requestId}] retrying request due to ${status} response. Retry count: ${i + 1}`);
+                    if (i < MAX_RETRY - 1) {
+                        const backoffTime = 200 * Math.pow(2, i);
+                        const jitter = backoffTime * 0.2 * Math.random();
+                        await new Promise(r => setTimeout(r, backoffTime + jitter));
+                    } else {
+                        throw error;
+                    }
+                } else {
+                    throw error;
+                }
+            } else {
+                throw error;
             }
-            errors.push(e);
         }
     }
-    return { error: errors };
-}
-
-const request = async (params, model) => {
-    const response = await postRequest(params, model);
-    const { error, data, cached } = response;
-    if (cached) {
-        console.info('=== Request served with cached response. ===');
-    }
-    if (error && error.length > 0) {
-        const lastError = error[error.length - 1];
-        return { error: lastError.toJSON() ?? lastError ?? error };
-    }
+};
 
-    return data;
+const request = async (params, model, requestId, pathway) => {
+    try {
+        const response = await postRequest(params, model, requestId, pathway);
+        const { error, data, cached } = response;
+        if (cached) {
+            console.info(`<<< [${requestId}] served with cached response.`);
+        }
+        if (error && error.length > 0) {
+            const lastError = error[error.length - 1];
+            return { error: lastError.toJSON() ?? lastError ?? error };
+        }
+        //console.log("<<< [${requestId}] response: ", data.choices[0].delta || data.choices[0])
+        return data;
+    } catch (error) {
+        console.error(`Error in request: ${error.message || error}`);
+        return { error: error };
+    } 
 }
 
 export {
-    axios,request, postRequest, buildLimiters
+    axios, request, postRequest, buildLimiters
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.9",
+  "version": "1.0.11",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/pathways/basePathway.js

@@ -14,11 +14,17 @@ export default {
     typeDef,
     rootResolver,
     resolver,
-    inputFormat: 'text',
-    useInputChunking: true,
-    useParallelChunkProcessing: false,
-    useInputSummarization: false,    
-    truncateFromFront: false,
-    timeout: 120, // in seconds
+    inputFormat: 'text', // text or html - changes the behavior of the input chunking
+    useInputChunking: true, // true or false - enables input to be split into multiple chunks to meet context window size
+    useParallelChunkProcessing: false, // true or false - enables parallel processing of chunks
+    useInputSummarization: false, // true or false - instead of chunking, summarize the input and act on the summary    
+    truncateFromFront: false, // true or false - if true, truncate from the front of the input instead of the back
+    timeout: 120, // seconds, cancels the pathway after this many seconds
+    duplicateRequestAfter: 10, // seconds, if the request is not completed after this many seconds, a backup request is sent
+    // override the default execution of the pathway
+    // callback signature: excuteOverride({args: object, runAllPrompts: function})
+    // args: the input arguments to the pathway
+    // runAllPrompts: a function that runs all prompts in the pathway and returns the result
+    executePathway: undefined,
 };
 

package/pathways/index.js

@@ -8,6 +8,7 @@ import sentiment from './sentiment.js';
 import summary from './summary.js';
 import sys_openai_chat from './sys_openai_chat.js';
 import sys_openai_completion from './sys_openai_completion.js';
+import test_cohere_summarize from './test_cohere_summarize.js';
 import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
@@ -24,6 +25,7 @@ export {
     summary,
     sys_openai_chat,
     sys_openai_completion,
+    test_cohere_summarize,
     test_langchain,
     test_palm_chat,
     transcribe,

package/pathways/summary.js

@@ -21,7 +21,7 @@ export default {
         const originalTargetLength = args.targetLength;
 
         // If targetLength is not provided, execute the prompt once and return the result.
-        if (originalTargetLength === 0) {
+        if (originalTargetLength === 0 || originalTargetLength === null) {
             let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
             return await pathwayResolver.resolve(args);
         }

package/pathways/test_cohere_summarize.js

@@ -0,0 +1,10 @@
+// test_cohere_summarize.js
+// Summarize text with the Cohere model
+
+export default {
+    // Uncomment the following line to enable caching for this prompt, if desired.
+    // enableCache: true,
+    
+    prompt: `{{text}}`,
+    model: 'cohere-summarize'
+};

package/pathways/transcribe.js

@@ -8,6 +8,7 @@ export default {
         wordTimestamped: false,
     },
     timeout: 3600, // in seconds
+    enableDuplicateRequests: false,
 };
 
 

package/pathways/translate.js

@@ -16,5 +16,6 @@ export default {
     // Set the timeout for the translation process, in seconds.
     timeout: 400,
     inputChunkSize: 500,
+    enableDuplicateRequests: false,
 };
 

package/server/chunker.js

@@ -152,7 +152,7 @@ const getSemanticChunks = (text, chunkSize, inputFormat = 'text') => {
       }
     });
 
-    if (chunks.some(chunk => encode(chunk).length > chunkSize)) {
+    if (chunks.filter(c => determineTextFormat(c) === 'html').some(chunk => encode(chunk).length > chunkSize)) {
       throw new Error('The HTML contains elements that are larger than the chunk size. Please try again with HTML that has smaller elements.');
     }
 

package/server/graphql.js

@@ -20,6 +20,7 @@ import { cancelRequestResolver } from './resolver.js';
 import { buildPathways, buildModels } from '../config.js';
 import { requestState } from './requestState.js';
 import { buildRestEndpoints } from './rest.js';
+import { startTestServer } from '../tests/server.js'
 
 // Utility functions
 // Server plugins
@@ -222,7 +223,7 @@ const build = async (config) => {
         });
     };
 
-    return { server, startServer, cache, plugins, typeDefs, resolvers }
+    return { server, startServer, startTestServer, cache, plugins, typeDefs, resolvers }
 }
 
 

package/server/parser.js

@@ -37,9 +37,21 @@ const parseCommaSeparatedList = (str) => {
     return str.split(',').map(s => s.trim()).filter(s => s.length);
 }
 
+const isCommaSeparatedList = (data) => {
+    const commaSeparatedPattern = /^([^,\n]+,)+[^,\n]+$/;
+    return commaSeparatedPattern.test(data.trim());
+}
+
+const isNumberedList = (data) => {
+    const numberedListPattern = /^\s*[\[\{\(]*\d+[\s.=\-:,;\]\)\}]/gm;
+    return numberedListPattern.test(data.trim());
+}
+
 export {
     regexParser,
     parseNumberedList,
     parseNumberedObjectList,
     parseCommaSeparatedList,
+    isCommaSeparatedList,
+    isNumberedList,
 };

package/server/pathwayPrompter.js

@@ -3,10 +3,13 @@ import OpenAIChatPlugin from './plugins/openAiChatPlugin.js';
 import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
 import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
 import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
+import OpenAIChatExtensionPlugin from './plugins/openAiChatExtensionPlugin.js';
 import LocalModelPlugin from './plugins/localModelPlugin.js';
 import PalmChatPlugin from './plugins/palmChatPlugin.js';
 import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
 import PalmCodeCompletionPlugin from './plugins/palmCodeCompletionPlugin.js';
+import CohereGeneratePlugin from './plugins/cohereGeneratePlugin.js';
+import CohereSummarizePlugin from './plugins/cohereSummarizePlugin.js';
 
 class PathwayPrompter {
     constructor(config, pathway, modelName, model) {
@@ -17,6 +20,9 @@ class PathwayPrompter {
             case 'OPENAI-CHAT':
                 plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
                 break;
+            case 'OPENAI-CHAT-EXTENSION':
+                plugin = new OpenAIChatExtensionPlugin(config, pathway, modelName, model);
+                break;
             case 'AZURE-TRANSLATE':
                 plugin = new AzureTranslatePlugin(config, pathway, modelName, model);
                 break;
@@ -38,6 +44,12 @@ class PathwayPrompter {
             case 'PALM-CODE-COMPLETION':
                 plugin = new PalmCodeCompletionPlugin(config, pathway, modelName, model);
                 break;
+            case 'COHERE-GENERATE':
+                plugin = new CohereGeneratePlugin(config, pathway, modelName, model);
+                break;
+            case 'COHERE-SUMMARIZE':
+                plugin = new CohereSummarizePlugin(config, pathway, modelName, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }