@aj-archipelago/cortex 1.0.8 → 1.0.10

package/config.js

@@ -9,42 +9,42 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 // Schema for config
 var config = convict({
-    pathwaysPath: {
+    basePathwayPath: {
         format: String,
-        default: path.join(process.cwd(), '/pathways'),
-        env: 'CORTEX_PATHWAYS_PATH'
+        default: path.join(__dirname, 'pathways', 'basePathway.js'),
+        env: 'CORTEX_BASE_PATHWAY_PATH'
     },
     corePathwaysPath: {
         format: String,
         default: path.join(__dirname, 'pathways'),
         env: 'CORTEX_CORE_PATHWAYS_PATH'
     },
-    basePathwayPath: {
+    cortexApiKey: {
         format: String,
-        default: path.join(__dirname, 'pathways', 'basePathway.js'),
-        env: 'CORTEX_BASE_PATHWAY_PATH'
-    },
-    storageConnectionString: {
-        doc: 'Connection string used for access to Storage',
-        format: '*',
-        default: '',
-        sensitive: true,
-        env: 'STORAGE_CONNECTION_STRING'
+        default: null,
+        env: 'CORTEX_API_KEY',
+        sensitive: true
     },
-    PORT: {
-        format: 'port',
-        default: 4000,
-        env: 'CORTEX_PORT'
+    cortexConfigFile: {
+        format: String,
+        default: null,
+        env: 'CORTEX_CONFIG_FILE'
     },
-    pathways: {
-        format: Object,
-        default: {}
+    defaultModelName: {
+        format: String,
+        default: null,
+        env: 'DEFAULT_MODEL_NAME'
     },
     enableCache: {
         format: Boolean,
         default: true,
         env: 'CORTEX_ENABLE_CACHE'
     },
+    enableDuplicateRequests: {
+        format: Boolean,
+        default: true,
+        env: 'CORTEX_ENABLE_DUPLICATE_REQUESTS'
+    },
     enableGraphqlCache: {
         format: Boolean,
         default: false,
@@ -55,17 +55,12 @@ var config = convict({
         default: false,
         env: 'CORTEX_ENABLE_REST'
     },
-    cortexApiKey: {
+    gcpServiceAccountKey: {
         format: String,
         default: null,
-        env: 'CORTEX_API_KEY',
+        env: 'GCP_SERVICE_ACCOUNT_KEY',
         sensitive: true
     },
-    defaultModelName: {
-        format: String,
-        default: null,
-        env: 'DEFAULT_MODEL_NAME'
-    },
     models: {
         format: Object,
         default: {
@@ -80,7 +75,8 @@ var config = convict({
                     "model": "gpt-3.5-turbo"
                 },
                 "requestsPerSecond": 10,
-                "maxTokenLength": 8192
+                "maxTokenLength": 8192,
+                "supportsStreaming": true,
             },
             "oai-whisper": {
                 "type": "OPENAI-WHISPER",
@@ -95,11 +91,6 @@ var config = convict({
         },
         env: 'CORTEX_MODELS'
     },
-    openaiDefaultModel: {
-        format: String,
-        default: 'gpt-3.5-turbo',
-        env: 'OPENAI_DEFAULT_MODEL'
-    },
     openaiApiKey: {
         format: String,
         default: null,
@@ -111,10 +102,31 @@ var config = convict({
         default: 'https://api.openai.com/v1/completions',
         env: 'OPENAI_API_URL'
     },
-    cortexConfigFile: {
+    openaiDefaultModel: {
         format: String,
-        default: null,
-        env: 'CORTEX_CONFIG_FILE'
+        default: 'gpt-3.5-turbo',
+        env: 'OPENAI_DEFAULT_MODEL'
+    },
+    pathways: {
+        format: Object,
+        default: {}
+    },
+    pathwaysPath: {
+        format: String,
+        default: path.join(process.cwd(), '/pathways'),
+        env: 'CORTEX_PATHWAYS_PATH'
+    },
+    PORT: {
+        format: 'port',
+        default: 4000,
+        env: 'CORTEX_PORT'
+    },
+    storageConnectionString: {
+        doc: 'Connection string used for access to Storage',
+        format: '*',
+        default: '',
+        sensitive: true,
+        env: 'STORAGE_CONNECTION_STRING'
     },
     whisperMediaApiUrl: {
         format: String,
@@ -126,12 +138,6 @@ var config = convict({
         default: 'null',
         env: 'WHISPER_TS_API_URL'
     },
-    gcpServiceAccountKey: {
-        format: String,
-        default: null,
-        env: 'GCP_SERVICE_ACCOUNT_KEY',
-        sensitive: true
-    },
 });
 
 // Read in environment variables and set up service configuration

package/helper_apps/MediaFileChunker/package-lock.json

@@ -18,7 +18,7 @@
         "ioredis": "^5.3.1",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "ytdl-core": "github:khlevon/node-ytdl-core#v4.11.4-patch.2"
+        "ytdl-core": "^4.11.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -1740,9 +1740,9 @@
       }
     },
     "node_modules/ytdl-core": {
-      "version": "0.0.0-development",
-      "resolved": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "license": "MIT",
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
       "dependencies": {
         "m3u8stream": "^0.8.6",
         "miniget": "^4.2.2",
@@ -2962,8 +2962,9 @@
       "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
     },
     "ytdl-core": {
-      "version": "git+ssh://git@github.com/khlevon/node-ytdl-core.git#87450450caabb91f81afa6e66758bf2f629664a1",
-      "from": "ytdl-core@github:khlevon/node-ytdl-core#v4.11.4-patch.2",
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
       "requires": {
         "m3u8stream": "^0.8.6",
         "miniget": "^4.2.2",

package/helper_apps/MediaFileChunker/package.json

@@ -18,6 +18,6 @@
     "ioredis": "^5.3.1",
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
-    "ytdl-core": "github:khlevon/node-ytdl-core#v4.11.4-patch.2"
+    "ytdl-core": "^4.11.5"
   }
 }

package/lib/request.js

@@ -62,48 +62,149 @@ const postWithMonitor = async (model, url, data, axiosConfigObj) => {
     return cortexAxios.post(url, data, axiosConfigObj);
 }
 
-const MAX_RETRY = 10;
-const postRequest = async ({ url, data, params, headers, cache }, model) => {
-    const errors = []
+const MAX_RETRY = 10; // retries for error handling
+const MAX_DUPLICATE_REQUESTS = 3; // duplicate requests to manage latency spikes
+const DUPLICATE_REQUEST_AFTER = 10; // 10 seconds
+
+const postRequest = async ({ url, data, params, headers, cache }, model, requestId, pathway) => {
+    let promises = [];
     for (let i = 0; i < MAX_RETRY; i++) {
+        const modelProperties = config.get('models')[model];
+        const enableDuplicateRequests = pathway.enableDuplicateRequests !== undefined ? pathway.enableDuplicateRequests : config.get('enableDuplicateRequests');
+        let maxDuplicateRequests = enableDuplicateRequests ? MAX_DUPLICATE_REQUESTS : 1;
+        let duplicateRequestAfter = (pathway.duplicateRequestAfter || DUPLICATE_REQUEST_AFTER) * 1000;
+
+        if (enableDuplicateRequests) {
+            //console.log(`>>> [${requestId}] Duplicate requests enabled after ${duplicateRequestAfter / 1000} seconds`);
+        }
+
+        const axiosConfigObj = { params, headers, cache };
+        const streamRequested = (params.stream || data.stream);
+        if (streamRequested && modelProperties.supportsStreaming) {
+            axiosConfigObj.responseType = 'stream';
+            promises.push(limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj)));
+        } else {
+            if (streamRequested) {
+                console.log(`>>> [${requestId}] ${model} does not support streaming - sending non-streaming request`);
+                axiosConfigObj.params.stream = false;
+                data.stream = false;
+            }
+            const controllers = Array.from({ length: maxDuplicateRequests }, () => new AbortController());
+            promises = controllers.map((controller, index) =>
+                new Promise((resolve, reject) => {
+                    const duplicateRequestTime = duplicateRequestAfter * Math.pow(2, index) - duplicateRequestAfter;
+                    const jitter = duplicateRequestTime * 0.2 * Math.random();
+                    const duplicateRequestTimeout = Math.max(0, duplicateRequestTime + jitter);
+                    setTimeout(async () => {
+                        try {
+                            if (!limiters[model]) {
+                                throw new Error(`No limiter for model ${model}!`);
+                            }
+                            const axiosConfigObj = { params, headers, cache };
+
+                            let response = null;
+
+                            if (!controller.signal?.aborted) {
+
+                                axiosConfigObj.signal = controller.signal;
+                                axiosConfigObj.headers['X-Cortex-Request-Index'] = index;
+
+                                if (index === 0) {
+                                    //console.log(`>>> [${requestId}] sending request to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`);
+                                } else {
+                                    if (modelProperties.supportsStreaming) {
+                                        axiosConfigObj.responseType = 'stream';
+                                        axiosConfigObj.cache = false;
+                                    }
+                                    const logMessage = `>>> [${requestId}] taking too long - sending duplicate request ${index} to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`;
+                                    const header = '>'.repeat(logMessage.length);
+                                    console.log(`\n${header}\n${logMessage}`);
+                                }
+
+                                response = await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
+
+                                if (!controller.signal?.aborted) {
+
+                                    //console.log(`<<< [${requestId}] received response for request ${index}`);
+
+                                    if (axiosConfigObj.responseType === 'stream') {
+                                        // Buffering and collecting the stream data
+                                        console.log(`<<< [${requestId}] buffering streaming response for request ${index}`);
+                                        response = await new Promise((resolve, reject) => {
+                                            let responseData = '';
+                                            response.data.on('data', (chunk) => {
+                                                responseData += chunk;
+                                                //console.log(`<<< [${requestId}] received chunk for request ${index}`);
+                                            });
+                                            response.data.on('end', () => {
+                                                response.data = JSON.parse(responseData);
+                                                resolve(response);
+                                            });
+                                            response.data.on('error', (error) => {
+                                                reject(error);
+                                            });
+                                        });
+                                    }
+                                }
+                            }
+
+                            resolve(response);
+
+                        } catch (error) {
+                            if (error.name === 'AbortError' || error.name === 'CanceledError') {
+                                //console.log(`XXX [${requestId}] request ${index} was cancelled`);
+                                reject(error);
+                            } else {
+                                console.log(`!!! [${requestId}] request ${index} failed with error: ${error?.response?.data?.error?.message || error}`);
+                                reject(error);
+                            }
+                        } finally {
+                            controllers.forEach(controller => controller.abort());
+                        }
+                    }, duplicateRequestTimeout);
+                })
+            );
+        }
+
         try {
-            if (i > 0) {
-                console.log(`Retrying request #retry ${i}: ${JSON.stringify(data)}...`);
-                await new Promise(r => setTimeout(r, 200 * Math.pow(2, i))); // exponential backoff
-            }           
-            if (!limiters[model]) {
-                throw new Error(`No limiter for model ${model}!`);
+            const response = await Promise.race(promises);
+
+            if (response.status === 200) {
+                return response;
+            } else {
+                throw new Error(`Received error response: ${response.status}`);
             }
-            const axiosConfigObj = { params, headers, cache };
-            if (params.stream || data.stream) {
-                axiosConfigObj.responseType = 'stream';
-            }           
-            return await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
-        } catch (e) {
-            console.error(`Failed request with data ${JSON.stringify(data)}: ${e} - ${e.response?.data?.error?.type || 'error'}: ${e.response?.data?.error?.message}`);
-            if (e.response?.status && e.response?.status === 429) {
+        } catch (error) {
+            //console.error(`!!! [${requestId}] failed request with data ${JSON.stringify(data)}: ${error}`);
+            if (error.response?.status === 429) {
                 monitors[model].incrementError429Count();
             }
-            errors.push(e);
+            console.log(`>>> [${requestId}] retrying request due to ${error.response?.status} response. Retry count: ${i + 1}`);
+            if (i < MAX_RETRY - 1) {
+                const backoffTime = 200 * Math.pow(2, i);
+                const jitter = backoffTime * 0.2 * Math.random();
+                await new Promise(r => setTimeout(r, backoffTime + jitter));
+            } else {
+                throw error;
+            }
         }
     }
-    return { error: errors };
-}
+};
 
-const request = async (params, model) => {
-    const response = await postRequest(params, model);
+const request = async (params, model, requestId, pathway) => {
+    const response = await postRequest(params, model, requestId, pathway);
     const { error, data, cached } = response;
     if (cached) {
-        console.info('=== Request served with cached response. ===');
+        console.info(`<<< [${requestId}] served with cached response.`);
     }
     if (error && error.length > 0) {
         const lastError = error[error.length - 1];
         return { error: lastError.toJSON() ?? lastError ?? error };
     }
-
+    //console.log("<<< [${requestId}] response: ", data.choices[0].delta || data.choices[0])
     return data;
 }
 
 export {
-    axios,request, postRequest, buildLimiters
+    axios, request, postRequest, buildLimiters
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/pathways/basePathway.js

@@ -14,11 +14,12 @@ export default {
     typeDef,
     rootResolver,
     resolver,
-    inputFormat: 'text',
-    useInputChunking: true,
-    useParallelChunkProcessing: false,
-    useInputSummarization: false,    
-    truncateFromFront: false,
-    timeout: 120, // in seconds
+    inputFormat: 'text', // text or html - changes the behavior of the input chunking
+    useInputChunking: true, // true or false - enables input to be split into multiple chunks to meet context window size
+    useParallelChunkProcessing: false, // true or false - enables parallel processing of chunks
+    useInputSummarization: false, // true or false - instead of chunking, summarize the input and act on the summary    
+    truncateFromFront: false, // true or false - if true, truncate from the front of the input instead of the back
+    timeout: 120, // seconds, cancels the pathway after this many seconds
+    duplicateRequestAfter: 10, // seconds, if the request is not completed after this many seconds, a backup request is sent
 };
 

package/pathways/index.js

@@ -8,6 +8,7 @@ import sentiment from './sentiment.js';
 import summary from './summary.js';
 import sys_openai_chat from './sys_openai_chat.js';
 import sys_openai_completion from './sys_openai_completion.js';
+import test_cohere_summarize from './test_cohere_summarize.js';
 import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
@@ -24,6 +25,7 @@ export {
     summary,
     sys_openai_chat,
     sys_openai_completion,
+    test_cohere_summarize,
     test_langchain,
     test_palm_chat,
     transcribe,

package/pathways/test_cohere_summarize.js

@@ -0,0 +1,10 @@
+// test_cohere_summarize.js
+// Summarize text with the Cohere model
+
+export default {
+    // Uncomment the following line to enable caching for this prompt, if desired.
+    // enableCache: true,
+    
+    prompt: `{{text}}`,
+    model: 'cohere-summarize'
+};

package/pathways/transcribe.js

@@ -8,6 +8,7 @@ export default {
         wordTimestamped: false,
     },
     timeout: 3600, // in seconds
+    enableDuplicateRequests: false,
 };
 
 

package/pathways/translate.js

@@ -16,5 +16,6 @@ export default {
     // Set the timeout for the translation process, in seconds.
     timeout: 400,
     inputChunkSize: 500,
+    enableDuplicateRequests: false,
 };
 

package/server/parser.js

@@ -37,9 +37,21 @@ const parseCommaSeparatedList = (str) => {
     return str.split(',').map(s => s.trim()).filter(s => s.length);
 }
 
+const isCommaSeparatedList = (data) => {
+    const commaSeparatedPattern = /^([^,\n]+,)+[^,\n]+$/;
+    return commaSeparatedPattern.test(data.trim());
+}
+
+const isNumberedList = (data) => {
+    const numberedListPattern = /^\s*[\[\{\(]*\d+[\s.=\-:,;\]\)\}]/gm;
+    return numberedListPattern.test(data.trim());
+}
+
 export {
     regexParser,
     parseNumberedList,
     parseNumberedObjectList,
     parseCommaSeparatedList,
+    isCommaSeparatedList,
+    isNumberedList,
 };

package/server/pathwayPrompter.js

@@ -7,6 +7,8 @@ import LocalModelPlugin from './plugins/localModelPlugin.js';
 import PalmChatPlugin from './plugins/palmChatPlugin.js';
 import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
 import PalmCodeCompletionPlugin from './plugins/palmCodeCompletionPlugin.js';
+import CohereGeneratePlugin from './plugins/cohereGeneratePlugin.js';
+import CohereSummarizePlugin from './plugins/cohereSummarizePlugin.js';
 
 class PathwayPrompter {
     constructor(config, pathway, modelName, model) {
@@ -38,6 +40,12 @@ class PathwayPrompter {
             case 'PALM-CODE-COMPLETION':
                 plugin = new PalmCodeCompletionPlugin(config, pathway, modelName, model);
                 break;
+            case 'COHERE-GENERATE':
+                plugin = new CohereGeneratePlugin(config, pathway, modelName, model);
+                break;
+            case 'COHERE-SUMMARIZE':
+                plugin = new CohereSummarizePlugin(config, pathway, modelName, model);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/server/pathwayResolver.js

@@ -9,6 +9,7 @@ import { Prompt } from './prompt.js';
 import { getv, setv } from '../lib/keyValueStorageClient.js';
 import { requestState } from './requestState.js';
 import { callPathway } from '../lib/pathwayTools.js';
+import { response } from 'express';
 
 class PathwayResolver {
     constructor({ config, pathway, args }) {
@@ -63,57 +64,116 @@ class PathwayResolver {
         this.pathwayPrompt = pathway.prompt;
     }
 
+    // This code handles async and streaming responses.  In either case, we use
+    // the graphql subscription to send progress updates to the client.  Most of 
+    // the time the client will be an external client, but it could also be the
+    // Cortex REST api code.
     async asyncResolve(args) {
-        const responseData = await this.promptAndParse(args);
+        const MAX_RETRY_COUNT = 3;
+        let attempt = 0;
+        let streamErrorOccurred = false;
+
+        while (attempt < MAX_RETRY_COUNT) {
+            const responseData = await this.promptAndParse(args);
+
+            if (args.async || typeof responseData === 'string') {
+                const { completedCount, totalCount } = requestState[this.requestId];
+                requestState[this.requestId].data = responseData;
+                pubsub.publish('REQUEST_PROGRESS', {
+                    requestProgress: {
+                        requestId: this.requestId,
+                        progress: completedCount / totalCount,
+                        data: JSON.stringify(responseData),
+                    }
+                });
+            } else {
+                try {
+                    const incomingMessage = responseData;
 
-        // Either we're dealing with an async request or a stream
-        if(args.async || typeof responseData === 'string') {
-            const { completedCount, totalCount } = requestState[this.requestId];
-            requestState[this.requestId].data = responseData;
-            pubsub.publish('REQUEST_PROGRESS', {
-                requestProgress: {
-                    requestId: this.requestId,
-                    progress: completedCount / totalCount,
-                    data: JSON.stringify(responseData),
-                }
-            });
-        } else { // stream
-            try {
-                const incomingMessage = Array.isArray(responseData) && responseData.length > 0 ? responseData[0] : responseData;
-                incomingMessage.on('data', data => {
-                    const events = data.toString().split('\n');
-            
-                    events.forEach(event => {
-                        if (event.trim() === '') return; // Skip empty lines
-            
-                        const message = event.replace(/^data: /, '');
-                        
-                        //console.log(`====================================`);
-                        //console.log(`STREAM EVENT: ${event}`);
-                        //console.log(`MESSAGE: ${message}`);
-            
-                        const requestProgress = {
-                            requestId: this.requestId,
-                            data: message,
-                        }
-            
-                        if (message.trim() === '[DONE]') {
-                            requestProgress.progress = 1;
-                        }
-            
+                    const processData = (data) => {
                         try {
-                            pubsub.publish('REQUEST_PROGRESS', {
-                                requestProgress: requestProgress
-                            });
+                            //console.log(`\n\nReceived stream data for requestId ${this.requestId}`, data.toString());
+                            let events = data.toString().split('\n');
+                            
+                            //events = "data: {\"id\":\"chatcmpl-20bf1895-2fa7-4ef9-abfe-4d142aba5817\",\"object\":\"chat.completion.chunk\",\"created\":1689303423723,\"model\":\"gpt-4\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":{\"error\":{\"message\":\"The server had an error while processing your request. Sorry about that!\",\"type\":\"server_error\",\"param\":null,\"code\":null}}},\"finish_reason\":null}]}\n\n".split("\n");
+
+                            for (let event of events) {
+                                if (streamErrorOccurred) break;
+                                
+                                // skip empty events
+                                if (!(event.trim() === '')) {
+                                    //console.log(`Processing stream event for requestId ${this.requestId}`, event);
+
+                                    let message = event.replace(/^data: /, '');
+
+                                    const requestProgress = {
+                                        requestId: this.requestId,
+                                        data: message,
+                                    }
+
+                                    // check for end of stream or in-stream errors
+                                    if (message.trim() === '[DONE]') {
+                                        requestProgress.progress = 1;
+                                    } else {
+                                        let parsedMessage;
+                                        try {
+                                            parsedMessage = JSON.parse(message);
+                                        } catch (error) {
+                                            console.error('Could not JSON parse stream message', message, error);
+                                            return;
+                                        }
+
+                                        const streamError = parsedMessage.error || parsedMessage?.choices?.[0]?.delta?.content?.error || parsedMessage?.choices?.[0]?.text?.error;
+                                        if (streamError) {
+                                            streamErrorOccurred = true;
+                                            console.error(`Stream error: ${streamError.message}`);
+                                            incomingMessage.off('data', processData); // Stop listening to 'data'
+                                            return;
+                                        }
+                                    }
+
+                                    try {
+                                        //console.log(`Publishing stream message to requestId ${this.requestId}`, message);
+                                        pubsub.publish('REQUEST_PROGRESS', {
+                                            requestProgress: requestProgress
+                                        });
+                                    } catch (error) {
+                                        console.error('Could not publish the stream message', message, error);
+                                    }
+                                };
+                            };
                         } catch (error) {
-                            console.error('Could not JSON parse stream message', message, error);
+                            console.error('Could not process stream data', error);
                         }
+                    };
+
+                    await new Promise((resolve, reject) => {
+                        incomingMessage.on('data', processData);
+                        incomingMessage.on('end', resolve);
+                        incomingMessage.on('error', reject);
                     });
-                });
-            } catch (error) {
-                console.error('Could not subscribe to stream', error);
+
+                } catch (error) {
+                    console.error('Could not subscribe to stream', error);
+                }
+            }
+
+            if (streamErrorOccurred) {
+                attempt++;
+                console.error(`Stream attempt ${attempt} failed. Retrying...`);
+                streamErrorOccurred = false; // Reset the flag for the next attempt
+            } else {
+                return;
             }
         }
+        // if all retries failed, publish the stream end message
+        pubsub.publish('REQUEST_PROGRESS', {
+            requestProgress: {
+                requestId: this.requestId,
+                progress: 1,
+                data: '[DONE]',
+            }
+        });
     }
 
     async resolve(args) {
@@ -167,7 +227,7 @@ class PathwayResolver {
         } else {
             chunkTokenLength = this.chunkMaxTokenLength;
         }
-        const encoded = encode(text);
+        const encoded = text ? encode(text) : [];
         if (!this.useInputChunking || encoded.length <= chunkTokenLength) { // no chunking, return as is
             if (encoded.length > 0 && encoded.length >= chunkTokenLength) {
                 const warnText = `Truncating long input text. Text length: ${text.length}`;
@@ -275,8 +335,11 @@ class PathwayResolver {
                     previousResult = this.truncate(previousResult, this.chunkMaxTokenLength);
                     result = await Promise.all(chunks.map(chunk =>
                         this.applyPrompt(this.prompts[i], chunk, currentParameters)));
-                    if (!currentParameters.stream) {
-                        result = result.join("\n\n")
+
+                    if (result.length === 1) {
+                        result = result[0];
+                    } else if (!currentParameters.stream) {
+                        result = result.join("\n\n");
                     }
                 }
 

package/server/pathwayResponseParser.js

@@ -1,29 +1,25 @@
-import { parseNumberedList, parseNumberedObjectList, parseCommaSeparatedList } from './parser.js';
+import { parseNumberedList, parseNumberedObjectList, parseCommaSeparatedList, isCommaSeparatedList, isNumberedList } from './parser.js';
 
 class PathwayResponseParser {
     constructor(pathway) {
         this.pathway = pathway;
     }
 
-    isCommaSeparatedList(data) {
-        const commaSeparatedPattern = /^([^,\n]+,)+[^,\n]+$/;
-        return commaSeparatedPattern.test(data.trim());
-    }
-
     parse(data) {
         if (this.pathway.parser) {
             return this.pathway.parser(data);
         }
 
         if (this.pathway.list) {
-            if (this.isCommaSeparatedList(data)) {
-                return parseCommaSeparatedList(data);
-            } else {
+            if (isNumberedList(data)) {
                 if (this.pathway.format) {
                     return parseNumberedObjectList(data, this.pathway.format);
                 }
                 return parseNumberedList(data);
+            } else if (isCommaSeparatedList(data)) {
+                return parseCommaSeparatedList(data);
             }
+            return [data];
         }
 
         return data;

package/server/plugins/azureTranslatePlugin.js

@@ -26,7 +26,7 @@ class AzureTranslatePlugin extends ModelPlugin {
     // Execute the request to the Azure Translate API
     async execute(text, parameters, prompt, pathwayResolver) {
         const requestParameters = this.getRequestParameters(text, parameters, prompt);
-        const requestId = pathwayResolver?.requestId;
+        const { requestId, pathway} = pathwayResolver;
 
         const url = this.requestUrl(text);
 
@@ -34,7 +34,7 @@ class AzureTranslatePlugin extends ModelPlugin {
         const params = requestParameters.params;
         const headers = this.model.headers || {};
 
-        return this.executeRequest(url, data, params, headers, prompt, requestId);
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
     }
     
     // Parse the response from the Azure Translate API

package/server/plugins/cohereGeneratePlugin.js

@@ -0,0 +1,60 @@
+// CohereGeneratePlugin.js
+import ModelPlugin from './modelPlugin.js';
+
+class CohereGeneratePlugin extends ModelPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+    }
+
+    // Set up parameters specific to the Cohere API
+    getRequestParameters(text, parameters, prompt) {
+        const { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+
+        // Define the model's max token length
+        const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
+    
+        // Check if the token length exceeds the model's max token length
+        if (tokenLength > modelTargetTokenLength) {
+            // Truncate the prompt text to fit within the token length
+            modelPromptText = modelPromptText.substring(0, modelTargetTokenLength);
+        }
+    
+        const requestParameters = {
+            model: "command",
+            prompt: modelPromptText,
+            max_tokens: this.getModelMaxReturnTokens(),
+            temperature: this.temperature ?? 0.7,
+            k: 0,
+            stop_sequences: parameters.stop_sequences || [],
+            return_likelihoods: parameters.return_likelihoods || "NONE"
+        };
+    
+        return requestParameters;
+    }
+
+    // Execute the request to the Cohere API
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const url = this.requestUrl();
+        const requestParameters = this.getRequestParameters(text, parameters, prompt);
+        const { requestId, pathway} = pathwayResolver;
+
+        const data = { ...(this.model.params || {}), ...requestParameters };
+        const params = {};
+        const headers = { 
+            ...this.model.headers || {}
+        };
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
+    }
+
+    // Parse the response from the Cohere API
+    parseResponse(data) {
+        const { generations } = data;
+        if (!generations || !generations.length) {
+            return data;
+        }
+        // Return the text of the first generation
+        return generations[0].text || null;
+    }
+}
+
+export default CohereGeneratePlugin;

package/server/plugins/cohereSummarizePlugin.js

@@ -0,0 +1,50 @@
+// CohereSummarizePlugin.js
+import ModelPlugin from './modelPlugin.js';
+
+class CohereSummarizePlugin extends ModelPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+    }
+
+    // Set up parameters specific to the Cohere Summarize API
+    getRequestParameters(text, parameters, prompt) {
+        const { modelPromptText } = this.getCompiledPrompt(text, parameters, prompt);
+
+        const requestParameters = {
+            length: parameters.length || "medium",
+            format: parameters.format || "paragraph",
+            model: "summarize-xlarge",
+            extractiveness: parameters.extractiveness || "low",
+            temperature: this.temperature ?? 0.3,
+            text: modelPromptText
+        };
+    
+        return requestParameters;
+    }
+
+    // Execute the request to the Cohere Summarize API
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const url = this.requestUrl();
+        const requestParameters = this.getRequestParameters(text, parameters, prompt);
+        const { requestId, pathway} = pathwayResolver;
+
+        const data = { ...(this.model.params || {}), ...requestParameters };
+        const params = {};
+        const headers = { 
+            ...this.model.headers || {}
+        };
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
+    }
+
+    // Parse the response from the Cohere Summarize API
+    parseResponse(data) {
+        const { summary } = data;
+        if (!summary) {
+            return data;
+        }
+        // Return the summary
+        return summary;
+    }
+}
+
+export default CohereSummarizePlugin;

package/server/plugins/modelPlugin.js

@@ -121,7 +121,16 @@ class ModelPlugin {
 
     // compile the Prompt    
     getCompiledPrompt(text, parameters, prompt) {
-        const combinedParameters = { ...this.promptParameters, ...parameters };
+        
+        const mergeParameters = (promptParameters, parameters) => {
+            let result = { ...promptParameters };
+            for (let key in parameters) {
+                if (parameters[key] !== null) result[key] = parameters[key];
+            }
+            return result;
+        }
+
+        const combinedParameters = mergeParameters(this.promptParameters, parameters);
         const modelPrompt = this.getModelPrompt(prompt, parameters);
         const modelPromptText = modelPrompt.prompt ? HandleBars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
         const modelPromptMessages = this.getModelPromptMessages(modelPrompt, combinedParameters, text);
@@ -202,6 +211,7 @@ class ModelPlugin {
     // Default simple logging
     logRequestStart(url, data) {
         this.requestCount++;
+        this.lastRequestStartTime = new Date();
         const logMessage = `>>> [${this.requestId}: ${this.pathwayName}.${this.requestCount}] request`;
         const header = '>'.repeat(logMessage.length);
         console.log(`\n${header}\n${logMessage}`);
@@ -211,7 +221,7 @@ class ModelPlugin {
     logAIRequestFinished() {
         const currentTime = new Date();
         const timeElapsed = (currentTime - this.lastRequestStartTime) / 1000;
-        const logMessage = `<<< [${this.requestId}: ${this.pathwayName}.${this.requestCount}] response - complete in ${timeElapsed}s - data:`;
+        const logMessage = `<<< [${this.requestId}: ${this.pathwayName}] response - complete in ${timeElapsed}s - data:`;
         const header = '<'.repeat(logMessage.length);
         console.log(`\n${header}\n${logMessage}\n`);
     };
@@ -229,11 +239,11 @@ class ModelPlugin {
         prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
     }
     
-    async executeRequest(url, data, params, headers, prompt, requestId) {
+    async executeRequest(url, data, params, headers, prompt, requestId, pathway) {
         this.aiRequestStartTime = new Date();
         this.requestId = requestId;
         this.logRequestStart(url, data);
-        const responseData = await request({ url, data, params, headers, cache: this.shouldCache }, this.modelName);
+        const responseData = await request({ url, data, params, headers, cache: this.shouldCache }, this.modelName, this.requestId, pathway);
         
         if (responseData.error) {
             throw new Error(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);

package/server/plugins/openAiChatPlugin.js

@@ -79,12 +79,12 @@ class OpenAIChatPlugin extends ModelPlugin {
     async execute(text, parameters, prompt, pathwayResolver) {
         const url = this.requestUrl(text);
         const requestParameters = this.getRequestParameters(text, parameters, prompt);
-        const requestId = pathwayResolver?.requestId;
+        const { requestId, pathway} = pathwayResolver;
 
         const data = { ...(this.model.params || {}), ...requestParameters };
-        const params = {};
+        const params = {}; // query params
         const headers = this.model.headers || {};
-        return this.executeRequest(url, data, params, headers, prompt, requestId);
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
     }
 
     // Parse the response from the OpenAI Chat API
@@ -122,7 +122,7 @@ class OpenAIChatPlugin extends ModelPlugin {
         }
     
         if (stream) {
-            console.log(`\x1b[34m> Response is streaming...\x1b[0m`);
+            console.log(`\x1b[34m> [response is an SSE stream]\x1b[0m`);
         } else {
             console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
         }

package/server/plugins/openAiCompletionPlugin.js

@@ -79,13 +79,13 @@ class OpenAICompletionPlugin extends ModelPlugin {
     async execute(text, parameters, prompt, pathwayResolver) {
         const url = this.requestUrl(text);
         const requestParameters = this.getRequestParameters(text, parameters, prompt, pathwayResolver);
-        const requestId = pathwayResolver?.requestId;
+        const { requestId, pathway} = pathwayResolver;
 
         const data = { ...(this.model.params || {}), ...requestParameters };
         const params = {};
         const headers = this.model.headers || {};
         
-        return this.executeRequest(url, data, params, headers, prompt, requestId);
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
     }
 
     // Parse the response from the OpenAI Completion API
@@ -115,7 +115,7 @@ class OpenAICompletionPlugin extends ModelPlugin {
         console.log(`\x1b[36m${modelInput}\x1b[0m`);
 
         if (stream) {
-            console.log(`\x1b[34m> Response is streaming...\x1b[0m`);
+            console.log(`\x1b[34m> [response is an SSE stream]\x1b[0m`);
         } else {
             console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
         }

package/server/plugins/openAiWhisperPlugin.js

@@ -129,7 +129,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
                 try {
                     // const res = await axios.post(WHISPER_TS_API_URL, { params: { fileurl: uri } });
-                    const res = await this.executeRequest(WHISPER_TS_API_URL, {fileurl:uri},{},{});
+                    const res = await this.executeRequest(WHISPER_TS_API_URL, {fileurl:uri}, {}, {}, {}, requestId, pathway);
                     return res;
                 } catch (err) {
                     console.log(`Error getting word timestamped data from api:`, err);
@@ -150,7 +150,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
                 language && formData.append('language', language);
                 modelPromptText && formData.append('prompt', modelPromptText);
 
-                return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() });
+                return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() }, {}, requestId, pathway);
             } catch (err) {
                 console.log(err);
                 throw err;
@@ -161,7 +161,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         let { file } = parameters;
         let totalCount = 0;
         let completedCount = 0;
-        const { requestId } = pathwayResolver;
+        const { requestId, pathway } = pathwayResolver;
 
         const sendProgress = () => {
             completedCount++;

package/server/plugins/palmChatPlugin.js

@@ -140,7 +140,7 @@ class PalmChatPlugin extends ModelPlugin {
     async execute(text, parameters, prompt, pathwayResolver) {
         const url = this.requestUrl(text);
         const requestParameters = this.getRequestParameters(text, parameters, prompt);
-        const requestId = pathwayResolver?.requestId;
+        const { requestId, pathway} = pathwayResolver;
 
         const data = { ...(this.model.params || {}), ...requestParameters };
         const params = {};
@@ -148,7 +148,7 @@ class PalmChatPlugin extends ModelPlugin {
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
         const authToken = await gcpAuthTokenHelper.getAccessToken();
         headers.Authorization = `Bearer ${authToken}`;
-        return this.executeRequest(url, data, params, headers, prompt, requestId);
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
     }
 
     // Parse the response from the PaLM Chat API

package/server/plugins/palmCompletionPlugin.js

@@ -55,7 +55,7 @@ class PalmCompletionPlugin extends ModelPlugin {
     async execute(text, parameters, prompt, pathwayResolver) {
         const url = this.requestUrl(text);
         const requestParameters = this.getRequestParameters(text, parameters, prompt, pathwayResolver);
-        const requestId = pathwayResolver?.requestId;
+        const { requestId, pathway} = pathwayResolver;
 
         const data = { ...requestParameters };
         const params = {};
@@ -63,7 +63,7 @@ class PalmCompletionPlugin extends ModelPlugin {
         const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
         const authToken = await gcpAuthTokenHelper.getAccessToken();
         headers.Authorization = `Bearer ${authToken}`;
-        return this.executeRequest(url, data, params, headers, prompt, requestId);
+        return this.executeRequest(url, data, params, headers, prompt, requestId, pathway);
     }
 
     // Parse the response from the PaLM API Text Completion API

package/server/rest.js

@@ -61,7 +61,7 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
     
     const finishStream = (res, jsonResponse) => {
-    
+
         // If we haven't sent the stop message yet, do it now
         if (jsonResponse.choices?.[0]?.finish_reason !== "stop") {
     
@@ -77,17 +77,20 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
                 jsonEndStream.choices[0].delta = {};
             }
     
-            //console.log(`REST SEND: data: ${JSON.stringify(jsonEndStream)}`);
-            res.write(`data: ${JSON.stringify(jsonEndStream)}\n\n`);
+            sendStreamData(jsonEndStream);
         }
     
-        //console.log(`REST SEND: data: [DONE]\n\n`);
-        res.write(`data: [DONE]\n\n`);
+        sendStreamData('[DONE]');
+        res.end();
     }
 
     const sendStreamData = (data) => {
         //console.log(`REST SEND: data: ${JSON.stringify(data)}`);
-        res.write(`data: ${JSON.stringify(data)}\n\n`);
+        const dataString = (data==='[DONE]') ? data : JSON.stringify(data);
+
+        if (!res.writableEnded) {
+            res.write(`data: ${dataString}\n\n`);
+        }
     }
 
     const fillJsonResponse = (jsonResponse, inputText, finishReason) => {
@@ -106,13 +109,18 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
 
     let subscription;
 
-    const unsubscribe = async () => {
-        if (subscription) {
-            pubsub.unsubscribe(await subscription);
+    subscription = pubsub.subscribe('REQUEST_PROGRESS', (data) => {
+        
+        const safeUnsubscribe = async () => {
+            if (subscription) {
+                try {
+                    pubsub.unsubscribe(await subscription);
+                } catch (error) {
+                    console.error(`Error unsubscribing from pubsub: ${error}`);
+                }
+            }
         }
-    }
 
-    subscription = pubsub.subscribe('REQUEST_PROGRESS', (data) => {
         if (data.requestProgress.requestId === requestId) {
             //console.log(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
             
@@ -121,7 +129,12 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
 
             try {
                 const messageJson = JSON.parse(progressData);
-                if (messageJson.choices) {
+                if (messageJson.error) {
+                    console.error(`Stream error REST:`, messageJson?.error?.message);
+                    safeUnsubscribe();
+                    finishStream(res, jsonResponse);
+                    return;
+                } else if (messageJson.choices) {
                     const { text, delta, finish_reason } = messageJson.choices[0];
 
                     if (messageJson.object === 'text_completion') {
@@ -133,20 +146,20 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
                     fillJsonResponse(jsonResponse, messageJson, null);
                 }
             } catch (error) {
-                console.log(`progressData not JSON: ${progressData}`);
+                //console.log(`progressData not JSON: ${progressData}`);
                 fillJsonResponse(jsonResponse, progressData, "stop");
             }
-
             if (progress === 1 && progressData.trim() === "[DONE]") {
+                safeUnsubscribe();
                 finishStream(res, jsonResponse);
-                unsubscribe();
                 return;
             }
+
             sendStreamData(jsonResponse);
 
             if (progress === 1) {
+                safeUnsubscribe();
                 finishStream(res, jsonResponse);
-                unsubscribe();
             }
         }
     });

package/server/subscriptions.js

@@ -14,9 +14,9 @@ const subscriptions = {
                 const { requestIds } = args;
                 for (const requestId of requestIds) {
                     if (!requestState[requestId]) {
-                        console.log(`requestProgress, requestId: ${requestId} not found`);
+                        console.error(`Subscription requestId: ${requestId} not found`);
                     } else {
-                        console.log(`starting async requestProgress, requestId: ${requestId}`);
+                        console.log(`Subscription starting async requestProgress, requestId: ${requestId}`);
                         const { resolver, args } = requestState[requestId];
                         resolver(args);
                     }

package/server/typeDef.js

@@ -7,7 +7,7 @@ const getGraphQlType = (value) => {
       return {type: 'String', defaultValue: `""`};
       break;
     case 'number':
-      return {type: 'Int', defaultValue: '0'};
+      return {type: 'Int', defaultValue: 'null'};
       break;
     case 'object':
       if (Array.isArray(value)) {

package/tests/openAiChatPlugin.test.js

@@ -77,7 +77,7 @@ test('execute', async (t) => {
         };
     };
 
-    const result = await plugin.execute(text, parameters, prompt);
+    const result = await plugin.execute(text, parameters, prompt, { requestId: 'foo', pathway: {} });
     t.deepEqual(result, {
         choices: [
             {