@aj-archipelago/cortex 1.0.10 → 1.0.11

package/config.js

@@ -151,7 +151,7 @@ if (configFile && fs.existsSync(configFile)) {
     const openaiApiKey = config.get('openaiApiKey');
     if (!openaiApiKey) {
         throw console.log('No config file or api key specified. Please set the OPENAI_API_KEY to use OAI or use CORTEX_CONFIG_FILE environment variable to point at the Cortex configuration for your project.');
-    }else {
+    } else {
         console.log(`Using default model with OPENAI_API_KEY environment variable`)
     }
 }
@@ -202,7 +202,7 @@ const buildPathways = async (config) => {
 const buildModels = (config) => {
     const { models } = config.getProperties();
 
-    for (const [key, model] of Object.entries(models)) {        
+    for (const [key, model] of Object.entries(models)) {
         // Compile handlebars templates for models
         models[key] = JSON.parse(HandleBars.compile(JSON.stringify(model))({ ...config.getEnv(), ...config.getProperties() }))
     }
@@ -219,8 +219,8 @@ const buildModels = (config) => {
     // Set default model name to the first model in the config in case no default is specified
     if (!config.get('defaultModelName')) {
         console.log('No default model specified, using first model as default.');
-        config.load({ defaultModelName: Object.keys(config.get('models'))[0] }); 
-    } 
+        config.load({ defaultModelName: Object.keys(config.get('models'))[0] });
+    }
 
     return models;
 }

package/lib/request.js

@@ -176,14 +176,23 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
             }
         } catch (error) {
             //console.error(`!!! [${requestId}] failed request with data ${JSON.stringify(data)}: ${error}`);
-            if (error.response?.status === 429) {
-                monitors[model].incrementError429Count();
-            }
-            console.log(`>>> [${requestId}] retrying request due to ${error.response?.status} response. Retry count: ${i + 1}`);
-            if (i < MAX_RETRY - 1) {
-                const backoffTime = 200 * Math.pow(2, i);
-                const jitter = backoffTime * 0.2 * Math.random();
-                await new Promise(r => setTimeout(r, backoffTime + jitter));
+            if (error.response) {
+                const status = error.response.status;
+                if ((status === 429) || (status >= 500 && status < 600)) {
+                    if (status === 429) {
+                        monitors[model].incrementError429Count();
+                    }
+                    console.log(`>>> [${requestId}] retrying request due to ${status} response. Retry count: ${i + 1}`);
+                    if (i < MAX_RETRY - 1) {
+                        const backoffTime = 200 * Math.pow(2, i);
+                        const jitter = backoffTime * 0.2 * Math.random();
+                        await new Promise(r => setTimeout(r, backoffTime + jitter));
+                    } else {
+                        throw error;
+                    }
+                } else {
+                    throw error;
+                }
             } else {
                 throw error;
             }
@@ -192,17 +201,22 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
 };
 
 const request = async (params, model, requestId, pathway) => {
-    const response = await postRequest(params, model, requestId, pathway);
-    const { error, data, cached } = response;
-    if (cached) {
-        console.info(`<<< [${requestId}] served with cached response.`);
-    }
-    if (error && error.length > 0) {
-        const lastError = error[error.length - 1];
-        return { error: lastError.toJSON() ?? lastError ?? error };
-    }
-    //console.log("<<< [${requestId}] response: ", data.choices[0].delta || data.choices[0])
-    return data;
+    try {
+        const response = await postRequest(params, model, requestId, pathway);
+        const { error, data, cached } = response;
+        if (cached) {
+            console.info(`<<< [${requestId}] served with cached response.`);
+        }
+        if (error && error.length > 0) {
+            const lastError = error[error.length - 1];
+            return { error: lastError.toJSON() ?? lastError ?? error };
+        }
+        //console.log("<<< [${requestId}] response: ", data.choices[0].delta || data.choices[0])
+        return data;
+    } catch (error) {
+        console.error(`Error in request: ${error.message || error}`);
+        return { error: error };
+    } 
 }
 
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.10",
+  "version": "1.0.11",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",

package/pathways/basePathway.js

@@ -21,5 +21,10 @@ export default {
     truncateFromFront: false, // true or false - if true, truncate from the front of the input instead of the back
     timeout: 120, // seconds, cancels the pathway after this many seconds
     duplicateRequestAfter: 10, // seconds, if the request is not completed after this many seconds, a backup request is sent
+    // override the default execution of the pathway
+    // callback signature: excuteOverride({args: object, runAllPrompts: function})
+    // args: the input arguments to the pathway
+    // runAllPrompts: a function that runs all prompts in the pathway and returns the result
+    executePathway: undefined,
 };
 

package/pathways/summary.js

@@ -21,7 +21,7 @@ export default {
         const originalTargetLength = args.targetLength;
 
         // If targetLength is not provided, execute the prompt once and return the result.
-        if (originalTargetLength === 0) {
+        if (originalTargetLength === 0 || originalTargetLength === null) {
             let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
             return await pathwayResolver.resolve(args);
         }

package/server/chunker.js

@@ -152,7 +152,7 @@ const getSemanticChunks = (text, chunkSize, inputFormat = 'text') => {
       }
     });
 
-    if (chunks.some(chunk => encode(chunk).length > chunkSize)) {
+    if (chunks.filter(c => determineTextFormat(c) === 'html').some(chunk => encode(chunk).length > chunkSize)) {
       throw new Error('The HTML contains elements that are larger than the chunk size. Please try again with HTML that has smaller elements.');
     }
 

package/server/graphql.js

@@ -20,6 +20,7 @@ import { cancelRequestResolver } from './resolver.js';
 import { buildPathways, buildModels } from '../config.js';
 import { requestState } from './requestState.js';
 import { buildRestEndpoints } from './rest.js';
+import { startTestServer } from '../tests/server.js'
 
 // Utility functions
 // Server plugins
@@ -222,7 +223,7 @@ const build = async (config) => {
         });
     };
 
-    return { server, startServer, cache, plugins, typeDefs, resolvers }
+    return { server, startServer, startTestServer, cache, plugins, typeDefs, resolvers }
 }
 
 

package/server/pathwayPrompter.js

@@ -3,6 +3,7 @@ import OpenAIChatPlugin from './plugins/openAiChatPlugin.js';
 import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
 import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
 import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
+import OpenAIChatExtensionPlugin from './plugins/openAiChatExtensionPlugin.js';
 import LocalModelPlugin from './plugins/localModelPlugin.js';
 import PalmChatPlugin from './plugins/palmChatPlugin.js';
 import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
@@ -19,6 +20,9 @@ class PathwayPrompter {
             case 'OPENAI-CHAT':
                 plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
                 break;
+            case 'OPENAI-CHAT-EXTENSION':
+                plugin = new OpenAIChatExtensionPlugin(config, pathway, modelName, model);
+                break;
             case 'AZURE-TRANSLATE':
                 plugin = new AzureTranslatePlugin(config, pathway, modelName, model);
                 break;

package/server/pathwayResolver.js

@@ -9,7 +9,6 @@ import { Prompt } from './prompt.js';
 import { getv, setv } from '../lib/keyValueStorageClient.js';
 import { requestState } from './requestState.js';
 import { callPathway } from '../lib/pathwayTools.js';
-import { response } from 'express';
 
 class PathwayResolver {
     constructor({ config, pathway, args }) {
@@ -26,7 +25,7 @@ class PathwayResolver {
             args?.model,
             pathway.inputParameters?.model,
             config.get('defaultModelName')
-            ].find(modelName => modelName && config.get('models').hasOwnProperty(modelName));
+            ].find(modelName => modelName && Object.prototype.hasOwnProperty.call(config.get('models'), modelName));
         this.model = config.get('models')[this.modelName];
 
         if (!this.model) {
@@ -74,7 +73,7 @@ class PathwayResolver {
         let streamErrorOccurred = false;
 
         while (attempt < MAX_RETRY_COUNT) {
-            const responseData = await this.promptAndParse(args);
+            const responseData = await this.executePathway(args);
 
             if (args.async || typeof responseData === 'string') {
                 const { completedCount, totalCount } = requestState[this.requestId];
@@ -140,18 +139,20 @@ class PathwayResolver {
                                     } catch (error) {
                                         console.error('Could not publish the stream message', message, error);
                                     }
-                                };
-                            };
+                                }
+                            }
                         } catch (error) {
                             console.error('Could not process stream data', error);
                         }
                     };
 
-                    await new Promise((resolve, reject) => {
-                        incomingMessage.on('data', processData);
-                        incomingMessage.on('end', resolve);
-                        incomingMessage.on('error', reject);
-                    });
+                    if (incomingMessage) {
+                        await new Promise((resolve, reject) => {
+                            incomingMessage.on('data', processData);
+                            incomingMessage.on('end', resolve);
+                            incomingMessage.on('error', reject);
+                        });
+                    }
 
                 } catch (error) {
                     console.error('Could not subscribe to stream', error);
@@ -187,6 +188,15 @@ class PathwayResolver {
         }
         else {
             // Syncronously process the request
+            return await this.executePathway(args);
+        }
+    }
+
+    async executePathway(args) {
+        if (this.pathway.executePathway && typeof this.pathway.executePathway === 'function') {
+            return await this.pathway.executePathway({ args, runAllPrompts: this.promptAndParse.bind(this) });
+        }
+        else {
             return await this.promptAndParse(args);
         }
     }

package/server/plugins/azureTranslatePlugin.js

@@ -55,7 +55,7 @@ class AzureTranslatePlugin extends ModelPlugin {
         console.log(`\x1b[36m${modelInput}\x1b[0m`);
         console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
     
-        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
     }
 }
 

package/server/plugins/modelPlugin.js

@@ -22,11 +22,11 @@ class ModelPlugin {
 
         // Make all of the parameters defined on the pathway itself available to the prompt
         for (const [k, v] of Object.entries(pathway)) {
-            this.promptParameters[k] = v.default ?? v;
+            this.promptParameters[k] = v?.default ?? v;
         }
         if (pathway.inputParameters) {
             for (const [k, v] of Object.entries(pathway.inputParameters)) {
-                this.promptParameters[k] = v.default ?? v;
+                this.promptParameters[k] = v?.default ?? v;
             }
         }
 
@@ -206,17 +206,17 @@ class ModelPlugin {
     }
 
     // Default response parsing
-    parseResponse(data) { return data; };
+    parseResponse(data) { return data; }
 
     // Default simple logging
-    logRequestStart(url, data) {
+    logRequestStart(url, _data) {
         this.requestCount++;
         this.lastRequestStartTime = new Date();
         const logMessage = `>>> [${this.requestId}: ${this.pathwayName}.${this.requestCount}] request`;
         const header = '>'.repeat(logMessage.length);
         console.log(`\n${header}\n${logMessage}`);
         console.log(`>>> Making API request to ${url}`);
-    };
+    }
 
     logAIRequestFinished() {
         const currentTime = new Date();
@@ -224,7 +224,7 @@ class ModelPlugin {
         const logMessage = `<<< [${this.requestId}: ${this.pathwayName}] response - complete in ${timeElapsed}s - data:`;
         const header = '<'.repeat(logMessage.length);
         console.log(`\n${header}\n${logMessage}\n`);
-    };
+    }
 
     logRequestData(data, responseData, prompt) {
         this.logAIRequestFinished(); 
@@ -236,21 +236,26 @@ class ModelPlugin {
     
         console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
     
-        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
     }
     
     async executeRequest(url, data, params, headers, prompt, requestId, pathway) {
-        this.aiRequestStartTime = new Date();
-        this.requestId = requestId;
-        this.logRequestStart(url, data);
-        const responseData = await request({ url, data, params, headers, cache: this.shouldCache }, this.modelName, this.requestId, pathway);
+        try {
+            this.aiRequestStartTime = new Date();
+            this.requestId = requestId;
+            this.logRequestStart(url, data);
+            const responseData = await request({ url, data, params, headers, cache: this.shouldCache }, this.modelName, this.requestId, pathway);
+            
+            if (responseData.error) {
+                throw new Error(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);
+            }
         
-        if (responseData.error) {
-            throw new Error(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);
+            this.logRequestData(data, responseData, prompt);
+            return this.parseResponse(responseData);
+        } catch (error) {
+            // Log the error and continue
+            console.error(error);
         }
-    
-        this.logRequestData(data, responseData, prompt);
-        return this.parseResponse(responseData);
     }
 
 }

package/server/plugins/openAiChatExtensionPlugin.js

@@ -0,0 +1,56 @@
+// OpenAIChatPlugin.js
+import OpenAIChatPlugin from './openAiChatPlugin.js';
+
+class OpenAIChatExtensionPlugin extends OpenAIChatPlugin {
+    constructor(config, pathway, modelName, model) {
+        super(config, pathway, modelName, model);
+        this.tool = '';
+    }
+
+    // Parse the response from the OpenAI Extension API
+    parseResponse(data) {
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            return data;
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        // otherwise, return the first choice messages based on role
+        const messageResult = [];
+        for(const message of choices[0].messages) {
+            if(message.role === "tool"){
+                this.tool = message.content;
+            }else{
+                messageResult.push(message.content);
+            }
+        }
+        return messageResult.join('\n\n') ?? null;
+    }
+
+    // Set up parameters specific to the OpenAI Chat API
+    getRequestParameters(text, parameters, prompt) {
+        const reqParams = super.getRequestParameters(text, parameters, prompt);
+        reqParams.dataSources = this.model.dataSources || reqParams.dataSources || []; // add dataSources to the request parameters
+        const {roleInformation, indexName } = parameters; // add roleInformation and indexName to the dataSource if given
+        for(const dataSource of reqParams.dataSources) {
+            if(!dataSource) continue;
+            if(!dataSource.parameters) dataSource.parameters = {};
+            roleInformation && (dataSource.parameters.roleInformation = roleInformation);
+            indexName && (dataSource.parameters.indexName = indexName);
+        }
+        return reqParams;
+    }
+
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const result = await super.execute(text, parameters, prompt, pathwayResolver);
+        pathwayResolver.tool = this.tool; // add tool info back 
+        return result;
+    }
+
+}
+
+export default OpenAIChatExtensionPlugin;

package/server/plugins/openAiChatPlugin.js

@@ -127,7 +127,7 @@ class OpenAIChatPlugin extends ModelPlugin {
             console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
         }
 
-        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
     }
 }
 

package/server/plugins/openAiCompletionPlugin.js

@@ -120,7 +120,7 @@ class OpenAICompletionPlugin extends ModelPlugin {
             console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
         }
     
-        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+        prompt && prompt.debugInfo && (prompt.debugInfo += `\n${JSON.stringify(data)}`);
     }
 }
 

package/server/plugins/palmChatPlugin.js

@@ -219,7 +219,7 @@ class PalmChatPlugin extends ModelPlugin {
         }
     
         if (prompt && prompt.debugInfo) {
-            prompt.debugInfo += `${separator}${JSON.stringify(data)}`;
+            prompt.debugInfo += `\n${JSON.stringify(data)}`;
         }
     }
 }

package/server/plugins/palmCompletionPlugin.js

@@ -124,7 +124,7 @@ class PalmCompletionPlugin extends ModelPlugin {
         }
 
         if (prompt && prompt.debugInfo) {
-            prompt.debugInfo += `${separator}${JSON.stringify(data)}`;
+            prompt.debugInfo += `\n${JSON.stringify(data)}`;
         }
     }
 }

package/server/resolver.js

@@ -17,12 +17,12 @@ const rootResolver = async (parent, args, contextValue, info) => {
 
     // Execute the request with timeout
     const result = await fulfillWithTimeout(pathway.resolver(parent, args, contextValue, info), pathway.timeout);
-    const { warnings, previousResult, savedContextId } = pathwayResolver;
+    const { warnings, previousResult, savedContextId, tool } = pathwayResolver;
     
     // Add request parameters back as debug
     const debug = pathwayResolver.prompts.map(prompt => prompt.debugInfo || '').join('\n').trim();
     
-    return { debug, result, warnings, previousResult, contextId: savedContextId }
+    return { debug, result, warnings, previousResult, tool, contextId: savedContextId }
 }
 
 // This resolver is used by the root resolver to process the request

package/server/subscriptions.js

@@ -13,9 +13,8 @@ const subscriptions = {
             (_, args, __, _info) => {
                 const { requestIds } = args;
                 for (const requestId of requestIds) {
-                    if (!requestState[requestId]) {
-                        console.error(`Subscription requestId: ${requestId} not found`);
-                    } else {
+                    if (requestState[requestId] && !requestState[requestId].started) {
+                        requestState[requestId].started = true;
                         console.log(`Subscription starting async requestProgress, requestId: ${requestId}`);
                         const { resolver, args } = requestState[requestId];
                         resolver(args);

package/server/typeDef.js

@@ -47,6 +47,7 @@ const typeDef = (pathway) => {
         previousResult: String
         warnings: [String]
         contextId: String
+        tool: String
 }`;
 
   const params = { ...defaultInputParameters, ...inputParameters };

package/tests/server.js

@@ -0,0 +1,23 @@
+import 'dotenv/config'
+import { ApolloServer } from '@apollo/server';
+import { config } from '../config.js';
+import typeDefsresolversFactory from '../index.js';
+
+let typeDefs;
+let resolvers;
+
+const initTypeDefsResolvers = async () => {
+    const result = await typeDefsresolversFactory();
+    typeDefs = result.typeDefs;
+    resolvers = result.resolvers;
+};
+
+export const startTestServer = async () => {
+    await initTypeDefsResolvers();
+
+    return new ApolloServer({
+        typeDefs,
+        resolvers,
+        context: () => ({ config, requestState: {} }),
+    });
+};