@aj-archipelago/cortex 1.0.4 → 1.0.6

package/pathways/index.js

@@ -3,10 +3,13 @@ import chat from './chat.js';
 import bias from './bias.js';
 import complete from './complete.js';
 import entities from './entities.js';
-import lc_test from './lc_test.mjs';
 import paraphrase from './paraphrase.js';
 import sentiment from './sentiment.js';
 import summary from './summary.js';
+import sys_openai_chat from './sys_openai_chat.js';
+import sys_openai_completion from './sys_openai_completion.js';
+import test_langchain from './test_langchain.mjs';
+import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
 import translate from './translate.js';
 
@@ -16,10 +19,13 @@ export {
     bias,
     complete,
     entities,
-    lc_test,
     paraphrase,
     sentiment,
     summary,
+    sys_openai_chat,
+    sys_openai_completion,
+    test_langchain,
+    test_palm_chat,
     transcribe,
     translate
 };

package/pathways/summary.js

@@ -3,8 +3,8 @@
 // This module exports a prompt that takes an input text and generates a summary using a custom resolver.
 
 // Import required modules
-import { semanticTruncate } from '../graphql/chunker.js';
-import { PathwayResolver } from '../graphql/pathwayResolver.js';
+import { semanticTruncate } from '../server/chunker.js';
+import { PathwayResolver } from '../server/pathwayResolver.js';
 
 export default {
     // The main prompt function that takes the input text and asks to generate a summary.

package/pathways/sys_openai_chat.js

@@ -0,0 +1,19 @@
+// sys_openai_chat.js
+// default handler for openAI chat endpoints when REST endpoints are enabled
+
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [],
+    },
+    model: 'oai-gpturbo',
+    useInputChunking: false,
+    emulateOpenAIChatModel: '*',
+}

package/pathways/sys_openai_completion.js

@@ -0,0 +1,11 @@
+// sys_openai_completion.js
+// default handler for openAI completion endpoints when REST endpoints are enabled
+
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: `{{text}}`,
+    model: 'oai-gpturbo',
+    useInputChunking: false,
+    emulateOpenAICompletionModel: '*',
+}

package/pathways/{lc_test.mjs → test_langchain.mjs}

@@ -1,4 +1,4 @@
-// lc_test.js
+// test_langchain.mjs
 // LangChain Cortex integration test
 
 // Import required modules

package/pathways/test_palm_chat.js

@@ -0,0 +1,31 @@
+//test_palm_chat.mjs
+// Test for handling of prompts in the PaLM chat format for Cortex
+
+import { Prompt } from '../server/prompt.js';
+
+// Description: Have a chat with a bot that uses context to understand the conversation
+export default {
+    prompt:
+        [
+            new Prompt({ 
+                context: "Instructions:\nYou an AI entity working a global media network. You are truthful, kind, and helpful. Your expertise includes journalism, journalistic ethics, researching and composing documents, and technology. You know the current date and time - it is {{now}}.",
+                examples: [
+                    { 
+                       input: {"content": "What is your expertise?"},
+                       output: {"content": "I am an expert in journalism and journalistic ethics."}
+                    }],
+                messages: [
+                {"author": "user", "content": "Hi how are you today?"},
+                {"author": "assistant", "content": "I am doing well. How are you?"},
+                {"author": "user", "content": "I am doing well. What is your name?"},
+                {"author": "assistant", "content": "My name is Hula. What is your name?"},
+                {"author": "user", "content": "My name is Bob. What is your expertise?"},
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [],
+        contextId: ``,
+    },
+    model: 'palm-chat',
+    useInputChunking: false,
+}

package/pathways/transcribe.js

@@ -4,8 +4,10 @@ export default {
     inputParameters: {
         file: ``,
         language: ``,
+        responseFormat: `text`,
+        wordTimestamped: false,
     },
-    timeout: 1800, // in seconds
+    timeout: 3600, // in seconds
 };
 
 

package/pathways/translate.js

@@ -14,6 +14,7 @@ export default {
     },
 
     // Set the timeout for the translation process, in seconds.
-    timeout: 300,
+    timeout: 400,
+    inputChunkSize: 500,
 };
 

package/{graphql → server}/graphql.js

@@ -1,24 +1,29 @@
-import { createServer } from 'http';
-import {
-    ApolloServerPluginDrainHttpServer,
-    ApolloServerPluginLandingPageLocalDefault,
-} from 'apollo-server-core';
+// graphql.js
+// Setup the Apollo server and Express middleware
+
+import { ApolloServerPluginDrainHttpServer } from '@apollo/server/plugin/drainHttpServer';
+import { ApolloServerPluginLandingPageLocalDefault } from '@apollo/server/plugin/landingPage/default';
+import { ApolloServer } from '@apollo/server';
+import { expressMiddleware } from '@apollo/server/express4';
 import { makeExecutableSchema } from '@graphql-tools/schema';
 import { WebSocketServer } from 'ws';
 import { useServer } from 'graphql-ws/lib/use/ws';
 import express from 'express';
-import { ApolloServer } from 'apollo-server-express';
+import http from 'http';
 import Keyv from 'keyv';
+import cors from 'cors';
 import { KeyvAdapter } from '@apollo/utils.keyvadapter';
-import responseCachePlugin from 'apollo-server-plugin-response-cache';
+import responseCachePlugin from '@apollo/server-plugin-response-cache';
 import subscriptions from './subscriptions.js';
 import { buildLimiters } from '../lib/request.js';
 import { cancelRequestResolver } from './resolver.js';
 import { buildPathways, buildModels } from '../config.js';
 import { requestState } from './requestState.js';
+import { buildRestEndpoints } from './rest.js';
 
+// Utility functions
+// Server plugins
 const getPlugins = (config) => {
-    // server plugins
     const plugins = [
         ApolloServerPluginLandingPageLocalDefault({ embed: true }), // For local development.   
     ];
@@ -39,41 +44,8 @@ const getPlugins = (config) => {
     return { plugins, cache };
 }
 
-const buildRestEndpoints = (pathways, app, server, config) => {
-  for (const [name, pathway] of Object.entries(pathways)) {
-    // Only expose endpoints for enabled pathways that explicitly want to expose a REST endpoint
-    if (pathway.disabled || !config.get('enableRestEndpoints')) continue;
-
-    const fieldVariableDefs = pathway.typeDef(pathway).restDefinition || [];
-
-    app.post(`/rest/${name}`, async (req, res) => {
-      const variables = fieldVariableDefs.reduce((acc, variableDef) => {
-        if (Object.prototype.hasOwnProperty.call(req.body, variableDef.name)) {
-          acc[variableDef.name] = req.body[variableDef.name];
-        }
-        return acc;
-      }, {});
-
-      const variableParams = fieldVariableDefs.map(({ name, type }) => `$${name}: ${type}`).join(', ');
-      const queryArgs = fieldVariableDefs.map(({ name }) => `${name}: $${name}`).join(', ');
-
-      const query = `
-                query ${name}(${variableParams}) {
-                    ${name}(${queryArgs}) {
-                        contextId
-                        previousResult
-                        result
-                    }
-                }
-            `;
-
-      const result = await server.executeOperation({ query, variables });
-      res.json(result.data[name]);
-    });
-  }
-};
 
-//typeDefs
+// Type Definitions for GraphQL
 const getTypedefs = (pathways) => {
 
     const defaultTypeDefs = `#graphql
@@ -111,6 +83,7 @@ const getTypedefs = (pathways) => {
     return typeDefs.join('\n');
 }
 
+// Resolvers for GraphQL
 const getResolvers = (config, pathways) => {
     const resolverFunctions = {};
     for (const [name, pathway] of Object.entries(pathways)) {
@@ -118,6 +91,7 @@ const getResolvers = (config, pathways) => {
         resolverFunctions[name] = (parent, args, contextValue, info) => {
             // add shared state to contextValue
             contextValue.pathway = pathway;
+            contextValue.config = config;
             return pathway.rootResolver(parent, args, contextValue, info);
         }
     }
@@ -131,7 +105,7 @@ const getResolvers = (config, pathways) => {
     return resolvers;
 }
 
-//graphql api build factory method
+// Build the server including the GraphQL schema and REST endpoints
 const build = async (config) => {
     // First perform config build
     await buildPathways(config);
@@ -150,9 +124,9 @@ const build = async (config) => {
 
     const { plugins, cache } = getPlugins(config);
 
-    const app = express()
+    const app = express();
 
-    const httpServer = createServer(app);
+    const httpServer = http.createServer(app);
 
     // Creating the WebSocket server
     const wsServer = new WebSocketServer({
@@ -182,35 +156,63 @@ const build = async (config) => {
                         },
                     };
                 },
-            }]),
-        context: ({ req, res }) => ({ req, res, config, requestState }),
+            }
+        ]),
     });
 
     // If CORTEX_API_KEY is set, we roll our own auth middleware - usually not used if you're being fronted by a proxy
     const cortexApiKey = config.get('cortexApiKey');
+    if (cortexApiKey) {
+        app.use((req, res, next) => {
+            if (cortexApiKey && req.headers['Cortex-Api-Key'] !== cortexApiKey && req.query['Cortex-Api-Key'] !== cortexApiKey) {
+                if (req.baseUrl === '/graphql' || req.headers['content-type'] === 'application/graphql') {
+                    res.status(401)
+                    .set('WWW-Authenticate', 'Cortex-Api-Key')
+                    .set('X-Cortex-Api-Key-Info', 'Server requires Cortex API Key')
+                    .json({
+                            errors: [
+                                {
+                                    message: 'Unauthorized',
+                                    extensions: {
+                                        code: 'UNAUTHENTICATED',
+                                    },
+                                },
+                            ],
+                        });
+                } else {
+                    res.status(401)
+                    .set('WWW-Authenticate', 'Cortex-Api-Key')
+                    .set('X-Cortex-Api-Key-Info', 'Server requires Cortex API Key')
+                    .send('Unauthorized');
+                }
+            } else {
+                next();
+            }
+        });
+    };
 
-    app.use((req, res, next) => {
-        if (cortexApiKey && req.headers.cortexApiKey !== cortexApiKey && req.query.cortexApiKey !== cortexApiKey) {
-            res.status(401).send('Unauthorized');
-        } else {
-            next();
-        }
-    });
-    
-    // Use the JSON body parser middleware for REST endpoints
+    // Parse the body for REST endpoints
     app.use(express.json());
-        
-    // add the REST endpoints
-    buildRestEndpoints(pathways, app, server, config);
 
-    // if local start server
+    // Server Startup Function
     const startServer = async () => {
         await server.start();
-        server.applyMiddleware({ app });
+        app.use(
+            '/graphql',
+
+            cors(),
+
+            expressMiddleware(server, {
+                context: async ({ req, res }) => ({ req, res, config, requestState }),
+            }),
+        );
+            
+        // add the REST endpoints
+        buildRestEndpoints(pathways, app, server, config);
 
         // Now that our HTTP server is fully set up, we can listen to it.
         httpServer.listen(config.get('PORT'), () => {
-            console.log(`🚀 Server is now running at http://localhost:${config.get('PORT')}${server.graphqlPath}`);
+            console.log(`🚀 Server is now running at http://localhost:${config.get('PORT')}/graphql`);
         });
     };
 

package/{graphql → server}/pathwayPrompter.js

@@ -4,6 +4,8 @@ import OpenAICompletionPlugin from './plugins/openAiCompletionPlugin.js';
 import AzureTranslatePlugin from './plugins/azureTranslatePlugin.js';
 import OpenAIWhisperPlugin from './plugins/openAiWhisperPlugin.js';
 import LocalModelPlugin from './plugins/localModelPlugin.js';
+import PalmChatPlugin from './plugins/palmChatPlugin.js';
+import PalmCompletionPlugin from './plugins/palmCompletionPlugin.js';
 
 class PathwayPrompter {
     constructor({ config, pathway }) {
@@ -27,12 +29,18 @@ class PathwayPrompter {
             case 'OPENAI-COMPLETION':
                 plugin = new OpenAICompletionPlugin(config, pathway);
                 break;
-            case 'OPENAI_WHISPER':
+            case 'OPENAI-WHISPER':
                 plugin = new OpenAIWhisperPlugin(config, pathway);
                 break;
             case 'LOCAL-CPP-MODEL':
                 plugin = new LocalModelPlugin(config, pathway);
                 break;
+            case 'PALM-CHAT':
+                plugin = new PalmChatPlugin(config, pathway);
+                break;
+            case 'PALM-COMPLETION':
+                plugin = new PalmCompletionPlugin(config, pathway);
+                break;
             default:
                 throw new Error(`Unsupported model type: ${model.type}`);
         }

package/{graphql → server}/pathwayResolver.js

@@ -42,66 +42,57 @@ class PathwayResolver {
     }
 
     async asyncResolve(args) {
-        // Wait with a sleep promise for the race condition to resolve
-        // const results = await Promise.all([this.promptAndParse(args), await new Promise(resolve => setTimeout(resolve, 250))]);
-        const data = await this.promptAndParse(args);
-        // Process the results for async
-        if(args.async || typeof data === 'string') { // if async flag set or processed async and got string response
+        const responseData = await this.promptAndParse(args);
+
+        // Either we're dealing with an async request or a stream
+        if(args.async || typeof responseData === 'string') {
             const { completedCount, totalCount } = requestState[this.requestId];
-            requestState[this.requestId].data = data;
+            requestState[this.requestId].data = responseData;
             pubsub.publish('REQUEST_PROGRESS', {
                 requestProgress: {
                     requestId: this.requestId,
                     progress: completedCount / totalCount,
-                    data: JSON.stringify(data),
+                    data: JSON.stringify(responseData),
                 }
             });
-        } else { //stream
-            for (const handle of data) {
-                handle.on('data', data => {
-                    console.log(data.toString());
-                    const lines = data.toString().split('\n').filter(line => line.trim() !== '');
-                    for (const line of lines) {
-                        const message = line.replace(/^data: /, '');
-                        if (message === '[DONE]') {
-                            // Send stream finished message
-                            pubsub.publish('REQUEST_PROGRESS', {
-                                requestProgress: {
-                                    requestId: this.requestId,
-                                    data: null,
-                                    progress: 1,
-                                }
-                            });
-                            return; // Stream finished
-                        }
-                        try {
-                            const parsed = JSON.parse(message);
-                            const result = this.pathwayPrompter.plugin.parseResponse(parsed)
-
-                            pubsub.publish('REQUEST_PROGRESS', {
-                                requestProgress: {
-                                    requestId: this.requestId,
-                                    data: JSON.stringify(result)
-                                }
-                            });
-                        } catch (error) {
-                            console.error('Could not JSON parse stream message', message, error);
-                        }
+        } else { // stream
+            const incomingMessage = Array.isArray(responseData) && responseData.length > 0 ? responseData[0] : responseData;
+            incomingMessage.on('data', data => {
+                const events = data.toString().split('\n');
+        
+                events.forEach(event => {
+                    if (event.trim() === '') return; // Skip empty lines
+        
+                    const message = event.replace(/^data: /, '');
+                    
+                    //console.log(`====================================`);
+                    //console.log(`STREAM EVENT: ${event}`);
+                    //console.log(`MESSAGE: ${message}`);
+        
+                    const requestProgress = {
+                        requestId: this.requestId,
+                        data: message,
+                    }
+        
+                    if (message.trim() === '[DONE]') {
+                        requestProgress.progress = 1;
+                    }
+        
+                    try {
+                        pubsub.publish('REQUEST_PROGRESS', {
+                            requestProgress: requestProgress
+                        });
+                    } catch (error) {
+                        console.error('Could not JSON parse stream message', message, error);
                     }
                 });
-
-                // data.on('end', () => {
-                //     console.log("stream done");
-                // });
-            }
-            
+            });
         }
     }
 
     async resolve(args) {
+        // Either we're dealing with an async request, stream, or regular request
         if (args.async || args.stream) {
-            // Asyncronously process the request
-            // this.asyncResolve(args);
             if (!requestState[this.requestId]) {
                 requestState[this.requestId] = {}
             }
@@ -289,7 +280,15 @@ class PathwayResolver {
         if (requestState[this.requestId].canceled) {
             return;
         }
-        const result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+        let result = '';
+
+        // If this text is empty, skip applying the prompt as it will likely be a nonsensical result
+        if (!/^\s*$/.test(text) || parameters?.file) {
+            result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt, this);
+        } else {
+            result = text;
+        }
+        
         requestState[this.requestId].completedCount++;
 
         const { completedCount, totalCount } = requestState[this.requestId];

package/{graphql → server}/plugins/azureTranslatePlugin.js

@@ -35,6 +35,28 @@ class AzureTranslatePlugin extends ModelPlugin {
 
         return this.executeRequest(url, data, params, headers, prompt);
     }
+    
+    // Parse the response from the Azure Translate API
+    parseResponse(data) {
+        if (Array.isArray(data) && data.length > 0 && data[0].translations) {
+            return data[0].translations[0].text.trim();
+        } else {
+            return data;
+        }
+    }
+    
+    // Override the logging function to display the request and response
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+    
+        const modelInput = data[0].Text;
+    
+        console.log(`\x1b[36m${modelInput}\x1b[0m`);
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+    
+        prompt && prompt.debugInfo && (prompt.debugInfo += `${separator}${JSON.stringify(data)}`);
+    }
 }
 
 export default AzureTranslatePlugin;

package/{graphql → server}/plugins/modelPlugin.js

@@ -62,7 +62,7 @@ class ModelPlugin {
             const message = tokenLengths[index].message;
 
             // Skip system messages
-            if (message.role === 'system') {
+            if (message?.role === 'system') {
                 index++;
                 continue;
             }
@@ -113,7 +113,7 @@ class ModelPlugin {
         let output = "";
         if (messages && messages.length) {
             for (let message of messages) {
-                output += (message.role && (message.content || message.content === '')) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
+                output += ((message.author || message.role) && (message.content || message.content === '')) ? `<|im_start|>${(message.author || message.role)}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
             }
             // you always want the assistant to respond next so add a
             // directive for that
@@ -124,6 +124,7 @@ class ModelPlugin {
         return output;
     }
 
+    // compile the Prompt    
     getCompiledPrompt(text, parameters, prompt) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const modelPrompt = this.getModelPrompt(prompt, parameters);
@@ -132,9 +133,9 @@ class ModelPlugin {
         const modelPromptMessagesML = this.messagesToChatML(modelPromptMessages);
 
         if (modelPromptMessagesML) {
-            return { modelPromptMessages, tokenLength: encode(modelPromptMessagesML).length };
+            return { modelPromptMessages, tokenLength: encode(modelPromptMessagesML).length, modelPrompt };
         } else {
-            return { modelPromptText, tokenLength: encode(modelPromptText).length };
+            return { modelPromptText, tokenLength: encode(modelPromptText).length, modelPrompt };
         }
     }
 
@@ -147,12 +148,11 @@ class ModelPlugin {
         return this.promptParameters.inputParameters?.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
     }
 
-
     getModelPrompt(prompt, parameters) {
         if (typeof(prompt) === 'function') {
-        return prompt(parameters);
+            return prompt(parameters);
         } else {
-        return prompt;
+            return prompt;
         }
     }
 
@@ -160,20 +160,20 @@ class ModelPlugin {
         if (!modelPrompt.messages) {
             return null;
         }
-
+    
         // First run handlebars compile on the pathway messages
         const compiledMessages = modelPrompt.messages.map((message) => {
             if (message.content) {
                 const compileText = HandleBars.compile(message.content);
                 return {
-                    role: message.role,
+                    ...message,
                     content: compileText({ ...combinedParameters, text }),
                 };
             } else {
                 return message;
             }
         });
-
+    
         // Next add in any parameters that are referenced by name in the array
         const expandedMessages = compiledMessages.flatMap((message) => {
             if (typeof message === 'string') {
@@ -188,7 +188,7 @@ class ModelPlugin {
                 return [message];
             }
         });
-
+     
         return expandedMessages;
     }
 
@@ -197,44 +197,17 @@ class ModelPlugin {
         return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
     }
 
-    //simples form string single or list return
-    parseResponse(data) {
-        const { choices } = data;
-        if (!choices || !choices.length) {
-            if (Array.isArray(data) && data.length > 0 && data[0].translations) {
-                return data[0].translations[0].text.trim();
-            } else {
-                return data;
-            }
-        }
-
-        // if we got a choices array back with more than one choice, return the whole array
-        if (choices.length > 1) {
-            return choices;
-        }
-
-        // otherwise, return the first choice
-        const textResult = choices[0].text && choices[0].text.trim();
-        const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
-
-        return messageResult ?? textResult ?? null;
-    }
+    // Default response parsing
+    parseResponse(data) { return data; };
 
+    // Default simple logging
     logRequestData(data, responseData, prompt) {
         const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
         console.log(separator);
     
         const modelInput = data.prompt || (data.messages && data.messages[0].content) || (data.length > 0 && data[0].Text) || null;
     
-        if (data && data.messages && data.messages.length > 1) {
-            data.messages.forEach((message, index) => {
-                const words = message.content.split(" ");
-                const tokenCount = encode(message.content).length;
-                const preview = words.length < 41 ? message.content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
-    
-                console.log(`\x1b[36mMessage ${index + 1}: Role: ${message.role}, Tokens: ${tokenCount}, Content: "${preview}"\x1b[0m`);
-            });
-        } else {
+        if (modelInput) {
             console.log(`\x1b[36m${modelInput}\x1b[0m`);
         }