@aj-archipelago/cortex 0.0.3 → 0.0.5

package/.release-it.json

@@ -0,0 +1,9 @@
+{
+    "git": {
+        "commitMessage": "chore: release v${version}"
+    },
+    "github": {
+        "release": true,
+        "autoGenerate": true
+    }
+}

package/config.js

@@ -50,6 +50,7 @@ var config = convict({
         format: Object,
         default: {
             "oai-td3": {
+                "type": "OPENAI-COMPLETION",
                 "url": "{{openaiApiUrl}}",
                 "headers": {
                     "Authorization": "Bearer {{openaiApiKey}}",

package/graphql/parser.js

@@ -1,22 +1,3 @@
-//simples form string single or list return
-const getResponseResult = (data) => {
-    const { choices } = data;
-    if (!choices || !choices.length) {
-        return; //TODO no choices case
-    }
-
-    // if we got a choices array back with more than one choice, return the whole array
-    if (choices.length > 1) {
-        return choices;
-    }
-
-    // otherwise, return the first choice
-    const textResult = choices[0].text && choices[0].text.trim();
-    const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
-
-    return messageResult || textResult || null;
-}
-
 //simply trim and parse with given regex
 const regexParser = (text, regex) => {
     return text.trim().split(regex).map(s => s.trim()).filter(s => s.length);
@@ -51,8 +32,7 @@ const parseNumberedObjectList = (text, format) => {
 }
 
 module.exports = {
-    getResponseResult,
     regexParser,
     parseNumberedList,
-    parseNumberedObjectList
+    parseNumberedObjectList,
 };

package/graphql/pathwayPrompter.js

@@ -1,145 +1,58 @@
-const { request } = require("../lib/request");
+// PathwayPrompter.js
+const OpenAIChatPlugin = require('./plugins/openAIChatPlugin');
+const OpenAICompletionPlugin = require('./plugins/openAICompletionPlugin');
+const AzureTranslatePlugin = require('./plugins/azureTranslatePlugin');
 const handlebars = require("handlebars");
-const { getResponseResult } = require("./parser");
 const { Exception } = require("handlebars");
-const { encode } = require("gpt-3-encoder");
-
-const DEFAULT_MAX_TOKENS = 4096;
-const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
 
 // register functions that can be called directly in the prompt markdown
-handlebars.registerHelper('stripHTML', function(value) {
+handlebars.registerHelper('stripHTML', function (value) {
     return value.replace(/<[^>]*>/g, '');
-    });
+});
 
-handlebars.registerHelper('now', function() {
+handlebars.registerHelper('now', function () {
     return new Date().toISOString();
-    });
-    
-class PathwayPrompter {
-    constructor({ config, pathway }) {
-        // If the pathway specifies a model, use that, otherwise use the default
-        this.modelName = pathway.model || config.get('defaultModelName');
-        // Get the model from the config
-        this.model = config.get('models')[this.modelName];
-        // If the model doesn't exist, throw an exception
-        if (!this.model) {
-            throw new Exception(`Model ${this.modelName} not found in config`);
-        }
-        this.environmentVariables = config.getEnv();
-        this.temperature = pathway.temperature;
-        this.pathwayPrompt = pathway.prompt;
-        this.pathwayName = pathway.name;
-        this.promptParameters = {}
-        // Make all of the parameters defined on the pathway itself available to the prompt
-        for (const [k, v] of Object.entries(pathway)) {
-            this.promptParameters[k] = v.default ?? v;
-        }
-        if (pathway.inputParameters) {
-            for (const [k, v] of Object.entries(pathway.inputParameters)) {
-                this.promptParameters[k] = v.default ?? v;
-            }
-        }
-        this.requestCount = 1
-    }
-
-    getModelMaxTokenLength() {
-        return (this.promptParameters.maxTokenLength ?? this.model.maxTokenLength ?? DEFAULT_MAX_TOKENS);
-    }
-
-    getPromptTokenRatio() {
-        return this.promptParameters.inputParameters.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
-    }
-
-    requestUrl() {
-        const generateUrl = handlebars.compile(this.model.url);
-        return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
-    }
-
-    requestParameters(text, parameters, prompt) {
-        // the prompt object will either have a messages property or a prompt propery
-        // or it could be a function that returns prompt text
-
-        const combinedParameters = { ...this.promptParameters, ...parameters };
+});
 
-        // if it's a messages prompt, compile the messages and send them directly
-        // to the API - a messages prompt automatically means its a chat-style
-        // conversation
-        if (prompt.messages)
-        {
-            const compiledMessages = prompt.messages.map((message) => {
-                const compileText = handlebars.compile(message.content);
-                return { role: message.role,
-                content: compileText({...combinedParameters, text})
-                }
-            })
+handlebars.registerHelper('toJSON', function(object) {
+    return JSON.stringify(object);
+});
+  
 
-            return {
-                messages: compiledMessages,
-                temperature: this.temperature ?? 0.7,
-            }
-        }
+class PathwayPrompter {
+    constructor({ config, pathway }) {
 
-        // otherwise, we need to get the prompt text
-        let promptText;
+        const modelName = pathway.model || config.get('defaultModelName');
+        const model = config.get('models')[modelName];
 
-        if (typeof (prompt) === 'function') {
-            promptText = prompt(parameters);
-        }
-        else {
-            promptText = prompt.prompt;
+        if (!model) {
+        throw new Exception(`Model ${modelName} not found in config`);
         }
 
-        const interpolatePrompt = handlebars.compile(promptText);
-        const constructedPrompt = interpolatePrompt({ ...combinedParameters, text });
-        
-        // this prompt could be for either a chat-style conversation or a completion-style
-        // conversation. They require different parameters.
-        
-        let params = {};
+        let plugin;
 
-        if (this.model.type === 'OPENAI_CHAT') {
-            params = {
-                messages: [ {"role": "user", "content": constructedPrompt} ],
-                temperature: this.temperature ?? 0.7,
-            }
-        } else {
-            params = {
-                prompt: constructedPrompt,
-                max_tokens: this.getModelMaxTokenLength() - encode(constructedPrompt).length - 1,
-                // model: "text-davinci-002",
-                temperature: this.temperature ?? 0.7,
-                // "top_p": 1,
-                // "n": 1,
-                // "presence_penalty": 0,
-                // "frequency_penalty": 0,
-                // "best_of": 1,
-            }
+        switch (model.type) {
+            case 'OPENAI-CHAT':
+                plugin = new OpenAIChatPlugin(config, pathway);
+                break;
+            case 'AZURE-TRANSLATE':
+                plugin = new AzureTranslatePlugin(config, pathway);
+                break;
+            case 'OPENAI-COMPLETION':
+                plugin = new OpenAICompletionPlugin(config, pathway);
+                break;
+            default:
+                throw new Exception(`Unsupported model type: ${model.type}`);
         }
 
-        return params;
+        this.plugin = plugin;
     }
 
     async execute(text, parameters, prompt) {
-        const requestParameters = this.requestParameters(text, parameters, prompt);
-
-        const url = this.requestUrl(text);
-        const params = { ...(this.model.params || {}), ...requestParameters }
-        const headers = this.model.headers || {};
-        const data = await request({ url, params, headers }, this.modelName);
-        const modelInput = params.prompt || params.messages[0].content;
-        console.log(`=== ${this.pathwayName}.${this.requestCount++} ===`)
-        console.log(`\x1b[36m${modelInput}\x1b[0m`)
-        console.log(`\x1b[34m> ${getResponseResult(data)}\x1b[0m`)
-
-        if (data.error) {
-            throw new Exception(`An error was returned from the server: ${JSON.stringify(data.error)}`);
-        }
-
-        return getResponseResult(data);
+        return await this.plugin.execute(text, parameters, prompt);
     }
 }
 
 module.exports = {
-    PathwayPrompter
-}
+  PathwayPrompter
+};

package/graphql/pathwayResolver.js

@@ -106,7 +106,7 @@ class PathwayResolver {
                 const warnText = `Your input is possibly too long, truncating! Text length: ${text.length}`;
                 this.warnings.push(warnText);
                 console.warn(warnText);
-                text = truncate(text, chunkMaxChunkTokenLength);
+                text = this.truncate(text, chunkMaxChunkTokenLength);
             }
             return [text];
         }
@@ -116,7 +116,7 @@ class PathwayResolver {
     }
 
     truncate(str, n) {
-        if (this.pathwayPrompter.promptParameters.truncateFromFront) {
+        if (this.pathwayPrompter.plugin.promptParameters.truncateFromFront) {
             return getFirstNToken(str, n);
         }
         return getLastNToken(str, n);
@@ -134,7 +134,7 @@ class PathwayResolver {
         // find the longest prompt
         const maxPromptTokenLength = Math.max(...this.prompts.map(({ prompt }) => prompt ? encode(String(prompt)).length : 0));
         const maxMessagesTokenLength = Math.max(...this.prompts.map(({ messages }) => messages ? messages.reduce((acc, {role, content}) => {
-            return acc + encode(role).length + encode(content).length;
+            return (role && content) ? acc + encode(role).length + encode(content).length : acc;
         }, 0) : 0));
 
         const maxTokenLength = Math.max(maxPromptTokenLength, maxMessagesTokenLength);
@@ -144,8 +144,8 @@ class PathwayResolver {
         
         // the token ratio is the ratio of the total prompt to the result text - both have to be included
         // in computing the max token length
-        const promptRatio = this.pathwayPrompter.getPromptTokenRatio();
-        let maxChunkToken = promptRatio * this.pathwayPrompter.getModelMaxTokenLength() - maxTokenLength;
+        const promptRatio = this.pathwayPrompter.plugin.getPromptTokenRatio();
+        let maxChunkToken = promptRatio * this.pathwayPrompter.plugin.getModelMaxTokenLength() - maxTokenLength;
 
         // if we have to deal with prompts that have both text input
         // and previous result, we need to split the maxChunkToken in half

package/graphql/plugins/azureTranslatePlugin.js

@@ -0,0 +1,42 @@
+// AzureTranslatePlugin.js
+const ModelPlugin = require('./modelPlugin');
+const handlebars = require("handlebars");
+
+class AzureTranslatePlugin extends ModelPlugin {
+    constructor(config, modelName, pathway) {
+        super(config, modelName, pathway);
+    }
+
+    // Set up parameters specific to the Azure Translate API
+    requestParameters(text, parameters, prompt) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const modelPrompt = this.getModelPrompt(prompt, parameters);
+        const modelPromptText = modelPrompt.prompt ? handlebars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
+
+        return {
+            data: [
+                {
+                Text: modelPromptText,
+                },
+            ],
+            params: {
+                to: combinedParameters.to
+            }
+        };
+    }
+
+    // Execute the request to the Azure Translate API
+    async execute(text, parameters, prompt) {
+        const requestParameters = this.requestParameters(text, parameters, prompt);
+
+        const url = this.requestUrl(text);
+
+        const data = requestParameters.data;
+        const params = requestParameters.params;
+        const headers = this.model.headers || {};
+
+        return this.executeRequest(url, data, params, headers);
+    }
+}
+
+module.exports = AzureTranslatePlugin;

package/graphql/plugins/modelPlugin.js

@@ -0,0 +1,138 @@
+// ModelPlugin.js
+const handlebars = require('handlebars');
+const { request } = require("../../lib/request");
+const { getResponseResult } = require("../parser");
+
+const DEFAULT_MAX_TOKENS = 4096;
+const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+
+class ModelPlugin {
+    constructor(config, pathway) {
+        // If the pathway specifies a model, use that, otherwise use the default
+        this.modelName = pathway.model || config.get('defaultModelName');
+        // Get the model from the config
+        this.model = config.get('models')[this.modelName];
+        // If the model doesn't exist, throw an exception
+        if (!this.model) {
+            throw new Error(`Model ${this.modelName} not found in config`);
+        }
+
+        this.config = config;
+        this.environmentVariables = config.getEnv();
+        this.temperature = pathway.temperature;
+        this.pathwayPrompt = pathway.prompt;
+        this.pathwayName = pathway.name;
+        this.promptParameters = {};
+
+        // Make all of the parameters defined on the pathway itself available to the prompt
+        for (const [k, v] of Object.entries(pathway)) {
+            this.promptParameters[k] = v.default ?? v;
+        }
+        if (pathway.inputParameters) {
+            for (const [k, v] of Object.entries(pathway.inputParameters)) {
+                this.promptParameters[k] = v.default ?? v;
+            }
+        }
+
+        this.requestCount = 1;
+    }
+
+    getModelMaxTokenLength() {
+        return (this.promptParameters.maxTokenLength ?? this.model.maxTokenLength ?? DEFAULT_MAX_TOKENS);
+    }
+
+    getPromptTokenRatio() {
+        // TODO: Is this the right order of precedence? inputParameters should maybe be second?
+        return this.promptParameters.inputParameters.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
+    }
+
+
+    getModelPrompt(prompt, parameters) {
+        if (typeof(prompt) === 'function') {
+        return prompt(parameters);
+        } else {
+        return prompt;
+        }
+    }
+
+    getModelPromptMessages(modelPrompt, combinedParameters, text) {
+        if (!modelPrompt.messages) {
+            return null;
+        }
+
+        // First run handlebars compile on the pathway messages
+        const compiledMessages = modelPrompt.messages.map((message) => {
+            if (message.content) {
+                const compileText = handlebars.compile(message.content);
+                return {
+                    role: message.role,
+                    content: compileText({ ...combinedParameters, text }),
+                };
+            } else {
+                return message;
+            }
+        });
+
+        // Next add in any parameters that are referenced by name in the array
+        const expandedMessages = compiledMessages.flatMap((message) => {
+            if (typeof message === 'string') {
+                const match = message.match(/{{(.+?)}}/);
+                const placeholder = match ? match[1] : null;
+                if (placeholder === null) {
+                    return message;
+                } else {
+                    return combinedParameters[placeholder] || [];
+                }
+            } else {
+                return [message];
+            }
+        });
+
+        return expandedMessages;
+    }
+
+    requestUrl() {
+        const generateUrl = handlebars.compile(this.model.url);
+        return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
+    }
+
+    //simples form string single or list return
+    parseResponse(data) {
+        const { choices } = data;
+        if (!choices || !choices.length) {
+            if (Array.isArray(data) && data.length > 0 && data[0].translations) {
+                return data[0].translations[0].text.trim();
+            }
+        }
+
+        // if we got a choices array back with more than one choice, return the whole array
+        if (choices.length > 1) {
+            return choices;
+        }
+
+        // otherwise, return the first choice
+        const textResult = choices[0].text && choices[0].text.trim();
+        const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
+
+        return messageResult || textResult || null;
+    }
+
+    async executeRequest(url, data, params, headers) {
+        const responseData = await request({ url, data, params, headers }, this.modelName);
+        const modelInput = data.prompt || (data.messages && data.messages[0].content) || data[0].Text || null;
+        console.log(`=== ${this.pathwayName}.${this.requestCount++} ===`)
+        console.log(`\x1b[36m${modelInput}\x1b[0m`)
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+
+        if (responseData.error) {
+            throw new Exception(`An error was returned from the server: ${JSON.stringify(responseData.error)}`);
+        }
+
+        return this.parseResponse(responseData);
+    }
+
+}
+
+module.exports = ModelPlugin;
+
+  

package/graphql/plugins/openAiChatPlugin.js

@@ -0,0 +1,35 @@
+// OpenAIChatPlugin.js
+const ModelPlugin = require('./modelPlugin');
+const handlebars = require("handlebars");
+
+class OpenAIChatPlugin extends ModelPlugin {
+    constructor(config, pathway) {
+        super(config, pathway);
+    }
+
+    // Set up parameters specific to the OpenAI Chat API
+    requestParameters(text, parameters, prompt) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const modelPrompt = this.getModelPrompt(prompt, parameters);
+        const modelPromptText = modelPrompt.prompt ? handlebars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
+        const modelPromptMessages = this.getModelPromptMessages(modelPrompt, combinedParameters, text);
+
+        return {
+            messages: modelPromptMessages || [{ "role": "user", "content": modelPromptText }],
+            temperature: this.temperature ?? 0.7,
+        };
+    }
+
+    // Execute the request to the OpenAI Chat API
+    async execute(text, parameters, prompt) {
+        const url = this.requestUrl(text);
+        const requestParameters = this.requestParameters(text, parameters, prompt);
+
+        const data = { ...(this.model.params || {}), ...requestParameters };
+        const params = {};
+        const headers = this.model.headers || {};
+        return this.executeRequest(url, data, params, headers);
+    }
+}
+
+module.exports = OpenAIChatPlugin;

package/graphql/plugins/openAiCompletionPlugin.js

@@ -0,0 +1,65 @@
+// OpenAICompletionPlugin.js
+const ModelPlugin = require('./modelPlugin');
+const handlebars = require("handlebars");
+const { encode } = require("gpt-3-encoder");
+
+//convert a messages array to a simple chatML format
+const messagesToChatML = (messages) => {
+    let output = "";
+    if (messages && messages.length) {
+        for (let message of messages) {
+            output += (message.role && message.content) ? `<|im_start|>${message.role}\n${message.content}\n<|im_end|>\n` : `${message}\n`;
+        }
+        // you always want the assistant to respond next so add a
+        // directive for that
+        output += "<|im_start|>assistant\n";
+    }
+    return output;
+}
+
+class OpenAICompletionPlugin extends ModelPlugin {
+    constructor(config, pathway) {
+        super(config, pathway);
+    }
+
+    // Set up parameters specific to the OpenAI Completion API
+    requestParameters(text, parameters, prompt) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const modelPrompt = this.getModelPrompt(prompt, parameters);
+        const modelPromptText = modelPrompt.prompt ? handlebars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
+        const modelPromptMessages = this.getModelPromptMessages(modelPrompt, combinedParameters, text);
+        const modelPromptMessagesML = messagesToChatML(modelPromptMessages);
+
+        if (modelPromptMessagesML) {
+        return {
+            prompt: modelPromptMessagesML,
+            max_tokens: this.getModelMaxTokenLength() - encode(modelPromptMessagesML).length - 1,
+            temperature: this.temperature ?? 0.7,
+            top_p: 0.95,
+            frequency_penalty: 0,
+            presence_penalty: 0,
+            stop: ["<|im_end|>"]
+        };
+        } else {
+        return {
+            prompt: modelPromptText,
+            max_tokens: this.getModelMaxTokenLength() - encode(modelPromptText).length - 1,
+            temperature: this.temperature ?? 0.7,
+        };
+        }
+    }
+
+    // Execute the request to the OpenAI Completion API
+    async execute(text, parameters, prompt) {
+        const url = this.requestUrl(text);
+        const requestParameters = this.requestParameters(text, parameters, prompt);
+    
+        const data = { ...(this.model.params || {}), ...requestParameters };
+        const params = {};
+        const headers = this.model.headers || {};
+        return this.executeRequest(url, data, params, headers);
+    }
+}
+
+module.exports = OpenAICompletionPlugin;
+

package/graphql/prompt.js

@@ -25,7 +25,7 @@ function promptContains(variable, prompt) {
     // if it's an array, it's the messages format
     if (Array.isArray(prompt)) {
       prompt.forEach(p => {
-        while ((match = p.content && regexp.exec(p.content)) !== null) {
+        while (match = p.content && regexp.exec(p.content)) {
           matches.push(match[1]);
         }
       });

package/graphql/resolver.js

@@ -16,7 +16,7 @@ const rootResolver = async (parent, args, contextValue, info) => {
     contextValue.pathwayResolver = pathwayResolver;
 
     // Add request parameters back as debug
-    const requestParameters = pathwayResolver.prompts.map((prompt) => pathwayResolver.pathwayPrompter.requestParameters(args.text, args, prompt));
+    const requestParameters = pathwayResolver.prompts.map((prompt) => pathwayResolver.pathwayPrompter.plugin.requestParameters(args.text, args, prompt));
     const debug = JSON.stringify(requestParameters);
 
     // Execute the request with timeout

package/graphql/typeDef.js

@@ -12,10 +12,11 @@ const typeDef = (pathway) => {
     const fieldsStr = !fields ? `` : fields.map(f => `${f}: String`).join('\n    ');
 
     const typeName = fields ? `${objName}Result` : `String`;
+    const messageType = `input Message { role: String, content: String }`;
+
     const type = fields ? `type ${typeName} {
     ${fieldsStr}
-}` : ``;
-
+    }` : ``;
 
     const resultStr = pathway.list ? `[${typeName}]` : typeName;
 
@@ -29,18 +30,21 @@ const typeDef = (pathway) => {
 
 
     const params = { ...defaultInputParameters, ...inputParameters };
-    const paramsStr = Object.entries(params).map(
-        ([key, value]) => `${key}: ${GRAPHQL_TYPE_MAP[typeof (value)]} = ${typeof (value) == `string` ? `"${value}"` : value}`).join('\n');
-
 
-    return `${type}
-
-${responseType}
-
-extend type Query {
-    ${name}(${paramsStr}): ${objName}
-}  
-`;
+    const paramsStr = Object.entries(params).map(
+        ([key, value]) => {
+            if (typeof value === 'object' && Array.isArray(value)) {
+                return `${key}: [Message] = []`;
+            } else {
+                return `${key}: ${GRAPHQL_TYPE_MAP[typeof (value)]} = ${typeof (value) === 'string' ? `"${value}"` : value}`;
+            }
+        }
+        ).join('\n');
+          
+
+    const definition = `${messageType}\n\n${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
+    //console.log(definition);
+    return definition;
 }
 
 module.exports = {

package/lib/request.js

@@ -14,21 +14,21 @@ const buildLimiters = (config) => {
 }
 
 const MAX_RETRY = 10;
-const postRequest = async ({ url, params, headers }, model) => {
+const postRequest = async ({ url, data, params, headers }, model) => {
     let retry = 0;
     const errors = []
     for (let i = 0; i < MAX_RETRY; i++) {
         try {
             if (i > 0) {
-                console.log(`Retrying request #retry ${i}: ${JSON.stringify(params)}...`);
+                console.log(`Retrying request #retry ${i}: ${JSON.stringify(data)}...`);
                 await new Promise(r => setTimeout(r, 200 * Math.pow(2, i))); // exponential backoff
             }
             if (!limiters[model]) {
                 throw new Error(`No limiter for model ${model}!`);
             }
-            return await limiters[model].schedule(() => axios.post(url, params, { headers }));
+            return await limiters[model].schedule(() => axios.post(url, data, { params, headers }));
         } catch (e) {
-            console.error(`Failed request with params ${JSON.stringify(params)}: ${e}`);
+            console.error(`Failed request with data ${JSON.stringify(data)}: ${e}`);
             errors.push(e);
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "0.0.3",
+  "version": "0.0.5",
   "description": "Project Cortex",
   "repository": {
     "type": "git",

package/pathways/basePathway.js

@@ -19,5 +19,5 @@ module.exports = {
     useParallelChunkProcessing: false,
     useInputSummarization: false,    
     truncateFromFront: false,
-    timeout: 60, // in seconds
+    timeout: 120, // in seconds
 }

package/pathways/paraphrase.js

@@ -1,3 +1,3 @@
 module.exports = {
-    prompt: `Rewrite the following:\n\n{{text}}`
+    prompt: `Rewrite the following:\n\n{{{text}}}`
 }