@aj-archipelago/cortex 0.0.3

package/graphql/graphql.js

@@ -0,0 +1,183 @@
+const { createServer } = require('http');
+const {
+    ApolloServerPluginDrainHttpServer,
+    ApolloServerPluginLandingPageLocalDefault,
+} = require("apollo-server-core");
+const { makeExecutableSchema } = require('@graphql-tools/schema');
+const { WebSocketServer } = require('ws');
+const { useServer } = require('graphql-ws/lib/use/ws');
+const express = require('express');
+
+/// Create apollo graphql server
+const Keyv = require("keyv");
+const { KeyvAdapter } = require("@apollo/utils.keyvadapter");
+const responseCachePlugin = require('apollo-server-plugin-response-cache').default
+
+const subscriptions = require('./subscriptions');
+const { buildLimiters } = require('../lib/request');
+const { cancelRequestResolver } = require('./resolver');
+const { buildPathways, buildModels } = require('../config');
+
+const requestState = {}; // Stores the state of each request
+
+const getPlugins = (config) => {
+    // server plugins
+    const plugins = [
+        ApolloServerPluginLandingPageLocalDefault({ embed: true }), // For local development.   
+    ];
+
+    //if cache is enabled and Redis is available, use it
+    let cache;
+    if (config.get('enableCache') && config.get('storageConnectionString')) {
+        cache = new KeyvAdapter(new Keyv(config.get('storageConnectionString'),{
+            ssl: true,
+            abortConnect: false,            
+        }));
+        //caching similar strings, embedding hashing, ... #delta similarity 
+        // TODO: custom cache key:
+        // https://www.apollographql.com/docs/apollo-server/performance/cache-backends#implementing-your-own-cache-backend
+        plugins.push(responseCachePlugin({ cache }));
+    }
+
+    return { plugins, cache };
+}
+
+//typeDefs
+const getTypedefs = (pathways) => {
+
+    const defaultTypeDefs = `#graphql
+    enum CacheControlScope {
+        PUBLIC
+        PRIVATE
+    }
+
+    directive @cacheControl(
+        maxAge: Int
+        scope: CacheControlScope
+        inheritMaxAge: Boolean
+    ) on FIELD_DEFINITION | OBJECT | INTERFACE | UNION
+
+    type Query {
+        _ : Boolean
+    }
+
+    type Mutation {
+        cancelRequest(requestId: String!): Boolean
+    }
+    
+    type RequestSubscription {
+        requestId: String
+        progress: Float
+        data: String
+    }
+
+    type Subscription {
+        requestProgress(requestId: String!): RequestSubscription
+    }
+`;
+
+    const typeDefs = [defaultTypeDefs, ...Object.values(pathways).filter(p=>!p.disabled).map(p => p.typeDef(p))];
+    return typeDefs.join('\n');
+}
+
+const getResolvers = (config, pathways) => {
+    const resolverFunctions = {};
+    for (const [name, pathway] of Object.entries(pathways)) {
+        if (pathway.disabled) continue; //skip disabled pathways
+        resolverFunctions[name] = (parent, args, contextValue, info) => {
+            // add shared state to contextValue
+            contextValue.pathway = pathway;
+            return pathway.rootResolver(parent, args, contextValue, info);
+        }
+    }
+
+    const resolvers = {
+        Query: resolverFunctions,
+        Mutation: { 'cancelRequest': cancelRequestResolver },
+        Subscription: subscriptions,
+    }
+
+    return resolvers;
+}
+
+//graphql api build factory method
+const build = (config) => {
+    // First perform config build
+    buildPathways(config);
+    buildModels(config);
+
+    // build api limiters 
+    buildLimiters(config);
+
+    //build api
+    const pathways = config.get('pathways');
+
+    const typeDefs = getTypedefs(pathways);
+    const resolvers = getResolvers(config, pathways);
+
+    const schema = makeExecutableSchema({ typeDefs, resolvers });
+
+    const { plugins, cache } = getPlugins(config);
+
+    const { ApolloServer, gql } = require('apollo-server-express');
+    const app = express()
+    const httpServer = createServer(app);
+
+    // Creating the WebSocket server
+    const wsServer = new WebSocketServer({
+        // This is the `httpServer` we created in a previous step.
+        server: httpServer,
+        // Pass a different path here if your ApolloServer serves at
+        // a different path.
+        path: '/graphql',
+    });
+
+    // Hand in the schema we just created and have the
+    // WebSocketServer start listening.
+    const serverCleanup = useServer({ schema }, wsServer);
+
+    const server = new ApolloServer({
+        schema,
+        csrfPrevention: true,
+        plugins: plugins.concat([// Proper shutdown for the HTTP server.
+            ApolloServerPluginDrainHttpServer({ httpServer }),
+
+            // Proper shutdown for the WebSocket server.
+            {
+                async serverWillStart() {
+                    return {
+                        async drainServer() {
+                            await serverCleanup.dispose();
+                        },
+                    };
+                },
+            }]),
+        context: ({ req, res }) => ({ req, res, config, requestState }),
+    });
+
+    // if local start server
+    const startServer = async () => {
+        await server.start();
+        server.applyMiddleware({ app });
+
+        // Now that our HTTP server is fully set up, we can listen to it.
+        httpServer.listen(config.get('PORT'), () => {
+            console.log(`🚀 Server is now running at http://localhost:${config.get('PORT')}${server.graphqlPath}`);
+        });
+    };
+
+    app.use((req, res, next) => {
+        if (process.env.API_KEY && req.headers.api_key !== process.env.API_KEY && req.query.api_key !== process.env.API_KEY) {
+            res.status(401).send('Unauthorized');
+        }
+
+        next();
+    })
+
+    return { server, startServer, cache, plugins, typeDefs, resolvers }
+}
+
+
+module.exports = {
+    build
+};

package/graphql/parser.js

@@ -0,0 +1,58 @@
+//simples form string single or list return
+const getResponseResult = (data) => {
+    const { choices } = data;
+    if (!choices || !choices.length) {
+        return; //TODO no choices case
+    }
+
+    // if we got a choices array back with more than one choice, return the whole array
+    if (choices.length > 1) {
+        return choices;
+    }
+
+    // otherwise, return the first choice
+    const textResult = choices[0].text && choices[0].text.trim();
+    const messageResult = choices[0].message && choices[0].message.content && choices[0].message.content.trim();
+
+    return messageResult || textResult || null;
+}
+
+//simply trim and parse with given regex
+const regexParser = (text, regex) => {
+    return text.trim().split(regex).map(s => s.trim()).filter(s => s.length);
+}
+
+// parse numbered list text format into list
+// this supports most common numbered list returns like "1.", "1)", "1-"
+const parseNumberedList = (str) => {
+    return regexParser(str, /^\s*[\[\{\(]*\d+[\s.=\-:,;\]\)\}]/gm);
+}
+
+// parse a numbered object list text format into list of objects
+const parseNumberedObjectList = (text, format) => {
+    const fields = format.match(/\b(\w+)\b/g);
+    const values = parseNumberedList(text);
+
+    const result = [];
+    for (const value of values) {
+        try {
+            const splitted = regexParser(value, /[:-](.*)/);
+            const obj = {};
+            for (let i = 0; i < fields.length; i++) {
+                obj[fields[i]] = splitted[i];
+            }
+            result.push(obj);
+        } catch (e) {
+            console.warn(`Failed to parse value in parseNumberedObjectList, value: ${value}, fields: ${fields}`);
+        }
+    }
+
+    return result;
+}
+
+module.exports = {
+    getResponseResult,
+    regexParser,
+    parseNumberedList,
+    parseNumberedObjectList
+};

package/graphql/pathwayPrompter.js

@@ -0,0 +1,145 @@
+const { request } = require("../lib/request");
+const handlebars = require("handlebars");
+const { getResponseResult } = require("./parser");
+const { Exception } = require("handlebars");
+const { encode } = require("gpt-3-encoder");
+
+const DEFAULT_MAX_TOKENS = 4096;
+const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
+
+// register functions that can be called directly in the prompt markdown
+handlebars.registerHelper('stripHTML', function(value) {
+    return value.replace(/<[^>]*>/g, '');
+    });
+
+handlebars.registerHelper('now', function() {
+    return new Date().toISOString();
+    });
+    
+class PathwayPrompter {
+    constructor({ config, pathway }) {
+        // If the pathway specifies a model, use that, otherwise use the default
+        this.modelName = pathway.model || config.get('defaultModelName');
+        // Get the model from the config
+        this.model = config.get('models')[this.modelName];
+        // If the model doesn't exist, throw an exception
+        if (!this.model) {
+            throw new Exception(`Model ${this.modelName} not found in config`);
+        }
+        this.environmentVariables = config.getEnv();
+        this.temperature = pathway.temperature;
+        this.pathwayPrompt = pathway.prompt;
+        this.pathwayName = pathway.name;
+        this.promptParameters = {}
+        // Make all of the parameters defined on the pathway itself available to the prompt
+        for (const [k, v] of Object.entries(pathway)) {
+            this.promptParameters[k] = v.default ?? v;
+        }
+        if (pathway.inputParameters) {
+            for (const [k, v] of Object.entries(pathway.inputParameters)) {
+                this.promptParameters[k] = v.default ?? v;
+            }
+        }
+        this.requestCount = 1
+    }
+
+    getModelMaxTokenLength() {
+        return (this.promptParameters.maxTokenLength ?? this.model.maxTokenLength ?? DEFAULT_MAX_TOKENS);
+    }
+
+    getPromptTokenRatio() {
+        return this.promptParameters.inputParameters.tokenRatio ?? this.promptParameters.tokenRatio ?? DEFAULT_PROMPT_TOKEN_RATIO;
+    }
+
+    requestUrl() {
+        const generateUrl = handlebars.compile(this.model.url);
+        return generateUrl({ ...this.model, ...this.environmentVariables, ...this.config });
+    }
+
+    requestParameters(text, parameters, prompt) {
+        // the prompt object will either have a messages property or a prompt propery
+        // or it could be a function that returns prompt text
+
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+
+        // if it's a messages prompt, compile the messages and send them directly
+        // to the API - a messages prompt automatically means its a chat-style
+        // conversation
+        if (prompt.messages)
+        {
+            const compiledMessages = prompt.messages.map((message) => {
+                const compileText = handlebars.compile(message.content);
+                return { role: message.role,
+                content: compileText({...combinedParameters, text})
+                }
+            })
+
+            return {
+                messages: compiledMessages,
+                temperature: this.temperature ?? 0.7,
+            }
+        }
+
+        // otherwise, we need to get the prompt text
+        let promptText;
+
+        if (typeof (prompt) === 'function') {
+            promptText = prompt(parameters);
+        }
+        else {
+            promptText = prompt.prompt;
+        }
+
+        const interpolatePrompt = handlebars.compile(promptText);
+        const constructedPrompt = interpolatePrompt({ ...combinedParameters, text });
+        
+        // this prompt could be for either a chat-style conversation or a completion-style
+        // conversation. They require different parameters.
+        
+        let params = {};
+
+        if (this.model.type === 'OPENAI_CHAT') {
+            params = {
+                messages: [ {"role": "user", "content": constructedPrompt} ],
+                temperature: this.temperature ?? 0.7,
+            }
+        } else {
+            params = {
+                prompt: constructedPrompt,
+                max_tokens: this.getModelMaxTokenLength() - encode(constructedPrompt).length - 1,
+                // model: "text-davinci-002",
+                temperature: this.temperature ?? 0.7,
+                // "top_p": 1,
+                // "n": 1,
+                // "presence_penalty": 0,
+                // "frequency_penalty": 0,
+                // "best_of": 1,
+            }
+        }
+
+        return params;
+    }
+
+    async execute(text, parameters, prompt) {
+        const requestParameters = this.requestParameters(text, parameters, prompt);
+
+        const url = this.requestUrl(text);
+        const params = { ...(this.model.params || {}), ...requestParameters }
+        const headers = this.model.headers || {};
+        const data = await request({ url, params, headers }, this.modelName);
+        const modelInput = params.prompt || params.messages[0].content;
+        console.log(`=== ${this.pathwayName}.${this.requestCount++} ===`)
+        console.log(`\x1b[36m${modelInput}\x1b[0m`)
+        console.log(`\x1b[34m> ${getResponseResult(data)}\x1b[0m`)
+
+        if (data.error) {
+            throw new Exception(`An error was returned from the server: ${JSON.stringify(data.error)}`);
+        }
+
+        return getResponseResult(data);
+    }
+}
+
+module.exports = {
+    PathwayPrompter
+}

package/graphql/pathwayResolver.js

@@ -0,0 +1,250 @@
+const { PathwayPrompter } = require('./pathwayPrompter');
+const {
+    v4: uuidv4,
+} = require('uuid');
+const pubsub = require('./pubsub');
+const { encode } = require('gpt-3-encoder')
+const { getFirstNToken, getLastNToken, getSemanticChunks } = require('./chunker');
+const { PathwayResponseParser } = require('./pathwayResponseParser');
+const { Prompt } = require('./prompt');
+const { getv, setv } = require('../lib/keyValueStorageClient');
+
+const MAX_PREVIOUS_RESULT_TOKEN_LENGTH = 1000;
+
+const callPathway = async (config, pathwayName, requestState, { text, ...parameters }) => {
+    const pathwayResolver = new PathwayResolver({ config, pathway: config.get(`pathways.${pathwayName}`), requestState });
+    return await pathwayResolver.resolve({ text, ...parameters });
+}
+
+class PathwayResolver {
+    constructor({ config, pathway, requestState }) {
+        this.config = config;
+        this.requestState = requestState;
+        this.pathway = pathway;
+        this.useInputChunking = pathway.useInputChunking;
+        this.chunkMaxTokenLength = 0;
+        this.warnings = [];
+        this.requestId = uuidv4();
+        this.responseParser = new PathwayResponseParser(pathway);
+        this.pathwayPrompter = new PathwayPrompter({ config, pathway });
+        this.previousResult = '';
+        this.prompts = [];
+        this._pathwayPrompt = '';
+
+        Object.defineProperty(this, 'pathwayPrompt', {
+            get() {
+                return this._pathwayPrompt;
+            },
+            set(value) {
+                this._pathwayPrompt = value;
+                if (!Array.isArray(this._pathwayPrompt)) {
+                    this._pathwayPrompt = [this._pathwayPrompt];
+                }
+                this.prompts = this._pathwayPrompt.map(p => (p instanceof Prompt) ? p : new Prompt({ prompt:p }));
+                this.chunkMaxTokenLength = this.getChunkMaxTokenLength();
+            }
+        });
+
+        this.pathwayPrompt = pathway.prompt;
+    }
+
+    async resolve(args) {
+        if (args.async) {
+            // Asynchronously process the request
+            this.promptAndParse(args).then((data) => {
+                this.requestState[this.requestId].data = data;
+                pubsub.publish('REQUEST_PROGRESS', {
+                    requestProgress: {
+                        requestId: this.requestId,
+                        data: JSON.stringify(data)
+                    }
+                });
+            });
+
+            return this.requestId;
+        }
+        else {
+            // Syncronously process the request
+            return await this.promptAndParse(args);
+        }
+    }
+
+    async promptAndParse(args) {
+
+        // Get saved context from contextId or change contextId if needed
+        const { contextId } = args;
+        this.savedContextId = contextId ? contextId : null;
+        this.savedContext = contextId ? (getv && await getv(contextId) || {}) : {};
+
+        // Save the context before processing the request
+        const savedContextStr = JSON.stringify(this.savedContext);
+
+        // Process the request
+        const data = await this.processRequest(args);
+
+        // Update saved context if it has changed, generating a new contextId if necessary
+        if (savedContextStr !== JSON.stringify(this.savedContext)) {
+            this.savedContextId = this.savedContextId || uuidv4();
+            setv && setv(this.savedContextId, this.savedContext);
+        }
+
+        // Return the result
+        return this.responseParser.parse(data);
+    }
+
+    // Here we choose how to handle long input - either summarize or chunk
+    processInputText(text) {
+        let chunkMaxChunkTokenLength = 0;
+        if (this.pathway.inputChunkSize) {
+            chunkMaxChunkTokenLength = Math.min(this.pathway.inputChunkSize, this.chunkMaxTokenLength);
+        } else {
+             chunkMaxChunkTokenLength = this.chunkMaxTokenLength;
+        }
+        const encoded = encode(text);
+        if (!this.useInputChunking || encoded.length <= chunkMaxChunkTokenLength) { // no chunking, return as is
+            if (encoded.length >= chunkMaxChunkTokenLength) {
+                const warnText = `Your input is possibly too long, truncating! Text length: ${text.length}`;
+                this.warnings.push(warnText);
+                console.warn(warnText);
+                text = truncate(text, chunkMaxChunkTokenLength);
+            }
+            return [text];
+        }
+
+        // chunk the text and return the chunks with newline separators
+        return getSemanticChunks({ text, maxChunkToken: chunkMaxChunkTokenLength });
+    }
+
+    truncate(str, n) {
+        if (this.pathwayPrompter.promptParameters.truncateFromFront) {
+            return getFirstNToken(str, n);
+        }
+        return getLastNToken(str, n);
+    }
+
+    async summarizeIfEnabled({ text, ...parameters }) {
+        if (this.pathway.useInputSummarization) {
+            return await callPathway(this.config, 'summary', this.requestState, { text, targetLength: 1000, ...parameters });
+        }
+        return text;
+    }
+
+    // Calculate the maximum token length for a chunk
+    getChunkMaxTokenLength() {
+        // find the longest prompt
+        const maxPromptTokenLength = Math.max(...this.prompts.map(({ prompt }) => prompt ? encode(String(prompt)).length : 0));
+        const maxMessagesTokenLength = Math.max(...this.prompts.map(({ messages }) => messages ? messages.reduce((acc, {role, content}) => {
+            return acc + encode(role).length + encode(content).length;
+        }, 0) : 0));
+
+        const maxTokenLength = Math.max(maxPromptTokenLength, maxMessagesTokenLength);
+
+        // find out if any prompts use both text input and previous result
+        const hasBothProperties = this.prompts.some(prompt => prompt.usesInputText && prompt.usesPreviousResult);
+        
+        // the token ratio is the ratio of the total prompt to the result text - both have to be included
+        // in computing the max token length
+        const promptRatio = this.pathwayPrompter.getPromptTokenRatio();
+        let maxChunkToken = promptRatio * this.pathwayPrompter.getModelMaxTokenLength() - maxTokenLength;
+
+        // if we have to deal with prompts that have both text input
+        // and previous result, we need to split the maxChunkToken in half
+        maxChunkToken = hasBothProperties ? maxChunkToken / 2 : maxChunkToken;
+
+        // detect if the longest prompt might be too long to allow any chunk size
+        if (maxChunkToken && maxChunkToken <= 0) {
+            throw new Error(`Your prompt is too long! Split to multiple prompts or reduce length of your prompt, prompt length: ${maxPromptLength}`);
+        }
+        return maxChunkToken;
+    }
+
+    // Process the request and return the result        
+    async processRequest({ text, ...parameters }) {
+
+        text = await this.summarizeIfEnabled({ text, ...parameters }); // summarize if flag enabled
+        const chunks = this.processInputText(text);
+
+        const anticipatedRequestCount = chunks.length * this.prompts.length;
+
+        if ((this.requestState[this.requestId] || {}).canceled) {
+            throw new Error('Request canceled');
+        }
+
+        // Store the request state
+        this.requestState[this.requestId] = { totalCount: anticipatedRequestCount, completedCount: 0 };
+
+        // If pre information is needed, apply current prompt with previous prompt info, only parallelize current call
+        if (this.pathway.useParallelChunkProcessing) {
+            // Apply each prompt across all chunks in parallel
+            // this.previousResult is not available at the object level as it is different for each chunk
+            this.previousResult = '';
+            const data = await Promise.all(chunks.map(chunk =>
+                this.applyPromptsSerially(chunk, parameters)));
+            // Join the chunks with newlines
+            return data.join("\n\n");
+        } else {
+            // Apply prompts one by one, serially, across all chunks
+            // This is the default processing mode and will make previousResult available at the object level
+            let previousResult = '';
+            let result = '';
+
+            for (let i = 0; i < this.prompts.length; i++) {
+                // If the prompt doesn't contain {{text}} then we can skip the chunking, and also give that token space to the previous result
+                if (!this.prompts[i].usesTextInput) {
+                    // Limit context to it's N + text's characters
+                    previousResult = this.truncate(previousResult, 2 * this.chunkMaxTokenLength);
+                    result = await this.applyPrompt(this.prompts[i], null, { ...parameters, previousResult });
+                } else {
+                    // Limit context to N characters
+                    previousResult = this.truncate(previousResult, this.chunkMaxTokenLength);
+                    result = await Promise.all(chunks.map(chunk =>
+                        this.applyPrompt(this.prompts[i], chunk, { ...parameters, previousResult })));
+                    result = result.join("\n\n")
+                }
+
+                // If this is any prompt other than the last, use the result as the previous context
+                if (i < this.prompts.length - 1) {
+                    previousResult = result;
+                }
+            }
+            // store the previous result in the PathwayResolver
+            this.previousResult = previousResult;
+            return result;
+        }
+
+    }
+
+    async applyPromptsSerially(text, parameters) {
+        let previousResult = '';
+        let result = '';
+        for (const prompt of this.prompts) {
+            previousResult = result;
+            result = await this.applyPrompt(prompt, text, { ...parameters, previousResult });
+        }
+        return result;
+    }
+
+    async applyPrompt(prompt, text, parameters) {
+        if (this.requestState[this.requestId].canceled) {
+            return;
+        }
+        const result = await this.pathwayPrompter.execute(text, { ...parameters, ...this.savedContext }, prompt);
+        this.requestState[this.requestId].completedCount++;
+
+        const { completedCount, totalCount } = this.requestState[this.requestId];
+
+        pubsub.publish('REQUEST_PROGRESS', {
+            requestProgress: {
+                requestId: this.requestId,
+                progress: completedCount / totalCount,
+            }
+        });
+
+        if (prompt.saveResultTo) {
+            this.savedContext[prompt.saveResultTo] = result;
+        }
+        return result;
+    }
+}
+
+module.exports = { PathwayResolver };

package/graphql/pathwayResponseParser.js

@@ -0,0 +1,24 @@
+const { parseNumberedList, parseNumberedObjectList } = require('./parser')
+
+class PathwayResponseParser {
+    constructor(pathway) {
+        this.pathway = pathway;
+    }
+
+    parse(data) {
+        if (this.pathway.parser) {
+            return this.pathway.parser(data);
+        }
+
+        if (this.pathway.list) {
+            if (this.pathway.format) {
+                return parseNumberedObjectList(data, this.pathway.format);
+            }
+            return parseNumberedList(data)
+        }
+
+        return data;
+    }
+}
+
+module.exports = { PathwayResponseParser };