@aj-archipelago/cortex 1.0.4 → 1.0.6

package/{graphql → server}/prompt.js

@@ -3,15 +3,21 @@ class Prompt {
         if (typeof params === 'string' || params instanceof String) {
             this.prompt = params;
         } else {
-            const { prompt, saveResultTo, messages } = params;
+            const { prompt, saveResultTo, messages, context, examples } = params;
             this.prompt = prompt;
             this.saveResultTo = saveResultTo;
             this.messages = messages;
+            this.context = context;
+            this.examples = examples;
             this.params = params;
         }
 
-        this.usesTextInput = promptContains('text', this.prompt ? this.prompt : this.messages);
-        this.usesPreviousResult = promptContains('previousResult', this.prompt ? this.prompt : this.messages);
+        this.usesTextInput = promptContains('text', this.prompt ? this.prompt : this.messages) ||
+                             (this.context && promptContains('text', this.context)) ||
+                             (this.examples && promptContains('text', this.examples));
+        this.usesPreviousResult = promptContains('previousResult', this.prompt ? this.prompt : this.messages) ||
+                                   (this.context && promptContains('previousResult', this.context)) ||
+                                   (this.examples && promptContains('previousResult', this.examples));
         this.debugInfo = '';
     }
 }
@@ -23,7 +29,8 @@ function promptContains(variable, prompt) {
     let matches = [];
     let match;
 
-    // if it's an array, it's the messages format
+    // if it's an array, it's either an OpenAI messages array or a PaLM messages
+    // array or a PaLM examples array, all of which have a content property
     if (Array.isArray(prompt)) {
       prompt.forEach(p => {
         // eslint-disable-next-line no-cond-assign

package/server/rest.js

@@ -0,0 +1,321 @@
+// rest.js
+// Implement the REST endpoints for the pathways
+
+import { json } from 'express';
+import pubsub from './pubsub.js';
+import { requestState } from './requestState.js';
+import { v4 as uuidv4 } from 'uuid';
+
+
+const processRestRequest = async (server, req, pathway, name, parameterMap = {}) => {
+    const fieldVariableDefs = pathway.typeDef(pathway).restDefinition || [];
+
+    const convertType = (value, type) => {
+        if (type === 'Boolean') {
+            return Boolean(value);
+        } else if (type === 'Int') {
+            return parseInt(value, 10);
+        } else {
+            return value;
+        }
+    };
+
+    const variables = fieldVariableDefs.reduce((acc, variableDef) => {
+        const requestBodyParamName = Object.keys(parameterMap).includes(variableDef.name)
+            ? parameterMap[variableDef.name]
+            : variableDef.name;
+
+        if (Object.prototype.hasOwnProperty.call(req.body, requestBodyParamName)) {
+            acc[variableDef.name] = convertType(req.body[requestBodyParamName], variableDef.type);
+        }
+        return acc;
+    }, {});
+
+    const variableParams = fieldVariableDefs.map(({ name, type }) => `$${name}: ${type}`).join(', ');
+    const queryArgs = fieldVariableDefs.map(({ name }) => `${name}: $${name}`).join(', ');
+
+    const query = `
+            query ${name}(${variableParams}) {
+                    ${name}(${queryArgs}) {
+                        contextId
+                        previousResult
+                        result
+                    }
+                }
+            `;
+
+    const result = await server.executeOperation({ query, variables });
+    const resultText = result?.body?.singleResult?.data?.[name]?.result || result?.body?.singleResult?.errors?.[0]?.message || "";
+
+    return resultText;
+};
+
+const processIncomingStream = (requestId, res, jsonResponse) => {
+
+    const startStream = (res) => {
+        // Set the headers for streaming
+        res.setHeader('Content-Type', 'text/event-stream');
+        res.setHeader('Cache-Control', 'no-cache');
+        res.setHeader('Connection', 'keep-alive');
+        res.flushHeaders();
+    }
+    
+    const finishStream = (res, jsonResponse) => {
+    
+        // If we haven't sent the stop message yet, do it now
+        if (jsonResponse.choices?.[0]?.finish_reason !== "stop") {
+    
+            let jsonEndStream = JSON.parse(JSON.stringify(jsonResponse));
+    
+            if (jsonEndStream.object === 'text_completion') {
+                jsonEndStream.choices[0].index = 0;
+                jsonEndStream.choices[0].finish_reason = "stop";
+                jsonEndStream.choices[0].text = "";
+            } else {
+                jsonEndStream.choices[0].finish_reason = "stop";
+                jsonEndStream.choices[0].index = 0;
+                jsonEndStream.choices[0].delta = {};
+            }
+    
+            //console.log(`REST SEND: data: ${JSON.stringify(jsonEndStream)}`);
+            res.write(`data: ${JSON.stringify(jsonEndStream)}\n\n`);
+        }
+    
+        //console.log(`REST SEND: data: [DONE]\n\n`);
+        res.write(`data: [DONE]\n\n`);
+    }
+
+    const sendStreamData = (data) => {
+        //console.log(`REST SEND: data: ${JSON.stringify(data)}`);
+        res.write(`data: ${JSON.stringify(data)}\n\n`);
+    }
+
+    const fillJsonResponse = (jsonResponse, inputText, finishReason) => {
+
+        jsonResponse.choices[0].finish_reason = finishReason;
+        if (jsonResponse.object === 'text_completion') {
+            jsonResponse.choices[0].text = inputText;
+        } else {
+            jsonResponse.choices[0].delta.content = inputText;
+        }
+
+        return jsonResponse;
+    }
+
+    startStream(res);
+
+    let subscription;
+
+    const unsubscribe = async () => {
+        if (subscription) {
+            pubsub.unsubscribe(await subscription);
+        }
+    }
+
+    subscription = pubsub.subscribe('REQUEST_PROGRESS', (data) => {
+        if (data.requestProgress.requestId === requestId) {
+            //console.log(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
+            
+            const progress = data.requestProgress.progress;
+            const progressData = data.requestProgress.data;
+
+            try {
+                const messageJson = JSON.parse(progressData);
+                if (messageJson.choices) {
+                    const { text, delta, finish_reason } = messageJson.choices[0];
+
+                    if (messageJson.object === 'text_completion') {
+                        fillJsonResponse(jsonResponse, text, finish_reason);
+                    } else {
+                        fillJsonResponse(jsonResponse, delta.content, finish_reason);
+                    }
+                } else {
+                    fillJsonResponse(jsonResponse, messageJson, null);
+                }
+            } catch (error) {
+                console.log(`progressData not JSON: ${progressData}`);
+                fillJsonResponse(jsonResponse, progressData, "stop");
+            }
+
+            if (progress === 1 && progressData.trim() === "[DONE]") {
+                finishStream(res, jsonResponse);
+                unsubscribe();
+                return;
+            }
+            sendStreamData(jsonResponse);
+
+            if (progress === 1) {
+                finishStream(res, jsonResponse);
+                unsubscribe();
+            }
+        }
+    });
+
+    // Fire the resolver for the async requestProgress
+    console.log(`Rest Endpoint starting async requestProgress, requestId: ${requestId}`);
+    const { resolver, args } = requestState[requestId];
+    resolver(args);
+
+    return subscription;
+  
+}
+
+function buildRestEndpoints(pathways, app, server, config) {
+
+    if (config.get('enableRestEndpoints')) {
+        const openAIChatModels = {};
+        const openAICompletionModels = {};
+
+        // Create normal REST endpoints or emulate OpenAI API per pathway
+        for (const [name, pathway] of Object.entries(pathways)) {
+            // Only expose endpoints for enabled pathways that explicitly want to expose a REST endpoint
+            if (pathway.disabled) continue;
+
+            // The pathway can either emulate an OpenAI endpoint or be a normal REST endpoint
+            if (pathway.emulateOpenAIChatModel || pathway.emulateOpenAICompletionModel) {
+                if (pathway.emulateOpenAIChatModel) {
+                    openAIChatModels[pathway.emulateOpenAIChatModel] = name;
+                }
+                if (pathway.emulateOpenAICompletionModel) {
+                        openAICompletionModels[pathway.emulateOpenAICompletionModel] = name;
+                }
+            } else {
+                app.post(`/rest/${name}`, async (req, res) => {
+                    const resultText = await processRestRequest(server, req, pathway, name);
+                    res.send(resultText);
+                });
+            }
+        }
+
+        // Create OpenAI compatible endpoints
+        app.post('/v1/completions', async (req, res) => {
+            const modelName = req.body.model || 'gpt-3.5-turbo';
+            const pathwayName = openAICompletionModels[modelName] || openAICompletionModels['*'];
+
+            if (!pathwayName) {
+                res.status(404).json({
+                    error: `Model ${modelName} not found.`,
+                });
+                return;
+            }
+
+            const pathway = pathways[pathwayName];
+
+            const parameterMap = {
+                text: 'prompt'
+            };
+
+            const resultText = await processRestRequest(server, req, pathway, pathwayName, parameterMap);
+
+            const jsonResponse = {
+                id: `cmpl`,
+                object: "text_completion",
+                created: Date.now(),
+                model: req.body.model,
+                choices: [
+                {
+                    text: resultText,
+                    index: 0,
+                    logprobs: null,
+                    finish_reason: "stop"
+                }
+                ],
+            };
+
+            if (Boolean(req.body.stream)) {
+                jsonResponse.id = `cmpl-${resultText}`;
+                jsonResponse.choices[0].finish_reason = null;
+                //jsonResponse.object = "text_completion.chunk";
+
+                const subscription = processIncomingStream(resultText, res, jsonResponse);
+            } else {
+                const requestId = uuidv4();
+                jsonResponse.id = `cmpl-${requestId}`;
+                res.json(jsonResponse);
+            };
+        });
+        
+        app.post('/v1/chat/completions', async (req, res) => {
+            const modelName = req.body.model || 'gpt-3.5-turbo';
+            const pathwayName = openAIChatModels[modelName] || openAIChatModels['*'];
+
+            if (!pathwayName) {
+                res.status(404).json({
+                    error: `Model ${modelName} not found.`,
+                });
+                return;
+            }
+
+            const pathway = pathways[pathwayName];
+
+            const resultText = await processRestRequest(server, req, pathway, pathwayName);
+
+            const jsonResponse = {
+                id: `chatcmpl`,
+                object: "chat.completion",
+                created: Date.now(),
+                model: req.body.model,
+                choices: [
+                    {
+                        message: {
+                            role: "assistant",
+                            content: resultText
+                        },
+                        index: 0,
+                        finish_reason: "stop"
+                    }
+                ],
+            };
+
+            if (Boolean(req.body.stream)) {
+                jsonResponse.id = `chatcmpl-${resultText}`;
+                jsonResponse.choices[0] = {
+                    delta: {
+                        role: "assistant",
+                        content: resultText
+                    },
+                    finish_reason: null
+                }
+                jsonResponse.object = "chat.completion.chunk";
+
+                const subscription = processIncomingStream(resultText, res, jsonResponse);
+            } else {
+                const requestId = uuidv4();
+                jsonResponse.id = `chatcmpl-${requestId}`;
+
+                res.json(jsonResponse);
+            }
+
+        });
+
+        app.get('/v1/models', async (req, res) => {
+            const openAIModels = { ...openAIChatModels, ...openAICompletionModels };
+            const defaultModelId = 'gpt-3.5-turbo';
+
+            const models = Object.entries(openAIModels)
+                .map(([modelId]) => {
+                    if (modelId.includes('*')) {
+                        modelId = defaultModelId;
+                    }
+                    return {
+                        id: modelId,
+                        object: 'model',
+                        owned_by: 'openai',
+                        permission: '',
+                    };
+                })
+                .filter((model, index, self) => {
+                    return index === self.findIndex((m) => m.id === model.id);
+                })
+                .sort((a, b) => a.id.localeCompare(b.id));
+
+            res.json({
+                data: models,
+                object: 'list',
+            });
+        });
+
+    }
+};
+
+export { buildRestEndpoints };

package/{graphql → server}/typeDef.js

@@ -1,8 +1,30 @@
-const GRAPHQL_TYPE_MAP = {
-    boolean: 'Boolean',
-    string: 'String',
-    number: 'Int',
-  };
+const getGraphQlType = (value) => {
+  switch (typeof value) {
+    case 'boolean':
+      return {type: 'Boolean', defaultValue: 'false'};
+      break;
+    case 'string':
+      return {type: 'String', defaultValue: `""`};
+      break;
+    case 'number':
+      return {type: 'Int', defaultValue: '0'};
+      break;
+    case 'object':
+      if (Array.isArray(value)) {
+        if (value.length > 0 && typeof(value[0]) === 'string') {
+          return {type: '[String]', defaultValue: '[]'};
+        }
+        else {
+          return {type: '[Message]', defaultValue: '[]'};
+        }
+      } else {
+        return {type: `[${value.objName}]`, defaultValue: 'null'};
+      }
+      break;
+    default:
+      return {type: 'String', defaultValue: `""`};
+  }
+};
 
 const typeDef = (pathway) => {
   const { name, objName, defaultInputParameters, inputParameters, format } = pathway;
@@ -31,20 +53,15 @@ const typeDef = (pathway) => {
 
   const paramsStr = Object.entries(params)
     .map(([key, value]) => {
-      if (typeof value === 'object' && Array.isArray(value)) {
-        return `${key}: [Message] = []`;
-      } else {
-        return `${key}: ${GRAPHQL_TYPE_MAP[typeof value]} = ${
-          typeof value === 'string' ? `"${value}"` : value
-        }`;
-      }
+      const { type, defaultValue } = getGraphQlType(value);
+      return `${key}: ${type} = ${defaultValue}`;
     })
     .join('\n');
 
   const restDefinition = Object.entries(params).map(([key, value]) => {
     return {
       name: key,
-      type: `${GRAPHQL_TYPE_MAP[typeof value]}${typeof value === 'object' && Array.isArray(value) ? '[]' : ''}`,
+      type: `${getGraphQlType(value).type}`,
     };
   });
 

package/tests/chunkfunction.test.js

@@ -1,5 +1,5 @@
 import test from 'ava';
-import { getSemanticChunks } from '../graphql/chunker.js';
+import { getSemanticChunks } from '../server/chunker.js';
 import { encode } from 'gpt-3-encoder';
 
 const testText = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.

package/tests/config.test.js

@@ -45,7 +45,7 @@ test('config enableRestEndpoints', (t) => {
 });
 
 test('config openaiDefaultModel', (t) => {
-    const expectedDefault = 'text-davinci-003';
+    const expectedDefault = 'gpt-3.5-turbo';
     t.is(config.get('openaiDefaultModel'), expectedDefault);
 });