@aj-archipelago/cortex 1.1.1 → 1.1.3

package/.eslintignore

@@ -18,6 +18,12 @@
 # Ignore documentation
 /docs
 
+# Ignore helper apps
+/helper_apps
+
+# Ignore tests
+/tests
+
 # Ignore any generated or bundled files
 *.min.js
 *.bundle.js

package/.eslintrc

@@ -16,8 +16,9 @@
     ],
     "rules": {
         "import/no-unresolved": "error",
-        "import/no-extraneous-dependencies": ["error", {"devDependencies": true, "dependencies": true}],
-        "no-unused-vars": ["error", { "argsIgnorePattern": "^_" }]
+        "import/no-extraneous-dependencies": ["error", {"devDependencies": true}],
+        "no-unused-vars": ["error", { "argsIgnorePattern": "^_" }],
+        "no-useless-escape": "off"
     },
     "settings": {
         "import/resolver": {

package/lib/redisSubscription.js

@@ -46,7 +46,7 @@ if (connectionString) {
             channels.forEach(channel => {
                 subscriptionClient.subscribe(channel, (error) => {
                     if (error) {
-                        logger.error(`Error subscribing to redis channel ${channel}: ${error}`);
+                        logger.error(`Error subscribing to Redis channel ${channel}: ${error}`);
                     } else {
                         logger.info(`Subscribed to channel ${channel}`);
                     }
@@ -55,26 +55,22 @@ if (connectionString) {
         });
 
         subscriptionClient.on('message', (channel, message) => {
-            logger.debug(`Received message from ${channel}: ${message}`);
+            logger.debug(`Received message from Redis channel ${channel}: ${message}`);
             
-            let decryptedMessage;
-
-            if (channel === requestProgressChannel && redisEncryptionKey) {
-                try {
-                    decryptedMessage = decrypt(message, redisEncryptionKey);
-                } catch (error) {
-                    logger.error(`Error decrypting message: ${error}`);
-                }
-            }
-
-            decryptedMessage = decryptedMessage || message;
-
             let parsedMessage;
 
             try {
-                parsedMessage = JSON.parse(decryptedMessage);
+                parsedMessage = JSON.parse(message);
             } catch (error) {
-                logger.error(`Error parsing message: ${error}`);
+                if (channel === requestProgressChannel && redisEncryptionKey) {
+                    try {
+                        parsedMessage = JSON.parse(decrypt(message, redisEncryptionKey));
+                    } catch (error) {
+                        logger.error(`Error parsing or decrypting message: ${error}`);
+                    }
+                } else {
+                    logger.error(`Error parsing message: ${error}`);
+                }
             }
 
             switch(channel) {
@@ -96,7 +92,7 @@ if (connectionString) {
 }
 
 async function publishRequestProgress(data) {
-    if (publisherClient) {
+    if (publisherClient && requestState?.[data?.requestId]?.useRedis) {
         try {
             let message = JSON.stringify(data);
             if (redisEncryptionKey) {
@@ -106,10 +102,10 @@ async function publishRequestProgress(data) {
                     logger.error(`Error encrypting message: ${error}`);
                 }
             }
-            logger.debug(`Publishing message ${message} to channel ${requestProgressChannel}`);
+            logger.debug(`Publishing request progress ${message} to Redis channel ${requestProgressChannel}`);
             await publisherClient.publish(requestProgressChannel, message);
         } catch (error) {
-            logger.error(`Error publishing message: ${error}`);
+            logger.error(`Error publishing request progress to Redis: ${error}`);
         }
     } else {
         pubsubHandleMessage(data);
@@ -119,11 +115,30 @@ async function publishRequestProgress(data) {
 async function publishRequestProgressSubscription(data) {
     if (publisherClient) {
         try {
-            const message = JSON.stringify(data);
-            logger.debug(`Publishing message ${message} to channel ${requestProgressSubscriptionsChannel}`);
-            await publisherClient.publish(requestProgressSubscriptionsChannel, message);
+            const requestIds = data;
+            const idsToForward = [];
+            // If any of these requests belong to this instance, we can just start and handle them locally
+            for (const requestId of requestIds) {
+                if (requestState[requestId]) {
+                    if (!requestState[requestId].started) {
+                        requestState[requestId].started = true;
+                        requestState[requestId].useRedis = false;
+                        logger.info(`Starting local execution for registered async request: ${requestId}`);
+                        const { resolver, args } = requestState[requestId];
+                        resolver(args, false);
+                    }
+                } else {
+                    idsToForward.push(requestId);
+                }
+            }
+
+            if (idsToForward.length > 0) {
+                const message = JSON.stringify(idsToForward);
+                logger.debug(`Sending subscription request(s) to channel ${requestProgressSubscriptionsChannel} for remote execution: ${message}`);
+                await publisherClient.publish(requestProgressSubscriptionsChannel, message);
+            }
         } catch (error) {
-            logger.error(`Error publishing message: ${error}`);
+            logger.error(`Error handling subscription: ${error}`);
         }
     } else {
         handleSubscription(data);
@@ -132,11 +147,11 @@ async function publishRequestProgressSubscription(data) {
 
 function pubsubHandleMessage(data){
     const message = JSON.stringify(data);
-    logger.debug(`Publishing message to pubsub: ${message}`);
+    logger.debug(`Publishing request progress to local subscribers: ${message}`);
     try {
         pubsub.publish('REQUEST_PROGRESS', { requestProgress: data });
     } catch (error) {
-        logger.error(`Error publishing data to pubsub: ${error}`);
+        logger.error(`Error publishing request progress to local subscribers: ${error}`);
     }
 }
 
@@ -145,7 +160,8 @@ function handleSubscription(data){
     for (const requestId of requestIds) {
         if (requestState[requestId] && !requestState[requestId].started) {
             requestState[requestId].started = true;
-            logger.info(`Subscription starting async requestProgress, requestId: ${requestId}`);
+            requestState[requestId].useRedis = true;
+            logger.info(`Starting execution for registered async request: ${requestId}`);
             const { resolver, args } = requestState[requestId];
             resolver(args);
         }

package/lib/request.js

@@ -46,7 +46,6 @@ const buildLimiters = (config) => {
         if (connection) {
             limiterOptions.id = `${cortexId}-${name}-limiter`; // Unique id for each limiter
             limiterOptions.connection = connection;  // Shared Redis connection
-            limiterOptions.clearDatastore = true;    // Clear Redis datastore on startup
         }
 
         limiters[name] = new Bottleneck(limiterOptions);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.1",
+  "version": "1.1.3",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/sys_openai_completion.js

@@ -1,8 +1,6 @@
 // sys_openai_completion.js
 // default handler for openAI completion endpoints when REST endpoints are enabled
 
-import { Prompt } from '../server/prompt.js';
-
 export default {
     prompt: `{{text}}`,
     model: 'oai-gpturbo',

package/server/plugins/azureCognitivePlugin.js

@@ -30,7 +30,7 @@ class AzureCognitivePlugin extends ModelPlugin {
     }
 
     // Set up parameters specific to the Azure Cognitive API
-    async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId,  {headers, requestId, pathway, url}) {
+    async getRequestParameters(text, parameters, prompt, mode, indexName, savedContextId,  {headers, requestId, pathway, _url}) {
         const combinedParameters = { ...this.promptParameters, ...parameters };
         const { modelPromptText } = this.getCompiledPrompt(text, combinedParameters, prompt);
         const { inputVector, calculateInputVector, privateData, filter, docId } = combinedParameters;

package/server/plugins/azureTranslatePlugin.js

@@ -1,5 +1,6 @@
 // AzureTranslatePlugin.js
 import ModelPlugin from './modelPlugin.js';
+import logger from '../../lib/logger.js';
 
 class AzureTranslatePlugin extends ModelPlugin {
     constructor(config, pathway, modelName, model) {

package/server/plugins/cohereGeneratePlugin.js

@@ -8,7 +8,7 @@ class CohereGeneratePlugin extends ModelPlugin {
 
     // Set up parameters specific to the Cohere API
     getRequestParameters(text, parameters, prompt) {
-        const { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+        let { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
 
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();

package/server/plugins/localModelPlugin.js

@@ -2,6 +2,7 @@
 import ModelPlugin from './modelPlugin.js';
 import { execFileSync } from 'child_process';
 import { encode } from 'gpt-3-encoder';
+import logger from '../../lib/logger.js';
 
 class LocalModelPlugin extends ModelPlugin {
     constructor(config, pathway, modelName, model) {

package/server/plugins/openAiCompletionPlugin.js

@@ -1,6 +1,5 @@
 // OpenAICompletionPlugin.js
 
-import { request } from 'https';
 import ModelPlugin from './modelPlugin.js';
 import { encode } from 'gpt-3-encoder';
 import logger from '../../lib/logger.js';

package/server/plugins/palmChatPlugin.js

@@ -75,7 +75,6 @@ class PalmChatPlugin extends ModelPlugin {
     // Set up parameters specific to the PaLM Chat API
     getRequestParameters(text, parameters, prompt) {
         const { modelPromptText, modelPromptMessages, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
-        const { stream } = parameters;
     
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();

package/server/plugins/palmCodeCompletionPlugin.js

@@ -11,7 +11,6 @@ class PalmCodeCompletionPlugin extends PalmCompletionPlugin {
     // Set up parameters specific to the PaLM API Code Completion API
     getRequestParameters(text, parameters, prompt, pathwayResolver) {
         const { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
-        const { stream } = parameters;
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
     

package/server/plugins/palmCompletionPlugin.js

@@ -22,7 +22,7 @@ class PalmCompletionPlugin extends ModelPlugin {
     // Set up parameters specific to the PaLM API Text Completion API
     getRequestParameters(text, parameters, prompt, pathwayResolver) {
         const { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
-        const { stream } = parameters;
+        
         // Define the model's max token length
         const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
     

package/server/rest.js

@@ -1,7 +1,6 @@
 // rest.js
 // Implement the REST endpoints for the pathways
 
-import { json } from 'express';
 import pubsub from './pubsub.js';
 import { requestState } from './requestState.js';
 import { v4 as uuidv4 } from 'uuid';
@@ -168,6 +167,9 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     // Fire the resolver for the async requestProgress
     logger.info(`Rest Endpoint starting async requestProgress, requestId: ${requestId}`);
     const { resolver, args } = requestState[requestId];
+    requestState[requestId].useRedis = false;
+    requestState[requestId].started = true;
+
     resolver(args);
 
     return subscription;
@@ -236,17 +238,18 @@ function buildRestEndpoints(pathways, app, server, config) {
                 ],
             };
 
+            // eslint-disable-next-line no-extra-boolean-cast
             if (Boolean(req.body.stream)) {
                 jsonResponse.id = `cmpl-${resultText}`;
                 jsonResponse.choices[0].finish_reason = null;
                 //jsonResponse.object = "text_completion.chunk";
 
-                const subscription = processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `cmpl-${requestId}`;
                 res.json(jsonResponse);
-            };
+            }
         });
         
         app.post('/v1/chat/completions', async (req, res) => {
@@ -281,6 +284,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 ],
             };
 
+            // eslint-disable-next-line no-extra-boolean-cast
             if (Boolean(req.body.stream)) {
                 jsonResponse.id = `chatcmpl-${resultText}`;
                 jsonResponse.choices[0] = {
@@ -292,7 +296,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 }
                 jsonResponse.object = "chat.completion.chunk";
 
-                const subscription = processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `chatcmpl-${requestId}`;
@@ -330,6 +334,6 @@ function buildRestEndpoints(pathways, app, server, config) {
         });
 
     }
-};
+}
 
 export { buildRestEndpoints };

package/server/subscriptions.js

@@ -1,12 +1,13 @@
 import pubsub from './pubsub.js';
-import logger from '../lib/logger.js';
 import { withFilter } from 'graphql-subscriptions';
 import { publishRequestProgressSubscription } from '../lib/redisSubscription.js';
+import logger from '../lib/logger.js';
 
 const subscriptions = {
     requestProgress: {
         subscribe: withFilter(
             (_, args, __, _info) => {
+                logger.debug(`Client requested subscription for request ids: ${args.requestIds}`);
                 publishRequestProgressSubscription(args.requestIds);
                 return pubsub.asyncIterator(['REQUEST_PROGRESS'])
             },