@aj-archipelago/cortex 1.1.2 → 1.1.4-0

package/server/rest.js

@@ -167,11 +167,10 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     // Fire the resolver for the async requestProgress
     logger.info(`Rest Endpoint starting async requestProgress, requestId: ${requestId}`);
     const { resolver, args } = requestState[requestId];
-    // The false here means never use a Redis subscription channel
-    // to handle these streaming messages. This is because we are
-    // guaranteed in this case that the stream is going to the same
-    // client.
-    resolver(args, false);
+    requestState[requestId].useRedis = false;
+    requestState[requestId].started = true;
+
+    resolver && resolver(args);
 
     return subscription;
   

package/server/subscriptions.js

@@ -1,11 +1,13 @@
 import pubsub from './pubsub.js';
 import { withFilter } from 'graphql-subscriptions';
 import { publishRequestProgressSubscription } from '../lib/redisSubscription.js';
+import logger from '../lib/logger.js';
 
 const subscriptions = {
     requestProgress: {
         subscribe: withFilter(
             (_, args, __, _info) => {
+                logger.debug(`Client requested subscription for request ids: ${args.requestIds}`);
                 publishRequestProgressSubscription(args.requestIds);
                 return pubsub.asyncIterator(['REQUEST_PROGRESS'])
             },

package/tests/config.test.js

@@ -25,7 +25,7 @@ test('config basePathwayPath', (t) => {
 });
 
 test('config PORT', (t) => {
-    const expectedDefault = 4000;
+    const expectedDefault = parseInt(process.env.CORTEX_PORT) || 4000;
     t.is(config.get('PORT'), expectedDefault);
 });
 

package/tests/mocks.js

@@ -6,6 +6,7 @@ export const mockConfig = {
         defaultModelName: 'testModel',
         models: {
           testModel: {
+            name: 'testModel', 
             url: 'https://api.example.com/testModel',
             type: 'OPENAI-COMPLETION',
           },
@@ -40,6 +41,7 @@ export const mockConfig = {
 
   export const mockPathwayResolverString = {
     model: {
+      name: 'testModel',
       url: 'https://api.example.com/testModel',
       type: 'OPENAI-COMPLETION',
     },
@@ -51,6 +53,7 @@ export const mockConfig = {
 
   export const mockPathwayResolverFunction = {
     model: {
+      name: 'testModel',
       url: 'https://api.example.com/testModel',
       type: 'OPENAI-COMPLETION',
     },
@@ -64,6 +67,7 @@ export const mockConfig = {
 
   export const mockPathwayResolverMessages = {
     model: {
+      name: 'testModel',
       url: 'https://api.example.com/testModel',
       type: 'OPENAI-COMPLETION',
     },
@@ -78,3 +82,4 @@ export const mockConfig = {
       }),
   };
 
+  export const mockModelEndpoints = { testModel: { name: 'testModel', url: 'https://api.example.com/testModel', type: 'OPENAI-COMPLETION' }};

package/tests/modelPlugin.test.js

@@ -8,10 +8,10 @@ const DEFAULT_MAX_TOKENS = 4096;
 const DEFAULT_PROMPT_TOKEN_RATIO = 0.5;
 
 // Mock configuration and pathway objects
-const { config, pathway, modelName, model } = mockPathwayResolverString;
+const { config, pathway, model } = mockPathwayResolverString;
 
 test('ModelPlugin constructor', (t) => {
-    const modelPlugin = new ModelPlugin(config, pathway, modelName, model);
+    const modelPlugin = new ModelPlugin(pathway, model);
 
     t.is(modelPlugin.modelName, pathway.model, 'modelName should be set from pathway');
     t.deepEqual(modelPlugin.model, config.get('models')[pathway.model], 'model should be set from config');
@@ -20,7 +20,7 @@ test('ModelPlugin constructor', (t) => {
 });
 
 test.beforeEach((t) => {
-  t.context.modelPlugin = new ModelPlugin(config, pathway, modelName, model);
+  t.context.modelPlugin = new ModelPlugin(pathway, model);
 });
 
 test('getCompiledPrompt - text and parameters', (t) => {
@@ -71,13 +71,6 @@ test('getPromptTokenRatio', (t) => {
     t.is(modelPlugin.getPromptTokenRatio(), DEFAULT_PROMPT_TOKEN_RATIO, 'getPromptTokenRatio should return default prompt token ratio');
 });
 
-test('requestUrl', (t) => {
-    const { modelPlugin } = t.context;
-
-    const expectedUrl = HandleBars.compile(modelPlugin.model.url)({ ...modelPlugin.model, ...config.getEnv(), ...config });
-    t.is(modelPlugin.requestUrl(), expectedUrl, 'requestUrl should return the correct URL');
-});
-
 test('default parseResponse', (t) => {
     const { modelPlugin } = t.context;
     const multipleChoicesResponse = {

package/tests/openAiChatPlugin.test.js

@@ -1,19 +1,20 @@
 import test from 'ava';
 import OpenAIChatPlugin from '../server/plugins/openAiChatPlugin.js';
 import { mockPathwayResolverMessages } from './mocks.js';
+import { config } from '../config.js';
 
-const { config, pathway, modelName, model } = mockPathwayResolverMessages;
+const { pathway, modelName, model } = mockPathwayResolverMessages;
 
 // Test the constructor
 test('constructor', (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
-    t.is(plugin.config, mockPathwayResolverMessages.config);
+    const plugin = new OpenAIChatPlugin(pathway, model);
+    t.is(plugin.config, config);
     t.is(plugin.pathwayPrompt, mockPathwayResolverMessages.pathway.prompt);
 });
 
 // Test the convertPalmToOpenAIMessages function
 test('convertPalmToOpenAIMessages', (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
+    const plugin = new OpenAIChatPlugin(pathway, model);
     const context = 'This is a test context.';
     const examples = [
         {
@@ -37,7 +38,7 @@ test('convertPalmToOpenAIMessages', (t) => {
 
 // Test the getRequestParameters function
 test('getRequestParameters', async (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
+    const plugin = new OpenAIChatPlugin(pathway, model);
     const text = 'Help me';
     const parameters = { name: 'John', age: 30 };
     const prompt = mockPathwayResolverMessages.pathway.prompt;
@@ -59,7 +60,7 @@ test('getRequestParameters', async (t) => {
 
 // Test the execute function
 test('execute', async (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
+    const plugin = new OpenAIChatPlugin(pathway, model);
     const text = 'Help me';
     const parameters = { name: 'John', age: 30 };
     const prompt = mockPathwayResolverMessages.pathway.prompt;
@@ -91,7 +92,7 @@ test('execute', async (t) => {
 
 // Test the parseResponse function
 test('parseResponse', (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
+    const plugin = new OpenAIChatPlugin(pathway, model);
     const data = {
         choices: [
             {
@@ -107,7 +108,7 @@ test('parseResponse', (t) => {
 
 // Test the logRequestData function
 test('logRequestData', (t) => {
-    const plugin = new OpenAIChatPlugin(config, pathway, modelName, model);
+    const plugin = new OpenAIChatPlugin(pathway, model);
     const data = {
         messages: [
             { role: 'user', content: 'User: Help me\nAssistant: Please help John who is 30 years old.' },

package/tests/openai_api.test.js

@@ -5,7 +5,7 @@ import got from 'got';
 import axios from 'axios';
 import serverFactory from '../index.js';
 
-const API_BASE = 'http://localhost:4000/v1';
+const API_BASE = `http://localhost:${process.env.CORTEX_PORT}/v1`;
 
 let testServer;
 
@@ -110,7 +110,7 @@ test('POST SSE: /v1/completions should send a series of events and a [DONE] even
         stream: true,
     };
     
-    const url = 'http://localhost:4000/v1';
+    const url = `http://localhost:${process.env.CORTEX_PORT}/v1`;
     
     const completionsAssertions = (t, messageJson) => {
         t.truthy(messageJson.id);
@@ -133,7 +133,7 @@ test('POST SSE: /v1/chat/completions should send a series of events and a [DONE]
         stream: true,
     };
     
-    const url = 'http://localhost:4000/v1';
+    const url = `http://localhost:${process.env.CORTEX_PORT}/v1`;
     
     const chatCompletionsAssertions = (t, messageJson) => {
         t.truthy(messageJson.id);

package/tests/palmChatPlugin.test.js

@@ -6,7 +6,7 @@ import { mockPathwayResolverMessages } from './mocks.js';
 const { config, pathway, modelName, model } = mockPathwayResolverMessages;
 
 test.beforeEach((t) => {
-  const palmChatPlugin = new PalmChatPlugin(config, pathway, modelName, model);
+  const palmChatPlugin = new PalmChatPlugin(pathway, model);
   t.context = { palmChatPlugin };
 });
 

package/tests/palmCompletionPlugin.test.js

@@ -7,7 +7,7 @@ import { mockPathwayResolverString } from './mocks.js';
 const { config, pathway, modelName, model } = mockPathwayResolverString;
 
 test.beforeEach((t) => {
-  const palmCompletionPlugin = new PalmCompletionPlugin(config, pathway, modelName, model);
+  const palmCompletionPlugin = new PalmCompletionPlugin(pathway, model);
   t.context = { palmCompletionPlugin };
 });
 

package/tests/pathwayResolver.test.js

@@ -1,7 +1,7 @@
 import test from 'ava';
 import { PathwayResolver } from '../server/pathwayResolver.js';
 import sinon from 'sinon';
-import { mockConfig, mockPathwayString } from './mocks.js';
+import { mockConfig, mockPathwayString, mockModelEndpoints } from './mocks.js';
 
 const mockPathway = mockPathwayString;
 mockPathway.useInputChunking = false;
@@ -16,6 +16,7 @@ test.beforeEach((t) => {
     config: mockConfig,
     pathway: mockPathway,
     args: mockArgs,
+    endpoints: mockModelEndpoints,
   });
 });
 

package/tests/requestMonitor.test.js

@@ -0,0 +1,94 @@
+import test from 'ava';
+import RequestMonitor from '../lib/requestMonitor.js'; // replace with actual path
+
+test('RequestMonitor: startCall', t => {
+  const rm = new RequestMonitor();
+
+  const callId = rm.startCall();
+
+  t.is(rm.callStartTimes.has(callId), true);
+});
+
+test('RequestMonitor: endCall', t => {
+  const rm = new RequestMonitor();
+
+  const callId = rm.startCall();
+  rm.endCall(callId);
+
+  t.is(rm.callStartTimes.has(callId), false);
+  t.is(rm.callCount.size(), 1);
+});
+
+test('RequestMonitor: getAverageCallDuration', async t => {
+  const rm = new RequestMonitor();
+
+  const callId1 = rm.startCall();
+  await new Promise(resolve => setTimeout(resolve, 1000));
+  rm.endCall(callId1);
+
+  const callId2 = rm.startCall();
+  await new Promise(resolve => setTimeout(resolve, 2000));
+  rm.endCall(callId2);
+
+  const average = rm.getAverageCallDuration();
+  t.truthy(average > 1400 && average < 1600);
+});
+
+test('RequestMonitor: incrementError429Count', t => {
+  const rm = new RequestMonitor();
+
+  rm.incrementError429Count();
+
+  t.is(rm.error429Count.size(), 1);
+});
+
+test('RequestMonitor: getCallRate', async t => {
+  const rm = new RequestMonitor();
+
+  rm.startCall();
+  rm.endCall();
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  const callRate = rm.getCallRate();
+  t.truthy(callRate > 0.9 && callRate < 1.1);
+});
+
+test('RequestMonitor: getPeakCallRate', async t => {
+  const rm = new RequestMonitor();
+
+  rm.startCall();
+  rm.endCall();
+
+  await new Promise(resolve => setTimeout(resolve, 1000));
+
+  rm.startCall();
+  rm.endCall();
+
+  const peakCallRate = rm.getPeakCallRate();
+  t.truthy(peakCallRate > 1.9 && peakCallRate < 2.1);
+});
+
+test('RequestMonitor: getError429Rate', t => {
+  const rm = new RequestMonitor();
+
+  rm.startCall();
+  rm.endCall();
+  rm.incrementError429Count();
+
+  t.is(rm.getError429Rate(), 1);
+});
+
+test('RequestMonitor: reset', t => {
+  const rm = new RequestMonitor();
+
+  rm.startCall();
+  rm.endCall();
+  rm.incrementError429Count();
+
+  rm.reset();
+
+  t.is(rm.callCount.size(), 0);
+  t.is(rm.error429Count.size(), 0);
+  t.is(rm.peakCallRate, 0);
+});

package/tests/{requestDurationEstimator.test.js → requestMonitorDurationEstimator.test.js}

@@ -1,14 +1,14 @@
 import test from 'ava';
-import RequestDurationEstimator from '../lib/requestDurationEstimator.js';
+import RequestMonitor from '../lib/requestMonitor.js';
 
 test('add and get average request duration', async (t) => {
-    const estimator = new RequestDurationEstimator(5);
+    const estimator = new RequestMonitor(5);
 
-    estimator.startRequest('req1');
+    const callid = estimator.startCall();
     await new Promise(resolve => setTimeout(() => {
-        estimator.endRequest();
+        estimator.endCall(callid);
 
-        const average = estimator.calculatePercentComplete();
+        const average = estimator.calculatePercentComplete(callid);
 
         // An average should be calculated after the first completed request  
         t.not(average, 0);
@@ -17,31 +17,31 @@ test('add and get average request duration', async (t) => {
 });
 
 test('add more requests than size of durations array', (t) => {
-    const estimator = new RequestDurationEstimator(5);
+    const estimator = new RequestMonitor(5);
 
     for (let i = 0; i < 10; i++) {
-        estimator.startRequest(`req${i}`);
-        estimator.endRequest();
+        const callid = estimator.startCall();
+        estimator.endCall(callid);
     }
 
     // Array size should not exceed maximum length (5 in this case)
-    t.is(estimator.durations.length, 5);
+    t.is(estimator.callDurations.size(), 5);
 });
 
 test('calculate percent complete of current request based on average of past durations', async (t) => {
-    const estimator = new RequestDurationEstimator(5);
+    const estimator = new RequestMonitor(5);
 
     for (let i = 0; i < 4; i++) {
-        estimator.startRequest(`req${i}`);
+        const callid = estimator.startCall();
         // wait 1 second
         await new Promise(resolve => setTimeout(resolve, 1000));
-        estimator.endRequest();
+        estimator.endCall(callid);
     }
 
-    estimator.startRequest('req5');
+    const callid = estimator.startCall();
 
     await new Promise(resolve => setTimeout(() => {
-        const percentComplete = estimator.calculatePercentComplete();
+        const percentComplete = estimator.calculatePercentComplete(callid);
 
         // Depending on how fast the operations are,
         // the percentage may not be exactly 50%, but
@@ -52,8 +52,12 @@ test('calculate percent complete of current request based on average of past dur
 });
 
 test('calculate percent complete based on average of past durations', async (t) => {
-    const estimator = new RequestDurationEstimator(5);
-    estimator.durations = [1000, 2000, 3000];
-    const average = estimator.getAverage();
+    const estimator = new RequestMonitor(5);
+    estimator.callDurations.clear;
+    estimator.callDurations.pushBack({endTime: new Date(), callDuration: 1000});
+    estimator.callDurations.pushBack({endTime: new Date(), callDuration: 2000});
+    estimator.callDurations.pushBack({endTime: new Date(), callDuration: 3000});
+
+    const average = estimator.getAverageCallDuration();
     t.is(average, 2000);
 });

package/tests/truncateMessages.test.js

@@ -6,7 +6,7 @@ import { mockPathwayResolverString } from './mocks.js';
 
 const { config, pathway, modelName, model } = mockPathwayResolverString;
 
-const modelPlugin = new ModelPlugin(config, pathway, modelName, model);
+const modelPlugin = new ModelPlugin(pathway, model);
 
 const generateMessage = (role, content) => ({ role, content });