@aj-archipelago/cortex 1.0.25 → 1.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc +1 -1
- package/config.js +23 -7
- package/lib/crypto.js +46 -0
- package/lib/keyValueStorageClient.js +19 -2
- package/lib/logger.js +48 -0
- package/lib/redisSubscription.js +123 -32
- package/lib/request.js +23 -22
- package/package.json +3 -2
- package/server/graphql.js +5 -4
- package/server/parser.js +3 -1
- package/server/pathwayResolver.js +15 -22
- package/server/plugins/azureCognitivePlugin.js +5 -4
- package/server/plugins/azureTranslatePlugin.js +2 -2
- package/server/plugins/localModelPlugin.js +3 -3
- package/server/plugins/modelPlugin.js +15 -6
- package/server/plugins/openAiChatPlugin.js +16 -4
- package/server/plugins/openAiCompletionPlugin.js +9 -3
- package/server/plugins/openAiDallE3Plugin.js +6 -7
- package/server/plugins/openAiImagePlugin.js +5 -8
- package/server/plugins/openAiWhisperPlugin.js +17 -17
- package/server/plugins/palmChatPlugin.js +14 -6
- package/server/plugins/palmCompletionPlugin.js +10 -3
- package/server/pubsub.js +1 -0
- package/server/rest.js +7 -6
- package/server/subscriptions.js +3 -15
- package/tests/palmChatPlugin.test.js +0 -38
- package/tests/palmCompletionPlugin.test.js +0 -30
package/.eslintrc
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
],
|
|
17
17
|
"rules": {
|
|
18
18
|
"import/no-unresolved": "error",
|
|
19
|
-
"import/no-extraneous-dependencies": ["error", {"devDependencies": true}],
|
|
19
|
+
"import/no-extraneous-dependencies": ["error", {"devDependencies": true, "dependencies": true}],
|
|
20
20
|
"no-unused-vars": ["error", { "argsIgnorePattern": "^_" }]
|
|
21
21
|
},
|
|
22
22
|
"settings": {
|
package/config.js
CHANGED
|
@@ -4,11 +4,17 @@ import HandleBars from './lib/handleBars.js';
|
|
|
4
4
|
import fs from 'fs';
|
|
5
5
|
import { fileURLToPath, pathToFileURL } from 'url';
|
|
6
6
|
import GcpAuthTokenHelper from './lib/gcpAuthTokenHelper.js';
|
|
7
|
+
import logger from './lib/logger.js';
|
|
7
8
|
|
|
8
9
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
9
10
|
|
|
10
11
|
// Schema for config
|
|
11
12
|
var config = convict({
|
|
13
|
+
env: {
|
|
14
|
+
format: String,
|
|
15
|
+
default: 'development',
|
|
16
|
+
env: 'NODE_ENV'
|
|
17
|
+
},
|
|
12
18
|
cortexId: {
|
|
13
19
|
format: String,
|
|
14
20
|
default: 'local',
|
|
@@ -168,6 +174,12 @@ var config = convict({
|
|
|
168
174
|
sensitive: true,
|
|
169
175
|
env: 'STORAGE_CONNECTION_STRING'
|
|
170
176
|
},
|
|
177
|
+
redisEncryptionKey: {
|
|
178
|
+
format: String,
|
|
179
|
+
default: null,
|
|
180
|
+
env: 'REDIS_ENCRYPTION_KEY',
|
|
181
|
+
sensitive: true
|
|
182
|
+
},
|
|
171
183
|
dalleImageApiUrl: {
|
|
172
184
|
format: String,
|
|
173
185
|
default: 'null',
|
|
@@ -195,14 +207,16 @@ const configFile = config.get('cortexConfigFile');
|
|
|
195
207
|
|
|
196
208
|
// Load config file
|
|
197
209
|
if (configFile && fs.existsSync(configFile)) {
|
|
198
|
-
|
|
210
|
+
logger.info(`Loading config from ${configFile}`);
|
|
199
211
|
config.loadFile(configFile);
|
|
200
212
|
} else {
|
|
201
213
|
const openaiApiKey = config.get('openaiApiKey');
|
|
202
214
|
if (!openaiApiKey) {
|
|
203
|
-
|
|
215
|
+
const errorString = 'No config file or api key specified. Please set the OPENAI_API_KEY to use OAI or use CORTEX_CONFIG_FILE environment variable to point at the Cortex configuration for your project.';
|
|
216
|
+
logger.error(errorString);
|
|
217
|
+
throw new Error(errorString);
|
|
204
218
|
} else {
|
|
205
|
-
|
|
219
|
+
logger.info(`Using default model with OPENAI_API_KEY environment variable`)
|
|
206
220
|
}
|
|
207
221
|
}
|
|
208
222
|
|
|
@@ -223,12 +237,12 @@ const buildPathways = async (config) => {
|
|
|
223
237
|
const basePathway = await import(basePathwayURL).then(module => module.default);
|
|
224
238
|
|
|
225
239
|
// Load core pathways, default from the Cortex package
|
|
226
|
-
|
|
240
|
+
logger.info(`Loading core pathways from ${corePathwaysPath}`)
|
|
227
241
|
let loadedPathways = await import(`${corePathwaysURL}/index.js`).then(module => module);
|
|
228
242
|
|
|
229
243
|
// Load custom pathways and override core pathways if same
|
|
230
244
|
if (pathwaysPath && fs.existsSync(pathwaysPath)) {
|
|
231
|
-
|
|
245
|
+
logger.info(`Loading custom pathways from ${pathwaysPath}`)
|
|
232
246
|
const customPathways = await import(`${pathwaysURL}/index.js`).then(module => module);
|
|
233
247
|
loadedPathways = { ...loadedPathways, ...customPathways };
|
|
234
248
|
}
|
|
@@ -263,12 +277,14 @@ const buildModels = (config) => {
|
|
|
263
277
|
|
|
264
278
|
// Check that models are specified, Cortex cannot run without a model
|
|
265
279
|
if (Object.keys(config.get('models')).length <= 0) {
|
|
266
|
-
|
|
280
|
+
const errorString = 'No models specified! Please set the models in your config file or via CORTEX_MODELS environment variable to point at the models for your project.';
|
|
281
|
+
logger.error(errorString);
|
|
282
|
+
throw new Error(errorString);
|
|
267
283
|
}
|
|
268
284
|
|
|
269
285
|
// Set default model name to the first model in the config in case no default is specified
|
|
270
286
|
if (!config.get('defaultModelName')) {
|
|
271
|
-
|
|
287
|
+
logger.warn('No default model specified, using first model as default.');
|
|
272
288
|
config.load({ defaultModelName: Object.keys(config.get('models'))[0] });
|
|
273
289
|
}
|
|
274
290
|
|
package/lib/crypto.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// This file is used to encrypt and decrypt data using the crypto library
|
|
2
|
+
import logger from './logger.js';
|
|
3
|
+
import crypto from 'crypto';
|
|
4
|
+
|
|
5
|
+
// Encryption function
|
|
6
|
+
function encrypt(text, key) {
|
|
7
|
+
if (!key) { return text; }
|
|
8
|
+
try {
|
|
9
|
+
key = tryBufferKey(key);
|
|
10
|
+
let iv = crypto.randomBytes(16);
|
|
11
|
+
let cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
|
|
12
|
+
let encrypted = cipher.update(text, 'utf8', 'hex');
|
|
13
|
+
encrypted += cipher.final('hex');
|
|
14
|
+
return iv.toString('hex') + ':' + encrypted;
|
|
15
|
+
} catch (error) {
|
|
16
|
+
logger.error(`Encryption failed: ${error.message}`);
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Decryption function
|
|
22
|
+
function decrypt(message, key) {
|
|
23
|
+
if (!key) { return message; }
|
|
24
|
+
try {
|
|
25
|
+
key = tryBufferKey(key);
|
|
26
|
+
let parts = message.split(':');
|
|
27
|
+
let iv = Buffer.from(parts.shift(), 'hex');
|
|
28
|
+
let encrypted = parts.join(':');
|
|
29
|
+
let decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
|
|
30
|
+
let decrypted = decipher.update(encrypted, 'hex', 'utf8');
|
|
31
|
+
decrypted += decipher.final('utf8');
|
|
32
|
+
return decrypted;
|
|
33
|
+
} catch (error) {
|
|
34
|
+
logger.error(`Decryption failed: ${error.message}`);
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function tryBufferKey(key) {
|
|
40
|
+
if (key.length === 64) {
|
|
41
|
+
return Buffer.from(key, 'hex');
|
|
42
|
+
}
|
|
43
|
+
return key;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export { encrypt, decrypt };
|
|
@@ -1,15 +1,32 @@
|
|
|
1
1
|
import Keyv from 'keyv';
|
|
2
2
|
import { config } from '../config.js';
|
|
3
|
+
import { encrypt, decrypt } from './crypto.js';
|
|
4
|
+
import logger from './logger.js';
|
|
3
5
|
|
|
4
6
|
const storageConnectionString = config.get('storageConnectionString');
|
|
5
7
|
const cortexId = config.get('cortexId');
|
|
8
|
+
const redisEncryptionKey = config.get('redisEncryptionKey');
|
|
6
9
|
|
|
7
10
|
// Create a keyv client to store data
|
|
8
11
|
const keyValueStorageClient = new Keyv(storageConnectionString, {
|
|
9
12
|
ssl: true,
|
|
10
13
|
abortConnect: false,
|
|
11
|
-
serialize: JSON.stringify,
|
|
12
|
-
deserialize:
|
|
14
|
+
serialize: (data) => redisEncryptionKey ? encrypt(JSON.stringify(data), redisEncryptionKey) : JSON.stringify(data),
|
|
15
|
+
deserialize: (data) => {
|
|
16
|
+
try {
|
|
17
|
+
// Try to parse the data normally
|
|
18
|
+
return JSON.parse(data);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
// If it fails, the data may be encrypted so attempt to decrypt it if we have a key
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(decrypt(data, redisEncryptionKey));
|
|
23
|
+
} catch (decryptError) {
|
|
24
|
+
// If decryption also fails, log an error and return an empty object
|
|
25
|
+
logger.error(`Failed to parse or decrypt stored key value data: ${decryptError}`);
|
|
26
|
+
return {};
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
},
|
|
13
30
|
namespace: `${cortexId}-cortex-context`
|
|
14
31
|
});
|
|
15
32
|
|
package/lib/logger.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// logger.js
|
|
2
|
+
import winston from 'winston';
|
|
3
|
+
|
|
4
|
+
winston.addColors({
|
|
5
|
+
debug: 'green',
|
|
6
|
+
verbose: 'blue',
|
|
7
|
+
http: 'gray',
|
|
8
|
+
info: 'cyan',
|
|
9
|
+
warn: 'yellow',
|
|
10
|
+
error: 'red'
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
const debugFormat = winston.format.combine(
|
|
14
|
+
winston.format.colorize({ all: true }),
|
|
15
|
+
winston.format.cli()
|
|
16
|
+
);
|
|
17
|
+
|
|
18
|
+
const prodFormat = winston.format.combine(
|
|
19
|
+
winston.format.simple()
|
|
20
|
+
);
|
|
21
|
+
|
|
22
|
+
const transports = process.env.NODE_ENV === 'production' ?
|
|
23
|
+
new winston.transports.Console({ level: 'info', format: prodFormat }) :
|
|
24
|
+
new winston.transports.Console({ level: 'debug', format: debugFormat });
|
|
25
|
+
|
|
26
|
+
const logger = winston.createLogger({ transports });
|
|
27
|
+
|
|
28
|
+
// Function to obscure sensitive URL parameters
|
|
29
|
+
export const obscureUrlParams = url => {
|
|
30
|
+
try {
|
|
31
|
+
const urlObject = new URL(url);
|
|
32
|
+
urlObject.searchParams.forEach((value, name) => {
|
|
33
|
+
if (/token|key|password|secret|auth|apikey|access|passwd|credential/i.test(name)) {
|
|
34
|
+
urlObject.searchParams.set(name, '******');
|
|
35
|
+
}
|
|
36
|
+
});
|
|
37
|
+
return urlObject.toString();
|
|
38
|
+
} catch (e) {
|
|
39
|
+
if (e instanceof TypeError) {
|
|
40
|
+
logger.error('Error obscuring URL parameters - invalid URL.');
|
|
41
|
+
return url;
|
|
42
|
+
} else {
|
|
43
|
+
throw e;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
export default logger;
|
package/lib/redisSubscription.js
CHANGED
|
@@ -1,63 +1,154 @@
|
|
|
1
1
|
import Redis from 'ioredis';
|
|
2
2
|
import { config } from '../config.js';
|
|
3
3
|
import pubsub from '../server/pubsub.js';
|
|
4
|
+
import { requestState } from '../server/requestState.js';
|
|
5
|
+
import logger from '../lib/logger.js';
|
|
6
|
+
import { encrypt, decrypt } from '../lib/crypto.js';
|
|
4
7
|
|
|
5
8
|
const connectionString = config.get('storageConnectionString');
|
|
6
|
-
const
|
|
7
|
-
|
|
9
|
+
const redisEncryptionKey = config.get('redisEncryptionKey');
|
|
10
|
+
const requestProgressChannel = 'requestProgress';
|
|
11
|
+
const requestProgressSubscriptionsChannel = 'requestProgressSubscriptions';
|
|
12
|
+
|
|
13
|
+
let subscriptionClient;
|
|
14
|
+
let publisherClient;
|
|
8
15
|
|
|
9
16
|
if (connectionString) {
|
|
10
|
-
|
|
17
|
+
logger.info(`Using Redis subscription for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
|
|
18
|
+
try {
|
|
19
|
+
subscriptionClient = connectionString && new Redis(connectionString);
|
|
20
|
+
} catch (error) {
|
|
21
|
+
logger.error(`Redis connection error: ${error}`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
logger.info(`Using Redis publish for channel(s) ${requestProgressChannel}, ${requestProgressSubscriptionsChannel}`);
|
|
11
25
|
try {
|
|
12
|
-
|
|
26
|
+
publisherClient = connectionString && new Redis(connectionString);
|
|
13
27
|
} catch (error) {
|
|
14
|
-
|
|
28
|
+
logger.error(`Redis connection error: ${error}`);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (redisEncryptionKey) {
|
|
32
|
+
logger.info('Using encryption for Redis');
|
|
33
|
+
} else {
|
|
34
|
+
logger.warn('REDIS_ENCRYPTION_KEY not set. Data stored in Redis will not be encrypted.');
|
|
15
35
|
}
|
|
16
36
|
|
|
17
|
-
if (
|
|
18
|
-
const channel = 'requestProgress';
|
|
37
|
+
if (subscriptionClient) {
|
|
19
38
|
|
|
20
|
-
|
|
21
|
-
|
|
39
|
+
subscriptionClient.on('error', (error) => {
|
|
40
|
+
logger.error(`Redis subscriptionClient error: ${error}`);
|
|
22
41
|
});
|
|
23
42
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
43
|
+
subscriptionClient.on('connect', () => {
|
|
44
|
+
const channels = [requestProgressChannel, requestProgressSubscriptionsChannel];
|
|
45
|
+
|
|
46
|
+
channels.forEach(channel => {
|
|
47
|
+
subscriptionClient.subscribe(channel, (error) => {
|
|
48
|
+
if (error) {
|
|
49
|
+
logger.error(`Error subscribing to redis channel ${channel}: ${error}`);
|
|
50
|
+
} else {
|
|
51
|
+
logger.info(`Subscribed to channel ${channel}`);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
31
54
|
});
|
|
32
55
|
});
|
|
33
56
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
57
|
+
subscriptionClient.on('message', (channel, message) => {
|
|
58
|
+
logger.debug(`Received message from ${channel}: ${message}`);
|
|
59
|
+
|
|
60
|
+
let decryptedMessage = message;
|
|
38
61
|
|
|
62
|
+
if (channel === requestProgressChannel && redisEncryptionKey) {
|
|
39
63
|
try {
|
|
40
|
-
|
|
64
|
+
decryptedMessage = decrypt(message, redisEncryptionKey);
|
|
41
65
|
} catch (error) {
|
|
42
|
-
|
|
66
|
+
logger.error(`Error decrypting message: ${error}`);
|
|
43
67
|
}
|
|
44
|
-
|
|
45
|
-
handleMessage(parsedMessage);
|
|
46
68
|
}
|
|
47
|
-
});
|
|
48
69
|
|
|
49
|
-
|
|
50
|
-
// Process the received data
|
|
51
|
-
console.log('Processing data:', data);
|
|
70
|
+
let parsedMessage = decryptedMessage;
|
|
52
71
|
try {
|
|
53
|
-
|
|
72
|
+
parsedMessage = JSON.parse(decryptedMessage);
|
|
54
73
|
} catch (error) {
|
|
55
|
-
|
|
74
|
+
logger.error(`Error parsing message: ${error}`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
switch(channel) {
|
|
78
|
+
case requestProgressChannel:
|
|
79
|
+
pubsubHandleMessage(parsedMessage);
|
|
80
|
+
break;
|
|
81
|
+
case requestProgressSubscriptionsChannel:
|
|
82
|
+
handleSubscription(parsedMessage);
|
|
83
|
+
break;
|
|
84
|
+
default:
|
|
85
|
+
logger.error(`Unsupported channel: ${channel}`);
|
|
86
|
+
break;
|
|
56
87
|
}
|
|
57
|
-
};
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
} else {
|
|
91
|
+
// No Redis connection, use pubsub for communication
|
|
92
|
+
logger.info(`Using pubsub publish for channel ${requestProgressChannel}`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async function publishRequestProgress(data) {
|
|
96
|
+
if (publisherClient) {
|
|
97
|
+
try {
|
|
98
|
+
let message = JSON.stringify(data);
|
|
99
|
+
if (redisEncryptionKey) {
|
|
100
|
+
try {
|
|
101
|
+
message = encrypt(message, redisEncryptionKey);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
logger.error(`Error encrypting message: ${error}`);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
logger.debug(`Publishing message ${message} to channel ${requestProgressChannel}`);
|
|
107
|
+
await publisherClient.publish(requestProgressChannel, message);
|
|
108
|
+
} catch (error) {
|
|
109
|
+
logger.error(`Error publishing message: ${error}`);
|
|
110
|
+
}
|
|
111
|
+
} else {
|
|
112
|
+
pubsubHandleMessage(data);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async function publishRequestProgressSubscription(data) {
|
|
117
|
+
if (publisherClient) {
|
|
118
|
+
try {
|
|
119
|
+
const message = JSON.stringify(data);
|
|
120
|
+
logger.debug(`Publishing message ${message} to channel ${requestProgressSubscriptionsChannel}`);
|
|
121
|
+
await publisherClient.publish(requestProgressSubscriptionsChannel, message);
|
|
122
|
+
} catch (error) {
|
|
123
|
+
logger.error(`Error publishing message: ${error}`);
|
|
124
|
+
}
|
|
125
|
+
} else {
|
|
126
|
+
handleSubscription(data);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function pubsubHandleMessage(data){
|
|
131
|
+
const message = JSON.stringify(data);
|
|
132
|
+
logger.debug(`Publishing message to pubsub: ${message}`);
|
|
133
|
+
try {
|
|
134
|
+
pubsub.publish('REQUEST_PROGRESS', { requestProgress: data });
|
|
135
|
+
} catch (error) {
|
|
136
|
+
logger.error(`Error publishing data to pubsub: ${error}`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function handleSubscription(data){
|
|
141
|
+
const requestIds = data;
|
|
142
|
+
for (const requestId of requestIds) {
|
|
143
|
+
if (requestState[requestId] && !requestState[requestId].started) {
|
|
144
|
+
requestState[requestId].started = true;
|
|
145
|
+
logger.info(`Subscription starting async requestProgress, requestId: ${requestId}`);
|
|
146
|
+
const { resolver, args } = requestState[requestId];
|
|
147
|
+
resolver(args);
|
|
148
|
+
}
|
|
58
149
|
}
|
|
59
150
|
}
|
|
60
151
|
|
|
61
152
|
export {
|
|
62
|
-
|
|
153
|
+
subscriptionClient, publishRequestProgress, publishRequestProgressSubscription
|
|
63
154
|
};
|
package/lib/request.js
CHANGED
|
@@ -4,13 +4,14 @@ import { config } from '../config.js';
|
|
|
4
4
|
import axios from 'axios';
|
|
5
5
|
import { setupCache } from 'axios-cache-interceptor';
|
|
6
6
|
import Redis from 'ioredis';
|
|
7
|
+
import logger from './logger.js';
|
|
7
8
|
|
|
8
9
|
const connectionString = config.get('storageConnectionString');
|
|
9
10
|
|
|
10
11
|
if (!connectionString) {
|
|
11
|
-
|
|
12
|
+
logger.info('No STORAGE_CONNECTION_STRING found in environment. Redis features (caching, pubsub, clustered limiters) disabled.')
|
|
12
13
|
} else {
|
|
13
|
-
|
|
14
|
+
logger.info('Using Redis connection specified in STORAGE_CONNECTION_STRING.');
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
let client;
|
|
@@ -19,7 +20,7 @@ if (connectionString) {
|
|
|
19
20
|
try {
|
|
20
21
|
client = new Redis(connectionString);
|
|
21
22
|
} catch (error) {
|
|
22
|
-
|
|
23
|
+
logger.error(`Redis connection error: ${error}`);
|
|
23
24
|
}
|
|
24
25
|
}
|
|
25
26
|
|
|
@@ -30,7 +31,7 @@ const limiters = {};
|
|
|
30
31
|
const monitors = {};
|
|
31
32
|
|
|
32
33
|
const buildLimiters = (config) => {
|
|
33
|
-
|
|
34
|
+
logger.info(`Building ${connection ? 'Redis clustered' : 'local'} model rate limiters for ${cortexId}...`);
|
|
34
35
|
for (const [name, model] of Object.entries(config.get('models'))) {
|
|
35
36
|
const rps = model.requestsPerSecond ?? 100;
|
|
36
37
|
let limiterOptions = {
|
|
@@ -50,7 +51,7 @@ const buildLimiters = (config) => {
|
|
|
50
51
|
|
|
51
52
|
limiters[name] = new Bottleneck(limiterOptions);
|
|
52
53
|
limiters[name].on('error', (err) => {
|
|
53
|
-
|
|
54
|
+
logger.error(`Limiter error for ${cortexId}-${name}: ${err}`);
|
|
54
55
|
});
|
|
55
56
|
monitors[name] = new RequestMonitor();
|
|
56
57
|
}
|
|
@@ -81,9 +82,9 @@ setInterval(() => {
|
|
|
81
82
|
const callRate = monitor.getPeakCallRate();
|
|
82
83
|
const error429Rate = monitor.getError429Rate();
|
|
83
84
|
if (callRate > 0) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
85
|
+
logger.info('------------------------');
|
|
86
|
+
logger.info(`${monitorName} Call rate: ${callRate} calls/sec, 429 errors: ${error429Rate * 100}%`);
|
|
87
|
+
logger.info('------------------------');
|
|
87
88
|
// Reset the rate monitor to start a new monitoring interval.
|
|
88
89
|
monitor.reset();
|
|
89
90
|
}
|
|
@@ -109,7 +110,7 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
109
110
|
let duplicateRequestAfter = (pathway?.duplicateRequestAfter || DUPLICATE_REQUEST_AFTER) * 1000;
|
|
110
111
|
|
|
111
112
|
if (enableDuplicateRequests) {
|
|
112
|
-
//
|
|
113
|
+
//logger.info(`>>> [${requestId}] Duplicate requests enabled after ${duplicateRequestAfter / 1000} seconds`);
|
|
113
114
|
}
|
|
114
115
|
|
|
115
116
|
const axiosConfigObj = { params, headers, cache };
|
|
@@ -119,7 +120,7 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
119
120
|
promises.push(limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj)));
|
|
120
121
|
} else {
|
|
121
122
|
if (streamRequested) {
|
|
122
|
-
|
|
123
|
+
logger.info(`>>> [${requestId}] ${model} does not support streaming - sending non-streaming request`);
|
|
123
124
|
axiosConfigObj.params.stream = false;
|
|
124
125
|
data.stream = false;
|
|
125
126
|
}
|
|
@@ -144,7 +145,7 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
144
145
|
axiosConfigObj.headers['X-Cortex-Request-Index'] = index;
|
|
145
146
|
|
|
146
147
|
if (index === 0) {
|
|
147
|
-
//
|
|
148
|
+
//logger.info(`>>> [${requestId}] sending request to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`);
|
|
148
149
|
} else {
|
|
149
150
|
if (modelProperties.supportsStreaming) {
|
|
150
151
|
axiosConfigObj.responseType = 'stream';
|
|
@@ -152,23 +153,23 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
152
153
|
}
|
|
153
154
|
const logMessage = `>>> [${requestId}] taking too long - sending duplicate request ${index} to ${model} API ${axiosConfigObj.responseType === 'stream' ? 'with streaming' : ''}`;
|
|
154
155
|
const header = '>'.repeat(logMessage.length);
|
|
155
|
-
|
|
156
|
+
logger.info(`\n${header}\n${logMessage}`);
|
|
156
157
|
}
|
|
157
158
|
|
|
158
159
|
response = await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
|
|
159
160
|
|
|
160
161
|
if (!controller.signal?.aborted) {
|
|
161
162
|
|
|
162
|
-
//
|
|
163
|
+
//logger.info(`<<< [${requestId}] received response for request ${index}`);
|
|
163
164
|
|
|
164
165
|
if (axiosConfigObj.responseType === 'stream') {
|
|
165
166
|
// Buffering and collecting the stream data
|
|
166
|
-
|
|
167
|
+
logger.info(`<<< [${requestId}] buffering streaming response for request ${index}`);
|
|
167
168
|
response = await new Promise((resolve, reject) => {
|
|
168
169
|
let responseData = '';
|
|
169
170
|
response.data.on('data', (chunk) => {
|
|
170
171
|
responseData += chunk;
|
|
171
|
-
//
|
|
172
|
+
//logger.info(`<<< [${requestId}] received chunk for request ${index}`);
|
|
172
173
|
});
|
|
173
174
|
response.data.on('end', () => {
|
|
174
175
|
response.data = JSON.parse(responseData);
|
|
@@ -186,10 +187,10 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
186
187
|
|
|
187
188
|
} catch (error) {
|
|
188
189
|
if (error.name === 'AbortError' || error.name === 'CanceledError') {
|
|
189
|
-
//
|
|
190
|
+
//logger.info(`XXX [${requestId}] request ${index} was cancelled`);
|
|
190
191
|
reject(error);
|
|
191
192
|
} else {
|
|
192
|
-
|
|
193
|
+
logger.info(`!!! [${requestId}] request ${index} failed with error: ${error?.response?.data?.error?.message || error}`);
|
|
193
194
|
reject(error);
|
|
194
195
|
}
|
|
195
196
|
} finally {
|
|
@@ -210,14 +211,14 @@ const postRequest = async ({ url, data, params, headers, cache }, model, request
|
|
|
210
211
|
throw new Error(`Received error response: ${response.status}`);
|
|
211
212
|
}
|
|
212
213
|
} catch (error) {
|
|
213
|
-
//
|
|
214
|
+
//logger.error(`!!! [${requestId}] failed request with data ${JSON.stringify(data)}: ${error}`);
|
|
214
215
|
if (error.response) {
|
|
215
216
|
const status = error.response.status;
|
|
216
217
|
if ((status === 429) || (status >= 500 && status < 600)) {
|
|
217
218
|
if (status === 429) {
|
|
218
219
|
monitors[model].incrementError429Count();
|
|
219
220
|
}
|
|
220
|
-
|
|
221
|
+
logger.info(`>>> [${requestId}] retrying request due to ${status} response. Retry count: ${i + 1}`);
|
|
221
222
|
if (i < MAX_RETRY - 1) {
|
|
222
223
|
const backoffTime = 200 * Math.pow(2, i);
|
|
223
224
|
const jitter = backoffTime * 0.2 * Math.random();
|
|
@@ -240,16 +241,16 @@ const request = async (params, model, requestId, pathway) => {
|
|
|
240
241
|
const response = await postRequest(params, model, requestId, pathway);
|
|
241
242
|
const { error, data, cached } = response;
|
|
242
243
|
if (cached) {
|
|
243
|
-
|
|
244
|
+
logger.info(`<<< [${requestId}] served with cached response.`);
|
|
244
245
|
}
|
|
245
246
|
if (error && error.length > 0) {
|
|
246
247
|
const lastError = error[error.length - 1];
|
|
247
248
|
return { error: lastError.toJSON() ?? lastError ?? error };
|
|
248
249
|
}
|
|
249
|
-
//
|
|
250
|
+
//logger.info(`<<< [${requestId}] response: ${data.choices[0].delta || data.choices[0]}`)
|
|
250
251
|
return data;
|
|
251
252
|
} catch (error) {
|
|
252
|
-
|
|
253
|
+
logger.error(`Error in request: ${error.message || error}`);
|
|
253
254
|
return { error: error };
|
|
254
255
|
}
|
|
255
256
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aj-archipelago/cortex",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.1.0-beta.1",
|
|
4
4
|
"description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
|
|
5
5
|
"private": false,
|
|
6
6
|
"repository": {
|
|
@@ -54,6 +54,7 @@
|
|
|
54
54
|
"langchain": "^0.0.47",
|
|
55
55
|
"subsrt": "^1.1.1",
|
|
56
56
|
"uuid": "^9.0.0",
|
|
57
|
+
"winston": "^3.11.0",
|
|
57
58
|
"ws": "^8.12.0"
|
|
58
59
|
},
|
|
59
60
|
"devDependencies": {
|
|
@@ -62,7 +63,7 @@
|
|
|
62
63
|
"eslint": "^8.38.0",
|
|
63
64
|
"eslint-plugin-import": "^2.27.5",
|
|
64
65
|
"got": "^13.0.0",
|
|
65
|
-
"sinon": "^
|
|
66
|
+
"sinon": "^17.0.1"
|
|
66
67
|
},
|
|
67
68
|
"publishConfig": {
|
|
68
69
|
"access": "restricted"
|
package/server/graphql.js
CHANGED
|
@@ -22,6 +22,7 @@ import { buildPathways, buildModels } from '../config.js';
|
|
|
22
22
|
import { requestState } from './requestState.js';
|
|
23
23
|
import { buildRestEndpoints } from './rest.js';
|
|
24
24
|
import { startTestServer } from '../tests/server.js'
|
|
25
|
+
import logger from '../lib/logger.js';
|
|
25
26
|
|
|
26
27
|
// Utility functions
|
|
27
28
|
// Server plugins
|
|
@@ -41,7 +42,7 @@ const getPlugins = (config) => {
|
|
|
41
42
|
// TODO: custom cache key:
|
|
42
43
|
// https://www.apollographql.com/docs/apollo-server/performance/cache-backends#implementing-your-own-cache-backend
|
|
43
44
|
plugins.push(responseCachePlugin({ cache }));
|
|
44
|
-
|
|
45
|
+
logger.info('Using Redis for GraphQL cache');
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
return { plugins, cache };
|
|
@@ -146,12 +147,12 @@ const build = async (config) => {
|
|
|
146
147
|
// Respects the keep alive setting in config in case you want to
|
|
147
148
|
// turn it off for deployments that don't route the ping/pong frames
|
|
148
149
|
const keepAlive = config.get('subscriptionKeepAlive');
|
|
149
|
-
|
|
150
|
+
logger.info(`Starting web socket server with subscription keep alive: ${keepAlive}`);
|
|
150
151
|
const serverCleanup = useServer({ schema }, wsServer, keepAlive);
|
|
151
152
|
|
|
152
153
|
const server = new ApolloServer({
|
|
153
154
|
schema,
|
|
154
|
-
introspection:
|
|
155
|
+
introspection: config.get('env') === 'development',
|
|
155
156
|
csrfPrevention: true,
|
|
156
157
|
plugins: plugins.concat([// Proper shutdown for the HTTP server.
|
|
157
158
|
ApolloServerPluginDrainHttpServer({ httpServer }),
|
|
@@ -232,7 +233,7 @@ const build = async (config) => {
|
|
|
232
233
|
|
|
233
234
|
// Now that our HTTP server is fully set up, we can listen to it.
|
|
234
235
|
httpServer.listen(config.get('PORT'), () => {
|
|
235
|
-
|
|
236
|
+
logger.info(`🚀 Server is now running at http://localhost:${config.get('PORT')}/graphql`);
|
|
236
237
|
});
|
|
237
238
|
};
|
|
238
239
|
|
package/server/parser.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import logger from '../lib/logger.js';
|
|
2
|
+
|
|
1
3
|
//simply trim and parse with given regex
|
|
2
4
|
const regexParser = (text, regex) => {
|
|
3
5
|
return text.trim().split(regex).map(s => s.trim()).filter(s => s.length);
|
|
@@ -25,7 +27,7 @@ const parseNumberedObjectList = (text, format) => {
|
|
|
25
27
|
}
|
|
26
28
|
result.push(obj);
|
|
27
29
|
} catch (e) {
|
|
28
|
-
|
|
30
|
+
logger.warn(`Failed to parse value in parseNumberedObjectList, value: ${value}, fields: ${fields}`);
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
|