@aj-archipelago/cortex 0.0.5 → 0.0.7

package/graphql/plugins/openAiWhisperPlugin.js

@@ -0,0 +1,79 @@
+// OpenAICompletionPlugin.js
+const ModelPlugin = require('./modelPlugin');
+const handlebars = require("handlebars");
+const { encode } = require("gpt-3-encoder");
+const FormData = require('form-data');
+const fs = require('fs');
+const { splitMediaFile, isValidYoutubeUrl, processYoutubeUrl, deleteTempPath } = require('../../lib/fileChunker');
+const pubsub = require('../pubsub');
+
+class OpenAIWhisperPlugin extends ModelPlugin {
+    constructor(config, pathway) {
+        super(config, pathway);
+    }
+
+    getCompiledPrompt(text, parameters, prompt) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        const modelPrompt = this.getModelPrompt(prompt, parameters);
+        const modelPromptText = modelPrompt.prompt ? handlebars.compile(modelPrompt.prompt)({ ...combinedParameters, text }) : '';
+    
+        return { modelPromptText, tokenLength: encode(modelPromptText).length };
+    }
+
+    // Execute the request to the OpenAI Whisper API
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const url = this.requestUrl(text);
+        const params = {};
+        const { modelPromptText } = this.getCompiledPrompt(text, parameters, prompt);
+
+        const processChunk = async (chunk) => {
+            try {
+                const formData = new FormData();
+                formData.append('file', fs.createReadStream(chunk));
+                formData.append('model', this.model.params.model);
+                formData.append('response_format', 'text');
+                // formData.append('language', 'tr');
+                modelPromptText && formData.append('prompt', modelPromptText);
+
+                return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() });
+            } catch (err) {
+                console.log(err);
+            }
+        }
+
+        let result;
+        let { file } = parameters;
+        let folder;
+        const isYoutubeUrl = isValidYoutubeUrl(file);
+
+        try {
+            if (isYoutubeUrl) {
+                file = await processYoutubeUrl(file);
+            }
+
+            const mediaSplit = await splitMediaFile(file);
+
+            const { requestId } = pathwayResolver;
+            pubsub.publish('REQUEST_PROGRESS', {
+                requestProgress: {
+                    requestId,
+                    progress: 0.5,
+                    data: null,
+                }
+            });
+
+            folder = mediaSplit.folder;
+            result = await Promise.all(mediaSplit.chunks.map(processChunk));
+
+        } catch (error) {
+            console.error("An error occurred:", error);
+        } finally {
+            isYoutubeUrl && (await deleteTempPath(file));
+            folder && (await deleteTempPath(folder));
+        }
+        return result.join('');
+    }
+}
+
+module.exports = OpenAIWhisperPlugin;
+

package/graphql/prompt.js

@@ -12,6 +12,7 @@ class Prompt {
 
         this.usesTextInput = promptContains('text', this.prompt ? this.prompt : this.messages);
         this.usesPreviousResult = promptContains('previousResult', this.prompt ? this.prompt : this.messages);
+        this.debugInfo = '';
     }
 }
 

package/graphql/requestState.js

@@ -0,0 +1,5 @@
+const requestState = {}; // Stores the state of each request
+
+module.exports = {
+    requestState
+};

package/graphql/resolver.js

@@ -5,23 +5,23 @@ const { PathwayResolver } = require("./pathwayResolver");
 // (parent, args, contextValue, info)
 const rootResolver = async (parent, args, contextValue, info) => {
     const { config, pathway, requestState } = contextValue;
-    const { temperature } = pathway;
+    const { temperature, enableGraphqlCache } = pathway;
 
-    // Turn off caching if temperature is 0
-    if (temperature == 0) {
+    // Turn on graphql caching if enableGraphqlCache true and temperature is 0
+    if (enableGraphqlCache && temperature == 0) { // || 
         info.cacheControl.setCacheHint({ maxAge: 60 * 60 * 24, scope: 'PUBLIC' });
     }
 
-    const pathwayResolver = new PathwayResolver({ config, pathway, requestState });
+    const pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
     contextValue.pathwayResolver = pathwayResolver;
 
-    // Add request parameters back as debug
-    const requestParameters = pathwayResolver.prompts.map((prompt) => pathwayResolver.pathwayPrompter.plugin.requestParameters(args.text, args, prompt));
-    const debug = JSON.stringify(requestParameters);
-
     // Execute the request with timeout
     const result = await fulfillWithTimeout(pathway.resolver(parent, args, contextValue, info), pathway.timeout);
     const { warnings, previousResult, savedContextId } = pathwayResolver;
+    
+    // Add request parameters back as debug
+    const debug = pathwayResolver.prompts.map(prompt => prompt.debugInfo || '').join('\n').trim();
+    
     return { debug, result, warnings, previousResult, contextId: savedContextId }
 }
 

package/graphql/subscriptions.js

@@ -4,14 +4,27 @@
 
 const pubsub = require("./pubsub");
 const { withFilter } = require("graphql-subscriptions");
+const { requestState } = require("./requestState");
 
 const subscriptions = {
     requestProgress: {
         subscribe: withFilter(
-            () => pubsub.asyncIterator(['REQUEST_PROGRESS']),
+            (_, args, __, info) => {
+                const { requestIds } = args;
+                for (const requestId of requestIds) {
+                    if (!requestState[requestId]) {
+                        console.log(`requestProgress, requestId: ${requestId} not found`);
+                    } else {
+                        console.log(`starting async requestProgress, requestId: ${requestId}`);
+                        const { resolver, args } = requestState[requestId];
+                        resolver(args);
+                    }
+                }
+                return pubsub.asyncIterator(['REQUEST_PROGRESS'])
+            },
             (payload, variables) => {
                 return (
-                    payload.requestProgress.requestId === variables.requestId
+                    variables.requestIds.includes(payload.requestProgress.requestId)
                 );
             },
         ),

package/graphql/typeDef.js

@@ -2,51 +2,60 @@ const GRAPHQL_TYPE_MAP = {
     boolean: 'Boolean',
     string: 'String',
     number: 'Int',
-}
-
-
-const typeDef = (pathway) => {
+  };
+  
+  const typeDef = (pathway) => {
     const { name, objName, defaultInputParameters, inputParameters, format } = pathway;
-
+  
     const fields = format ? format.match(/\b(\w+)\b/g) : null;
-    const fieldsStr = !fields ? `` : fields.map(f => `${f}: String`).join('\n    ');
-
+    const fieldsStr = !fields ? `` : fields.map((f) => `${f}: String`).join('\n    ');
+  
     const typeName = fields ? `${objName}Result` : `String`;
     const messageType = `input Message { role: String, content: String }`;
-
+  
     const type = fields ? `type ${typeName} {
-    ${fieldsStr}
-    }` : ``;
-
+      ${fieldsStr}
+      }` : ``;
+  
     const resultStr = pathway.list ? `[${typeName}]` : typeName;
-
+  
     const responseType = `type ${objName} {
-        debug: String
-        result: ${resultStr}
-        previousResult: String
-        warnings: [String]
-        contextId: String
-}`;
-
-
+          debug: String
+          result: ${resultStr}
+          previousResult: String
+          warnings: [String]
+          contextId: String
+  }`;
+  
     const params = { ...defaultInputParameters, ...inputParameters };
-
-    const paramsStr = Object.entries(params).map(
-        ([key, value]) => {
-            if (typeof value === 'object' && Array.isArray(value)) {
-                return `${key}: [Message] = []`;
-            } else {
-                return `${key}: ${GRAPHQL_TYPE_MAP[typeof (value)]} = ${typeof (value) === 'string' ? `"${value}"` : value}`;
-            }
+  
+    const paramsStr = Object.entries(params)
+      .map(([key, value]) => {
+        if (typeof value === 'object' && Array.isArray(value)) {
+          return `${key}: [Message] = []`;
+        } else {
+          return `${key}: ${GRAPHQL_TYPE_MAP[typeof value]} = ${
+            typeof value === 'string' ? `"${value}"` : value
+          }`;
         }
-        ).join('\n');
-          
-
-    const definition = `${messageType}\n\n${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
-    //console.log(definition);
-    return definition;
-}
-
-module.exports = {
+      })
+      .join('\n');
+  
+    const restDefinition = Object.entries(params).map(([key, value]) => {
+      return {
+        name: key,
+        type: `${GRAPHQL_TYPE_MAP[typeof value]}${typeof value === 'object' && Array.isArray(value) ? '[]' : ''}`,
+      };
+    });
+  
+    const gqlDefinition = `${messageType}\n\n${type}\n\n${responseType}\n\nextend type Query {${name}(${paramsStr}): ${objName}}`;
+  
+    return {
+      gqlDefinition,
+      restDefinition,
+    };
+  };
+  
+  module.exports = {
     typeDef,
-}
+  };

package/lib/fileChunker.js

@@ -0,0 +1,152 @@
+const fs = require('fs');
+const ffmpeg = require('fluent-ffmpeg');
+const path = require('path');
+const { v4: uuidv4 } = require('uuid');
+const os = require('os');
+const util = require('util');
+const ffmpegProbe = util.promisify(ffmpeg.ffprobe);
+const pipeline = util.promisify(require('stream').pipeline);
+const ytdl = require('ytdl-core');
+
+
+async function processChunk(inputPath, outputFileName, start, duration) {
+    return new Promise((resolve, reject) => {
+        ffmpeg(inputPath)
+            .seekInput(start)
+            .duration(duration)
+            .on('start', (cmd) => {
+                console.log(`Started FFmpeg with command: ${cmd}`);
+            })
+            .on('error', (err) => {
+                console.error(`Error occurred while processing chunk:`, err);
+                reject(err);
+            })
+            .on('end', () => {
+                console.log(`Finished processing chunk`);
+                resolve(outputFileName);
+            })
+            .save(outputFileName);
+    });
+}
+
+const generateUniqueFolderName = () => {
+    const uniqueFolderName = uuidv4();
+    const tempFolderPath = os.tmpdir(); // Get the system's temporary folder
+    const uniqueOutputPath = path.join(tempFolderPath, uniqueFolderName);
+    return uniqueOutputPath;
+}
+
+const generateUniqueTempFileName = () => {
+    return path.join(os.tmpdir(), uuidv4());
+}
+
+async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
+    try {
+        const metadata = await ffmpegProbe(inputPath);
+        const duration = metadata.format.duration;
+        const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
+
+        const chunkPromises = [];
+
+        const uniqueOutputPath = generateUniqueFolderName();
+
+        // Create unique folder
+        fs.mkdirSync(uniqueOutputPath, { recursive: true });
+
+
+        for (let i = 0; i < numChunks; i++) {
+            const outputFileName = path.join(
+                uniqueOutputPath,
+                `chunk-${i + 1}-${path.basename(inputPath)}`
+            );
+
+            const chunkPromise = processChunk(
+                inputPath,
+                outputFileName,
+                i * chunkDurationInSeconds,
+                chunkDurationInSeconds
+            );
+
+            chunkPromises.push(chunkPromise);
+        }
+
+        const chunkedFiles = await Promise.all(chunkPromises);
+        console.log('All chunks processed. Chunked file names:', chunkedFiles);
+        return { chunks: chunkedFiles, folder: uniqueOutputPath }
+    } catch (err) {
+        console.error('Error occurred during the splitting process:', err);
+    }
+}
+
+async function deleteTempPath(path) {
+    try {
+        if (!path) return;
+        const stats = fs.statSync(path);
+        if (stats.isFile()) {
+            fs.unlinkSync(path);
+            console.log(`Temporary file ${path} deleted successfully.`);
+        } else if (stats.isDirectory()) {
+            fs.rmdirSync(path, { recursive: true });
+            console.log(`Temporary folder ${path} and its contents deleted successfully.`);
+        }
+    } catch (err) {
+        console.error('Error occurred while deleting the temporary path:', err);
+    }
+}
+
+
+function isValidYoutubeUrl(url) {
+    const regex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
+    return regex.test(url);
+}
+
+function convertYoutubeToMp3Stream(video) {
+    // Configure ffmpeg to convert the video to mp3
+    const mp3Stream = ffmpeg(video)
+        .withAudioCodec('libmp3lame')
+        .toFormat('mp3')
+        .on('error', (err) => {
+            console.error(`An error occurred during conversion: ${err.message}`);
+        });
+
+    return mp3Stream;
+}
+
+async function pipeStreamToFile(stream, filePath) {
+    try {
+        await pipeline(stream, fs.createWriteStream(filePath));
+        console.log('Stream piped to file successfully.');
+    } catch (error) {
+        console.error(`Error piping stream to file: ${error.message}`);
+    }
+}
+
+
+const processYoutubeUrl = async (url) => {
+    const info = await ytdl.getInfo(url);
+    const audioFormat = ytdl.chooseFormat(info.formats, { quality: 'highestaudio' });
+
+    if (!audioFormat) {
+        throw new Error('No suitable audio format found');
+    }
+
+    const stream = ytdl.downloadFromInfo(info, { format: audioFormat });
+
+    const mp3Stream = convertYoutubeToMp3Stream(stream);
+    const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
+    await pipeStreamToFile(mp3Stream, outputFileName); // You can also pipe the stream to a file
+    return outputFileName;
+}
+
+function deleteFile(filePath) {
+    try {
+        fs.unlinkSync(filePath);
+        console.log(`File ${filePath} cleaned successfully.`);
+    } catch (error) {
+        console.error(`Error deleting file ${filePath}:`, error);
+    }
+}
+
+module.exports = {
+    splitMediaFile, deleteTempPath, processYoutubeUrl, isValidYoutubeUrl
+};

package/lib/request.js

@@ -1,20 +1,67 @@
-const axios = require('axios');
 const Bottleneck = require("bottleneck/es5");
+const RequestMonitor = require('./requestMonitor');
+const { config } = require('../config');
+let axios = require('axios');
+
+if (config.get('enableCache')) {
+    // Setup cache
+    const { setupCache } = require('axios-cache-interceptor');
+    axios = setupCache(axios, {
+        // enable cache for all requests by default
+        methods: ['get', 'post', 'put', 'delete', 'patch'],
+        interpretHeader: false,
+        ttl: 1000 * 60 * 60 * 24 * 7, // 7 days
+    }); 
+}
 
 const limiters = {};
+const monitors = {};
 
 const buildLimiters = (config) => {
     console.log('Building limiters...');
     for (const [name, model] of Object.entries(config.get('models'))) {
+        const rps = model.requestsPerSecond ?? 100;
         limiters[name] = new Bottleneck({
-            minTime: 1000 / (model.requestsPerSecond ?? 100),
-            // maxConcurrent: 20,
-        })
+            minTime: 1000 / rps,
+            maxConcurrent: rps,
+            reservoir: rps,      // Number of tokens available initially
+            reservoirRefreshAmount: rps,     // Number of tokens added per interval
+            reservoirRefreshInterval: 1000, // Interval in milliseconds
+        });
+        monitors[name] = new RequestMonitor();
+    }
+}
+
+setInterval(() => {
+    const monitorKeys = Object.keys(monitors);
+
+    // Skip logging if the monitors object does not exist or is empty
+    if (!monitorKeys || monitorKeys.length === 0) {
+      return;
     }
+
+    monitorKeys.forEach((monitorName) => {
+        const monitor = monitors[monitorName];
+        const callRate = monitor.getPeakCallRate();
+        const error429Rate = monitor.getError429Rate();
+        if (callRate > 0) {
+            console.log('------------------------');
+            console.log(`${monitorName} Call rate: ${callRate} calls/sec, 429 errors: ${error429Rate * 100}%`);
+            console.log('------------------------');
+            // Reset the rate monitor to start a new monitoring interval.
+            monitor.reset();
+        }
+    });
+  }, 10000); // Log rates every 10 seconds (10000 ms).
+
+const postWithMonitor = async (model, url, data, axiosConfigObj) => {
+    const monitor = monitors[model];
+    monitor.incrementCallCount();
+    return axios.post(url, data, axiosConfigObj);
 }
 
 const MAX_RETRY = 10;
-const postRequest = async ({ url, data, params, headers }, model) => {
+const postRequest = async ({ url, data, params, headers, cache }, model) => {
     let retry = 0;
     const errors = []
     for (let i = 0; i < MAX_RETRY; i++) {
@@ -22,13 +69,20 @@ const postRequest = async ({ url, data, params, headers }, model) => {
             if (i > 0) {
                 console.log(`Retrying request #retry ${i}: ${JSON.stringify(data)}...`);
                 await new Promise(r => setTimeout(r, 200 * Math.pow(2, i))); // exponential backoff
-            }
+            }           
             if (!limiters[model]) {
                 throw new Error(`No limiter for model ${model}!`);
             }
-            return await limiters[model].schedule(() => axios.post(url, data, { params, headers }));
+            const axiosConfigObj = { params, headers, cache };
+            if (params.stream || data.stream) {
+                axiosConfigObj.responseType = 'stream';
+            }           
+            return await limiters[model].schedule(() => postWithMonitor(model, url, data, axiosConfigObj));
         } catch (e) {
             console.error(`Failed request with data ${JSON.stringify(data)}: ${e}`);
+            if (e.response.status === 429) {
+                monitors[model].incrementError429Count();
+            }
             errors.push(e);
         }
     }
@@ -37,7 +91,10 @@ const postRequest = async ({ url, data, params, headers }, model) => {
 
 const request = async (params, model) => {
     const response = await postRequest(params, model);
-    const { error, data } = response;
+    const { error, data, cached } = response;
+    if (cached) {
+        console.info('/Request served with cached response.');
+    }
     if (error && error.length > 0) {
         const lastError = error[error.length - 1];
         return { error: lastError.toJSON() ?? lastError ?? error };

package/lib/requestMonitor.js

@@ -0,0 +1,43 @@
+class RequestMonitor {
+    constructor() {
+      this.callCount = 0;
+      this.peakCallRate = 0;
+      this.error429Count = 0;
+      this.startTime = new Date();
+    }
+  
+    incrementCallCount() {
+      this.callCount++;
+      if (this.getCallRate() > this.peakCallRate) {
+        this.peakCallRate = this.getCallRate();
+      }
+    }
+  
+    incrementError429Count() {
+      this.error429Count++;
+    }
+  
+    getCallRate() {
+      const currentTime = new Date();
+      const timeElapsed = (currentTime - this.startTime) / 1000; // time elapsed in seconds
+      return timeElapsed < 1 ? this.callCount : this.callCount / timeElapsed;
+    }
+
+    getPeakCallRate() {
+      return this.peakCallRate;
+    }
+  
+    getError429Rate() {
+      return this.error429Count / this.callCount;
+    }
+  
+    reset() {
+      this.callCount = 0;
+      this.error429Count = 0;
+      this.peakCallRate = 0;
+      this.startTime = new Date();
+    }
+  }
+  
+  module.exports = RequestMonitor;
+  

package/package.json

@@ -1,14 +1,22 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "0.0.5",
-  "description": "Project Cortex",
+  "version": "0.0.7",
+  "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/aj-archipelago/cortex.git"
   },
   "keywords": [
     "cortex",
-    "ai"
+    "AI",
+    "prompt engineering",
+    "LLM",
+    "OpenAI",
+    "Azure",
+    "GPT-3",
+    "GPT-4",
+    "chatGPT",
+    "GraphQL"
   ],
   "main": "index.js",
   "scripts": {
@@ -22,22 +30,26 @@
     "@apollo/utils.keyvadapter": "^1.1.2",
     "@graphql-tools/schema": "^9.0.12",
     "@keyv/redis": "^2.5.4",
-    "apollo-server": "^3.11.1",
+    "apollo-server": "^3.12.0",
     "apollo-server-core": "^3.11.1",
     "apollo-server-express": "^3.11.1",
     "apollo-server-plugin-response-cache": "^3.8.1",
-    "axios": "^1.2.0",
+    "axios": "^1.3.4",
+    "axios-cache-interceptor": "^1.0.1",
     "bottleneck": "^2.19.5",
     "compromise": "^14.8.1",
     "compromise-paragraphs": "^0.1.0",
     "convict": "^6.2.3",
+    "fluent-ffmpeg": "^2.1.2",
+    "form-data": "^4.0.0",
     "gpt-3-encoder": "^1.1.4",
     "graphql": "^16.6.0",
     "graphql-subscriptions": "^2.0.0",
     "graphql-ws": "^5.11.2",
     "handlebars": "^4.7.7",
     "keyv": "^4.5.2",
-    "ws": "^8.12.0"
+    "ws": "^8.12.0",
+    "ytdl-core": "^4.11.2"
   },
   "devDependencies": {
     "dotenv": "^16.0.3",

package/pathways/basePathway.js

@@ -1,4 +1,3 @@
-const { parseResponse } = require("../graphql/parser");
 const { rootResolver, resolver } = require("../graphql/resolver");
 const { typeDef } = require('../graphql/typeDef')
 
@@ -7,9 +6,9 @@ module.exports = {
     prompt: `{{text}}`,
     defaultInputParameters: {
         text: ``,
-        // Add the option of making every call async
-        async: false,
-        contextId : ``, // used to identify the context of the request
+        async: false, // switch to enable async mode
+        contextId: ``, // used to identify the context of the request,
+        stream: false, // switch to enable stream mode
     },
     inputParameters: {},
     typeDef,

package/pathways/bias.js

@@ -1,3 +1,10 @@
+// bias.js
+// Objectivity analysis of text
+// This module exports a prompt that analyzes the given text and determines if it's written objectively. It also provides a detailed explanation of the decision.
+
 module.exports = {
+    // Uncomment the following line to enable caching for this prompt, if desired.
+    // enableCache: true,
+    
     prompt: `{{text}}\n\nIs the above text written objectively?  Why or why not, explain with details:\n`
 }

package/pathways/chat.js

@@ -1,4 +1,7 @@
-// Description: Have a chat with a bot that uses context to understand the conversation
+// chat.js
+// Simple context-aware chat bot
+// This is a two prompt implementation of a context aware chat bot. The first prompt generates content that will be stored in the previousResult variable and will be returned to the client. In the optimum implementation, the client will then update their chatContext variable for the next call. The second prompt actually responds to the user. The second prompt *could* use previousResult instead of chatContext, but in this situation previousResult will also include the current turn of the conversation to which it is responding. That can get a little confusing as it tends to overemphasize the current turn in the response.
+
 module.exports = {
     prompt:
         [

package/pathways/complete.js

@@ -1,3 +1,7 @@
+// complete.js
+// Text completion module
+// This module exports a prompt that takes an input text and completes it by generating a continuation of the given text.
+
 module.exports = {
     prompt: `Continue and complete the following:\n\n{{text}}`
 }