@aj-archipelago/cortex 1.0.3 → 1.0.5

package/graphql/plugins/palmChatPlugin.js

@@ -0,0 +1,229 @@
+// palmChatPlugin.js
+import ModelPlugin from './modelPlugin.js';
+import { encode } from 'gpt-3-encoder';
+import HandleBars from '../../lib/handleBars.js';
+
+class PalmChatPlugin extends ModelPlugin {
+    constructor(config, pathway) {
+        super(config, pathway);
+    }
+
+    // Convert to PaLM messages array format if necessary
+    convertMessagesToPalm(messages) {
+        let context = '';
+        let modifiedMessages = [];
+        let lastAuthor = '';
+    
+        messages.forEach(message => {
+            const { role, author, content } = message;
+    
+            // Extract system messages into the context string
+            if (role === 'system') {
+                context += (context.length > 0 ? '\n' : '') + content;
+                return;
+            }
+     
+            // Aggregate consecutive author messages, appending the content
+            if ((role === lastAuthor || author === lastAuthor) && modifiedMessages.length > 0) {
+                modifiedMessages[modifiedMessages.length - 1].content += '\n' + content;
+            }
+            // Only push messages with role 'user' or 'assistant' or existing author messages
+            else if (role === 'user' || role === 'assistant' || author) {
+                modifiedMessages.push({
+                    author: author || role,
+                    content,
+                });
+                lastAuthor = author || role;
+            }
+        });
+    
+        return {
+            modifiedMessages,
+            context,
+        };
+    }
+
+    // Handlebars compiler for context (PaLM chat specific)
+    getCompiledContext(text, parameters, context) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+        return context ? HandleBars.compile(context)({ ...combinedParameters, text}) : '';
+    }
+
+    // Handlebars compiler for examples (PaLM chat specific)
+    getCompiledExamples(text, parameters, examples = []) {
+        const combinedParameters = { ...this.promptParameters, ...parameters };
+
+        const compileContent = (content) => {
+            const compile = HandleBars.compile(content);
+            return compile({ ...combinedParameters, text });
+        };
+
+        const processExample = (example, key) => {
+            if (example[key]?.content) {
+                return { ...example[key], content: compileContent(example[key].content) };
+            }
+            return { ...example[key] };
+        };
+
+        return examples.map((example) => ({
+            input: example.input ? processExample(example, 'input') : undefined,
+            output: example.output ? processExample(example, 'output') : undefined,
+        }));
+    }
+
+    // Set up parameters specific to the PaLM Chat API
+    getRequestParameters(text, parameters, prompt) {
+        const { modelPromptText, modelPromptMessages, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+        const { stream } = parameters;
+    
+        // Define the model's max token length
+        const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
+    
+        const palmMessages = this.convertMessagesToPalm(modelPromptMessages || [{ "author": "user", "content": modelPromptText }]);
+        
+        let requestMessages = palmMessages.modifiedMessages;
+
+        // Check if the token length exceeds the model's max token length
+        if (tokenLength > modelTargetTokenLength) {
+            // Remove older messages until the token length is within the model's limit
+            requestMessages = this.truncateMessagesToTargetLength(requestMessages, modelTargetTokenLength);
+        }
+
+        const context = this.getCompiledContext(text, parameters, prompt.context || palmMessages.context || '');
+        const examples = this.getCompiledExamples(text, parameters, prompt.examples || []);
+        
+        // For PaLM right now, the max return tokens is 1024, regardless of the max context length
+        // I can't think of a time you'd want to constrain it to fewer at the moment.
+        const max_tokens = 1024//this.getModelMaxTokenLength() - tokenLength;
+    
+        if (max_tokens < 0) {
+            throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens.  The model will not be called.`);
+        }
+    
+        // Ensure there are an even number of messages (PaLM requires an even number of messages)
+        if (requestMessages.length % 2 === 0) {
+            requestMessages = requestMessages.slice(1);
+        }
+
+        const requestParameters = {
+            instances: [{
+                context: context,
+                examples: examples,
+                messages: requestMessages,
+            }],
+            parameters: {
+                temperature: this.temperature ?? 0.7,
+                maxOutputTokens: max_tokens,
+                topP: parameters.topP ?? 0.95,
+                topK: parameters.topK ?? 40,
+            }
+        };
+    
+        return requestParameters;
+    }
+
+    // Get the safetyAttributes from the PaLM Chat API response data
+    getSafetyAttributes(data) {
+        const { predictions } = data;
+        if (!predictions || !predictions.length) {
+            return null;
+        }
+
+        // if we got a predictions array back with more than one prediction, return the safetyAttributes of the first prediction
+        if (predictions.length > 1) {
+            return predictions[0].safetyAttributes ?? null;
+        }
+
+        // otherwise, return the safetyAttributes of the content of the first prediction
+        return predictions[0].safetyAttributes ?? null;
+    }
+
+    // Execute the request to the PaLM Chat API
+    async execute(text, parameters, prompt) {
+        const url = this.requestUrl(text);
+        const requestParameters = this.getRequestParameters(text, parameters, prompt);
+
+        const data = { ...(this.model.params || {}), ...requestParameters };
+        const params = {};
+        const headers = this.model.headers || {};
+        const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
+        const authToken = await gcpAuthTokenHelper.getAccessToken();
+        headers.Authorization = `Bearer ${authToken}`;
+        return this.executeRequest(url, data, params, headers, prompt);
+    }
+
+    // Parse the response from the PaLM Chat API
+    parseResponse(data) {
+        const { predictions } = data;
+        if (!predictions || !predictions.length) {
+            return null;
+        }
+    
+        // Get the candidates array from the first prediction
+        const { candidates } = predictions[0];
+
+        // if it was blocked, return the blocked message
+        if (predictions[0].safetyAttributes?.blocked) {
+            return 'The response is blocked because the input or response potentially violates Google policies. Try rephrasing the prompt or adjusting the parameter settings. Currently, only English is supported.';
+        }
+
+        if (!candidates || !candidates.length) {
+            return null;
+        }
+    
+        // If we got a candidates array back with more than one candidate, return the whole array
+        if (candidates.length > 1) {
+            return candidates;
+        }
+    
+        // Otherwise, return the content of the first candidate
+        const messageResult = candidates[0].content && candidates[0].content.trim();
+        return messageResult ?? null;
+    }
+
+    // Override the logging function to display the messages and responses
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+    
+        const instances = data && data.instances;
+        const messages = instances && instances[0] && instances[0].messages;
+        const { context, examples } = instances && instances [0] || {};
+    
+        if (context) {
+            console.log(`\x1b[36mContext: ${context}\x1b[0m`);
+        }
+
+        if (examples && examples.length) {
+            examples.forEach((example, index) => {
+                console.log(`\x1b[36mExample ${index + 1}: Input: "${example.input.content}", Output: "${example.output.content}"\x1b[0m`);
+            });
+        }
+        
+        if (messages && messages.length > 1) {
+            messages.forEach((message, index) => {
+                const words = message.content.split(" ");
+                const tokenCount = encode(message.content).length;
+                const preview = words.length < 41 ? message.content : words.slice(0, 20).join(" ") + " ... " + words.slice(-20).join(" ");
+    
+                console.log(`\x1b[36mMessage ${index + 1}: Author: ${message.author}, Tokens: ${tokenCount}, Content: "${preview}"\x1b[0m`);
+            });
+        } else if (messages && messages.length === 1) {
+            console.log(`\x1b[36m${messages[0].content}\x1b[0m`);
+        }
+
+        const safetyAttributes = this.getSafetyAttributes(responseData);
+
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+
+        if (safetyAttributes) {
+            console.log(`\x1b[33mSafety Attributes: ${JSON.stringify(safetyAttributes, null, 2)}\x1b[0m`);
+        }
+    
+        if (prompt && prompt.debugInfo) {
+            prompt.debugInfo += `${separator}${JSON.stringify(data)}`;
+        }
+    }
+}
+
+export default PalmChatPlugin;

package/graphql/plugins/palmCompletionPlugin.js

@@ -0,0 +1,134 @@
+// palmCompletionPlugin.js
+
+import ModelPlugin from './modelPlugin.js';
+
+// Helper function to truncate the prompt if it is too long
+const truncatePromptIfNecessary = (text, textTokenCount, modelMaxTokenCount, targetTextTokenCount, pathwayResolver) => {
+    const maxAllowedTokens = textTokenCount + ((modelMaxTokenCount - targetTextTokenCount) * 0.5);
+
+    if (textTokenCount > maxAllowedTokens) {
+        pathwayResolver.logWarning(`Prompt is too long at ${textTokenCount} tokens (this target token length for this pathway is ${targetTextTokenCount} tokens because the response is expected to take up the rest of the model's max tokens (${modelMaxTokenCount}). Prompt will be truncated.`);
+        return pathwayResolver.truncate(text, maxAllowedTokens);
+    }
+    return text;
+}
+
+// PalmCompletionPlugin class for handling requests and responses to the PaLM API Text Completion API
+class PalmCompletionPlugin extends ModelPlugin {
+    constructor(config, pathway) {
+        super(config, pathway);
+    }
+
+    // Set up parameters specific to the PaLM API Text Completion API
+    getRequestParameters(text, parameters, prompt, pathwayResolver) {
+        const { modelPromptText, tokenLength } = this.getCompiledPrompt(text, parameters, prompt);
+        const { stream } = parameters;
+        // Define the model's max token length
+        const modelTargetTokenLength = this.getModelMaxTokenLength() * this.getPromptTokenRatio();
+    
+        const truncatedPrompt = truncatePromptIfNecessary(modelPromptText, tokenLength, this.getModelMaxTokenLength(), modelTargetTokenLength, pathwayResolver);
+    
+        const max_tokens = 1024//this.getModelMaxTokenLength() - tokenLength;
+    
+        if (max_tokens < 0) {
+            throw new Error(`Prompt is too long to successfully call the model at ${tokenLength} tokens.  The model will not be called.`);
+        }
+
+        if (!truncatedPrompt) {
+            throw new Error(`Prompt is empty.  The model will not be called.`);
+        }
+    
+        const requestParameters = {
+            instances: [
+                { prompt: truncatedPrompt }
+            ],
+            parameters: {
+                temperature: this.temperature ?? 0.7,
+                maxOutputTokens: max_tokens,
+                topP: parameters.topP ?? 0.95,
+                topK: parameters.topK ?? 40,
+            }
+        };
+    
+        return requestParameters;
+    }
+
+    // Execute the request to the PaLM API Text Completion API
+    async execute(text, parameters, prompt, pathwayResolver) {
+        const url = this.requestUrl(text);
+        const requestParameters = this.getRequestParameters(text, parameters, prompt, pathwayResolver);
+
+        const data = { ...requestParameters };
+        const params = {};
+        const headers = this.model.headers || {};
+        const gcpAuthTokenHelper = this.config.get('gcpAuthTokenHelper');
+        const authToken = await gcpAuthTokenHelper.getAccessToken();
+        headers.Authorization = `Bearer ${authToken}`;
+        return this.executeRequest(url, data, params, headers, prompt);
+    }
+
+    // Parse the response from the PaLM API Text Completion API
+    parseResponse(data) {
+        const { predictions } = data;
+        if (!predictions || !predictions.length) {
+            return data;
+        }
+
+        // if we got a predictions array back with more than one prediction, return the whole array
+        if (predictions.length > 1) {
+            return predictions;
+        }
+
+        // otherwise, return the content of the first prediction
+        // if it was blocked, return the blocked message
+        if (predictions[0].safetyAttributes?.blocked) {
+            return 'The response is blocked because the input or response potentially violates Google policies. Try rephrasing the prompt or adjusting the parameter settings. Currently, only English is supported.';
+        }
+
+        const contentResult = predictions[0].content && predictions[0].content.trim();
+        return contentResult ?? null;
+    }
+
+    // Get the safetyAttributes from the PaLM API Text Completion API response data
+    getSafetyAttributes(data) {
+        const { predictions } = data;
+        if (!predictions || !predictions.length) {
+            return null;
+        }
+
+        // if we got a predictions array back with more than one prediction, return the safetyAttributes of the first prediction
+        if (predictions.length > 1) {
+            return predictions[0].safetyAttributes ?? null;
+        }
+
+        // otherwise, return the safetyAttributes of the content of the first prediction
+        return predictions[0].safetyAttributes ?? null;
+    }
+
+    // Override the logging function to log the prompt and response
+    logRequestData(data, responseData, prompt) {
+        const separator = `\n=== ${this.pathwayName}.${this.requestCount++} ===\n`;
+        console.log(separator);
+
+        const safetyAttributes = this.getSafetyAttributes(responseData);
+
+        const instances = data && data.instances;
+        const modelInput = instances && instances[0] && instances[0].prompt;
+
+        if (modelInput) {
+            console.log(`\x1b[36m${modelInput}\x1b[0m`);
+        }
+
+        console.log(`\x1b[34m> ${this.parseResponse(responseData)}\x1b[0m`);
+
+        if (safetyAttributes) {
+            console.log(`\x1b[33mSafety Attributes: ${JSON.stringify(safetyAttributes, null, 2)}\x1b[0m`);
+        }
+
+        if (prompt && prompt.debugInfo) {
+            prompt.debugInfo += `${separator}${JSON.stringify(data)}`;
+        }
+    }
+}
+
+export default PalmCompletionPlugin;

package/graphql/prompt.js

@@ -3,15 +3,21 @@ class Prompt {
         if (typeof params === 'string' || params instanceof String) {
             this.prompt = params;
         } else {
-            const { prompt, saveResultTo, messages } = params;
+            const { prompt, saveResultTo, messages, context, examples } = params;
             this.prompt = prompt;
             this.saveResultTo = saveResultTo;
             this.messages = messages;
+            this.context = context;
+            this.examples = examples;
             this.params = params;
         }
 
-        this.usesTextInput = promptContains('text', this.prompt ? this.prompt : this.messages);
-        this.usesPreviousResult = promptContains('previousResult', this.prompt ? this.prompt : this.messages);
+        this.usesTextInput = promptContains('text', this.prompt ? this.prompt : this.messages) ||
+                             (this.context && promptContains('text', this.context)) ||
+                             (this.examples && promptContains('text', this.examples));
+        this.usesPreviousResult = promptContains('previousResult', this.prompt ? this.prompt : this.messages) ||
+                                   (this.context && promptContains('previousResult', this.context)) ||
+                                   (this.examples && promptContains('previousResult', this.examples));
         this.debugInfo = '';
     }
 }
@@ -23,7 +29,8 @@ function promptContains(variable, prompt) {
     let matches = [];
     let match;
 
-    // if it's an array, it's the messages format
+    // if it's an array, it's either an OpenAI messages array or a PaLM messages
+    // array or a PaLM examples array, all of which have a content property
     if (Array.isArray(prompt)) {
       prompt.forEach(p => {
         // eslint-disable-next-line no-cond-assign

package/helper_apps/MediaFileChunker/Dockerfile

@@ -0,0 +1,20 @@
+FROM node:18-alpine
+
+WORKDIR /usr/src/app
+
+COPY package*.json ./
+
+RUN npm install
+
+## following 3 lines are for installing ffmepg
+RUN apk update
+RUN apk add
+RUN apk add ffmpeg
+
+COPY . .
+
+EXPOSE 7071
+
+# RUN npm run build
+
+CMD [ "node", "start.js" ]

package/helper_apps/MediaFileChunker/fileChunker.js

@@ -7,6 +7,8 @@ import { v4 as uuidv4 } from 'uuid';
 import os from 'os';
 import ytdl from 'ytdl-core';
 import { promisify } from 'util';
+import axios from 'axios';
+import { ensureEncoded } from './helper.js';
 
 ffmpeg.setFfmpegPath(ffmpegPath);
 ffmpeg.setFfprobePath(ffprobePath);
@@ -15,12 +17,15 @@ console.log(`ffprobePath: ${ffprobePath}`);
 
 const ffmpegProbe = promisify(ffmpeg.ffprobe);
 
+
 async function processChunk(inputPath, outputFileName, start, duration) {
     return new Promise((resolve, reject) => {
         ffmpeg(inputPath)
             .seekInput(start)
             .duration(duration)
-            .audioCodec('libmp3lame') // Ensure output is always in MP3 format
+            .format('mp3')
+            .audioCodec('libmp3lame')
+            .audioBitrate(128)
             .on('start', (cmd) => {
                 console.log(`Started FFmpeg with command: ${cmd}`);
             })
@@ -43,18 +48,57 @@ const generateUniqueFolderName = () => {
     return uniqueOutputPath;
 }
 
+async function downloadFile(url, outputPath) {
+    try {
+        // Make an HTTP request for the file
+        const response = await axios.get(url, { responseType: 'stream' });
+
+        // Create a writable file stream to save the file
+        const fileStream = fs.createWriteStream(outputPath);
+
+        // Pipe the response data into the file stream
+        response.data.pipe(fileStream);
+
+        // Wait for the file stream to finish writing
+        await new Promise((resolve, reject) => {
+            fileStream.on('finish', resolve);
+            fileStream.on('error', reject);
+        });
+
+        console.log(`Downloaded file saved to: ${outputPath}`);
+    } catch (error) {
+        console.error(`Error downloading file from ${url}:`, error);
+        throw error;
+    }
+}
+
 async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
     try {
+        // Create unique folder
+        const uniqueOutputPath = generateUniqueFolderName();
+        fs.mkdirSync(uniqueOutputPath, { recursive: true });
+
+        // Download the file if it's not a local file
+        const isUrl = /^(https?|ftp):\/\/[^\s/$.?#].[^\s]*$/i.test(inputPath);
+        if (isUrl) {
+            inputPath = ensureEncoded(inputPath);
+            // Extract the original file name from the URL
+            const urlObj = new URL(inputPath);
+            const originalFileName = path.basename(urlObj.pathname);
+
+            // Use the original file name when saving the downloaded file
+            const downloadPath = path.join(uniqueOutputPath, originalFileName);
+            await downloadFile(inputPath, downloadPath);
+            inputPath = downloadPath;
+        }
+
+        
         const metadata = await ffmpegProbe(inputPath);
         const duration = metadata.format.duration;
         const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
 
         const chunkPromises = [];
 
-        const uniqueOutputPath = generateUniqueFolderName();
-
-        // Create unique folder
-        fs.mkdirSync(uniqueOutputPath, { recursive: true });
 
 
         for (let i = 0; i < numChunks; i++) {
@@ -73,7 +117,7 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
             chunkPromises.push(chunkPromise);
         }
 
-        return { chunkPromises, uniqueOutputPath }
+        return { chunkPromises, uniqueOutputPath };
     } catch (err) {
         console.error('Error occurred during the splitting process:', err);
     }

package/helper_apps/MediaFileChunker/helper.js

@@ -28,6 +28,18 @@ async function deleteTempPath(path) {
     }
 }
 
+function ensureEncoded(url) {
+    try {
+        const decodedUrl = decodeURI(url);
+        if (decodedUrl === url) {
+            return encodeURI(url);
+        }
+        return url;
+    } catch (e) {
+        return url;
+    }
+}
+
 export {
-    isValidYoutubeUrl, deleteTempPath
+    isValidYoutubeUrl, deleteTempPath, ensureEncoded
 }

package/helper_apps/MediaFileChunker/index.js

@@ -1,7 +1,7 @@
 import { processYoutubeUrl, splitMediaFile } from './fileChunker.js';
 import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
 import { publishRequestProgress } from './redis.js';
-import { deleteTempPath, isValidYoutubeUrl } from './helper.js';
+import { deleteTempPath, ensureEncoded, isValidYoutubeUrl } from './helper.js';
 import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
 
 const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
@@ -10,7 +10,6 @@ console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
 
 async function main(context, req) {
     context.log('Starting req processing..');
-    // await publishRequestProgress({ requestId:222, progress: 0, data: null });
 
     // Clean up blob when request delete which means processing marked completed
     if (req.method.toLowerCase() === `delete`) {
@@ -48,7 +47,7 @@ async function main(context, req) {
     let completedCount = 0;
     let numberOfChunks;
 
-    let file = uri;
+    let file = ensureEncoded(uri); // encode url to handle special characters
     let folder;
     const isYoutubeUrl = isValidYoutubeUrl(uri);
 
@@ -111,6 +110,5 @@ async function main(context, req) {
     };
 }
 
-// main(console, { query: { uri: "https://www.youtube.com/watch?v=QH2-TGUlwu4" } });
 
 export default main;