@aj-archipelago/cortex 1.0.1 → 1.0.3

package/graphql/plugins/openAiWhisperPlugin.js

@@ -1,16 +1,90 @@
 // openAiWhisperPlugin.js
 import ModelPlugin from './modelPlugin.js';
-
 import FormData from 'form-data';
 import fs from 'fs';
-import { splitMediaFile, isValidYoutubeUrl, processYoutubeUrl, deleteTempPath } from '../../lib/fileChunker.js';
 import pubsub from '../pubsub.js';
+import { axios } from '../../lib/request.js';
+import stream from 'stream';
+import os from 'os';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import { config } from '../../config.js';
+import { deleteTempPath } from '../../helper_apps/MediaFileChunker/helper.js';
+import http from 'http';
+import https from 'https';
+import url from 'url';
+import { promisify } from 'util';
+const pipeline = promisify(stream.pipeline);
+
+
+const API_URL = config.get('whisperMediaApiUrl');
+
+function generateUniqueFilename(extension) {
+    return `${uuidv4()}.${extension}`;
+}
+
+const downloadFile = async (fileUrl) => {
+    const fileExtension = path.extname(fileUrl).slice(1);
+    const uniqueFilename = generateUniqueFilename(fileExtension);
+    const tempDir = os.tmpdir();
+    const localFilePath = `${tempDir}/${uniqueFilename}`;
+
+    // eslint-disable-next-line no-async-promise-executor
+    return new Promise(async (resolve, reject) => {
+        try {
+            const parsedUrl = url.parse(fileUrl);
+            const protocol = parsedUrl.protocol === 'https:' ? https : http;
+
+            const response = await new Promise((resolve, reject) => {
+                protocol.get(parsedUrl, (res) => {
+                    if (res.statusCode === 200) {
+                        resolve(res);
+                    } else {
+                        reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
+                    }
+                }).on('error', reject);
+            });
+
+            await pipeline(response, fs.createWriteStream(localFilePath));
+            console.log(`Downloaded file to ${localFilePath}`);
+            resolve(localFilePath);
+        } catch (error) {
+            fs.unlink(localFilePath, () => {
+                reject(error);
+            });
+        }
+    });
+};
 
 class OpenAIWhisperPlugin extends ModelPlugin {
     constructor(config, pathway) {
         super(config, pathway);
     }
 
+    async getMediaChunks(file, requestId) {
+        try {
+            if (API_URL) {
+                //call helper api and get list of file uris
+                const res = await axios.get(API_URL, { params: { uri: file, requestId } });
+                return res.data;
+            } else {
+                console.log(`No API_URL set, returning file as chunk`);
+                return [file];
+            }
+        } catch (err) {
+            console.log(`Error getting media chunks list from api:`, err);
+        }
+    }
+
+    async markCompletedForCleanUp(requestId) {
+        if (API_URL) {
+            //call helper api to mark processing as completed
+            const res = await axios.delete(API_URL, { params: { requestId } });
+            console.log(`Marked request ${requestId} as completed:`, res.data);
+            return res.data;
+        }
+    }
+
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, pathwayResolver) {
         const url = this.requestUrl(text);
@@ -19,11 +93,12 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
         const processChunk = async (chunk) => {
             try {
+                const { language } = parameters;
                 const formData = new FormData();
                 formData.append('file', fs.createReadStream(chunk));
                 formData.append('model', this.model.params.model);
                 formData.append('response_format', 'text');
-                // formData.append('language', 'tr');
+                language && formData.append('language', language);
                 modelPromptText && formData.append('prompt', modelPromptText);
 
                 return this.executeRequest(url, formData, params, { ...this.model.headers, ...formData.getHeaders() });
@@ -34,14 +109,13 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
         let result = ``;
         let { file } = parameters;
-        let folder;
-        const isYoutubeUrl = isValidYoutubeUrl(file);
         let totalCount = 0;
         let completedCount = 0;
         const { requestId } = pathwayResolver;
 
         const sendProgress = () => {
             completedCount++;
+            if (completedCount >= totalCount) return;
             pubsub.publish('REQUEST_PROGRESS', {
                 requestProgress: {
                     requestId,
@@ -51,24 +125,23 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             });
         }
 
+        let chunks = []; // array of local file paths
         try {
-            if (isYoutubeUrl) {
-                // totalCount += 1; // extra 1 step for youtube download
-                file = await processYoutubeUrl(file);
-            }
 
-            const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
-            folder = uniqueOutputPath;
-            totalCount += chunkPromises.length * 2; // 2 steps for each chunk (download and upload)
-            // isYoutubeUrl && sendProgress(); // send progress for youtube download after total count is calculated
+            const uris = await this.getMediaChunks(file, requestId); // array of remote file uris
+            if (!uris || !uris.length) {
+                throw new Error(`Error in getting chunks from media helper for file ${file}`);
+            }
+            totalCount = uris.length * 4; // 4 steps for each chunk (download and upload)
+            API_URL && (completedCount = uris.length); // api progress is already calculated
 
             // sequential download of chunks
-            const chunks = [];
-            for (const chunkPromise of chunkPromises) {
+            for (const uri of uris) {
+                chunks.push(await downloadFile(uri));
                 sendProgress();
-                chunks.push(await chunkPromise);
             }
 
+
             // sequential processing of chunks
             for (const chunk of chunks) {
                 result += await processChunk(chunk);
@@ -80,9 +153,29 @@ class OpenAIWhisperPlugin extends ModelPlugin {
 
         } catch (error) {
             console.error("An error occurred:", error);
-        } finally {
-            isYoutubeUrl && (await deleteTempPath(file));
-            folder && (await deleteTempPath(folder));
+        }
+        finally {
+            // isYoutubeUrl && (await deleteTempPath(file));
+            // folder && (await deleteTempPath(folder));
+            try {
+                for (const chunk of chunks) {
+                    await deleteTempPath(chunk);
+                }
+
+                await this.markCompletedForCleanUp(requestId);
+
+                //check cleanup for whisper temp uploaded files url
+                const regex = /whispertempfiles\/([a-z0-9-]+)/;
+                const match = file.match(regex);
+                if (match && match[1]) {
+                    const extractedValue = match[1];
+                    await this.markCompletedForCleanUp(extractedValue);
+                    console.log(`Cleaned temp whisper file ${file} with request id ${extractedValue}`);
+                }
+
+            } catch (error) {
+                console.error("An error occurred while deleting:", error);
+            }
         }
         return result;
     }

package/graphql/prompt.js

@@ -26,6 +26,7 @@ function promptContains(variable, prompt) {
     // if it's an array, it's the messages format
     if (Array.isArray(prompt)) {
       prompt.forEach(p => {
+        // eslint-disable-next-line no-cond-assign
         while (match = p.content && regexp.exec(p.content)) {
           matches.push(match[1]);
         }

package/graphql/resolver.js

@@ -26,12 +26,12 @@ const rootResolver = async (parent, args, contextValue, info) => {
 }
 
 // This resolver is used by the root resolver to process the request
-const resolver = async (parent, args, contextValue, info) => {
+const resolver = async (parent, args, contextValue, _info) => {
     const { pathwayResolver } = contextValue;
     return await pathwayResolver.resolve(args);
 }
 
-const cancelRequestResolver = (parent, args, contextValue, info) => {
+const cancelRequestResolver = (parent, args, contextValue, _info) => {
     const { requestId } = args;
     const { requestState } = contextValue;
     requestState[requestId] = { canceled: true };

package/graphql/subscriptions.js

@@ -10,7 +10,7 @@ import { requestState } from './requestState.js';
 const subscriptions = {
     requestProgress: {
         subscribe: withFilter(
-            (_, args, __, info) => {
+            (_, args, __, _info) => {
                 const { requestIds } = args;
                 for (const requestId of requestIds) {
                     if (!requestState[requestId]) {

package/helper_apps/MediaFileChunker/blobHandler.js

@@ -0,0 +1,150 @@
+import fs from 'fs';
+import path from 'path';
+import { BlobServiceClient } from '@azure/storage-blob';
+import { v4 as uuidv4 } from 'uuid';
+import Busboy from 'busboy';
+import { PassThrough } from 'stream';
+import { pipeline as _pipeline } from 'stream';
+import { promisify } from 'util';
+const pipeline = promisify(_pipeline);
+import { join } from 'path';
+
+
+import { publicFolder, port, ipAddress } from "./start.js";
+
+const getBlobClient = () => {
+    const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+    const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
+    if (!connectionString || !containerName) {
+        throw new Error('Missing Azure Storage connection string or container name environment variable');
+    }
+
+    const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+    const containerClient = blobServiceClient.getContainerClient(containerName);
+
+    return { blobServiceClient, containerClient };
+}
+
+async function saveFileToBlob(chunkPath, requestId) {
+    const { containerClient } = getBlobClient();
+    // Use the filename with a UUID as the blob name
+    const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+
+    // Create a read stream for the chunk file
+    const fileStream = fs.createReadStream(chunkPath);
+
+    // Upload the chunk to Azure Blob Storage using the stream
+    const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+    await blockBlobClient.uploadStream(fileStream);
+
+    // Return the full URI of the uploaded blob
+    const blobUrl = blockBlobClient.url;
+    return blobUrl;
+}
+
+//deletes blob that has the requestId
+async function deleteBlob(requestId) {
+    if (!requestId) throw new Error('Missing requestId parameter');
+    const { containerClient } = getBlobClient();
+    // List the blobs in the container with the specified prefix
+    const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
+
+    const result = []
+    // Iterate through the blobs
+    for await (const blob of blobs) {
+        // Delete the matching blob
+        const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
+        await blockBlobClient.delete();
+        console.log(`Cleaned blob: ${blob.name}`);
+        result.push(blob.name);
+    }
+
+    return result
+}
+
+async function uploadBlob(context, req, saveToLocal = false) {
+    return new Promise((resolve, reject) => {
+        try {
+            const busboy = Busboy({ headers: req.headers });
+            let requestId = uuidv4();
+
+            busboy.on('field', (fieldname, value) => {
+                if (fieldname === 'requestId') {
+                    requestId = value;
+                }
+            });
+
+            busboy.on('file', async (fieldname, file, info) => {
+                if (saveToLocal) {
+                    // Create the target folder if it doesn't exist
+                    const localPath = join(publicFolder, requestId);
+                    fs.mkdirSync(localPath, { recursive: true });
+
+                    const filename = `${uuidv4()}_${info.filename}`;
+                    const destinationPath = `${localPath}/${filename}`;
+
+                    await pipeline(file, fs.createWriteStream(destinationPath));
+
+                    const message = `File '${filename}' saved to folder successfully.`;
+                    context.log(message);
+
+                    const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
+
+                    const body = { message, url };
+
+                    context.res = {
+                        status: 200,
+                        body,
+                    };
+
+
+                    resolve(body); // Resolve the promise
+                } else {
+                    const { containerClient } = getBlobClient();
+                    const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+
+                    const blockBlobClient = containerClient.getBlockBlobClient(filename);
+
+                    const passThroughStream = new PassThrough();
+                    file.pipe(passThroughStream);
+
+                    await blockBlobClient.uploadStream(passThroughStream);
+
+                    const message = `File '${filename}' uploaded successfully.`;
+                    const url = blockBlobClient.url;
+                    context.log(message);
+                    const body = { message, url };
+
+                    context.res = {
+                        status: 200,
+                        body,
+                    };
+
+                    resolve(body); // Resolve the promise
+                }
+            });
+
+            busboy.on('error', (error) => {
+                context.log.error('Error processing file upload:', error);
+                context.res = {
+                    status: 500,
+                    body: 'Error processing file upload.',
+                };
+                reject(error); // Reject the promise
+            });
+
+            req.pipe(busboy);
+        } catch (error) {
+            context.log.error('Error processing file upload:', error);
+            context.res = {
+                status: 500,
+                body: 'Error processing file upload.',
+            };
+            reject(error); // Reject the promise
+        }
+    });
+}
+
+export {
+    saveFileToBlob, deleteBlob, uploadBlob
+}

package/helper_apps/MediaFileChunker/fileChunker.js

@@ -0,0 +1,123 @@
+import fs from 'fs';
+import { path as ffmpegPath } from '@ffmpeg-installer/ffmpeg';
+import { path as ffprobePath } from '@ffprobe-installer/ffprobe';
+import path from 'path';
+import ffmpeg from 'fluent-ffmpeg';
+import { v4 as uuidv4 } from 'uuid';
+import os from 'os';
+import ytdl from 'ytdl-core';
+import { promisify } from 'util';
+
+ffmpeg.setFfmpegPath(ffmpegPath);
+ffmpeg.setFfprobePath(ffprobePath);
+console.log(`ffmpegPath: ${ffmpegPath}`);
+console.log(`ffprobePath: ${ffprobePath}`);
+
+const ffmpegProbe = promisify(ffmpeg.ffprobe);
+
+async function processChunk(inputPath, outputFileName, start, duration) {
+    return new Promise((resolve, reject) => {
+        ffmpeg(inputPath)
+            .seekInput(start)
+            .duration(duration)
+            .audioCodec('libmp3lame') // Ensure output is always in MP3 format
+            .on('start', (cmd) => {
+                console.log(`Started FFmpeg with command: ${cmd}`);
+            })
+            .on('error', (err) => {
+                console.error(`Error occurred while processing chunk:`, err);
+                reject(err);
+            })
+            .on('end', () => {
+                console.log(`Finished processing chunk`);
+                resolve(outputFileName);
+            })
+            .save(outputFileName);
+    });
+}
+
+const generateUniqueFolderName = () => {
+    const uniqueFolderName = uuidv4();
+    const tempFolderPath = os.tmpdir(); // Get the system's temporary folder
+    const uniqueOutputPath = path.join(tempFolderPath, uniqueFolderName);
+    return uniqueOutputPath;
+}
+
+async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
+    try {
+        const metadata = await ffmpegProbe(inputPath);
+        const duration = metadata.format.duration;
+        const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
+
+        const chunkPromises = [];
+
+        const uniqueOutputPath = generateUniqueFolderName();
+
+        // Create unique folder
+        fs.mkdirSync(uniqueOutputPath, { recursive: true });
+
+
+        for (let i = 0; i < numChunks; i++) {
+            const outputFileName = path.join(
+                uniqueOutputPath,
+                `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`
+            );
+
+            const chunkPromise = processChunk(
+                inputPath,
+                outputFileName,
+                i * chunkDurationInSeconds,
+                chunkDurationInSeconds
+            );
+
+            chunkPromises.push(chunkPromise);
+        }
+
+        return { chunkPromises, uniqueOutputPath }
+    } catch (err) {
+        console.error('Error occurred during the splitting process:', err);
+    }
+}
+
+const ytdlDownload = async (url, filename) => {
+    return new Promise((resolve, reject) => {
+        const video = ytdl(url, { quality: 'highestaudio' });
+        let lastLoggedTime = Date.now();
+
+        video.on('error', (error) => {
+            reject(error);
+        });
+
+        video.on('progress', (chunkLength, downloaded, total) => {
+            const currentTime = Date.now();
+            if (currentTime - lastLoggedTime >= 2000) { // Log every 2 seconds
+                const percent = downloaded / total;
+                console.log(`${(percent * 100).toFixed(2)}% downloaded ${url}`);
+                lastLoggedTime = currentTime;
+            }
+        });
+
+        video.pipe(fs.createWriteStream(filename))
+            .on('finish', () => {
+                resolve();
+            })
+            .on('error', (error) => {
+                reject(error);
+            });
+    });
+};
+
+const processYoutubeUrl = async (url) => {
+    try {
+        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
+        await ytdlDownload(url, outputFileName);
+        return outputFileName;
+    } catch (e) {
+        console.log(e);
+        throw e;
+    }
+}
+
+export {
+    splitMediaFile, processYoutubeUrl
+};

package/helper_apps/MediaFileChunker/function.json

@@ -0,0 +1,20 @@
+{
+  "bindings": [
+    {
+      "authLevel": "function",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post",
+        "delete"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ]
+}

package/helper_apps/MediaFileChunker/helper.js

@@ -0,0 +1,33 @@
+import fs from 'fs';
+
+function isValidYoutubeUrl(url) {
+    const regex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
+    return regex.test(url);
+}
+
+async function deleteTempPath(path) {
+    try {
+        if (!path) {
+            console.log('Temporary path is not defined.');
+            return;
+        }
+        if (!fs.existsSync(path)) {
+            console.log(`Temporary path ${path} does not exist.`);
+            return;
+        }
+        const stats = fs.statSync(path);
+        if (stats.isFile()) {
+            fs.unlinkSync(path);
+            console.log(`Temporary file ${path} deleted successfully.`);
+        } else if (stats.isDirectory()) {
+            fs.rmSync(path, { recursive: true });
+            console.log(`Temporary folder ${path} and its contents deleted successfully.`);
+        }
+    } catch (err) {
+        console.error('Error occurred while deleting the temporary path:', err);
+    }
+}
+
+export {
+    isValidYoutubeUrl, deleteTempPath
+}

package/helper_apps/MediaFileChunker/index.js

@@ -0,0 +1,116 @@
+import { processYoutubeUrl, splitMediaFile } from './fileChunker.js';
+import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
+import { publishRequestProgress } from './redis.js';
+import { deleteTempPath, isValidYoutubeUrl } from './helper.js';
+import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
+
+const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
+console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
+
+
+async function main(context, req) {
+    context.log('Starting req processing..');
+    // await publishRequestProgress({ requestId:222, progress: 0, data: null });
+
+    // Clean up blob when request delete which means processing marked completed
+    if (req.method.toLowerCase() === `delete`) {
+        const { requestId } = req.query;
+        if (!requestId) {
+            context.res = {
+                status: 400,
+                body: "Please pass a requestId on the query string"
+            };
+            return;
+        }
+        const result = useAzure ? await deleteBlob(requestId) : await deleteFolder(requestId);
+        context.res = {
+            body: result
+        };
+        return;
+    }
+
+    if (req.method.toLowerCase() === `post`) {
+        const { url } = await uploadBlob(context, req, !useAzure);
+        context.log(`File url: ${url}`);
+        return
+    }
+
+    const { uri, requestId } = req.body?.params || req.query;
+    if (!uri || !requestId) {
+        context.res = {
+            status: 400,
+            body: "Please pass a uri and requestId on the query string or in the request body"
+        };
+        return;
+    }
+
+    let totalCount = 0;
+    let completedCount = 0;
+    let numberOfChunks;
+
+    let file = uri;
+    let folder;
+    const isYoutubeUrl = isValidYoutubeUrl(uri);
+
+    const result = [];
+
+    const sendProgress = async (data = null) => {
+        completedCount++;
+        const progress = completedCount / totalCount;
+        await publishRequestProgress({ requestId, progress, completedCount, totalCount, numberOfChunks, data });
+    }
+
+    try {
+        if (isYoutubeUrl) {
+            // totalCount += 1; // extra 1 step for youtube download
+            file = await processYoutubeUrl(file);
+        }
+
+        const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+        folder = uniqueOutputPath;
+
+        numberOfChunks = chunkPromises.length; // for progress reporting
+        totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload)
+        // isYoutubeUrl && sendProgress(); // send progress for youtube download after total count is calculated
+
+        // sequential download of chunks
+        const chunks = [];
+        for (const chunkPromise of chunkPromises) {
+            chunks.push(await chunkPromise);
+            sendProgress();
+        }
+
+        // sequential processing of chunks
+        for (const chunk of chunks) {
+            const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
+            result.push(blobName);
+            context.log(`Saved chunk as: ${blobName}`);
+            sendProgress();
+        }
+
+        // parallel processing, dropped 
+        // result = await Promise.all(mediaSplit.chunks.map(processChunk));
+
+    } catch (error) {
+        console.error("An error occurred:", error);
+    } finally {
+        try {
+            isYoutubeUrl && (await deleteTempPath(file));
+            folder && (await deleteTempPath(folder));
+        } catch (error) {
+            console.error("An error occurred while deleting:", error);
+        }
+    }
+
+
+    console.log(`result: ${result}`);
+
+    context.res = {
+        // status: 200, /* Defaults to 200 */
+        body: result
+    };
+}
+
+// main(console, { query: { uri: "https://www.youtube.com/watch?v=QH2-TGUlwu4" } });
+
+export default main;