@aj-archipelago/cortex 1.0.2 → 1.0.3

package/helper_apps/MediaFileChunker/blobHandler.js

@@ -0,0 +1,150 @@
+import fs from 'fs';
+import path from 'path';
+import { BlobServiceClient } from '@azure/storage-blob';
+import { v4 as uuidv4 } from 'uuid';
+import Busboy from 'busboy';
+import { PassThrough } from 'stream';
+import { pipeline as _pipeline } from 'stream';
+import { promisify } from 'util';
+const pipeline = promisify(_pipeline);
+import { join } from 'path';
+
+
+import { publicFolder, port, ipAddress } from "./start.js";
+
+const getBlobClient = () => {
+    const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
+    const containerName = process.env.AZURE_STORAGE_CONTAINER_NAME;
+    if (!connectionString || !containerName) {
+        throw new Error('Missing Azure Storage connection string or container name environment variable');
+    }
+
+    const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString);
+    const containerClient = blobServiceClient.getContainerClient(containerName);
+
+    return { blobServiceClient, containerClient };
+}
+
+async function saveFileToBlob(chunkPath, requestId) {
+    const { containerClient } = getBlobClient();
+    // Use the filename with a UUID as the blob name
+    const blobName = `${requestId}/${uuidv4()}_${path.basename(chunkPath)}`;
+
+    // Create a read stream for the chunk file
+    const fileStream = fs.createReadStream(chunkPath);
+
+    // Upload the chunk to Azure Blob Storage using the stream
+    const blockBlobClient = containerClient.getBlockBlobClient(blobName);
+    await blockBlobClient.uploadStream(fileStream);
+
+    // Return the full URI of the uploaded blob
+    const blobUrl = blockBlobClient.url;
+    return blobUrl;
+}
+
+//deletes blob that has the requestId
+async function deleteBlob(requestId) {
+    if (!requestId) throw new Error('Missing requestId parameter');
+    const { containerClient } = getBlobClient();
+    // List the blobs in the container with the specified prefix
+    const blobs = containerClient.listBlobsFlat({ prefix: `${requestId}/` });
+
+    const result = []
+    // Iterate through the blobs
+    for await (const blob of blobs) {
+        // Delete the matching blob
+        const blockBlobClient = containerClient.getBlockBlobClient(blob.name);
+        await blockBlobClient.delete();
+        console.log(`Cleaned blob: ${blob.name}`);
+        result.push(blob.name);
+    }
+
+    return result
+}
+
+async function uploadBlob(context, req, saveToLocal = false) {
+    return new Promise((resolve, reject) => {
+        try {
+            const busboy = Busboy({ headers: req.headers });
+            let requestId = uuidv4();
+
+            busboy.on('field', (fieldname, value) => {
+                if (fieldname === 'requestId') {
+                    requestId = value;
+                }
+            });
+
+            busboy.on('file', async (fieldname, file, info) => {
+                if (saveToLocal) {
+                    // Create the target folder if it doesn't exist
+                    const localPath = join(publicFolder, requestId);
+                    fs.mkdirSync(localPath, { recursive: true });
+
+                    const filename = `${uuidv4()}_${info.filename}`;
+                    const destinationPath = `${localPath}/${filename}`;
+
+                    await pipeline(file, fs.createWriteStream(destinationPath));
+
+                    const message = `File '${filename}' saved to folder successfully.`;
+                    context.log(message);
+
+                    const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
+
+                    const body = { message, url };
+
+                    context.res = {
+                        status: 200,
+                        body,
+                    };
+
+
+                    resolve(body); // Resolve the promise
+                } else {
+                    const { containerClient } = getBlobClient();
+                    const filename = `${requestId}/${uuidv4()}_${info.filename}`;
+
+                    const blockBlobClient = containerClient.getBlockBlobClient(filename);
+
+                    const passThroughStream = new PassThrough();
+                    file.pipe(passThroughStream);
+
+                    await blockBlobClient.uploadStream(passThroughStream);
+
+                    const message = `File '${filename}' uploaded successfully.`;
+                    const url = blockBlobClient.url;
+                    context.log(message);
+                    const body = { message, url };
+
+                    context.res = {
+                        status: 200,
+                        body,
+                    };
+
+                    resolve(body); // Resolve the promise
+                }
+            });
+
+            busboy.on('error', (error) => {
+                context.log.error('Error processing file upload:', error);
+                context.res = {
+                    status: 500,
+                    body: 'Error processing file upload.',
+                };
+                reject(error); // Reject the promise
+            });
+
+            req.pipe(busboy);
+        } catch (error) {
+            context.log.error('Error processing file upload:', error);
+            context.res = {
+                status: 500,
+                body: 'Error processing file upload.',
+            };
+            reject(error); // Reject the promise
+        }
+    });
+}
+
+export {
+    saveFileToBlob, deleteBlob, uploadBlob
+}

package/helper_apps/MediaFileChunker/fileChunker.js

@@ -0,0 +1,123 @@
+import fs from 'fs';
+import { path as ffmpegPath } from '@ffmpeg-installer/ffmpeg';
+import { path as ffprobePath } from '@ffprobe-installer/ffprobe';
+import path from 'path';
+import ffmpeg from 'fluent-ffmpeg';
+import { v4 as uuidv4 } from 'uuid';
+import os from 'os';
+import ytdl from 'ytdl-core';
+import { promisify } from 'util';
+
+ffmpeg.setFfmpegPath(ffmpegPath);
+ffmpeg.setFfprobePath(ffprobePath);
+console.log(`ffmpegPath: ${ffmpegPath}`);
+console.log(`ffprobePath: ${ffprobePath}`);
+
+const ffmpegProbe = promisify(ffmpeg.ffprobe);
+
+async function processChunk(inputPath, outputFileName, start, duration) {
+    return new Promise((resolve, reject) => {
+        ffmpeg(inputPath)
+            .seekInput(start)
+            .duration(duration)
+            .audioCodec('libmp3lame') // Ensure output is always in MP3 format
+            .on('start', (cmd) => {
+                console.log(`Started FFmpeg with command: ${cmd}`);
+            })
+            .on('error', (err) => {
+                console.error(`Error occurred while processing chunk:`, err);
+                reject(err);
+            })
+            .on('end', () => {
+                console.log(`Finished processing chunk`);
+                resolve(outputFileName);
+            })
+            .save(outputFileName);
+    });
+}
+
+const generateUniqueFolderName = () => {
+    const uniqueFolderName = uuidv4();
+    const tempFolderPath = os.tmpdir(); // Get the system's temporary folder
+    const uniqueOutputPath = path.join(tempFolderPath, uniqueFolderName);
+    return uniqueOutputPath;
+}
+
+async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
+    try {
+        const metadata = await ffmpegProbe(inputPath);
+        const duration = metadata.format.duration;
+        const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
+
+        const chunkPromises = [];
+
+        const uniqueOutputPath = generateUniqueFolderName();
+
+        // Create unique folder
+        fs.mkdirSync(uniqueOutputPath, { recursive: true });
+
+
+        for (let i = 0; i < numChunks; i++) {
+            const outputFileName = path.join(
+                uniqueOutputPath,
+                `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`
+            );
+
+            const chunkPromise = processChunk(
+                inputPath,
+                outputFileName,
+                i * chunkDurationInSeconds,
+                chunkDurationInSeconds
+            );
+
+            chunkPromises.push(chunkPromise);
+        }
+
+        return { chunkPromises, uniqueOutputPath }
+    } catch (err) {
+        console.error('Error occurred during the splitting process:', err);
+    }
+}
+
+const ytdlDownload = async (url, filename) => {
+    return new Promise((resolve, reject) => {
+        const video = ytdl(url, { quality: 'highestaudio' });
+        let lastLoggedTime = Date.now();
+
+        video.on('error', (error) => {
+            reject(error);
+        });
+
+        video.on('progress', (chunkLength, downloaded, total) => {
+            const currentTime = Date.now();
+            if (currentTime - lastLoggedTime >= 2000) { // Log every 2 seconds
+                const percent = downloaded / total;
+                console.log(`${(percent * 100).toFixed(2)}% downloaded ${url}`);
+                lastLoggedTime = currentTime;
+            }
+        });
+
+        video.pipe(fs.createWriteStream(filename))
+            .on('finish', () => {
+                resolve();
+            })
+            .on('error', (error) => {
+                reject(error);
+            });
+    });
+};
+
+const processYoutubeUrl = async (url) => {
+    try {
+        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
+        await ytdlDownload(url, outputFileName);
+        return outputFileName;
+    } catch (e) {
+        console.log(e);
+        throw e;
+    }
+}
+
+export {
+    splitMediaFile, processYoutubeUrl
+};

package/helper_apps/MediaFileChunker/function.json

@@ -0,0 +1,20 @@
+{
+  "bindings": [
+    {
+      "authLevel": "function",
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post",
+        "delete"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ]
+}

package/helper_apps/MediaFileChunker/helper.js

@@ -0,0 +1,33 @@
+import fs from 'fs';
+
+function isValidYoutubeUrl(url) {
+    const regex = /^(https?:\/\/)?(www\.)?(youtube\.com|youtu\.?be)\/.+$/;
+    return regex.test(url);
+}
+
+async function deleteTempPath(path) {
+    try {
+        if (!path) {
+            console.log('Temporary path is not defined.');
+            return;
+        }
+        if (!fs.existsSync(path)) {
+            console.log(`Temporary path ${path} does not exist.`);
+            return;
+        }
+        const stats = fs.statSync(path);
+        if (stats.isFile()) {
+            fs.unlinkSync(path);
+            console.log(`Temporary file ${path} deleted successfully.`);
+        } else if (stats.isDirectory()) {
+            fs.rmSync(path, { recursive: true });
+            console.log(`Temporary folder ${path} and its contents deleted successfully.`);
+        }
+    } catch (err) {
+        console.error('Error occurred while deleting the temporary path:', err);
+    }
+}
+
+export {
+    isValidYoutubeUrl, deleteTempPath
+}

package/helper_apps/MediaFileChunker/index.js

@@ -0,0 +1,116 @@
+import { processYoutubeUrl, splitMediaFile } from './fileChunker.js';
+import { saveFileToBlob, deleteBlob, uploadBlob } from './blobHandler.js';
+import { publishRequestProgress } from './redis.js';
+import { deleteTempPath, isValidYoutubeUrl } from './helper.js';
+import { moveFileToPublicFolder, deleteFolder } from './localFileHandler.js';
+
+const useAzure = process.env.AZURE_STORAGE_CONNECTION_STRING ? true : false;
+console.log(useAzure ? 'Using Azure Storage' : 'Using local file system');
+
+
+async function main(context, req) {
+    context.log('Starting req processing..');
+    // await publishRequestProgress({ requestId:222, progress: 0, data: null });
+
+    // Clean up blob when request delete which means processing marked completed
+    if (req.method.toLowerCase() === `delete`) {
+        const { requestId } = req.query;
+        if (!requestId) {
+            context.res = {
+                status: 400,
+                body: "Please pass a requestId on the query string"
+            };
+            return;
+        }
+        const result = useAzure ? await deleteBlob(requestId) : await deleteFolder(requestId);
+        context.res = {
+            body: result
+        };
+        return;
+    }
+
+    if (req.method.toLowerCase() === `post`) {
+        const { url } = await uploadBlob(context, req, !useAzure);
+        context.log(`File url: ${url}`);
+        return
+    }
+
+    const { uri, requestId } = req.body?.params || req.query;
+    if (!uri || !requestId) {
+        context.res = {
+            status: 400,
+            body: "Please pass a uri and requestId on the query string or in the request body"
+        };
+        return;
+    }
+
+    let totalCount = 0;
+    let completedCount = 0;
+    let numberOfChunks;
+
+    let file = uri;
+    let folder;
+    const isYoutubeUrl = isValidYoutubeUrl(uri);
+
+    const result = [];
+
+    const sendProgress = async (data = null) => {
+        completedCount++;
+        const progress = completedCount / totalCount;
+        await publishRequestProgress({ requestId, progress, completedCount, totalCount, numberOfChunks, data });
+    }
+
+    try {
+        if (isYoutubeUrl) {
+            // totalCount += 1; // extra 1 step for youtube download
+            file = await processYoutubeUrl(file);
+        }
+
+        const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+        folder = uniqueOutputPath;
+
+        numberOfChunks = chunkPromises.length; // for progress reporting
+        totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload)
+        // isYoutubeUrl && sendProgress(); // send progress for youtube download after total count is calculated
+
+        // sequential download of chunks
+        const chunks = [];
+        for (const chunkPromise of chunkPromises) {
+            chunks.push(await chunkPromise);
+            sendProgress();
+        }
+
+        // sequential processing of chunks
+        for (const chunk of chunks) {
+            const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
+            result.push(blobName);
+            context.log(`Saved chunk as: ${blobName}`);
+            sendProgress();
+        }
+
+        // parallel processing, dropped 
+        // result = await Promise.all(mediaSplit.chunks.map(processChunk));
+
+    } catch (error) {
+        console.error("An error occurred:", error);
+    } finally {
+        try {
+            isYoutubeUrl && (await deleteTempPath(file));
+            folder && (await deleteTempPath(folder));
+        } catch (error) {
+            console.error("An error occurred while deleting:", error);
+        }
+    }
+
+
+    console.log(`result: ${result}`);
+
+    context.res = {
+        // status: 200, /* Defaults to 200 */
+        body: result
+    };
+}
+
+// main(console, { query: { uri: "https://www.youtube.com/watch?v=QH2-TGUlwu4" } });
+
+export default main;

package/helper_apps/MediaFileChunker/localFileHandler.js

@@ -0,0 +1,36 @@
+import { promises as fs } from 'fs';
+import { join, basename } from 'path';
+import { v4 as uuidv4 } from 'uuid';
+
+import { publicFolder, port, ipAddress } from "./start.js";
+
+
+async function moveFileToPublicFolder(chunkPath, requestId) {
+    // Use the filename with a UUID as the blob name
+    const filename = `${requestId}/${uuidv4()}_${basename(chunkPath)}`;
+
+    // Create the target folder if it doesn't exist
+    const targetFolder = join(publicFolder, requestId);
+    await fs.mkdir(targetFolder, { recursive: true });
+
+    // Move the file to the target folder
+    const targetPath = join(targetFolder, basename(filename));
+    await fs.rename(chunkPath, targetPath);
+
+    // Return the complete URL of the file
+    const fileUrl = `http://${ipAddress}:${port}/files/${filename}`;
+    // const fileUrl = `http://localhost:${port}/files/${filename}`;
+    return fileUrl;
+}
+
+async function deleteFolder(requestId) {
+    if (!requestId) throw new Error('Missing requestId parameter');
+    const targetFolder = join(publicFolder, requestId);
+    await fs.rm(targetFolder, { recursive: true });
+    console.log(`Cleaned folder: ${targetFolder}`);
+}
+
+
+export {
+    moveFileToPublicFolder, deleteFolder
+};