@aj-archipelago/cortex 1.1.12 → 1.1.14

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -87,32 +87,24 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
             inputPath = downloadPath;
         }
 
-        
         const metadata = await ffmpegProbe(inputPath);
         const duration = metadata.format.duration;
         const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
 
         const chunkPromises = [];
-
-
+        const chunkOffsets = [];
 
         for (let i = 0; i < numChunks; i++) {
-            const outputFileName = path.join(
-                uniqueOutputPath,
-                `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`
-            );
-
-            const chunkPromise = processChunk(
-                inputPath,
-                outputFileName,
-                i * chunkDurationInSeconds,
-                chunkDurationInSeconds
-            );
+            const outputFileName = path.join(uniqueOutputPath, `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`);
+            const offset = i * chunkDurationInSeconds;
 
+            const chunkPromise = processChunk(inputPath, outputFileName, offset, chunkDurationInSeconds);
+            
             chunkPromises.push(chunkPromise);
+            chunkOffsets.push(offset);
         }
 
-        return { chunkPromises, uniqueOutputPath };
+        return { chunkPromises, chunkOffsets, uniqueOutputPath }; 
     } catch (err) {
         const msg = `Error processing media file, check if the file is a valid media file or is accessible`;
         console.error(msg, err);

package/helper-apps/cortex-file-handler/index.js

@@ -143,7 +143,7 @@ async function main(context, req) {
                 file = await processYoutubeUrl(file);
             }
 
-            const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+            const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);
             folder = uniqueOutputPath;
 
             numberOfChunks = chunkPromises.length; // for progress reporting
@@ -158,9 +158,11 @@ async function main(context, req) {
             }
 
             // sequential processing of chunks
-            for (const chunk of chunks) {
+            for (let index = 0; index < chunks.length; index++) {
+                const chunk = chunks[index];
                 const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
-                result.push(blobName);
+                const chunkOffset = chunkOffsets[index];
+                result.push({ uri:blobName, offset:chunkOffset });
                 context.log(`Saved chunk as: ${blobName}`);
                 sendProgress();
             }
@@ -182,7 +184,10 @@ async function main(context, req) {
         }
     }
 
-    console.log(`result: ${result}`);
+    console.log('result:', result.map(item =>
+        typeof item === 'object' ? JSON.stringify(item, null, 2) : item
+    ).join('\n'));
+    
     context.res = {
         body: result
     };

package/lib/requestExecutor.js

@@ -323,8 +323,13 @@ const makeRequest = async (cortexRequest) => {
                         return { response, duration };
                     }
                 } else {
-                    // if there are multiple endpoints, retry everything as it
-                    // could be going to a different host
+                    // if there are multiple endpoints, retry everything by default
+                    // as it could be a temporary issue with one endpoint
+                    // certain errors (e.g. 400) are problems with the request itself
+                    // and should not be retried
+                    if (status == 400) {
+                        return { response, duration };
+                    }
                     cortexRequest.selectNewEndpoint();
                 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.12",
+  "version": "1.1.14",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/modelPlugin.js

@@ -276,7 +276,9 @@ class ModelPlugin {
             
             const errorData = Array.isArray(responseData) ? responseData[0] : responseData;
             if (errorData && errorData.error) {
-                throw new Error(`Server error: ${JSON.stringify(errorData.error)}`);
+                const newError = new Error(errorData.error.message);
+                newError.data = errorData;
+                throw newError;
             }
         
             this.logAIRequestFinished(requestDuration);

package/server/plugins/openAiDallE3Plugin.js

@@ -55,8 +55,12 @@ class OpenAIDallE3Plugin extends ModelPlugin {
         .catch((error) => handleResponse(error));
 
         function handleResponse(response) {
-            const status = response?.error ? "failed" : "succeeded";
-            const data = JSON.stringify(response?.error ? response : response);
+            let status = "succeeded";
+            let data = JSON.stringify(response);
+            if (response.data.error) {
+                status = "failed";
+                data = JSON.stringify(response.data);
+            }
 
             const requestProgress = {
                 requestId,

package/server/plugins/openAiWhisperPlugin.js

@@ -15,6 +15,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { promisify } from 'util';
 import { publishRequestProgress } from '../../lib/redisSubscription.js';
 import logger from '../../lib/logger.js';
+import CortexRequest from '../../lib/cortexRequest.js';
 const pipeline = promisify(stream.pipeline);
 
 const API_URL = config.get('whisperMediaApiUrl');
@@ -25,7 +26,7 @@ if(WHISPER_TS_API_URL){
     logger.warn(`WHISPER API URL not set using default OpenAI API Whisper`);
 }
 
-const OFFSET_CHUNK = 1000 * 500; // 500 seconds chunk offset
+const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
 async function deleteTempPath(path) {
     try {
@@ -96,7 +97,7 @@ function convertToText(str) {
       .join(' ');
 }
 
-function alignSubtitles(subtitles, format) {
+function alignSubtitles(subtitles, format, offsets) {
     const result = [];
 
     function preprocessStr(str) {
@@ -116,7 +117,7 @@ function alignSubtitles(subtitles, format) {
     }
 
     for (let i = 0; i < subtitles.length; i++) {
-        result.push(...shiftSubtitles(subtitles[i], i * OFFSET_CHUNK));
+        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
     }
     
     try {
@@ -171,12 +172,14 @@ class OpenAIWhisperPlugin extends ModelPlugin {
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, cortexRequest) {
         const { pathwayResolver } = cortexRequest;
+
         const { responseFormat, wordTimestamped, highlightWords, maxLineWidth, maxLineCount, maxWordsPerLine } = parameters;
-        cortexRequest.url = this.requestUrl(text);
 
         const chunks = [];
         const processChunk = async (uri) => {
             try {
+                const cortexRequest = new CortexRequest({ pathwayResolver });
+
                 const chunk = await downloadFile(uri);
                 chunks.push(chunk);
 
@@ -205,6 +208,8 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         const processTS = async (uri) => {
+            const cortexRequest = new CortexRequest({ pathwayResolver });
+
             const tsparams = { fileurl:uri };
             const { language } = parameters;
             if(language) tsparams.language = language;
@@ -315,14 +320,20 @@ async function processURI(uri) {
     return result;
 }
 
+let offsets = [];
+let uris = []
+
 try {
-    const uris = await this.getMediaChunks(file, requestId);
+    const mediaChunks = await this.getMediaChunks(file, requestId);
     
-    if (!uris || !uris.length) {
+    if (!mediaChunks || !mediaChunks.length) {
         throw new Error(`Error in getting chunks from media helper for file ${file}`);
     }
 
-    totalCount = uris.length + 1; // total number of chunks that will be processed
+    uris = mediaChunks.map((chunk) => chunk?.uri || chunk);
+    offsets = mediaChunks.map((chunk, index) => chunk?.offset || index * OFFSET_CHUNK);
+
+    totalCount = mediaChunks.length + 1; // total number of chunks that will be processed
 
     const batchSize = 2;
     sendProgress();
@@ -369,7 +380,7 @@ try {
         }
 
         if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
-            return alignSubtitles(result, responseFormat);
+            return alignSubtitles(result, responseFormat, offsets);
         }
         return result.join(` `);
     }

package/server/plugins/openAiWhisperPlugin_parallel.js

@@ -1,358 +0,0 @@
-// openAiWhisperPlugin.js
-import ModelPlugin from './modelPlugin.js';
-import { config } from '../../config.js';
-import subsrt from 'subsrt';
-import FormData from 'form-data';
-import fs from 'fs';
-import { axios } from '../../lib/requestExecutor.js';
-import stream from 'stream';
-import os from 'os';
-import path from 'path';
-import http from 'http';
-import https from 'https';
-import { URL } from 'url';
-import { v4 as uuidv4 } from 'uuid';
-import { promisify } from 'util';
-import { publishRequestProgress } from '../../lib/redisSubscription.js';
-import logger from '../../lib/logger.js';
-const pipeline = promisify(stream.pipeline);
-
-const API_URL = config.get('whisperMediaApiUrl');
-const WHISPER_TS_API_URL  = config.get('whisperTSApiUrl');
-if(WHISPER_TS_API_URL){
-    logger.info(`WHISPER API URL using ${WHISPER_TS_API_URL}`);
-}else{
-    logger.warn(`WHISPER API URL not set using default OpenAI API Whisper`);
-}
-
-const OFFSET_CHUNK = 1000 * 60 * 10; // 10 minutes for each chunk
-
-async function deleteTempPath(path) {
-    try {
-        if (!path) {
-            logger.warn('Temporary path is not defined.');
-            return;
-        }
-        if (!fs.existsSync(path)) {
-            logger.warn(`Temporary path ${path} does not exist.`);
-            return;
-        }
-        const stats = fs.statSync(path);
-        if (stats.isFile()) {
-            fs.unlinkSync(path);
-            logger.info(`Temporary file ${path} deleted successfully.`);
-        } else if (stats.isDirectory()) {
-            fs.rmSync(path, { recursive: true });
-            logger.info(`Temporary folder ${path} and its contents deleted successfully.`);
-        }
-    } catch (err) {
-        logger.error(`Error occurred while deleting the temporary path: ${err}`);
-    }
-}
-
-function generateUniqueFilename(extension) {
-    return `${uuidv4()}.${extension}`;
-}
-
-const downloadFile = async (fileUrl) => {
-    const fileExtension = path.extname(fileUrl).slice(1);
-    const uniqueFilename = generateUniqueFilename(fileExtension);
-    const tempDir = os.tmpdir();
-    const localFilePath = `${tempDir}/${uniqueFilename}`;
-
-    // eslint-disable-next-line no-async-promise-executor
-    return new Promise(async (resolve, reject) => {
-        try {
-            const parsedUrl = new URL(fileUrl);
-            const protocol = parsedUrl.protocol === 'https:' ? https : http;
-
-            const response = await new Promise((resolve, reject) => {
-                protocol.get(parsedUrl, (res) => {
-                    if (res.statusCode === 200) {
-                        resolve(res);
-                    } else {
-                        reject(new Error(`HTTP request failed with status code ${res.statusCode}`));
-                    }
-                }).on('error', reject);
-            });
-
-            await pipeline(response, fs.createWriteStream(localFilePath));
-            logger.info(`Downloaded file to ${localFilePath}`);
-            resolve(localFilePath);
-        } catch (error) {
-            fs.unlink(localFilePath, () => {
-                reject(error);
-            });
-            //throw error;
-        }
-    });
-};
-
-// convert srt format to text
-function convertToText(str) {
-    return str
-      .split('\n')
-      .filter(line => !line.match(/^\d+$/) && !line.match(/^\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}$/) && line !== '')
-      .join(' ');
-}
-
-function alignSubtitles(subtitles, format) {
-    const result = [];
-
-    function preprocessStr(str) {
-        try{
-            if(!str) return '';
-            return str.trim().replace(/(\n\n)(?!\n)/g, '\n\n\n');
-        }catch(e){
-            logger.error(`An error occurred in content text preprocessing: ${e}`);
-            return '';
-        }
-    }
-
-    function shiftSubtitles(subtitle, shiftOffset) {
-        const captions = subsrt.parse(preprocessStr(subtitle));
-        const resynced = subsrt.resync(captions, { offset: shiftOffset });
-        return resynced;
-    }
-
-    for (let i = 0; i < subtitles.length; i++) {
-        result.push(...shiftSubtitles(subtitles[i], i * OFFSET_CHUNK));
-    }
-    
-    try {
-        //if content has needed html style tags, keep them
-        for(const obj of result) {
-            if(obj && obj.content){ 
-                obj.text = obj.content;
-            }
-        }
-    } catch (error) {
-        logger.error(`An error occurred in content text parsing: ${error}`);
-    }
-    
-    return subsrt.build(result, { format: format === 'vtt' ? 'vtt' : 'srt' });
-}
-
-
-class OpenAIWhisperPlugin extends ModelPlugin {
-    constructor(pathway, model) {
-        super(pathway, model);
-    }
-
-    async getMediaChunks(file, requestId) {
-        try {
-            if (API_URL) {
-                //call helper api and get list of file uris
-                const res = await axios.get(API_URL, { params: { uri: file, requestId } });
-                return res.data;
-            } else {
-                logger.info(`No API_URL set, returning file as chunk`);
-                return [file];
-            }
-        } catch (err) {
-            logger.error(`Error getting media chunks list from api: ${err}`);
-            throw err;
-        }
-    }
-
-    async markCompletedForCleanUp(requestId) {
-        try {
-            if (API_URL) {
-                //call helper api to mark processing as completed
-                const res = await axios.delete(API_URL, { params: { requestId } });
-                logger.info(`Marked request ${requestId} as completed:`, res.data);
-                return res.data;
-            }
-        } catch (err) {
-            logger.error(`Error marking request ${requestId} as completed: ${err}`);
-        }
-    }
-
-    // Execute the request to the OpenAI Whisper API
-    async execute(text, parameters, prompt, cortexRequest) {
-        const { pathwayResolver } = cortexRequest;
-        const { responseFormat, wordTimestamped, highlightWords, maxLineWidth, maxLineCount, maxWordsPerLine } = parameters;
-        cortexRequest.url = this.requestUrl(text);
-
-        const chunks = [];
-        const processChunk = async (uri) => {
-            try {
-                const chunk = await downloadFile(uri);
-                chunks.push(chunk);
-
-                const { language, responseFormat } = parameters;
-                cortexRequest.url = this.requestUrl(text);
-                const params = {};
-                const { modelPromptText } = this.getCompiledPrompt(text, parameters, prompt);
-                const response_format = responseFormat || 'text';
-
-                const formData = new FormData();
-                formData.append('file', fs.createReadStream(chunk));
-                formData.append('model', this.model.params.model);
-                formData.append('response_format', response_format);
-                language && formData.append('language', language);
-                modelPromptText && formData.append('prompt', modelPromptText);
-
-                cortexRequest.data = formData;
-                cortexRequest.params = params;
-                cortexRequest.headers = { ...cortexRequest.headers, ...formData.getHeaders() };
-
-                return this.executeRequest(cortexRequest);
-            } catch (err) {
-                logger.error(`Error getting word timestamped data from api: ${err}`);
-                throw err;
-            }
-        }
-
-        const processTS = async (uri) => {
-            try {
-                const tsparams = { fileurl:uri };
-
-                const { language } = parameters;
-                if(language) tsparams.language = language;
-                if(highlightWords) tsparams.highlight_words = highlightWords ? "True" : "False";
-                if(maxLineWidth) tsparams.max_line_width = maxLineWidth;
-                if(maxLineCount) tsparams.max_line_count = maxLineCount;
-                if(maxWordsPerLine) tsparams.max_words_per_line = maxWordsPerLine;
-                if(wordTimestamped!=null) {
-                    if(!wordTimestamped) {
-                        tsparams.word_timestamps = "False";
-                    }else{
-                        tsparams.word_timestamps = wordTimestamped;    
-                    }
-                }
-
-                cortexRequest.url = WHISPER_TS_API_URL;
-                cortexRequest.data = tsparams;
-
-                const res = await this.executeRequest(cortexRequest);
-
-                if(!wordTimestamped && !responseFormat){ 
-                    //if no response format, convert to text
-                    return convertToText(res);
-                }
-                return res;
-            } catch (err) {
-                logger.error(`Error getting word timestamped data from api: ${err}`);
-                throw err;
-            }
-        }
-
-        let result = [];
-        let { file } = parameters;
-        let totalCount = 0;
-        let completedCount = 0;
-        let partialCount = 0;
-        const { requestId } = pathwayResolver;
-
-        const MAXPARTIALCOUNT = 60;
-        const sendProgress = (partial=false) => {
-            if(partial){
-                partialCount = Math.min(partialCount + 1, MAXPARTIALCOUNT-1);
-            }else {
-                partialCount = 0;
-                completedCount++;
-            }
-            if (completedCount >= totalCount) return;
-
-            const progress = (partialCount / MAXPARTIALCOUNT + completedCount) / totalCount;
-            logger.info(`Progress for ${requestId}: ${progress}`);
-
-            publishRequestProgress({
-                requestId,
-                progress,
-                data: null,
-            });
-        }
-
-        async function processInBatches(tasks, batchSize) {
-            const batches = chunkArray(tasks,batchSize);
-            const result = [];
-            for(let i=0; i < batches.length; i++){
-                let batch = batches[i];
-                // execute all tasks in current batch and wait for them to finish
-                const curBatchResults = await Promise.all(batch.map(task => task()));
-                // accumulate the results
-                result.push(...curBatchResults);
-            }
-            return result;
-        }
-        
-        // helper function to chunk an array into smaller arrays of size n
-        function chunkArray(array, chunkSize) {
-            const results = [];
-            while (array.length) {
-                results.push(array.splice(0, chunkSize));
-            }
-            return results;
-        }
-
-        async function processURI(uri) {
-            let result = null;
-            let _promise = null;
-            if(WHISPER_TS_API_URL){
-                _promise = processTS
-            }else {
-                _promise = processChunk;
-            }
-            _promise(uri).then((ts) => { result = ts;});
-
-            //send updates while waiting for result
-            while(!result) {
-                sendProgress(true);
-                await new Promise(r => setTimeout(r, 3000));
-            }
-            return result;
-        }
-
-        try {
-            const uris = await this.getMediaChunks(file, requestId); // array of remote file uris
-            if (!uris || !uris.length) {
-                throw new Error(`Error in getting chunks from media helper for file ${file}`);
-            }
-            totalCount = uris.length + 1; // total number of chunks that will be processed
-
-            // parallel process of chunks with limit
-            const tasks = uris.map(uri => () => processURI(uri)); // map each uri to a function that returns a Promise
-            result = await processInBatches(tasks, 2); // wait for all Promises to resolve, 2 at a time
-
-
-        } catch (error) {
-            const errMsg = `Transcribe error: ${error?.response?.data || error?.message || error}`;
-            logger.error(errMsg);
-            return errMsg;
-        }
-        finally {
-            try {
-                for (const chunk of chunks) {
-                    try {
-                        await deleteTempPath(chunk);
-                    } catch (error) {
-                        //ignore error
-                    } 
-                }
-
-                await this.markCompletedForCleanUp(requestId);
-
-                //check cleanup for whisper temp uploaded files url
-                const regex = /whispertempfiles\/([a-z0-9-]+)/;
-                const match = file.match(regex);
-                if (match && match[1]) {
-                    const extractedValue = match[1];
-                    await this.markCompletedForCleanUp(extractedValue);
-                    logger.info(`Cleaned temp whisper file ${file} with request id ${extractedValue}`);
-                }
-
-            } catch (error) {
-                logger.error(`An error occurred while deleting: ${error}`);
-            }
-        }
-
-        if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
-            return alignSubtitles(result, responseFormat);
-        }
-        return result.join(` `);
-    }
-}
-
-export default OpenAIWhisperPlugin;