@aj-archipelago/cortex 1.1.11 → 1.1.13

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -87,32 +87,24 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
             inputPath = downloadPath;
         }
 
-        
         const metadata = await ffmpegProbe(inputPath);
         const duration = metadata.format.duration;
         const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
 
         const chunkPromises = [];
-
-
+        const chunkOffsets = [];
 
         for (let i = 0; i < numChunks; i++) {
-            const outputFileName = path.join(
-                uniqueOutputPath,
-                `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`
-            );
-
-            const chunkPromise = processChunk(
-                inputPath,
-                outputFileName,
-                i * chunkDurationInSeconds,
-                chunkDurationInSeconds
-            );
+            const outputFileName = path.join(uniqueOutputPath, `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`);
+            const offset = i * chunkDurationInSeconds;
 
+            const chunkPromise = processChunk(inputPath, outputFileName, offset, chunkDurationInSeconds);
+            
             chunkPromises.push(chunkPromise);
+            chunkOffsets.push(offset);
         }
 
-        return { chunkPromises, uniqueOutputPath };
+        return { chunkPromises, chunkOffsets, uniqueOutputPath }; 
     } catch (err) {
         const msg = `Error processing media file, check if the file is a valid media file or is accessible`;
         console.error(msg, err);

package/helper-apps/cortex-file-handler/index.js

@@ -143,7 +143,7 @@ async function main(context, req) {
                 file = await processYoutubeUrl(file);
             }
 
-            const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+            const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);
             folder = uniqueOutputPath;
 
             numberOfChunks = chunkPromises.length; // for progress reporting
@@ -158,9 +158,11 @@ async function main(context, req) {
             }
 
             // sequential processing of chunks
-            for (const chunk of chunks) {
+            for (let index = 0; index < chunks.length; index++) {
+                const chunk = chunks[index];
                 const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
-                result.push(blobName);
+                const chunkOffset = chunkOffsets[index];
+                result.push({ uri:blobName, offset:chunkOffset });
                 context.log(`Saved chunk as: ${blobName}`);
                 sendProgress();
             }
@@ -182,7 +184,10 @@ async function main(context, req) {
         }
     }
 
-    console.log(`result: ${result}`);
+    console.log('result:', result.map(item =>
+        typeof item === 'object' ? JSON.stringify(item, null, 2) : item
+    ).join('\n'));
+    
     context.res = {
         body: result
     };

package/lib/requestExecutor.js

@@ -323,8 +323,13 @@ const makeRequest = async (cortexRequest) => {
                         return { response, duration };
                     }
                 } else {
-                    // if there are multiple endpoints, retry everything as it
-                    // could be going to a different host
+                    // if there are multiple endpoints, retry everything by default
+                    // as it could be a temporary issue with one endpoint
+                    // certain errors (e.g. 400) are problems with the request itself
+                    // and should not be retried
+                    if (status == 400) {
+                        return { response, duration };
+                    }
                     cortexRequest.selectNewEndpoint();
                 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.11",
+  "version": "1.1.13",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/openAiWhisperPlugin.js

@@ -15,6 +15,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { promisify } from 'util';
 import { publishRequestProgress } from '../../lib/redisSubscription.js';
 import logger from '../../lib/logger.js';
+import CortexRequest from '../../lib/cortexRequest.js';
 const pipeline = promisify(stream.pipeline);
 
 const API_URL = config.get('whisperMediaApiUrl');
@@ -25,7 +26,7 @@ if(WHISPER_TS_API_URL){
     logger.warn(`WHISPER API URL not set using default OpenAI API Whisper`);
 }
 
-const OFFSET_CHUNK = 1000 * 500; // 500 seconds chunk offset
+const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
 async function deleteTempPath(path) {
     try {
@@ -96,7 +97,7 @@ function convertToText(str) {
       .join(' ');
 }
 
-function alignSubtitles(subtitles, format) {
+function alignSubtitles(subtitles, format, offsets) {
     const result = [];
 
     function preprocessStr(str) {
@@ -116,7 +117,7 @@ function alignSubtitles(subtitles, format) {
     }
 
     for (let i = 0; i < subtitles.length; i++) {
-        result.push(...shiftSubtitles(subtitles[i], i * OFFSET_CHUNK));
+        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
     }
     
     try {
@@ -171,12 +172,14 @@ class OpenAIWhisperPlugin extends ModelPlugin {
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, cortexRequest) {
         const { pathwayResolver } = cortexRequest;
+
         const { responseFormat, wordTimestamped, highlightWords, maxLineWidth, maxLineCount, maxWordsPerLine } = parameters;
-        cortexRequest.url = this.requestUrl(text);
 
         const chunks = [];
         const processChunk = async (uri) => {
             try {
+                const cortexRequest = new CortexRequest({ pathwayResolver });
+
                 const chunk = await downloadFile(uri);
                 chunks.push(chunk);
 
@@ -205,6 +208,8 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         const processTS = async (uri) => {
+            const cortexRequest = new CortexRequest({ pathwayResolver });
+
             const tsparams = { fileurl:uri };
             const { language } = parameters;
             if(language) tsparams.language = language;
@@ -223,7 +228,24 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             cortexRequest.url = WHISPER_TS_API_URL;
             cortexRequest.data = tsparams;
 
-            const res = await this.executeRequest(cortexRequest);
+            const MAX_RETRIES = 3;
+            let attempt = 0;
+            let res = null;
+            while(attempt < MAX_RETRIES){
+                sendProgress(true, true);
+                try {
+                    res = await this.executeRequest(cortexRequest);
+                    if(res.statusCode && res.statusCode >= 400){
+                        throw new Error(res.message || 'An error occurred.');
+                    }
+                    break;
+                }
+                catch(err){
+                    logger.warn(`Error calling timestamped API: ${err}. Retrying ${attempt+1} of ${MAX_RETRIES}...`);
+                    attempt++;
+                }
+            }
+
             if (res.statusCode && res.statusCode >= 400) {
                 throw new Error(res.message || 'An error occurred.');
             }
@@ -241,18 +263,23 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         let completedCount = 0;
         let partialCount = 0;
         const { requestId } = pathwayResolver;
+        let partialRatio = 0;
+
+        const sendProgress = (partial=false, resetCount=false) => {
+            partialCount = resetCount ? 0 : partialCount;
 
-        const MAXPARTIALCOUNT = 60;
-        const sendProgress = (partial=false) => {
             if(partial){
-                partialCount = Math.min(partialCount + 1, MAXPARTIALCOUNT-1);
-            }else {
+                partialCount++;
+                const increment = 0.02 / Math.log2(partialCount + 1); // logarithmic diminishing increment
+                partialRatio = Math.min(partialRatio + increment, 0.99); // limit to 0.99
+            }else{
                 partialCount = 0;
+                partialRatio = 0;
                 completedCount++;
             }
-            if (completedCount >= totalCount) return;
+            if(completedCount >= totalCount) return;
 
-            const progress = (partialCount / MAXPARTIALCOUNT + completedCount) / totalCount;
+            const progress = (completedCount + partialRatio) / totalCount;
             logger.info(`Progress for ${requestId}: ${progress}`);
 
             publishRequestProgress({
@@ -262,57 +289,70 @@ class OpenAIWhisperPlugin extends ModelPlugin {
             });
         }
 
-        async function processURI(uri) {
-            let result = null;
-            let _promise = null;
-            let errorOccurred = false;
-        
-            const useTS = WHISPER_TS_API_URL && (wordTimestamped || highlightWords);
-        
-            if (useTS) {
-                _promise = processTS;
-            } else {
-                _promise = processChunk;
-            }
-            
-            _promise(uri).then((ts) => {
-                result = ts;
-            }).catch((err) => {
-                logger.error(`Error occurred while processing URI: ${err}`);
-                errorOccurred = err;
-            });
-        
-            while(result === null && !errorOccurred) {
-                sendProgress(true);
-                await new Promise(r => setTimeout(r, 3000));
-            }
+async function processURI(uri) {
+    let result = null;
+    let _promise = null;
+    let errorOccurred = false;
 
-            if(errorOccurred) {
-                throw errorOccurred;
-            }
-            
-            return result;
-        }
+    const intervalId = setInterval(() => sendProgress(true), 3000);
 
-        try {
-            const uris = await this.getMediaChunks(file, requestId); // array of remote file uris
-            if (!uris || !uris.length) {
-                throw new Error(`Error in getting chunks from media helper for file ${file}`);
-            }
-            totalCount = uris.length + 1; // total number of chunks that will be processed
+    const useTS = WHISPER_TS_API_URL && (wordTimestamped || highlightWords);
 
-            // sequential process of chunks
-            for (const uri of uris) {
-                sendProgress(); 
-                const ts = await processURI(uri);
-                result.push(ts);
-            }
+    if (useTS) {
+        _promise = processTS;
+    } else {
+        _promise = processChunk;
+    }
 
-        } catch (error) {
-            const errMsg = `Transcribe error: ${error?.response?.data || error?.message || error}`;
-            logger.error(errMsg);
-            return errMsg;
+    await _promise(uri).then((ts) => {
+        result = ts;
+    }).catch((err) => {
+        errorOccurred = err;
+    }).finally(() => {
+        clearInterval(intervalId);
+        sendProgress();
+    });
+
+    if(errorOccurred) {
+        throw errorOccurred;
+    }
+
+    return result;
+}
+
+let offsets = [];
+let uris = []
+
+try {
+    const mediaChunks = await this.getMediaChunks(file, requestId);
+    
+    if (!mediaChunks || !mediaChunks.length) {
+        throw new Error(`Error in getting chunks from media helper for file ${file}`);
+    }
+
+    uris = mediaChunks.map((chunk) => chunk?.uri || chunk);
+    offsets = mediaChunks.map((chunk, index) => chunk?.offset || index * OFFSET_CHUNK);
+
+    totalCount = mediaChunks.length + 1; // total number of chunks that will be processed
+
+    const batchSize = 2;
+    sendProgress();
+
+    for (let i = 0; i < uris.length; i += batchSize) {
+        const currentBatchURIs = uris.slice(i, i + batchSize);
+        const promisesToProcess = currentBatchURIs.map(uri => processURI(uri));
+        const results = await Promise.all(promisesToProcess); 
+          
+        for(const res of results) {
+            result.push(res);
         }
+    }
+
+} catch (error) {
+    const errMsg = `Transcribe error: ${error?.response?.data || error?.message || error}`;
+    logger.error(errMsg);
+    return errMsg;
+}
         finally {
             try {
                 for (const chunk of chunks) {
@@ -340,7 +380,7 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
-            return alignSubtitles(result, responseFormat);
+            return alignSubtitles(result, responseFormat, offsets);
         }
         return result.join(` `);
     }