@aj-archipelago/cortex 1.1.12 → 1.1.13

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -87,32 +87,24 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
             inputPath = downloadPath;
         }
 
-        
         const metadata = await ffmpegProbe(inputPath);
         const duration = metadata.format.duration;
         const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds);
 
         const chunkPromises = [];
-
-
+        const chunkOffsets = [];
 
         for (let i = 0; i < numChunks; i++) {
-            const outputFileName = path.join(
-                uniqueOutputPath,
-                `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`
-            );
-
-            const chunkPromise = processChunk(
-                inputPath,
-                outputFileName,
-                i * chunkDurationInSeconds,
-                chunkDurationInSeconds
-            );
+            const outputFileName = path.join(uniqueOutputPath, `chunk-${i + 1}-${path.parse(inputPath).name}.mp3`);
+            const offset = i * chunkDurationInSeconds;
 
+            const chunkPromise = processChunk(inputPath, outputFileName, offset, chunkDurationInSeconds);
+            
             chunkPromises.push(chunkPromise);
+            chunkOffsets.push(offset);
         }
 
-        return { chunkPromises, uniqueOutputPath };
+        return { chunkPromises, chunkOffsets, uniqueOutputPath }; 
     } catch (err) {
         const msg = `Error processing media file, check if the file is a valid media file or is accessible`;
         console.error(msg, err);

package/helper-apps/cortex-file-handler/index.js

@@ -143,7 +143,7 @@ async function main(context, req) {
                 file = await processYoutubeUrl(file);
             }
 
-            const { chunkPromises, uniqueOutputPath } = await splitMediaFile(file);
+            const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);
             folder = uniqueOutputPath;
 
             numberOfChunks = chunkPromises.length; // for progress reporting
@@ -158,9 +158,11 @@ async function main(context, req) {
             }
 
             // sequential processing of chunks
-            for (const chunk of chunks) {
+            for (let index = 0; index < chunks.length; index++) {
+                const chunk = chunks[index];
                 const blobName = useAzure ? await saveFileToBlob(chunk, requestId) : await moveFileToPublicFolder(chunk, requestId);
-                result.push(blobName);
+                const chunkOffset = chunkOffsets[index];
+                result.push({ uri:blobName, offset:chunkOffset });
                 context.log(`Saved chunk as: ${blobName}`);
                 sendProgress();
             }
@@ -182,7 +184,10 @@ async function main(context, req) {
         }
     }
 
-    console.log(`result: ${result}`);
+    console.log('result:', result.map(item =>
+        typeof item === 'object' ? JSON.stringify(item, null, 2) : item
+    ).join('\n'));
+    
     context.res = {
         body: result
     };

package/lib/requestExecutor.js

@@ -323,8 +323,13 @@ const makeRequest = async (cortexRequest) => {
                         return { response, duration };
                     }
                 } else {
-                    // if there are multiple endpoints, retry everything as it
-                    // could be going to a different host
+                    // if there are multiple endpoints, retry everything by default
+                    // as it could be a temporary issue with one endpoint
+                    // certain errors (e.g. 400) are problems with the request itself
+                    // and should not be retried
+                    if (status == 400) {
+                        return { response, duration };
+                    }
                     cortexRequest.selectNewEndpoint();
                 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.12",
+  "version": "1.1.13",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/server/plugins/openAiWhisperPlugin.js

@@ -15,6 +15,7 @@ import { v4 as uuidv4 } from 'uuid';
 import { promisify } from 'util';
 import { publishRequestProgress } from '../../lib/redisSubscription.js';
 import logger from '../../lib/logger.js';
+import CortexRequest from '../../lib/cortexRequest.js';
 const pipeline = promisify(stream.pipeline);
 
 const API_URL = config.get('whisperMediaApiUrl');
@@ -25,7 +26,7 @@ if(WHISPER_TS_API_URL){
     logger.warn(`WHISPER API URL not set using default OpenAI API Whisper`);
 }
 
-const OFFSET_CHUNK = 1000 * 500; // 500 seconds chunk offset
+const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
 async function deleteTempPath(path) {
     try {
@@ -96,7 +97,7 @@ function convertToText(str) {
       .join(' ');
 }
 
-function alignSubtitles(subtitles, format) {
+function alignSubtitles(subtitles, format, offsets) {
     const result = [];
 
     function preprocessStr(str) {
@@ -116,7 +117,7 @@ function alignSubtitles(subtitles, format) {
     }
 
     for (let i = 0; i < subtitles.length; i++) {
-        result.push(...shiftSubtitles(subtitles[i], i * OFFSET_CHUNK));
+        result.push(...shiftSubtitles(subtitles[i], offsets[i]*1000)); // convert to milliseconds
     }
     
     try {
@@ -171,12 +172,14 @@ class OpenAIWhisperPlugin extends ModelPlugin {
     // Execute the request to the OpenAI Whisper API
     async execute(text, parameters, prompt, cortexRequest) {
         const { pathwayResolver } = cortexRequest;
+
         const { responseFormat, wordTimestamped, highlightWords, maxLineWidth, maxLineCount, maxWordsPerLine } = parameters;
-        cortexRequest.url = this.requestUrl(text);
 
         const chunks = [];
         const processChunk = async (uri) => {
             try {
+                const cortexRequest = new CortexRequest({ pathwayResolver });
+
                 const chunk = await downloadFile(uri);
                 chunks.push(chunk);
 
@@ -205,6 +208,8 @@ class OpenAIWhisperPlugin extends ModelPlugin {
         }
 
         const processTS = async (uri) => {
+            const cortexRequest = new CortexRequest({ pathwayResolver });
+
             const tsparams = { fileurl:uri };
             const { language } = parameters;
             if(language) tsparams.language = language;
@@ -315,14 +320,20 @@ async function processURI(uri) {
     return result;
 }
 
+let offsets = [];
+let uris = []
+
 try {
-    const uris = await this.getMediaChunks(file, requestId);
+    const mediaChunks = await this.getMediaChunks(file, requestId);
     
-    if (!uris || !uris.length) {
+    if (!mediaChunks || !mediaChunks.length) {
         throw new Error(`Error in getting chunks from media helper for file ${file}`);
     }
 
-    totalCount = uris.length + 1; // total number of chunks that will be processed
+    uris = mediaChunks.map((chunk) => chunk?.uri || chunk);
+    offsets = mediaChunks.map((chunk, index) => chunk?.offset || index * OFFSET_CHUNK);
+
+    totalCount = mediaChunks.length + 1; // total number of chunks that will be processed
 
     const batchSize = 2;
     sendProgress();
@@ -369,7 +380,7 @@ try {
         }
 
         if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
-            return alignSubtitles(result, responseFormat);
+            return alignSubtitles(result, responseFormat, offsets);
         }
         return result.join(` `);
     }