@aj-archipelago/cortex 1.1.2 → 1.1.4-0

package/.eslintignore

@@ -15,12 +15,12 @@
 # Ignore coverage reports
 /coverage
 
+# Ignore helper apps
+/helper-apps
+
 # Ignore documentation
 /docs
 
-# Ignore helper apps
-/helper_apps
-
 # Ignore tests
 /tests
 

package/README.md

@@ -212,12 +212,12 @@ export default {
 
     // Custom resolver to generate summaries by reprompting if they are too long or too short.
     resolver: async (parent, args, contextValue, info) => {
-        const { config, pathway, requestState } = contextValue;
+        const { config, pathway } = contextValue;
         const originalTargetLength = args.targetLength;
 
         // If targetLength is not provided, execute the prompt once and return the result.
         if (originalTargetLength === 0) {
-            let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
+            let pathwayResolver = new PathwayResolver({ config, pathway, args });
             return await pathwayResolver.resolve(args);
         }
 
@@ -232,7 +232,7 @@ export default {
 
         const MAX_ITERATIONS = 5;
         let summary = '';
-        let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
+        let pathwayResolver = new PathwayResolver({ config, pathway, args });
 
         // Modify the prompt to be words-based instead of characters-based.
         pathwayResolver.pathwayPrompt = `Write a summary of all of the text below. If the text is in a language other than english, make sure the summary is written in the same language. Your summary should be ${targetWords} words in length.\n\nText:\n\n{{{text}}}\n\nSummary:\n\n`
@@ -440,6 +440,19 @@ The `config` object can be used to access configuration values throughout the pr
 ```js
 config.get('PORT')
 ```
+
+## Helper Apps
+The Cortex project includes a set of utility applications, which are located in the `helper-apps`` directory. Each of these applications comes with a Dockerfile. This Dockerfile can be used to create a Docker image of the application, which in turn allows the application to be run in a standalone manner using Docker.
+
+- cortex-file-handler
+Extends Cortex with several file processing units. Handles file operations (download, split, upload) with local file system or Azure Storage. It can process different file types including documents, files ( .pdf, .docx, .xlsx, .csv .txt, .json, .md, .xml, .js, .html, .css) and additionally YouTube URLs. It also manages deletion requests and cleanup operations, and provides progress reporting for requests.  
+
+- cortex-whisper-wrapper
+The cortex-whisper-wrapper is a custom API wrapper for the Whisper package from OpenAI. Designed as a FastAPI server, it aids in transcribing audio files using the Whisper library. 
+The server provides an HTTP endpoint ("/") that accepts POST requests with a JSON payload containing a "fileurl" parameter specifying the URL of the audio file to transcribe. Upon receiving a request, the server calls the transcribe function to perform the transcription using the Whisper model, saves the transcription as an SRT file, and returns the SRT content as the response.
+It helps Cortex to make use of Whisper OS parameters which currently are not available in OpenAI API. Parameters supported are: 'word_timestamps', 'highlight_words', 'max_line_count', 'max_line_width', 'max_words_per_line'. These parameters customizes transcription output, for more info on the parameters see open source Whisper package https://github.com/openai/whisper 
+
+
 ## Troubleshooting
 If you encounter any issues while using Cortex, there are a few things you can do. First, check the Cortex documentation for any common errors and their solutions. If that does not help, you can also open an issue on the Cortex GitHub repository.
 

package/config.js

@@ -101,7 +101,7 @@ var config = convict({
             },
             "azure-cognitive": {
                 "type": "AZURE-COGNITIVE",
-                "url": "https://archipelago-cognitive-search.search.windows.net/indexes/indexcortex/docs/search?api-version=2023-07-01-Preview",
+                "url": "{{{AZURE_COGNITIVE_API_URL}}}",
                 "headers": {
                     "api-key": "{{AZURE_COGNITIVE_API_KEY}}",
                     "Content-Type": "application/json"
@@ -110,7 +110,7 @@ var config = convict({
             },
             "oai-embeddings": {
                 "type": "OPENAI-EMBEDDINGS",
-                "url": "https://archipelago-openai.openai.azure.com/openai/deployments/archipelago-embedding/embeddings?api-version=2023-05-15",
+                "url": "https://archipelago-openai.openai.azure.com/openai/deployments/archipelago-embedding/embeddings?api-version=2023-12-01",
                 "headers": {
                     "api-key": "{{ARCHIPELAGO_OPENAI_KEY}}",
                     "Content-Type": "application/json"
@@ -192,7 +192,7 @@ var config = convict({
     },
     whisperTSApiUrl: {
         format: String,
-        default: 'null',
+        default: null,
         env: 'WHISPER_TS_API_URL'
     },
     subscriptionKeepAlive: {
@@ -264,17 +264,41 @@ const buildPathways = async (config) => {
 
 // Build and load models to config
 const buildModels = (config) => {
-    const { models } = config.getProperties();
+    let { models } = config.getProperties();
+
+    // iterate over each model
+    for (let [key, model] of Object.entries(models)) {
+        if (!model.name) {
+            model.name = key;
+        }
+        
+        // if model is in old format, convert it to new format
+        if (!model.endpoints) {
+            model = {
+                ...model,
+                endpoints: [
+                    {
+                        name: "default",
+                        url: model.url,
+                        headers: model.headers,
+                        params: model.params,
+                        requestsPerSecond: model.requestsPerSecond
+                    }
+                ]
+            };
+        }
 
-    for (const [key, model] of Object.entries(models)) {
-        // Compile handlebars templates for models
-        models[key] = JSON.parse(HandleBars.compile(JSON.stringify(model))({ ...config.getEnv(), ...config.getProperties() }))
+        // compile handlebars templates for each endpoint
+        model.endpoints = model.endpoints.map(endpoint => 
+            JSON.parse(HandleBars.compile(JSON.stringify(endpoint))({ ...model, ...config.getEnv(), ...config.getProperties() }))
+        );
+
+        models[key] = model;
     }
 
     // Add constructed models to config
     config.load({ models });
 
-
     // Check that models are specified, Cortex cannot run without a model
     if (Object.keys(config.get('models')).length <= 0) {
         const errorString = 'No models specified! Please set the models in your config file or via CORTEX_MODELS environment variable to point at the models for your project.';

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/Dockerfile

@@ -8,7 +8,7 @@ RUN npm install
 
 ## installing ffmepg
 RUN apk update && \
-    apk add ffmpeg=6.0-r15
+    apk add ffmpeg
 
 COPY . .
 

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/fileChunker.js

@@ -6,6 +6,7 @@ import os from 'os';
 import { promisify } from 'util';
 import axios from 'axios';
 import { ensureEncoded } from './helper.js';
+import ytdl from 'ytdl-core';
 
 
 const ffmpegProbe = promisify(ffmpeg.ffprobe);

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/package-lock.json

@@ -9,7 +9,6 @@
       "version": "1.0.0",
       "dependencies": {
         "@azure/storage-blob": "^12.13.0",
-        "@distube/ytdl-core": "^4.13.2",
         "axios": "^1.3.6",
         "busboy": "^1.6.0",
         "cors": "^2.8.5",
@@ -21,7 +20,8 @@
         "pdfjs-dist": "^3.9.179",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "xlsx": "^0.18.5"
+        "xlsx": "^0.18.5",
+        "ytdl-core": "^4.11.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -156,29 +156,6 @@
         "node": ">=14.0.0"
       }
     },
-    "node_modules/@distube/ytdl-core": {
-      "version": "4.13.2",
-      "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.13.2.tgz",
-      "integrity": "sha512-FBjy4LXsBv34qFrmEF1dSbKWQtqB+yd0dt4MYYUq8oYys2Bvqps3QSFGTUZO5QA6nKD/XRsMJfcG5lgEJaUqiQ==",
-      "dependencies": {
-        "http-cookie-agent": "^5.0.4",
-        "m3u8stream": "^0.8.6",
-        "sax": "^1.2.4",
-        "tough-cookie": "^4.1.3",
-        "undici": "^5.25.2"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
@@ -1190,44 +1167,6 @@
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
       "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
     },
-    "node_modules/http-cookie-agent": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.4.tgz",
-      "integrity": "sha512-OtvikW69RvfyP6Lsequ0fN5R49S+8QcS9zwd58k6VSr6r57T8G29BkPdyrBcSwLq6ExLs9V+rBlfxu7gDstJag==",
-      "dependencies": {
-        "agent-base": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=14.18.0 <15.0.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/3846masa"
-      },
-      "peerDependencies": {
-        "deasync": "^0.1.26",
-        "tough-cookie": "^4.0.0",
-        "undici": "^5.11.0"
-      },
-      "peerDependenciesMeta": {
-        "deasync": {
-          "optional": true
-        },
-        "undici": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/http-cookie-agent/node_modules/agent-base": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
-      "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
-      "dependencies": {
-        "debug": "^4.3.4"
-      },
-      "engines": {
-        "node": ">= 14"
-      }
-    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -1889,11 +1828,6 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
-    "node_modules/psl": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
-      "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
-    },
     "node_modules/public-ip": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/public-ip/-/public-ip-6.0.1.tgz",
@@ -1911,14 +1845,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/punycode": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/qs": {
       "version": "6.11.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
@@ -1933,11 +1859,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
-    },
     "node_modules/quick-lru": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
@@ -2004,11 +1925,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
-    },
     "node_modules/resolve-alpn": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
@@ -2329,20 +2245,6 @@
         "node": ">=0.6"
       }
     },
-    "node_modules/tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
-      "dependencies": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
-      },
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@@ -2378,25 +2280,6 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
       "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
     },
-    "node_modules/undici": {
-      "version": "5.27.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.2.tgz",
-      "integrity": "sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=14.0"
-      }
-    },
-    "node_modules/universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
-      "engines": {
-        "node": ">= 4.0.0"
-      }
-    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -2405,15 +2288,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "dependencies": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -2544,6 +2418,19 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
+    },
+    "node_modules/ytdl-core": {
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
+      "dependencies": {
+        "m3u8stream": "^0.8.6",
+        "miniget": "^4.2.2",
+        "sax": "^1.1.3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
     }
   },
   "dependencies": {
@@ -2651,23 +2538,6 @@
         "tslib": "^2.2.0"
       }
     },
-    "@distube/ytdl-core": {
-      "version": "4.13.2",
-      "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.13.2.tgz",
-      "integrity": "sha512-FBjy4LXsBv34qFrmEF1dSbKWQtqB+yd0dt4MYYUq8oYys2Bvqps3QSFGTUZO5QA6nKD/XRsMJfcG5lgEJaUqiQ==",
-      "requires": {
-        "http-cookie-agent": "^5.0.4",
-        "m3u8stream": "^0.8.6",
-        "sax": "^1.2.4",
-        "tough-cookie": "^4.1.3",
-        "undici": "^5.25.2"
-      }
-    },
-    "@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA=="
-    },
     "@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
@@ -3433,24 +3303,6 @@
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
       "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
     },
-    "http-cookie-agent": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.4.tgz",
-      "integrity": "sha512-OtvikW69RvfyP6Lsequ0fN5R49S+8QcS9zwd58k6VSr6r57T8G29BkPdyrBcSwLq6ExLs9V+rBlfxu7gDstJag==",
-      "requires": {
-        "agent-base": "^7.1.0"
-      },
-      "dependencies": {
-        "agent-base": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
-          "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
-          "requires": {
-            "debug": "^4.3.4"
-          }
-        }
-      }
-    },
     "http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -3946,11 +3798,6 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
-    "psl": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
-      "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
-    },
     "public-ip": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/public-ip/-/public-ip-6.0.1.tgz",
@@ -3962,11 +3809,6 @@
         "is-ip": "^4.0.0"
       }
     },
-    "punycode": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
-    },
     "qs": {
       "version": "6.11.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
@@ -3975,11 +3817,6 @@
         "side-channel": "^1.0.4"
       }
     },
-    "querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
-    },
     "quick-lru": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
@@ -4025,11 +3862,6 @@
         "redis-errors": "^1.0.0"
       }
     },
-    "requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
-    },
     "resolve-alpn": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
@@ -4271,17 +4103,6 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
-    "tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
-      "requires": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
-      }
-    },
     "tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@@ -4311,33 +4132,11 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
       "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
     },
-    "undici": {
-      "version": "5.27.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.2.tgz",
-      "integrity": "sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==",
-      "requires": {
-        "@fastify/busboy": "^2.0.0"
-      }
-    },
-    "universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg=="
-    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
       "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
     },
-    "url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "requires": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -4438,6 +4237,16 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
+    },
+    "ytdl-core": {
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
+      "requires": {
+        "m3u8stream": "^0.8.6",
+        "miniget": "^4.2.2",
+        "sax": "^1.1.3"
+      }
     }
   }
 }

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/package.json

@@ -9,7 +9,6 @@
   },
   "dependencies": {
     "@azure/storage-blob": "^12.13.0",
-    "@distube/ytdl-core": "^4.13.2",
     "axios": "^1.3.6",
     "busboy": "^1.6.0",
     "cors": "^2.8.5",
@@ -21,6 +20,7 @@
     "pdfjs-dist": "^3.9.179",
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
-    "xlsx": "^0.18.5"
+    "xlsx": "^0.18.5",
+    "ytdl-core": "^4.11.5"
   }
 }

package/helper-apps/cortex-whisper-wrapper/.dockerignore

@@ -0,0 +1,27 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

package/helper-apps/cortex-whisper-wrapper/Dockerfile

@@ -0,0 +1,32 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:3.10-slim
+
+EXPOSE 8000
+
+## following 3 lines are for installing ffmepg
+RUN apt-get -y update
+RUN apt-get -y upgrade
+RUN apt-get install -y ffmpeg
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+# Install pip requirements
+COPY requirements.txt .
+RUN python -m pip install -r requirements.txt
+
+WORKDIR /app
+COPY ./models /app/models
+COPY . /app
+
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
+USER appuser
+
+# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
+# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout", "0", "-k", "uvicorn.workers.UvicornWorker", "app:app"]