@aj-archipelago/cortex 1.1.3 → 1.1.4

package/.eslintignore

@@ -15,12 +15,12 @@
 # Ignore coverage reports
 /coverage
 
+# Ignore helper apps
+/helper-apps
+
 # Ignore documentation
 /docs
 
-# Ignore helper apps
-/helper_apps
-
 # Ignore tests
 /tests
 

package/README.md

@@ -212,12 +212,12 @@ export default {
 
     // Custom resolver to generate summaries by reprompting if they are too long or too short.
     resolver: async (parent, args, contextValue, info) => {
-        const { config, pathway, requestState } = contextValue;
+        const { config, pathway } = contextValue;
         const originalTargetLength = args.targetLength;
 
         // If targetLength is not provided, execute the prompt once and return the result.
         if (originalTargetLength === 0) {
-            let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
+            let pathwayResolver = new PathwayResolver({ config, pathway, args });
             return await pathwayResolver.resolve(args);
         }
 
@@ -232,7 +232,7 @@ export default {
 
         const MAX_ITERATIONS = 5;
         let summary = '';
-        let pathwayResolver = new PathwayResolver({ config, pathway, args, requestState });
+        let pathwayResolver = new PathwayResolver({ config, pathway, args });
 
         // Modify the prompt to be words-based instead of characters-based.
         pathwayResolver.pathwayPrompt = `Write a summary of all of the text below. If the text is in a language other than english, make sure the summary is written in the same language. Your summary should be ${targetWords} words in length.\n\nText:\n\n{{{text}}}\n\nSummary:\n\n`
@@ -422,7 +422,7 @@ Configuration of Cortex is done via a [convict](https://github.com/mozilla/node-
 - `enableCache`: A boolean flag indicating whether to enable Axios-level request caching. Default is true. The value can be set using the `CORTEX_ENABLE_CACHE` environment variable.
 - `enableGraphqlCache`: A boolean flag indicating whether to enable GraphQL query caching. Default is false. The value can be set using the `CORTEX_ENABLE_GRAPHQL_CACHE` environment variable.
 - `enableRestEndpoints`: A boolean flag indicating whether create REST endpoints for pathways as well as GraphQL queries. Default is false. The value can be set using the `CORTEX_ENABLE_REST` environment variable.
-- `cortexApiKey`: A string containing an API key that the client must pass to Cortex for authorization. Default is null in which case Cortex is unprotected. The value can be set using the `CORTEX_API_KEY` environment variable
+- `cortexApiKeys`: A string containing one or more comma separated API keys that the client must pass to Cortex for authorization. Default is null in which case Cortex is unprotected. The value can be set using the `CORTEX_API_KEY` environment variable
 - `models`: An object containing the different models used by the project. The value can be set using the `CORTEX_MODELS` environment variable. Cortex is model and vendor agnostic - you can use this config to set up models of any type from any vendor.
 - `openaiApiKey`: The API key used for accessing the OpenAI API. This is sensitive information and has no default value. The value can be set using the `OPENAI_API_KEY` environment variable.
 - `openaiApiUrl`: The URL used for accessing the OpenAI API. Default is https://api.openai.com/v1/completions. The value can be set using the `OPENAI_API_URL` environment variable.
@@ -440,6 +440,19 @@ The `config` object can be used to access configuration values throughout the pr
 ```js
 config.get('PORT')
 ```
+
+## Helper Apps
+The Cortex project includes a set of utility applications, which are located in the `helper-apps`` directory. Each of these applications comes with a Dockerfile. This Dockerfile can be used to create a Docker image of the application, which in turn allows the application to be run in a standalone manner using Docker.
+
+- cortex-file-handler
+Extends Cortex with several file processing units. Handles file operations (download, split, upload) with local file system or Azure Storage. It can process different file types including documents, files ( .pdf, .docx, .xlsx, .csv .txt, .json, .md, .xml, .js, .html, .css) and additionally YouTube URLs. It also manages deletion requests and cleanup operations, and provides progress reporting for requests.  
+
+- cortex-whisper-wrapper
+The cortex-whisper-wrapper is a custom API wrapper for the Whisper package from OpenAI. Designed as a FastAPI server, it aids in transcribing audio files using the Whisper library. 
+The server provides an HTTP endpoint ("/") that accepts POST requests with a JSON payload containing a "fileurl" parameter specifying the URL of the audio file to transcribe. Upon receiving a request, the server calls the transcribe function to perform the transcription using the Whisper model, saves the transcription as an SRT file, and returns the SRT content as the response.
+It helps Cortex to make use of Whisper OS parameters which currently are not available in OpenAI API. Parameters supported are: 'word_timestamps', 'highlight_words', 'max_line_count', 'max_line_width', 'max_words_per_line'. These parameters customizes transcription output, for more info on the parameters see open source Whisper package https://github.com/openai/whisper 
+
+
 ## Troubleshooting
 If you encounter any issues while using Cortex, there are a few things you can do. First, check the Cortex documentation for any common errors and their solutions. If that does not help, you can also open an issue on the Cortex GitHub repository.
 

package/config.js

@@ -8,6 +8,18 @@ import logger from './lib/logger.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
+convict.addFormat({
+    name: 'string-array',
+    validate: function(val) {
+      if (!Array.isArray(val)) {
+        throw new Error('must be of type Array');
+      }
+    },
+    coerce: function(val) {
+      return val.split(',');
+    },
+});
+
 // Schema for config
 var config = convict({
     env: {
@@ -30,8 +42,8 @@ var config = convict({
         default: path.join(__dirname, 'pathways'),
         env: 'CORTEX_CORE_PATHWAYS_PATH'
     },
-    cortexApiKey: {
-        format: String,
+    cortexApiKeys: {
+        format: 'string-array',
         default: null,
         env: 'CORTEX_API_KEY',
         sensitive: true
@@ -101,7 +113,7 @@ var config = convict({
             },
             "azure-cognitive": {
                 "type": "AZURE-COGNITIVE",
-                "url": "https://archipelago-cognitive-search.search.windows.net/indexes/indexcortex/docs/search?api-version=2023-07-01-Preview",
+                "url": "{{{AZURE_COGNITIVE_API_URL}}}",
                 "headers": {
                     "api-key": "{{AZURE_COGNITIVE_API_KEY}}",
                     "Content-Type": "application/json"
@@ -110,7 +122,7 @@ var config = convict({
             },
             "oai-embeddings": {
                 "type": "OPENAI-EMBEDDINGS",
-                "url": "https://archipelago-openai.openai.azure.com/openai/deployments/archipelago-embedding/embeddings?api-version=2023-05-15",
+                "url": "https://archipelago-openai.openai.azure.com/openai/deployments/archipelago-embedding/embeddings?api-version=2023-12-01",
                 "headers": {
                     "api-key": "{{ARCHIPELAGO_OPENAI_KEY}}",
                     "Content-Type": "application/json"
@@ -192,7 +204,7 @@ var config = convict({
     },
     whisperTSApiUrl: {
         format: String,
-        default: 'null',
+        default: null,
         env: 'WHISPER_TS_API_URL'
     },
     subscriptionKeepAlive: {
@@ -266,15 +278,39 @@ const buildPathways = async (config) => {
 const buildModels = (config) => {
     const { models } = config.getProperties();
 
-    for (const [key, model] of Object.entries(models)) {
-        // Compile handlebars templates for models
-        models[key] = JSON.parse(HandleBars.compile(JSON.stringify(model))({ ...config.getEnv(), ...config.getProperties() }))
+    // iterate over each model
+    for (let [key, model] of Object.entries(models)) {
+        if (!model.name) {
+            model.name = key;
+        }
+        
+        // if model is in old format, convert it to new format
+        if (!model.endpoints) {
+            model = {
+                ...model,
+                endpoints: [
+                    {
+                        name: "default",
+                        url: model.url,
+                        headers: model.headers,
+                        params: model.params,
+                        requestsPerSecond: model.requestsPerSecond
+                    }
+                ]
+            };
+        }
+
+        // compile handlebars templates for each endpoint
+        model.endpoints = model.endpoints.map(endpoint => 
+            JSON.parse(HandleBars.compile(JSON.stringify(endpoint))({ ...model, ...config.getEnv(), ...config.getProperties() }))
+        );
+
+        models[key] = model;
     }
 
     // Add constructed models to config
     config.load({ models });
 
-
     // Check that models are specified, Cortex cannot run without a model
     if (Object.keys(config.get('models')).length <= 0) {
         const errorString = 'No models specified! Please set the models in your config file or via CORTEX_MODELS environment variable to point at the models for your project.';

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/Dockerfile

@@ -8,7 +8,7 @@ RUN npm install
 
 ## installing ffmepg
 RUN apk update && \
-    apk add ffmpeg=6.0-r15
+    apk add ffmpeg
 
 COPY . .
 

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/fileChunker.js

@@ -6,6 +6,7 @@ import os from 'os';
 import { promisify } from 'util';
 import axios from 'axios';
 import { ensureEncoded } from './helper.js';
+import ytdl from 'ytdl-core';
 
 
 const ffmpegProbe = promisify(ffmpeg.ffprobe);
@@ -112,7 +113,9 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 600) {
 
         return { chunkPromises, uniqueOutputPath };
     } catch (err) {
-        console.error('Error occurred during the splitting process:', err);
+        const msg = `Error processing media file, check if the file is a valid media file or is accessible`;
+        console.error(msg, err);
+        throw new Error(msg);
     }
 }
 

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/package-lock.json

@@ -9,7 +9,6 @@
       "version": "1.0.0",
       "dependencies": {
         "@azure/storage-blob": "^12.13.0",
-        "@distube/ytdl-core": "^4.13.2",
         "axios": "^1.3.6",
         "busboy": "^1.6.0",
         "cors": "^2.8.5",
@@ -21,7 +20,8 @@
         "pdfjs-dist": "^3.9.179",
         "public-ip": "^6.0.1",
         "uuid": "^9.0.0",
-        "xlsx": "^0.18.5"
+        "xlsx": "^0.18.5",
+        "ytdl-core": "^4.11.5"
       }
     },
     "node_modules/@azure/abort-controller": {
@@ -156,29 +156,6 @@
         "node": ">=14.0.0"
       }
     },
-    "node_modules/@distube/ytdl-core": {
-      "version": "4.13.2",
-      "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.13.2.tgz",
-      "integrity": "sha512-FBjy4LXsBv34qFrmEF1dSbKWQtqB+yd0dt4MYYUq8oYys2Bvqps3QSFGTUZO5QA6nKD/XRsMJfcG5lgEJaUqiQ==",
-      "dependencies": {
-        "http-cookie-agent": "^5.0.4",
-        "m3u8stream": "^0.8.6",
-        "sax": "^1.2.4",
-        "tough-cookie": "^4.1.3",
-        "undici": "^5.25.2"
-      },
-      "engines": {
-        "node": ">=12"
-      }
-    },
-    "node_modules/@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
-      "engines": {
-        "node": ">=14"
-      }
-    },
     "node_modules/@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
@@ -1190,44 +1167,6 @@
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
       "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
     },
-    "node_modules/http-cookie-agent": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.4.tgz",
-      "integrity": "sha512-OtvikW69RvfyP6Lsequ0fN5R49S+8QcS9zwd58k6VSr6r57T8G29BkPdyrBcSwLq6ExLs9V+rBlfxu7gDstJag==",
-      "dependencies": {
-        "agent-base": "^7.1.0"
-      },
-      "engines": {
-        "node": ">=14.18.0 <15.0.0 || >=16.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/3846masa"
-      },
-      "peerDependencies": {
-        "deasync": "^0.1.26",
-        "tough-cookie": "^4.0.0",
-        "undici": "^5.11.0"
-      },
-      "peerDependenciesMeta": {
-        "deasync": {
-          "optional": true
-        },
-        "undici": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/http-cookie-agent/node_modules/agent-base": {
-      "version": "7.1.0",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
-      "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
-      "dependencies": {
-        "debug": "^4.3.4"
-      },
-      "engines": {
-        "node": ">= 14"
-      }
-    },
     "node_modules/http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -1889,11 +1828,6 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
-    "node_modules/psl": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
-      "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
-    },
     "node_modules/public-ip": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/public-ip/-/public-ip-6.0.1.tgz",
@@ -1911,14 +1845,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/punycode": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/qs": {
       "version": "6.11.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
@@ -1933,11 +1859,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
-    },
     "node_modules/quick-lru": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
@@ -2004,11 +1925,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
-    },
     "node_modules/resolve-alpn": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
@@ -2329,20 +2245,6 @@
         "node": ">=0.6"
       }
     },
-    "node_modules/tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
-      "dependencies": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
-      },
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@@ -2378,25 +2280,6 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
       "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
     },
-    "node_modules/undici": {
-      "version": "5.27.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.2.tgz",
-      "integrity": "sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==",
-      "dependencies": {
-        "@fastify/busboy": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=14.0"
-      }
-    },
-    "node_modules/universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
-      "engines": {
-        "node": ">= 4.0.0"
-      }
-    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -2405,15 +2288,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "dependencies": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -2544,6 +2418,19 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
+    },
+    "node_modules/ytdl-core": {
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
+      "dependencies": {
+        "m3u8stream": "^0.8.6",
+        "miniget": "^4.2.2",
+        "sax": "^1.1.3"
+      },
+      "engines": {
+        "node": ">=12"
+      }
     }
   },
   "dependencies": {
@@ -2651,23 +2538,6 @@
         "tslib": "^2.2.0"
       }
     },
-    "@distube/ytdl-core": {
-      "version": "4.13.2",
-      "resolved": "https://registry.npmjs.org/@distube/ytdl-core/-/ytdl-core-4.13.2.tgz",
-      "integrity": "sha512-FBjy4LXsBv34qFrmEF1dSbKWQtqB+yd0dt4MYYUq8oYys2Bvqps3QSFGTUZO5QA6nKD/XRsMJfcG5lgEJaUqiQ==",
-      "requires": {
-        "http-cookie-agent": "^5.0.4",
-        "m3u8stream": "^0.8.6",
-        "sax": "^1.2.4",
-        "tough-cookie": "^4.1.3",
-        "undici": "^5.25.2"
-      }
-    },
-    "@fastify/busboy": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
-      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA=="
-    },
     "@ioredis/commands": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz",
@@ -3433,24 +3303,6 @@
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz",
       "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="
     },
-    "http-cookie-agent": {
-      "version": "5.0.4",
-      "resolved": "https://registry.npmjs.org/http-cookie-agent/-/http-cookie-agent-5.0.4.tgz",
-      "integrity": "sha512-OtvikW69RvfyP6Lsequ0fN5R49S+8QcS9zwd58k6VSr6r57T8G29BkPdyrBcSwLq6ExLs9V+rBlfxu7gDstJag==",
-      "requires": {
-        "agent-base": "^7.1.0"
-      },
-      "dependencies": {
-        "agent-base": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz",
-          "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==",
-          "requires": {
-            "debug": "^4.3.4"
-          }
-        }
-      }
-    },
     "http-errors": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz",
@@ -3946,11 +3798,6 @@
       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
       "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
     },
-    "psl": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
-      "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
-    },
     "public-ip": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/public-ip/-/public-ip-6.0.1.tgz",
@@ -3962,11 +3809,6 @@
         "is-ip": "^4.0.0"
       }
     },
-    "punycode": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
-      "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
-    },
     "qs": {
       "version": "6.11.0",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
@@ -3975,11 +3817,6 @@
         "side-channel": "^1.0.4"
       }
     },
-    "querystringify": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
-      "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
-    },
     "quick-lru": {
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
@@ -4025,11 +3862,6 @@
         "redis-errors": "^1.0.0"
       }
     },
-    "requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
-    },
     "resolve-alpn": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz",
@@ -4271,17 +4103,6 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
       "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="
     },
-    "tough-cookie": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
-      "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
-      "requires": {
-        "psl": "^1.1.33",
-        "punycode": "^2.1.1",
-        "universalify": "^0.2.0",
-        "url-parse": "^1.5.3"
-      }
-    },
     "tr46": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
@@ -4311,33 +4132,11 @@
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz",
       "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A=="
     },
-    "undici": {
-      "version": "5.27.2",
-      "resolved": "https://registry.npmjs.org/undici/-/undici-5.27.2.tgz",
-      "integrity": "sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==",
-      "requires": {
-        "@fastify/busboy": "^2.0.0"
-      }
-    },
-    "universalify": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
-      "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg=="
-    },
     "unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
       "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
     },
-    "url-parse": {
-      "version": "1.5.10",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
-      "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
-      "requires": {
-        "querystringify": "^2.1.1",
-        "requires-port": "^1.0.0"
-      }
-    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -4438,6 +4237,16 @@
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
       "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
       "optional": true
+    },
+    "ytdl-core": {
+      "version": "4.11.5",
+      "resolved": "https://registry.npmjs.org/ytdl-core/-/ytdl-core-4.11.5.tgz",
+      "integrity": "sha512-27LwsW4n4nyNviRCO1hmr8Wr5J1wLLMawHCQvH8Fk0hiRqrxuIu028WzbJetiYH28K8XDbeinYW4/wcHQD1EXA==",
+      "requires": {
+        "m3u8stream": "^0.8.6",
+        "miniget": "^4.2.2",
+        "sax": "^1.1.3"
+      }
     }
   }
 }

package/{helper_apps/CortexFileHandler → helper-apps/cortex-file-handler}/package.json

@@ -9,7 +9,6 @@
   },
   "dependencies": {
     "@azure/storage-blob": "^12.13.0",
-    "@distube/ytdl-core": "^4.13.2",
     "axios": "^1.3.6",
     "busboy": "^1.6.0",
     "cors": "^2.8.5",
@@ -21,6 +20,7 @@
     "pdfjs-dist": "^3.9.179",
     "public-ip": "^6.0.1",
     "uuid": "^9.0.0",
-    "xlsx": "^0.18.5"
+    "xlsx": "^0.18.5",
+    "ytdl-core": "^4.11.5"
   }
 }

package/helper-apps/cortex-whisper-wrapper/.dockerignore

@@ -0,0 +1,27 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

package/helper-apps/cortex-whisper-wrapper/Dockerfile

@@ -0,0 +1,32 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:3.10-slim
+
+EXPOSE 8000
+
+## following 3 lines are for installing ffmepg
+RUN apt-get -y update
+RUN apt-get -y upgrade
+RUN apt-get install -y ffmpeg
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+# Install pip requirements
+COPY requirements.txt .
+RUN python -m pip install -r requirements.txt
+
+WORKDIR /app
+COPY ./models /app/models
+COPY . /app
+
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
+USER appuser
+
+# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
+# CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout", "0", "-k", "uvicorn.workers.UvicornWorker", "app:app"]