@aj-archipelago/cortex 1.0.5 → 1.0.7

package/README.md

@@ -198,8 +198,8 @@ The core pathway `summary.js` below is implemented using custom pathway logic an
 // This module exports a prompt that takes an input text and generates a summary using a custom resolver.
 
 // Import required modules
-import { semanticTruncate } from '../graphql/chunker.js';
-import { PathwayResolver } from '../graphql/pathwayResolver.js';
+import { semanticTruncate } from '../server/chunker.js';
+import { PathwayResolver } from '../server/pathwayResolver.js';
 
 export default {
     // The main prompt function that takes the input text and asks to generate a summary.

package/config/default.example.json

@@ -58,7 +58,8 @@
                 "Content-Type": "application/json"
             },
             "requestsPerSecond": 10,
-            "maxTokenLength": 2048
+            "maxTokenLength": 2048,
+            "maxReturnTokens": 1024
         },
         "palm-chat": {
             "type": "PALM-CHAT",
@@ -67,7 +68,8 @@
                 "Content-Type": "application/json"
             },
             "requestsPerSecond": 10,
-            "maxTokenLength": 2048
+            "maxTokenLength": 2048,
+            "maxReturnTokens": 1024   
         },
         "local-llama13B": {
             "type": "LOCAL-CPP-MODEL",

package/config.js

@@ -69,20 +69,21 @@ var config = convict({
     models: {
         format: Object,
         default: {
-            "oai-td3": {
-                "type": "OPENAI-COMPLETION",
-                "url": "{{openaiApiUrl}}",
+            "oai-gpturbo": {
+                "type": "OPENAI-CHAT",
+                "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
-                    "Authorization": "Bearer {{openaiApiKey}}",
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
                     "Content-Type": "application/json"
                 },
                 "params": {
-                    "model": "{{openaiDefaultModel}}"
+                    "model": "gpt-3.5-turbo"
                 },
-                "requestsPerSecond": 2,
+                "requestsPerSecond": 10,
+                "maxTokenLength": 8192
             },
             "oai-whisper": {
-                "type": "OPENAI_WHISPER",
+                "type": "OPENAI-WHISPER",
                 "url": "https://api.openai.com/v1/audio/transcriptions",
                 "headers": {
                     "Authorization": "Bearer {{OPENAI_API_KEY}}"
@@ -96,7 +97,7 @@ var config = convict({
     },
     openaiDefaultModel: {
         format: String,
-        default: 'text-davinci-003',
+        default: 'gpt-3.5-turbo',
         env: 'OPENAI_DEFAULT_MODEL'
     },
     openaiApiKey: {
@@ -120,6 +121,11 @@ var config = convict({
         default: 'null',
         env: 'WHISPER_MEDIA_API_URL'
     },
+    whisperTSApiUrl: {
+        format: String,
+        default: 'null',
+        env: 'WHISPER_TS_API_URL'
+    },
     gcpServiceAccountKey: {
         format: String,
         default: null,

package/helper_apps/WhisperX/.dockerignore

@@ -0,0 +1,27 @@
+**/__pycache__
+**/.venv
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/bin
+**/charts
+**/docker-compose*
+**/compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md

package/helper_apps/WhisperX/Dockerfile

@@ -0,0 +1,31 @@
+# For more information, please refer to https://aka.ms/vscode-docker-python
+FROM python:3.10-slim
+
+EXPOSE 8000
+
+## following 3 lines are for installing ffmepg
+RUN apt-get -y update
+RUN apt-get -y upgrade
+RUN apt-get install -y ffmpeg
+
+# Keeps Python from generating .pyc files in the container
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Turns off buffering for easier container logging
+ENV PYTHONUNBUFFERED=1
+
+# Install pip requirements
+COPY requirements.txt .
+RUN python -m pip install -r requirements.txt
+
+WORKDIR /app
+COPY ./models /app/models
+COPY . /app
+
+# Creates a non-root user with an explicit UID and adds permission to access the /app folder
+# For more info, please refer to https://aka.ms/vscode-docker-python-configure-containers
+RUN adduser -u 5678 --disabled-password --gecos "" appuser && chown -R appuser /app
+USER appuser
+
+# During debugging, this entry point will be overridden. For more information, please refer to https://aka.ms/vscode-docker-python-debug
+CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout", "0", "-k", "uvicorn.workers.UvicornWorker", "app:app"]

package/helper_apps/WhisperX/app-ts.py

@@ -0,0 +1,76 @@
+import uvicorn
+from fastapi import FastAPI
+import stable_whisper
+from uuid import uuid4
+import requests
+import os
+
+model_download_root = './models'
+model = stable_whisper.load_model('large', download_root=model_download_root) #large, tiny
+
+app = FastAPI()
+
+save_directory = "./tmp"  # folder for downloaded files
+os.makedirs(save_directory, exist_ok=True)
+
+
+def download_remote_file(url, save_directory):
+    # Generate a unique file name with a UUID
+    unique_name = str(uuid4()) + os.path.splitext(url)[-1]
+    save_path = os.path.join(save_directory, unique_name)
+
+    # Download the remote file
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    # Save the downloaded file with the unique name
+    with open(save_path, 'wb') as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+
+    return [unique_name, save_path]
+
+
+def delete_tmp_file(file_path):
+    try:
+        os.remove(file_path)
+        print(f"Temporary file '{file_path}' has been deleted.")
+    except OSError as e:
+        print(f"Error: {e.strerror}")
+
+
+async def transcribe(fileurl):
+    print(f"Downloading file from: {fileurl}")
+    [unique_file_name, save_path] = download_remote_file(
+        fileurl, save_directory)
+    print(f"Downloaded file saved as: {unique_file_name}")
+
+    print(f"Transcribing file")
+    result = model.transcribe(save_path)
+
+    srtpath = os.path.join(save_directory, str(uuid4()) + ".srt")
+
+    print(f"Saving transcription as : {srtpath}")
+    result.to_srt_vtt(srtpath, segment_level=False)
+
+    with open(srtpath, "r") as f:
+        srtstr = f.read()
+
+    # clean up tmp files
+    delete_tmp_file(save_path)
+    delete_tmp_file(srtpath)
+
+    print(f"Transcription done.")
+    return srtstr
+
+
+@app.get("/")
+async def root(fileurl: str):
+    if not fileurl:
+        return "No fileurl given!"
+    result = await transcribe(fileurl)
+    return result
+
+if __name__ == "__main__":
+    print("Starting server", flush=True)
+    uvicorn.run(app, host="0.0.0.0", port=8000)

package/helper_apps/WhisperX/app.py

@@ -0,0 +1,115 @@
+import uvicorn
+from fastapi import FastAPI, HTTPException, Request
+from uuid import uuid4
+import os
+import requests
+import asyncio
+import whisper
+from whisper.utils import get_writer
+from fastapi.encoders import jsonable_encoder
+
+model_download_root = './models'
+model = whisper.load_model("large", download_root=model_download_root) #large, tiny
+
+# Create a semaphore with a limit of 1
+semaphore = asyncio.Semaphore(1)
+
+app = FastAPI()
+
+save_directory = "./tmp"  # folder for downloaded files
+os.makedirs(save_directory, exist_ok=True)
+
+
+def download_remote_file(url, save_directory):
+    # Generate a unique file name with a UUID
+    unique_name = str(uuid4()) + os.path.splitext(url)[-1]
+    save_path = os.path.join(save_directory, unique_name)
+
+    # Download the remote file
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    # Save the downloaded file with the unique name
+    with open(save_path, 'wb') as file:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+
+    return [unique_name, save_path]
+
+def delete_tmp_file(file_path):
+    try:
+        os.remove(file_path)
+        print(f"Temporary file '{file_path}' has been deleted.")
+    except OSError as e:
+        print(f"Error: {e.strerror}")
+
+def modify_segments(result):
+    modified_segments = []
+    
+    id = 0
+    for segment in result["segments"]:
+        for word_info in segment['words']:
+            word = word_info['word']
+            start = word_info['start']
+            end = word_info['end']
+            
+            modified_segment = {} #segment.copy()
+            modified_segment['id'] = id
+            modified_segment['text'] = word
+            modified_segment['start'] = start
+            modified_segment['end'] = end
+            modified_segments.append(modified_segment)
+            id+=1
+    
+    result["segments"] = modified_segments
+    
+def transcribe(fileurl):
+    print(f"Downloading file from: {fileurl}")
+    [unique_file_name, save_path] = download_remote_file(
+        fileurl, save_directory)
+    print(f"Downloaded file saved as: {unique_file_name}")
+
+    print(f"Transcribing file")
+    result = model.transcribe(save_path, word_timestamps=True)
+
+    modify_segments(result)
+
+    srtpath = os.path.join(save_directory, str(uuid4()) + ".srt")
+
+    print(f"Saving transcription as : {srtpath}")
+    writer = get_writer("srt", save_directory)
+    with open(srtpath, 'w', encoding='utf-8') as file_obj :
+        writer.write_result(result, file_obj)
+
+    with open(srtpath, "r") as f:
+        srtstr = f.read()
+
+    # clean up tmp files
+    delete_tmp_file(save_path)
+    delete_tmp_file(srtpath)
+
+    print(f"Transcription done.")
+    return srtstr
+
+
+@app.get("/")
+@app.post("/")
+async def root(request: Request):
+    if request.method == "POST":
+        body = jsonable_encoder(await request.json())
+        fileurl = body.get("fileurl")
+    else:
+        fileurl = request.query_params.get("fileurl")
+    if not fileurl:
+        return "No fileurl given!"
+
+    if semaphore.locked():
+        raise HTTPException(status_code=429, detail="Too Many Requests")
+    
+    async with semaphore:
+        result = await asyncio.to_thread(transcribe, fileurl)
+        return result
+
+if __name__ == "__main__":
+    print("Starting APPWhisper server", flush=True)
+    uvicorn.run(app, host="0.0.0.0", port=8000)

package/helper_apps/WhisperX/docker-compose.debug.yml

@@ -0,0 +1,12 @@
+version: '3.4'
+
+services:
+  cortex:
+    image: cortex
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    command: ["sh", "-c", "pip install debugpy -t /tmp && python /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 -m uvicorn helper_apps.WhisperX/app:app --host 0.0.0.0 --port 8000"]
+    ports:
+      - 8000:8000
+      - 5678:5678

package/helper_apps/WhisperX/docker-compose.yml

@@ -0,0 +1,10 @@
+version: '3.4'
+
+services:
+  cortex:
+    image: cortex
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    ports:
+      - 8000:8000

package/helper_apps/WhisperX/requirements.txt

@@ -0,0 +1,6 @@
+# To ensure app dependencies are ported from your virtual environment/host machine into your container, run 'pip freeze > requirements.txt' in the terminal to overwrite this file
+fastapi[all]==0.89.0
+uvicorn[standard]==0.20.0
+gunicorn==20.1.0
+whisper
+stable-ts

package/index.js

@@ -1,5 +1,5 @@
 import { config } from './config.js';
-import { build } from './graphql/graphql.js';
+import { build } from './server/graphql.js';
 
 export default async (configParams) => {
     configParams && config.load(configParams);

package/lib/redisSubscription.js

@@ -1,6 +1,6 @@
 import Redis from 'ioredis';
 import { config } from '../config.js';
-import pubsub from '../graphql/pubsub.js';
+import pubsub from '../server/pubsub.js';
 
 const connectionString = config.get('storageConnectionString');
 const client = new Redis(connectionString);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "repository": {
     "type": "git",
@@ -28,16 +28,15 @@
   "type": "module",
   "homepage": "https://github.com/aj-archipelago/cortex#readme",
   "dependencies": {
-    "@apollo/utils.keyvadapter": "^1.1.2",
+    "@apollo/server": "^4.7.3",
+    "@apollo/server-plugin-response-cache": "^4.1.2",
+    "@apollo/utils.keyvadapter": "^3.0.0",
     "@graphql-tools/schema": "^9.0.12",
     "@keyv/redis": "^2.5.4",
-    "apollo-server": "^3.12.0",
-    "apollo-server-core": "^3.11.1",
-    "apollo-server-express": "^3.11.1",
-    "apollo-server-plugin-response-cache": "^3.8.1",
     "axios": "^1.3.4",
     "axios-cache-interceptor": "^1.0.1",
     "bottleneck": "^2.19.5",
+    "cheerio": "^1.0.0-rc.12",
     "compromise": "^14.8.1",
     "compromise-paragraphs": "^0.1.0",
     "convict": "^6.2.3",
@@ -61,6 +60,7 @@
     "dotenv": "^16.0.3",
     "eslint": "^8.38.0",
     "eslint-plugin-import": "^2.27.5",
+    "got": "^13.0.0",
     "sinon": "^15.0.3"
   },
   "publishConfig": {
@@ -72,6 +72,7 @@
     ],
     "require": [
       "dotenv/config"
-    ]
+    ],
+    "concurrency": 1
   }
 }

package/pathways/basePathway.js

@@ -1,5 +1,5 @@
-import { rootResolver, resolver } from '../graphql/resolver.js';
-import { typeDef } from '../graphql/typeDef.js';
+import { rootResolver, resolver } from '../server/resolver.js';
+import { typeDef } from '../server/typeDef.js';
 
 // all default definitions of a single pathway
 export default {
@@ -14,6 +14,7 @@ export default {
     typeDef,
     rootResolver,
     resolver,
+    inputFormat: 'text',
     useInputChunking: true,
     useParallelChunkProcessing: false,
     useInputSummarization: false,    

package/pathways/index.js

@@ -6,6 +6,8 @@ import entities from './entities.js';
 import paraphrase from './paraphrase.js';
 import sentiment from './sentiment.js';
 import summary from './summary.js';
+import sys_openai_chat from './sys_openai_chat.js';
+import sys_openai_completion from './sys_openai_completion.js';
 import test_langchain from './test_langchain.mjs';
 import test_palm_chat from './test_palm_chat.js';
 import transcribe from './transcribe.js';
@@ -20,6 +22,8 @@ export {
     paraphrase,
     sentiment,
     summary,
+    sys_openai_chat,
+    sys_openai_completion,
     test_langchain,
     test_palm_chat,
     transcribe,

package/pathways/summary.js

@@ -3,8 +3,8 @@
 // This module exports a prompt that takes an input text and generates a summary using a custom resolver.
 
 // Import required modules
-import { semanticTruncate } from '../graphql/chunker.js';
-import { PathwayResolver } from '../graphql/pathwayResolver.js';
+import { semanticTruncate } from '../server/chunker.js';
+import { PathwayResolver } from '../server/pathwayResolver.js';
 
 export default {
     // The main prompt function that takes the input text and asks to generate a summary.

package/pathways/sys_openai_chat.js

@@ -0,0 +1,19 @@
+// sys_openai_chat.js
+// default handler for openAI chat endpoints when REST endpoints are enabled
+
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt:
+    [
+        new Prompt({ messages: [
+            "{{messages}}",
+        ]}),
+    ],
+    inputParameters: {
+        messages: [],
+    },
+    model: 'oai-gpturbo',
+    useInputChunking: false,
+    emulateOpenAIChatModel: '*',
+}

package/pathways/sys_openai_completion.js

@@ -0,0 +1,11 @@
+// sys_openai_completion.js
+// default handler for openAI completion endpoints when REST endpoints are enabled
+
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: `{{text}}`,
+    model: 'oai-gpturbo',
+    useInputChunking: false,
+    emulateOpenAICompletionModel: '*',
+}

package/pathways/test_palm_chat.js

@@ -1,7 +1,7 @@
 //test_palm_chat.mjs
 // Test for handling of prompts in the PaLM chat format for Cortex
 
-import { Prompt } from '../graphql/prompt.js';
+import { Prompt } from '../server/prompt.js';
 
 // Description: Have a chat with a bot that uses context to understand the conversation
 export default {

package/pathways/transcribe.js

@@ -5,8 +5,9 @@ export default {
         file: ``,
         language: ``,
         responseFormat: `text`,
+        wordTimestamped: false,
     },
-    timeout: 1800, // in seconds
+    timeout: 3600, // in seconds
 };
 
 

package/{graphql → server}/chunker.js

@@ -1,4 +1,5 @@
 import { encode, decode } from 'gpt-3-encoder';
+import cheerio from 'cheerio';
 
 const getLastNToken = (text, maxTokenLen) => { 
     const encoded = encode(text);
@@ -18,8 +19,18 @@ const getFirstNToken = (text, maxTokenLen) => {
     return text;
 }
 
-const getSemanticChunks = (text, chunkSize) => {
+const determineTextFormat = (text) => {
+  const htmlTagPattern = /<[^>]*>/g;
+  
+  if (htmlTagPattern.test(text)) {
+    return 'html';
+  }
+  else {
+    return 'text';
+  }
+}
 
+const getSemanticChunks = (text, chunkSize, inputFormat = 'text') => {
   const breakByRegex = (str, regex, preserveWhitespace = false) => {
     const result = [];
     let match;
@@ -46,6 +57,19 @@ const getSemanticChunks = (text, chunkSize) => {
   const breakBySentences = (str) => breakByRegex(str, /(?<=[.。؟！?!\n])\s+/, true);
   const breakByWords = (str) => breakByRegex(str, /(\s,;:.+)/);
 
+  const breakByHtmlElements = (str) => {
+    const $ = cheerio.load(str, null, true);
+
+    // the .filter() call is important to get the text nodes
+    // https://stackoverflow.com/questions/54878673/cheerio-get-normal-text-nodes
+    let rootNodes = $('body').contents();
+
+    // create an array with the outerHTML of each node
+    const nodes = rootNodes.map((i, el) => $(el).prop('outerHTML') || $(el).text()).get();
+
+    return nodes;
+};
+
   const createChunks = (tokens) => {
     let chunks = [];
     let currentChunk = '';
@@ -115,7 +139,28 @@ const getSemanticChunks = (text, chunkSize) => {
     return createChunks([...str]); // Split by characters
   };
 
-  return breakText(text);
+  if (inputFormat === 'html') {
+    const tokens = breakByHtmlElements(text);
+    let chunks = createChunks(tokens);
+    chunks = combineChunks(chunks);
+
+    chunks = chunks.flatMap(chunk => {
+      if (determineTextFormat(chunk) === 'text') {
+        return getSemanticChunks(chunk, chunkSize);
+      } else {
+        return chunk;
+      }
+    });
+
+    if (chunks.some(chunk => encode(chunk).length > chunkSize)) {
+      throw new Error('The HTML contains elements that are larger than the chunk size. Please try again with HTML that has smaller elements.');
+    }
+
+    return chunks;
+  }
+  else {
+      return breakText(text);
+  }
 }
 
 
@@ -133,5 +178,5 @@ const semanticTruncate = (text, maxLength) => {
 };
 
 export {
-    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken
+    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat
 };