@aj-archipelago/cortex 1.1.31 → 1.1.33

package/config.js

@@ -154,6 +154,36 @@ var config = convict({
                 "maxReturnTokens": 4096,
                 "supportsStreaming": true
             },
+            "oai-o1-mini": {
+                "type": "OPENAI-REASONING",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "o1-mini"
+                },
+                "requestsPerSecond": 10,
+                "maxTokenLength": 128000,
+                "maxReturnTokens": 65536,
+                "supportsStreaming": false
+            },
+            "oai-o1-preview": {
+                "type": "OPENAI-REASONING",
+                "url": "https://api.openai.com/v1/chat/completions",
+                "headers": {
+                    "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                    "Content-Type": "application/json"
+                },
+                "params": {
+                    "model": "o1-preview"
+                },
+                "requestsPerSecond": 10,
+                "maxTokenLength": 128000,
+                "maxReturnTokens": 32768,
+                "supportsStreaming": false
+            },
             "azure-bing": {
                 "type": "AZURE-BING",
                 "url": "https://api.bing.microsoft.com/v7.0/search",

package/helper-apps/cortex-autogen/.funcignore

@@ -0,0 +1,8 @@
+.git*
+.vscode
+__azurite_db*__.json
+__blobstorage__
+__queuestorage__
+local.settings.json
+test
+.venv

package/helper-apps/cortex-autogen/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "main.py"]

package/helper-apps/cortex-autogen/OAI_CONFIG_LIST

@@ -0,0 +1,5 @@
+[
+    {
+        "model": "claude-3.5-sonnet"
+    }
+]

package/helper-apps/cortex-autogen/function_app.py

@@ -0,0 +1,29 @@
+import azure.functions as func
+import logging
+import json
+from azure.storage.queue import QueueClient
+import os
+import redis
+from myautogen import process_message
+
+app = func.FunctionApp()
+
+connection_string = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
+queue_name = os.environ.get("QUEUE_NAME", "autogen-message-queue")
+queue_client = QueueClient.from_connection_string(connection_string, queue_name)
+
+redis_client = redis.from_url(os.environ['REDIS_CONNECTION_STRING'])
+channel = 'requestProgress'
+
+
+@app.queue_trigger(arg_name="msg", queue_name=queue_name, connection="AZURE_STORAGE_CONNECTION_STRING")
+def queue_trigger(msg: func.QueueMessage):
+    logging.info(f"Queue trigger Message ID: {msg.id}")
+    try:
+        message_data = json.loads(msg.get_body().decode('utf-8'))
+        if "requestId" not in message_data:
+            message_data['requestId'] = msg.id
+        process_message(message_data, msg)
+
+    except Exception as e:
+        logging.error(f"Error processing message: {str(e)}")

package/helper-apps/cortex-autogen/host.json

@@ -0,0 +1,15 @@
+{
+  "version": "2.0",
+  "logging": {
+    "applicationInsights": {
+      "samplingSettings": {
+        "isEnabled": true,
+        "excludedTypes": "Request"
+      }
+    }
+  },
+  "extensionBundle": {
+    "id": "Microsoft.Azure.Functions.ExtensionBundle",
+    "version": "[4.*, 5.0.0)"
+  }
+}

package/helper-apps/cortex-autogen/main.py

@@ -0,0 +1,38 @@
+import os
+from azure.storage.queue import QueueClient
+import base64
+import json
+from myautogen import process_message
+import time
+
+def main():
+    print("Starting message processing loop")
+    connection_string = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
+    queue_name = os.environ.get("QUEUE_NAME", "autogen-message-queue")
+
+    queue_client = QueueClient.from_connection_string(connection_string, queue_name)
+    
+    attempts = 0
+    max_attempts = 100
+    
+
+    while attempts < max_attempts:
+        messages = queue_client.receive_messages(messages_per_page=1)
+        
+        if messages:
+            for message in messages:
+                decoded_content = base64.b64decode(message.content).decode('utf-8')
+                message_data = json.loads(decoded_content)
+                if "requestId" not in message_data:
+                    message_data['requestId'] = message.id
+                process_message(message_data, message)
+                queue_client.delete_message(message)
+            attempts = 0  # Reset attempts if a message was processed
+        else:
+            attempts += 1
+            time.sleep(1)  # Wait for 1 second before checking again
+
+    print("No messages received after 100 attempts. Exiting.")
+
+if __name__ == "__main__":
+    main()

package/helper-apps/cortex-autogen/myautogen.py

@@ -0,0 +1,228 @@
+import azure.functions as func
+import logging
+import json
+import autogen
+from autogen import AssistantAgent, UserProxyAgent, config_list_from_json, register_function
+from azure.storage.queue import QueueClient
+import os
+import tempfile
+import redis
+from dotenv import load_dotenv
+import requests
+import pathlib
+import pymongo
+import logging
+from datetime import datetime, timezone
+from tools.sasfileuploader import autogen_sas_uploader
+import shutil
+load_dotenv()
+
+DEFAULT_SUMMARY_PROMPT = "Summarize the takeaway from the conversation. Do not add any introductory phrases."
+try:
+    with open("prompt_summary.txt", "r") as file:
+        summary_prompt = file.read() or DEFAULT_SUMMARY_PROMPT
+except FileNotFoundError:
+    summary_prompt = DEFAULT_SUMMARY_PROMPT 
+
+
+def store_in_mongo(data):
+    try:
+        if 'MONGO_URI' in os.environ:
+            client = pymongo.MongoClient(os.environ['MONGO_URI'])
+            collection = client.get_default_database()[os.environ.get('MONGO_COLLECTION_NAME', 'autogenruns')]
+            collection.insert_one(data)
+        else:
+            logging.warning("MONGO_URI not found in environment variables")
+    except Exception as e:
+        logging.error(f"An error occurred while storing data in MongoDB: {str(e)}")
+
+app = func.FunctionApp()
+
+connection_string = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
+queue_name = os.environ.get("QUEUE_NAME", "autogen-message-queue")
+queue_client = QueueClient.from_connection_string(connection_string, queue_name)
+
+redis_client = redis.from_url(os.environ['REDIS_CONNECTION_STRING'])
+channel = 'requestProgress'
+
+def connect_redis():
+    if not redis_client.ping():
+        try:
+            redis_client.ping()
+        except redis.ConnectionError as e:
+            logging.error(f"Error reconnecting to Redis: {e}")
+            return False
+    return True
+
+def publish_request_progress(data):
+    if connect_redis():
+        try:
+            message = json.dumps(data)
+            logging.info(f"Publishing message {message} to channel {channel}")
+            redis_client.publish(channel, message)
+        except Exception as e:
+            logging.error(f"Error publishing message: {e}")
+
+
+def get_given_system_message():
+    env_context = os.environ.get("ENV_SYSTEM_MESSAGE_CONTEXT")
+    
+    if not env_context:
+        return read_local_file("prompt.txt")
+
+    if env_context.startswith(("http://", "https://")):
+        return fetch_from_url(env_context)
+
+    if pathlib.Path(env_context).suffix:
+        return read_local_file(env_context)
+
+    return env_context
+
+def read_local_file(filename):
+    try:
+        with open(filename, "r") as file:
+            return file.read()
+    except FileNotFoundError:
+        logging.error(f"{filename} not found")
+        return ""
+
+def fetch_from_url(url):
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.text
+    except requests.RequestException as e:
+        logging.error(f"Error fetching from URL: {e}")
+        return ""
+
+def process_message(message_data, original_request_message):
+    logging.info(f"Processing Message: {message_data}")
+    try:
+        started_at = datetime.now()
+        message = message_data['message']
+        request_id = message_data.get('requestId') or msg.id
+
+        config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
+        base_url = os.environ.get("CORTEX_API_BASE_URL")
+        api_key = os.environ.get("CORTEX_API_KEY")
+        llm_config = {"config_list": config_list, "base_url": base_url, "api_key": api_key, "cache_seed": None, "timeout": 600}
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            #copy /tools directory to temp_dir
+            shutil.copytree(os.path.join(os.getcwd(), "tools"), temp_dir, dirs_exist_ok=True)
+            
+            code_executor = autogen.coding.LocalCommandLineCodeExecutor(work_dir=temp_dir)
+
+            message_count = 0
+            total_messages = 20 * 2
+            all_messages = []
+
+            def is_termination_msg(m):
+                content = m.get("content", "")
+                if message_count == 0:
+                    return False
+                return (m.get("role") == "assistant" and not content.strip()) or \
+                    content.rstrip().endswith("TERMINATE") or \
+                    "first message must use the" in content.lower() or \
+                    len(content.strip()) == 0
+
+            system_message_given = get_given_system_message()
+            system_message_assistant = AssistantAgent.DEFAULT_SYSTEM_MESSAGE 
+
+            if system_message_given:
+                system_message_assistant = system_message_given
+            else:
+                print("No extra system message given for assistant")
+
+            assistant = AssistantAgent("assistant", 
+                llm_config=llm_config, 
+                system_message=system_message_assistant,
+                code_execution_config={"executor": code_executor},
+                is_termination_msg=is_termination_msg,
+            )
+            
+            user_proxy = UserProxyAgent(
+                "user_proxy",
+                llm_config=llm_config,
+                system_message=system_message_given,
+                code_execution_config={"executor": code_executor},
+                human_input_mode="NEVER",
+                max_consecutive_auto_reply=20,
+            )
+
+            # description = "Upload a file to Azure Blob Storage and get URL back with a SAS token. Requires AZURE_STORAGE_CONNECTION_STRING and AZURE_BLOB_CONTAINER environment variables. Input: file_path (str). Output: SAS URL (str) or error message."
+
+            # register_function(
+            #     autogen_sas_uploader,
+            #     caller=assistant,
+            #     executor=user_proxy,
+            #     name="autogen_sas_uploader",
+            #     description=description,    
+            # )
+
+            # register_function(
+            #     autogen_sas_uploader,
+            #     caller=user_proxy,
+            #     executor=assistant,
+            #     name="autogen_sas_uploader",
+            #     description=description,    
+            # )
+
+            original_assistant_send = assistant.send
+            original_user_proxy_send = user_proxy.send
+
+            def logged_send(sender, original_send, message, recipient, request_reply=None, silent=True):
+                nonlocal message_count, all_messages
+                if not message:
+                    return
+                logging.info(f"Message from {sender.name} to {recipient.name}: {message}")
+                message_count += 1
+                progress = min(message_count / total_messages, 1)
+                all_messages.append({"sender": sender.name, "message": message})
+                publish_request_progress({
+                    "requestId": request_id,
+                    "progress": progress,
+                    "info": message
+                })
+                return original_send(message, recipient, request_reply, silent)
+
+            assistant.send = lambda message, recipient, request_reply=None, silent=False: logged_send(assistant, original_assistant_send, message, recipient, request_reply, silent)
+            user_proxy.send = lambda message, recipient, request_reply=None, silent=False: logged_send(user_proxy, original_user_proxy_send, message, recipient, request_reply, silent)
+
+            #summary_method="reflection_with_llm", "last_msg"
+            chat_result = user_proxy.initiate_chat(assistant, message=message, summary_method="reflection_with_llm", summary_args={"summary_role": "user", "summary_prompt": summary_prompt})
+
+            msg = ""
+            try:
+                msg = all_messages[-1 if all_messages[-2]["message"] else -3]["message"] 
+                logging.info(f"####Final message: {msg}")     
+            except Exception as e:
+                logging.error(f"Error getting final message: {e}")
+                msg = f"Finished, with errors 🤖 ... {e}"
+
+            msg = chat_result.summary if chat_result.summary else msg
+
+            finalData = {
+                "requestId": request_id,
+                "requestMessage": message_data.get("message"),
+                "progress": 1,
+                "data": msg,
+                "contextId": message_data.get("contextId"),
+                "conversation": all_messages,
+                "createdAt": datetime.now(timezone.utc).isoformat(),
+                "insertionTime": original_request_message.insertion_time.astimezone(timezone.utc).isoformat() if original_request_message else None,
+                "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+            }       
+
+            # Final message to indicate completion
+            publish_request_progress(finalData)
+            store_in_mongo(finalData)
+
+    except Exception as e:
+        logging.error(f"Error processing message: {str(e)}")
+        if request_id:
+            publish_request_progress({
+                "requestId": request_id,
+                "progress": 1,
+                "error": str(e)
+            })

package/helper-apps/cortex-autogen/prompt_summary.txt

@@ -0,0 +1,28 @@
+Provide a detailed summary of the conversation, including key points, decisions, and action items, and so on.
+Do not add any introductory phrases.
+Avoid expressing gratitude or using pleasantries.
+Maintain a professional and direct tone throughout responses.
+Include most recent meaningful messages from the conversation in the summary.
+You must include all your uploaded URLs, and url of your uploaded final code URL.
+Reply must be in markdown format, including images and videos as UI can show markdown directly to user in a nice way, so make sure to include all visuals, you may do as follows:
+    For images: ![Alt Text](IMAGE_URL)
+    For videos: <video src="VIDEO_URL" controls></video>
+    For urls: [Link Text](URL)
+Your reply will be only thing that finally gets to surface so make sure it is complete.
+Do not mention words like "Summary of the conversation", "Response", "Task", "The conversation" or so as it doesn't makes sense. 
+Also no need for "Request", user already know its request and task.
+Be as detailed as possible without being annoying.
+Start with the result as that is the most important part, do not mention "Result" as user already know its result.
+No need to say information about generated SAS urls just include them, only include the latest versions of same file.
+No need to say none of this as user already 'll be aware as has got the result: 
+    - Code executed successfully, producing correct result ...
+    - File uploaded to Azure Blob Storage with unique timestamp ...
+    - SAS URL generated for file access, valid for ...
+    - File accessibility verified ...
+    - Code execution details ... 
+    - Current date and time ...
+    - Script executed twice due to debugging environment ...
+    - Verification code ...
+    - Issues encountered and resolved: ...
+
+

package/helper-apps/cortex-autogen/requirements.txt

@@ -0,0 +1,8 @@
+azure-storage-queue
+azure-functions
+pyautogen
+redis
+pymongo
+requests
+azure-storage-blob
+mysql-connector-python

package/helper-apps/cortex-autogen/tools/sasfileuploader.py

@@ -0,0 +1,66 @@
+import os
+import sys
+from datetime import datetime, timedelta
+from typing import Annotated
+from pydantic import BaseModel, Field
+
+def install_azure_storage_blob():
+    import subprocess
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "azure-storage-blob"])
+
+try:
+    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
+except ImportError:
+    install_azure_storage_blob()
+    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
+
+class SasUploaderInput(BaseModel):
+    file_path: Annotated[str, Field(description="Path to the file to upload")]
+    container_name: Annotated[str, Field(description="Azure Blob container name")]
+    blob_name: Annotated[str, Field(description="Name for the blob in Azure storage")]
+
+def autogen_sas_uploader(file_path: str) -> str:
+    """
+    Upload a file to Azure Blob Storage and generate a SAS URL.
+
+    This function uploads the specified file to Azure Blob Storage using the container name
+    from the AZURE_BLOB_CONTAINER environment variable. It then generates and returns a
+    Shared Access Signature (SAS) URL for the uploaded blob.
+
+    Args:
+    file_path (str): Path to the local file to be uploaded.
+
+    Returns:
+    str: SAS URL of the uploaded blob if successful, or an error message if the upload fails.
+
+    Note: 
+    - Requires AZURE_STORAGE_CONNECTION_STRING and AZURE_BLOB_CONTAINER environment variables.
+    - The blob name in Azure will be the same as the input file name.
+    """
+    connect_str = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
+    container_name = os.environ.get('AZURE_BLOB_CONTAINER')
+    
+    if not connect_str or not container_name:
+        return "Error: AZURE_STORAGE_CONNECTION_STRING or AZURE_BLOB_CONTAINER not set."
+
+    blob_service_client = BlobServiceClient.from_connection_string(connect_str)
+    blob_client = blob_service_client.get_blob_client(container=container_name, blob=file_path)
+
+    try:
+        with open(file_path, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+        
+        sas_token = generate_blob_sas(
+            account_name=blob_service_client.account_name,
+            container_name=container_name,
+            blob_name=file_path,
+            account_key=blob_service_client.credential.account_key,
+            permission=BlobSasPermissions(read=True),
+            expiry=datetime.utcnow() + timedelta(days=30)
+        )
+        
+        sas_url = f"https://{blob_service_client.account_name}.blob.core.windows.net/{container_name}/{file_path}?{sas_token}"
+        return sas_url
+    except Exception as e:
+        return f"Error uploading file: {str(e)}"
+    

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -86,9 +86,16 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
             // Extract the original file name from the URL
             const urlObj = new URL(inputPath);
             const originalFileName = path.basename(urlObj.pathname);
+            const maxLength = 200; // Set the maximum length for the filename
+            let truncatedFileName = originalFileName;
+            if (originalFileName.length > maxLength) {
+                const extension = path.extname(originalFileName); // Preserve the file extension
+                const basename = path.basename(originalFileName, extension); // Get the filename without the extension
+                truncatedFileName = basename.substring(0, maxLength) + extension; // Truncate the filename and append the extension
+            }
 
-            // Use the original file name when saving the downloaded file
-            const downloadPath = path.join(uniqueOutputPath, originalFileName);
+            // Use the original-truncated file name when saving the downloaded file
+            const downloadPath = path.join(uniqueOutputPath, truncatedFileName);
             await downloadFile(inputPath, downloadPath);
             inputPath = downloadPath;
         }

package/lib/pathwayTools.js

@@ -1,6 +1,10 @@
 // pathwayTools.js
 import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
+import { publishRequestProgress } from "../lib/redisSubscription.js";
+import { getSemanticChunks } from "../server/chunker.js";
+import logger from '../lib/logger.js';
+import { requestState } from '../server/requestState.js';
 
 // callPathway - call a pathway from another pathway
 const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
@@ -12,14 +16,26 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
     if (!pathway) {
         throw new Error(`Pathway ${pathwayName} not found`);
     }
-    const requestState = {};
+
     const parent = {};
-    const data = await pathway.rootResolver(parent, args, { config, pathway, requestState } );
-    
-    // Merge the results into the pathwayResolver if it was provided
-    if (pathwayResolver) {
-        pathwayResolver.mergeResults(data);
+    let rootRequestId = pathwayResolver?.rootRequestId || pathwayResolver?.requestId;
+
+    let data = await pathway.rootResolver(parent, {...args, rootRequestId}, { config, pathway, requestState } );
+
+    if (args.async || args.stream) {
+        const { result: requestId } = data;
+
+        // Fire the resolver for the async requestProgress
+        logger.info(`Callpathway starting async requestProgress, requestId: ${requestId}`);
+        const { resolver, args } = requestState[requestId];
+        requestState[requestId].useRedis = false;
+        requestState[requestId].started = true;
+
+        data = resolver && await resolver(args);
     }
+    
+    // Update pathwayResolver with new data if available
+    pathwayResolver?.mergeResults(data);
 
     return data?.result;
 };
@@ -32,4 +48,33 @@ const gpt3Decode = (text) => {
     return decode(text);
 }
 
-export { callPathway, gpt3Encode, gpt3Decode };
+const say = async (requestId, message, maxMessageLength = Infinity) => {
+    try {
+        const chunks = getSemanticChunks(message, maxMessageLength);
+
+        for (let chunk of chunks) {
+            await publishRequestProgress({
+                requestId,
+                progress: 0.5,
+                data: chunk
+            });
+        }
+
+        await publishRequestProgress({
+            requestId,
+            progress: 0.5,
+            data: " ... "
+        });
+
+        await publishRequestProgress({
+            requestId,
+            progress: 0.5,
+            data: "\n\n"
+        });
+
+    } catch (error) {
+        logger.error(`Say error: ${error.message}`);
+    }
+};
+
+export { callPathway, gpt3Encode, gpt3Decode, say };

package/lib/requestExecutor.js

@@ -236,7 +236,7 @@ const makeRequest = async (cortexRequest) => {
             promises.push(selectedEndpoint.limiter.schedule({expiration: pathway.timeout * 1000 + 1000, id: `${requestId}_${uuidv4()}`},() => requestWithMonitor(selectedEndpoint, url, data, axiosConfigObj)));
         } else {
             if (streamRequested) {
-                logger.info(`>>> [${requestId}] ${model} does not support streaming - sending non-streaming request`);
+                logger.info(`>>> [${requestId}] ${model.name || 'This model'} does not support streaming - sending non-streaming request`);
                 axiosConfigObj.params.stream = false;
                 data.stream = false;
             }

package/lib/util.js

@@ -2,7 +2,6 @@ import logger from "./logger.js";
 import stream from 'stream';
 import subsrt from 'subsrt';
 import os from 'os';
-import path from 'path';
 import http from 'http';
 import https from 'https';
 import { URL } from 'url';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.31",
+  "version": "1.1.33",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/styleguide/styleguide.js

@@ -1,4 +1,5 @@
 import { Prompt } from '../../server/prompt.js';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import * as Diff from "diff";
 
 const prompt = new Prompt({

package/pathways/timeline.js

@@ -1,5 +1,6 @@
 import { Prompt } from '../server/prompt.js';
 import * as chrono from 'chrono-node';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import dayjs from 'dayjs';
 
 const getLastOccurrenceOfMonth = (month) => {

package/server/chunker.js

@@ -217,6 +217,11 @@ const semanticTruncate = (text, maxLength) => {
     : truncatedText + "...";
 };
 
+const getSingleTokenChunks = (text) => {
+  if (text === '') return [''];
+  return encode(text).map(token => decode([token]));
+}
+
 export {
-    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat
+    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat, getSingleTokenChunks
 };

package/server/modelExecutor.js

@@ -17,6 +17,7 @@ import OpenAiEmbeddingsPlugin from './plugins/openAiEmbeddingsPlugin.js';
 import OpenAIImagePlugin from './plugins/openAiImagePlugin.js';
 import OpenAIDallE3Plugin from './plugins/openAiDallE3Plugin.js';
 import OpenAIVisionPlugin from './plugins/openAiVisionPlugin.js';
+import OpenAIReasoningPlugin from './plugins/openAiReasoningPlugin.js';
 import GeminiChatPlugin from './plugins/geminiChatPlugin.js';
 import GeminiVisionPlugin from './plugins/geminiVisionPlugin.js';
 import Gemini15ChatPlugin from './plugins/gemini15ChatPlugin.js';
@@ -82,6 +83,9 @@ class ModelExecutor {
             case 'OPENAI-VISION':
                 plugin = new OpenAIVisionPlugin(pathway, model);
                 break;
+            case 'OPENAI-REASONING':
+                plugin = new OpenAIReasoningPlugin(pathway, model);
+                break;
             case 'GEMINI-CHAT':
                 plugin = new GeminiChatPlugin(pathway, model);
                 break;

package/server/pathwayResolver.js

@@ -27,6 +27,7 @@ class PathwayResolver {
         this.warnings = [];
         this.errors = [];
         this.requestId = uuidv4();
+        this.rootRequestId = null;
         this.responseParser = new PathwayResponseParser(pathway);
         this.tool = null;
         this.modelName = [
@@ -84,7 +85,7 @@ class PathwayResolver {
         catch (error) {
             if (!args.async) {
                 publishRequestProgress({
-                    requestId: this.requestId,
+                    requestId: this.rootRequestId || this.requestId,
                     progress: 1,
                     data: '[DONE]',
                 });
@@ -100,9 +101,9 @@ class PathwayResolver {
             // some models don't support progress updates
             if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
                 await publishRequestProgress({
-                        requestId: this.requestId,
+                        requestId: this.rootRequestId || this.requestId,
                         progress: completedCount / totalCount,
-                        data: JSON.stringify(responseData),
+                        data: typeof responseData === 'string' ? responseData : JSON.stringify(responseData),
                 });
             }
         // If the response is an object, it's a streaming response
@@ -113,7 +114,7 @@ class PathwayResolver {
 
                 const onParse = (event) => {
                     let requestProgress = {
-                        requestId: this.requestId
+                        requestId: this.rootRequestId || this.requestId
                     };
 
                     logger.debug(`Received event: ${event.type}`);
@@ -138,8 +139,10 @@ class PathwayResolver {
 
                     try {
                         if (!streamEnded && requestProgress.data) {
-                            //logger.info(`Publishing stream message to requestId ${this.requestId}: ${message}`);
-                            publishRequestProgress(requestProgress);
+                            if (!(this.rootRequestId && requestProgress.progress === 1)) {
+                                logger.debug(`Publishing stream message to requestId ${this.requestId}: ${requestProgress.data}`);
+                                publishRequestProgress(requestProgress);
+                            }
                             streamEnded = requestProgress.progress === 1;
                         }
                     } catch (error) {
@@ -195,6 +198,7 @@ class PathwayResolver {
             if (!requestState[this.requestId]) {
                 requestState[this.requestId] = {}
             }
+            this.rootRequestId = args.rootRequestId ?? null;
             requestState[this.requestId] = { ...requestState[this.requestId], args, resolver: this.asyncResolve.bind(this) };
             return this.requestId;
         }

package/server/plugins/openAiReasoningPlugin.js

@@ -0,0 +1,61 @@
+import OpenAIChatPlugin from './openAiChatPlugin.js';
+
+class OpenAIReasoningPlugin extends OpenAIChatPlugin {
+    
+    tryParseMessages(messages) {
+        let newMessages = [];
+
+        for (const message of messages) {
+            if (message.role === 'user' || message.role === 'assistant') {
+                newMessages.push({
+                    role: message.role,
+                    content: this.parseContent(message.content)
+                });
+            }
+            // System messages are simply ignored
+        }
+
+        messages.length = 0;
+        messages.push(...newMessages);
+    }
+
+    parseContent(content) {
+        if (typeof content === 'string') {
+            return [{ type: 'text', text: content }];
+        }
+        if (Array.isArray(content)) {
+            return content.map(item => {
+                if (typeof item === 'string') {
+                    return { type: 'text', text: item };
+                }
+                const { type, text } = item;
+                return { type, text: text || '' };
+            });
+        }
+        return [];
+    }
+
+    getRequestParameters(text, parameters, prompt) {
+        const requestParameters = super.getRequestParameters(text, parameters, prompt);
+
+        this.tryParseMessages(requestParameters.messages);
+
+        const modelMaxReturnTokens = this.getModelMaxReturnTokens();
+        const maxTokensPrompt = this.promptParameters.max_tokens;
+        const maxTokensModel = this.getModelMaxTokenLength() * (1 - this.getPromptTokenRatio());
+
+        const maxTokens = maxTokensPrompt || maxTokensModel;
+
+        requestParameters.max_completion_tokens = maxTokens ? Math.min(maxTokens, modelMaxReturnTokens) : modelMaxReturnTokens;
+        requestParameters.temperature = 1;
+
+        if (this.promptParameters.json) {
+            //requestParameters.response_format = { type: "json_object", }
+        }
+
+        return requestParameters;
+    }
+
+}
+
+export default OpenAIReasoningPlugin;

package/server/rest.js

@@ -5,7 +5,20 @@ import pubsub from './pubsub.js';
 import { requestState } from './requestState.js';
 import { v4 as uuidv4 } from 'uuid';
 import logger from '../lib/logger.js';
-
+import { getSingleTokenChunks } from './chunker.js';
+
+const chunkTextIntoTokens = (() => {
+    let partialToken = '';
+    return (text, isLast = false, useSingleTokenStream = false) => {
+        const tokens = useSingleTokenStream ? getSingleTokenChunks(partialToken + text) : [text];
+        if (isLast) {
+            partialToken = '';
+            return tokens;
+        }
+        partialToken = useSingleTokenStream ? tokens.pop() : '';
+        return tokens;
+    };
+})();
 
 const processRestRequest = async (server, req, pathway, name, parameterMap = {}) => {
     const fieldVariableDefs = pathway.typeDef(pathway).restDefinition || [];
@@ -50,7 +63,8 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
     return resultText;
 };
 
-const processIncomingStream = (requestId, res, jsonResponse) => {
+const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
+    const useSingleTokenStream = pathway.useSingleTokenStream || false;
 
     const startStream = (res) => {
         // Set the headers for streaming
@@ -61,6 +75,14 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
     
     const finishStream = (res, jsonResponse) => {
+        // Send the last partial token if it exists
+        const lastTokens = chunkTextIntoTokens('', true, useSingleTokenStream);
+        if (lastTokens.length > 0) {
+            lastTokens.forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        }
 
         // If we haven't sent the stop message yet, do it now
         if (jsonResponse.choices?.[0]?.finish_reason !== "stop") {
@@ -85,11 +107,11 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
 
     const sendStreamData = (data) => {
-        logger.debug(`REST SEND: data: ${JSON.stringify(data)}`);
         const dataString = (data==='[DONE]') ? data : JSON.stringify(data);
 
         if (!res.writableEnded) {
             res.write(`data: ${dataString}\n\n`);
+            logger.debug(`REST SEND: data: ${dataString}`);
         }
     }
 
@@ -115,63 +137,68 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
             if (subscription) {
                 try {
                     const subPromiseResult = await subscription;
-                    if (subPromiseResult) {
-                        pubsub.unsubscribe(subPromiseResult);
-                    }
+                    subPromiseResult && pubsub.unsubscribe(subPromiseResult);
                 } catch (error) {
                     logger.error(`Error unsubscribing from pubsub: ${error}`);
                 }
             }
         }
 
-        if (data.requestProgress.requestId === requestId) {
-            logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
-            
-            const progress = data.requestProgress.progress;
-            const progressData = data.requestProgress.data;
+        if (data.requestProgress.requestId !== requestId) return;
 
-            try {
-                const messageJson = JSON.parse(progressData);
-                if (messageJson.error) {
-                    logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
-                    safeUnsubscribe();
-                    finishStream(res, jsonResponse);
-                    return;
-                } else if (messageJson.choices) {
-                    const { text, delta, finish_reason } = messageJson.choices[0];
+        logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
+        
+        const { progress, data: progressData } = data.requestProgress;
 
-                    if (messageJson.object === 'text_completion') {
-                        fillJsonResponse(jsonResponse, text, finish_reason);
-                    } else {
-                        fillJsonResponse(jsonResponse, delta.content, finish_reason);
-                    }
-                } else if (messageJson.candidates) {
-                    const { content, finishReason } = messageJson.candidates[0];
-                    fillJsonResponse(jsonResponse, content.parts[0].text, finishReason);
-                } else if (messageJson.content) {
-                    const text = messageJson.content?.[0]?.text || '';
-                    const finishReason = messageJson.stop_reason;
-                    fillJsonResponse(jsonResponse, text, finishReason);
-                } else {
-                    fillJsonResponse(jsonResponse, messageJson, null);
-                }
-            } catch (error) {
-                //logger.info(`progressData not JSON: ${progressData}`);
-                fillJsonResponse(jsonResponse, progressData, "stop");
-            }
-            if (progress === 1 && progressData.trim() === "[DONE]") {
+        try {
+            const messageJson = JSON.parse(progressData);
+            if (messageJson.error) {
+                logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
                 safeUnsubscribe();
                 finishStream(res, jsonResponse);
                 return;
             }
 
-            sendStreamData(jsonResponse);
+            let content = '';
+            if (messageJson.choices) {
+                const { text, delta } = messageJson.choices[0];
+                content = messageJson.object === 'text_completion' ? text : delta.content;
+            } else if (messageJson.candidates) {
+                content = messageJson.candidates[0].content.parts[0].text;
+            } else if (messageJson.content) {
+                content = messageJson.content?.[0]?.text || '';
+            } else {
+                content = messageJson;
+            }
 
-            if (progress === 1) {
-                safeUnsubscribe();
-                finishStream(res, jsonResponse);
+            chunkTextIntoTokens(content, false, useSingleTokenStream).forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        } catch (error) {
+            logger.debug(`progressData not JSON: ${progressData}`);
+            if (typeof progressData === 'string') {
+                if (progress === 1 && progressData.trim() === "[DONE]") {
+                    fillJsonResponse(jsonResponse, progressData, "stop");
+                    safeUnsubscribe();
+                    finishStream(res, jsonResponse);
+                    return;
+                }
+
+                chunkTextIntoTokens(progressData, false, useSingleTokenStream).forEach(token => {
+                    fillJsonResponse(jsonResponse, token, null);
+                    sendStreamData(jsonResponse);
+                });
+            } else {
+                fillJsonResponse(jsonResponse, progressData, "stop");
+                sendStreamData(jsonResponse);
             }
         }
+
+        if (progress === 1) {
+            safeUnsubscribe();
+            finishStream(res, jsonResponse);
+        }
     });
 
     // Fire the resolver for the async requestProgress
@@ -254,7 +281,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 jsonResponse.choices[0].finish_reason = null;
                 //jsonResponse.object = "text_completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `cmpl-${requestId}`;
@@ -306,7 +333,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 }
                 jsonResponse.object = "chat.completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `chatcmpl-${requestId}`;
@@ -346,4 +373,4 @@ function buildRestEndpoints(pathways, app, server, config) {
     }
 }
 
-export { buildRestEndpoints };
+export { buildRestEndpoints };

package/tests/chunkfunction.test.js

@@ -1,5 +1,5 @@
 import test from 'ava';
-import { getSemanticChunks, determineTextFormat } from '../server/chunker.js';
+import { getSemanticChunks, determineTextFormat, getSingleTokenChunks } from '../server/chunker.js';
 import { encode } from '../lib/encodeCache.js';
 
 const testText = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.
@@ -207,4 +207,18 @@ test('should return identical text that chunker was passed, given weird spaces a
     t.assert(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
     const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
     t.assert(recomposedText === testTextShortWeirdSpaces); //check recomposition
+});
+
+test('should correctly split text into single token chunks', t => {
+    const testString = 'Hello, world!';
+    const chunks = getSingleTokenChunks(testString);
+    
+    // Check that each chunk is a single token
+    t.true(chunks.every(chunk => encode(chunk).length === 1));
+    
+    // Check that joining the chunks recreates the original string
+    t.is(chunks.join(''), testString);
+    
+    // Check specific tokens (this may need adjustment based on your tokenizer)
+    t.deepEqual(chunks, ['Hello', ',', ' world', '!']);
 });