@aj-archipelago/cortex 1.1.32 → 1.1.33

package/helper-apps/cortex-autogen/function_app.py

@@ -1,11 +1,8 @@
 import azure.functions as func
 import logging
 import json
-import autogen
-from autogen import AssistantAgent, UserProxyAgent, config_list_from_json
 from azure.storage.queue import QueueClient
 import os
-import tempfile
 import redis
 from myautogen import process_message
 
@@ -26,7 +23,7 @@ def queue_trigger(msg: func.QueueMessage):
         message_data = json.loads(msg.get_body().decode('utf-8'))
         if "requestId" not in message_data:
             message_data['requestId'] = msg.id
-        process_message(message_data)
+        process_message(message_data, msg)
 
     except Exception as e:
         logging.error(f"Error processing message: {str(e)}")

package/helper-apps/cortex-autogen/main.py

@@ -25,7 +25,7 @@ def main():
                 message_data = json.loads(decoded_content)
                 if "requestId" not in message_data:
                     message_data['requestId'] = message.id
-                process_message(message_data)
+                process_message(message_data, message)
                 queue_client.delete_message(message)
             attempts = 0  # Reset attempts if a message was processed
         else:

package/helper-apps/cortex-autogen/myautogen.py

@@ -2,7 +2,7 @@ import azure.functions as func
 import logging
 import json
 import autogen
-from autogen import AssistantAgent, UserProxyAgent, config_list_from_json
+from autogen import AssistantAgent, UserProxyAgent, config_list_from_json, register_function
 from azure.storage.queue import QueueClient
 import os
 import tempfile
@@ -10,9 +10,32 @@ import redis
 from dotenv import load_dotenv
 import requests
 import pathlib
-
+import pymongo
+import logging
+from datetime import datetime, timezone
+from tools.sasfileuploader import autogen_sas_uploader
+import shutil
 load_dotenv()
 
+DEFAULT_SUMMARY_PROMPT = "Summarize the takeaway from the conversation. Do not add any introductory phrases."
+try:
+    with open("prompt_summary.txt", "r") as file:
+        summary_prompt = file.read() or DEFAULT_SUMMARY_PROMPT
+except FileNotFoundError:
+    summary_prompt = DEFAULT_SUMMARY_PROMPT 
+
+
+def store_in_mongo(data):
+    try:
+        if 'MONGO_URI' in os.environ:
+            client = pymongo.MongoClient(os.environ['MONGO_URI'])
+            collection = client.get_default_database()[os.environ.get('MONGO_COLLECTION_NAME', 'autogenruns')]
+            collection.insert_one(data)
+        else:
+            logging.warning("MONGO_URI not found in environment variables")
+    except Exception as e:
+        logging.error(f"An error occurred while storing data in MongoDB: {str(e)}")
+
 app = func.FunctionApp()
 
 connection_string = os.environ["AZURE_STORAGE_CONNECTION_STRING"]
@@ -72,18 +95,22 @@ def fetch_from_url(url):
         logging.error(f"Error fetching from URL: {e}")
         return ""
 
-def process_message(message_data):
+def process_message(message_data, original_request_message):
     logging.info(f"Processing Message: {message_data}")
     try:
+        started_at = datetime.now()
         message = message_data['message']
         request_id = message_data.get('requestId') or msg.id
 
         config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
         base_url = os.environ.get("CORTEX_API_BASE_URL")
         api_key = os.environ.get("CORTEX_API_KEY")
-        llm_config = {"config_list": config_list, "base_url": base_url, "api_key": api_key, "cache_seed": None}
+        llm_config = {"config_list": config_list, "base_url": base_url, "api_key": api_key, "cache_seed": None, "timeout": 600}
 
         with tempfile.TemporaryDirectory() as temp_dir:
+            #copy /tools directory to temp_dir
+            shutil.copytree(os.path.join(os.getcwd(), "tools"), temp_dir, dirs_exist_ok=True)
+            
             code_executor = autogen.coding.LocalCommandLineCodeExecutor(work_dir=temp_dir)
 
             message_count = 0
@@ -103,26 +130,51 @@ def process_message(message_data):
             system_message_assistant = AssistantAgent.DEFAULT_SYSTEM_MESSAGE 
 
             if system_message_given:
-                system_message_assistant = f"{system_message_assistant}\n\n{system_message_given}"
+                system_message_assistant = system_message_given
             else:
                 print("No extra system message given for assistant")
 
-            assistant = AssistantAgent("assistant", llm_config=llm_config, system_message=system_message_assistant)
-
+            assistant = AssistantAgent("assistant", 
+                llm_config=llm_config, 
+                system_message=system_message_assistant,
+                code_execution_config={"executor": code_executor},
+                is_termination_msg=is_termination_msg,
+            )
+            
             user_proxy = UserProxyAgent(
                 "user_proxy",
+                llm_config=llm_config,
                 system_message=system_message_given,
                 code_execution_config={"executor": code_executor},
                 human_input_mode="NEVER",
                 max_consecutive_auto_reply=20,
-                is_termination_msg=is_termination_msg,
             )
 
+            # description = "Upload a file to Azure Blob Storage and get URL back with a SAS token. Requires AZURE_STORAGE_CONNECTION_STRING and AZURE_BLOB_CONTAINER environment variables. Input: file_path (str). Output: SAS URL (str) or error message."
+
+            # register_function(
+            #     autogen_sas_uploader,
+            #     caller=assistant,
+            #     executor=user_proxy,
+            #     name="autogen_sas_uploader",
+            #     description=description,    
+            # )
+
+            # register_function(
+            #     autogen_sas_uploader,
+            #     caller=user_proxy,
+            #     executor=assistant,
+            #     name="autogen_sas_uploader",
+            #     description=description,    
+            # )
+
             original_assistant_send = assistant.send
             original_user_proxy_send = user_proxy.send
 
             def logged_send(sender, original_send, message, recipient, request_reply=None, silent=True):
                 nonlocal message_count, all_messages
+                if not message:
+                    return
                 logging.info(f"Message from {sender.name} to {recipient.name}: {message}")
                 message_count += 1
                 progress = min(message_count / total_messages, 1)
@@ -134,19 +186,37 @@ def process_message(message_data):
                 })
                 return original_send(message, recipient, request_reply, silent)
 
-            assistant.send = lambda message, recipient, request_reply=None, silent=True: logged_send(assistant, original_assistant_send, message, recipient, request_reply, silent)
-            user_proxy.send = lambda message, recipient, request_reply=None, silent=True: logged_send(user_proxy, original_user_proxy_send, message, recipient, request_reply, silent)
+            assistant.send = lambda message, recipient, request_reply=None, silent=False: logged_send(assistant, original_assistant_send, message, recipient, request_reply, silent)
+            user_proxy.send = lambda message, recipient, request_reply=None, silent=False: logged_send(user_proxy, original_user_proxy_send, message, recipient, request_reply, silent)
 
-            chat_result = user_proxy.initiate_chat(assistant, message=message)
+            #summary_method="reflection_with_llm", "last_msg"
+            chat_result = user_proxy.initiate_chat(assistant, message=message, summary_method="reflection_with_llm", summary_args={"summary_role": "user", "summary_prompt": summary_prompt})
 
-            msg = all_messages[-3]["message"] if len(all_messages) >= 3 else ""
-            logging.info(f"####Final message: {msg}")            
+            msg = ""
+            try:
+                msg = all_messages[-1 if all_messages[-2]["message"] else -3]["message"] 
+                logging.info(f"####Final message: {msg}")     
+            except Exception as e:
+                logging.error(f"Error getting final message: {e}")
+                msg = f"Finished, with errors 🤖 ... {e}"
 
-            publish_request_progress({
+            msg = chat_result.summary if chat_result.summary else msg
+
+            finalData = {
                 "requestId": request_id,
+                "requestMessage": message_data.get("message"),
                 "progress": 1,
-                "data": msg
-            })
+                "data": msg,
+                "contextId": message_data.get("contextId"),
+                "conversation": all_messages,
+                "createdAt": datetime.now(timezone.utc).isoformat(),
+                "insertionTime": original_request_message.insertion_time.astimezone(timezone.utc).isoformat() if original_request_message else None,
+                "startedAt": started_at.astimezone(timezone.utc).isoformat(),
+            }       
+
+            # Final message to indicate completion
+            publish_request_progress(finalData)
+            store_in_mongo(finalData)
 
     except Exception as e:
         logging.error(f"Error processing message: {str(e)}")

package/helper-apps/cortex-autogen/prompt_summary.txt

@@ -0,0 +1,28 @@
+Provide a detailed summary of the conversation, including key points, decisions, and action items, and so on.
+Do not add any introductory phrases.
+Avoid expressing gratitude or using pleasantries.
+Maintain a professional and direct tone throughout responses.
+Include most recent meaningful messages from the conversation in the summary.
+You must include all your uploaded URLs, and url of your uploaded final code URL.
+Reply must be in markdown format, including images and videos as UI can show markdown directly to user in a nice way, so make sure to include all visuals, you may do as follows:
+    For images: ![Alt Text](IMAGE_URL)
+    For videos: <video src="VIDEO_URL" controls></video>
+    For urls: [Link Text](URL)
+Your reply will be only thing that finally gets to surface so make sure it is complete.
+Do not mention words like "Summary of the conversation", "Response", "Task", "The conversation" or so as it doesn't makes sense. 
+Also no need for "Request", user already know its request and task.
+Be as detailed as possible without being annoying.
+Start with the result as that is the most important part, do not mention "Result" as user already know its result.
+No need to say information about generated SAS urls just include them, only include the latest versions of same file.
+No need to say none of this as user already 'll be aware as has got the result: 
+    - Code executed successfully, producing correct result ...
+    - File uploaded to Azure Blob Storage with unique timestamp ...
+    - SAS URL generated for file access, valid for ...
+    - File accessibility verified ...
+    - Code execution details ... 
+    - Current date and time ...
+    - Script executed twice due to debugging environment ...
+    - Verification code ...
+    - Issues encountered and resolved: ...
+
+

package/helper-apps/cortex-autogen/requirements.txt

@@ -2,5 +2,7 @@ azure-storage-queue
 azure-functions
 pyautogen
 redis
+pymongo
 requests
-azure-storage-blob
+azure-storage-blob
+mysql-connector-python

package/helper-apps/cortex-autogen/tools/sasfileuploader.py

@@ -0,0 +1,66 @@
+import os
+import sys
+from datetime import datetime, timedelta
+from typing import Annotated
+from pydantic import BaseModel, Field
+
+def install_azure_storage_blob():
+    import subprocess
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "azure-storage-blob"])
+
+try:
+    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
+except ImportError:
+    install_azure_storage_blob()
+    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
+
+class SasUploaderInput(BaseModel):
+    file_path: Annotated[str, Field(description="Path to the file to upload")]
+    container_name: Annotated[str, Field(description="Azure Blob container name")]
+    blob_name: Annotated[str, Field(description="Name for the blob in Azure storage")]
+
+def autogen_sas_uploader(file_path: str) -> str:
+    """
+    Upload a file to Azure Blob Storage and generate a SAS URL.
+
+    This function uploads the specified file to Azure Blob Storage using the container name
+    from the AZURE_BLOB_CONTAINER environment variable. It then generates and returns a
+    Shared Access Signature (SAS) URL for the uploaded blob.
+
+    Args:
+    file_path (str): Path to the local file to be uploaded.
+
+    Returns:
+    str: SAS URL of the uploaded blob if successful, or an error message if the upload fails.
+
+    Note: 
+    - Requires AZURE_STORAGE_CONNECTION_STRING and AZURE_BLOB_CONTAINER environment variables.
+    - The blob name in Azure will be the same as the input file name.
+    """
+    connect_str = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
+    container_name = os.environ.get('AZURE_BLOB_CONTAINER')
+    
+    if not connect_str or not container_name:
+        return "Error: AZURE_STORAGE_CONNECTION_STRING or AZURE_BLOB_CONTAINER not set."
+
+    blob_service_client = BlobServiceClient.from_connection_string(connect_str)
+    blob_client = blob_service_client.get_blob_client(container=container_name, blob=file_path)
+
+    try:
+        with open(file_path, "rb") as data:
+            blob_client.upload_blob(data, overwrite=True)
+        
+        sas_token = generate_blob_sas(
+            account_name=blob_service_client.account_name,
+            container_name=container_name,
+            blob_name=file_path,
+            account_key=blob_service_client.credential.account_key,
+            permission=BlobSasPermissions(read=True),
+            expiry=datetime.utcnow() + timedelta(days=30)
+        )
+        
+        sas_url = f"https://{blob_service_client.account_name}.blob.core.windows.net/{container_name}/{file_path}?{sas_token}"
+        return sas_url
+    except Exception as e:
+        return f"Error uploading file: {str(e)}"
+    

package/lib/pathwayTools.js

@@ -1,6 +1,10 @@
 // pathwayTools.js
 import { encode, decode } from '../lib/encodeCache.js';
 import { config } from '../config.js';
+import { publishRequestProgress } from "../lib/redisSubscription.js";
+import { getSemanticChunks } from "../server/chunker.js";
+import logger from '../lib/logger.js';
+import { requestState } from '../server/requestState.js';
 
 // callPathway - call a pathway from another pathway
 const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
@@ -12,14 +16,26 @@ const callPathway = async (pathwayName, inArgs, pathwayResolver) => {
     if (!pathway) {
         throw new Error(`Pathway ${pathwayName} not found`);
     }
-    const requestState = {};
+
     const parent = {};
-    const data = await pathway.rootResolver(parent, args, { config, pathway, requestState } );
-    
-    // Merge the results into the pathwayResolver if it was provided
-    if (pathwayResolver) {
-        pathwayResolver.mergeResults(data);
+    let rootRequestId = pathwayResolver?.rootRequestId || pathwayResolver?.requestId;
+
+    let data = await pathway.rootResolver(parent, {...args, rootRequestId}, { config, pathway, requestState } );
+
+    if (args.async || args.stream) {
+        const { result: requestId } = data;
+
+        // Fire the resolver for the async requestProgress
+        logger.info(`Callpathway starting async requestProgress, requestId: ${requestId}`);
+        const { resolver, args } = requestState[requestId];
+        requestState[requestId].useRedis = false;
+        requestState[requestId].started = true;
+
+        data = resolver && await resolver(args);
     }
+    
+    // Update pathwayResolver with new data if available
+    pathwayResolver?.mergeResults(data);
 
     return data?.result;
 };
@@ -32,4 +48,33 @@ const gpt3Decode = (text) => {
     return decode(text);
 }
 
-export { callPathway, gpt3Encode, gpt3Decode };
+const say = async (requestId, message, maxMessageLength = Infinity) => {
+    try {
+        const chunks = getSemanticChunks(message, maxMessageLength);
+
+        for (let chunk of chunks) {
+            await publishRequestProgress({
+                requestId,
+                progress: 0.5,
+                data: chunk
+            });
+        }
+
+        await publishRequestProgress({
+            requestId,
+            progress: 0.5,
+            data: " ... "
+        });
+
+        await publishRequestProgress({
+            requestId,
+            progress: 0.5,
+            data: "\n\n"
+        });
+
+    } catch (error) {
+        logger.error(`Say error: ${error.message}`);
+    }
+};
+
+export { callPathway, gpt3Encode, gpt3Decode, say };

package/lib/util.js

@@ -2,7 +2,6 @@ import logger from "./logger.js";
 import stream from 'stream';
 import subsrt from 'subsrt';
 import os from 'os';
-import path from 'path';
 import http from 'http';
 import https from 'https';
 import { URL } from 'url';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.32",
+  "version": "1.1.33",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {

package/pathways/styleguide/styleguide.js

@@ -1,4 +1,5 @@
 import { Prompt } from '../../server/prompt.js';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import * as Diff from "diff";
 
 const prompt = new Prompt({

package/pathways/timeline.js

@@ -1,5 +1,6 @@
 import { Prompt } from '../server/prompt.js';
 import * as chrono from 'chrono-node';
+// eslint-disable-next-line import/no-extraneous-dependencies
 import dayjs from 'dayjs';
 
 const getLastOccurrenceOfMonth = (month) => {

package/server/chunker.js

@@ -217,6 +217,11 @@ const semanticTruncate = (text, maxLength) => {
     : truncatedText + "...";
 };
 
+const getSingleTokenChunks = (text) => {
+  if (text === '') return [''];
+  return encode(text).map(token => decode([token]));
+}
+
 export {
-    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat
+    getSemanticChunks, semanticTruncate, getLastNToken, getFirstNToken, determineTextFormat, getSingleTokenChunks
 };

package/server/pathwayResolver.js

@@ -27,6 +27,7 @@ class PathwayResolver {
         this.warnings = [];
         this.errors = [];
         this.requestId = uuidv4();
+        this.rootRequestId = null;
         this.responseParser = new PathwayResponseParser(pathway);
         this.tool = null;
         this.modelName = [
@@ -84,7 +85,7 @@ class PathwayResolver {
         catch (error) {
             if (!args.async) {
                 publishRequestProgress({
-                    requestId: this.requestId,
+                    requestId: this.rootRequestId || this.requestId,
                     progress: 1,
                     data: '[DONE]',
                 });
@@ -100,9 +101,9 @@ class PathwayResolver {
             // some models don't support progress updates
             if (!modelTypesExcludedFromProgressUpdates.includes(this.model.type)) {
                 await publishRequestProgress({
-                        requestId: this.requestId,
+                        requestId: this.rootRequestId || this.requestId,
                         progress: completedCount / totalCount,
-                        data: JSON.stringify(responseData),
+                        data: typeof responseData === 'string' ? responseData : JSON.stringify(responseData),
                 });
             }
         // If the response is an object, it's a streaming response
@@ -113,7 +114,7 @@ class PathwayResolver {
 
                 const onParse = (event) => {
                     let requestProgress = {
-                        requestId: this.requestId
+                        requestId: this.rootRequestId || this.requestId
                     };
 
                     logger.debug(`Received event: ${event.type}`);
@@ -138,8 +139,10 @@ class PathwayResolver {
 
                     try {
                         if (!streamEnded && requestProgress.data) {
-                            //logger.info(`Publishing stream message to requestId ${this.requestId}: ${message}`);
-                            publishRequestProgress(requestProgress);
+                            if (!(this.rootRequestId && requestProgress.progress === 1)) {
+                                logger.debug(`Publishing stream message to requestId ${this.requestId}: ${requestProgress.data}`);
+                                publishRequestProgress(requestProgress);
+                            }
                             streamEnded = requestProgress.progress === 1;
                         }
                     } catch (error) {
@@ -195,6 +198,7 @@ class PathwayResolver {
             if (!requestState[this.requestId]) {
                 requestState[this.requestId] = {}
             }
+            this.rootRequestId = args.rootRequestId ?? null;
             requestState[this.requestId] = { ...requestState[this.requestId], args, resolver: this.asyncResolve.bind(this) };
             return this.requestId;
         }

package/server/rest.js

@@ -5,7 +5,20 @@ import pubsub from './pubsub.js';
 import { requestState } from './requestState.js';
 import { v4 as uuidv4 } from 'uuid';
 import logger from '../lib/logger.js';
-
+import { getSingleTokenChunks } from './chunker.js';
+
+const chunkTextIntoTokens = (() => {
+    let partialToken = '';
+    return (text, isLast = false, useSingleTokenStream = false) => {
+        const tokens = useSingleTokenStream ? getSingleTokenChunks(partialToken + text) : [text];
+        if (isLast) {
+            partialToken = '';
+            return tokens;
+        }
+        partialToken = useSingleTokenStream ? tokens.pop() : '';
+        return tokens;
+    };
+})();
 
 const processRestRequest = async (server, req, pathway, name, parameterMap = {}) => {
     const fieldVariableDefs = pathway.typeDef(pathway).restDefinition || [];
@@ -50,7 +63,8 @@ const processRestRequest = async (server, req, pathway, name, parameterMap = {})
     return resultText;
 };
 
-const processIncomingStream = (requestId, res, jsonResponse) => {
+const processIncomingStream = (requestId, res, jsonResponse, pathway) => {
+    const useSingleTokenStream = pathway.useSingleTokenStream || false;
 
     const startStream = (res) => {
         // Set the headers for streaming
@@ -61,6 +75,14 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
     
     const finishStream = (res, jsonResponse) => {
+        // Send the last partial token if it exists
+        const lastTokens = chunkTextIntoTokens('', true, useSingleTokenStream);
+        if (lastTokens.length > 0) {
+            lastTokens.forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        }
 
         // If we haven't sent the stop message yet, do it now
         if (jsonResponse.choices?.[0]?.finish_reason !== "stop") {
@@ -85,11 +107,11 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
     }
 
     const sendStreamData = (data) => {
-        logger.debug(`REST SEND: data: ${JSON.stringify(data)}`);
         const dataString = (data==='[DONE]') ? data : JSON.stringify(data);
 
         if (!res.writableEnded) {
             res.write(`data: ${dataString}\n\n`);
+            logger.debug(`REST SEND: data: ${dataString}`);
         }
     }
 
@@ -115,63 +137,68 @@ const processIncomingStream = (requestId, res, jsonResponse) => {
             if (subscription) {
                 try {
                     const subPromiseResult = await subscription;
-                    if (subPromiseResult) {
-                        pubsub.unsubscribe(subPromiseResult);
-                    }
+                    subPromiseResult && pubsub.unsubscribe(subPromiseResult);
                 } catch (error) {
                     logger.error(`Error unsubscribing from pubsub: ${error}`);
                 }
             }
         }
 
-        if (data.requestProgress.requestId === requestId) {
-            logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
-            
-            const progress = data.requestProgress.progress;
-            const progressData = data.requestProgress.data;
+        if (data.requestProgress.requestId !== requestId) return;
 
-            try {
-                const messageJson = JSON.parse(progressData);
-                if (messageJson.error) {
-                    logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
-                    safeUnsubscribe();
-                    finishStream(res, jsonResponse);
-                    return;
-                } else if (messageJson.choices) {
-                    const { text, delta, finish_reason } = messageJson.choices[0];
+        logger.debug(`REQUEST_PROGRESS received progress: ${data.requestProgress.progress}, data: ${data.requestProgress.data}`);
+        
+        const { progress, data: progressData } = data.requestProgress;
 
-                    if (messageJson.object === 'text_completion') {
-                        fillJsonResponse(jsonResponse, text, finish_reason);
-                    } else {
-                        fillJsonResponse(jsonResponse, delta.content, finish_reason);
-                    }
-                } else if (messageJson.candidates) {
-                    const { content, finishReason } = messageJson.candidates[0];
-                    fillJsonResponse(jsonResponse, content.parts[0].text, finishReason);
-                } else if (messageJson.content) {
-                    const text = messageJson.content?.[0]?.text || '';
-                    const finishReason = messageJson.stop_reason;
-                    fillJsonResponse(jsonResponse, text, finishReason);
-                } else {
-                    fillJsonResponse(jsonResponse, messageJson, null);
-                }
-            } catch (error) {
-                //logger.info(`progressData not JSON: ${progressData}`);
-                fillJsonResponse(jsonResponse, progressData, "stop");
-            }
-            if (progress === 1 && progressData.trim() === "[DONE]") {
+        try {
+            const messageJson = JSON.parse(progressData);
+            if (messageJson.error) {
+                logger.error(`Stream error REST: ${messageJson?.error?.message || 'unknown error'}`);
                 safeUnsubscribe();
                 finishStream(res, jsonResponse);
                 return;
             }
 
-            sendStreamData(jsonResponse);
+            let content = '';
+            if (messageJson.choices) {
+                const { text, delta } = messageJson.choices[0];
+                content = messageJson.object === 'text_completion' ? text : delta.content;
+            } else if (messageJson.candidates) {
+                content = messageJson.candidates[0].content.parts[0].text;
+            } else if (messageJson.content) {
+                content = messageJson.content?.[0]?.text || '';
+            } else {
+                content = messageJson;
+            }
 
-            if (progress === 1) {
-                safeUnsubscribe();
-                finishStream(res, jsonResponse);
+            chunkTextIntoTokens(content, false, useSingleTokenStream).forEach(token => {
+                fillJsonResponse(jsonResponse, token, null);
+                sendStreamData(jsonResponse);
+            });
+        } catch (error) {
+            logger.debug(`progressData not JSON: ${progressData}`);
+            if (typeof progressData === 'string') {
+                if (progress === 1 && progressData.trim() === "[DONE]") {
+                    fillJsonResponse(jsonResponse, progressData, "stop");
+                    safeUnsubscribe();
+                    finishStream(res, jsonResponse);
+                    return;
+                }
+
+                chunkTextIntoTokens(progressData, false, useSingleTokenStream).forEach(token => {
+                    fillJsonResponse(jsonResponse, token, null);
+                    sendStreamData(jsonResponse);
+                });
+            } else {
+                fillJsonResponse(jsonResponse, progressData, "stop");
+                sendStreamData(jsonResponse);
             }
         }
+
+        if (progress === 1) {
+            safeUnsubscribe();
+            finishStream(res, jsonResponse);
+        }
     });
 
     // Fire the resolver for the async requestProgress
@@ -254,7 +281,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 jsonResponse.choices[0].finish_reason = null;
                 //jsonResponse.object = "text_completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `cmpl-${requestId}`;
@@ -306,7 +333,7 @@ function buildRestEndpoints(pathways, app, server, config) {
                 }
                 jsonResponse.object = "chat.completion.chunk";
 
-                processIncomingStream(resultText, res, jsonResponse);
+                processIncomingStream(resultText, res, jsonResponse, pathway);
             } else {
                 const requestId = uuidv4();
                 jsonResponse.id = `chatcmpl-${requestId}`;
@@ -346,4 +373,4 @@ function buildRestEndpoints(pathways, app, server, config) {
     }
 }
 
-export { buildRestEndpoints };
+export { buildRestEndpoints };

package/tests/chunkfunction.test.js

@@ -1,5 +1,5 @@
 import test from 'ava';
-import { getSemanticChunks, determineTextFormat } from '../server/chunker.js';
+import { getSemanticChunks, determineTextFormat, getSingleTokenChunks } from '../server/chunker.js';
 import { encode } from '../lib/encodeCache.js';
 
 const testText = `Lorem ipsum dolor sit amet, consectetur adipiscing elit. In id erat sem. Phasellus ac dapibus purus, in fermentum nunc. Mauris quis rutrum magna. Quisque rutrum, augue vel blandit posuere, augue magna convallis turpis, nec elementum augue mauris sit amet nunc. Aenean sit amet leo est. Nunc ante ex, blandit et felis ut, iaculis lacinia est. Phasellus dictum orci id libero ullamcorper tempor.
@@ -207,4 +207,18 @@ test('should return identical text that chunker was passed, given weird spaces a
     t.assert(chunks.every(chunk => encode(chunk).length <= maxChunkToken)); //check chunk size
     const recomposedText = chunks.reduce((acc, chunk) => acc + chunk, '');
     t.assert(recomposedText === testTextShortWeirdSpaces); //check recomposition
+});
+
+test('should correctly split text into single token chunks', t => {
+    const testString = 'Hello, world!';
+    const chunks = getSingleTokenChunks(testString);
+    
+    // Check that each chunk is a single token
+    t.true(chunks.every(chunk => encode(chunk).length === 1));
+    
+    // Check that joining the chunks recreates the original string
+    t.is(chunks.join(''), testString);
+    
+    // Check specific tokens (this may need adjustment based on your tokenizer)
+    t.deepEqual(chunks, ['Hello', ',', ' world', '!']);
 });

package/helper-apps/cortex-autogen/sasfileuploader.py

@@ -1,93 +0,0 @@
-import os
-import sys
-from datetime import datetime, timedelta
-
-def install_azure_storage_blob():
-    print("Installing azure-storage-blob...")
-    import subprocess
-    subprocess.check_call([sys.executable, "-m", "pip", "install", "azure-storage-blob"])
-    print("azure-storage-blob installed successfully.")
-
-try:
-    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
-except ImportError:
-    install_azure_storage_blob()
-    from azure.storage.blob import BlobServiceClient, BlobClient, generate_blob_sas, BlobSasPermissions
-
-def generate_sas_url(blob_service_client, container_name, blob_name):
-    """
-    Generates a SAS URL for a blob.
-    """
-    sas_token = generate_blob_sas(
-        account_name=blob_service_client.account_name,
-        container_name=container_name,
-        blob_name=blob_name,
-        account_key=blob_service_client.credential.account_key,
-        permission=BlobSasPermissions(read=True, write=True),
-        expiry=datetime.utcnow() + timedelta(hours=1)
-    )
-    return f"https://{blob_service_client.account_name}.blob.core.windows.net/{container_name}/{blob_name}?{sas_token}"
-
-def upload_file_to_blob(file_path, sas_url):
-    """
-    Uploads a single file to Azure Blob Storage using a SAS URL.
-    """
-    try:
-        blob_client = BlobClient.from_blob_url(sas_url)
-        with open(file_path, "rb") as data:
-            blob_client.upload_blob(data, overwrite=True)
-        print(f"Successfully uploaded {os.path.basename(file_path)} to Azure Blob Storage.")
-        return True
-    except Exception as e:
-        print(f"Error uploading file: {e}")
-        return False
-
-def main():
-    # Get Azure Storage connection string from environment variable
-    connect_str = os.environ.get('AZURE_STORAGE_CONNECTION_STRING')
-    if not connect_str:
-        print("Error: AZURE_STORAGE_CONNECTION_STRING is not set in environment variables.")
-        sys.exit(1)
-
-    # Create the BlobServiceClient object
-    blob_service_client = BlobServiceClient.from_connection_string(connect_str)
-
-    # Get the container name from environment variable or use a default
-    container_name = os.environ.get('AZURE_BLOB_CONTAINER', 'testcontainer')
-
-    # Test file details
-    file_path = "/tmp/test_file.txt"
-    blob_name = "test_file.txt"
-
-    # Create a test file
-    with open(file_path, "w") as f:
-        f.write("This is a test file for Azure Blob Storage upload.")
-
-    print(f"Test file created at: {file_path}")
-
-    # Generate SAS URL
-    sas_url = generate_sas_url(blob_service_client, container_name, blob_name)
-    print(f"Generated SAS URL: {sas_url}")
-
-    # Upload file
-    if upload_file_to_blob(file_path, sas_url):
-        print("File upload completed successfully.")
-    else:
-        print("File upload failed.")
-
-    # Clean up the test file
-    os.remove(file_path)
-    print(f"Test file removed: {file_path}")
-
-    # Upload this script to Azure Blob Storage
-    script_path = os.path.abspath(__file__)
-    script_name = os.path.basename(script_path)
-    script_sas_url = generate_sas_url(blob_service_client, container_name, script_name)
-    
-    if upload_file_to_blob(script_path, script_sas_url):
-        print(f"Script uploaded successfully. You can access it at: {script_sas_url}")
-    else:
-        print("Failed to upload the script.")
-
-if __name__ == "__main__":
-    main()