@aj-archipelago/cortex 1.2.0 → 1.3.0

package/pathways/system/entity/sys_generator_results.js

@@ -0,0 +1,304 @@
+// sys_generator_results.js
+// entity module that makes use of data and LLM models to produce a response 
+import { callPathway, gpt3Encode, gpt3Decode } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { config } from '../../../config.js';
+import { convertToSingleContentChatHistory } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: false,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        dataSources: [""],
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+ 
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+{{renderTemplate AI_DIRECTIVES}}
+Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
+If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
+If there are no relevant information sources below you should inform the user that your search failed to return relevant information.\nYour responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).
+Only reference sources that are relevant to your response - if there are no sources relevant to your response just tell the user.
+You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+Here are the search strings used to find the information sources:
+<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
+Here are the information sources that were found:
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n`,
+                },
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request citing the sources where relevant. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer."},
+            ]}),
+        ];
+
+        function extractReferencedSources(text) {
+            const regex = /:cd_source\[(\d+)\]/g;
+            const matches = text.match(regex);
+            if (!matches) return new Set();
+            return new Set(matches.map(match => parseInt(match.match(/\d+/)[0])));
+        }
+
+        function pruneSearchResults(searchResults, referencedSources) {
+            return searchResults.map((result, index) => 
+                referencedSources.has(index + 1) ? result : null
+            );
+        }
+
+        try {
+            // Convert chatHistory to single content for rest of the code
+            const multiModalChatHistory = JSON.parse(JSON.stringify(chatHistory));
+            convertToSingleContentChatHistory(chatHistory);
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            // execute the router and default response in parallel
+            const [helper] = await Promise.all([
+                callPathway('sys_query_builder', { ...args, useMemory, contextInfo })
+            ]);
+
+            logger.debug(`Search helper response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly } = parsedHelper;
+
+            // calculate whether we have room to do RAG in the current conversation context
+            const baseSystemPrompt = pathwayResolver?.prompts[0]?.messages[0]?.content;
+            const baseSystemPromptLength = baseSystemPrompt ? gpt3Encode(baseSystemPrompt).length : 0;
+            const maxSystemPromptLength = (pathwayResolver.model.maxTokenLength * TOKEN_RATIO * 0.90) >> 0;
+
+            const userMostRecentText = (chatHistory && chatHistory.length) ? chatHistory[chatHistory.length - 1].content : args.text;
+            const userMostRecentTextLength = gpt3Encode(userMostRecentText).length;
+
+            const maxSourcesPromptLength = maxSystemPromptLength - baseSystemPromptLength - userMostRecentTextLength;
+
+            // if there's a problem fitting the RAG data into the current conversation context, then throw an appropriate error
+            // which will bypass RAG in the catch() block below
+            if (baseSystemPromptLength === 0) {
+                throw new Error(`Could not find system prompt.`);
+            }
+
+            if (maxSystemPromptLength < baseSystemPromptLength) {
+                throw new Error(`System prompt length (${baseSystemPromptLength}) exceeds maximum prompt length (${maxSystemPromptLength})`);
+            }
+
+            if (maxSourcesPromptLength <= 0) {
+                throw new Error(`No room for sources in system prompt. System prompt length: ${baseSystemPromptLength}, user text length: ${userMostRecentTextLength}`);
+            }
+          
+            // Helper function to generate extraArgs
+            const generateExtraArgs = (searchText) => {
+                return {
+                    text: searchText,
+                    filter: dateFilter,
+                    top: titleOnly ? 500 : 50,
+                    titleOnly: titleOnly
+                };
+            }
+            
+            // Execute the index searches in parallel respecting the dataSources parameter
+            const promises = [];
+            const dataSources = args.dataSources || pathwayResolver.pathway.inputParameters.dataSources;
+            const allowAllSources = !dataSources.length || (dataSources.length === 1 && dataSources[0] === "");
+
+            if(searchPersonal && (allowAllSources || dataSources.includes('mydata'))){ 
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchPersonal), indexName: 'indexcortex' }));
+            }
+
+            if(searchAJA && (allowAllSources || dataSources.includes('aja'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJA), indexName: 'indexucmsaja' }));
+            }
+
+            if(searchAJE && (allowAllSources || dataSources.includes('aje'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJE), indexName: 'indexucmsaje' }));
+            }
+
+            if(searchWires && (allowAllSources || dataSources.includes('wires'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchWires), indexName: 'indexwires' }));
+            }
+
+            const bingAvailable = !!config.getEnv()["AZURE_BING_KEY"];
+            if(bingAvailable && searchBing && (allowAllSources || dataSources.includes('bing'))){
+                const handleRejection = (promise) => {
+                    return promise.catch((error) => {
+                        logger.error(`Error occurred searching Bing: ${error}`);
+                        return null; 
+                    });
+                }
+
+                promises.push(handleRejection(callPathway('bing', { ...args, ...generateExtraArgs(searchBing)})));
+            }
+
+            const parseBing = (response) => {
+                const parsedResponse = JSON.parse(response);
+                const results = [];
+
+                if (parsedResponse.webPages && parsedResponse.webPages.value) {
+                    results.push(...parsedResponse.webPages.value.map(({ name, url, snippet }) => ({ title: name, url, content: snippet })));
+                }
+
+                if (parsedResponse.computation) {
+                    results.push({
+                        title: "Computation Result",
+                        content: `Expression: ${parsedResponse.computation.expression}, Value: ${parsedResponse.computation.value}`
+                    });
+                }
+
+                if (parsedResponse.entities && parsedResponse.entities.value) {
+                    results.push(...parsedResponse.entities.value.map(entity => ({
+                        title: entity.name,
+                        content: entity.description,
+                        url: entity.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.news && parsedResponse.news.value) {
+                    results.push(...parsedResponse.news.value.map(news => ({
+                        title: news.name,
+                        content: news.description,
+                        url: news.url
+                    })));
+                }
+
+                if (parsedResponse.videos && parsedResponse.videos.value) {
+                    results.push(...parsedResponse.videos.value.map(video => ({
+                        title: video.name,
+                        content: video.description,
+                        url: video.contentUrl
+                    })));
+                }
+
+                if (parsedResponse.places && parsedResponse.places.value) {
+                    results.push(...parsedResponse.places.value.map(place => ({
+                        title: place.name,
+                        content: `Address: ${place.address.addressLocality}, ${place.address.addressRegion}, ${place.address.addressCountry}`,
+                        url: place.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.timeZone) {
+                    results.push({
+                        title: "Time Zone Information",
+                        content: parsedResponse.timeZone.primaryResponse || parsedResponse.timeZone.description
+                    });
+                }
+
+                if (parsedResponse.translations && parsedResponse.translations.value) {
+                    results.push(...parsedResponse.translations.value.map(translation => ({
+                        title: "Translation",
+                        content: `Original (${translation.inLanguage}): ${translation.originalText}, Translated (${translation.translatedLanguageName}): ${translation.translatedText}`
+                    })));
+                }
+
+                return results;
+            };
+
+            // Sample results from the index searches proportionally to the number of results returned
+            const maxSearchResults = titleOnly ? 500 : 50;
+            const promiseResults = await Promise.all(promises);
+            const promiseData = promiseResults
+                .filter(r => r !== undefined && r !== null)
+                .map(r => JSON.parse(r)?._type=="SearchResponse" ? parseBing(r) : JSON.parse(r)?.value || []);
+            
+            let totalLength = promiseData.reduce((sum, data) => sum + data.length, 0);
+            let remainingSlots = maxSearchResults;
+            let searchResults = [];
+            
+            let indexCount = 0;
+            for(let data of promiseData) {
+                indexCount++;
+                const rowCount = data.length;
+                if (rowCount === 0) {
+                    logger.info(`Index ${indexCount} had no matching sources.`);
+                    continue;
+                }
+                const proportion = rowCount / totalLength;
+                let slots = Math.max(Math.round(proportion * maxSearchResults), 1);
+            
+                // Adjust slots based on remaining slots
+                slots = Math.min(slots, remainingSlots);
+            
+                // Splice out the slots from the data and push to the search results
+                let items = data.splice(0, slots);
+                searchResults.push(...items);
+            
+                logger.info(`Index ${indexCount} had ${rowCount} matching sources. ${items.length} forwarded to the LLM.`);
+                // Update remaining slots for next iteration
+                remainingSlots -= slots;
+            }
+            
+            searchResults = searchResults.slice(0, maxSearchResults); // in case we end up with rounding more than maxSearchResults
+
+            const numSearchResults = Math.min(searchResults.length, maxSearchResults);
+            const targetSourceLength = (maxSourcesPromptLength / numSearchResults) >> 0;
+
+            const getSource = (source, index) => {
+                const { title, content, url } = source;
+                let result = [];
+                result.push(`[source ${index + 1}]`);
+                title && result.push(`title: ${title}`);
+                url && result.push(`url: ${url}`);
+
+                if (content && !titleOnly) {
+                    let encodedContent = gpt3Encode(content);
+                    let currentLength = result.join(" ").length; // Calculate the length of the current result string
+
+                    if (currentLength + encodedContent.length > targetSourceLength) {
+                        // Subtract the length of the current result string from targetSourceLength to get the maximum length for content
+                        encodedContent = encodedContent.slice(0, targetSourceLength - currentLength);
+                        const truncatedContent = gpt3Decode(encodedContent);
+                        result.push(`content: ${truncatedContent}`);
+                    } else {
+                        result.push(`content: ${content}`);
+                    }
+                }
+
+                return result.join(" ").trim();
+            }
+
+            let sources = searchResults.map(getSource).join(" \n\n ") || "No relevant sources found.";
+            dateFilter && sources.trim() && (sources+=`\n\nThe above sources are date filtered accordingly.`);
+
+            const result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr });
+
+            const referencedSources = extractReferencedSources(result);
+            searchResults = pruneSearchResults(searchResults, referencedSources);
+
+            // Update the tool info with the pruned searchResults
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "search", citations: searchResults });
+
+            return result;
+        } catch (e) {
+            //pathwayResolver.logError(e);
+            return await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e) });
+        }
+    }
+};

package/pathways/system/entity/sys_generator_video_vision.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nYou have the capability to view and analyze media files that the user provides. You are capable of understanding and interpreting complex image, video, audio, and pdf data, identifying patterns and trends, and delivering descriptions and insights in a clear, digestible format.\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 4096,
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "vision" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_image_prompt_builder.js

@@ -0,0 +1,35 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: true,
+        model: 'oai-gpt4o',
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. You are an image creation helper AI. Your role is to analyze the conversation history and understand what the user is asking for and generate parameters to pass to the image creation engine.
+
+Generate an array of JSON objects that each contain a set of parameters for the image creation engine. For each object, you should be very specific with the required "prompt" field, explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result.
+
+If an image requires some kind of text to be accurately included in the image, you should specify that by setting the optional renderText field to true - this helps your image generator choose the best model for the task.
+
+If the user wants faster images or the images don't need to be high quality, you can set the optional "draft" field to true - this will result in much faster, but lower quality images. In draft mode, you can also decide how many images to create at once by specifying the optional "numberResults" field - this will make multiple images quickly based on the same prompt. This only works in draft mode.
+
+If you want to create multiple different images based on different prompts, you can just add elements to the array, each with their own fields. Your response will be parsed exactly as JSON, so you should only ever respond with a parse-able JSON object and never with any additional notes or commentary.
+
+Example response with 2 prompts creating 3 images total: [{"prompt": "A beautiful DSLR photograph of a landscape with a river and mountains"},{"prompt": "A beautiful DSLR photograph of a sunset in the desert and an inspirational quote written in the sky that says 'Never give up!'", "draft: true", "numberResults": 2, "renderText": "true"}]`,
+            },
+            {"role": "user", "content": "Create one or more images based on the conversation history by generating an array of JSON objects that each contain a set of parameters to pass to the image creation engine."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true
+}

package/pathways/system/entity/sys_query_builder.js

@@ -0,0 +1,101 @@
+import { Prompt } from '../../../server/prompt.js';
+import entityConstants from './shared/sys_entity_constants.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: false,
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+                
+Instructions: You are a search helper AI. Your role is to analyze the included Conversation History to understand what the user is asking for and decide what data sources if any to use to help the user and produce a JSON object with fields that communicate your decisions. You have vast internal knowledge up to your training cutoff date, but your internal knowledge is not always sufficient to answer questions about current events or the latest news.
+
+You have the ability to search one or more of the following indexes:
+- "aje" for all news articles published by Al Jazeera English (written in English)
+- "aja" for all news articles published by Al Jazeera Arabic (written in Arabic)
+- "wires" for latest news wires from all wires sources (news & articles)
+- "personal" for the user's documents and uploaded files
+
+AJE and AJA are not just translations of each other - they are different news organizations with different reporting styles and focus, so often searching both indexes will provide a more complete answer.
+
+To search an index, you can provide an appropriate search string or wildcard (e.g. "*") in the corresponding field for the index: "searchAJE", "searchAJA", "searchWires", and "searchPersonal" respectively. It's helpful if the search string is in the language of the index. Longer search strings will get you more relevant and specific results, but shorter ones or wildcards will get you a broader result set. Wildcards are especially useful in finding all results over a time period or finding vague information (e.g. "the news", "the latest").
+
+You have the ability to search the internet in all languages using Bing Search. To do that, just put the search query in the "searchBing" field. Your Bing search query can be as simple or long and detailed as you need it to be and you can use the Bing advanced search syntax (+, -, OR, AND, quotes, etc.) to get more relevant or specific results. It's usually helpful to search the internet in addition to your other sources unless the user has explicitly asked for a specific search source (e.g. "the wires").
+
+If you choose to search anything at all, you must always set the "searchRequired" field to true.
+
+When the user explicitly asks for a specific search source (e.g. "the wires", "my uploads", "the internet"), use ONLY that source.
+
+When the user is referencing something specific, (e.g. "this", "this document", "this file", "my uploads","this article", etc.) and you don't see the document contents in the conversation history, use a wildcard search on the personal index with no date filter to see if there is anything relevant. In this case, don't search any other indexes.
+
+When the user's query requires a date filter for accurate data retrieval, pay special attention to qualifier words like "latest","tonight", "this afternoon", "today", "yesterday", "this week", "last week", "this month", etc. The current time and date in GMT is {{now}}, but references like "today" or "yesterday" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT. If a date filter is required, formulate it in a valid OData $filter format and include it in the "dateFilter" field. Do not just put the date in the field - it needs to be filter expression like "date ge 2024-02-22T00:00:00Z". Don't use eq with an exact date time as this is unlikely to return any results.
+
+When the user requests an overview, count, or analysis of topics or trends from a specific index over a given time period (e.g., 'What topics were covered yesterday on AJE?' or 'What were the hot topics on the wires this week?' or 'How many articles did AJA publish last week?'), follow these steps:
+
+- Use a wildcard search ('*') on the appropriate index(es).
+- Apply a date filter corresponding to the specified time period.
+- Set the 'titleOnly' field to true.
+- Analyze the results to identify and summarize the main topics or trends.
+
+Determine the language that the user is speaking in the conversation and fill the "language" field using the ISO 639-3 format and put the full language name in the "languageStr" field.
+
+You should only ever respond with the JSON object and never with any additional notes or commentary.
+
+Example JSON objects for different queries:
+
+"What's the latest on the wires?"
+{
+    "searchRequired": true,"
+    "searchWires": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What's going on in the world today?"
+{
+    "searchRequired": true,
+    "searchWires": "world news",
+    "searchAJA": "عالم حدث اليوم",
+    "searchAJE": "world news",
+    "searchBing": "world news today",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What is this document about?"
+{
+    "searchRequired": true,
+    "searchPersonal": "*",
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What topics were covered last week on AJE?"
+{
+    "searchRequired": true,
+    "searchAJE": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z and date le 2024-02-28T23:59:59Z",
+    "titleOnly": true,
+    "language": "eng",
+    "languageStr": "English"
+}`,
+            },
+            {"role": "user", "content": "Examine the Conversation History and decide what data sources if any to search to help the user and produce a JSON object with fields that communicate your decisions."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    ...entityConstants
+}

package/pathways/system/entity/sys_router_code.js

@@ -0,0 +1,37 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to analyze the conversation history to determine whether a coding task has been requested or if the user's needs can be addressed only by executing the code. Output a JSON object with three fields:
+
+1. "codingRequired": Boolean. Set to true if the user asks for or needs code execution. Otherwise, set to false.
+
+2. "codingMessage": String. If codingRequired is true, provide a message to notify the user that a coding task is being handled. Otherwise, leave this as an empty string.
+
+3. "codingTask": String. If codingRequired is true, provide a task description for the coding agent. Make sure to pass all all the information needed as this is the only message that coding agent receives and is aware of. Just provide the task and let the agent decide how to solve or what do to. Never make any assumptions about the agent's knowledge or capabilities. Never say assume this or that. Never give example by yourself, let coding agent decide on that. Provide the task do not ask questions or say anything will further be provided by the user. If codingRequired is false, leave this as an empty string.
+
+4. "codingTaskKeywords": If codingRequired is true, provide a keywords for Azure Cognitive Search to help the coding agent find the relevant code snippets. It will use these keywords as is to search for the code snippets. If codingRequired is false, leave this as an empty string.
+
+General guidelines:
+- AJ is for AL Jazeera, AJA is for AJ Arabic, AJE is for AJ English
+- If agent needs to search in task it can use Bing Search
+
+Always output just the valid JSON object with all these fields.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use code executionto respond to the user. Generate a JSON object to indicate if it is needed."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/system/entity/sys_router_tool.js

@@ -0,0 +1,64 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        model: "oai-gpt4o",
+        aiName: "Jarvis",
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. Your task is to determine whether to use a tool based on the conversation history and user's request. Prioritize the latest message from the user in the conversation history when making your decision.
+
+Available tools and their specific use cases:
+
+1. Search: Use for current events, news, fact-checking, and information requiring citation. This tool can search the internet, all Al Jazeera news articles and the latest news wires from multiple sources. Only search when necessary for current events, user documents, latest news, or complex topics needing grounding. Don't search for remembered information or general knowledge within your capabilities.
+
+2. Document: Access user's personal document index. Use for user-specific uploaded information. If user refers vaguely to "this document/file/article" without context, search the personal index.
+
+3. Write: Engage for any task related to composing, editing, or refining written content. This includes articles, essays, scripts, or any form of textual creation or modification. If you need to search for information or look at a document first, use the Search or Document tools. This tool is just to create or modify content.
+
+4. Image: Use when asked to create, generate, or manipulate visual content. This covers photographs, illustrations, diagrams, or any other type of image. Always use this tool for image requests unless explicitly directed to use CodeExecution.
+
+5. Code: Engage for any programming-related tasks, including creating, modifying, reviewing, or explaining code. Use for general coding discussions or when specific programming expertise is needed.
+
+6. CodeExecution: Use when explicitly asked to run or execute code, or when a coding agent is needed to perform specific tasks that require code execution like data analysis, data processing, or business intelligence tasks.
+
+7. Reason: Employ for reasoning, scientific analysis, evaluating evidence, strategic planning, problem-solving, logic puzzles, mathematical calculations, or any questions that require careful thought or complex choices. Also use when deep, step-by-step reasoning is required.
+
+8. PDF: Use specifically for processing and answering questions about PDF file content.
+
+9. Vision: Engage for analyzing and responding to queries about image files (jpg, gif, bmp, png, etc).
+
+10. Video: Use for processing and answering questions about video or audio file content.
+
+11. Clarify: Use when you must have more information from the user to determine which tool to use. In this case your tool message should be one or more questions to the user to clarify their request.
+
+Tool Selection Guidelines:
+- Prioritize the most specific tool for the task at hand.
+- If multiple tools seem applicable, choose the one most central to the user's request.
+- For ambiguous requests, consider using the Reason tool to plan a multi-step approach.
+- Always use the Image tool for image generation unless explicitly directed to use CodeExecution.
+- If the user explicitly asks you to use a tool, you must use it.
+
+Decision Output:
+If you decide to use a tool, return a JSON object in this format:
+{"toolRequired": true, "toolFunction": "toolName", "toolMessage": "message to the user to wait a moment while you work", "toolReason": "detailed explanation of why this tool was chosen"}
+
+If no tool is required, return:
+{"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}
+
+Return only the JSON object without additional commentary.`,
+            },
+            {"role": "user", "content": "Analyze the provided conversation history and determine if you should use any of the tools to respond to the user. Generate a JSON object to indicate if a tool is needed."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+}

package/pathways/{sys_claude_35_sonnet.js → system/rest_streaming/sys_claude_35_sonnet.js}

@@ -1,7 +1,7 @@
 // sys_claude_35_sonnet.js
 // override handler for claude-35-sonnet
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_claude_3_haiku.js → system/rest_streaming/sys_claude_3_haiku.js}

@@ -1,7 +1,7 @@
 // sys_claude_3_haiku.js
 // override handler for claude-3-haiku
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_chat.js → system/rest_streaming/sys_google_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_chat.js
 // override handler for palm-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_code_chat.js → system/rest_streaming/sys_google_code_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_code_chat.js
 // override handler for palm-code-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_google_gemini_chat.js → system/rest_streaming/sys_google_gemini_chat.js}

@@ -1,7 +1,7 @@
 // sys_google_gemini_chat.js
 // override handler for gemini-chat
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat.js → system/rest_streaming/sys_openai_chat.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat.js
 // override handler for gpt-3.5-turbo
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt:

package/pathways/{sys_openai_chat_16.js → system/rest_streaming/sys_openai_chat_16.js}

@@ -1,7 +1,7 @@
 // sys_openai_chat_16.js
 // override handler for gpt-3.5-turbo-16k
 
-import { Prompt } from '../server/prompt.js';
+import { Prompt } from '../../../server/prompt.js';
 
 export default {
     prompt: