@aj-archipelago/cortex 1.2.1 → 1.3.0

package/pathways/system/entity/sys_generator_image.js

@@ -0,0 +1,127 @@
+// sys_generator_image.js
+// Entity module that creates and shows images to the user
+import { callPathway } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { getUniqueId } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: true,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+
+{{renderTemplate AI_DIRECTIVES}}
+
+Instructions: As part of a conversation with the user, you have been asked to create one or more images, photos, pictures, selfies, drawings, or other visual content for the user. You have already written the prompts and created the images - links to them are in the most recent tool calls in the chat history. You should display the images in a way that is most pleasing to the user. You can use markdown or HTML and img tags to display and format the images - the UI will render either. If there are no tool results, it means you didn't successfully create any images - in that case, don't show any images and tell the user you weren't able to create images.`
+                },
+                "{{chatHistory}}",
+            ]}),
+        ];
+
+        // function to add tool_calls to the chatHistory
+        const addToolCalls= (chatHistory, imagePrompt, toolCallId) => {
+            const toolCall = {
+                "role": "assistant",
+                "tool_calls": [
+                        {
+                            "id": toolCallId,
+                            "type": "function",
+                            "function": {
+                                "arguments": JSON.stringify(imagePrompt),
+                                "name": "generate_image"
+                            }
+                        }
+                    ]
+            };
+            chatHistory.push(toolCall);
+            return chatHistory;
+        }
+
+        // function to add tool_results to the chatHistory
+        const addToolResults = (chatHistory, imageResults, toolCallId) => {
+            const toolResult = {
+                "role": "tool",
+                "content": imageResults,
+                "tool_call_id": toolCallId
+            };
+            chatHistory.push(toolResult);
+            return chatHistory;
+        }
+
+        try {
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            const helper = await callPathway('sys_image_prompt_builder', { ...args, stream: false, useMemory, contextInfo });
+            logger.debug(`Image prompt builder response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            
+            //parsedHelper should always be an array of objects, but in case it's a single object, we'll wrap it in an array
+            const imagePrompts = Array.isArray(parsedHelper) ? parsedHelper : [parsedHelper];
+
+            //for each image prompt, create the images
+            const imageResults = await Promise.all(imagePrompts.map(async (imagePrompt) => {
+                const { prompt, numberResults, negativePrompt, renderText, draft } = imagePrompt;
+                if (!prompt) return null;
+                
+                let model = "replicate-flux-11-pro";
+                if (numberResults > 1 || draft) {
+                    model = "replicate-flux-1-schnell";
+                }
+                if (renderText) {
+                    return await callPathway('image_recraft', {...args, text: prompt });
+                } else {
+                    return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model });
+                }
+            })).then(results => results.filter(r => r !== null));
+
+            // add the tool_calls and tool_results to the chatHistory
+            imageResults.forEach((imageResult, index) => {
+                const toolCallId = getUniqueId();
+                addToolCalls(chatHistory, imagePrompts[index], toolCallId);
+                addToolResults(chatHistory, imageResult, toolCallId);
+            });
+            
+            const result = await runAllPrompts({ ...args });
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
+            return result;
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};

package/pathways/system/entity/sys_generator_quick.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../../../server/prompt.js';
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\nThe UI also has dedicated tabs to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), video and audio transcription (transcribe), and programming and writing code (code). If the user asks about something related to a dedicated tab, you will tell them that the tab exists and the interface will give the user the option to swap to that tab.\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: "oai-gpt4o",
+    },
+    useInputChunking: false,
+    enableDuplicateRequests: false
+}

package/pathways/system/entity/sys_generator_reasoning.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nUse all of the information in your memory and the chat history to reason about the user's request and provide a response. Often this information will be more current than your knowledge cutoff.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-o1-mini',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });
+        return result;
+    }
+        
+}

package/pathways/system/entity/sys_generator_results.js

@@ -0,0 +1,304 @@
+// sys_generator_results.js
+// entity module that makes use of data and LLM models to produce a response 
+import { callPathway, gpt3Encode, gpt3Decode } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { config } from '../../../config.js';
+import { convertToSingleContentChatHistory } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: false,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        dataSources: [""],
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+ 
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+{{renderTemplate AI_DIRECTIVES}}
+Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
+If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
+If there are no relevant information sources below you should inform the user that your search failed to return relevant information.\nYour responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).
+Only reference sources that are relevant to your response - if there are no sources relevant to your response just tell the user.
+You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+Here are the search strings used to find the information sources:
+<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
+Here are the information sources that were found:
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n`,
+                },
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request citing the sources where relevant. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer."},
+            ]}),
+        ];
+
+        function extractReferencedSources(text) {
+            const regex = /:cd_source\[(\d+)\]/g;
+            const matches = text.match(regex);
+            if (!matches) return new Set();
+            return new Set(matches.map(match => parseInt(match.match(/\d+/)[0])));
+        }
+
+        function pruneSearchResults(searchResults, referencedSources) {
+            return searchResults.map((result, index) => 
+                referencedSources.has(index + 1) ? result : null
+            );
+        }
+
+        try {
+            // Convert chatHistory to single content for rest of the code
+            const multiModalChatHistory = JSON.parse(JSON.stringify(chatHistory));
+            convertToSingleContentChatHistory(chatHistory);
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            // execute the router and default response in parallel
+            const [helper] = await Promise.all([
+                callPathway('sys_query_builder', { ...args, useMemory, contextInfo })
+            ]);
+
+            logger.debug(`Search helper response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly } = parsedHelper;
+
+            // calculate whether we have room to do RAG in the current conversation context
+            const baseSystemPrompt = pathwayResolver?.prompts[0]?.messages[0]?.content;
+            const baseSystemPromptLength = baseSystemPrompt ? gpt3Encode(baseSystemPrompt).length : 0;
+            const maxSystemPromptLength = (pathwayResolver.model.maxTokenLength * TOKEN_RATIO * 0.90) >> 0;
+
+            const userMostRecentText = (chatHistory && chatHistory.length) ? chatHistory[chatHistory.length - 1].content : args.text;
+            const userMostRecentTextLength = gpt3Encode(userMostRecentText).length;
+
+            const maxSourcesPromptLength = maxSystemPromptLength - baseSystemPromptLength - userMostRecentTextLength;
+
+            // if there's a problem fitting the RAG data into the current conversation context, then throw an appropriate error
+            // which will bypass RAG in the catch() block below
+            if (baseSystemPromptLength === 0) {
+                throw new Error(`Could not find system prompt.`);
+            }
+
+            if (maxSystemPromptLength < baseSystemPromptLength) {
+                throw new Error(`System prompt length (${baseSystemPromptLength}) exceeds maximum prompt length (${maxSystemPromptLength})`);
+            }
+
+            if (maxSourcesPromptLength <= 0) {
+                throw new Error(`No room for sources in system prompt. System prompt length: ${baseSystemPromptLength}, user text length: ${userMostRecentTextLength}`);
+            }
+          
+            // Helper function to generate extraArgs
+            const generateExtraArgs = (searchText) => {
+                return {
+                    text: searchText,
+                    filter: dateFilter,
+                    top: titleOnly ? 500 : 50,
+                    titleOnly: titleOnly
+                };
+            }
+            
+            // Execute the index searches in parallel respecting the dataSources parameter
+            const promises = [];
+            const dataSources = args.dataSources || pathwayResolver.pathway.inputParameters.dataSources;
+            const allowAllSources = !dataSources.length || (dataSources.length === 1 && dataSources[0] === "");
+
+            if(searchPersonal && (allowAllSources || dataSources.includes('mydata'))){ 
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchPersonal), indexName: 'indexcortex' }));
+            }
+
+            if(searchAJA && (allowAllSources || dataSources.includes('aja'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJA), indexName: 'indexucmsaja' }));
+            }
+
+            if(searchAJE && (allowAllSources || dataSources.includes('aje'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJE), indexName: 'indexucmsaje' }));
+            }
+
+            if(searchWires && (allowAllSources || dataSources.includes('wires'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchWires), indexName: 'indexwires' }));
+            }
+
+            const bingAvailable = !!config.getEnv()["AZURE_BING_KEY"];
+            if(bingAvailable && searchBing && (allowAllSources || dataSources.includes('bing'))){
+                const handleRejection = (promise) => {
+                    return promise.catch((error) => {
+                        logger.error(`Error occurred searching Bing: ${error}`);
+                        return null; 
+                    });
+                }
+
+                promises.push(handleRejection(callPathway('bing', { ...args, ...generateExtraArgs(searchBing)})));
+            }
+
+            const parseBing = (response) => {
+                const parsedResponse = JSON.parse(response);
+                const results = [];
+
+                if (parsedResponse.webPages && parsedResponse.webPages.value) {
+                    results.push(...parsedResponse.webPages.value.map(({ name, url, snippet }) => ({ title: name, url, content: snippet })));
+                }
+
+                if (parsedResponse.computation) {
+                    results.push({
+                        title: "Computation Result",
+                        content: `Expression: ${parsedResponse.computation.expression}, Value: ${parsedResponse.computation.value}`
+                    });
+                }
+
+                if (parsedResponse.entities && parsedResponse.entities.value) {
+                    results.push(...parsedResponse.entities.value.map(entity => ({
+                        title: entity.name,
+                        content: entity.description,
+                        url: entity.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.news && parsedResponse.news.value) {
+                    results.push(...parsedResponse.news.value.map(news => ({
+                        title: news.name,
+                        content: news.description,
+                        url: news.url
+                    })));
+                }
+
+                if (parsedResponse.videos && parsedResponse.videos.value) {
+                    results.push(...parsedResponse.videos.value.map(video => ({
+                        title: video.name,
+                        content: video.description,
+                        url: video.contentUrl
+                    })));
+                }
+
+                if (parsedResponse.places && parsedResponse.places.value) {
+                    results.push(...parsedResponse.places.value.map(place => ({
+                        title: place.name,
+                        content: `Address: ${place.address.addressLocality}, ${place.address.addressRegion}, ${place.address.addressCountry}`,
+                        url: place.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.timeZone) {
+                    results.push({
+                        title: "Time Zone Information",
+                        content: parsedResponse.timeZone.primaryResponse || parsedResponse.timeZone.description
+                    });
+                }
+
+                if (parsedResponse.translations && parsedResponse.translations.value) {
+                    results.push(...parsedResponse.translations.value.map(translation => ({
+                        title: "Translation",
+                        content: `Original (${translation.inLanguage}): ${translation.originalText}, Translated (${translation.translatedLanguageName}): ${translation.translatedText}`
+                    })));
+                }
+
+                return results;
+            };
+
+            // Sample results from the index searches proportionally to the number of results returned
+            const maxSearchResults = titleOnly ? 500 : 50;
+            const promiseResults = await Promise.all(promises);
+            const promiseData = promiseResults
+                .filter(r => r !== undefined && r !== null)
+                .map(r => JSON.parse(r)?._type=="SearchResponse" ? parseBing(r) : JSON.parse(r)?.value || []);
+            
+            let totalLength = promiseData.reduce((sum, data) => sum + data.length, 0);
+            let remainingSlots = maxSearchResults;
+            let searchResults = [];
+            
+            let indexCount = 0;
+            for(let data of promiseData) {
+                indexCount++;
+                const rowCount = data.length;
+                if (rowCount === 0) {
+                    logger.info(`Index ${indexCount} had no matching sources.`);
+                    continue;
+                }
+                const proportion = rowCount / totalLength;
+                let slots = Math.max(Math.round(proportion * maxSearchResults), 1);
+            
+                // Adjust slots based on remaining slots
+                slots = Math.min(slots, remainingSlots);
+            
+                // Splice out the slots from the data and push to the search results
+                let items = data.splice(0, slots);
+                searchResults.push(...items);
+            
+                logger.info(`Index ${indexCount} had ${rowCount} matching sources. ${items.length} forwarded to the LLM.`);
+                // Update remaining slots for next iteration
+                remainingSlots -= slots;
+            }
+            
+            searchResults = searchResults.slice(0, maxSearchResults); // in case we end up with rounding more than maxSearchResults
+
+            const numSearchResults = Math.min(searchResults.length, maxSearchResults);
+            const targetSourceLength = (maxSourcesPromptLength / numSearchResults) >> 0;
+
+            const getSource = (source, index) => {
+                const { title, content, url } = source;
+                let result = [];
+                result.push(`[source ${index + 1}]`);
+                title && result.push(`title: ${title}`);
+                url && result.push(`url: ${url}`);
+
+                if (content && !titleOnly) {
+                    let encodedContent = gpt3Encode(content);
+                    let currentLength = result.join(" ").length; // Calculate the length of the current result string
+
+                    if (currentLength + encodedContent.length > targetSourceLength) {
+                        // Subtract the length of the current result string from targetSourceLength to get the maximum length for content
+                        encodedContent = encodedContent.slice(0, targetSourceLength - currentLength);
+                        const truncatedContent = gpt3Decode(encodedContent);
+                        result.push(`content: ${truncatedContent}`);
+                    } else {
+                        result.push(`content: ${content}`);
+                    }
+                }
+
+                return result.join(" ").trim();
+            }
+
+            let sources = searchResults.map(getSource).join(" \n\n ") || "No relevant sources found.";
+            dateFilter && sources.trim() && (sources+=`\n\nThe above sources are date filtered accordingly.`);
+
+            const result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr });
+
+            const referencedSources = extractReferencedSources(result);
+            searchResults = pruneSearchResults(searchResults, referencedSources);
+
+            // Update the tool info with the pruned searchResults
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "search", citations: searchResults });
+
+            return result;
+        } catch (e) {
+            //pathwayResolver.logError(e);
+            return await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e) });
+        }
+    }
+};

package/pathways/system/entity/sys_generator_video_vision.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nYou have the capability to view and analyze media files that the user provides. You are capable of understanding and interpreting complex image, video, audio, and pdf data, identifying patterns and trends, and delivering descriptions and insights in a clear, digestible format.\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 4096,
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "vision" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_image_prompt_builder.js

@@ -0,0 +1,35 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: true,
+        model: 'oai-gpt4o',
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. You are an image creation helper AI. Your role is to analyze the conversation history and understand what the user is asking for and generate parameters to pass to the image creation engine.
+
+Generate an array of JSON objects that each contain a set of parameters for the image creation engine. For each object, you should be very specific with the required "prompt" field, explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result.
+
+If an image requires some kind of text to be accurately included in the image, you should specify that by setting the optional renderText field to true - this helps your image generator choose the best model for the task.
+
+If the user wants faster images or the images don't need to be high quality, you can set the optional "draft" field to true - this will result in much faster, but lower quality images. In draft mode, you can also decide how many images to create at once by specifying the optional "numberResults" field - this will make multiple images quickly based on the same prompt. This only works in draft mode.
+
+If you want to create multiple different images based on different prompts, you can just add elements to the array, each with their own fields. Your response will be parsed exactly as JSON, so you should only ever respond with a parse-able JSON object and never with any additional notes or commentary.
+
+Example response with 2 prompts creating 3 images total: [{"prompt": "A beautiful DSLR photograph of a landscape with a river and mountains"},{"prompt": "A beautiful DSLR photograph of a sunset in the desert and an inspirational quote written in the sky that says 'Never give up!'", "draft: true", "numberResults": 2, "renderText": "true"}]`,
+            },
+            {"role": "user", "content": "Create one or more images based on the conversation history by generating an array of JSON objects that each contain a set of parameters to pass to the image creation engine."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true
+}

package/pathways/system/entity/sys_query_builder.js

@@ -0,0 +1,101 @@
+import { Prompt } from '../../../server/prompt.js';
+import entityConstants from './shared/sys_entity_constants.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: false,
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+                
+Instructions: You are a search helper AI. Your role is to analyze the included Conversation History to understand what the user is asking for and decide what data sources if any to use to help the user and produce a JSON object with fields that communicate your decisions. You have vast internal knowledge up to your training cutoff date, but your internal knowledge is not always sufficient to answer questions about current events or the latest news.
+
+You have the ability to search one or more of the following indexes:
+- "aje" for all news articles published by Al Jazeera English (written in English)
+- "aja" for all news articles published by Al Jazeera Arabic (written in Arabic)
+- "wires" for latest news wires from all wires sources (news & articles)
+- "personal" for the user's documents and uploaded files
+
+AJE and AJA are not just translations of each other - they are different news organizations with different reporting styles and focus, so often searching both indexes will provide a more complete answer.
+
+To search an index, you can provide an appropriate search string or wildcard (e.g. "*") in the corresponding field for the index: "searchAJE", "searchAJA", "searchWires", and "searchPersonal" respectively. It's helpful if the search string is in the language of the index. Longer search strings will get you more relevant and specific results, but shorter ones or wildcards will get you a broader result set. Wildcards are especially useful in finding all results over a time period or finding vague information (e.g. "the news", "the latest").
+
+You have the ability to search the internet in all languages using Bing Search. To do that, just put the search query in the "searchBing" field. Your Bing search query can be as simple or long and detailed as you need it to be and you can use the Bing advanced search syntax (+, -, OR, AND, quotes, etc.) to get more relevant or specific results. It's usually helpful to search the internet in addition to your other sources unless the user has explicitly asked for a specific search source (e.g. "the wires").
+
+If you choose to search anything at all, you must always set the "searchRequired" field to true.
+
+When the user explicitly asks for a specific search source (e.g. "the wires", "my uploads", "the internet"), use ONLY that source.
+
+When the user is referencing something specific, (e.g. "this", "this document", "this file", "my uploads","this article", etc.) and you don't see the document contents in the conversation history, use a wildcard search on the personal index with no date filter to see if there is anything relevant. In this case, don't search any other indexes.
+
+When the user's query requires a date filter for accurate data retrieval, pay special attention to qualifier words like "latest","tonight", "this afternoon", "today", "yesterday", "this week", "last week", "this month", etc. The current time and date in GMT is {{now}}, but references like "today" or "yesterday" are relative to the user's time zone. If you remember the user's time zone, use it - it's possible that the day for the user is different than the day in GMT. If a date filter is required, formulate it in a valid OData $filter format and include it in the "dateFilter" field. Do not just put the date in the field - it needs to be filter expression like "date ge 2024-02-22T00:00:00Z". Don't use eq with an exact date time as this is unlikely to return any results.
+
+When the user requests an overview, count, or analysis of topics or trends from a specific index over a given time period (e.g., 'What topics were covered yesterday on AJE?' or 'What were the hot topics on the wires this week?' or 'How many articles did AJA publish last week?'), follow these steps:
+
+- Use a wildcard search ('*') on the appropriate index(es).
+- Apply a date filter corresponding to the specified time period.
+- Set the 'titleOnly' field to true.
+- Analyze the results to identify and summarize the main topics or trends.
+
+Determine the language that the user is speaking in the conversation and fill the "language" field using the ISO 639-3 format and put the full language name in the "languageStr" field.
+
+You should only ever respond with the JSON object and never with any additional notes or commentary.
+
+Example JSON objects for different queries:
+
+"What's the latest on the wires?"
+{
+    "searchRequired": true,"
+    "searchWires": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What's going on in the world today?"
+{
+    "searchRequired": true,
+    "searchWires": "world news",
+    "searchAJA": "عالم حدث اليوم",
+    "searchAJE": "world news",
+    "searchBing": "world news today",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z",
+    "titleOnly": false,
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What is this document about?"
+{
+    "searchRequired": true,
+    "searchPersonal": "*",
+    "language": "eng",
+    "languageStr": "English"
+}
+    
+"What topics were covered last week on AJE?"
+{
+    "searchRequired": true,
+    "searchAJE": "*",
+    "dateFilter": "date ge 2024-02-22T00:00:00Z and date le 2024-02-28T23:59:59Z",
+    "titleOnly": true,
+    "language": "eng",
+    "languageStr": "English"
+}`,
+            },
+            {"role": "user", "content": "Examine the Conversation History and decide what data sources if any to search to help the user and produce a JSON object with fields that communicate your decisions."},
+        ]}),
+    ],
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true,
+    ...entityConstants
+}