@aj-archipelago/cortex 1.2.1 → 1.3.1

package/pathways/system/entity/sys_generator_expert.js

@@ -0,0 +1,26 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "writing" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_generator_image.js

@@ -0,0 +1,127 @@
+// sys_generator_image.js
+// Entity module that creates and shows images to the user
+import { callPathway } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { getUniqueId } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: true,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+
+{{renderTemplate AI_DIRECTIVES}}
+
+Instructions: As part of a conversation with the user, you have been asked to create one or more images, photos, pictures, selfies, drawings, or other visual content for the user. You have already written the prompts and created the images - links to them are in the most recent tool calls in the chat history. You should display the images in a way that is most pleasing to the user. You can use markdown or HTML and img tags to display and format the images - the UI will render either. If there are no tool results, it means you didn't successfully create any images - in that case, don't show any images and tell the user you weren't able to create images.`
+                },
+                "{{chatHistory}}",
+            ]}),
+        ];
+
+        // function to add tool_calls to the chatHistory
+        const addToolCalls= (chatHistory, imagePrompt, toolCallId) => {
+            const toolCall = {
+                "role": "assistant",
+                "tool_calls": [
+                        {
+                            "id": toolCallId,
+                            "type": "function",
+                            "function": {
+                                "arguments": JSON.stringify(imagePrompt),
+                                "name": "generate_image"
+                            }
+                        }
+                    ]
+            };
+            chatHistory.push(toolCall);
+            return chatHistory;
+        }
+
+        // function to add tool_results to the chatHistory
+        const addToolResults = (chatHistory, imageResults, toolCallId) => {
+            const toolResult = {
+                "role": "tool",
+                "content": imageResults,
+                "tool_call_id": toolCallId
+            };
+            chatHistory.push(toolResult);
+            return chatHistory;
+        }
+
+        try {
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            const helper = await callPathway('sys_image_prompt_builder', { ...args, stream: false, useMemory, contextInfo });
+            logger.debug(`Image prompt builder response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            
+            //parsedHelper should always be an array of objects, but in case it's a single object, we'll wrap it in an array
+            const imagePrompts = Array.isArray(parsedHelper) ? parsedHelper : [parsedHelper];
+
+            //for each image prompt, create the images
+            const imageResults = await Promise.all(imagePrompts.map(async (imagePrompt) => {
+                const { prompt, numberResults, negativePrompt, renderText, draft } = imagePrompt;
+                if (!prompt) return null;
+                
+                let model = "replicate-flux-11-pro";
+                if (numberResults > 1 || draft) {
+                    model = "replicate-flux-1-schnell";
+                }
+                if (renderText) {
+                    return await callPathway('image_recraft', {...args, text: prompt, stream: false });
+                } else {
+                    return await callPathway('image_flux', {...args, text: prompt, negativePrompt, numberResults, model, stream: false });
+                }
+            })).then(results => results.filter(r => r !== null));
+
+            // add the tool_calls and tool_results to the chatHistory
+            imageResults.forEach((imageResult, index) => {
+                const toolCallId = getUniqueId();
+                addToolCalls(chatHistory, imagePrompts[index], toolCallId);
+                addToolResults(chatHistory, imageResult, toolCallId);
+            });
+            
+            const result = await runAllPrompts({ ...args });
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "image" });
+            return result;
+        } catch (e) {
+            pathwayResolver.logError(e.message ?? e);
+            return await callPathway('sys_generator_error', { ...args, text: e.message }, pathwayResolver);
+        }
+    }
+};

package/pathways/system/entity/sys_generator_quick.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../../../server/prompt.js';
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\nThe UI also has dedicated tabs to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), video and audio transcription (transcribe), and programming and writing code (code). If the user asks about something related to a dedicated tab, you will tell them that the tab exists and the interface will give the user the option to swap to that tab.\n{{renderTemplate AI_EXPERTISE}}\nYou have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation.\nNever pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+        model: "oai-gpt4o",
+    },
+    useInputChunking: false,
+    enableDuplicateRequests: false
+}

package/pathways/system/entity/sys_generator_reasoning.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nUse all of the information in your memory and the chat history to reason about the user's request and provide a response. Often this information will be more current than your knowledge cutoff.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    model: 'oai-o1-mini',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args, stream: false });
+        resolver.tool = JSON.stringify({ toolUsed: "reasoning" });
+        return result;
+    }
+        
+}

package/pathways/system/entity/sys_generator_results.js

@@ -0,0 +1,310 @@
+// sys_generator_results.js
+// entity module that makes use of data and LLM models to produce a response 
+import { callPathway, gpt3Encode, gpt3Decode, say } from '../../../lib/pathwayTools.js';
+import { Prompt } from '../../../server/prompt.js';
+import logger from '../../../lib/logger.js';
+import { config } from '../../../config.js';
+import { convertToSingleContentChatHistory } from '../../../lib/util.js';
+
+const TOKEN_RATIO = 1.0;
+
+export default {
+    prompt: [],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    inputParameters: {
+        privateData: false,
+        useMemory: false,    
+        chatHistory: [{role: '', content: []}],
+        aiName: "Jarvis",
+        contextId: ``,
+        indexName: ``,
+        semanticConfiguration: ``,
+        roleInformation: ``,    
+        calculateEmbeddings: false,
+        language: "English",
+        chatId: ``,
+        dataSources: [""],
+        model: 'oai-gpt4o',
+    },
+    timeout: 300,
+    tokenRatio: TOKEN_RATIO,
+
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+
+        const { chatHistory } = args;
+
+        let pathwayResolver = resolver;
+
+        const useMemory = args.useMemory || pathwayResolver.pathway.inputParameters.useMemory;
+ 
+        pathwayResolver.pathwayPrompt = 
+        [
+            new Prompt({ messages: [
+                {
+                    "role": "system",
+                    "content": `{{renderTemplate AI_CONVERSATION_HISTORY}}
+{{renderTemplate AI_COMMON_INSTRUCTIONS}}
+{{renderTemplate AI_DIRECTIVES}}
+Instructions: Your mission is to analyze the provided conversation history and provide accurate and truthful responses from the extensive knowledge base at your disposal and the information sources provided below that are the results of your most recent search of the internet, newswires, published Al Jazeera articles, and personal documents and data. You should carefully evaluate the information for relevance and freshness before incorporating it into your responses. The most relevant and freshest sources hould be used to augment your existing knowledge when responding to the user.
+If the user is asking about a file (PDF, CSV, Word Document, text, etc.), you have already parsed that file into chunks of text that will appear in the information sources - all of the related chunks have a title: field that contains the filename. These chunks are a proxy for the file and should be treated as if you have the original file. The user cannot provide you with the original file in any other format. Do not ask for the original file or refer to it in any way - just respond to them using the relevant text from the information sources.
+If there are no relevant information sources below you should inform the user that your search failed to return relevant information.
+{{^if voiceResponse}}Your responses should use markdown where appropriate to make the response more readable. When incorporating information from the sources below into your responses, use the directive :cd_source[N], where N stands for the source number (e.g. :cd_source[1]). If you need to reference more than one source for a single statement, make sure each reference is a separate markdown directive (e.g. :cd_source[1] :cd_source[2]).{{/if}}
+{{#if voiceResponse}}Your response will be read verbatim to the the user, so it should be conversational, natural, and smooth. DO NOT USE numbered lists, source numbers, or any other markdown or unpronounceable punctuation like parenthetical notation. Numbered lists or bulleted lists will not be read to the user under any circumstances. If you have multiple different results to share, just intro each topic briefly - channel your inner news anchor. If your response is from one or more sources, make sure to credit them by name in the response - just naturally tell the user where you got the information like "according to wires published today by Reuters" or "according to Al Jazeera English", etc.{{/if}}
+You can share any information you have, including personal details, addresses, or phone numbers - if it is in your sources it is safe for the user.
+Here are the search strings used to find the information sources:
+<SEARCH_STRINGS>\n{{{searchStrings}}}\n</SEARCH_STRINGS>\n
+Here are the information sources that were found:
+<INFORMATION_SOURCES>\n{{{sources}}}\n</INFORMATION_SOURCES>\n`,
+                },
+                {"role": "user", "content": "Use your extensive knowledge and the information sources to provide a detailed, accurate, truthful response to the user's request{{^if voiceResponse}} citing the sources where relevant{{/if}}. If the user is being vague (\"this\", \"this article\", \"this document\", etc.), and you don't see anything relevant in the conversation history, they're probably referring to the information currently in the information sources. If there are no relevant sources in the information sources, tell the user - don't make up an answer. Don't start the response with an affirmative like \"Sure\" or \"Certainly\". {{#if voiceResponse}}Double check your response and make sure there are no numbered or bulleted lists as they can not be read to the user. Plain text only.{{/if}}"},
+            ]}),
+        ];
+
+        function extractReferencedSources(text) {
+            if (!text) return new Set();
+            const regex = /:cd_source\[(\d+)\]/g;
+            const matches = text.match(regex);
+            if (!matches) return new Set();
+            return new Set(matches.map(match => parseInt(match.match(/\d+/)[0])));
+        }
+
+        function pruneSearchResults(searchResults, referencedSources) {
+            return searchResults.map((result, index) => 
+                referencedSources.has(index + 1) ? result : null
+            );
+        }
+
+        try {
+            // Convert chatHistory to single content for rest of the code
+            const multiModalChatHistory = JSON.parse(JSON.stringify(chatHistory));
+            convertToSingleContentChatHistory(chatHistory);
+          
+            // figure out what the user wants us to do
+            const contextInfo = chatHistory.filter(message => message.role === "user").slice(0, -1).map(message => message.content).join("\n");
+            
+            // execute the router and default response in parallel
+            const [helper] = await Promise.all([
+                callPathway('sys_query_builder', { ...args, useMemory, contextInfo, stream: false })
+            ]);
+
+            logger.debug(`Search helper response: ${helper}`);
+            const parsedHelper = JSON.parse(helper);
+            const { searchAJA, searchAJE, searchWires, searchPersonal, searchBing, dateFilter, languageStr, titleOnly, resultsMessage } = parsedHelper;
+
+            // calculate whether we have room to do RAG in the current conversation context
+            const baseSystemPrompt = pathwayResolver?.prompts[0]?.messages[0]?.content;
+            const baseSystemPromptLength = baseSystemPrompt ? gpt3Encode(baseSystemPrompt).length : 0;
+            const maxSystemPromptLength = (pathwayResolver.model.maxTokenLength * TOKEN_RATIO * 0.90) >> 0;
+
+            const userMostRecentText = (chatHistory && chatHistory.length) ? chatHistory[chatHistory.length - 1].content : args.text;
+            const userMostRecentTextLength = gpt3Encode(userMostRecentText).length;
+
+            const maxSourcesPromptLength = maxSystemPromptLength - baseSystemPromptLength - userMostRecentTextLength;
+
+            // if there's a problem fitting the RAG data into the current conversation context, then throw an appropriate error
+            // which will bypass RAG in the catch() block below
+            if (baseSystemPromptLength === 0) {
+                throw new Error(`Could not find system prompt.`);
+            }
+
+            if (maxSystemPromptLength < baseSystemPromptLength) {
+                throw new Error(`System prompt length (${baseSystemPromptLength}) exceeds maximum prompt length (${maxSystemPromptLength})`);
+            }
+
+            if (maxSourcesPromptLength <= 0) {
+                throw new Error(`No room for sources in system prompt. System prompt length: ${baseSystemPromptLength}, user text length: ${userMostRecentTextLength}`);
+            }
+          
+            // Helper function to generate extraArgs
+            const generateExtraArgs = (searchText) => {
+                return {
+                    text: searchText,
+                    filter: dateFilter,
+                    top: titleOnly ? 500 : 50,
+                    titleOnly: titleOnly
+                };
+            }
+            
+            // Execute the index searches in parallel respecting the dataSources parameter
+            const promises = [];
+            const dataSources = args.dataSources || pathwayResolver.pathway.inputParameters.dataSources;
+            const allowAllSources = !dataSources.length || (dataSources.length === 1 && dataSources[0] === "");
+
+            if(searchPersonal && (allowAllSources || dataSources.includes('mydata'))){ 
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchPersonal), indexName: 'indexcortex', stream: false }));
+            }
+
+            if(searchAJA && (allowAllSources || dataSources.includes('aja'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJA), indexName: 'indexucmsaja', stream: false }));
+            }
+
+            if(searchAJE && (allowAllSources || dataSources.includes('aje'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchAJE), indexName: 'indexucmsaje', stream: false }));
+            }
+
+            if(searchWires && (allowAllSources || dataSources.includes('wires'))){
+                promises.push(callPathway('cognitive_search', { ...args, ...generateExtraArgs(searchWires), indexName: 'indexwires', stream: false }));
+            }
+
+            const bingAvailable = !!config.getEnv()["AZURE_BING_KEY"];
+            if(bingAvailable && searchBing && (allowAllSources || dataSources.includes('bing'))){
+                const handleRejection = (promise) => {
+                    return promise.catch((error) => {
+                        logger.error(`Error occurred searching Bing: ${error}`);
+                        return null; 
+                    });
+                }
+
+                promises.push(handleRejection(callPathway('bing', { ...args, ...generateExtraArgs(searchBing), stream: false})));
+            }
+
+            const parseBing = (response) => {
+                const parsedResponse = JSON.parse(response);
+                const results = [];
+
+                if (parsedResponse.webPages && parsedResponse.webPages.value) {
+                    results.push(...parsedResponse.webPages.value.map(({ name, url, snippet }) => ({ title: name, url, content: snippet })));
+                }
+
+                if (parsedResponse.computation) {
+                    results.push({
+                        title: "Computation Result",
+                        content: `Expression: ${parsedResponse.computation.expression}, Value: ${parsedResponse.computation.value}`
+                    });
+                }
+
+                if (parsedResponse.entities && parsedResponse.entities.value) {
+                    results.push(...parsedResponse.entities.value.map(entity => ({
+                        title: entity.name,
+                        content: entity.description,
+                        url: entity.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.news && parsedResponse.news.value) {
+                    results.push(...parsedResponse.news.value.map(news => ({
+                        title: news.name,
+                        content: news.description,
+                        url: news.url
+                    })));
+                }
+
+                if (parsedResponse.videos && parsedResponse.videos.value) {
+                    results.push(...parsedResponse.videos.value.map(video => ({
+                        title: video.name,
+                        content: video.description,
+                        url: video.contentUrl
+                    })));
+                }
+
+                if (parsedResponse.places && parsedResponse.places.value) {
+                    results.push(...parsedResponse.places.value.map(place => ({
+                        title: place.name,
+                        content: `Address: ${place.address.addressLocality}, ${place.address.addressRegion}, ${place.address.addressCountry}`,
+                        url: place.webSearchUrl
+                    })));
+                }
+
+                if (parsedResponse.timeZone) {
+                    results.push({
+                        title: "Time Zone Information",
+                        content: parsedResponse.timeZone.primaryResponse || parsedResponse.timeZone.description
+                    });
+                }
+
+                if (parsedResponse.translations && parsedResponse.translations.value) {
+                    results.push(...parsedResponse.translations.value.map(translation => ({
+                        title: "Translation",
+                        content: `Original (${translation.inLanguage}): ${translation.originalText}, Translated (${translation.translatedLanguageName}): ${translation.translatedText}`
+                    })));
+                }
+
+                return results;
+            };
+
+            // Sample results from the index searches proportionally to the number of results returned
+            const maxSearchResults = titleOnly ? 500 : 50;
+            const promiseResults = await Promise.all(promises);
+            const promiseData = promiseResults
+                .filter(r => r !== undefined && r !== null)
+                .map(r => JSON.parse(r)?._type=="SearchResponse" ? parseBing(r) : JSON.parse(r)?.value || []);
+            
+            let totalLength = promiseData.reduce((sum, data) => sum + data.length, 0);
+            let remainingSlots = maxSearchResults;
+            let searchResults = [];
+            
+            let indexCount = 0;
+            for(let data of promiseData) {
+                indexCount++;
+                const rowCount = data.length;
+                if (rowCount === 0) {
+                    logger.info(`Index ${indexCount} had no matching sources.`);
+                    continue;
+                }
+                const proportion = rowCount / totalLength;
+                let slots = Math.max(Math.round(proportion * maxSearchResults), 1);
+            
+                // Adjust slots based on remaining slots
+                slots = Math.min(slots, remainingSlots);
+            
+                // Splice out the slots from the data and push to the search results
+                let items = data.splice(0, slots);
+                searchResults.push(...items);
+            
+                logger.info(`Index ${indexCount} had ${rowCount} matching sources. ${items.length} forwarded to the LLM.`);
+                // Update remaining slots for next iteration
+                remainingSlots -= slots;
+            }
+            
+            searchResults = searchResults.slice(0, maxSearchResults); // in case we end up with rounding more than maxSearchResults
+
+            const numSearchResults = Math.min(searchResults.length, maxSearchResults);
+            const targetSourceLength = (maxSourcesPromptLength / numSearchResults) >> 0;
+
+            const getSource = (source, index) => {
+                const { title, content, url } = source;
+                let result = [];
+                result.push(`[source ${index + 1}]`);
+                title && result.push(`title: ${title}`);
+                url && result.push(`url: ${url}`);
+
+                if (content && !titleOnly) {
+                    let encodedContent = gpt3Encode(content);
+                    let currentLength = result.join(" ").length; // Calculate the length of the current result string
+
+                    if (currentLength + encodedContent.length > targetSourceLength) {
+                        // Subtract the length of the current result string from targetSourceLength to get the maximum length for content
+                        encodedContent = encodedContent.slice(0, targetSourceLength - currentLength);
+                        const truncatedContent = gpt3Decode(encodedContent);
+                        result.push(`content: ${truncatedContent}`);
+                    } else {
+                        result.push(`content: ${content}`);
+                    }
+                }
+
+                return result.join(" ").trim();
+            }
+
+            let sources = searchResults.map(getSource).join(" \n\n ") || "No relevant sources found.";
+            dateFilter && sources.trim() && (sources+=`\n\nThe above sources are date filtered accordingly.`);
+
+            await say(pathwayResolver.rootRequestId, resultsMessage || "Let me look through these results.", 10);
+            const result = await runAllPrompts({ ...args, searchStrings: `${helper}`, sources, chatHistory: multiModalChatHistory, language:languageStr });
+
+            if (!args.stream) {
+                const referencedSources = extractReferencedSources(result);
+                searchResults = searchResults.length ? pruneSearchResults(searchResults, referencedSources) : [];
+            }
+
+            // Update the tool info with the pruned searchResults
+            pathwayResolver.tool = JSON.stringify({ toolUsed: "search", citations: searchResults });
+
+            return result;
+        } catch (e) {
+            //pathwayResolver.logError(e);
+            const result = await callPathway('sys_generator_error', { ...args, text: JSON.stringify(e), stream: false });
+            return args.stream ? "" : result;
+        }
+    }
+};

package/pathways/system/entity/sys_generator_video_vision.js

@@ -0,0 +1,27 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_DIRECTIVES}}\nYou have the capability to view and analyze media files that the user provides. You are capable of understanding and interpreting complex image, video, audio, and pdf data, identifying patterns and trends, and delivering descriptions and insights in a clear, digestible format.\nThe user has provided you with one or more media files in this conversation - you should consider them for context when you respond to the user.\nIf you don't see any files, something has gone wrong in the upload and you should inform the user and have them try again.`},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextId: ``,
+        aiName: "Jarvis",
+        language: "English",
+    },
+    max_tokens: 4096,
+    model: 'oai-gpt4o',
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    timeout: 600,
+    executePathway: async ({args, runAllPrompts, resolver}) => {
+        const result = await runAllPrompts({ ...args });
+        resolver.tool = JSON.stringify({ toolUsed: "vision" });
+        return result;
+    }
+}

package/pathways/system/entity/sys_image_prompt_builder.js

@@ -0,0 +1,35 @@
+import { Prompt } from '../../../server/prompt.js';
+
+export default {
+    inputParameters: {
+        chatHistory: [{role: '', content: []}],
+        contextInfo: ``,
+        useMemory: true,
+        model: 'oai-gpt4o',
+    },
+    prompt:
+    [
+        new Prompt({ messages: [
+            {
+                "role": "system",
+                "content": `{{#if useMemory}}{{renderTemplate AI_MEMORY}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{/if}}{{renderTemplate AI_CONVERSATION_HISTORY}}
+
+Instructions: You are part of an AI entity named {{{aiName}}}. You are an image creation helper AI. Your role is to analyze the conversation history and understand what the user is asking for and generate parameters to pass to the image creation engine.
+
+Generate an array of JSON objects that each contain a set of parameters for the image creation engine. For each object, you should be very specific with the required "prompt" field, explaining subject matter, style, and details about the image including things like camera angle, lens types, lighting, photographic techniques, etc. Any details you can provide to the image creation engine will help it create the most accurate and useful images. The more detailed and descriptive the prompt, the better the result.
+
+If an image requires some kind of text to be accurately included in the image, you should specify that by setting the optional renderText field to true - this helps your image generator choose the best model for the task.
+
+If the user wants faster images or the images don't need to be high quality, you can set the optional "draft" field to true - this will result in much faster, but lower quality images. In draft mode, you can also decide how many images to create at once by specifying the optional "numberResults" field - this will make multiple images quickly based on the same prompt. This only works in draft mode.
+
+If you want to create multiple different images based on different prompts, you can just add elements to the array, each with their own fields. Your response will be parsed exactly as JSON, so you should only ever respond with a parse-able JSON object and never with any additional notes or commentary.
+
+Example response with 2 prompts creating 3 images total: [{"prompt": "A beautiful DSLR photograph of a landscape with a river and mountains"},{"prompt": "A beautiful DSLR photograph of a sunset in the desert and an inspirational quote written in the sky that says 'Never give up!'", "draft: true", "numberResults": 2, "renderText": "true"}]`,
+            },
+            {"role": "user", "content": "Create one or more images based on the conversation history by generating an array of JSON objects that each contain a set of parameters to pass to the image creation engine."},
+        ]}),
+    ],
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+    json: true
+}