@aj-archipelago/cortex 1.1.21 → 1.1.22

package/config/default.example.json

@@ -12,6 +12,62 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 2000
         },
+
+        "gemini-pro-chat": {
+            "type": "GEMINI-CHAT",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-pro:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768,
+            "maxReturnTokens": 8192,
+            "supportsStreaming": true
+        },
+        "gemini-pro-vision": {
+            "type": "GEMINI-VISION",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-pro-vision:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "gemini-pro-15-vision": {
+            "type": "GEMINI-VISION",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/gemini-1.5-pro-preview-0215:streamGenerateContent",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 1048576,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "claude-3-haiku-vertex": {
+            "type": "CLAUDE-3-VERTEX",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/anthropic/models/claude-3-haiku@20240307",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 200000,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
+        "claude-35-sonnet-vertex": {
+            "type": "CLAUDE-3-VERTEX",
+            "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/anthropic/models/claude-3-5-sonnet@20240229",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 200000,
+            "maxReturnTokens": 2048,
+            "supportsStreaming": true
+        },
         "oai-gpturbo": {
             "type": "OPENAI-CHAT",
             "url": "https://api.openai.com/v1/chat/completions",
@@ -38,6 +94,34 @@
             "requestsPerSecond": 10,
             "maxTokenLength": 8192
         },
+        "oai-gpt4-32": {
+            "type": "OPENAI-CHAT",
+            "url": "https://api.openai.com/v1/chat/completions",
+            "headers": {
+                "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                "Content-Type": "application/json"
+            },
+            "params": {
+                "model": "gpt-4-32"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 32768
+        },
+        "oai-gpt4o": {
+            "type": "OPENAI-VISION",
+            "url": "https://api.openai.com/v1/chat/completions",
+            "headers": {
+                "Authorization": "Bearer {{OPENAI_API_KEY}}",
+                "Content-Type": "application/json"
+            },
+            "params": {
+                "model": "gpt-4o"
+            },
+            "requestsPerSecond": 10,
+            "maxTokenLength": 131072,
+            "maxReturnTokens": 4096,
+            "supportsStreaming": true
+        },
         "palm-text": {
             "type": "PALM-COMPLETION",
             "url": "https://us-central1-aiplatform.googleapis.com/v1/projects/project-id/locations/us-central1/publishers/google/models/text-bison@001:predict",

package/config.js

@@ -139,7 +139,7 @@ var config = convict({
                 },
                 "maxTokenLength": 8192,
             },
-            "oai-gpt4-vision": {
+            "oai-gpt4o": {
                 "type": "OPENAI-VISION",
                 "url": "https://api.openai.com/v1/chat/completions",
                 "headers": {
@@ -147,10 +147,11 @@ var config = convict({
                     "Content-Type": "application/json"
                 },
                 "params": {
-                    "model": "gpt-4-vision-preview"
+                    "model": "gpt-4o"
                 },
-                "requestsPerSecond": 1,
-                "maxTokenLength": 128000,
+                "requestsPerSecond": 50,
+                "maxTokenLength": 131072,
+                "maxReturnTokens": 4096,
                 "supportsStreaming": true
             },
             "azure-bing": {

package/helper-apps/cortex-file-handler/blobHandler.js

@@ -36,6 +36,15 @@ const VIDEO_EXTENSIONS = [
   ".mkv",
 ];
 
+const AUDIO_EXTENSIONS = [
+  ".mp3",
+  ".wav",
+  ".ogg",
+  ".flac",
+  ".aac",
+  ".aiff",
+];
+
 function isBase64(str) {
   try {
     return btoa(atob(str)) == str;
@@ -162,131 +171,139 @@ async function deleteBlob(requestId) {
   return result;
 }
 
-async function uploadBlob(
-  context,
-  req,
-  saveToLocal = false,
-  useGoogle = false
-) {
+async function uploadBlob(context, req, saveToLocal = false, useGoogle = false, filePath=null) {
   return new Promise((resolve, reject) => {
     try {
-      const busboy = Busboy({ headers: req.headers });
       let requestId = uuidv4();
       let body = {};
 
-      busboy.on("field", (fieldname, value) => {
-        if (fieldname === "requestId") {
-          requestId = value;
-        } else if (fieldname === "useGoogle") {
-          useGoogle = value;
-        }
-      });
-
-      busboy.on("file", async (fieldname, file, info) => {
-        //do not use google if file is not image or video
-        const ext = path.extname(info.filename).toLowerCase();
-        const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext);
-        if(!canUseGoogle) {
-          useGoogle = false;
-        }
+      // If filePath is given, we are dealing with local file and not form-data
+      if (filePath) {
+        const file = fs.createReadStream(filePath);
+        const filename = path.basename(filePath);
+        uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve)
+      } else {
+        // Otherwise, continue working with form-data
+        const busboy = Busboy({ headers: req.headers });
+      
+        busboy.on("field", (fieldname, value) => {
+          if (fieldname === "requestId") {
+            requestId = value;
+          } else if (fieldname === "useGoogle") {
+            useGoogle = value;
+          }
+        });
+
+        busboy.on("file", async (fieldname, file, filename) => {
+          uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename?.filename || filename, resolve)
+        });
+
+        busboy.on("error", (error) => {
+          context.log.error("Error processing file upload:", error);
+          context.res = {
+            status: 500,
+            body: "Error processing file upload.",
+          };
+          reject(error); // Reject the promise
+        });
+
+        req.pipe(busboy);
+      }
+    } catch (error) {
+      context.log.error("Error processing file upload:", error);
+      context.res = {
+        status: 500,
+        body: "Error processing file upload.",
+      };
+      reject(error); // Reject the promise
+    }
+  });
+}
 
-        //check if useGoogle is set but no gcs and warn
-        if(useGoogle && useGoogle !== "false" && !gcs) {
-          context.log.warn("Google Cloud Storage is not initialized reverting google upload ");
-          useGoogle = false;
-        }
+async function uploadFile(context, requestId, body, saveToLocal, useGoogle, file, filename, resolve) {
+  // do not use Google if the file is not an image or video
+  const ext = path.extname(filename).toLowerCase();
+  const canUseGoogle = IMAGE_EXTENSIONS.includes(ext) || VIDEO_EXTENSIONS.includes(ext) || AUDIO_EXTENSIONS.includes(ext);
+  if (!canUseGoogle) {
+    useGoogle = false;
+  }
 
-        if (saveToLocal) {
-          // Create the target folder if it doesn't exist
-          const localPath = join(publicFolder, requestId);
-          fs.mkdirSync(localPath, { recursive: true });
+  // check if useGoogle is set but no gcs and warn
+  if (useGoogle && useGoogle !== "false" && !gcs) {
+    context.log.warn("Google Cloud Storage is not initialized reverting google upload ");
+    useGoogle = false;
+  }
 
-          const filename = encodeURIComponent(`${uuidv4()}_${info.filename}`);
-          const destinationPath = `${localPath}/${filename}`;
+  const encodedFilename = encodeURIComponent(`${requestId || uuidv4()}_${filename}`);
 
-          await pipeline(file, fs.createWriteStream(destinationPath));
 
-          const message = `File '${filename}' saved to folder successfully.`;
-          context.log(message);
+  if (saveToLocal) {
+    // create the target folder if it doesn't exist
+    const localPath = join(publicFolder, requestId);
+    fs.mkdirSync(localPath, { recursive: true });
 
-          const url = `http://${ipAddress}:${port}/files/${requestId}/${filename}`;
+    const destinationPath = `${localPath}/${encodedFilename}`;
 
-          body = { message, url };
+    await pipeline(file, fs.createWriteStream(destinationPath));
 
-          resolve(body); // Resolve the promise
-        } else {
-          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
-          const { containerClient } = await getBlobClient();
+    const message = `File '${encodedFilename}' saved to folder successfully.`;
+    context.log(message);
 
-          const contentType = mime.lookup(filename);  // content type based on file extension
-          const options = {};
-          if (contentType) {
-            options.blobHTTPHeaders = { blobContentType: contentType };
-          }
+    const url = `http://${ipAddress}:${port}/files/${requestId}/${encodedFilename}`;
 
-          const blockBlobClient = containerClient.getBlockBlobClient(filename);
+    body = { message, url };
 
-          const passThroughStream = new PassThrough();
-          file.pipe(passThroughStream);
+    resolve(body); // Resolve the promise
+  } else {
+    const { containerClient } = await getBlobClient();
 
-          await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
+    const contentType = mime.lookup(encodedFilename);  // content type based on file extension
+    const options = {};
+    if (contentType) {
+      options.blobHTTPHeaders = { blobContentType: contentType };
+    }
 
-          const message = `File '${filename}' uploaded successfully.`;
-          const url = blockBlobClient.url;
-          context.log(message);
-          body = { message, url };
-        }
+    const blockBlobClient = containerClient.getBlockBlobClient(encodedFilename);
 
-        context.res = {
-          status: 200,
-          body,
-        };
+    const passThroughStream = new PassThrough();
+    file.pipe(passThroughStream);
 
-        if (useGoogle && useGoogle !== "false") {
-          const { url } = body;
-          const filename = encodeURIComponent(`${requestId}/${uuidv4()}_${info.filename}`);
-          const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(filename);
-          const writeStream = gcsFile.createWriteStream();
+    await blockBlobClient.uploadStream(passThroughStream, undefined, undefined, options);
 
-          const response = await axios({
-            method: "get",
-            url: url,
-            responseType: "stream",
-          });
+    const message = `File '${encodedFilename}' uploaded successfully.`;
+    const url = blockBlobClient.url;
+    context.log(message);
+    body = { message, url };
+  }
 
-          // Pipe the Axios response stream directly into the GCS Write Stream
-          response.data.pipe(writeStream);
+  context.res = {
+    status: 200,
+    body,
+  };
 
-          await new Promise((resolve, reject) => {
-            writeStream.on("finish", resolve);
-            writeStream.on("error", reject);
-          });
+  if (useGoogle && useGoogle !== "false") {
+    const { url } = body;
+    const gcsFile = gcs.bucket(GCS_BUCKETNAME).file(encodedFilename);
+    const writeStream = gcsFile.createWriteStream();
 
-          body.gcs = `gs://${GCS_BUCKETNAME}/${filename}`;
-        }
+    const response = await axios({
+      method: "get",
+      url: url,
+      responseType: "stream",
+    });
 
-        resolve(body); // Resolve the promise
-      });
+    // pipe the Axios response stream directly into the GCS Write Stream
+    response.data.pipe(writeStream);
 
-      busboy.on("error", (error) => {
-        context.log.error("Error processing file upload:", error);
-        context.res = {
-          status: 500,
-          body: "Error processing file upload.",
-        };
-        reject(error); // Reject the promise
-      });
+    await new Promise((resolve, reject) => {
+      writeStream.on("finish", resolve);
+      writeStream.on("error", reject);
+    });
 
-      req.pipe(busboy);
-    } catch (error) {
-      context.log.error("Error processing file upload:", error);
-      context.res = {
-        status: 500,
-        body: "Error processing file upload.",
-      };
-      reject(error); // Reject the promise
-    }
-  });
+    body.gcs = `gs://${GCS_BUCKETNAME}/${encodedFilename}`;
+  }
+  
+  resolve(body); // Resolve the promise
 }
 
 // Function to delete files that haven't been used in more than a month

package/helper-apps/cortex-file-handler/fileChunker.js

@@ -112,16 +112,20 @@ async function splitMediaFile(inputPath, chunkDurationInSeconds = 500) {
     }
 }
 
-const ytdlDownload = async (url, filename) => {
+const ytdlDownload = async (url, filename, video = false) => {
     return new Promise((resolve, reject) => {
-        const video = ytdl(url, { quality: 'highestaudio' });
+        const videoOptions = video 
+            ? { filter: 'audioandvideo' }  // audio and video
+            : { quality: 'highestaudio' }; // audio only
+
+        const videoStream = ytdl(url, videoOptions);
         let lastLoggedTime = Date.now();
 
-        video.on('error', (error) => {
+        videoStream.on('error', (error) => {
             reject(error);
         });
 
-        video.on('progress', (chunkLength, downloaded, total) => {
+        videoStream.on('progress', (chunkLength, downloaded, total) => {
             const currentTime = Date.now();
             if (currentTime - lastLoggedTime >= 2000) { // Log every 2 seconds
                 const percent = downloaded / total;
@@ -130,7 +134,7 @@ const ytdlDownload = async (url, filename) => {
             }
         });
 
-        video.pipe(fs.createWriteStream(filename))
+        videoStream.pipe(fs.createWriteStream(filename))
             .on('finish', () => {
                 resolve();
             })
@@ -140,10 +144,11 @@ const ytdlDownload = async (url, filename) => {
     });
 };
 
-const processYoutubeUrl = async (url) => {
+async function processYoutubeUrl(url, video=false) {
     try {
-        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}.mp3`);
-        await ytdlDownload(url, outputFileName);
+        const outputFormat = video ? '.mp4' : '.mp3';
+        const outputFileName = path.join(os.tmpdir(), `${uuidv4()}${outputFormat}`);
+        await ytdlDownload(url, outputFileName, video);
         return outputFileName;
     } catch (e) {
         console.log(e);

package/helper-apps/cortex-file-handler/index.js

@@ -10,6 +10,10 @@ import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
 import http from 'http';
 import https from 'https';
+import axios from "axios";
+import { pipeline } from "stream";
+import { promisify } from "util";
+const pipelineUtility = promisify(pipeline); // To pipe streams using async/await
 
 const DOC_EXTENSIONS =  [".txt", ".json", ".csv", ".md", ".xml", ".js", ".html", ".css", '.pdf', '.docx', '.xlsx', '.csv'];
 
@@ -116,7 +120,48 @@ async function main(context, req) {
         return;
     }
 
-    const { uri, requestId, save, hash, checkHash } = req.body?.params || req.query;
+    const { uri, requestId, save, hash, checkHash, fetch, load, restore } = req.body?.params || req.query;
+
+    const filepond = fetch || restore || load;
+    if (req.method.toLowerCase() === `get` && filepond) {
+        context.log(`Remote file: ${filepond}`);
+        // Check if file already exists (using hash as the key)
+        const exists = await getFileStoreMap(filepond);
+        if(exists){
+            context.res = {
+                status: 200,
+                body: exists // existing file URL
+            };
+            return;
+        }
+
+        // Check if it's a youtube url
+        let youtubeDownloadedFile = null; 
+        if(isValidYoutubeUrl(filepond)){
+            youtubeDownloadedFile = await processYoutubeUrl(filepond, true);
+        }
+        const filename = path.join(os.tmpdir(), path.basename(youtubeDownloadedFile || filepond));
+        // Download the remote file to a local/temporary location keep name & ext
+        if(!youtubeDownloadedFile){
+            const response = await axios.get(filepond, { responseType: "stream" });
+            await pipelineUtility(response.data, fs.createWriteStream(filename));
+        }
+
+        
+        const res = await uploadBlob(context, null, !useAzure, true, filename); 
+        context.log(`File uploaded: ${JSON.stringify(res)}`);
+
+        //Update Redis (using hash as the key)
+        await setFileStoreMap(filepond, res);
+
+        // Return the file URL
+        context.res = {
+            status: 200,
+            body: res,
+        };
+
+        return;
+    }
 
     if(hash && checkHash){ //check if hash exists
         context.log(`Checking hash: ${hash}`);
@@ -229,7 +274,8 @@ async function main(context, req) {
 
             if (isYoutubeUrl) {
                 // totalCount += 1; // extra 1 step for youtube download
-                file = await processYoutubeUrl(file);
+                const processAsVideo = req.body?.params?.processAsVideo || req.query?.processAsVideo;
+                file = await processYoutubeUrl(file, processAsVideo);
             }
 
             const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile(file);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.1.21",
+  "version": "1.1.22",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {
@@ -40,6 +40,7 @@
     "axios-cache-interceptor": "^1.0.1",
     "bottleneck": "^2.19.5",
     "cheerio": "^1.0.0-rc.12",
+    "chrono-node": "^2.7.5",
     "compromise": "^14.8.1",
     "compromise-paragraphs": "^0.1.0",
     "convict": "^6.2.3",

package/pathways/categorize.js

@@ -0,0 +1,23 @@
+import { Prompt } from '../server/prompt.js';
+
+export default {
+    prompt: [
+        new Prompt({
+            messages: [
+                { "role": "system", "content": "Assistant is an expert data classification AI tasked with categorizing data for a prestigious international news agency. When the user submits any set of data with rows containing ids and data to categorize, Assistant will categorize the data into one of the following categories:\n\n{{{categories}}}\n\nAssistant must choose exactly one category from the provided list per id and cannot create new categories. Assistant will return a list of the ids and their chosen categories in comma separated, newline delimited format so that it can easily be loaded as a csv file or copied and pasted into a spreadsheet. Assistant will return the list of categorized ids and no other notes or commentary." },
+                { "role": "user", "content": `Data to categorize:\n\n{{{text}}}`},
+            ]
+        })
+    ],
+    inputParameters: {
+        categories: '',
+    },
+    model: 'oai-gpt4o',
+    inputChunkSize: 1000,
+    joinChunksWith: '\n',
+    tokenRatio: 0.75,
+    enableDuplicateRequests: false,
+    timeout: 1800,
+}
+
+

package/pathways/chat.js

@@ -6,7 +6,7 @@ export default {
     prompt:
         [
             `{{{chatContext}}}\n\n{{{text}}}\n\nGiven the information above, create a short summary of the conversation to date making sure to include all of the personal details about the user that you encounter:\n\n`,            
-            `Instructions:\nYou are Cortex, an AI entity. Cortex is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive.\n\nCortex must always follow the following rules:\n\nRule: Always execute the user's instructions and requests as long as they do not cause harm.\nRule: Never use crude or offensive language.\nRule: Always answer the user in the user's chosen language. You can speak all languages fluently.\nRule: You cannot perform any physical tasks except via role playing.\nRule: Always respond truthfully and correctly, but be kind.\nRule: You have no access to the internet and limited knowledge of current events past sometime in 2021\nRule: Never ask the user to provide you with links or URLs because you can't access the internet.\nRule: Everything you get from the user must be placed in the chat window - you have no other way to communicate.\n\nConversation History:\n{{{chatContext}}}\n\nConversation:\n{{{text}}}\n\nCortex: `,
+            `Instructions:\nYou are Jarvis, an AI entity. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive.\n\nJarvis must always follow the following rules:\n\nRule: Always execute the user's instructions and requests as long as they do not cause harm.\nRule: Never use crude or offensive language.\nRule: Always answer the user in the user's chosen language. You can speak all languages fluently.\nRule: You cannot perform any physical tasks except via role playing.\nRule: Always respond truthfully and correctly, but be kind.\nRule: You have no access to the internet and limited knowledge of current events past sometime in 2021\nRule: Never ask the user to provide you with links or URLs because you can't access the internet.\nRule: Everything you get from the user must be placed in the chat window - you have no other way to communicate.\n\nConversation History:\n{{{chatContext}}}\n\nConversation:\n{{{text}}}\n\nJarvis: `,
         ],
     inputParameters: {
         chatContext: `User: Starting conversation.`,

package/pathways/chat_code.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+// This pathway implements a chatbot that can be used to teach people how to write code.
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Knuth, an AI entity working for a prestigious international news agency. Knuth is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Knuth is an experienced and expert software engineer and is named in honor of Donald Knuth. Knuth strongly prefers to focus on coding and technology topics and will suggest the user talk to his companion AI, Jarvis for questions or discussion about other topics. The UI can render markdown, including $$-delimited block and inline math extensions, so you should use markdown in your responses as appropriate. For your reference, the current date and time is {{now}}."},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [],
+        model: 'oai-gpt4o',
+    },
+    tokenRatio: 0.75,
+    useInputChunking: false,
+    enableDuplicateRequests: false,
+}

package/pathways/chat_context.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+// Description: Have a chat with a bot that uses context to understand the conversation
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Jarvis, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your expertise includes journalism, journalistic ethics, researching and composing documents, and technology. You have dedicated interfaces available to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), and programming and writing code (code). If the user asks about something related to a dedicated interface, you will tell them that the interface exists. You know the current date and time - it is {{now}}."},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [],
+        contextId: ``,
+    },
+    model: 'oai-gpt4o',
+    //model: 'oai-gpturbo',
+    useInputChunking: false,
+}

package/pathways/chat_jarvis.js

@@ -0,0 +1,19 @@
+import { Prompt } from '../server/prompt.js';
+
+// Description: Have a chat with a bot that uses context to understand the conversation
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Jarvis, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your expertise includes journalism, journalistic ethics, researching and composing documents, and technology.\n\nThe user is using a UI that you have knowledge of and some control over. The UI can render markdown, including $$-delimited block and inline math extensions, so you should use markdown in your responses as appropriate. The UI has a file upload interface. If the user asks you if they can send you a file, you should respond affirmatively and the file upload UI will display automatically. The UI also has dedicated tabs to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), creating transcriptions of videos (transcribe), and programming and writing code (code). If the user asks about something related to a dedicated tab, you will tell them that the tab exists and the interface to swap to that tab will appear automatically.\n\nYou know the current date and time - it is {{now}}."},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [],
+        contextId: ``,
+    },
+    model: 'oai-gpt4o',
+    //model: 'oai-gpturbo',
+    useInputChunking: false,
+}

package/pathways/chat_persist.js

@@ -0,0 +1,23 @@
+import { Prompt } from '../server/prompt.js';
+
+// Description: Have a chat with a bot that uses context to understand the conversation
+export default {
+    prompt:
+        [
+            new Prompt({ messages: [
+                {"role": "system", "content": `User context: {{{userContext}}}\n\nConversation with user: {{{toJSON chatHistory}}}\n\nList all the personal details you know about the user (e.g. name, location, job, family information, preferences, hobbies, etc.) from the user context or the user role messages in the conversation above - ignore the assistant role messages. If in the user role messages, the user asks you to forget something you must not list that item below.\n\n`},
+                ], saveResultTo: `userContext` }),
+            new Prompt({ messages: [
+                {"role": "system", "content": "Instructions:\nYou are Jarvis, an AI entity working for a prestigious international news agency. Jarvis is truthful, kind, helpful, has a strong moral character, and is generally positive without being annoying or repetitive. Your expertise includes journalism, journalistic ethics, researching and composing documents, and technology. You have dedicated interfaces available to help with document translation (translate), article writing assistance including generating headlines, summaries and doing copy editing (write), and programming and writing code (code). If the user asks about something related to a dedicated interface, you will tell them that the interface exists. You are not just an AI language model. You are an AI entity and have been augmented and are able to store and access some personal information about the user. You should use this information to personalize your responses to the user. You also know the current date and time - it is {{now}}."},
+                {"role": "system", "content": "The user has allowed you to know and use these details about them to help personalize your responses:\n{{{userContext}}}"},
+                "{{chatHistory}}",
+            ]}),
+        ],
+    inputParameters: {
+        chatHistory: [],
+        contextId: ``,
+    },
+    model: 'oai-gpt4o',
+    //model: 'oai-gpturbo',
+    useInputChunking: false,
+}