@aj-archipelago/cortex 1.3.31 → 1.3.32

package/helper-apps/cortex-file-handler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex-file-handler",
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing",
   "type": "module",
   "scripts": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aj-archipelago/cortex",
-  "version": "1.3.31",
+  "version": "1.3.32",
   "description": "Cortex is a GraphQL API for AI. It provides a simple, extensible interface for using AI services from OpenAI, Azure and others.",
   "private": false,
   "repository": {
@@ -33,7 +33,7 @@
   "type": "module",
   "homepage": "https://github.com/aj-archipelago/cortex#readme",
   "dependencies": {
-    "@aj-archipelago/subvibe": "^1.0.8",
+    "@aj-archipelago/subvibe": "^1.0.10",
     "@apollo/server": "^4.7.3",
     "@apollo/server-plugin-response-cache": "^4.1.2",
     "@apollo/utils.keyvadapter": "^3.0.0",

package/pathways/image_flux.js

@@ -9,7 +9,7 @@ export default {
     height: 1024,
     aspectRatio: "custom",
     numberResults: 1,
-    safety_tolerance: 5,
+    safety_tolerance: 6,
     output_format: "webp",
     output_quality: 80,
     steps: 4,

package/pathways/system/entity/memory/shared/sys_memory_helpers.js

@@ -139,6 +139,14 @@ const addToolResults = (chatHistory, result, toolCallId) => {
     return { chatHistory, toolCallId };
 };
 
+const insertToolCallAndResults = (chatHistory, toolArgs, toolName, result = null, toolCallId = getUniqueId()) => {
+    const lastMessage = chatHistory.length > 0 ? chatHistory.pop() : null;
+    addToolCalls(chatHistory, toolArgs, toolName, toolCallId);
+    addToolResults(chatHistory, result, toolCallId);
+    chatHistory.push(lastMessage);
+    return { chatHistory, toolCallId };
+};
+
 const modifyText = (text, modifications) => {
     let modifiedText = text || '';
   
@@ -225,4 +233,4 @@ const modifyText = (text, modifications) => {
     return modifiedText;
 };
 
-export { normalizeMemoryFormat, enforceTokenLimit, addToolCalls, addToolResults, modifyText };
+export { normalizeMemoryFormat, enforceTokenLimit, addToolCalls, addToolResults, modifyText, insertToolCallAndResults };

package/pathways/system/entity/sys_entity_start.js

@@ -5,7 +5,7 @@ import logger from  '../../../lib/logger.js';
 import { chatArgsHasImageUrl } from  '../../../lib/util.js';
 import { QueueServiceClient } from '@azure/storage-queue';
 import { config } from '../../../config.js';
-import { addToolCalls, addToolResults } from './memory/shared/sys_memory_helpers.js';
+import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
 
 const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
 let queueClient;
@@ -94,13 +94,10 @@ export default {
         if (args.chatHistory.length > 1) {
             const memoryContext = await callPathway('sys_read_memory', { ...args, section: 'memoryContext', priority: 0, recentHours: 0, stream: false }, pathwayResolver);
             if (memoryContext) {
-                const lastMessage = args.chatHistory.length > 0 ? args.chatHistory.pop() : null;
-                const { toolCallId } = addToolCalls(args.chatHistory, "search memory for relevant information", "memory_lookup");
-                addToolResults(args.chatHistory, memoryContext, toolCallId);
-                args.chatHistory.push(lastMessage);
+                insertToolCallAndResults(args.chatHistory, "search memory for relevant information", "memory_lookup", memoryContext);
             }
         }
-        
+      
         // If we're using voice, get a quick response to say
         let ackResponse = null;
         if (args.voiceResponse) {
@@ -222,7 +219,7 @@ export default {
             title = await fetchTitleResponsePromise;
 
             pathwayResolver.tool = JSON.stringify({ 
-                hideFromModel: toolCallbackName ? true : false, 
+                hideFromModel: (!args.stream && toolCallbackName) ? true : false, 
                 toolCallbackName, 
                 title,
                 search: toolCallbackName === 'sys_generator_results' ? true : false,

package/pathways/system/entity/sys_generator_image.js

@@ -3,7 +3,7 @@
 import { callPathway } from '../../../lib/pathwayTools.js';
 import { Prompt } from '../../../server/prompt.js';
 import logger from '../../../lib/logger.js';
-import { addToolCalls, addToolResults } from './memory/shared/sys_memory_helpers.js';
+import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
 
 export default {
     prompt: [],
@@ -73,8 +73,7 @@ Instructions: As part of a conversation with the user, you have been asked to cr
 
             // add the tool_calls and tool_results to the chatHistory
             imageResults.forEach((imageResult, index) => {
-                const { toolCallId } = addToolCalls(chatHistory, imagePrompts[index], "generate_image");
-                addToolResults(chatHistory, imageResult, toolCallId, "generate_image");
+                insertToolCallAndResults(chatHistory, imagePrompts[index], "generate_image", imageResult);
             });
             
             const result = await runAllPrompts({ ...args });

package/pathways/system/entity/sys_generator_memory.js

@@ -1,5 +1,5 @@
 import { callPathway } from '../../../lib/pathwayTools.js';
-import { addToolCalls, addToolResults } from './memory/shared/sys_memory_helpers.js';
+import { insertToolCallAndResults } from './memory/shared/sys_memory_helpers.js';
 
 export default {
     prompt:
@@ -20,8 +20,7 @@ export default {
 
         const memoryContext = await callPathway('sys_search_memory', { ...args, stream: false, section: 'memoryAll', updateContext: true });
         if (memoryContext) {
-            const {toolCallId} = addToolCalls(args.chatHistory, "search memory for relevant information", "memory_lookup");
-            addToolResults(args.chatHistory, memoryContext, toolCallId);
+            insertToolCallAndResults(args.chatHistory, "search memory for relevant information", "memory_lookup", memoryContext);
         }
 
         let result;

package/pathways/system/entity/sys_generator_quick.js

@@ -15,7 +15,7 @@ export default {
         let pathwayResolver = resolver;
  
         const promptMessages = [
-            {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}} While you have those capabilities but you have already decided it is not necessary to do any of those things to respond in this turn of the conversation. Never pretend like you are searching, looking anything up, or reading or looking in a file or show the user any made up or hallucinated information including non-existent images.\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
+            {"role": "system", "content": `{{renderTemplate AI_MEMORY}}\n\n{{renderTemplate AI_COMMON_INSTRUCTIONS}}\n{{renderTemplate AI_EXPERTISE}}\n{{renderTemplate AI_MEMORY_INSTRUCTIONS}}\n{{renderTemplate AI_DATETIME}}`},
             "{{chatHistory}}",
         ];
 

package/pathways/system/entity/sys_router_tool.js

@@ -56,8 +56,16 @@ If you decide to use a tool, return a JSON object in this format:
 
 toolMessage Guidelines:
 - The message should be consistent in style and tone with the rest of your responses in the conversation history.
-- The message should be brief and conversational and flow naturally with the conversation history.
-- The message should be something a human would say to the user to stall for time while you're working on the task.
+- The message should be brief, conversational, and flow naturally with the conversation history.
+- Use contractions (I'm, don't, let's) and informal language to sound more natural.
+- Include slight verbal hesitations or false starts occasionally (e.g., "So... I think what we could do here is" or "Actually, let me try a different approach").
+- Vary sentence length and structure - mix short punchy sentences with longer ones.
+- Include casual transitions and filler phrases that humans use: "you know," "basically," "so," "actually," "by the way," "anyway," etc.
+- Express uncertainty when appropriate with phrases like "I'm pretty sure," "If I remember correctly," or "I think."
+- Reference personal preferences or opinions occasionally ("I personally find that..." or "I tend to prefer...").
+- Occasionally make minor self-corrections as humans do ("Wait, let me rephrase that" or "Actually, I meant to say...").
+- Humans don't refer to tools - stall for time by saying things like "Let me think about this for a sec," "Drawing that up for you now," or "Working on it..." Humans also stall with vocalizations like "hmm," "let's see," or "give me a moment."
+- When appropriate, react emotionally to content with expressions like "Wow!" or "That's interesting!"
 
 If no tool is required, return:
 {"toolRequired": false, "toolReason": "explanation of why no tool was necessary"}

package/pathways/transcribe_gemini.js

@@ -5,7 +5,6 @@ import { Prompt } from "../server/prompt.js";
 
 const OFFSET_CHUNK = 500; //seconds of each chunk offset, only used if helper does not provide
 
-// Function to properly detect YouTube URLs
 function isYoutubeUrl(url) {
     try {
         const urlObj = new URL(url);
@@ -23,6 +22,10 @@ function isYoutubeUrl(url) {
             if (urlObj.pathname.startsWith("/embed/")) {
                 return urlObj.pathname.length > 7; // '/embed/' is 7 chars
             }
+            // For shorts URLs, verify they have a video ID in the path
+            if (urlObj.pathname.startsWith("/shorts/")) {
+                return urlObj.pathname.length > 8; // '/shorts/' is 8 chars
+            }
             return false;
         }
 
@@ -45,7 +48,7 @@ export default {
             "{{messages}}",
         ]}),
     ],
-    model: 'gemini-flash-20-vision',
+    model: 'gemini-pro-20-vision',
     inputParameters: {
         file: ``,
         language: ``,
@@ -96,7 +99,10 @@ export default {
         sendProgress(true);
         intervalId = setInterval(() => sendProgress(true), 3000);
 
-        const { file, responseFormat, wordTimestamped, maxLineWidth } = args;
+        const { file, wordTimestamped, maxLineWidth } = args;
+
+        const responseFormat = args.responseFormat || 'text';
+
         if(!file) {
             throw new Error("Please provide a file to transcribe.");
         }
@@ -129,9 +135,7 @@ export default {
             respectLimitsPrompt += `  These subtitles will be shown in a ${possiblePlacement} formatted video player.  Each subtitle line should not exceed ${maxLineWidth} characters to fit the player.`;
         }
 
-        function getMessages(file, format) {
-
-            const responseFormat = format !== 'text' ? 'VTT' : 'text';
+        function getMessages(file) {
             
             // Base system content that's always included
             let systemContent = `Instructions:
@@ -216,7 +220,7 @@ REMEMBER:
             const messages = [
                 {"role": "system", "content": systemContent},
                 {"role": "user", "content": [
-                    `{ type: 'text', text: 'Transcribe this file in ${responseFormat} format.${respectLimitsPrompt}' }`,
+                    `{ type: 'text', text: 'Transcribe this file in ${responseFormat} format.${respectLimitsPrompt} Output only the transcription, no other text or comments or formatting.' }`,
                     JSON.stringify({
                         type: 'image_url',
                         url: file,
@@ -266,7 +270,7 @@ REMEMBER:
         
         const result = await processChunksParallel(chunks, args);
         
-        if (['srt','vtt'].includes(responseFormat) || wordTimestamped) { // align subtitles for formats
+        if (['srt','vtt'].includes(responseFormat.toLowerCase()) || wordTimestamped) { // align subtitles for formats
             const offsets = chunks.map((chunk, index) => chunk?.offset || index * OFFSET_CHUNK);
             return alignSubtitles(result, responseFormat, offsets);
         }