@minded-ai/mindedjs 1.0.103-beta-1 → 1.0.103-beta-4

package/src/nodes/addAppToolNode.ts

@@ -26,14 +26,14 @@ export const addAppToolNode = async ({
   const cleanedParameters = Object.fromEntries(Object.entries(node.parameters || {}).filter(([, value]) => value !== ''));
   const appRunnerTool = getAppActionRunnerTool(node.displayName!);
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
-    logger.info({ message: `Executing tool node ${appRunnerTool.name}` });
+    logger.debug({ msg: `[Node] Executing tool node`, node: appRunnerTool.name });
 
     const executeWrapper = async (input: z.infer<typeof appRunnerTool.input>) => {
       try {
         const response = await appRunnerTool.execute({ input, state, agent });
         return response || {};
       } catch (error) {
-        logger.error({ msg: 'Error executing tool', error });
+        logger.error({ msg: `[Node] Error executing tool`, error, node: node.name });
         throw error;
       }
     };
@@ -75,11 +75,12 @@ export const addAppToolNode = async ({
     };
     const toolCallMessage = await tool.invoke(AIToolCallMessage.tool_calls[0]);
     AIToolCallMessage.additional_kwargs = {
-      appActionInvocation: {
-        nodeTitle: node.name,
+      mindedMetadata: {
+        nodeType: NodeType.APP_TOOL,
       },
     };
     return {
+      goto: null,
       messages: [AIToolCallMessage, toolCallMessage],
       history: createHistoryStep<AppActionInvocationHistoryStep>(state.history, {
         type: NodeType.APP_TOOL,

package/src/nodes/addBrowserTaskNode.ts

@@ -0,0 +1,231 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { BrowserTaskNode, NodeType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { HistoryStep } from '../types/Agent.types';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { AIMessage, SystemMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { Agent } from '../agent';
+import { createCloudTask, waitForLiveUrl } from '../browserTask/executeBrowserTask';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { LLMProviders } from '../types/LLM.types';
+import { compilePlaybooks } from '../playbooks/playbooks';
+import * as ejs from 'ejs';
+
+/**
+ * Compile prompt with parameters using EJS and placeholder replacement
+ */
+function compilePrompt(prompt: string, params: Record<string, any> = {}): string {
+  try {
+    // First, render with EJS
+    let compiledPrompt = ejs.render(prompt, params);
+
+    // Then, replace placeholders in {} format
+    compiledPrompt = replacePlaceholders(compiledPrompt, params);
+
+    return compiledPrompt;
+  } catch (error) {
+    logger.error({ message: 'Error compiling prompt', error });
+    return prompt; // Return uncompiled if there's an error
+  }
+}
+
+/**
+ * Replace placeholders in {key} format
+ */
+function replacePlaceholders(text: string, params: Record<string, any>): string {
+  return text.replace(/\{([^}]+)\}/g, (match, key) => {
+    const keys = key.split('.');
+    let value: any = params;
+
+    for (const k of keys) {
+      if (value && typeof value === 'object' && k in value) {
+        value = value[k];
+      } else {
+        return match; // Return original if key not found
+      }
+    }
+
+    return String(value);
+  });
+}
+
+type AddBrowserTaskNodeParams = {
+  graph: PreCompiledGraph;
+  node: BrowserTaskNode;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowserTaskNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.info({ msg: `Executing browser task node ${node.displayName}`, prompt: node.prompt });
+
+    // Create Zod schema from inputSchema
+    const schemaFields: Record<string, z.ZodTypeAny> = {};
+    if (node.inputSchema) {
+      for (const field of node.inputSchema) {
+        let fieldSchema: z.ZodTypeAny;
+
+        if (field.type === 'string') {
+          fieldSchema = z.string();
+        } else if (field.type === 'number') {
+          fieldSchema = z.number();
+        } else {
+          fieldSchema = z.string(); // Default to string
+        }
+
+        if (field.description) {
+          fieldSchema = fieldSchema.describe(field.description);
+        }
+
+        if (field.required === false) {
+          fieldSchema = fieldSchema.optional();
+        }
+
+        schemaFields[field.name] = fieldSchema;
+      }
+    }
+
+    const zodSchema = z.object(schemaFields);
+
+    // Create langchain tool
+    const tool = langchainTool(() => {}, {
+      name: 'browser-task',
+      description: node.prompt,
+      schema: zodSchema,
+    });
+
+    // Get compiled playbooks
+    const playbookParams = {
+      ...state.memory,
+      state,
+      currentTime: new Date().toISOString(),
+    };
+    const compiledPlaybooks = compilePlaybooks(agent.playbooks, playbookParams) || '';
+
+    const systemPrompt = `${compiledPlaybooks ? compiledPlaybooks + '\n\n' : ''}
+Additional context:
+- workflow memory: ${JSON.stringify(state.memory)}`;
+
+    try {
+      // Use LLM to generate tool call
+      const AIToolCallMessage: AIMessage = await llm
+        .bindTools([tool], {
+          tool_choice: tool.name,
+        })
+        .invoke([...state.messages, new SystemMessage(systemPrompt)]);
+
+      if (!AIToolCallMessage.tool_calls || AIToolCallMessage.tool_calls.length === 0) {
+        throw new Error('No tool calls generated by LLM');
+      }
+
+      const toolCall = AIToolCallMessage.tool_calls[0];
+      const inputParams = toolCall.args || {};
+
+      // Prepare parameters for prompt compilation
+      const promptParams = {
+        input: inputParams,
+        state,
+        currentTime: new Date().toISOString(),
+      };
+
+      // Compile the prompt with parameters
+      const compiledPrompt = compilePrompt(node.prompt, promptParams);
+
+      // Build the full prompt with compiled content
+      const fullPrompt = `
+Follow the instructions. Any retrieved data should be printed as string and not saved to a file.
+
+# Task instructions:
+${compiledPrompt}
+
+${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(inputParams, null, 2)}\n\n` : ''}
+${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
+`;
+
+      // Create the browser task in the cloud
+      const taskId = await createCloudTask(fullPrompt, node.model);
+      logger.debug({ msg: 'Browser task created', taskId });
+
+      // Wait for live_url to become available
+      const taskDetails = await waitForLiveUrl(taskId);
+      logger.debug({ msg: 'Live URL available', taskId, liveUrl: taskDetails.live_url });
+
+      // Update the tool call with taskId and other metadata
+      const updatedToolCall = {
+        ...toolCall,
+        args: {
+          ...inputParams,
+          taskId,
+          prompt: fullPrompt,
+        },
+      };
+
+      const toolCallingMessage = new AIMessage({
+        id: AIToolCallMessage.id,
+        content: AIToolCallMessage.content,
+        tool_calls: [updatedToolCall],
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: node.displayName,
+            taskId,
+            liveUrl: taskDetails.live_url,
+          },
+        },
+      });
+
+      return {
+        history: createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: { taskId, liveUrl: taskDetails.live_url, inputParams },
+          messageIds: [toolCallingMessage.id!],
+        }),
+        messages: [toolCallingMessage],
+      };
+    } catch (error: any) {
+      logger.error(`Error creating browser task node ${node.displayName}: ${error}`);
+
+      const errorMessageId = uuidv4();
+      const errorToolCallId = uuidv4();
+
+      const toolCallingMessage = new AIMessage({
+        id: errorMessageId,
+        content: '',
+        tool_calls: [
+          {
+            id: errorToolCallId,
+            name: 'browser-task',
+            args: {
+              error: error.message,
+            },
+          },
+        ],
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: node.displayName,
+            error: error.message,
+          },
+        },
+      });
+
+      return {
+        history: createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: error,
+          messageIds: [errorMessageId],
+        }),
+        messages: [toolCallingMessage],
+      };
+    }
+  };
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addBrowserTaskRunNode.ts

@@ -0,0 +1,144 @@
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { internalNodesSuffix, NodeType, BrowserTaskNode } from '../types/Flows.types';
+import { ToolMessage, AIMessage } from '@langchain/core/messages';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { HistoryStep } from '../types/Agent.types';
+import { v4 as uuidv4 } from 'uuid';
+import { waitForCompletion } from '../browserTask/executeBrowserTask';
+
+type AddBrowserTaskRunNodeParams = {
+  graph: PreCompiledGraph;
+  browserTaskNode: BrowserTaskNode;
+  attachedToNodeName: string;
+  agent: Agent;
+};
+
+export const buildBrowserTaskRunNodeName = (nodeName: string) => `${nodeName}${internalNodesSuffix.BROWSER_TASK_RUN}`;
+
+export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeName }: AddBrowserTaskRunNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.info(`Executing browser task run node for ${browserTaskNode.displayName}`);
+
+    const toolCallObj = state.messages[state.messages.length - 1] as any;
+    if (!toolCallObj.tool_calls) {
+      throw new Error('Tool call not found');
+    }
+    const toolCall = toolCallObj.tool_calls[0];
+    const { taskId, ...inputParams } = toolCall.args;
+    try {
+      if (!taskId) {
+        throw new Error('Task ID not found in tool call arguments');
+      }
+
+      // Wait for task completion
+      const completedTask = await waitForCompletion(taskId);
+      logger.debug({ msg: 'Browser task completed', taskId, output: completedTask.output });
+
+      // Create tool message with the result
+      const toolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          result: completedTask.output || 'Task completed successfully',
+          steps: completedTask.steps || [],
+          inputParams: inputParams,
+        }),
+        name: 'browser-task',
+        tool_call_id: toolCall.id,
+        status: 'success',
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: browserTaskNode.displayName,
+            taskId,
+            steps: completedTask.steps,
+            inputParams: inputParams,
+          },
+        },
+      });
+
+      // Update the AI message with steps using the message reducer pattern
+      const updatedAIMessage = new AIMessage({
+        id: toolCallObj.id,
+        content: toolCallObj.content,
+        tool_calls: toolCallObj.tool_calls,
+        additional_kwargs: {
+          ...toolCallObj.additional_kwargs,
+          mindedMetadata: {
+            ...toolCallObj.additional_kwargs?.mindedMetadata,
+            nodeDisplayName: browserTaskNode.displayName,
+            steps: completedTask.steps,
+            status: 'completed',
+          },
+          update: true, // This triggers the message reducer to update the existing message
+        },
+      });
+
+      return {
+        messages: [toolMessage, updatedAIMessage],
+        history: createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: browserTaskNode.name,
+          nodeDisplayName: browserTaskNode.displayName,
+          raw: completedTask,
+          messageIds: [],
+        }),
+      };
+    } catch (error: any) {
+      logger.error({ msg: 'Error executing browser task run node', error });
+
+      const errorToolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          error: error instanceof Error ? error.message : error.toString(),
+          inputParams: inputParams,
+        }),
+        status: 'error',
+        name: 'browser-task',
+        tool_call_id: toolCall.id,
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: browserTaskNode.displayName,
+            taskId,
+            error: error.message,
+            inputParams: inputParams,
+          },
+        },
+      });
+
+      // Update the AI message with error status
+      const updatedAIMessage = new AIMessage({
+        id: toolCallObj.id,
+        content: toolCallObj.content,
+        tool_calls: toolCallObj.tool_calls,
+        additional_kwargs: {
+          ...toolCallObj.additional_kwargs,
+          mindedMetadata: {
+            ...toolCallObj.additional_kwargs?.mindedMetadata,
+            nodeDisplayName: browserTaskNode.displayName,
+            status: 'failed',
+            error: error.message,
+          },
+          update: true,
+        },
+      });
+
+      return {
+        messages: [errorToolMessage, updatedAIMessage],
+        history: createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: browserTaskNode.name,
+          nodeDisplayName: browserTaskNode.displayName,
+          raw: error,
+          messageIds: [],
+        }),
+      };
+    }
+  };
+
+  graph.addNode(buildBrowserTaskRunNodeName(attachedToNodeName), callback);
+  graph.addEdge(attachedToNodeName as any, buildBrowserTaskRunNodeName(attachedToNodeName) as any);
+};

package/src/nodes/addHumanInTheLoopNode.ts

@@ -14,7 +14,8 @@ export const buildHumanInTheLoopNodeName = (nodeName: string) => `${nodeName}${i
 
 export const addHumanInTheLoopNode = async ({ graph, attachedToNodeName }: AddHumanInTheLoopNodeParams) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
-    logger.info({ message: `Executing "human in the loop" node for the attached node${attachedToNodeName}` });
+    logger.debug({ msg: `[Node] Waiting for human input`, node: attachedToNodeName });
+
     if (state.messages[state.messages.length - 1].getType() === 'ai') {
       const value = interrupt({ type: InterruptType.HUMAN_IN_THE_LOOP });
       return value;

package/src/nodes/addJumpToNode.ts

@@ -7,9 +7,10 @@ import { createHistoryStep } from '../utils/history';
 
 export const addJumpToNode = async ({ graph, node }: { graph: PreCompiledGraph; node: JumpToNode }) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
-    logger.info({ message: `Executing jump node ${node.displayName} – jumping to ${node.targetNodeId}` });
+    logger.info({ msg: `Executing jump node ${node.displayName} – jumping to ${node.targetNodeId}` });
     // No state modifications are necessary; control flow is handled via edges.
     return {
+      goto: null,
       history: createHistoryStep<HistoryStep>(state.history, {
         type: NodeType.JUMP_TO_NODE,
         nodeId: node.name,

package/src/nodes/addJunctionNode.ts

@@ -7,6 +7,7 @@ import { HistoryStep } from '../types/Agent.types';
 export const addJunctionNode = ({ graph, node }: { graph: PreCompiledGraph; node: JunctionNode }) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
     return {
+      goto: null,
       history: createHistoryStep<HistoryStep>(state.history, {
         type: NodeType.JUNCTION,
         nodeId: node.name,

package/src/nodes/addPromptNode.ts

@@ -14,10 +14,9 @@ import { Agent } from '../agent';
 import { logger } from '../utils/logger';
 import { compilePlaybooks } from '../playbooks/playbooks';
 import { createHistoryStep } from '../utils/history';
+import { v4 as uuidv4 } from 'uuid';
 import { wait } from '../utils/wait';
 import { sendPlaceholderMessage } from '../internalTools/sendPlaceholderMessage';
-import { v4 as uuidv4 } from 'uuid';
-
 type AddPromptNodeParams = {
   graph: PreCompiledGraph;
   node: PromptNode;
@@ -31,7 +30,7 @@ let shouldWait = true;
 export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: AddPromptNodeParams) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
-    logger.info({ message: `Executing prompt node ${node.displayName}` });
+    logger.info({ msg: `[Node] Executing prompt node`, node: node.displayName });
     const llmToUse = node.llmConfig ? createLlmInstance(node.llmConfig) : llm;
 
     const globalTools = tools
@@ -55,16 +54,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
     ${state.memory ? `# Task context:\n${JSON.stringify(state.memory)}\n\n` : ''}
     ${compiledPlaybooks ? `# General guidelines:\n${compiledPlaybooks}\n\n` : ''}
     `;
-    if (true) {
-      logger.info({ msg: `waiting 15 seconds` });
-      shouldWait = false;
 
-      await sendPlaceholderMessage({ sessionId: state.sessionId, message: 'just a moment please... ' });
-      await wait(10000);
-      logger.info({ msg: `done waiting 15 seconds` });
-    } else {
-      logger.info({ msg: `not waiting` });
-    }
     const result: AIMessage = await llmToUse.bindTools(globalTools).invoke([...state.messages, new SystemMessage(message)]);
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     // Check if the result contains tool calls
@@ -75,11 +65,12 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
 
       for (const toolCall of result.tool_calls) {
         const matchedTool = globalTools.find((t) => t.name === toolCall.name);
-        logger.info({ msg: `Model called tool ${matchedTool?.name}` });
+        logger.info({ msg: `[Model] Calling tool`, tool: matchedTool?.name });
         if (matchedTool) {
           try {
             // Invoke the LangChain tool directly
             const toolResult = await matchedTool.invoke(toolCall);
+            logger.debug({ msg: `[Tool] Tool result`, tool: matchedTool?.name, result: toolResult });
             //check for queue after tool call
             const systemMessageId = uuidv4();
 
@@ -113,7 +104,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
             toolResults.push(toolResult);
           } catch (error: any) {
             if (error?.name === 'GraphInterrupt') throw error;
-            logger.error({ msg: `Error executing tool ${toolCall.name}:`, error });
+            logger.error({ msg: `[Tool] Error executing tool`, tool: toolCall.name, error });
             const errorMessage = new ToolMessage({
               content: JSON.stringify({ error: error instanceof Error ? error.message : String(error) }),
               tool_call_id: toolCall.id!,
@@ -121,7 +112,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
             toolResults.push(errorMessage);
           }
         } else {
-          logger.error({ msg: `Model tried to call tool ${toolCall.name} but it was not found` });
+          logger.error({ msg: `[Tool] Model called tool but it was not found`, tool: toolCall.name });
         }
       }
 
@@ -157,9 +148,10 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
         message: result.content as string,
         state,
       });
-      console.log('AI Message', result.content);
+      logger.info({ msg: `[Model] Response`, content: result.content });
     }
     return {
+      goto: null,
       history: createHistoryStep<HistoryStep>(state.history, {
         type: NodeType.PROMPT_NODE,
         nodeId: node.name,

package/src/nodes/addToolNode.ts

@@ -31,7 +31,8 @@ export const addToolNode = async ({
   }
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
-    logger.info({ message: `Executing tool node ${toolNode.displayName}` });
+    logger.debug({ msg: `[Node] Executing tool node`, node: toolNode.displayName });
+
     const tool = langchainTool(() => {}, {
       name: matchedTool.name,
       description: matchedTool.description,
@@ -56,9 +57,9 @@ export const addToolNode = async ({
         tool_choice: tool.name,
       })
       .invoke([...state.messages, new SystemMessage(prompt)]);
-    logger.info({ message: 'after llm tool invoke' });
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
     return {
+      goto: null,
       messages: [AIToolCallMessage],
       history: createHistoryStep<HistoryStep>(state.history, {
         type: NodeType.TOOL,

package/src/nodes/addToolRunNode.ts

@@ -36,7 +36,7 @@ export const addToolRunNode = async ({ graph, tools, toolNode, attachedToNodeNam
         const response = await matchedTool.execute({ input, state, agent });
         return response || {};
       } catch (error) {
-        logger.error({ msg: 'Error executing tool', error });
+        logger.error({ msg: '[Tool] Error executing tool', error, node: toolNode.displayName });
         throw error;
       }
     };
@@ -56,7 +56,9 @@ export const addToolRunNode = async ({ graph, tools, toolNode, attachedToNodeNam
     // Push the toolCallMessage into the messages array from toolStateUpdate
     const updatedMessages = [toolCallMessage, ...((toolStateUpdate as any).messages || [])];
     // Return the full state update from the tool with the updated messages
+
     return {
+      goto: null,
       ...toolStateUpdate,
       messages: updatedMessages,
       history: createHistoryStep<HistoryStep>(state.history, {

package/src/nodes/addTriggerNode.ts

@@ -7,9 +7,10 @@ import { createHistoryStep } from '../utils/history';
 
 export const addTriggerNode = async ({ graph, node }: { graph: PreCompiledGraph; node: TriggerNode }) => {
   const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
-    logger.info({ message: `Executing trigger node ${node.displayName}` });
+    logger.debug({ msg: `[Trigger] Executing trigger node`, node: node.displayName });
     if (node.triggerType === TriggerType.MANUAL) {
       return {
+        goto: null,
         history: createHistoryStep<HistoryStep>(state.history, {
           type: NodeType.TRIGGER,
           nodeId: node.name,

package/src/nodes/nodeFactory.ts

@@ -11,6 +11,7 @@ import { LLMProviders } from '../types/LLM.types';
 import { Agent } from '../agent';
 import { addJumpToNode } from './addJumpToNode';
 import { addJunctionNode } from './addJunctionNode';
+import { addBrowserTaskNode } from './addBrowserTaskNode';
 
 export const nodeFactory = ({
   graph,
@@ -47,7 +48,10 @@ export const nodeFactory = ({
     case NodeType.JUMP_TO_NODE:
       addJumpToNode({ graph, node });
       break;
+    case NodeType.BROWSER_TASK:
+      addBrowserTaskNode({ graph, node, agent, llm });
+      break;
     default:
       throw new Error(`Unsupported node type: ${nodeType}`);
   }
-};
+};