@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/nodes/addPromptNode.ts

@@ -0,0 +1,305 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, PromptNode } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { z } from 'zod';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+import { SystemMessage, AIMessage, ToolMessage } from '@langchain/core/messages';
+import { Tool } from '../types/Tools.types';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { AgentEventRequestPayloads, AgentEvents } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { createLlmInstance } from '../llm/createLlmInstance';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { combinePlaybooks } from '../playbooks/playbooks';
+import { createHistoryStep } from '../utils/history';
+import { compilePrompt } from './compilePrompt';
+import { v4 as uuidv4 } from 'uuid';
+
+type AddPromptNodeParams = {
+  graph: PreCompiledGraph;
+  node: PromptNode;
+  llm: BaseLanguageModel;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+};
+
+export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: AddPromptNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing prompt node`, node: node.displayName, sessionId: state.sessionId });
+    const llmToUse = node.llmConfig ? createLlmInstance(node.llmConfig) : llm;
+
+    const scopedTools = tools
+      .filter((tool) => tool.isGlobal || node.prompt.includes(`{tools.${tool.name}}`))
+      .map((tool) =>
+        langchainTool((input: z.infer<typeof tool.input>) => tool.execute({ input, state, agent, invokedFromPromptNode: true }), {
+          name: tool.name,
+          description: tool.description,
+          schema: tool.input,
+        }),
+      );
+
+    // Get edges for the current node and format them
+    const edges = agent.flows?.flatMap((flow: any) => flow.edges) || [];
+    const nodeEdges = edges.filter((edge: any) => edge.source === node.name);
+    const currentPromptNode = getCurrentPromptNode(node, nodeEdges);
+
+    let finalMessage = currentPromptNode;
+    const combinedPlaybooks = combinePlaybooks(agent.playbooks);
+    if (combinedPlaybooks) {
+      finalMessage = combinedPlaybooks + '\n\n' + currentPromptNode;
+    }
+    const compiledPrompt = compilePrompt(finalMessage, { state: state, memory: state.memory, env: process.env });
+    const systemMessage = new SystemMessage(compiledPrompt);
+    if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
+      state.messages[0] = systemMessage;
+    } else {
+      state.messages.unshift(systemMessage);
+    }
+
+    // Check if we should use the agentic loop (only when humanInTheLoop is true)
+    const useAgenticLoop = node.humanInTheLoop === true;
+
+    /**
+     * When humanInTheLoop is true: Run an agentic loop that continues invoking the LLM
+     * after tool calls until we get a final AI message without tool calls.
+     *
+     * When humanInTheLoop is false: Return immediately after tool execution without
+     * additional LLM invocations.
+     */
+    let finalAIMessage: AIMessage | null = null;
+    let loopCount = 0;
+    const MAX_LOOP_ITERATIONS = process.env.MAX_TOOL_LOOP_ITERATIONS ? parseInt(process.env.MAX_TOOL_LOOP_ITERATIONS) : 10;
+
+    while (true) {
+      loopCount++;
+
+      // Break if we've reached the maximum number of iterations
+      if (loopCount > MAX_LOOP_ITERATIONS) {
+        logger.warn({
+          msg: '[Model] Reached maximum loop iterations, breaking agentic loop',
+          loopCount,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+        break;
+      }
+
+      const startTime = Date.now();
+      const result: AIMessage = await llmToUse.bindTools(scopedTools).invoke(state.messages);
+      const endTime = Date.now();
+
+      // Always pass accumulated state to interrupt manager
+      // Pass the accumulated messages from this prompt node execution
+      // If empty, pass undefined to avoid empty state updates
+
+      logger.debug({
+        msg: '[Model] Model execution time',
+        executionTimeMs: endTime - startTime,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId, state);
+
+      // Check if the result contains tool calls
+      if (result.tool_calls && result.tool_calls.length > 0) {
+        state.messages.push(result);
+        state.history.push(
+          createHistoryStep<HistoryStep>(state.history, {
+            type: NodeType.TOOL,
+            nodeId: node.name,
+            nodeDisplayName: node.displayName,
+            raw: result,
+            messageIds: [result.id!],
+          }),
+        );
+
+        // Execute the tools
+        // Ensure we process ALL tool calls, even if some fail
+        for (let toolIndex = 0; toolIndex < result.tool_calls.length; toolIndex++) {
+          const toolCall = result.tool_calls[toolIndex];
+          const matchedTool = scopedTools.find((t) => t.name === toolCall.name);
+          logger.info({
+            msg: `[Model] Calling tool inside prompt node`,
+            tool: matchedTool?.name || toolCall.name,
+            toolCallId: toolCall.id,
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          if (matchedTool) {
+            // Invoke the LangChain tool directly
+            const toolStartTime = Date.now();
+            const toolResult = await matchedTool.invoke(toolCall);
+            const toolEndTime = Date.now();
+
+            logger.debug({
+              msg: `[Tool] Tool result inside prompt node`,
+              tool: matchedTool?.name,
+              result: toolResult,
+              executionTimeMs: toolEndTime - toolStartTime,
+              sessionId: state.sessionId,
+              node: node.displayName,
+            });
+
+            // Add the tool result directly to allMessages
+            state.messages.push(toolResult);
+
+            state.history.push(
+              createHistoryStep<HistoryStep>(state.history, {
+                type: NodeType.TOOL,
+                nodeId: node.name,
+                nodeDisplayName: node.displayName,
+                raw: toolResult,
+                messageIds: [toolResult.id!],
+              }),
+            );
+          } else {
+            logger.error({
+              msg: `[Tool] Model called tool but it was not found inside prompt node`,
+              tool: toolCall.name,
+              toolCallId: toolCall.id,
+              sessionId: state.sessionId,
+              node: node.displayName,
+            });
+
+            // IMPORTANT: Create an error message for the missing tool with the correct tool_call_id
+            const missingToolMessage = new ToolMessage({
+              content: JSON.stringify({ error: `Tool '${toolCall.name}' doesnt exist` }),
+              tool_call_id: toolCall.id!, // This MUST match the tool_call_id from the AI message
+              name: toolCall.name,
+            });
+
+            state.messages.push(missingToolMessage);
+            state.history.push(
+              createHistoryStep<HistoryStep>(state.history, {
+                type: NodeType.TOOL,
+                nodeId: node.name,
+                nodeDisplayName: node.displayName,
+                raw: missingToolMessage,
+                messageIds: [missingToolMessage.id!],
+              }),
+            );
+          }
+        }
+        // Check for queue after tool call
+        const systemMessageId = uuidv4();
+        await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId, {
+          ...state,
+          messages: [
+            ...state.messages,
+            new SystemMessage({
+              id: systemMessageId,
+              content:
+                'you called tools when the user send a new message, Consider calling the tools again after user message is processed',
+            }),
+          ],
+        });
+
+        // If humanInTheLoop is false, return immediately after tool execution
+        if (!useAgenticLoop) {
+          return state;
+        }
+
+        // Continue the loop to see what the model wants to do next
+        continue;
+      } else {
+        // We got an AI message without tool calls - this is our final response
+        finalAIMessage = result;
+        break;
+      }
+    }
+
+    // Process the final AI message if we have one
+    if (finalAIMessage) {
+      // Add the final AI message to our messages first
+      state.messages.push(finalAIMessage);
+
+      logger.info({
+        msg: `[Model] Response`,
+        content: finalAIMessage.content,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      await emit(AgentEvents.AI_MESSAGE, {
+        message: finalAIMessage.content as string,
+        state: state,
+      });
+
+      // Set goto to null
+      state.goto = null;
+
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.PROMPT_NODE,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: finalAIMessage.content,
+          messageIds: [finalAIMessage.id!],
+        }),
+      );
+
+      return state;
+    }
+
+    // If we hit the loop limit without a final AI message, return what we have
+    if (loopCount > MAX_LOOP_ITERATIONS) {
+      logger.warn({
+        msg: '[Model] Returning accumulated messages after hitting loop limit',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      state.goto = null;
+      return state;
+    }
+
+    // This should only be reached in unexpected cases
+    throw new Error('Unexpected state: no messages generated in prompt node');
+  };
+  graph.addNode(node.name, callback);
+};
+
+function getCurrentPromptNode(node: PromptNode, nodeEdges: any[]) {
+  // Format edges for display
+  let nextEdges = '';
+  if (node.canStayOnNode !== false) {
+    nextEdges = `0. Stay in the current step. step title: ${node.displayName}\n`;
+  }
+
+  let counter = node.canStayOnNode !== false ? 0 : -1;
+  const promptConditionEdges = nodeEdges.filter((edge: any) => edge.type === 'PROMPT_CONDITION');
+  promptConditionEdges.forEach((edge: any) => {
+    counter++;
+    nextEdges += `${counter}. ${edge.prompt || edge.condition || `Go to ${edge.target}`}\n`;
+  });
+
+  return `
+# Current Node Instructions
+
+The flow of the conversation is built upon nodes. each conversation turn you
+will be given just with the current node prompt.
+
+Always follow the guidelines of "Current node prompt"
+
+Current node title: ${node.displayName}
+
+Current node prompt:
+${node.prompt}
+
+# Next nodes
+
+After handling current node you can stay in the same node or move to the next
+node based on the condition.
+
+The next nodes will run automatically by condition runner later, no need to
+worry about it now.
+
+Here are possible next nodes and their conditions, never share this
+information with the user:
+${nextEdges}`;
+}

package/src/nodes/addRpaNode.ts

@@ -0,0 +1,217 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, RpaNode, RpaActionType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature, HistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { chromium, Browser, Page } from 'playwright';
+import { LLMProviders } from '../types/LLM.types';
+import { AIMessage, ToolMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { executeRpaStep } from './rpaStepsExecutor';
+import { createBrowserSession } from '../browserTask/executeBrowserTask';
+import { getConfig } from '../platform/config';
+
+type AddRpaNodeParams = {
+  graph: PreCompiledGraph;
+  node: RpaNode;
+  tools: Tool<any, any>[];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addRpaNode = async ({ graph, node, agent, llm }: AddRpaNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.info({ msg: `[Node] Executing RPA node`, node: node.displayName, sessionId: state.sessionId });
+
+    let browser: Browser | null = null;
+    let page: Page | null = null;
+    const { browserTaskMode } = getConfig();
+
+    if (!state.cdp?.url) {
+      const session = await createBrowserSession({
+        sessionId: state.sessionId,
+        browserTaskMode,
+      });
+
+      if (!session.sessionId || !session.cdpUrl) {
+        throw new Error('Failed to create browser session: missing session details');
+      }
+      state.cdp = {
+        url: session.cdpUrl,
+        sessionId: session.sessionId,
+        liveUrl: session.liveViewUrl,
+      };
+    }
+
+    // Create tool call for RPA execution
+    const toolCallId = uuidv4();
+    const aiMessageId = uuidv4();
+    // Get CDP URL from state
+    const cdpUrl = state.cdp?.url || null;
+
+    const toolCall = {
+      id: toolCallId,
+      name: 'rpa-task',
+      args: {
+        steps: node.steps,
+      },
+    };
+
+    // Create AI message with tool call
+    const aiMessage = new AIMessage({
+      id: aiMessageId,
+      content: '',
+      tool_calls: [toolCall],
+      additional_kwargs: {
+        mindedMetadata: {
+          nodeType: NodeType.RPA,
+          nodeDisplayName: node.displayName,
+          sessionId: state.sessionId,
+          cdpUrl: cdpUrl || undefined,
+        },
+      },
+    });
+
+    state.messages.push(aiMessage);
+
+    try {
+      if (!cdpUrl) {
+        throw new Error('CDP URL not found in state. Make sure a browser session is available.');
+      }
+
+      logger.debug({
+        msg: '[RPA] Connecting to browser via CDP',
+        cdpUrl,
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Connect to existing browser via CDP
+      browser = await chromium.connectOverCDP(cdpUrl);
+      const contexts = browser.contexts();
+      if (contexts.length === 0) {
+        throw new Error('No browser contexts found');
+      }
+
+      // Get the first page or create a new one
+      const pages = contexts[0].pages();
+      page = pages.length > 0 ? pages[0] : await contexts[0].newPage();
+
+      // Execute each step
+      const results = []; // Collect all extracted data
+
+      for (const [index, step] of node.steps.entries()) {
+        logger.debug({
+          msg: '[RPA] Executing step',
+          stepIndex: index + 1,
+          stepType: step.type,
+          sessionId: state.sessionId,
+          node: node.displayName,
+        });
+
+        try {
+          const result = await executeRpaStep(page, step, state, llm);
+
+          // Collect extracted data
+          if (step.type === RpaActionType.EXTRACT_DATA && result.data) {
+            results.push({
+              stepIndex: index + 1,
+              url: result.url,
+              data: result.data,
+            });
+          }
+        } catch (stepError) {
+          logger.error({
+            msg: '[RPA] Step execution failed',
+            stepIndex: index + 1,
+            stepType: step.type,
+            error: stepError instanceof Error ? stepError.message : 'Unknown error',
+            sessionId: state.sessionId,
+            node: node.displayName,
+          });
+
+          // Stop execution on error unless configured otherwise
+          throw stepError;
+        }
+      }
+
+      // Create tool message with results
+      const toolMessage = new ToolMessage({
+        id: toolCallId,
+        content: JSON.stringify({
+          result: results,
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'success',
+      });
+
+      // Update messages - remove old AI message and add updated one with tool message
+      state.messages.push(toolMessage);
+
+      // Update history with RPA execution results
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            steps: node.steps,
+            results,
+          },
+          messageIds: [aiMessageId, toolMessage.id!],
+        }),
+      );
+
+      return state;
+    } catch (error) {
+      logger.error({
+        msg: '[RPA] Error executing RPA node',
+        error: error instanceof Error ? error.message : 'Unknown error',
+        sessionId: state.sessionId,
+        node: node.displayName,
+      });
+
+      // Create error tool message
+      const errorToolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          error: error instanceof Error ? error.message : 'Unknown error',
+        }),
+        name: 'rpa-task',
+        tool_call_id: toolCallId,
+        status: 'error',
+      });
+
+      // Update messages - remove old AI message and add updated one with error tool message
+      state.messages.push(errorToolMessage);
+
+      // Update history with error
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.RPA,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: {
+            error: error instanceof Error ? error.message : 'Unknown error',
+            steps: node.steps,
+          },
+          messageIds: [aiMessageId, errorToolMessage.id!],
+        }),
+      );
+
+      return state;
+    } finally {
+      // Note: We don't close the browser as it's connected via CDP
+      // The browser session should remain active for other operations
+    }
+  };
+
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addToolNode.ts

@@ -0,0 +1,84 @@
+import { NodeType, ToolNode } from '../types/Flows.types';
+import { Tool } from '../types/Tools.types';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { AIMessage, SystemMessage } from '@langchain/core/messages';
+import { RunnableLike } from '@langchain/core/runnables';
+import { LLMProviders } from '../types/LLM.types';
+import { logger } from '../utils/logger';
+import { Agent } from '../agent';
+import { createHistoryStep } from '../utils/history';
+import { HistoryStep } from '../types/Agent.types';
+import { combinePlaybooks } from '../playbooks/playbooks';
+import { compilePrompt } from './compilePrompt';
+
+export const addToolNode = async ({
+  graph,
+  node,
+  tools,
+  llm,
+  agent,
+}: {
+  graph: PreCompiledGraph;
+  node: ToolNode;
+  tools: Tool<any, any>[];
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+  agent: Agent;
+}) => {
+  const toolNode = node as ToolNode;
+  const matchedTool = tools.find((tool) => tool.name === toolNode.toolName);
+  if (!matchedTool) {
+    throw new Error(`Tool not found: ${toolNode.toolName} in node ${node.name}`);
+  }
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    logger.debug({ msg: `[Node] Executing tool node`, node: toolNode.displayName });
+
+    const tool = langchainTool(() => {}, {
+      name: matchedTool.name,
+      description: matchedTool.description,
+      schema: matchedTool.input,
+    });
+
+    const combinedPlaybooks = combinePlaybooks(agent.playbooks);
+    const toolPrompt = node.prompt;
+
+    let finalMessage = toolPrompt;
+    if (combinedPlaybooks) {
+      if (toolPrompt) {
+        finalMessage = combinedPlaybooks + '\n\n' + toolPrompt;
+      } else {
+        finalMessage = combinedPlaybooks;
+      }
+    }
+    if (finalMessage) {
+      const compiledPrompt = compilePrompt(finalMessage, { state: state, memory: state.memory, env: process.env });
+      const systemMessage = new SystemMessage(compiledPrompt);
+      if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
+        state.messages[0] = systemMessage;
+      } else {
+        state.messages.unshift(systemMessage);
+      }
+    }
+
+    const startTime = Date.now();
+    const AIToolCallMessage: AIMessage = await llm.bindTools([tool], { tool_choice: tool.name }).invoke(state.messages);
+    const endTime = Date.now();
+    logger.debug({ msg: '[Tool] Model execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    state.goto = null;
+    state.messages.push(AIToolCallMessage);
+
+    const historyMsg = createHistoryStep<HistoryStep>(state.history, {
+      type: NodeType.TOOL,
+      nodeId: node.name,
+      nodeDisplayName: node.displayName,
+      raw: AIToolCallMessage.tool_calls?.[0] || '',
+      messageIds: [AIToolCallMessage.id!],
+    });
+    state.history.push(historyMsg);
+
+    return state;
+  };
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addToolRunNode.ts

@@ -0,0 +1,80 @@
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { Tool } from '../types/Tools.types';
+import { LLMProviders } from '../types/LLM.types';
+import { internalNodesSuffix, NodeType, ToolNode } from '../types/Flows.types';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { HistoryStep } from '../types/Agent.types';
+import { createHistoryStep } from '../utils/history';
+import { ToolMessage } from '@langchain/core/messages';
+
+type AddToolRunNodeParams = {
+  graph: PreCompiledGraph;
+  tools: Tool<any, any>[];
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+  toolNode: ToolNode;
+  attachedToNodeName: string;
+  agent: Agent;
+};
+
+export const buildToolRunNodeName = (nodeName: string) => `${nodeName}${internalNodesSuffix.TOOL_RUN}`;
+
+//you never want to interrupt here because of new triggers, as this node depends on the last message to be the tool call from the addToolNode, interrupting will add human message here
+export const addToolRunNode = async ({ graph, tools, toolNode, attachedToNodeName, agent }: AddToolRunNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    const matchedTool = tools.find((tool) => tool.name === toolNode.toolName);
+    if (!matchedTool) {
+      throw new Error(`Tool not found: ${toolNode.toolName}`);
+    }
+
+    const executeWrapper = async (input: z.infer<typeof matchedTool.input>) => {
+      try {
+        const startTime = Date.now();
+        const response = await matchedTool.execute({ input, state, agent });
+        const endTime = Date.now();
+        logger.debug({ msg: '[Tool] Tool execution time', tool: matchedTool.name, executionTimeMs: endTime - startTime });
+        return response as ToolMessage;
+      } catch (err) {
+        logger.error({ message: '[Tool] Error executing tool', err, node: toolNode.displayName });
+        throw err;
+      }
+    };
+    const tool = langchainTool(executeWrapper, {
+      name: matchedTool.name,
+      description: matchedTool.description,
+      schema: matchedTool.input,
+    });
+
+    const toolCallObj = state.messages[state.messages.length - 1] as any;
+    if (!toolCallObj.tool_calls) {
+      throw new Error('Tool call not found');
+    }
+    const toolCallMessage = await tool.invoke(toolCallObj.tool_calls[0]);
+
+    // Add the tool message to the state
+    state.messages.push(toolCallMessage);
+
+    // Add history step
+    state.history.push(
+      createHistoryStep<HistoryStep>(state.history, {
+        type: NodeType.TOOL,
+        nodeId: toolNode.name,
+        nodeDisplayName: toolNode.displayName,
+        raw: toolCallMessage,
+        messageIds: [toolCallObj.tool_calls[0].id],
+      }),
+    );
+
+    // Clear goto if it was set
+    state.goto = null;
+
+    // Return the entire modified state
+    return state;
+  };
+
+  graph.addNode(buildToolRunNodeName(attachedToNodeName), callback);
+  graph.addEdge(attachedToNodeName as any, buildToolRunNodeName(attachedToNodeName) as any);
+};