@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/nodes/addAppToolNode.ts

@@ -0,0 +1,111 @@
+import { AppToolNode, NodeType } from '../types/Flows.types';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { SystemMessage } from '@langchain/core/messages';
+import { RunnableLike } from '@langchain/core/runnables';
+import { z } from 'zod';
+import { LLMProviders } from '../types/LLM.types';
+import { AppActionInvocationHistoryStep } from '../types/Agent.types';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { Tool } from '../types/Tools.types';
+import { combinePlaybooks } from '../playbooks/playbooks';
+import { compilePrompt } from './compilePrompt';
+import { ToolMessage } from '@langchain/core/messages';
+
+export const addAppToolNode = async ({
+  graph,
+  node,
+  llm,
+  agent,
+  tools,
+}: {
+  graph: PreCompiledGraph;
+  node: AppToolNode;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+  agent: Agent;
+  tools: Tool<any, any>[];
+}) => {
+  const cleanedParameters = Object.fromEntries(Object.entries(node.parameters || {}).filter(([, value]) => value !== ''));
+
+  // Find the tool from the tools array by name
+  const appRunnerTool = tools.find((tool) => tool.name === node.displayName);
+
+  if (!appRunnerTool) {
+    throw new Error(`Tool not found for node: ${node.displayName}`);
+  }
+
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.debug({ msg: `[Node] Executing tool node`, node: appRunnerTool.name });
+
+    const executeWrapper = async (input: z.infer<typeof appRunnerTool.input>) => {
+      try {
+        const response = await appRunnerTool.execute({ input, state, agent });
+        return response || {};
+      } catch (err) {
+        logger.error({ message: `[Node] Error executing tool`, err, node: node.name });
+        throw err;
+      }
+    };
+    const tool = langchainTool(executeWrapper, {
+      name: appRunnerTool.name,
+      description: appRunnerTool.description,
+      schema: appRunnerTool.input,
+    });
+
+    const combinedPlaybooks = combinePlaybooks(agent.playbooks) || '';
+
+    const message = `${combinedPlaybooks ? combinedPlaybooks + '\n\n' : ''}
+    Additional context:
+    previous messages are available for context.
+    Your goal is execute the tool with the correct parameters, some of them already chosen by the user and the rest should be generated.
+    You have 4 things to base your parameters on:
+    - Previous messages
+    - Parameters manually configured by the user (if any)
+    - User instructions for choosing tool parameters that are not set by the user (if any)
+    - Workflow memory
+    Parameters manually configured by the user are:
+    ${JSON.stringify(cleanedParameters)}
+    User instructions for choosing tool parameters are:
+        ${node.prompt ? `${node.prompt}` : 'no instructions set by the user'}`;
+
+    const compiledPrompt = compilePrompt(message, { state: state, memory: state.memory, env: process.env });
+    const systemMessage = new SystemMessage(compiledPrompt);
+    if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
+      state.messages[0] = systemMessage;
+    } else {
+      state.messages.unshift(systemMessage);
+    }
+
+    const AIToolCallMessage = await llm.bindTools([tool], { tool_choice: tool.name }).invoke(state.messages);
+    AIToolCallMessage.tool_calls[0].args = {
+      ...AIToolCallMessage.tool_calls[0].args,
+      ...cleanedParameters, //user set parameters have priority over ai generated parameters
+    };
+    const toolCallMessage = await tool.invoke(AIToolCallMessage.tool_calls[0]);
+    AIToolCallMessage.additional_kwargs = {
+      mindedMetadata: {
+        nodeType: NodeType.APP_TOOL,
+      },
+    };
+
+    state.goto = null;
+    state.messages.push(AIToolCallMessage);
+    if (toolCallMessage instanceof ToolMessage) {
+      state.messages.push(toolCallMessage);
+    }
+    state.history.push(
+      createHistoryStep<AppActionInvocationHistoryStep>(state.history, {
+        type: NodeType.APP_TOOL,
+        nodeId: node.name,
+        nodeDisplayName: node.displayName!,
+        raw: AIToolCallMessage.tool_calls[0],
+        appName: node.appName,
+        messageIds: [AIToolCallMessage.tool_calls[0].id],
+      }),
+    );
+    return state;
+  };
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addBrowserTaskNode.ts

@@ -0,0 +1,191 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { BrowserTaskNode, NodeType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { HistoryStep } from '../types/Agent.types';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { AIMessage, SystemMessage } from '@langchain/core/messages';
+import { v4 as uuidv4 } from 'uuid';
+import { Agent } from '../agent';
+import { createBrowserSession } from '../browserTask/executeBrowserTask';
+import { tool as langchainTool } from '@langchain/core/tools';
+import { LLMProviders } from '../types/LLM.types';
+import { compilePrompt } from './compilePrompt';
+import { combinePlaybooks } from '../playbooks/playbooks';
+import { BrowserTaskMode } from '../browserTask/types';
+import { getConfig } from '../platform/config';
+import { createZodSchemaFromFields } from '../utils/schemaUtils';
+
+type AddBrowserTaskNodeParams = {
+  graph: PreCompiledGraph;
+  node: BrowserTaskNode;
+  agent: Agent;
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+};
+
+export const addBrowserTaskNode = async ({ graph, node, agent, llm }: AddBrowserTaskNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.info({ msg: `Executing browser task node ${node.displayName}`, prompt: node.prompt });
+    await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+    // Create Zod schema from inputSchema
+    const zodSchema = createZodSchemaFromFields(node.inputSchema);
+
+    // Create langchain tool
+    const tool = langchainTool(() => {}, {
+      name: 'browser-task',
+      description: node.prompt,
+      schema: zodSchema as any,
+    });
+
+    const combinedPlaybooks = combinePlaybooks(agent.playbooks);
+    if (combinedPlaybooks) {
+      const compiledPrompt = compilePrompt(combinedPlaybooks, { state: state, memory: state.memory });
+      const systemMessage = new SystemMessage(compiledPrompt);
+      if (state.messages.length === 0 || state.messages[0].getType() === 'system') {
+        state.messages[0] = systemMessage;
+      } else {
+        state.messages.unshift(systemMessage);
+      }
+    }
+
+    try {
+      // Use LLM to generate tool call
+      const AIToolCallMessage: AIMessage = await llm.bindTools([tool], { tool_choice: tool.name }).invoke(state.messages);
+      await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId);
+      if (!AIToolCallMessage.tool_calls || AIToolCallMessage.tool_calls.length === 0) {
+        throw new Error('No tool calls generated by LLM');
+      }
+
+      const toolCall = AIToolCallMessage.tool_calls[0];
+      const inputParams = toolCall.args || {};
+
+      // Prepare parameters for prompt compilation
+      const promptParams = {
+        input: inputParams,
+        memory: state.memory,
+        state: state,
+        system: { currentTime: new Date().toISOString() },
+      };
+
+      // Compile the prompt with parameters
+      const compiledPrompt = compilePrompt(node.prompt, { ...promptParams });
+
+      // Build the full prompt with compiled content
+      const fullPrompt = `
+# Task instructions:
+${compiledPrompt}
+
+${Object.keys(inputParams).length > 0 ? `# Input parameters:\n${JSON.stringify(inputParams, null, 2)}\n\n` : ''}`;
+
+      const { browserTaskMode } = getConfig();
+      // Determine keepAlive based on NODE_ENV
+      const keepAlive = process.env.NODE_ENV !== 'production';
+      // Create browser session using socket
+      if (!state.cdp?.url) {
+        const session = await createBrowserSession({
+          sessionId: state.sessionId,
+          proxy: node.proxy,
+          browserTaskMode,
+        });
+        if (!session.sessionId || !session.cdpUrl) {
+          throw new Error('Failed to create browser session: missing session details');
+        }
+
+        logger.debug({
+          msg: 'Browser session created',
+          sessionId: session.sessionId,
+          hasLiveUrl: !!session.liveViewUrl,
+          keepAlive,
+          nodeEnv: process.env.NODE_ENV,
+        });
+        state.cdp = {
+          url: session.cdpUrl,
+          liveUrl: session.liveViewUrl,
+          sessionId: session.sessionId,
+        };
+      }
+
+      // Update the tool call with session info and prompt
+      const updatedToolCall = {
+        ...toolCall,
+        args: {
+          ...inputParams,
+          prompt: fullPrompt,
+        },
+      };
+
+      const toolCallingMessage = new AIMessage({
+        id: AIToolCallMessage.id,
+        content: AIToolCallMessage.content,
+        tool_calls: [updatedToolCall],
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: node.displayName,
+            sessionId: state.cdp?.sessionId,
+            cdpUrl: state.cdp?.url,
+            liveUrl: state.cdp?.liveUrl,
+            keepAlive,
+            hooks: node.hooks || [],
+            onPrem: browserTaskMode === BrowserTaskMode.ON_PREM,
+            localRun: browserTaskMode === BrowserTaskMode.LOCAL,
+            outputSchema: node.outputSchema,
+          },
+        },
+      });
+
+      state.messages.push(toolCallingMessage);
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: { taskId: state.cdp?.sessionId, liveUrl: state.cdp?.liveUrl, inputParams },
+          messageIds: [toolCallingMessage.id!],
+        }),
+      );
+      return state;
+    } catch (error: any) {
+      logger.error(`Error creating browser task node ${node.displayName}: ${error}`);
+
+      const errorMessageId = uuidv4();
+      const errorToolCallId = uuidv4();
+
+      const toolCallingMessage = new AIMessage({
+        id: errorMessageId,
+        content: '',
+        tool_calls: [
+          {
+            id: errorToolCallId,
+            name: 'browser-task',
+            args: {
+              error: error.message,
+            },
+          },
+        ],
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: node.displayName,
+            error: error.message,
+            hooks: node.hooks || [],
+            outputSchema: node.outputSchema,
+          },
+        },
+      });
+
+      state.messages.push(toolCallingMessage);
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: error,
+          messageIds: [errorMessageId],
+        }),
+      );
+      return state;
+    }
+  };
+  graph.addNode(node.name, callback);
+};

package/src/nodes/addBrowserTaskRunNode.ts

@@ -0,0 +1,190 @@
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { internalNodesSuffix, NodeType, BrowserTaskNode } from '../types/Flows.types';
+import { ToolMessage, AIMessage } from '@langchain/core/messages';
+import { Agent } from '../agent';
+import { logger } from '../utils/logger';
+import { createHistoryStep } from '../utils/history';
+import { HistoryStep } from '../types/Agent.types';
+import { v4 as uuidv4 } from 'uuid';
+import { invokeBrowserTask } from '../browserTask/executeBrowserTask';
+import { zodSchemaToParams } from '../platform/utils/tools';
+import { getConfig } from '../platform/config';
+import { compilePrompt } from './compilePrompt';
+
+type AddBrowserTaskRunNodeParams = {
+  graph: PreCompiledGraph;
+  browserTaskNode: BrowserTaskNode;
+  attachedToNodeName: string;
+  agent: Agent;
+};
+
+export const buildBrowserTaskRunNodeName = (nodeName: string) => `${nodeName}${internalNodesSuffix.BROWSER_TASK_RUN}`;
+
+export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedToNodeName, agent }: AddBrowserTaskRunNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.info(`Executing browser task run node for ${browserTaskNode.displayName}`);
+
+    const toolCallObj = state.messages[state.messages.length - 1] as any;
+    if (!toolCallObj.tool_calls) {
+      throw new Error('Tool call not found');
+    }
+    const toolCallMindedMetadata = toolCallObj.additional_kwargs?.mindedMetadata;
+    const toolCall = toolCallObj.tool_calls[0];
+    const { prompt, ...inputParams } = toolCall.args;
+    const { sessionId, cdpUrl, keepAlive, hooks, outputSchema } = toolCallMindedMetadata;
+
+    try {
+      if (inputParams.error) {
+        throw new Error(inputParams.error);
+      }
+
+      if (!sessionId || !cdpUrl || !prompt) {
+        throw new Error('Missing required parameters: sessionId, cdpUrl, or prompt');
+      }
+
+      const toolSchemas = agent.tools
+        .filter((tool) => tool.allowExecutionRequests === true)
+        .map((tool) => ({
+          name: tool.name,
+          description: tool.description,
+          inputParams: zodSchemaToParams(tool.input),
+        }));
+
+      // We compile the env variables to avoid having to pass them to the platform in the tool input
+      const promptCompiledWithEnv = compilePrompt(prompt, { env: process.env });
+
+      const { browserTaskMode } = getConfig();
+      // Invoke browser task via socket
+      const result = await invokeBrowserTask({
+        sessionId,
+        cdpUrl,
+        task: promptCompiledWithEnv,
+        keepAlive,
+        hooks,
+        browserTaskMode,
+        toolSchemas,
+        outputSchema,
+      });
+
+      logger.debug({
+        msg: 'Browser task completed',
+        sessionId,
+        hasResult: !!result.result,
+        stepCount: result.steps?.length || 0,
+        recordingCount: result.recordings?.length || 0,
+      });
+
+      // Create tool message with the result
+      const toolMessage = new ToolMessage({
+        id: toolCall.id,
+        content: JSON.stringify({
+          result: result.result || 'Failed to execute browser task',
+          steps: result.steps || [],
+          recordings: result.recordings || [],
+          inputParams: inputParams,
+        }),
+        name: 'browser-task',
+        tool_call_id: toolCall.id,
+        status: 'success',
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: browserTaskNode.displayName,
+            sessionId,
+            steps: result.steps,
+            recordings: result.recordings,
+            inputParams: inputParams,
+          },
+        },
+      });
+
+      // Update the AI message with steps using the message reducer pattern
+      const updatedAIMessage = new AIMessage({
+        id: toolCallObj.id,
+        content: toolCallObj.content,
+        tool_calls: toolCallObj.tool_calls,
+        additional_kwargs: {
+          ...toolCallObj.additional_kwargs,
+          mindedMetadata: {
+            ...toolCallObj.additional_kwargs?.mindedMetadata,
+            nodeDisplayName: browserTaskNode.displayName,
+            steps: result.steps,
+            recordings: result.recordings,
+            status: 'completed',
+          },
+          update: true, // This triggers the message reducer to update the existing message
+        },
+      });
+
+      state.messages.pop();
+      state.messages.push(updatedAIMessage, toolMessage);
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: browserTaskNode.name,
+          nodeDisplayName: browserTaskNode.displayName,
+          raw: result,
+          messageIds: [toolMessage.id!],
+        }),
+      );
+      return state;
+    } catch (error: any) {
+      logger.error({ msg: 'Error executing browser task run node', error: error.toString() });
+
+      const errorToolMessage = new ToolMessage({
+        id: uuidv4(),
+        content: JSON.stringify({
+          error: error instanceof Error ? error.message : error.toString(),
+          inputParams: inputParams,
+          message: 'Failed to execute browser task',
+        }),
+        status: 'error',
+        name: 'browser-task',
+        tool_call_id: toolCall.id,
+        additional_kwargs: {
+          mindedMetadata: {
+            nodeType: NodeType.BROWSER_TASK,
+            nodeDisplayName: browserTaskNode.displayName,
+            sessionId,
+            error: error.message,
+            inputParams: inputParams,
+          },
+        },
+      });
+
+      // Update the AI message with error status
+      const updatedAIMessage = new AIMessage({
+        id: toolCallObj.id,
+        content: toolCallObj.content,
+        tool_calls: toolCallObj.tool_calls,
+        additional_kwargs: {
+          ...toolCallObj.additional_kwargs,
+          mindedMetadata: {
+            ...toolCallObj.additional_kwargs?.mindedMetadata,
+            nodeDisplayName: browserTaskNode.displayName,
+            status: 'failed',
+            error: error.message,
+          },
+          update: true,
+        },
+      });
+
+      state.messages.pop();
+      state.messages.push(updatedAIMessage, errorToolMessage);
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.BROWSER_TASK,
+          nodeId: browserTaskNode.name,
+          nodeDisplayName: browserTaskNode.displayName,
+          raw: error,
+          messageIds: [],
+        }),
+      );
+      return state;
+    }
+  };
+
+  graph.addNode(buildBrowserTaskRunNodeName(attachedToNodeName), callback);
+  graph.addEdge(attachedToNodeName as any, buildBrowserTaskRunNodeName(attachedToNodeName) as any);
+};

package/src/nodes/addHumanInTheLoopNode.ts

@@ -0,0 +1,26 @@
+import { interrupt } from '@langchain/langgraph';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { logger } from '../utils/logger';
+import { internalNodesSuffix } from '../types/Flows.types';
+import { InterruptType } from '../interrupts/BaseInterruptSessionManager';
+
+type AddHumanInTheLoopNodeParams = {
+  graph: PreCompiledGraph;
+  attachedToNodeName: string;
+};
+
+export const buildHumanInTheLoopNodeName = (nodeName: string) => `${nodeName}${internalNodesSuffix.HUMAN_IN_THE_LOOP}`;
+
+export const addHumanInTheLoopNode = async ({ graph, attachedToNodeName }: AddHumanInTheLoopNodeParams) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.debug({ msg: `[Node] Waiting for human input`, node: attachedToNodeName });
+
+    if (state.messages[state.messages.length - 1].getType() === 'ai') {
+      interrupt({ type: InterruptType.HUMAN_IN_THE_LOOP });
+      return state;
+    }
+  };
+  graph.addNode(buildHumanInTheLoopNodeName(attachedToNodeName), callback);
+  graph.addEdge(attachedToNodeName as any, buildHumanInTheLoopNodeName(attachedToNodeName) as any);
+};

package/src/nodes/addJumpToNode.ts

@@ -0,0 +1,26 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { JumpToNode, NodeType } from '../types/Flows.types';
+import { logger } from '../utils/logger';
+import { HistoryStep } from '../types/Agent.types';
+import { createHistoryStep } from '../utils/history';
+
+export const addJumpToNode = async ({ graph, node }: { graph: PreCompiledGraph; node: JumpToNode }) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.info({ msg: `Executing jump node ${node.displayName} – jumping to ${node.targetNodeId}` });
+    // No state modifications are necessary; control flow is handled via edges.
+    state.goto = node.targetNodeId;
+    state.history.push(
+      createHistoryStep<HistoryStep>(state.history, {
+        type: NodeType.JUMP_TO_NODE,
+        nodeId: node.name,
+        nodeDisplayName: node.displayName,
+        raw: '',
+      }),
+    );
+    return state;
+  };
+
+  graph.addNode(node.name, callback);
+  graph.addEdge(node.name as any, node.targetNodeId as any);
+};

package/src/nodes/addJunctionNode.ts

@@ -0,0 +1,21 @@
+import { JunctionNode, NodeType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { RunnableLike } from '@langchain/core/runnables';
+import { createHistoryStep } from '../utils/history';
+import { HistoryStep } from '../types/Agent.types';
+
+export const addJunctionNode = ({ graph, node }: { graph: PreCompiledGraph; node: JunctionNode }) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    state.goto = null;
+    state.history.push(
+      createHistoryStep<HistoryStep>(state.history, {
+        type: NodeType.JUNCTION,
+        nodeId: node.name,
+        nodeDisplayName: node.displayName,
+        raw: '',
+      }),
+    );
+    return state;
+  };
+  graph.addNode(node.name, callback);
+};