@minded-ai/mindedjs 2.0.43 → 2.0.45-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "2.0.43",
+  "version": "2.0.45-beta.1",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -72,4 +72,4 @@
   "peerDependencies": {
     "playwright": "^1.55.0"
   }
-}
+}

package/src/agent.ts

@@ -490,6 +490,21 @@ export class Agent {
     } catch (err) {
       const { baseUrl } = getConfig();
       logger.error({ msg: '[Trigger] Agent initialization failed', err, sessionId, baseUrl });
+
+      // Notify backend about the initialization error
+      if (mindedConnection.isConnected()) {
+        try {
+          await mindedConnection.emit(mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR, {
+            type: mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR,
+            sessionId,
+            error: err instanceof Error ? err.message : String(err),
+            stack: err instanceof Error ? err.stack : undefined,
+          });
+        } catch (emitErr) {
+          logger.error({ msg: '[Agent] Failed to emit initialization error to backend', emitErr });
+        }
+      }
+
       throw err;
     }
 
@@ -644,6 +659,20 @@ export class Agent {
       // Release the session lock on error
       await this.interruptSessionManager.release(sessionId);
 
+      // Notify backend about the error
+      if (mindedConnection.isConnected()) {
+        try {
+          await mindedConnection.emit(mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR, {
+            type: mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR,
+            sessionId,
+            error: err instanceof Error ? err.message : String(err),
+            stack: err instanceof Error ? err.stack : undefined,
+          });
+        } catch (emitErr) {
+          logger.error({ msg: '[Agent] Failed to emit error to backend', emitErr });
+        }
+      }
+
       const state = await this.compiledGraph.getState(this.getLangraphConfig(sessionId));
       const results = await this.emit(AgentEvents.ERROR, {
         error: err instanceof Error ? err : new Error(JSON.stringify(err)),
@@ -854,7 +883,7 @@ export class Agent {
     } else if (runLocally) {
       throw new Error(
         'No LLM configuration found in minded.json. When running locally, you must provide LLM configuration in minded.json. ' +
-        'Add an "llm" section to your minded.json file with provider and model settings.',
+          'Add an "llm" section to your minded.json file with provider and model settings.',
       );
     } else {
       throw new Error('No LLM configuration found. Please configure LLM in Organization Settings or add it to minded.json');
@@ -1224,7 +1253,7 @@ export class Agent {
   /**
    * Handle tool execution mode when agent is run with 'runTool' argument
    * This allows running tools directly via: npm run tool <toolName> [params...] [cookies=false]
-   * 
+   *
    * @private
    */
   private async handleToolExecutionMode(): Promise<void> {
@@ -1252,12 +1281,15 @@ export class Agent {
         flags,
       });
 
-      const result = await this.executeTool({
-        toolName,
-        toolParams: params,
-        sessionId: uuidv4(),
-        requestId: uuidv4()
-      }, 'direct');
+      const result = await this.executeTool(
+        {
+          toolName,
+          toolParams: params,
+          sessionId: uuidv4(),
+          requestId: uuidv4(),
+        },
+        'direct',
+      );
 
       if (result.error) {
         logger.error({

package/src/browserTask/executeBrowserTask.ts

@@ -43,26 +43,26 @@ const loadCookiesForSession = async (sessionId: string): Promise<void> => {
   // Check if cookies should be disabled
   const noCookies = process.env.MINDED_NO_COOKIES === 'true';
   if (noCookies) {
-    logger.info({ message: 'Cookies disabled via cookies=false flag', sessionId });
+    logger.info({ message: '[Browser] Cookies disabled via cookies=false flag', sessionId });
     return;
   }
 
   const stored = await cookieStore.load();
   if (!stored || stored.length === 0) {
-    logger.info({ message: 'No stored cookies to load', sessionId });
+    logger.info({ message: '[Browser] No stored cookies to load', sessionId });
     return;
   }
   const cdp = cdpClients.get(sessionId);
   if (!cdp) {
-    logger.warn({ message: 'No CDP client found for session', sessionId });
+    logger.warn({ message: '[Browser] No CDP client found for session', sessionId });
     return;
   }
 
   try {
     await cdp.Network.setCookies({ cookies: stored });
-    logger.info({ message: 'Loaded cookies into browser session', sessionId, count: stored.length });
+    logger.info({ message: '[Browser] Loaded cookies into browser session', sessionId, count: stored.length });
   } catch (error) {
-    logger.warn({ message: 'Failed to set cookies via CDP', sessionId, error });
+    logger.warn({ message: '[Browser] Failed to set cookies via CDP', sessionId, error });
   }
 };
 
@@ -77,19 +77,19 @@ const saveCookiesForSession = async (sessionId: string): Promise<void> => {
   try {
     const cdp = cdpClients.get(sessionId);
     if (!cdp) {
-      logger.warn({ message: 'No CDP client found for session', sessionId });
+      logger.warn({ message: '[Browser] No CDP client found for session', sessionId });
       return;
     }
 
     const result = await cdp.Network.getAllCookies();
     if (result) {
       await cookieStore.save(result.cookies);
-      logger.info({ message: 'Saved cookies from browser session', sessionId, count: result.cookies.length });
+      logger.info({ message: '[Browser] Saved cookies from browser session', sessionId, count: result.cookies.length });
     } else {
-      logger.warn({ message: 'No cookies found in browser session', sessionId });
+      logger.warn({ message: '[Browser] No cookies found in browser session', sessionId });
     }
   } catch (error) {
-    logger.warn({ message: 'Failed to get/save cookies from session', sessionId, error: JSON.stringify(error) });
+    logger.warn({ message: '[Browser] Failed to get/save cookies from session', sessionId, error: JSON.stringify(error) });
   }
 };
 
@@ -100,7 +100,7 @@ const cleanupCdpClient = async (sessionId: string): Promise<void> => {
     try {
       await cdp.close();
     } catch (error) {
-      logger.warn({ message: 'Failed to close CDP client', sessionId, error });
+      logger.warn({ message: '[Browser] Failed to close CDP client', sessionId, error });
     }
     cdpClients.delete(sessionId);
   }
@@ -116,9 +116,8 @@ export const createBrowserSession = async ({
   proxy?: string;
   browserTaskMode: BrowserTaskMode;
 }): Promise<CreateBrowserSessionResponse> => {
-  logger.debug({ msg: 'Creating browser session via socket', proxy });
-
-  if (browserTaskMode === BrowserTaskMode.LOCAL) {
+  if (browserTaskMode === BrowserTaskMode.LOCAL && process.env.MINDED_CLOUD !== 'true') {
+    logger.debug({ msg: '[Browser] Creating local browser session', proxy });
     const { cdpUrl, instanceId } = await getOrStartLocalCDP({
       headless: false,
     });
@@ -128,19 +127,19 @@ export const createBrowserSession = async ({
 
     const port = extractPortFromCdpUrl(cdpUrl);
     if (!port) {
-      logger.warn({ message: 'Failed to extract port from CDP URL', sessionId, cdpUrl });
+      logger.warn({ message: '[Browser] Failed to extract port from CDP URL', sessionId, cdpUrl });
     } else {
       const cdp = await CDP({ host: 'localhost', port });
       await cdp.Network.enable();
       cdpClients.set(sessionId, cdp);
-      logger.info({ message: 'Created CDP client for session', sessionId, port });
+      logger.info({ message: '[Browser] Created CDP client for session', sessionId, port });
     }
 
     // Load cookies for this session (best-effort)
     try {
       await loadCookiesForSession(sessionId);
     } catch (error) {
-      logger.warn({ message: 'Failed to load cookies for session', sessionId, error });
+      logger.warn({ message: '[Browser] Failed to load cookies for session', sessionId, error });
     }
 
     return {
@@ -149,6 +148,7 @@ export const createBrowserSession = async ({
     };
   }
 
+  logger.debug({ msg: '[Browser] Creating browser session via socket', proxy });
   const response = await mindedConnection.awaitEmit<CreateBrowserSessionRequest, CreateBrowserSessionResponse>(
     mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION,
     {
@@ -160,12 +160,12 @@ export const createBrowserSession = async ({
   );
 
   if (response.error) {
-    logger.error({ msg: 'Failed to create browser session', error: response.error });
+    logger.error({ msg: '[Browser] Failed to create browser session', error: response.error });
     throw new Error(response.error);
   }
 
   logger.debug({
-    msg: 'Browser session created successfully',
+    msg: '[Browser] Browser session created successfully',
     sessionId: response.sessionId,
     hasLiveUrl: !!response.liveViewUrl,
   });
@@ -182,7 +182,7 @@ export const destroyBrowserSession = async ({
   onPrem?: boolean;
   localRun?: boolean;
 }): Promise<DestroyBrowserSessionResponse> => {
-  logger.debug({ msg: 'Destroying browser session via socket', sessionId, onPrem });
+  logger.debug({ msg: '[Browser] Destroying browser session via socket', sessionId, onPrem });
 
   if (localRun || process.env.BROWSER_TASK_MODE === BrowserTaskMode.LOCAL) {
     // Kill specific instance if we have the mapping
@@ -192,7 +192,7 @@ export const destroyBrowserSession = async ({
       localSessionInstances.delete(sessionId);
     } else {
       // Fallback to killing all if no specific instance found
-      logger.warn({ msg: 'No instance ID found for session, killing all instances', sessionId });
+      logger.warn({ msg: '[Browser] No instance ID found for session, killing all instances', sessionId });
       await kill();
     }
     return {
@@ -211,11 +211,11 @@ export const destroyBrowserSession = async ({
   );
 
   if (response.error) {
-    logger.error({ msg: 'Failed to destroy browser session', error: response.error, sessionId });
+    logger.error({ msg: '[Browser] Failed to destroy browser session', error: response.error, sessionId });
     throw new Error(response.error);
   }
 
-  logger.debug({ msg: 'Browser session destroyed successfully', sessionId });
+  logger.debug({ msg: '[Browser] Browser session destroyed successfully', sessionId });
   return response;
 };
 
@@ -237,7 +237,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
 
         const args = ['run', pythonScriptPath];
         logger.info({
-          message: 'Spawning Python process',
+          message: '[Browser] Spawning Python process',
           args,
         });
 
@@ -286,7 +286,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
         const interval = setInterval(() => {
           // Check if chromium process stopped running
           if (!isLocalBrowserRunning()) {
-            logger.error({ message: 'Local browser process stopped running, killing browser task' });
+            logger.error({ message: '[Browser] Local browser process stopped running, killing browser task' });
             child.kill();
             clearInterval(interval);
           }
@@ -305,18 +305,18 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
         clearInterval(interval);
 
         if (exitCode !== 0) {
-          logger.error({ message: 'Operator failed', exitCode, stderr: stderrBuffer });
+          logger.error({ message: '[Browser] Operator failed', exitCode, stderr: stderrBuffer });
           throw new Error(`Local browser task failed with exit code ${exitCode}`);
         }
 
-        logger.info({ message: 'Operator finished' });
+        logger.info({ message: '[Browser] Operator finished' });
 
         let result = stdoutBuffer.split('___RESULT___')[1]?.trim() || stdoutBuffer;
         if (outputSchema?.length) {
           try {
             result = JSON.parse(result);
           } catch (error) {
-            logger.debug({ message: 'Failed to parse result', error });
+            logger.debug({ message: '[Browser] Failed to parse result', error });
           }
         }
 
@@ -326,9 +326,9 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
           try {
             const files = readdirSync(folderPath);
             downloadedFiles = files.map((file) => path.join(folderPath, file));
-            logger.debug({ message: 'Found downloaded files', count: downloadedFiles.length, files: downloadedFiles });
+            logger.debug({ message: '[Browser] Found downloaded files', count: downloadedFiles.length, files: downloadedFiles });
           } catch (error) {
-            logger.error({ message: 'Failed to read downloads folder', error });
+            logger.error({ message: '[Browser] Failed to read downloads folder', error });
           }
         }
 
@@ -336,7 +336,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
         try {
           await saveCookiesForSession(sessionId);
         } catch (error) {
-          logger.warn({ message: 'Failed to save cookies after task completion', sessionId, error });
+          logger.warn({ message: '[Browser] Failed to save cookies after task completion', sessionId, error });
         }
 
         return {
@@ -354,7 +354,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
     }
 
     logger.debug({
-      msg: 'Invoking browser task via socket',
+      msg: '[Browser] Invoking browser task via socket',
       sessionId,
       taskLength: task.length,
       keepAlive,
@@ -380,12 +380,12 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
     );
 
     if (response.error) {
-      logger.error({ msg: 'Failed to invoke browser task', error: response.error });
+      logger.error({ msg: '[Browser] Failed to invoke browser task', error: response.error });
       throw new Error(response.error);
     }
 
     logger.debug({
-      msg: 'Browser task completed successfully',
+      msg: '[Browser] Browser task completed successfully',
       sessionId,
       hasResult: !!response.result,
       stepCount: response.steps?.length || 0,
@@ -394,7 +394,7 @@ export const invokeBrowserTask = async (options: InvokeBrowserTaskOptions): Prom
 
     return response;
   } catch (err) {
-    logger.error({ message: 'Error invoking browser task', err });
+    logger.error({ message: '[Browser] Error invoking browser task', err });
     throw err;
   }
 };

package/src/nodes/addPromptNode.ts

@@ -95,7 +95,7 @@ export const addPromptNode = async ({ graph, node, llm, tools, emit, agent }: Ad
                 });
 
                 return toolResult;
-              }
+              },
             );
 
             return result;

package/src/nodes/addThinkingNode.ts

@@ -13,6 +13,8 @@ import { createHistoryStep } from '../utils/history';
 import { compilePrompt } from './compilePrompt';
 import { AnalyticsEventName } from '../types/Analytics.types';
 import { trackAnalyticsEvent } from '../internalTools/analytics';
+import { convertArraySchemaToZod } from '../utils/schemaConverter';
+import { v4 as uuidv4 } from 'uuid';
 
 type AddThinkingNodeParams = {
   graph: PreCompiledGraph;
@@ -51,8 +53,17 @@ export const addThinkingNode = async ({ graph, node, llm, agent }: AddThinkingNo
     const messagesForModel = [...state.messages];
     messagesForModel.push(systemMessage);
 
+    // Check if we need structured output
+    const useStructuredOutput = node.outputSchema && node.outputSchema.length > 0;
+    let llmWithStructuredOutput = llmToUse;
+
+    if (useStructuredOutput) {
+      const zodSchema = convertArraySchemaToZod(node.outputSchema!);
+      llmWithStructuredOutput = llmToUse.withStructuredOutput(zodSchema, { name: 'extract' });
+    }
+
     const startTime = Date.now();
-    const result: AIMessage = await llmToUse.invoke(messagesForModel);
+    const result = await llmWithStructuredOutput.invoke(messagesForModel);
     const endTime = Date.now();
 
     logger.debug({
@@ -65,17 +76,35 @@ export const addThinkingNode = async ({ graph, node, llm, agent }: AddThinkingNo
     await agent.interruptSessionManager.checkQueueAndInterrupt(state.sessionId, state);
 
     // Inject metadata to identify this as a thinking node output
-    const messageWithMetadata = new AIMessage({
-      content: result.content,
-      additional_kwargs: {
-        ...result.additional_kwargs,
-        mindedMetadata: {
-          nodeType: NodeType.THINKING,
-          nodeDisplayName: node.displayName,
-          prompt: node.prompt,
-        },
+    const additionalKwargs: Record<string, any> = {
+      mindedMetadata: {
+        nodeType: NodeType.THINKING,
+        nodeDisplayName: node.displayName,
+        prompt: node.prompt,
       },
-      id: result.id,
+    };
+
+    let messageId: string;
+    let messageContent: string;
+
+    // When structured output is used, result is the raw extracted data (not an AIMessage)
+    // When not used, result is an AIMessage with content and id
+    if (useStructuredOutput) {
+      additionalKwargs.structuredOutput = result;
+      messageId = uuidv4();
+      messageContent = JSON.stringify(result);
+    } else {
+      // result is an AIMessage, preserve its additional_kwargs and id
+      const aiResult = result as AIMessage;
+      Object.assign(additionalKwargs, aiResult.additional_kwargs);
+      messageId = aiResult.id!;
+      messageContent = aiResult.content as string;
+    }
+
+    const messageWithMetadata = new AIMessage({
+      content: messageContent,
+      additional_kwargs: additionalKwargs,
+      id: messageId,
     });
 
     // Add to state messages
@@ -100,4 +129,3 @@ export const addThinkingNode = async ({ graph, node, llm, agent }: AddThinkingNo
   };
   graph.addNode(node.name, callback);
 };
-

package/src/nodes/compilePrompt.ts

@@ -1,5 +1,69 @@
 import * as ejs from 'ejs';
 import { logger } from '../utils/logger';
+import { HistoryStep } from '../types/Agent.types';
+
+/**
+ * Extract node outputs from state.history and state.messages
+ * Returns a map of nodeId/nodeDisplayName -> output object
+ */
+function extractNodeOutputs(state: any): Record<string, any> {
+  if (!state?.history || !state?.messages) {
+    return {};
+  }
+
+  const outputs: Record<string, any> = {};
+
+  // Build lookup maps by message type
+  const messagesByType = {
+    tool: new Map(
+      state.messages
+        .filter((msg: any) => msg.constructor?.name === 'ToolMessage' && msg.tool_call_id)
+        .map((msg: any) => [msg.tool_call_id, msg]),
+    ),
+    ai: new Map(state.messages.filter((msg: any) => msg.constructor?.name === 'AIMessage' && msg.id).map((msg: any) => [msg.id, msg])),
+  };
+
+  const setOutput = (item: HistoryStep, value: any) => {
+    if (item.nodeId) outputs[item.nodeId] = value;
+    if (item.nodeDisplayName) outputs[item.nodeDisplayName] = value;
+  };
+
+  const extractors: Record<string, (item: HistoryStep) => void> = {
+    tool: (item) => {
+      const toolCallId = item.messageIds?.[1] ?? item.messageIds?.[0];
+      const toolMessage: any = messagesByType.tool.get(toolCallId);
+      if (!toolMessage) return;
+
+      try {
+        const content = typeof toolMessage.content === 'string' ? JSON.parse(toolMessage.content) : toolMessage.content;
+        setOutput(item, content?.result ?? content);
+      } catch (err) {
+        logger.debug({ message: 'Failed to parse tool message content', toolCallId, err });
+      }
+    },
+
+    thinking: (item) => {
+      const messageId = item.messageIds?.[0];
+      if (!messageId) return;
+
+      const aiMessage: any = messagesByType.ai.get(messageId);
+      if (!aiMessage) return;
+
+      const structuredOutput = aiMessage.additional_kwargs?.structuredOutput;
+
+      if (structuredOutput) {
+        setOutput(item, structuredOutput);
+      }
+    },
+  };
+
+  for (const item of state.history) {
+    if (!item.messageIds?.length) continue;
+    extractors[item.type]?.(item);
+  }
+
+  return outputs;
+}
 
 /**
  * Compile prompt with parameters using EJS and placeholder replacement
@@ -9,6 +73,17 @@ export function compilePrompt(prompt: string, params: Record<string, any> = {}):
     // Define system parameters
     params.system = { currentTime: new Date().toISOString() };
 
+    // Extract node outputs from state if available
+    // This enables placeholder replacement like {tools.NodeName.field} or {NodeName.field}
+    // by making previous node outputs available in the params object.
+    // Example: If ExtractUser node outputs { userName: "Bob", userAge: 35 },
+    // the prompt "Hello {tools.ExtractUser.userName}" becomes "Hello Bob"
+    if (params.state) {
+      const nodeOutputs = extractNodeOutputs(params.state);
+      // Merge node outputs into params so they're accessible during placeholder replacement
+      params = { ...params, ...nodeOutputs };
+    }
+
     // First, render with EJS
     let compiledPrompt = ejs.render(prompt, params);
 
@@ -17,24 +92,37 @@ export function compilePrompt(prompt: string, params: Record<string, any> = {}):
 
     return compiledPrompt;
   } catch (err) {
-    logger.error({ message: 'Error compiling prompt', err });
+    logger.error({ message: 'Error compiling prompt', err, prompt });
     return prompt; // Return uncompiled if there's an error
   }
 }
 
 /**
  * Replace placeholders in {key} format
+ * Supports both:
+ * - {NodeName.field} - direct node reference
+ * - {tools.NodeName.field} - node reference with 'tools' prefix (alias)
  */
 function replacePlaceholders(text: string, params: Record<string, any>): string {
   return text.replace(/\{([^}]+)\}/g, (match, key) => {
-    const keys = key.split('.');
+    let keys = key.split('.');
+
+    // Handle 'tools.' prefix as an alias for node outputs
+    // {tools.NodeName.field} -> {NodeName.field}
+    if (keys[0] === 'tools' && keys.length >= 3) {
+      keys = keys.slice(1); // Remove 'tools' prefix
+    }
+
     let value: any = params;
 
-    for (const k of keys) {
+    for (let i = 0; i < keys.length; i++) {
+      const k = keys[i];
       if (value && typeof value === 'object' && k in value) {
         value = value[k];
       } else {
-        logger.warn({ message: `Placeholder {${key}} in prompt not found in params. It will remain as placeholder.` });
+        logger.warn({
+          message: `Placeholder {${key}} in prompt not found in params. It will remain as placeholder.`,
+        });
         return match; // Return original if key not found
       }
     }

package/src/platform/mindedConnectionTypes.ts

@@ -72,6 +72,7 @@ export enum mindedConnectionSocketMessageType {
   RPA_LOGS_UPLOAD = 'rpa-logs-upload',
   // RPA Error Upload
   RPA_ERROR_UPLOAD = 'rpa-error-upload',
+  ON_AGENT_RUN_ERROR = 'on-agent-run-error',
 }
 
 export type mindedConnectionSocketMessageTypeMap = {
@@ -125,6 +126,7 @@ export type mindedConnectionSocketMessageTypeMap = {
   [mindedConnectionSocketMessageType.RPA_SCREENSHOT_UPLOAD]: RPAScreenshotUploadRequest;
   [mindedConnectionSocketMessageType.RPA_LOGS_UPLOAD]: RPALogsUploadRequest;
   [mindedConnectionSocketMessageType.RPA_ERROR_UPLOAD]: RPAErrorUploadRequest;
+  [mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR]: OnAgentRunErrorRequest;
 };
 
 export interface BasemindedConnectionSocketMessage {
@@ -633,3 +635,14 @@ export interface RPAErrorUploadResponse extends BaseSdkConnectionSocketMessageRe
   screenshotS3Url?: string;
   htmlS3Url?: string;
 }
+
+export interface OnAgentRunErrorRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.ON_AGENT_RUN_ERROR;
+  sessionId: string;
+  error: string;
+  stack?: string;
+}
+
+export interface OnAgentRunErrorResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  success?: boolean;
+}