@minded-ai/mindedjs 1.0.116 → 1.0.118-beta-1

package/src/nodes/addBrowserTaskRunNode.ts

@@ -7,7 +7,7 @@ import { logger } from '../utils/logger';
 import { createHistoryStep } from '../utils/history';
 import { HistoryStep } from '../types/Agent.types';
 import { v4 as uuidv4 } from 'uuid';
-import { waitForCompletion } from '../browserTask/executeBrowserTask';
+import { invokeBrowserTask } from '../browserTask/executeBrowserTask';
 
 type AddBrowserTaskRunNodeParams = {
   graph: PreCompiledGraph;
@@ -26,23 +26,33 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
     if (!toolCallObj.tool_calls) {
       throw new Error('Tool call not found');
     }
+    const toolCallMindedMetadata = toolCallObj.additional_kwargs?.mindedMetadata;
     const toolCall = toolCallObj.tool_calls[0];
-    const { taskId, ...inputParams } = toolCall.args;
+    const { prompt, ...inputParams } = toolCall.args;
+    const { sessionId, cdpUrl, keepAlive } = toolCallMindedMetadata;
     try {
-      if (!taskId) {
-        throw new Error('Task ID not found in tool call arguments');
+      if (!sessionId || !cdpUrl || !prompt) {
+        throw new Error('Missing required parameters: sessionId, cdpUrl, or prompt');
       }
 
-      // Wait for task completion
-      const completedTask = await waitForCompletion(taskId);
-      logger.debug({ msg: 'Browser task completed', taskId, output: completedTask.output });
+      // Invoke browser task via socket
+      const result = await invokeBrowserTask(sessionId, cdpUrl, prompt, keepAlive);
+
+      logger.debug({
+        msg: 'Browser task completed',
+        sessionId,
+        hasResult: !!result.result,
+        stepCount: result.steps?.length || 0,
+        recordingCount: result.recordings?.length || 0,
+      });
 
       // Create tool message with the result
       const toolMessage = new ToolMessage({
         id: uuidv4(),
         content: JSON.stringify({
-          result: completedTask.output || 'Task completed successfully',
-          steps: completedTask.steps || [],
+          result: result.result || 'Task completed successfully',
+          steps: result.steps || [],
+          recordings: result.recordings || [],
           inputParams: inputParams,
         }),
         name: 'browser-task',
@@ -52,8 +62,9 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: browserTaskNode.displayName,
-            taskId,
-            steps: completedTask.steps,
+            sessionId,
+            steps: result.steps,
+            recordings: result.recordings,
             inputParams: inputParams,
           },
         },
@@ -69,7 +80,8 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             ...toolCallObj.additional_kwargs?.mindedMetadata,
             nodeDisplayName: browserTaskNode.displayName,
-            steps: completedTask.steps,
+            steps: result.steps,
+            recordings: result.recordings,
             status: 'completed',
           },
           update: true, // This triggers the message reducer to update the existing message
@@ -82,7 +94,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           type: NodeType.BROWSER_TASK,
           nodeId: browserTaskNode.name,
           nodeDisplayName: browserTaskNode.displayName,
-          raw: completedTask,
+          raw: result,
           messageIds: [],
         }),
       };
@@ -102,7 +114,7 @@ export const addBrowserTaskRunNode = async ({ graph, browserTaskNode, attachedTo
           mindedMetadata: {
             nodeType: NodeType.BROWSER_TASK,
             nodeDisplayName: browserTaskNode.displayName,
-            taskId,
+            sessionId,
             error: error.message,
             inputParams: inputParams,
           },

package/src/platform/mindedConnection.ts

@@ -1,12 +1,10 @@
 import { io, Socket } from 'socket.io-client';
-import {
-  mindedConnectionSocketMessageType,
-  mindedConnectionSocketMessageTypeMap,
-} from './mindedConnectionTypes';
+import { mindedConnectionSocketMessageType, mindedConnectionSocketMessageTypeMap } from './mindedConnectionTypes';
 import { stringify } from 'flatted';
 import { getConfig } from './config';
 import { logger } from '../utils/logger';
 import { wait } from '../utils/wait';
+import * as packageJson from '../../package.json';
 
 // Module-level singleton state
 let socket: Socket | null = null;
@@ -28,20 +26,13 @@ export const on = <E extends keyof mindedConnectionSocketMessageTypeMap>(
   listeners[event].push(callback);
 };
 
-export const emit = <E extends keyof mindedConnectionSocketMessageTypeMap>(
-  event: E,
-  message: mindedConnectionSocketMessageTypeMap[E]
-) => {
+export const emit = <E extends keyof mindedConnectionSocketMessageTypeMap>(event: E, message: mindedConnectionSocketMessageTypeMap[E]) => {
   if (socket) {
     socket.emit(event, message);
   }
 };
 
-export const awaitEmit = async <T, R>(
-  event: mindedConnectionSocketMessageType,
-  message: T,
-  timeoutMs: number = 5000
-): Promise<R> => {
+export const awaitEmit = async <T, R>(event: mindedConnectionSocketMessageType, message: T, timeoutMs: number = 5000): Promise<R> => {
   if (!socket) {
     throw new Error('Socket is not connected');
   }
@@ -81,6 +72,15 @@ const waitForConnection = async (): Promise<void> => {
   }
 };
 
+const getSdkVersion = (): string => {
+  try {
+    return packageJson.version;
+  } catch {
+    logger.warn('Could not determine SDK version');
+    return 'unknown';
+  }
+};
+
 const connect = async (token: string): Promise<void> => {
   const { isDeployed, baseUrl } = getConfig();
   return new Promise<void>((resolve, reject) => {
@@ -89,6 +89,7 @@ const connect = async (token: string): Promise<void> => {
       query: {
         isDeployedAgent: isDeployed,
         token,
+        sdkVersion: getSdkVersion(),
       },
     });
 

package/src/platform/mindedConnectionTypes.ts

@@ -39,7 +39,9 @@ export enum mindedConnectionSocketMessageType {
   TIMER_TRIGGER = 'timer-trigger',
   RETELL_CALL = 'retell-call',
   RETELL_GET_CALL = 'retell-get-call',
-
+  SDK_VERSION_MISMATCH = 'sdk-version-mismatch', // Browser Task
+  CREATE_BROWSER_SESSION = 'create-browser-session',
+  INVOKE_BROWSER_TASK = 'invoke-browser-task',
 }
 
 export type mindedConnectionSocketMessageTypeMap = {
@@ -67,7 +69,10 @@ export type mindedConnectionSocketMessageTypeMap = {
   [mindedConnectionSocketMessageType.RESTORE_CHECKPOINT]: RestoreCheckpointRequest;
   [mindedConnectionSocketMessageType.RETELL_CALL]: RetellCall;
   [mindedConnectionSocketMessageType.RETELL_GET_CALL]: RetellGetCall;
+  [mindedConnectionSocketMessageType.SDK_VERSION_MISMATCH]: SdkVersionMismatchMessage;
   [mindedConnectionSocketMessageType.UPDATE_STATE]: UpdateStateRequest;
+  [mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION]: CreateBrowserSessionRequest;
+  [mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK]: InvokeBrowserTaskRequest;
 };
 
 export interface BasemindedConnectionSocketMessage {
@@ -234,7 +239,38 @@ export interface RetellGetCallResponse extends BaseSdkConnectionSocketMessageRes
   error?: string;
 }
 
+export interface SdkVersionMismatchMessage extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.SDK_VERSION_MISMATCH;
+  currentVersion: string;
+  requiredVersion: string;
+}
 export interface UpdateStateRequest extends BasemindedConnectionSocketMessage {
   sessionId: string;
   state: Partial<typeof stateAnnotation.State>;
-}
+}
+
+// Browser Task Messages
+export interface CreateBrowserSessionRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.CREATE_BROWSER_SESSION;
+  proxy?: string; // 2-digit country code like 'IL'
+}
+
+export interface CreateBrowserSessionResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  sessionId?: string;
+  cdpUrl?: string;
+  liveViewUrl?: string;
+}
+
+export interface InvokeBrowserTaskRequest extends BasemindedConnectionSocketMessage {
+  type: mindedConnectionSocketMessageType.INVOKE_BROWSER_TASK;
+  cdpUrl: string;
+  task: string;
+  sessionId: string;
+  keepAlive?: boolean;
+}
+
+export interface InvokeBrowserTaskResponse extends BaseSdkConnectionSocketMessageResponseCallbackAck {
+  result?: any;
+  steps?: any[];
+  recordings?: any[];
+}

package/src/toolsLibrary/classifier.ts

@@ -0,0 +1,273 @@
+import { z } from 'zod';
+import { Tool } from '../types/Tools.types';
+import { logger } from '../utils/logger';
+import { JsonOutputParser } from '@langchain/core/output_parsers';
+import { SystemMessage } from '@langchain/core/messages';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+// Type definitions for classifier configuration
+export interface ClassDefinition {
+  name: string;
+  description: string;
+}
+
+export interface ClassifierConfig {
+  classes: ClassDefinition[];
+  systemPrompt?: string;
+  outputFormat?: 'json' | 'text';
+  includeReason?: boolean;
+  defaultClass?: string;
+  defaultReason?: string;
+}
+
+export interface ClassificationResult {
+  class: string;
+  reason?: string;
+  confidence?: number;
+  [key: string]: any; // Allow additional fields
+}
+
+// Default configuration
+const DEFAULT_CONFIG: Partial<ClassifierConfig> = {
+  outputFormat: 'json',
+  includeReason: true,
+  defaultClass: 'unknown',
+  defaultReason: 'Unable to determine classification',
+};
+
+/**
+ * Generic classifier utility that can be used standalone
+ * @param content The content to classify
+ * @param config The classifier configuration
+ * @param llm The language model to use for classification
+ * @returns The classification result
+ */
+export async function classify(content: string, config: ClassifierConfig, llm: BaseLanguageModel): Promise<ClassificationResult> {
+  const mergedConfig = { ...DEFAULT_CONFIG, ...config };
+
+  try {
+    // Build the classification prompt
+    const classesDescription = mergedConfig.classes.map((c) => `${c.name}: ${c.description}`).join('\n');
+
+    const basePrompt =
+      mergedConfig.systemPrompt || 'You are a classifier. Your task is to classify the given content into one of the following categories:';
+
+    let prompt = `${basePrompt}\n\n${classesDescription}\n\n`;
+
+    if (mergedConfig.outputFormat === 'json') {
+      prompt += `You should output the result in the following JSON format: {
+        "class": "<selected class name>",
+        ${mergedConfig.includeReason ? '"reason": "<explanation for the classification>",' : ''}
+        "confidence": <confidence score between 0 and 1>
+      }.\nReturn JSON and nothing more.`;
+    } else {
+      prompt += 'Return only the class name.';
+    }
+
+    prompt += `\n\nContent to classify:\n${content}`;
+
+    // Make the classification request
+    if (mergedConfig.outputFormat === 'json') {
+      const parser = new JsonOutputParser();
+      const result = await llm.pipe(parser).invoke([new SystemMessage(prompt)]);
+      return result as ClassificationResult;
+    } else {
+      const result = await llm.invoke([new SystemMessage(prompt)]);
+      const classText = typeof result.content === 'string' ? result.content.trim() : '';
+      return { class: classText };
+    }
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    logger.error({ message: 'Classification failed, using default', error: errorMessage });
+
+    // Return default classification on error
+    return {
+      class: mergedConfig.defaultClass || 'unknown',
+      reason: mergedConfig.defaultReason || errorMessage,
+      confidence: 0,
+    };
+  }
+}
+
+/**
+ * Create a classifier from a simple class list
+ * @param classes Array of class names or [name, description] tuples
+ * @param options Additional configuration options
+ * @returns A configured classify function
+ */
+export function createClassifier(classes: string[] | [string, string][], options?: Partial<ClassifierConfig>) {
+  const classDefinitions: ClassDefinition[] = classes.map((c) => {
+    if (typeof c === 'string') {
+      return { name: c, description: '' };
+    }
+    return { name: c[0], description: c[1] };
+  });
+
+  const config: ClassifierConfig = {
+    ...options,
+    classes: classDefinitions,
+  };
+
+  return (content: string, llm: BaseLanguageModel) => classify(content, config, llm);
+}
+
+// Schema for the classifier tool
+export const schema = z.object({
+  content: z.string().describe('The content to classify'),
+  classes: z
+    .array(
+      z.union([
+        z.string(),
+        z.tuple([z.string(), z.string()]),
+        z.object({
+          name: z.string(),
+          description: z.string(),
+        }),
+      ]),
+    )
+    .optional()
+    .describe('Classes to classify into. Can be strings, [name, description] tuples, or objects with name and description'),
+  systemPrompt: z.string().optional().describe('Custom system prompt for classification'),
+  includeReason: z.boolean().optional().default(true).describe('Whether to include reasoning in the response'),
+  outputFormat: z.enum(['json', 'text']).optional().default('json').describe('Output format for the classification'),
+  defaultClass: z.string().optional().describe('Default class to use if classification fails'),
+  defaultReason: z.string().optional().describe('Default reason to use if classification fails'),
+});
+
+const classifierTool: Tool<typeof schema, any> = {
+  name: 'minded-classifier',
+  description:
+    'Classify content into predefined categories using AI. Supports custom classes, system prompts, and various output formats. Can be configured with default fallback values.',
+  input: schema,
+  isGlobal: false,
+  execute: async ({ input, state, agent }) => {
+    const { content, classes, systemPrompt, includeReason, outputFormat, defaultClass, defaultReason } = input;
+
+    logger.info({
+      message: 'Classifying content',
+      sessionId: state.sessionId,
+      contentLength: content.length,
+      classCount: classes?.length,
+    });
+
+    // Convert input classes to ClassDefinition format
+    let classDefinitions: ClassDefinition[] = [];
+    if (classes) {
+      classDefinitions = classes.map((c) => {
+        if (typeof c === 'string') {
+          return { name: c, description: '' };
+        } else if (Array.isArray(c)) {
+          return { name: c[0], description: c[1] };
+        } else {
+          return c as ClassDefinition;
+        }
+      });
+    }
+
+    // If no classes provided, try to get from state memory
+    if (classDefinitions.length === 0 && state.memory?.classifierConfig?.classes) {
+      classDefinitions = state.memory.classifierConfig.classes;
+    }
+
+    if (classDefinitions.length === 0) {
+      throw new Error('No classes provided for classification');
+    }
+
+    const config: ClassifierConfig = {
+      classes: classDefinitions,
+      systemPrompt,
+      includeReason,
+      outputFormat,
+      defaultClass,
+      defaultReason,
+    };
+
+    try {
+      const result = await classify(content, config, agent.llm);
+
+      logger.info({
+        message: 'Classification completed',
+        sessionId: state.sessionId,
+        class: result.class,
+        confidence: result.confidence,
+      });
+
+      return {
+        result,
+        state: {
+          memory: {
+            lastClassification: {
+              content: content.substring(0, 100) + (content.length > 100 ? '...' : ''),
+              result,
+              timestamp: new Date().toISOString(),
+            },
+          },
+        },
+      };
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : String(error);
+      logger.error({
+        message: 'Classification failed',
+        sessionId: state.sessionId,
+        error: errorMessage,
+      });
+
+      throw error;
+    }
+  },
+};
+
+export default classifierTool;
+
+// Export utility for multi-label classification
+export async function multiClassify(
+  content: string,
+  config: ClassifierConfig & { maxClasses?: number },
+  llm: BaseLanguageModel,
+): Promise<ClassificationResult[]> {
+  const mergedConfig = { ...DEFAULT_CONFIG, ...config };
+  const maxClasses = mergedConfig.maxClasses || 3;
+
+  try {
+    const classesDescription = mergedConfig.classes.map((c) => `${c.name}: ${c.description}`).join('\n');
+
+    const basePrompt =
+      mergedConfig.systemPrompt || 'You are a multi-label classifier. Select all applicable categories for the given content:';
+
+    const prompt = `${basePrompt}\n\n${classesDescription}\n\n
+You should output the result as a JSON array of up to ${maxClasses} classifications, ordered by relevance:
+[
+  {
+    "class": "<class name>",
+    ${mergedConfig.includeReason ? '"reason": "<explanation>",' : ''}
+    "confidence": <confidence score between 0 and 1>
+  },
+  ...
+]
+Return JSON and nothing more.
+
+Content to classify:
+${content}`;
+
+    const parser = new JsonOutputParser();
+    const result = await llm.pipe(parser).invoke([new SystemMessage(prompt)]);
+
+    if (Array.isArray(result)) {
+      return result as ClassificationResult[];
+    }
+
+    // If single result returned, wrap in array
+    return [result as ClassificationResult];
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    logger.error({ message: 'Multi-classification failed', error: errorMessage });
+
+    return [
+      {
+        class: mergedConfig.defaultClass || 'unknown',
+        reason: mergedConfig.defaultReason || errorMessage,
+        confidence: 0,
+      },
+    ];
+  }
+}

package/src/toolsLibrary/index.ts

@@ -1,6 +1,8 @@
 import * as parseDocument from './parseDocument';
+import * as classifier from './classifier';
 
 // Export all tools as a collection for easy discovery
 export const tools = {
   'minded-parse-documents': parseDocument,
+  'minded-classifier': classifier,
 };

package/src/types/Flows.types.ts

@@ -69,6 +69,8 @@ export interface InterfaceTriggerNode extends BaseTriggerNode {
   parameters: Record<string, any>;
   appName: string;
   appImgSrc: string;
+  actionName: string;
+  actionKey: string;
 }
 
 export interface VoiceTriggerNode extends BaseTriggerNode {
@@ -108,6 +110,9 @@ export interface BrowserTaskNode extends BaseNode {
     description?: string;
     required?: boolean;
   }[];
+  autoTrigger?: boolean;
+  defaultPayload?: string;
+  proxy?: string; // 2-digit country code like 'IL'
 }
 
 export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;