@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/types/Flows.types.ts

@@ -0,0 +1,262 @@
+import { LLMConfig } from './LLM.types';
+export interface Position {
+  x: number;
+  y: number;
+}
+
+export enum NodeType {
+  TRIGGER = 'trigger',
+  JUNCTION = 'junction',
+  TOOL = 'tool',
+  APP_TOOL = 'appTool',
+  PROMPT_NODE = 'promptNode',
+  JUMP_TO_NODE = 'jumpToNode',
+  BROWSER_TASK = 'browserTask',
+  RPA = 'rpa',
+}
+
+export enum EdgeType {
+  LOGICAL_CONDITION = 'logicalCondition',
+  PROMPT_CONDITION = 'promptCondition',
+  STEP_FORWARD = 'stepForward',
+}
+
+export interface BaseNode {
+  name: string;
+  type: NodeType;
+  position?: Position;
+  humanInTheLoop?: boolean;
+  canStayOnNode?: boolean;
+  displayName: string;
+}
+
+export enum TriggerType {
+  APP = 'app',
+  WEBHOOK = 'webhook',
+  MANUAL = 'manual',
+  VOICE = 'voice',
+  INTERFACE = 'interface',
+}
+
+export interface BaseTriggerNode extends BaseNode {
+  type: NodeType.TRIGGER;
+  triggerType: TriggerType;
+}
+
+export interface BaseAppNode {
+  appName: string;
+  appImgSrc: string;
+  account?: string;
+  mockDataInstructions?: string;
+  textSentToModel?: string;
+  parameters?: Record<string, any>;
+}
+
+export interface AppTriggerNode extends BaseTriggerNode, BaseAppNode {
+  triggerType: TriggerType.APP;
+  appTriggerId: string;
+  appTriggerName: string;
+  promptInstructions?: string;
+}
+
+export interface WebhookTriggerNode extends BaseTriggerNode {
+  triggerType: TriggerType.WEBHOOK;
+  autoTrigger?: boolean;
+  defaultPayload?: string;
+}
+
+export interface InterfaceTriggerNode extends BaseTriggerNode {
+  triggerType: TriggerType.INTERFACE;
+  parameters: Record<string, any>;
+  appName: string;
+  appImgSrc: string;
+  actionName: string;
+  actionKey: string;
+}
+
+export interface VoiceTriggerNode extends BaseTriggerNode {
+  triggerType: TriggerType.VOICE;
+  /**
+   * Text that will be spoken as the very first message in a voice interaction.
+   */
+  firstMessage: string;
+  /**
+   * Optional ElevenLabs voice ID to be used for TTS.
+   */
+  voiceId?: string;
+}
+
+export interface ManualTriggerNode extends BaseTriggerNode {
+  triggerType: TriggerType.MANUAL;
+}
+
+export interface PromptNode extends BaseNode {
+  type: NodeType.PROMPT_NODE;
+  prompt: string;
+  llmConfig?: LLMConfig;
+}
+
+export interface JumpToNode extends BaseNode {
+  type: NodeType.JUMP_TO_NODE;
+  targetNodeId: string;
+}
+
+export interface BrowserTaskNode extends BaseNode {
+  type: NodeType.BROWSER_TASK;
+  prompt: string;
+  model?: string;
+  inputSchema?: {
+    name: string;
+    type: 'string' | 'number';
+    description?: string;
+    required?: boolean;
+  }[];
+  outputSchema?: {
+    name: string;
+    type: 'string' | 'number';
+    description?: string;
+    required?: boolean;
+  }[];
+  autoTrigger?: boolean;
+  defaultPayload?: string;
+  proxy?: string; // 2-digit country code like 'IL'
+  hooks?: { name: string }[]; // Array of hooks to be passed to the browser-use lambda
+}
+
+export enum RpaActionType {
+  CLICK = 'click',
+  TYPE = 'type',
+  WAIT = 'wait',
+  SCREENSHOT = 'screenshot',
+  SELECT = 'select',
+  PRESS = 'press',
+  GOTO = 'goto',
+  EXTRACT_DATA = 'extract_data',
+}
+
+export interface RpaStep {
+  id: string;
+  type: RpaActionType;
+  xpath?: string;
+  selector?: string;
+  text?: string;
+  value?: string;
+  key?: string;
+  url?: string;
+  shouldReplaceExistingText?: boolean;
+  waitTime?: number;
+  description?: string;
+  outputSchema?: {
+    name: string;
+    type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+    description?: string;
+    required?: boolean;
+    items?: {
+      type: 'string' | 'number' | 'boolean' | 'object';
+      properties?: any;
+    };
+    properties?: any;
+  }[];
+}
+
+export interface RpaNode extends BaseNode {
+  type: NodeType.RPA;
+  steps: RpaStep[];
+}
+
+export type TriggerNode = AppTriggerNode | WebhookTriggerNode | ManualTriggerNode | VoiceTriggerNode | InterfaceTriggerNode;
+
+export interface JunctionNode extends BaseNode {
+  type: NodeType.JUNCTION;
+}
+
+export interface ToolNode extends BaseNode {
+  type: NodeType.TOOL;
+  toolName: string;
+  prompt: string;
+  appImgSrc?: string;
+}
+
+export interface AppToolNode extends BaseNode, BaseAppNode {
+  type: NodeType.APP_TOOL;
+  prompt: string;
+  appName: string;
+  actionName: string;
+  metadata: AppNodeMetadata;
+  actionKey: string;
+}
+
+export type Node = TriggerNode | JunctionNode | ToolNode | AppToolNode | PromptNode | JumpToNode | BrowserTaskNode | RpaNode;
+
+export interface BaseEdge {
+  source: string;
+  target: string;
+  type: EdgeType;
+}
+
+export interface LogicalConditionEdge extends BaseEdge {
+  type: EdgeType.LOGICAL_CONDITION;
+  condition: string;
+}
+
+export interface PromptConditionEdge extends BaseEdge {
+  type: EdgeType.PROMPT_CONDITION;
+  prompt: string;
+}
+
+export interface StepForwardEdge extends BaseEdge {
+  type: EdgeType.STEP_FORWARD;
+}
+
+export type Edge = LogicalConditionEdge | PromptConditionEdge | StepForwardEdge;
+
+export enum AppNodeMetadataType {
+  APP_TOOL = 'appTool',
+}
+export interface NodeMetadata {
+  type: AppNodeMetadataType;
+}
+export interface AppNodeMetadata extends NodeMetadata {
+  type: AppNodeMetadataType.APP_TOOL;
+  schema: ActionInputParam[];
+}
+
+export type ActionInputParam = {
+  name: string;
+  type: ActionInputParamType;
+  description: string;
+  example: string | null;
+  required: boolean;
+  enum?: string[];
+  items?: {
+    type: ActionInputParamType;
+    properties?: ActionInputParam[];
+  };
+};
+
+export enum ActionInputParamType {
+  String = 'string',
+  Number = 'number',
+  Boolean = 'boolean',
+  Array = 'array',
+  Object = 'object',
+  Any = 'any',
+}
+
+export interface Flow {
+  name: string;
+  nodes: Node[];
+  edges: Edge[];
+}
+
+export enum KnownTriggerNames {
+  WEBHOOK = 'webhook',
+  DASHBOARD_MESSAGE = 'dashboard_message',
+  VOICE_MESSAGE = 'voice_message',
+}
+
+export enum internalNodesSuffix {
+  HUMAN_IN_THE_LOOP = '_humanInTheLoop',
+  TOOL_RUN = '_toolRun',
+  BROWSER_TASK_RUN = '_browserTaskRun',
+}

package/src/types/LLM.types.ts

@@ -0,0 +1,15 @@
+import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
+import { MindedChatOpenAI } from '../platform/models/mindedChatOpenAI';
+
+export type LLMConfig = {
+  name: string;
+  properties: Record<string, any>;
+};
+
+export type LLMProvider = 'ChatOpenAI' | 'AzureChatOpenAI' | 'MindedChatOpenAI';
+
+export const LLMProviders: Record<LLMProvider, any> = {
+  ChatOpenAI,
+  AzureChatOpenAI,
+  MindedChatOpenAI,
+};

package/src/types/LangGraph.types.ts

@@ -0,0 +1,86 @@
+import { BaseMessage } from '@langchain/core/messages';
+import { Annotation, CompiledStateGraph, StateGraph } from '@langchain/langgraph';
+import { HistoryStep, SessionType } from './Agent.types';
+import { z } from 'zod';
+import { logger } from '../utils/logger';
+
+export const createStateAnnotation = <Memory = any>(memorySchema?: z.ZodSchema<Memory>) =>
+  Annotation.Root({
+    messages: Annotation<Array<BaseMessage>>({
+      default: () => [] as BaseMessage[],
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    memory: Annotation<Memory>({
+      default: () => {
+        if (memorySchema) {
+          const parseResult = memorySchema.safeParse({});
+          if (!parseResult.success) {
+            logger.error({ message: 'Failed to parse memory schema', err: parseResult.error });
+            throw new Error('Failed to parse memory schema');
+          }
+          return parseResult.data;
+        }
+        return {} as Memory;
+      },
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    history: Annotation<Array<HistoryStep>>({
+      default: () => [] as HistoryStep[],
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    sessionId: Annotation<string>({
+      default: () => '',
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    goto: Annotation<string | null | undefined>({
+      default: () => null as string | null | undefined,
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    sessionType: Annotation<SessionType>({
+      default: () => SessionType.TEXT,
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    scenario: Annotation<Record<string, string | boolean>>({
+      default: () => ({} as Record<string, string | boolean>),
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+    // Available only when executing browser-use
+    cdp: Annotation<{
+      url: string | null;
+      liveUrl: string | null | undefined;
+      sessionId: string | null | undefined;
+    } | null>({
+      default: () => null,
+      // Direct replacement - no reducer, just replace the entire value
+      reducer: (_, newValue) => newValue,
+    }),
+  });
+
+// Default state annotation with any memory type
+export const stateAnnotation = createStateAnnotation();
+
+export const getInitialState = <Memory = any>(memorySchema?: z.ZodSchema<Memory>): State<Memory> => {
+  const memory = memorySchema ? memorySchema.parse({}) : ({} as Memory);
+
+  return {
+    messages: [],
+    history: [],
+    sessionId: '',
+    goto: null,
+    sessionType: SessionType.TEXT,
+    scenario: {},
+    cdp: null,
+    memory,
+  } as State<Memory>;
+};
+
+export type CompiledGraph = CompiledStateGraph<any, any, string>;
+export type PreCompiledGraph = StateGraph<any, any, string>;
+export type State<Memory = any> = ReturnType<typeof createStateAnnotation<Memory>>['State'];

package/src/types/Platform.types.ts

@@ -0,0 +1,6 @@
+export enum Environment {
+  DEVELOPMENT = 'development',
+  STAGING = 'staging',
+  PRODUCTION = 'production',
+  TEST = 'test',
+}

package/src/types/Tools.types.ts

@@ -0,0 +1,28 @@
+import { z } from 'zod';
+import { Agent } from '../';
+import { State } from './LangGraph.types';
+
+// Valid return type for new API
+export type ToolReturnType = { result?: any } | void;
+
+export interface Tool<Input extends z.ZodSchema, Memory = any> {
+  name: string;
+  description: string;
+  input: Input;
+  isGlobal?: boolean;
+  allowExecutionRequests?: boolean;
+  execute: (input: ToolExecuteInput<Input, Memory>) => Promise<ToolReturnType>;
+}
+
+export interface ToolExecuteInput<Input extends z.ZodSchema, Memory = any> {
+  input: z.infer<Input>;
+  state: State<Memory>;
+  agent: Agent;
+  invokedFromPromptNode?: boolean;
+}
+
+export interface AppActionRunnerInput {
+  appName: string;
+  actionName: string;
+  actionInput: object;
+}

package/src/types/Voice.types.ts

@@ -0,0 +1,4 @@
+export type ElevenLabsContext = {
+  sessionId: string;
+  mindedToken: string;
+}

package/src/utils/agentUtils.ts

@@ -0,0 +1,69 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as yaml from 'js-yaml';
+
+import { getConfig } from '../platform/config';
+import { AppToolNode, Flow, NodeType } from '../types/Flows.types';
+import { mindedConnection } from '../platform/mindedConnection';
+import { mindedConnectionSocketMessageType } from '../platform/mindedConnectionTypes';
+import libraryActionRunnerToolCreator from '../internalTools/libraryActionRunnerTool';
+import { Environment } from '../types/Platform.types';
+
+export async function loadFlows(flowsDirectories: string[]) {
+  const { env, isDeployed } = getConfig();
+  if ([Environment.DEVELOPMENT].includes(env) && isDeployed) {
+    const response = await mindedConnection.awaitEmit<object, { flows: Record<string, Flow> }>(
+      mindedConnectionSocketMessageType.GET_FLOWS,
+      {},
+    );
+    if (!response?.flows) {
+      throw new Error('Could not load flows from the platform connection');
+    }
+    return Object.values(response.flows);
+  }
+  return loadFlowsFromDirectory(flowsDirectories);
+}
+
+export function loadFlowsFromDirectory(flowsDirectories: string[]): Flow[] {
+  const flows: Flow[] = [];
+  for (const flowsDirectory of flowsDirectories) {
+    if (!fs.existsSync(flowsDirectory)) {
+      throw new Error(`Flows directory does not exist: ${flowsDirectory}`);
+    }
+
+    const files = fs.readdirSync(flowsDirectory);
+
+    for (const file of files) {
+      if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+        const filePath = path.join(flowsDirectory, file);
+        try {
+          const fileContent = fs.readFileSync(filePath, 'utf8');
+          const parsedFlow = yaml.load(fileContent) as Flow;
+
+          // Validate that the parsed flow has the required structure
+          if (!parsedFlow.name || !parsedFlow.nodes || !parsedFlow.edges) {
+            throw new Error(`Invalid flow structure in ${file}. Flow must have name, nodes, and edges.`);
+          }
+
+          flows.push(parsedFlow);
+        } catch (err) {
+          throw new Error(`Failed to parse flow file ${file}: ${err}`);
+        }
+      }
+    }
+
+    if (flows.length === 0) {
+      throw new Error(`No YAML flow files found in directory: ${flowsDirectory}`);
+    }
+  }
+
+  return flows;
+}
+
+export function initLibraryActionsRunnerTools(flows: Flow[]) {
+  return flows
+    .flatMap((flow) =>
+      flow.nodes.filter((node): node is AppToolNode => node.type === NodeType.APP_TOOL && (node as AppToolNode).appName === 'Minded'),
+    )
+    .map((node) => libraryActionRunnerToolCreator(node.actionKey, node.displayName!));
+}

package/src/utils/history.ts

@@ -0,0 +1,38 @@
+import { AppTriggerHistoryStep, HistoryStep, TriggerHistoryStep } from '../types/Agent.types';
+import { NodeType } from '../types/Flows.types';
+
+export const createHistoryStep = <T extends HistoryStep>(currentHistory: HistoryStep[], stepPayload: Omit<T, 'step'>): HistoryStep => {
+  const lastHistoryStep = currentHistory?.[currentHistory?.length - 1];
+  return {
+    ...stepPayload,
+    step: lastHistoryStep?.step + 1 || 0,
+  } as HistoryStep;
+};
+
+export function createTriggerHistoryStep(
+  currentHistory: HistoryStep[],
+  nodeId: string,
+  messageIds: string[],
+  triggerName: string,
+  triggerBody: any,
+  appName?: string,
+): HistoryStep {
+  const baseStep = {
+    nodeId: nodeId,
+    nodeDisplayName: triggerName,
+    messageIds,
+  } as HistoryStep;
+
+  return appName
+    ? createHistoryStep<AppTriggerHistoryStep>(currentHistory, {
+        ...baseStep,
+        type: NodeType.TRIGGER,
+        triggerBody,
+        appName,
+      })
+    : createHistoryStep<TriggerHistoryStep>(currentHistory, {
+        ...baseStep,
+        type: NodeType.TRIGGER,
+        triggerBody,
+      });
+}

package/src/utils/logger.ts

@@ -0,0 +1,38 @@
+import pino from 'pino';
+import dotenv from 'dotenv';
+dotenv.config();
+
+// Determine if we're running on Minded Cloud
+const isMindedCloud = process.env.MINDED_CLOUD === 'true' || process.env.IS_DEPLOYED_AGENT === 'true';
+
+// Configure transport based on environment
+// In Minded Cloud, use standard JSON output (no transport) for single-line logs
+const transport = isMindedCloud
+  ? undefined
+  : {
+      target: 'pino-pretty',
+      options: {
+        colorize: true,
+        crlf: true,
+        translateTime: 'SYS:standard',
+      },
+    };
+
+// Create a Pino logger with a custom log format
+export const logger = pino(
+  {
+    level: process.env.LOG_LEVEL || 'trace',
+    base: null,
+    transport,
+    formatters: {
+      level(label) {
+        return { level: label };
+      },
+    },
+    timestamp: () => `,"time":"${new Date().toISOString()}"`,
+    serializers: {
+      err: pino.stdSerializers.err,
+    },
+  },
+  pino.destination({ sync: true }),
+);

package/src/utils/schemaUtils.ts

@@ -0,0 +1,68 @@
+import { z } from 'zod';
+
+export interface SchemaField {
+  name: string;
+  type: 'string' | 'number' | 'boolean' | 'array' | 'object';
+  description?: string;
+  required?: boolean;
+}
+
+/**
+ * Converts an array of schema field definitions to a Zod object schema
+ * @param fields Array of field definitions
+ * @param defaultSchema Optional default schema to use if no fields are provided
+ * @returns Zod object schema
+ */
+export function createZodSchemaFromFields(
+  fields?: SchemaField[],
+  defaultSchema?: Record<string, z.ZodTypeAny>
+): z.ZodObject<Record<string, z.ZodTypeAny>> {
+  const schemaFields: Record<string, z.ZodTypeAny> = {};
+
+  if (fields && fields.length > 0) {
+    for (const field of fields) {
+      let fieldSchema: z.ZodTypeAny;
+
+      // Create appropriate Zod type based on field type
+      switch (field.type) {
+        case 'string':
+          fieldSchema = z.string();
+          break;
+        case 'number':
+          fieldSchema = z.number();
+          break;
+        case 'boolean':
+          fieldSchema = z.boolean();
+          break;
+        case 'array':
+          // For arrays, we'll default to array of any unless more specific
+          fieldSchema = z.array(z.any());
+          break;
+        case 'object':
+          // For objects, we'll default to record of any unless more specific
+          fieldSchema = z.record(z.any());
+          break;
+        default:
+          // Default to string for unknown types
+          fieldSchema = z.string();
+      }
+
+      // Add description if available
+      if (field.description) {
+        fieldSchema = fieldSchema.describe(field.description);
+      }
+
+      // Handle optional fields
+      if (field.required === false) {
+        fieldSchema = fieldSchema.optional();
+      }
+
+      schemaFields[field.name] = fieldSchema;
+    }
+  } else if (defaultSchema) {
+    // Use default schema if no fields provided
+    return z.object(defaultSchema);
+  }
+
+  return z.object(schemaFields);
+}

package/src/utils/wait.ts

@@ -0,0 +1 @@
+export const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));

package/src/voice/elevenLabsUtils.ts

@@ -0,0 +1,81 @@
+// ----------------- ElevenLabs WebSocket event types -----------------
+
+interface BaseEvent {
+  type: string;
+}
+
+interface UserTranscriptEvent extends BaseEvent {
+  type: 'user_transcript';
+  user_transcription_event: {
+    user_transcript: string;
+  };
+}
+
+interface AgentResponseEvent extends BaseEvent {
+  type: 'agent_response';
+  agent_response_event: {
+    agent_response: string;
+  };
+}
+
+interface AudioResponseEvent extends BaseEvent {
+  type: 'audio';
+  audio_event: {
+    audio_base_64: string;
+    event_id: number;
+  };
+}
+
+interface InterruptionEvent extends BaseEvent {
+  type: 'interruption';
+  interruption_event: {
+    event_id: number;
+  };
+}
+
+interface PingEvent extends BaseEvent {
+  type: 'ping';
+  ping_event: {
+    event_id: number;
+    ping_ms: number;
+  };
+}
+
+interface ConversationInitiationMetadataEvent extends BaseEvent {
+  type: 'conversation_initiation_metadata';
+  conversation_initiation_metadata_event: {
+    conversation_id: string;
+  };
+}
+
+export interface AgentResponseCorrectionEvent extends BaseEvent {
+  type: 'agent_response_correction';
+  agent_response_correction_event: {
+    corrected_agent_response: string;
+    original_agent_response: string;
+  };
+}
+
+/** Union of all possible WebSocket events we care about */
+export type ElevenLabsWebSocketEvent =
+  | UserTranscriptEvent
+  | AgentResponseEvent
+  | AudioResponseEvent
+  | InterruptionEvent
+  | PingEvent
+  | ConversationInitiationMetadataEvent
+  | AgentResponseCorrectionEvent;
+
+/** Shape of the message sent to ElevenLabs when starting a conversation */
+export interface ConversationInitiationClientData {
+  type: 'conversation_initiation_client_data';
+  conversation_config_override: {
+    agent: {
+      first_message: string;
+      language: string;
+    };
+    tts?: {
+      voice_id: string;
+    };
+  };
+}