@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/nodes/addTriggerNode.ts

@@ -0,0 +1,27 @@
+import { RunnableLike } from '@langchain/core/runnables';
+import { NodeType, TriggerNode, TriggerType } from '../types/Flows.types';
+import { PreCompiledGraph, stateAnnotation } from '../types/LangGraph.types';
+import { logger } from '../utils/logger';
+import { HistoryStep } from '../types/Agent.types';
+import { createHistoryStep } from '../utils/history';
+
+export const addTriggerNode = async ({ graph, node }: { graph: PreCompiledGraph; node: TriggerNode }) => {
+  const callback: RunnableLike = async (state: typeof stateAnnotation.State) => {
+    logger.debug({ msg: `[Trigger] Executing trigger node`, node: node.displayName });
+    if (node.triggerType === TriggerType.MANUAL) {
+      state.goto = null;
+      state.history.push(
+        createHistoryStep<HistoryStep>(state.history, {
+          type: NodeType.TRIGGER,
+          nodeId: node.name,
+          nodeDisplayName: node.displayName,
+          raw: '',
+          messageIds: [],
+        }),
+      );
+    }
+    return state;
+  };
+
+  graph.addNode(node.name, callback);
+};

package/src/nodes/compilePrompt.ts

@@ -0,0 +1,44 @@
+import * as ejs from 'ejs';
+import { logger } from '../utils/logger';
+
+/**
+ * Compile prompt with parameters using EJS and placeholder replacement
+ */
+export function compilePrompt(prompt: string, params: Record<string, any> = {}): string {
+  try {
+    // Define system parameters
+    params.system = { currentTime: new Date().toISOString() };
+
+    // First, render with EJS
+    let compiledPrompt = ejs.render(prompt, params);
+
+    // Then, replace placeholders in {} format
+    compiledPrompt = replacePlaceholders(compiledPrompt, params);
+
+    return compiledPrompt;
+  } catch (err) {
+    logger.error({ message: 'Error compiling prompt', err });
+    return prompt; // Return uncompiled if there's an error
+  }
+}
+
+/**
+ * Replace placeholders in {key} format
+ */
+function replacePlaceholders(text: string, params: Record<string, any>): string {
+  return text.replace(/\{([^}]+)\}/g, (match, key) => {
+    const keys = key.split('.');
+    let value: any = params;
+
+    for (const k of keys) {
+      if (value && typeof value === 'object' && k in value) {
+        value = value[k];
+      } else {
+        logger.warn({ message: `Placeholder {${key}} in prompt not found in params. It will remain as placeholder.` });
+        return match; // Return original if key not found
+      }
+    }
+
+    return String(value);
+  });
+}

package/src/nodes/nodeFactory.ts

@@ -0,0 +1,61 @@
+import { Node, NodeType } from '../types/Flows.types';
+import { PreCompiledGraph } from '../types/LangGraph.types';
+import { Tool } from '../types/Tools.types';
+import { addToolNode } from './addToolNode';
+import { addPromptNode } from './addPromptNode';
+import { AgentEventRequestPayloads } from '../events/AgentEvents';
+import { EmitSignature } from '../types/Agent.types';
+import { addAppToolNode } from './addAppToolNode';
+import { addTriggerNode } from './addTriggerNode';
+import { LLMProviders } from '../types/LLM.types';
+import { Agent } from '../agent';
+import { addJumpToNode } from './addJumpToNode';
+import { addJunctionNode } from './addJunctionNode';
+import { addBrowserTaskNode } from './addBrowserTaskNode';
+import { addRpaNode } from './addRpaNode';
+
+export const nodeFactory = ({
+  graph,
+  node,
+  tools,
+  llm,
+  emit,
+  agent,
+}: {
+  graph: PreCompiledGraph;
+  node: Node;
+  tools: Tool<any, any>[];
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders];
+  emit: EmitSignature<any, keyof AgentEventRequestPayloads<any>>;
+  agent: Agent;
+}) => {
+  const nodeType = node.type;
+  switch (nodeType) {
+    case NodeType.TRIGGER:
+      addTriggerNode({ graph, node });
+      break;
+    case NodeType.TOOL:
+      addToolNode({ graph, node, tools, llm, agent });
+      break;
+    case NodeType.JUNCTION:
+      addJunctionNode({ graph, node });
+      break;
+    case NodeType.APP_TOOL:
+      addAppToolNode({ graph, node, llm, agent, tools });
+      break;
+    case NodeType.PROMPT_NODE:
+      addPromptNode({ graph, node, tools, llm, emit, agent });
+      break;
+    case NodeType.JUMP_TO_NODE:
+      addJumpToNode({ graph, node });
+      break;
+    case NodeType.BROWSER_TASK:
+      addBrowserTaskNode({ graph, node, agent, llm });
+      break;
+    case NodeType.RPA:
+      addRpaNode({ graph, node, tools, emit, agent, llm });
+      break;
+    default:
+      throw new Error(`Unsupported node type: ${nodeType}`);
+  }
+};

package/src/nodes/rpaStepsExecutor.ts

@@ -0,0 +1,175 @@
+import { Page } from 'playwright';
+import { RpaActionType } from '../types/Flows.types';
+import { LLMProviders } from '../types/LLM.types';
+import { logger } from '../utils/logger';
+import { z } from 'zod';
+import { SystemMessage } from '@langchain/core/messages';
+import { compilePrompt } from './compilePrompt';
+import { stateAnnotation } from '../types/LangGraph.types';
+import { createZodSchemaFromFields } from '../utils/schemaUtils';
+
+// Helper function to execute individual RPA steps
+export async function executeRpaStep(
+  page: Page,
+  step: any,
+  state: typeof stateAnnotation.State,
+  llm: (typeof LLMProviders)[keyof typeof LLMProviders],
+): Promise<any> {
+  // Create params object with memory and secrets
+  const params = {
+    env: process.env,
+    state: state,
+  };
+
+  switch (step.type) {
+    case RpaActionType.CLICK: {
+      // Compile selectors in case they contain placeholders
+      const compiledXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (compiledXpath) {
+        await page.locator(`xpath=${compiledXpath}`).click({ timeout: 15000 });
+      } else if (compiledSelector) {
+        await page.click(compiledSelector, { timeout: 15000 });
+      }
+      return { action: 'clicked' };
+    }
+
+    case RpaActionType.TYPE: {
+      // Compile text and selectors
+      const compiledText = step.text ? compilePrompt(step.text, params) : '';
+      const compiledTypeXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledTypeSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (step.shouldReplaceExistingText) {
+        if (compiledTypeXpath) {
+          await page.locator(`xpath=${compiledTypeXpath}`).fill(compiledText, { timeout: 15000 });
+        } else if (compiledTypeSelector) {
+          await page.fill(compiledTypeSelector, compiledText, { timeout: 15000 });
+        }
+      } else {
+        if (compiledTypeXpath) {
+          await page.locator(`xpath=${compiledTypeXpath}`).type(compiledText, { timeout: 15000 });
+        } else if (compiledTypeSelector) {
+          await page.type(compiledTypeSelector, compiledText, { timeout: 15000 });
+        }
+      }
+      return { action: 'typed', text: compiledText };
+    }
+
+    case RpaActionType.WAIT:
+      await page.waitForTimeout(step.waitTime || 3000);
+      return { action: 'waited', duration: step.waitTime };
+
+    case RpaActionType.GOTO: {
+      // Compile URL
+      const compiledUrl = step.url ? compilePrompt(step.url, params) : '';
+      await page.goto(compiledUrl, { waitUntil: 'load', timeout: 15000 });
+      return { action: 'navigated', url: compiledUrl };
+    }
+
+    case RpaActionType.PRESS: {
+      // Compile key
+      const compiledKey = step.key ? compilePrompt(step.key, params) : 'Enter';
+      await page.keyboard.press(compiledKey);
+      return { action: 'pressed', key: compiledKey };
+    }
+
+    case RpaActionType.SELECT: {
+      // Compile value and selectors
+      const compiledValue = step.value ? compilePrompt(step.value, params) : '';
+      const compiledSelectXpath = step.xpath ? compilePrompt(step.xpath, params) : undefined;
+      const compiledSelectSelector = step.selector ? compilePrompt(step.selector, params) : undefined;
+
+      if (compiledSelectXpath) {
+        await page.locator(`xpath=${compiledSelectXpath}`).selectOption(compiledValue, { timeout: 15000 });
+      } else if (compiledSelectSelector) {
+        await page.selectOption(compiledSelectSelector, compiledValue, { timeout: 15000 });
+      }
+      return { action: 'selected', value: compiledValue };
+    }
+
+    case RpaActionType.SCREENSHOT: {
+      const screenshot = await page.screenshot({ type: 'png' });
+      // Compile description if provided
+      const compiledDescription = step.description ? compilePrompt(step.description, params) : undefined;
+      return {
+        action: 'screenshot',
+        description: compiledDescription,
+        data: screenshot.toString('base64'),
+      };
+    }
+
+    case RpaActionType.EXTRACT_DATA: {
+      if (!llm) {
+        throw new Error('LLM is required for EXTRACT_DATA action');
+      }
+
+      // Get the page content for extraction
+      const pageContent = await page.content();
+      const url = page.url();
+
+      // Convert outputSchema to Zod schema
+      const zodSchema = createZodSchemaFromFields(step.outputSchema, { result: z.string().describe('The extracted data') });
+
+      const prompt = `Extract data from the following web page according to the specified structure.
+                      Extract the requested data and return it as a structured object.
+                      WEB PAGE CONTENT:
+                      ${pageContent}`;
+
+      try {
+        // Check if LLM supports structured output
+        const supportsStructuredOutput = 'withStructuredOutput' in llm && typeof (llm as any).withStructuredOutput === 'function';
+
+        let extractedData: any;
+
+        if (supportsStructuredOutput) {
+          // Use withStructuredOutput for guaranteed schema compliance
+          const structuredLLM = (llm as any).withStructuredOutput(zodSchema);
+          const messages = [new SystemMessage(prompt)];
+          extractedData = await structuredLLM.invoke(messages);
+        } else {
+          // Fallback to regular LLM call if withStructuredOutput is not available
+          logger.warn({
+            msg: '[RPA] LLM does not support withStructuredOutput, using regular invoke',
+            url,
+          });
+
+          const response = await llm.invoke([new SystemMessage(prompt)]);
+
+          // Try to parse the response as JSON
+          const responseContent = typeof response.content === 'string' ? response.content : JSON.stringify(response.content);
+          extractedData = JSON.parse(responseContent);
+
+          // Validate against schema
+          try {
+            extractedData = zodSchema.parse(extractedData);
+          } catch (validationError) {
+            logger.error({
+              msg: '[RPA] Extracted data validation failed',
+              error: validationError instanceof Error ? validationError.message : 'Unknown error',
+              url,
+            });
+            throw validationError;
+          }
+        }
+
+        return {
+          action: 'extract_data',
+          url,
+          data: extractedData,
+        };
+      } catch (error) {
+        logger.error({
+          msg: '[RPA] Failed to extract data',
+          error: error instanceof Error ? error.message : 'Unknown error',
+          url,
+        });
+        throw error;
+      }
+    }
+
+    default:
+      throw new Error(`Unknown RPA action type: ${step.type}`);
+  }
+}

package/src/platform/config.ts

@@ -0,0 +1,89 @@
+import { resolve } from 'path';
+import { config } from 'dotenv';
+import { Environment } from '../types/Platform.types';
+import { BrowserTaskMode } from '../browserTask/types';
+import { logger } from '../utils/logger';
+const path = resolve(__dirname, '../../.env');
+config({ path });
+
+type MindedConfig = {
+  baseUrl: string;
+  env: Environment;
+  token: string | null;
+  runLocally: boolean;
+  isDeployed: boolean;
+  dashboardConnected: boolean;
+  elevenLabsKey: string | null;
+  elevenLabsAgentId: string | null;
+  browserTaskMode: BrowserTaskMode;
+};
+
+const parseBrowserTaskMode = (value: string | undefined): BrowserTaskMode => {
+  if (!value) {
+    return BrowserTaskMode.CLOUD;
+  }
+  const allowedModes = new Set<string>(Object.values(BrowserTaskMode));
+  if (allowedModes.has(value)) {
+    return value as BrowserTaskMode;
+  }
+  logger.warn({
+    message: 'Invalid BROWSER_TASK_MODE provided; defaulting to cloud',
+    provided: value,
+    allowed: Array.from(allowedModes),
+  });
+  return BrowserTaskMode.CLOUD;
+};
+
+export const getConfig = (): MindedConfig => {
+  const environment = (process.env.NODE_ENV as Environment) || Environment.DEVELOPMENT;
+  switch (environment) {
+    case Environment.TEST:
+      return {
+        baseUrl: process.env.MINDED_API_URL || process.env.MINDED_DASHBOARD_API_URL || 'http://localhost:8888',
+        env: Environment.TEST,
+        token: process.env.MINDED_CONNECTION_TOKEN || '',
+        runLocally: process.env.RUN_LOCALLY !== 'false',
+        isDeployed: process.env.MINDED_CLOUD === 'true',
+        dashboardConnected: false,
+        elevenLabsKey: process.env.ELEVEN_LABS_API_KEY || null,
+        elevenLabsAgentId: process.env.ELEVEN_LABS_AGENT_ID || null,
+        browserTaskMode: parseBrowserTaskMode(process.env.BROWSER_TASK_MODE),
+      };
+    case Environment.DEVELOPMENT:
+      return {
+        baseUrl: process.env.MINDED_DASHBOARD_API_URL || 'https://dashboard-api.minded.com',
+        env: Environment.DEVELOPMENT,
+        token: process.env.MINDED_CONNECTION_TOKEN || '',
+        runLocally: process.env.RUN_LOCALLY === 'true',
+        isDeployed: process.env.MINDED_CLOUD === 'true',
+        elevenLabsKey: process.env.ELEVEN_LABS_API_KEY || null,
+        dashboardConnected: true,
+        elevenLabsAgentId: process.env.ELEVEN_LABS_AGENT_ID || 'agent_01jye194psezr909nwec27m2t9',
+        browserTaskMode: parseBrowserTaskMode(process.env.BROWSER_TASK_MODE),
+      };
+    case Environment.STAGING:
+      return {
+        baseUrl: process.env.MINDED_API_URL || 'https://api.minded.com',
+        env: Environment.STAGING,
+        token: process.env.MINDED_CONNECTION_TOKEN || '',
+        runLocally: false,
+        isDeployed: process.env.MINDED_CLOUD === 'true',
+        elevenLabsKey: process.env.ELEVEN_LABS_API_KEY || null,
+        dashboardConnected: false,
+        elevenLabsAgentId: process.env.ELEVEN_LABS_AGENT_ID || 'agent_01jye1ajkaf17axpcxg220qwcm',
+        browserTaskMode: parseBrowserTaskMode(process.env.BROWSER_TASK_MODE),
+      };
+    case Environment.PRODUCTION:
+      return {
+        baseUrl: process.env.MINDED_API_URL || 'https://api.minded.com',
+        env: Environment.PRODUCTION,
+        token: process.env.MINDED_CONNECTION_TOKEN || '',
+        runLocally: false,
+        isDeployed: process.env.MINDED_CLOUD === 'true',
+        elevenLabsKey: process.env.ELEVEN_LABS_API_KEY || null,
+        dashboardConnected: false,
+        elevenLabsAgentId: process.env.ELEVEN_LABS_AGENT_ID || 'agent_01jye1ajkaf17axpcxg220qwcm',
+        browserTaskMode: parseBrowserTaskMode(process.env.BROWSER_TASK_MODE),
+      };
+  }
+};

package/src/platform/mindedCheckpointSaver.ts

@@ -0,0 +1,146 @@
+import {
+  BaseCheckpointSaver,
+  CheckpointTuple,
+  Checkpoint,
+  CheckpointMetadata,
+  ChannelVersions,
+  PendingWrite,
+  CheckpointListOptions,
+  CheckpointPendingWrite,
+} from '@langchain/langgraph-checkpoint';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import * as mindedConnection from './mindedConnection';
+import {
+  mindedConnectionSocketMessageType,
+  OnCheckpointGetTuple,
+  OnCheckpointGetTupleResponse,
+  OnCheckpointList,
+  OnCheckpointListResponse,
+  OnCheckpointPut,
+  OnCheckpointPutResponse,
+  OnCheckpointPutWrites,
+} from './mindedConnectionTypes';
+import { logger } from '../utils/logger';
+
+export class MindedCheckpointSaver extends BaseCheckpointSaver<number> {
+  constructor() {
+    super();
+  }
+
+  async getTuple(config: RunnableConfig<Record<string, any>>): Promise<CheckpointTuple | undefined> {
+    const response = await mindedConnection.awaitEmit<OnCheckpointGetTuple, OnCheckpointGetTupleResponse>(
+      mindedConnectionSocketMessageType.CHECKPOINT_GET_TUPLE,
+      {
+        type: mindedConnectionSocketMessageType.CHECKPOINT_GET_TUPLE,
+        config,
+      },
+    );
+
+    if (!response.tuple) {
+      return undefined;
+    }
+
+    const tuple = response.tuple;
+
+    // Re-serialize and deserialize checkpoint to restore class information
+    const [checkpointType, serializedCheckpoint] = this.serde.dumpsTyped(tuple.checkpoint);
+    const checkpoint = (await this.serde.loadsTyped(checkpointType, serializedCheckpoint)) as Checkpoint;
+
+    // Re-serialize and deserialize metadata to restore class information
+    const metadata = tuple.metadata
+      ? (() => {
+          const [metadataType, serializedMetadata] = this.serde.dumpsTyped(tuple.metadata);
+          return this.serde.loadsTyped(metadataType, serializedMetadata) as CheckpointMetadata;
+        })()
+      : undefined;
+
+    // Re-serialize and deserialize pending writes to restore class information
+    const pendingWrites: CheckpointPendingWrite[] = tuple.pendingWrites
+      ? await Promise.all(
+          tuple.pendingWrites.map(async (write) => {
+            const [taskId, channel, value] = write;
+            const [valueType, serializedValue] = this.serde.dumpsTyped(value);
+            const deserializedValue = await this.serde.loadsTyped(valueType, serializedValue);
+            return [taskId, channel, deserializedValue] as CheckpointPendingWrite;
+          }),
+        )
+      : [];
+
+    return {
+      config: tuple.config,
+      checkpoint,
+      metadata: await metadata,
+      parentConfig: tuple.parentConfig,
+      pendingWrites,
+    };
+  }
+
+  async *list(config: RunnableConfig<Record<string, any>>, options?: CheckpointListOptions): AsyncGenerator<CheckpointTuple, any, any> {
+    const response = await mindedConnection.awaitEmit<OnCheckpointList, OnCheckpointListResponse>(
+      mindedConnectionSocketMessageType.CHECKPOINT_LIST,
+      {
+        type: mindedConnectionSocketMessageType.CHECKPOINT_LIST,
+        config,
+        options,
+      },
+    );
+
+    for (const tuple of response.checkpoints || []) {
+      // Re-serialize and deserialize checkpoint to restore class information
+      const [checkpointType, serializedCheckpoint] = this.serde.dumpsTyped(tuple.checkpoint);
+      const checkpoint = (await this.serde.loadsTyped(checkpointType, serializedCheckpoint)) as Checkpoint;
+
+      // Re-serialize and deserialize metadata to restore class information
+      const metadata = tuple.metadata
+        ? (() => {
+            const [metadataType, serializedMetadata] = this.serde.dumpsTyped(tuple.metadata);
+            return this.serde.loadsTyped(metadataType, serializedMetadata) as CheckpointMetadata;
+          })()
+        : undefined;
+
+      yield {
+        config: tuple.config,
+        checkpoint,
+        metadata: await metadata,
+        parentConfig: tuple.parentConfig,
+      };
+    }
+  }
+
+  async put(
+    config: RunnableConfig<Record<string, any>>,
+    checkpoint: Checkpoint<string, string>,
+    metadata: CheckpointMetadata,
+    newVersions: ChannelVersions,
+  ): Promise<RunnableConfig<Record<string, any>>> {
+    try {
+      const response = await mindedConnection.awaitEmit<OnCheckpointPut, OnCheckpointPutResponse>(
+        mindedConnectionSocketMessageType.CHECKPOINT_PUT,
+        {
+          type: mindedConnectionSocketMessageType.CHECKPOINT_PUT,
+          config,
+          checkpoint,
+          metadata,
+          newVersions,
+        },
+      );
+      return response.config;
+    } catch (err) {
+      logger.error({ message: 'Error putting checkpoint:', err });
+      return config;
+    }
+  }
+
+  async putWrites(config: RunnableConfig<Record<string, any>>, writes: PendingWrite[], taskId: string): Promise<void> {
+    try {
+      await mindedConnection.awaitEmit<OnCheckpointPutWrites, any>(mindedConnectionSocketMessageType.CHECKPOINT_PUT_WRITES, {
+        type: mindedConnectionSocketMessageType.CHECKPOINT_PUT_WRITES,
+        config,
+        writes,
+        taskId,
+      });
+    } catch (err) {
+      logger.error({ message: 'Error putting writes:', err });
+    }
+  }
+}