@minded-ai/mindedjs 1.0.0-ec2-beta-1

package/src/toolsLibrary/extraction.ts

@@ -0,0 +1,393 @@
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { Tool } from '../types/Tools.types';
+import { logger } from '../utils/logger';
+import { JsonOutputParser } from '@langchain/core/output_parsers';
+import { SystemMessage } from '@langchain/core/messages';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+// Type guard for checking if LLM supports structured output
+interface StructuredOutputLLM extends BaseLanguageModel {
+  withStructuredOutput<T extends z.ZodType>(schema: T): BaseLanguageModel;
+}
+
+function supportsStructuredOutput(llm: BaseLanguageModel): llm is StructuredOutputLLM {
+  return 'withStructuredOutput' in llm && typeof (llm as any).withStructuredOutput === 'function';
+}
+
+// Type definitions for extraction configuration using Zod
+export type ExtractionSchema = z.ZodTypeAny;
+
+export interface ExtractionConfig {
+  schema: ExtractionSchema;
+  systemPrompt?: string;
+  examples?: Array<{ input: string; output?: any }>;
+  defaultValue?: any;
+  maxRetries?: number;
+  strictMode?: boolean; // If true, will validate against schema
+}
+
+export type ExtractionResult = z.infer<ExtractionSchema>;
+
+// Default configuration
+const DEFAULT_CONFIG: Partial<ExtractionConfig> = {
+  maxRetries: 3,
+  strictMode: true,
+};
+
+/**
+ * Convert Zod schema to JSON schema format for prompt
+ */
+function schemaToJsonFormat(schema: ExtractionSchema): string {
+  // Use zod-to-json-schema for robust conversion
+  const jsonSchema = zodToJsonSchema(schema, {
+    target: 'openApi3',
+    $refStrategy: 'none', // Inline all references for simpler prompts
+  });
+
+  // Remove the $schema property for cleaner output
+  if (typeof jsonSchema === 'object' && jsonSchema !== null && '$schema' in jsonSchema) {
+    delete (jsonSchema as any).$schema;
+  }
+
+  return JSON.stringify(jsonSchema, null, 2);
+}
+
+/**
+ * Build prompt with schema for JSON parsing mode
+ */
+function buildPromptWithSchema(
+  basePrompt: string,
+  schema: ExtractionSchema,
+  examples: Array<{ input: string; output?: any }> | undefined,
+  content: string,
+): string {
+  const schemaDescription = schemaToJsonFormat(schema);
+  let prompt = `${basePrompt}\n\nExpected JSON format:\n${schemaDescription}\n\n`;
+
+  // Add examples if provided
+  if (examples && examples.length > 0) {
+    prompt += 'Examples:\n';
+    for (const example of examples) {
+      prompt += `Input: ${example.input}\nOutput: ${JSON.stringify(example.output)}\n\n`;
+    }
+  }
+
+  prompt += 'Return ONLY valid JSON that matches the schema. No additional text or explanation.\n\n';
+  prompt += `Content to extract from:\n${content}`;
+
+  return prompt;
+}
+
+/**
+ * Validate extraction result against Zod schema
+ */
+function validateResult(result: any, schema: ExtractionSchema): { valid: boolean; errors: string[] } {
+  try {
+    // Use Zod's built-in validation
+    schema.parse(result);
+    return { valid: true, errors: [] };
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      // Extract error messages from Zod validation
+      const errors = error.errors.map((err) => {
+        const path = err.path.join('.');
+        return path ? `${path}: ${err.message}` : err.message;
+      });
+      return { valid: false, errors };
+    }
+    // Handle non-Zod errors
+    return { valid: false, errors: [error instanceof Error ? error.message : 'Unknown validation error'] };
+  }
+}
+
+/**
+ * Generic extraction utility that can be used standalone
+ *
+ * When strictMode is true:
+ * - If the LLM supports withStructuredOutput, it will use that for guaranteed schema compliance
+ * - Otherwise, it falls back to JSON parsing with Zod validation and retries
+ *
+ * When strictMode is false:
+ * - Uses JSON parsing without validation for more flexibility
+ *
+ * @param content The content to extract from
+ * @param config The extraction configuration
+ * @param llm The language model to use for extraction
+ * @param attemptNumber Current attempt number (used internally for retries)
+ * @returns The extraction result
+ */
+export async function extract(
+  content: string,
+  config: ExtractionConfig,
+  llm: BaseLanguageModel,
+  attemptNumber: number = 0,
+): Promise<ExtractionResult> {
+  const mergedConfig = { ...DEFAULT_CONFIG, ...config };
+
+  try {
+    const basePrompt = mergedConfig.systemPrompt || 'Extract structured information from the given content:';
+
+    let result: any;
+
+    if (mergedConfig.strictMode) {
+      // Use structured output when strict mode is enabled
+      try {
+        // Check if the LLM supports structured output
+        if (supportsStructuredOutput(llm)) {
+          // When using withStructuredOutput, we don't need to include the schema in the prompt
+          // The LLM already knows the structure from the withStructuredOutput method
+          let prompt = basePrompt;
+
+          // Add examples if provided (still useful for context)
+          if (mergedConfig.examples && mergedConfig.examples.length > 0) {
+            prompt += '\n\nExamples:\n';
+            for (const example of mergedConfig.examples) {
+              prompt += `Input: ${example.input}\nOutput: ${JSON.stringify(example.output)}\n\n`;
+            }
+          }
+
+          prompt += `\n\nContent to extract from:\n${content}`;
+
+          const structuredLLM = llm.withStructuredOutput(mergedConfig.schema);
+          const messages = [new SystemMessage(prompt)];
+          result = await structuredLLM.invoke(messages);
+
+          // The result should already be validated by withStructuredOutput
+          return result as ExtractionResult;
+        } else {
+          // Fallback to JSON parser with validation if withStructuredOutput is not available
+          logger.info({
+            message: 'LLM does not support withStructuredOutput, falling back to JSON parser with validation',
+          });
+
+          // For JSON parsing mode, we need to include the schema in the prompt
+          const prompt = buildPromptWithSchema(basePrompt, mergedConfig.schema, mergedConfig.examples, content);
+
+          const parser = new JsonOutputParser();
+          const messages = [new SystemMessage(prompt)];
+          result = await llm.pipe(parser).invoke(messages);
+
+          // Validate the result
+          const validation = validateResult(result, mergedConfig.schema);
+
+          if (!validation.valid && attemptNumber < (mergedConfig.maxRetries || 3)) {
+            logger.warn({
+              message: 'Extraction validation failed, retrying',
+              errors: validation.errors,
+              attempt: attemptNumber + 1,
+            });
+
+            // Add validation errors to prompt for retry
+            const retryConfig = {
+              ...config,
+              systemPrompt: `${basePrompt}\n\nPrevious attempt had these errors:\n${validation.errors.join(
+                '\n',
+              )}\n\nPlease correct these issues and provide valid JSON.`,
+            };
+
+            return extract(content, retryConfig, llm, attemptNumber + 1);
+          } else if (!validation.valid) {
+            logger.error({
+              message: 'Extraction validation failed after max retries',
+              errors: validation.errors,
+            });
+
+            if (mergedConfig.defaultValue) {
+              return mergedConfig.defaultValue;
+            }
+
+            throw new Error(`Extraction validation failed: ${validation.errors.join(', ')}`);
+          }
+        }
+      } catch (err) {
+        // If structured output fails, retry with adjusted config
+        if (attemptNumber < (mergedConfig.maxRetries || 3)) {
+          logger.warn({
+            message: 'Structured output failed, retrying',
+            error: err instanceof Error ? err.message : 'Unknown error',
+            attempt: attemptNumber + 1,
+          });
+
+          return extract(content, config, llm, attemptNumber + 1);
+        } else {
+          throw err;
+        }
+      }
+    } else {
+      // Non-strict mode: use JSON parser without validation
+      const prompt = buildPromptWithSchema(basePrompt, mergedConfig.schema, mergedConfig.examples, content);
+
+      const parser = new JsonOutputParser();
+      const messages = [new SystemMessage(prompt)];
+      result = await llm.pipe(parser).invoke(messages);
+    }
+
+    return result as ExtractionResult;
+  } catch (err) {
+    logger.error({ message: 'Extraction failed', err });
+
+    if (mergedConfig.defaultValue) {
+      return mergedConfig.defaultValue;
+    }
+
+    throw err;
+  }
+}
+
+/**
+ * Create a simple extractor for common use cases
+ * @param schemaOrPrompt Either a Zod schema or a simple prompt string
+ * @param options Additional configuration options
+ * @returns A configured extract function
+ */
+export function createExtractor(schemaOrPrompt: ExtractionSchema | string, options?: Partial<ExtractionConfig>) {
+  let config: ExtractionConfig;
+
+  if (typeof schemaOrPrompt === 'string') {
+    // Simple prompt-based extraction - create a Zod schema for a single string result
+    config = {
+      schema: z.object({
+        result: z.string().describe('The extracted result'),
+      }),
+      systemPrompt: schemaOrPrompt,
+      ...options,
+    };
+  } else {
+    // Schema-based extraction
+    config = {
+      schema: schemaOrPrompt,
+      ...options,
+    };
+  }
+
+  return (content: string, llm: BaseLanguageModel) => extract(content, config, llm);
+}
+
+// Schema for the extraction tool
+export const schema = z.object({
+  content: z.string().describe('The content to extract information from'),
+  schema: z.record(z.any()).optional().describe('Extraction schema defining the structure of data to extract'),
+  systemPrompt: z.string().optional().describe('Custom system prompt for extraction'),
+  examples: z
+    .array(
+      z.object({
+        input: z.string(),
+        output: z.any(),
+      }),
+    )
+    .optional()
+    .describe('Examples to guide the extraction'),
+  strictMode: z.boolean().optional().default(true).describe('Whether to validate the extraction against the schema'),
+  maxRetries: z.number().optional().default(3).describe('Maximum number of retries if validation fails'),
+  defaultValue: z.any().optional().describe('Default value to return if extraction fails'),
+});
+
+const extractionTool: Tool<typeof schema, any> = {
+  name: 'minded-extraction',
+  description:
+    'Extract structured data from unstructured text using AI. Supports custom Zod schemas, validation, examples, and retries. When strictMode is enabled, uses LLM structured output (if available) for guaranteed schema compliance. Perfect for extracting specific information like country codes, emails, dates, or any custom data structure.',
+  input: schema,
+  isGlobal: false,
+  execute: async ({ input, state, agent }) => {
+    const { content, schema: inputSchema, systemPrompt, examples, strictMode, maxRetries, defaultValue } = input;
+
+    logger.info({
+      message: 'Extracting structured data',
+      sessionId: state.sessionId,
+      contentLength: content.length,
+      hasSchema: !!inputSchema,
+    });
+
+    // Convert input schema to Zod schema if provided
+    let extractionSchema: ExtractionSchema;
+    if (inputSchema) {
+      // Build a Zod schema from the input schema object
+      const zodShape: Record<string, z.ZodType> = {};
+
+      for (const [key, value] of Object.entries(inputSchema)) {
+        if (typeof value === 'object' && value !== null) {
+          const fieldConfig = value as any;
+          let zodType: z.ZodType;
+
+          // Create appropriate Zod type based on the field configuration
+          if (fieldConfig.type === 'string') {
+            zodType = z.string();
+          } else if (fieldConfig.type === 'number') {
+            zodType = z.number();
+          } else if (fieldConfig.type === 'boolean') {
+            zodType = z.boolean();
+          } else if (fieldConfig.type === 'array') {
+            zodType = z.array(z.any());
+          } else if (fieldConfig.type === 'object') {
+            zodType = z.object({});
+          } else {
+            // Default to string if type not specified
+            zodType = z.string();
+          }
+
+          // Add description if available
+          if (fieldConfig.description) {
+            zodType = zodType.describe(fieldConfig.description);
+          }
+
+          // Handle optional fields
+          if (fieldConfig.required === false) {
+            zodType = zodType.optional();
+          }
+
+          zodShape[key] = zodType;
+        } else {
+          // Default to string for simple values
+          zodShape[key] = z.string();
+        }
+      }
+
+      extractionSchema = z.object(zodShape);
+    } else {
+      // Default schema if none provided
+      extractionSchema = z.object({
+        result: z.string().describe('The extracted result'),
+      });
+    }
+
+    const config: ExtractionConfig = {
+      schema: extractionSchema,
+      systemPrompt,
+      examples,
+      strictMode,
+      maxRetries,
+      defaultValue,
+    };
+
+    try {
+      const result = await extract(content, config, agent.llm);
+
+      logger.info({
+        message: 'Extraction completed',
+        sessionId: state.sessionId,
+        resultKeys: Object.keys(result),
+      });
+
+      state.memory.lastExtraction = {
+        content: content.substring(0, 100) + (content.length > 100 ? '...' : ''),
+        result,
+        timestamp: new Date().toISOString(),
+      };
+
+      return {
+        result,
+      };
+    } catch (err) {
+      logger.error({
+        message: 'Extraction failed',
+        sessionId: state.sessionId,
+        err,
+      });
+
+      throw err;
+    }
+  },
+};
+
+export default extractionTool;

package/src/toolsLibrary/index.ts

@@ -0,0 +1,10 @@
+import * as parseDocument from './parseDocument';
+import * as classifier from './classifier';
+import * as extraction from './extraction';
+
+// Export all tools as a collection for easy discovery
+export const tools = {
+  'minded-parse-documents': parseDocument,
+  'minded-classifier': classifier,
+  'minded-extraction': extraction,
+};

package/src/toolsLibrary/parseDocument.ts

@@ -0,0 +1,129 @@
+import { z } from 'zod';
+import { Tool } from '../types/Tools.types';
+import { extractFromDocument } from '../internalTools/documentExtraction/documentExtraction';
+import { logger } from '../utils/logger';
+
+// Schema for the tool - these are the parameters that can be inferred by LLM
+export const schema = z.object({
+  // Document source parameters - only one should be provided based on loadFrom setting
+  documentPath: z.string().optional().nullable().describe('Path to the document file to parse (when loadFrom is "path")'),
+  documentContent: z
+    .union([z.instanceof(Buffer), z.string()])
+    .optional()
+    .nullable()
+    .describe('Document content as Buffer or string (when loadFrom is "buffer" or "string")'),
+  documentUrl: z.string().optional().nullable().describe('URL to fetch the document from (when loadFrom is "url")'),
+
+  // Processing parameters - these can be predefined in the UI
+  extractRaw: z.boolean().optional().nullable().describe('Extract raw text without AI processing'),
+  schema: z.any().optional().nullable().describe('Zod schema for structured data extraction (when not extracting raw)'),
+
+  // Load source indicator - this would be set from UI
+  loadFrom: z.enum(['url', 'path', 'buffer', 'string']).optional().nullable().describe('Source type for the document'),
+  systemPrompt: z.string().optional().nullable().describe('Prompt for guiding extraction (when not using schema or extracting raw)'),
+});
+
+const parseDocumentTool: Tool<typeof schema, any> = {
+  name: 'minded-parse-documents',
+  description:
+    'Parse and extract data from documents (PDFs, images, Word docs, etc.). Supports multiple input sources (URL, file path, buffer, or string) and can extract raw text, structured data with a schema, or unstructured data with a prompt. Parameters can be predefined in the flow configuration.',
+  input: schema,
+  isGlobal: false,
+  execute: async ({ input, state, agent }) => {
+    // The input here will be the combined input from both LLM and UI properties
+    const combinedInput = input as z.infer<typeof schema>;
+
+    logger.info({
+      msg: '*Action: Parse document*',
+      sessionId: state.sessionId,
+      loadFrom: combinedInput.loadFrom,
+      hasPath: !!combinedInput.documentPath,
+      hasContent: !!combinedInput.documentContent,
+      hasUrl: !!combinedInput.documentUrl,
+      hasSchema: !!combinedInput.schema,
+      hasPrompt: !!combinedInput.systemPrompt,
+      extractRaw: combinedInput.extractRaw,
+    });
+
+    try {
+      // Validate document source based on loadFrom parameter
+      if (combinedInput.loadFrom) {
+        switch (combinedInput.loadFrom) {
+          case 'url':
+            if (!combinedInput.documentUrl) {
+              throw new Error('documentUrl is required when loadFrom is "url"');
+            }
+            break;
+          case 'path':
+            if (!combinedInput.documentPath) {
+              throw new Error('documentPath is required when loadFrom is "path"');
+            }
+            break;
+          case 'buffer':
+          case 'string':
+            if (!combinedInput.documentContent) {
+              throw new Error('documentContent is required when loadFrom is "buffer" or "string"');
+            }
+            break;
+        }
+      } else {
+        // Fallback to original validation if loadFrom is not specified
+        if (!combinedInput.documentPath && !combinedInput.documentContent && !combinedInput.documentUrl) {
+          throw new Error('At least one document source must be provided: documentPath, documentContent, or documentUrl');
+        }
+      }
+
+      // Prepare extraction options, filtering out null/undefined values
+      const extractionOptions: Parameters<typeof extractFromDocument>[0] = {};
+
+      if (combinedInput.documentPath) {
+        extractionOptions.documentPath = combinedInput.documentPath;
+      }
+      if (combinedInput.documentContent) {
+        extractionOptions.documentContent = combinedInput.documentContent;
+      }
+      if (combinedInput.documentUrl) {
+        extractionOptions.documentUrl = combinedInput.documentUrl;
+      }
+
+      // Only include LLM if not extracting raw text
+      if (!combinedInput.extractRaw) {
+        extractionOptions.llm = agent.llm;
+
+        // Include schema or system prompt if provided
+        if (combinedInput.schema) {
+          extractionOptions.schema = combinedInput.schema;
+        } else if (combinedInput.systemPrompt) {
+          extractionOptions.systemPrompt = combinedInput.systemPrompt;
+        }
+      }
+
+      // Extract from document using the SDK's document extraction capabilities
+      const result = await extractFromDocument(extractionOptions);
+
+      state.memory.lastParsedDocument = {
+        source: combinedInput.documentPath || combinedInput.documentUrl || 'content',
+        extractedAt: new Date().toISOString(),
+        extractedRaw: !!combinedInput.extractRaw,
+      };
+
+      // Return only the extracted data, not the metadata as requested
+      return {
+        result: result.data,
+      };
+    } catch (err) {
+      logger.error({
+        message: 'Failed to parse document',
+        sessionId: state.sessionId,
+        err,
+      });
+
+      state.memory.documentParsingError = err instanceof Error ? err.message : String(err);
+      return {
+        result: `Failed to parse document: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+  },
+};
+
+export default parseDocumentTool;

package/src/triggers/triggerTypeToDefaultMessage.ts

@@ -0,0 +1,9 @@
+import { BaseMessage, HumanMessage } from '@langchain/core/messages';
+
+const triggerTypeToDefaultMessage: Record<string, Record<string, (triggerBody: any) => Array<BaseMessage>>> = {
+  ['Slack']: {
+    'New Direct Message (Instant)': (triggerBody: any) => [new HumanMessage({ content: triggerBody.text })],
+  },
+};
+
+export default triggerTypeToDefaultMessage;

package/src/types/Agent.types.ts

@@ -0,0 +1,73 @@
+import { AgentEventRequestPayloads, AgentEventResponsePayloads } from '../events/AgentEvents';
+import { LLMConfig } from './LLM.types';
+import { NodeType } from './Flows.types';
+
+export type EmitSignature<Memory, E extends keyof AgentEventRequestPayloads<Memory>> = (
+  event: E,
+  payload: AgentEventRequestPayloads<Memory>[E],
+) => Promise<AgentEventResponsePayloads[E][]>;
+
+export type MindedSDKConfig = {
+  flows: string[];
+  llm: LLMConfig;
+  tools: string[];
+  /**
+   * Optional directories containing playbook definition files. If omitted, no playbooks will be loaded.
+   */
+  playbooks?: string[];
+};
+
+/**
+ * Parameters for the Agent invoke method
+ */
+export interface AgentInvokeParams extends TriggerEvent {
+  /** Internal flag to bypass session processing check for recursive calls */
+  bypassSessionCheck?: boolean;
+}
+
+export interface HistoryStep {
+  type: NodeType;
+  step: number;
+  raw: any;
+  nodeId: string;
+  nodeDisplayName: string;
+  messageIds?: string[];
+}
+
+export interface AppTriggerHistoryStep extends HistoryStep {
+  type: NodeType.TRIGGER;
+  appName: string;
+  triggerBody: any;
+}
+
+export interface TriggerHistoryStep extends HistoryStep {
+  type: NodeType.TRIGGER;
+  triggerBody: any;
+}
+
+export interface AppActionInvocationHistoryStep extends HistoryStep {
+  type: NodeType.APP_TOOL;
+  appName: string;
+}
+
+export interface CustomActionInvocationHistoryStep extends HistoryStep {
+  type: NodeType.TOOL;
+}
+
+export interface JunctionHistoryStep extends HistoryStep {
+  type: NodeType.JUNCTION;
+}
+
+export enum SessionType {
+  VOICE = 'voice',
+  TEXT = 'text',
+}
+
+export interface TriggerEvent {
+  /** The payload/data associated with the trigger */
+  triggerBody: any;
+  /** The name/type of the trigger being invoked */
+  triggerName: string;
+  /** Optional name of the application triggering the agent in case of an app trigger */
+  appName?: string;
+}