@minded-ai/mindedjs 3.0.8-beta.12 → 3.1.9-beta.1

package/docs/tooling/document-processing.md

@@ -1,132 +1,243 @@
 # Document Processing
 
-Parse and extract data from images, PDFs, Word documents, spreadsheets, and more using AI-powered document processing. This tool handles both document parsing and data extraction in a single step.
+Parse and extract data from images, PDFs, Word documents, spreadsheets, and more using AI-powered document processing. This tool handles both document parsing and data extraction.
 
 ## Overview
 
 **Supported formats:** Images (JPG, PNG, GIF, BMP, WebP, TIFF), Documents (PDF, DOC, DOCX, TXT, RTF, ODT), Spreadsheets (XLS, XLSX, CSV, ODS), Presentations (PPT, PPTX, ODP), Web formats (HTML, HTM, MD, XML)
 
-**Built-in extraction modes:**
+**Available extraction modes:**
 
-1. **Structured Extraction**: Extract data into a predefined Zod schema using AI
-2. **Unstructured Extraction**: Extract information based on prompt instructions
-3. **Raw Text Extraction**: Extract plain text without AI processing
+1. **Structured Extraction with Schema**: Extract data into a predefined Zod schema using AI
+2. **Structured Extraction with Prompt**: Guide extraction using custom prompts
+3. **Raw Text Extraction**: Parse document and extract plain text without AI processing
 
 ## Processing Modes
 
-Processing mode is controlled via the `DOCUMENT_PROCESSING_MODE` environment variable:
+Document processing supports two modes: **managed** (default) and **local**. In managed mode documents are uploaded to Minded cloud and processed there, providing secure API key storage, automatic cost tracking, and no SDK configuration. Local mode processes documents directly in your SDK using LlamaCloud.
 
-### Managed (Default)
+To use local mode, set the `DOCUMENT_PROCESSING_MODE` environment variable to `local` and provide a `LLAMA_CLOUD_API_KEY`:
 
-Backend handles processing. Benefits: secure API key storage, automatic cost tracking, no SDK configuration required.
+```bash
+DOCUMENT_PROCESSING_MODE=local
+LLAMA_CLOUD_API_KEY=your_llama_cloud_api_key
+```
 
-### Local
+## Using in Flows
 
-SDK handles processing. Set `DOCUMENT_PROCESSING_MODE=local` in your environment. Requires `LLAMA_CLOUD_API_KEY` in SDK environment.
+Document processing includes built-in AI extraction - use the node's `prompt` and `outputSchema` properties to specify what data to extract. No additional extraction tool is needed.
 
-## Using in Flows
+**Available properties:**
+
+- `parameters.documentSource` (string, required): URL or file path to the document (auto-detected)
+- `parameters.returnStructuredOutput` (boolean, optional, default: `false`): Set to `true` to enable AI-powered extraction, `false` for raw text only. When `true`, requires either `prompt` or `outputSchema` (or both)
+- `prompt` (string, optional): Instructions for AI-powered extraction. Ignored when `returnStructuredOutput` is `false`
+- `outputSchema` (schema object, optional): Define the structure of extracted data for structured extraction. Ignored when `returnStructuredOutput` is `false`
 
-Document processing includes built-in AI extraction - use `systemPrompt` to specify what data to extract. No additional extraction tool is needed.
+### Examples
 
-### Example: Document Processing
+Raw text extraction without AI processing:
 
 ```yaml
-- name: 'parse-invoice' # Must be unique within the flow
-  type: appTool
-  displayName: 'Parse Invoice'
-  actionKey: 'minded-parse-documents'
-  actionName: 'Parse Document'
-  appName: 'Minded'
-  parameters:
-    documentSource: '{state.memory.invoiceUrl}'
-    extractRaw: true
-
-- name: 'parse-uploaded-document' # Must be unique within the flow
-  type: appTool
-  displayName: 'Parse Uploaded Document'
-  actionKey: 'minded-parse-documents'
-  actionName: 'Parse Document'
-  appName: 'Minded'
-  parameters:
-    documentSource: '{state.memory.filePath}'
-    systemPrompt: 'Extract invoice number, amount, date, and vendor'
-
-- name: 'extract-names-addresses' # Must be unique within the flow
-  type: appTool
-  displayName: 'Extract Names and Addresses'
-  actionKey: 'minded-parse-documents'
-  actionName: 'Parse Document'
-  appName: 'Minded'
-  parameters:
-    documentSource: '{state.memory.uploadedFile}'
-    systemPrompt: 'Extract all names and addresses from this document'
+name: Main flow
+nodes:
+  - name: 'parse-invoice'
+    type: appTool
+    displayName: 'Parse Invoice'
+    actionKey: 'minded-parse-documents'
+    actionName: 'Parse Document'
+    appName: 'Minded'
+    parameters:
+      documentSource: '{state.memory.invoiceUrl}'
+      returnStructuredOutput: false
 ```
 
-**Available parameters:**
+Structured extraction with schema and prompt:
 
-- `documentSource`: URL or file path to the document (auto-detected)
-- `extractRaw`: Set to `true` for raw text without AI
-- `schema`: Zod schema for structured extraction
-- `systemPrompt`: Instructions for AI-powered extraction
+```yaml
+name: Main flow
+nodes:
+  - name: 'parse-uploaded-document'
+    type: appTool
+    displayName: 'Parse Uploaded Document'
+    actionKey: 'minded-parse-documents'
+    actionName: 'Parse Document'
+    appName: 'Minded'
+    parameters:
+      documentSource: '{state.memory.filePath}'
+      returnStructuredOutput: true
+    prompt: 'Extract invoice number, amount, date, and vendor'
+    outputSchema:
+      - name: invoiceNumber
+        type: string
+        description: Invoice number
+      - name: amount
+        type: number
+        description: Total amount
+      - name: date
+        type: string
+        description: Invoice date
+      - name: vendor
+        type: string
+        description: Vendor name
+```
 
-## Programmatic Usage
+Unstructured extraction guided by prompt only:
+
+```yaml
+name: Main flow
+nodes:
+  - name: 'extract-names-addresses'
+    type: appTool
+    displayName: 'Extract Names and Addresses'
+    actionKey: 'minded-parse-documents'
+    actionName: 'Parse Document'
+    appName: 'Minded'
+    parameters:
+      documentSource: '{state.memory.uploadedFile}'
+      returnStructuredOutput: true
+    prompt: 'Extract all names and addresses from this document'
+```
 
-### Structured Extraction
+## Programmatic Usage
 
-Note: use nullable() instead of optional() for fields that are not required.
+The SDK provides three main functions for document processing:
+
+1. **`parseDocumentAndExtractStructuredData`** - Parse and optionally extract structured data with AI
+
+   ```typescript
+   parseDocumentAndExtractStructuredData<T>({
+     documentSource: string,                  // Required: URL or file path
+     sessionId: string,                       // Required: Session identifier
+     returnStructuredOutput: boolean,         // Required: Enable/disable AI extraction
+     llm?: BaseLanguageModel,                 // Optional: LLM instance (required when returnStructuredOutput is true)
+     outputSchema?: ZodType<T>,               // Optional: Zod schema for structured extraction
+     outputSchemaPrompt?: string,             // Optional: Instructions for extraction
+     processingMode?: DocumentProcessingMode, // Optional: Processing mode (default: DocumentProcessingMode.MANAGED)
+     llamaCloudApiKey?: string                // Optional: API key for local mode
+   }): Promise<{
+     rawContent?: string,
+     structuredContent?: T | string,
+     metadata?: { fileSize?: number, fileType: string, processingTime: number, contentLength: number }
+   }>
+   ```
+
+2. **`parseDocument`** - Parse document and extract raw text only
+
+   ```typescript
+   parseDocument({
+     documentSource: string,                  // Required: URL or file path
+     sessionId: string,                       // Required: Session identifier
+     processingMode?: DocumentProcessingMode, // Optional: Processing mode (default: DocumentProcessingMode.MANAGED)
+     llamaCloudApiKey?: string                // Optional: API key for local mode
+   }): Promise<{
+     rawContent?: string,
+     metadata?: { fileSize?: number, fileType: string, processingTime: number, contentLength: number }
+   }>
+   ```
+
+3. **`extractStructuredDataFromString`** - Extract structured data from already parsed text
+
+   ```typescript
+   extractStructuredDataFromString<T>({
+     content: string,                  // Required: Text content to extract from
+     llm: BaseLanguageModel,           // Required: LLM instance
+     sessionId: string,                // Required: Session identifier
+     schema?: ZodType<T>,              // Optional: Zod schema for structured extraction
+     prompt?: string                   // Optional: Instructions for extraction
+   }): Promise<T | string>
+   ```
+
+### Structured Extraction with Schema
+
+Extract data matching a predefined Zod schema.
+
+{% hint style="info" %}
+**Note:** Use `nullable()` instead of `optional()` for fields that are not required.
+{% endhint %}
 
 ```typescript
-import { extractFromDocument } from '@minded-ai/mindedjs';
+import { parseDocumentAndExtractStructuredData } from '@minded-ai/mindedjs';
 import { z } from 'zod';
 
-const schema = z.object({
-  name: z.string(),
-  email: z.string(),
-  phoneNumber: z.string().nullable(),
+// Invoice processing with structured schema
+const invoiceSchema = z.object({
+  invoiceNumber: z.string(),
+  vendor: z.string(),
+  amount: z.number(),
+  dueDate: z.string(),
+  lineItems: z.array(
+    z.object({
+      description: z.string(),
+      quantity: z.number(),
+      unitPrice: z.number(),
+    }),
+  ),
 });
 
-const result = await extractFromDocument({
+const result = await parseDocumentAndExtractStructuredData({
+  documentSource: './invoice.pdf',
+  sessionId: state.sessionId,
+  returnStructuredOutput: true,
   llm: agent.llm,
-  documentPath: './id-card.jpg',
-  schema,
-  systemPrompt: 'Extract personal information from this ID document',
+  outputSchema: invoiceSchema,
+  outputSchemaPrompt: 'Extract all invoice details including line items',
 });
 
-console.log(result.data.name); // Typed data matching your schema
+console.log(result.structuredContent); // Typed data matching your schema
+console.log(result.rawContent); // Original raw text
+console.log(result.metadata); // Processing metadata
 ```
 
-### Unstructured Extraction
+### Structured Extraction with Prompt Only
+
+Extract data using AI with a custom prompt, without a predefined schema.
 
 ```typescript
-const result = await extractFromDocument({
+import { parseDocumentAndExtractStructuredData } from '@minded-ai/mindedjs';
+
+// Contract analysis with prompt guidance
+const result = await parseDocumentAndExtractStructuredData({
+  documentSource: './contract.pdf',
+  sessionId: state.sessionId,
+  returnStructuredOutput: true,
   llm: agent.llm,
-  documentPath: './contract.pdf',
-  systemPrompt: 'Extract parties involved, key dates, payment terms, and termination clauses',
+  outputSchemaPrompt: 'Extract parties involved, key dates, payment terms, and termination clauses',
 });
 
-console.log(result.data); // String with extracted information
+console.log(result.structuredContent); // String or object with extracted information
 ```
 
 ### Raw Text Extraction
 
+Parse document without AI processing to get plain text only.
+
 ```typescript
-const result = await extractFromDocument({
-  documentPath: './document.pdf',
-  config: {
-    llamaCloudApiKey: process.env.LLAMA_CLOUD_API_KEY,
-  },
+import { parseDocument } from '@minded-ai/mindedjs';
+
+const result = await parseDocument({
+  documentSource: './document.pdf',
+  sessionId: state.sessionId,
 });
 
-console.log(result.data); // Raw text content
+console.log(result.rawContent); // Raw extracted text
+console.log(result.metadata); // File size, type, processing time, content length
 ```
 
 ### Using URLs
 
+All functions accept URLs in addition to file paths via the `documentSource` parameter.
+
 ```typescript
-const result = await extractFromDocument({
+import { parseDocumentAndExtractStructuredData } from '@minded-ai/mindedjs';
+import { z } from 'zod';
+
+const result = await parseDocumentAndExtractStructuredData({
+  documentSource: 'https://example.com/invoice.pdf',
+  sessionId: state.sessionId,
+  returnStructuredOutput: true,
   llm: agent.llm,
-  documentUrl: 'https://example.com/invoice.pdf',
-  schema: z.object({
+  outputSchema: z.object({
     invoiceNumber: z.string(),
     amount: z.number(),
     dueDate: z.string(),
@@ -134,138 +245,88 @@ const result = await extractFromDocument({
 });
 ```
 
-## Configuration
-
-### Environment Variables
-
-```bash
-# Processing mode: 'managed' (default) or 'local'
-DOCUMENT_PROCESSING_MODE=managed
-
-# Required for local mode only:
-LLAMA_CLOUD_API_KEY=your_llama_cloud_api_key
-```
-
-### Document Processor Options
-
-```typescript
-const result = await extractFromDocument({
-  llm: agent.llm,
-  documentPath: './image.jpg',
-  schema: yourSchema,
-  config: {
-    llamaCloudApiKey: 'your-key',
-    useBase64: true,
-    maxImageWidth: 1600,
-    imageQuality: 90,
-  },
-});
-```
-
-### LLM Configuration
-
-```typescript
-const result = await extractFromDocument({
-  llm: agent.llm,
-  documentPath: './document.pdf',
-  schema: yourSchema,
-  llmConfig: {
-    model: 'gpt-4o',
-    temperature: 0.1,
-  },
-});
-```
-
-## Example Use Cases
+### Identity Document Verification
 
-### Invoice Processing
+Extract personal information from ID documents with structured validation.
 
 ```typescript
-const invoiceSchema = z.object({
-  invoiceNumber: z.string(),
-  vendor: z.string(),
-  amount: z.number(),
-  dueDate: z.string(),
-  lineItems: z.array(
-    z.object({
-      description: z.string(),
-      quantity: z.number(),
-      unitPrice: z.number(),
-    }),
-  ),
-});
-
-const result = await extractFromDocument({
-  llm: agent.llm,
-  documentPath: './invoice.pdf',
-  schema: invoiceSchema,
-  systemPrompt: 'Extract all invoice details including line items',
-});
-```
-
-### Identity Document Verification
+import { parseDocumentAndExtractStructuredData } from '@minded-ai/mindedjs';
+import { z } from 'zod';
 
-```typescript
 const idSchema = z.object({
   documentType: z.enum(['passport', 'driver_license', 'national_id']),
   fullName: z.string(),
   dateOfBirth: z.string(),
   documentNumber: z.string(),
   expiryDate: z.string().nullable(),
+  address: z.string().nullable(),
 });
 
-const result = await extractFromDocument({
+const result = await parseDocumentAndExtractStructuredData({
+  documentSource: './id-card.jpg',
+  sessionId: state.sessionId,
+  returnStructuredOutput: true,
   llm: agent.llm,
-  documentPath: './id-document.jpg',
-  schema: idSchema,
+  outputSchema: idSchema,
+  outputSchemaPrompt: 'Extract personal information from this ID document',
 });
 ```
 
-### Extract Names and Addresses
+### Extract from Already Parsed Content
+
+Use `extractStructuredDataFromString` to extract structured data from text you've already obtained.
 
 ```typescript
-const contactsSchema = z.object({
-  contacts: z.array(
-    z.object({
-      name: z.string(),
-      address: z.string(),
-    }),
-  ),
+import { extractStructuredDataFromString } from '@minded-ai/mindedjs';
+import { z } from 'zod';
+
+// With schema for structured extraction
+const structured = await extractStructuredDataFromString({
+  content: 'Invoice #12345\nTotal: $500.00\nDate: 2024-01-15',
+  llm: agent.llm,
+  schema: z.object({
+    invoiceNumber: z.string(),
+    totalAmount: z.number(),
+    date: z.string(),
+  }),
+  sessionId: state.sessionId,
 });
+// Returns: {invoiceNumber: "12345", totalAmount: 500, date: "2024-01-15"}
 
-const result = await extractFromDocument({
+// Without schema for unstructured extraction
+const unstructured = await extractStructuredDataFromString({
+  content: 'Invoice #12345\nTotal: $500.00\nDate: 2024-01-15',
   llm: agent.llm,
-  documentPath: './document.pdf',
-  schema: contactsSchema,
-  systemPrompt: 'Extract all names and addresses from this document',
+  prompt: 'Extract the invoice number, total amount, and date',
+  sessionId: state.sessionId,
 });
+// Returns: String or object with LLM's analysis
 ```
 
-## Troubleshooting
+### Local Processing Mode
 
-**PDF Processing Fails**
+By default, document processing uses the Minded cloud service. To process documents locally using LlamaCloud, set the `DOCUMENT_PROCESSING_MODE` environment variable and provide a LlamaCloud API key:
 
-```
-Error: PDF processing requires LLAMA_CLOUD_API_KEY
-```
-
-Solution (local mode only): Set the `LLAMA_CLOUD_API_KEY` environment variable.
-
-**Image Too Large**
+```bash
+# Processing mode: 'managed' (default) or 'local'
+DOCUMENT_PROCESSING_MODE=local
 
-```
-Error: Image processing failed: Input image exceeds pixel limit
+# Required for local mode:
+LLAMA_CLOUD_API_KEY=your_llama_cloud_api_key
 ```
 
-Solution: Reduce `maxImageWidth` in configuration.
+You can also pass the API key and processing mode directly to the function:
 
-**Schema Validation Errors**
+```typescript
+import { parseDocument, DocumentProcessingMode } from '@minded-ai/mindedjs';
 
+const result = await parseDocument({
+  documentSource: './contract.pdf',
+  processingMode: DocumentProcessingMode.LOCAL,
+  sessionId: state.sessionId,
+  llamaCloudApiKey: process.env.LLAMA_CLOUD_API_KEY,
+});
 ```
-Error: LLM extraction failed: Invalid schema
-```
-
-Solution: Verify your Zod schema matches the expected data structure.
 
 ## Supported File Types
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "3.0.8-beta.12",
+  "version": "3.1.9-beta.1",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -43,6 +43,7 @@
     "globals": "^16.2.0",
     "mocha": "^10.3.0",
     "nodemon": "^3.1.10",
+    "prettier": "^3.7.4",
     "sinon": "^20.0.0",
     "ts-node": "^10.9.2",
     "typedoc": "^0.28.5",

package/src/cli/index.ts

@@ -181,17 +181,9 @@ async function main() {
     const agentPath = args[1]; // Optional path argument
     runValidateCommand(agentPath);
   } else if (command === 'run') {
-    // Command to run the agent with inline JSON input
-    // Usage: npx minded run <json_input>
+    // Command to run the agent with optional inline JSON input
+    // Usage: npx minded run [json_input]
     const jsonInput = args[1];
-    if (!jsonInput) {
-      logger.error({ msg: 'Please provide inline JSON input' });
-      console.error('\nUsage:');
-      console.error('  npx minded run \'{"body": {...}}\'');
-      console.error('  npx minded run \'{"body": {...}, "headers": {...}}\'');
-      process.exit(1);
-    }
-
     await runCommand(jsonInput);
   } else {
     logger.error({

package/src/cli/runCommand.ts

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
 import * as fs from 'fs';
 import * as path from 'path';
 import { z } from 'zod';
@@ -9,7 +10,7 @@ import { logger } from '../utils/logger';
  * - headers: Optional, HTTP-like headers
  */
 const runInputSchema = z.object({
-  body: z.any(),
+  body: z.record(z.any()),
   headers: z.record(z.string()).optional(),
 });
 
@@ -107,7 +108,7 @@ async function loadAgent(projectDir: string, agentPath: string): Promise<any> {
 /**
  * Main run command handler
  */
-export async function runCommand(jsonInput: string): Promise<void> {
+export async function runCommand(jsonInput?: string): Promise<void> {
   const projectDir = process.cwd();
   const mindedConfigPath = path.join(projectDir, 'minded.json');
 
@@ -134,31 +135,36 @@ export async function runCommand(jsonInput: string): Promise<void> {
     process.exit(1);
   }
 
-  // Parse and validate the JSON input
-  let parsedInput: unknown;
-  try {
-    parsedInput = parseJsonInput(jsonInput);
-  } catch (err) {
-    logger.error({ message: 'Failed to parse input', err: err instanceof Error ? err.message : String(err) });
-    process.exit(1);
-  }
+  // Use default empty body if no input provided
+  let input: RunInput;
+  if (jsonInput) {
+    // Parse and validate the JSON input
+    let parsedInput: unknown;
+    try {
+      parsedInput = parseJsonInput(jsonInput);
+    } catch (err) {
+      logger.error({ message: 'Failed to parse input', err: err instanceof Error ? err.message : String(err) });
+      process.exit(1);
+    }
 
-  // Validate against schema
-  const validationResult = runInputSchema.safeParse(parsedInput);
-  if (!validationResult.success) {
-    logger.error({
-      msg: 'Invalid input schema',
-      errors: validationResult.error.errors.map((e) => `${e.path.join('.')}: ${e.message}`),
-    });
-    console.error('\nExpected schema:');
-    console.error('  {');
-    console.error('    "body": <any>,        // Required');
-    console.error('    "headers": {...}      // Optional');
-    console.error('  }');
-    process.exit(1);
+    // Validate against schema
+    const validationResult = runInputSchema.safeParse(parsedInput);
+    if (!validationResult.success) {
+      logger.error({
+        msg: 'Invalid input schema',
+        errors: validationResult.error.errors.map((e) => `${e.path.join('.')}: ${e.message}`),
+      });
+      console.error('\nExpected schema:');
+      console.error('  {');
+      console.error('    "body": <any>,        // Required');
+      console.error('    "headers": {...}      // Optional');
+      console.error('  }');
+      process.exit(1);
+    }
+    input = validationResult.data;
+  } else {
+    input = { body: {} };
   }
-
-  const input: RunInput = validationResult.data;
   const triggerName = 'cli';
 
   console.log(`\n🚀 Running agent with trigger: ${triggerName}\n`);

package/src/index.ts

@@ -71,7 +71,8 @@ export type { Tool, ToolExecuteInput } from './types/Tools.types';
 export { ToolType } from './types/Tools.types';
 
 // Document processing utilities
-export { DocumentProcessor, extractFromDocument } from './internalTools/documentExtraction/documentExtraction';
+export { parseDocumentAndExtractStructuredData, parseDocument, DocumentProcessingMode } from './internalTools/documentExtraction/documentExtraction';
+export { extractStructuredDataFromString } from './internalTools/documentExtraction/extractStructuredData';
 export type {
   DocumentProcessorConfig,
   DocumentExtractionOptions,