@minded-ai/mindedjs 3.0.8-beta.12 → 3.1.9-beta.1

package/src/internalTools/documentExtraction/parseDocumentManaged.ts

@@ -0,0 +1,152 @@
+import fsp from 'fs/promises';
+import path from 'path';
+import {
+  DocumentProcessRequest,
+  DocumentProcessResponse,
+  FileUploadRequest,
+  FileUploadResponse,
+  FileUploadType,
+  mindedConnectionSocketMessageType,
+} from '../../platform/mindedConnectionTypes';
+import { logger } from '../../utils/logger';
+import * as mindedConnection from '../../platform/mindedConnection';
+
+/**
+ * Process document using managed backend service
+ */
+export async function parseDocumentWithManagedService({
+  documentSource,
+  isDocumentUrl,
+  sessionId,
+}: {
+  isDocumentUrl: boolean;
+  documentSource: string;
+  sessionId: string;
+}): Promise<{ rawContent: string; metadata?: DocumentProcessResponse['metadata'] }> {
+  logger.info({
+    msg: 'Parsing document with managed service',
+    sessionId,
+    documentSource,
+    sourceType: isDocumentUrl ? 'url' : 'path',
+  });
+
+  let uploadId: string | undefined;
+  let documentUrl: string | undefined;
+
+  if (isDocumentUrl) {
+    documentUrl = documentSource;
+  } else {
+    // Upload file for processing
+    logger.info({
+      msg: 'Uploading document for parsing',
+      sessionId,
+    });
+
+    uploadId = await uploadDocumentForProcessing({
+      sessionId,
+      documentPath: documentSource,
+    });
+  }
+
+  logger.info({
+    msg: 'Requesting document parsing via socket',
+    sessionId,
+    documentSource,
+    hasUrl: !!documentUrl,
+    hasUploadId: !!uploadId,
+  });
+
+  // Use 2-minute timeout and maxRetries=1 for document processing
+  // No retries on timeout to avoid duplicate processing
+  const response = await mindedConnection.awaitEmit<DocumentProcessRequest, DocumentProcessResponse>(
+    mindedConnectionSocketMessageType.DOCUMENT_PROCESS,
+    {
+      type: mindedConnectionSocketMessageType.DOCUMENT_PROCESS,
+      sessionId,
+      documentUrl,
+      uploadId,
+    },
+    180000, // 2 minutes timeout
+    1, // No retries on timeout to prevent duplicate processing
+  );
+
+  if (!response.success) {
+    throw new Error(response.error || 'Failed to process document on backend');
+  }
+
+  logger.info({
+    msg: 'Got document parsing response',
+    sessionId,
+  });
+
+  return {
+    rawContent: response.rawContent || '',
+    metadata: response.metadata,
+  };
+}
+
+/**
+ * Upload document for processing
+ */
+async function uploadDocumentForProcessing({ sessionId, documentPath }: { sessionId: string; documentPath: string }): Promise<string> {
+  const fileBuffer = await fsp.readFile(documentPath);
+  const originalFileName = path.basename(documentPath);
+  const fileSize = fileBuffer.length;
+
+  logger.info({
+    msg: 'Requesting upload URL for document',
+    sessionId,
+    fileName: originalFileName,
+    fileSize,
+  });
+
+  const urlResponse = await mindedConnection.awaitEmit<FileUploadRequest, FileUploadResponse>(
+    mindedConnectionSocketMessageType.UPLOAD_FILE_REQUEST,
+    {
+      type: mindedConnectionSocketMessageType.UPLOAD_FILE_REQUEST,
+      sessionId,
+      fileName: originalFileName,
+      fileSize,
+      uploadType: FileUploadType.DOCUMENT_PROCESSING,
+    },
+    30000,
+  );
+
+  if (!urlResponse.success || !urlResponse.uploadUrl || !urlResponse.uploadId) {
+    throw new Error(urlResponse.error || 'Failed to generate upload URL');
+  }
+
+  logger.info({
+    msg: 'Received upload URL, uploading document',
+    sessionId,
+    uploadId: urlResponse.uploadId,
+    expiresIn: urlResponse.expiresIn,
+  });
+
+  const urlObj = new URL(urlResponse.uploadUrl);
+  const signedHeaders = urlObj.searchParams.get('X-Amz-SignedHeaders');
+  const headers: Record<string, string> = {};
+
+  if (signedHeaders?.includes('x-amz-server-side-encryption')) {
+    headers['x-amz-server-side-encryption'] = 'aws:kms';
+  }
+
+  const uploadResponse = await fetch(urlResponse.uploadUrl, {
+    method: 'PUT',
+    headers,
+    body: new Uint8Array(fileBuffer),
+  });
+
+  if (!uploadResponse.ok) {
+    const errorText = await uploadResponse.text();
+    throw new Error(`Failed to upload document: ${uploadResponse.status} - ${errorText}`);
+  }
+
+  logger.info({
+    msg: 'Successfully uploaded document',
+    sessionId,
+    uploadId: urlResponse.uploadId,
+  });
+
+  return urlResponse.uploadId;
+}

package/src/nodes/addAppToolNode.ts

@@ -94,12 +94,15 @@ export const addAppToolNode = async ({
       logger.debug({
         message: '[Node] Omitting parameters from system prompt due to length',
         node: node.name,
-        parameterLengths: Object.entries(compiledParameters).reduce((acc, [key, value]) => {
-          if (typeof value === 'string') {
-            acc[key] = value.length;
-          }
-          return acc;
-        }, {} as Record<string, number>),
+        parameterLengths: Object.entries(compiledParameters).reduce(
+          (acc, [key, value]) => {
+            if (typeof value === 'string') {
+              acc[key] = value.length;
+            }
+            return acc;
+          },
+          {} as Record<string, number>,
+        ),
       });
       parametersString = '[Parameters omitted - one or more values exceed 1000 characters]';
     } else {
@@ -145,12 +148,32 @@ export const addAppToolNode = async ({
     if (toolCallMessage instanceof ToolMessage) {
       state.messages.push(toolCallMessage);
     }
+
+    // Try to parse content if it's a JSON string and extract the result object
+    let parsedContent: any;
+    if (toolCallMessage instanceof ToolMessage) {
+      parsedContent = toolCallMessage.content;
+      try {
+        if (typeof toolCallMessage.content === 'string') {
+          parsedContent = JSON.parse(toolCallMessage.content);
+        }
+      } catch {
+        // If parsing fails, use the original content
+      }
+      if ('result' in parsedContent) {
+        parsedContent = parsedContent.result;
+      }
+    }
+
     state.history.push(
       createHistoryStep<AppActionInvocationHistoryStep>(state.history, {
         type: NodeType.APP_TOOL,
         nodeId: node.name,
         nodeDisplayName: node.displayName!,
-        raw: AIToolCallMessage.tool_calls[0],
+        raw: {
+          ...AIToolCallMessage.tool_calls[0],
+          result: parsedContent,
+        },
         appName: node.appName,
         messageIds: [AIToolCallMessage.tool_calls[0].id],
       }),

package/src/toolsLibrary/parseDocument.ts

@@ -1,109 +1,67 @@
 import { z } from 'zod';
 import { Tool } from '../types/Tools.types';
-import { extractFromDocument } from '../internalTools/documentExtraction/documentExtraction';
+import { parseDocumentAndExtractStructuredData, DocumentProcessingMode } from '../internalTools/documentExtraction/documentExtraction';
 import { logger } from '../utils/logger';
-import * as fs from 'fs';
-import * as path from 'path';
-import {
-  FileUploadRequest,
-  FileUploadResponse,
-  FileUploadType,
-  DocumentProcessRequest,
-  DocumentProcessResponse,
-  mindedConnectionSocketMessageType,
-} from '../platform/mindedConnectionTypes';
-import * as mindedConnection from '../platform/mindedConnection';
+import { NodeType } from '../types/Flows.types';
+import { createZodSchemaFromFields } from '../utils/schemaUtils';
 
-/**
- * Check if a string is a URL
- */
-function isUrl(source: string): boolean {
-  return source.startsWith('http://') || source.startsWith('https://');
-}
-
-// Schema for the tool
 export const schema = z.object({
-  // Document source - can be a URL or file path (auto-detected)
   documentSource: z.string().describe('URL or file path to the document to parse'),
-
-  // Processing parameters
-  extractRaw: z.boolean().optional().nullable().describe('Extract raw text without AI processing'),
-  schema: z.any().optional().nullable().describe('Zod schema for structured data extraction'),
-  systemPrompt: z.string().optional().nullable().describe('Prompt for guiding extraction'),
+  returnStructuredOutput: z.boolean().optional().default(false).describe('Whether to return structured output'),
 });
 
-const parseDocumentTool: Tool<typeof schema, any> = {
+/**
+ * Document parsing tool for flows.
+ * Parses documents and optionally extracts structured data using AI.
+ *
+ * - Raw mode: Returns plain text from document
+ * - Structured mode: Uses node's prompt and outputSchema for AI extraction
+ */
+const parseDocumentTool: Tool<typeof schema> = {
   name: 'minded-parse-documents',
   description:
     'Parse and extract data from documents (PDFs, images, Word docs, etc.). Provide a URL or file path and optionally a schema or prompt for extraction. Includes built-in AI extraction - no separate extraction tool needed.',
   input: schema,
   isGlobal: false,
   execute: async ({ input, state, agent }) => {
-    const combinedInput = input as z.infer<typeof schema>;
-
     try {
-      if (!combinedInput.documentSource) {
+      if (!input.documentSource) {
         throw new Error('documentSource is required - provide a URL or file path');
       }
 
-      const isDocumentUrl = isUrl(combinedInput.documentSource);
-      let rawContent: string;
+      const node = await agent.getCurrentNode(state.sessionId);
+      const appToolNode = node?.type === NodeType.APP_TOOL ? node : undefined;
 
       // Get processing mode from environment variable
-      const processingMode = process.env.DOCUMENT_PROCESSING_MODE === 'local' ? 'local' : 'managed';
-
-      if (processingMode === 'managed') {
-        logger.info({
-          msg: 'Using managed document processing',
-          sessionId: state.sessionId,
-          sourceType: isDocumentUrl ? 'url' : 'path',
-        });
-
-        const result = await processManagedDocument({
-          sessionId: state.sessionId,
-          documentSource: combinedInput.documentSource,
-          isUrl: isDocumentUrl,
-        });
-
-        rawContent = result.rawContent;
-      } else {
-        logger.info({
-          msg: 'Using local document processing',
-          sessionId: state.sessionId,
-          sourceType: isDocumentUrl ? 'url' : 'path',
-        });
-
-        const extractionOptions: Parameters<typeof extractFromDocument>[0] = isDocumentUrl
-          ? { documentUrl: combinedInput.documentSource }
-          : { documentPath: combinedInput.documentSource };
-
-        const localResult = await extractFromDocument(extractionOptions);
-        rawContent = localResult.data as string;
-      }
-
-      // Process with LLM if needed
-      let finalData: any = rawContent;
-
-      if (!combinedInput.extractRaw && agent.llm) {
-        const llmResult = await extractFromDocument({
-          documentContent: rawContent,
-          llm: agent.llm,
-          schema: combinedInput.schema,
-          systemPrompt: combinedInput.systemPrompt || undefined,
-        });
-
-        finalData = llmResult.data;
-      }
+      const processingMode =
+        process.env.DOCUMENT_PROCESSING_MODE === 'local' ? DocumentProcessingMode.LOCAL : DocumentProcessingMode.MANAGED;
+      const llamaCloudApiKey: string | undefined = process.env.LLAMA_CLOUD_API_KEY;
+      const outputSchema =
+        input.returnStructuredOutput && agent.llm && appToolNode && appToolNode.outputSchema?.length
+          ? createZodSchemaFromFields(appToolNode.outputSchema)
+          : undefined;
+
+      const result = await parseDocumentAndExtractStructuredData({
+        documentSource: input.documentSource,
+        processingMode,
+        llamaCloudApiKey,
+        sessionId: state.sessionId,
+        llm: agent.llm,
+        returnStructuredOutput: input.returnStructuredOutput,
+        outputSchema: outputSchema,
+        outputSchemaPrompt: appToolNode?.prompt,
+      });
 
       state.memory.lastParsedDocument = {
-        source: combinedInput.documentSource,
+        source: input.documentSource,
         extractedAt: new Date().toISOString(),
-        extractedRaw: !!combinedInput.extractRaw,
+        returnStructuredOutput: input.returnStructuredOutput,
         processingMode,
+        structuredContent: result.structuredContent,
       };
 
       return {
-        result: finalData,
+        result: result.structuredContent ?? result.rawContent,
       };
     } catch (err) {
       logger.error({
@@ -113,6 +71,7 @@ const parseDocumentTool: Tool<typeof schema, any> = {
       });
 
       state.memory.documentParsingError = err instanceof Error ? err.message : String(err);
+
       return {
         result: `Failed to parse document: ${err instanceof Error ? err.message : String(err)}`,
       };
@@ -120,131 +79,4 @@ const parseDocumentTool: Tool<typeof schema, any> = {
   },
 };
 
-/**
- * Process document using managed backend service
- */
-async function processManagedDocument(params: {
-  sessionId: string;
-  documentSource: string;
-  isUrl: boolean;
-}): Promise<{ rawContent: string; metadata: any }> {
-  const { sessionId, documentSource, isUrl: isDocumentUrl } = params;
-
-  let uploadId: string | undefined;
-  let documentUrl: string | undefined;
-
-  if (isDocumentUrl) {
-    documentUrl = documentSource;
-  } else {
-    // Upload file for processing
-    logger.info({
-      msg: 'Uploading document for processing',
-      sessionId,
-    });
-
-    uploadId = await uploadDocumentForProcessing({
-      sessionId,
-      documentPath: documentSource,
-    });
-  }
-
-  logger.info({
-    msg: 'Requesting document processing via socket',
-    sessionId,
-    hasUrl: !!documentUrl,
-    hasUploadId: !!uploadId,
-  });
-
-  // Use 2-minute timeout and maxRetries=1 for document processing
-  // No retries on timeout to avoid duplicate processing
-  const response = await mindedConnection.awaitEmit<DocumentProcessRequest, DocumentProcessResponse>(
-    mindedConnectionSocketMessageType.DOCUMENT_PROCESS,
-    {
-      type: mindedConnectionSocketMessageType.DOCUMENT_PROCESS,
-      sessionId,
-      documentUrl,
-      uploadId,
-    },
-    180000, // 2 minutes timeout
-    1, // No retries on timeout to prevent duplicate processing
-  );
-
-  if (!response.success) {
-    throw new Error(response.error || 'Failed to process document on backend');
-  }
-
-  return {
-    rawContent: response.rawContent || '',
-    metadata: response.metadata,
-  };
-}
-
-/**
- * Upload document for processing
- */
-async function uploadDocumentForProcessing(params: { sessionId: string; documentPath: string }): Promise<string> {
-  const { sessionId, documentPath } = params;
-
-  const fileBuffer = fs.readFileSync(documentPath);
-  const originalFileName = path.basename(documentPath);
-  const fileSize = fileBuffer.length;
-
-  logger.info({
-    msg: 'Requesting upload URL for document',
-    sessionId,
-    fileName: originalFileName,
-    fileSize,
-  });
-
-  const urlResponse = await mindedConnection.awaitEmit<FileUploadRequest, FileUploadResponse>(
-    mindedConnectionSocketMessageType.UPLOAD_FILE_REQUEST,
-    {
-      type: mindedConnectionSocketMessageType.UPLOAD_FILE_REQUEST,
-      sessionId,
-      fileName: originalFileName,
-      fileSize,
-      uploadType: FileUploadType.DOCUMENT_PROCESSING,
-    },
-    30000,
-  );
-
-  if (!urlResponse.success || !urlResponse.uploadUrl || !urlResponse.uploadId) {
-    throw new Error(urlResponse.error || 'Failed to generate upload URL');
-  }
-
-  logger.info({
-    msg: 'Received upload URL, uploading document',
-    sessionId,
-    uploadId: urlResponse.uploadId,
-    expiresIn: urlResponse.expiresIn,
-  });
-
-  const urlObj = new URL(urlResponse.uploadUrl);
-  const signedHeaders = urlObj.searchParams.get('X-Amz-SignedHeaders');
-  const headers: Record<string, string> = {};
-
-  if (signedHeaders?.includes('x-amz-server-side-encryption')) {
-    headers['x-amz-server-side-encryption'] = 'aws:kms';
-  }
-
-  const uploadResponse = await fetch(urlResponse.uploadUrl, {
-    method: 'PUT',
-    headers,
-    body: new Uint8Array(fileBuffer),
-  });
-
-  if (!uploadResponse.ok) {
-    const errorText = await uploadResponse.text();
-    throw new Error(`Failed to upload document: ${uploadResponse.status} - ${errorText}`);
-  }
-
-  logger.info({
-    msg: 'Successfully uploaded document',
-    sessionId,
-    uploadId: urlResponse.uploadId,
-  });
-
-  return urlResponse.uploadId;
-}
-
 export default parseDocumentTool;

package/src/toolsLibrary/withBrowserSession.ts

@@ -1,4 +1,4 @@
-import { Browser, BrowserContext, chromium, Page } from 'playwright';
+import { Browser, BrowserContext, chromium, Locator, Page } from 'playwright';
 import { BrowserTaskMode } from '../browserTask/types';
 import { createBrowserSession, destroyBrowserSession } from '../browserTask/executeBrowserTask';
 import { logger } from '../utils/logger';
@@ -170,6 +170,50 @@ class LogsCapture {
   }
 }
 
+/**
+ * Create a proxy locator that intercepts actions and captures screenshots and logs
+ */
+const createProxyLocator = (
+  locator: Locator,
+  page: Page,
+  screenshotCapture: ScreenshotCapture,
+  logsCapture: LogsCapture,
+  selector?: string,
+): Locator => {
+  // List of methods to intercept on locators
+  const locatorMethodsToIntercept = ['click', 'fill', 'type', 'selectOption', 'check', 'uncheck', 'setInputFiles'];
+
+  const proxyLocator = new Proxy(locator, {
+    get(target, prop, receiver) {
+      const originalValue = Reflect.get(target, prop, receiver);
+
+      // If it's one of the methods we want to intercept
+      if (typeof prop === 'string' && locatorMethodsToIntercept.includes(prop)) {
+        return async function (this: any, ...args: any[]) {
+          // Log the action (this will trigger immediate upload)
+          const actionDescription = formatLocatorActionLog(prop, selector, args);
+          await logsCapture.addLog(actionDescription);
+
+          // Capture "before" screenshot
+          await screenshotCapture.captureScreenshot(page, 'before');
+
+          // Call the original method
+          const result = await originalValue.apply(target, args);
+
+          // Capture "after" screenshot
+          await screenshotCapture.captureScreenshot(page, 'after');
+
+          return result;
+        };
+      }
+
+      return originalValue;
+    },
+  });
+
+  return proxyLocator;
+};
+
 /**
  * Create a proxy page that intercepts actions and captures screenshots and logs
  */
@@ -179,13 +223,25 @@ const createProxyPage = (
   logsCapture: LogsCapture,
 ): Page => {
   // List of methods to intercept for screenshot capture and logging
-  const methodsToIntercept = ['click', 'fill', 'type', 'goto', 'selectOption', 'check', 'uncheck', 'setInputFiles'];
+  const methodsToIntercept = ['click', 'fill', 'type', 'goto', 'selectOption', 'check', 'uncheck', 'setInputFiles', 'waitForTimeout', 'evaluate'];
+  // Methods that return locators and need special handling
+  const locatorMethods = ['locator'];
 
   const proxyPage = new Proxy(page, {
     get(target, prop, receiver) {
       const originalValue = Reflect.get(target, prop, receiver);
 
-      // If it's one of the methods we want to intercept
+      // Handle locator method - return a proxied locator
+      if (typeof prop === 'string' && locatorMethods.includes(prop)) {
+        return function (this: any, ...args: any[]) {
+          const locator = originalValue.apply(target, args);
+          // Extract selector from args if available (first argument is usually the selector)
+          const selector = args[0] && typeof args[0] === 'string' ? args[0] : undefined;
+          return createProxyLocator(locator, target, screenshotCapture, logsCapture, selector);
+        };
+      }
+
+      // If it's one of the methods we want to intercept with screenshots
       if (typeof prop === 'string' && methodsToIntercept.includes(prop)) {
         return async function (this: any, ...args: any[]) {
           // Log the action (this will trigger immediate upload)
@@ -216,7 +272,6 @@ const createProxyPage = (
  * Format action logs for display
  */
 const formatActionLog = (action: string, args: any[]): string => {
-
   switch (action) {
     case 'goto':
       return `Navigate to: ${args[0]}`;
@@ -234,11 +289,41 @@ const formatActionLog = (action: string, args: any[]): string => {
       return `Uncheck: ${args[0]}`;
     case 'setInputFiles':
       return `Set files for "${args[0]}"`;
+    case 'waitForTimeout':
+      return `Wait for timeout: ${args[0]}ms`;
+    case 'evaluate':
+      return `Evaluate: ${typeof args[0] === 'function' ? 'function' : String(args[0]).substring(0, 100)}`;
     default:
       return `${action}: ${JSON.stringify(args[0])}`;
   }
 };
 
+/**
+ * Format locator action logs for display
+ */
+const formatLocatorActionLog = (action: string, selector: string | undefined, args: any[]): string => {
+  const locatorDescription = selector || 'locator';
+
+  switch (action) {
+    case 'click':
+      return `Click locator: ${locatorDescription}`;
+    case 'fill':
+      return `Fill locator "${locatorDescription}" with: ${args[0]}`;
+    case 'type':
+      return `Type in locator "${locatorDescription}": ${args[0]}`;
+    case 'selectOption':
+      return `Select option in locator "${locatorDescription}": ${JSON.stringify(args[0])}`;
+    case 'check':
+      return `Check locator: ${locatorDescription}`;
+    case 'uncheck':
+      return `Uncheck locator: ${locatorDescription}`;
+    case 'setInputFiles':
+      return `Set files for locator "${locatorDescription}"`;
+    default:
+      return `Locator ${action}: ${JSON.stringify(args[0])}`;
+  }
+};
+
 // Helper function to load cookies for a session
 const loadCookiesForSession = async (context: BrowserContext, sessionId: string): Promise<void> => {
   const stored = await cookieStore.load();

package/src/types/Flows.types.ts

@@ -250,6 +250,7 @@ export interface AppToolNode extends BaseNode, BaseAppNode {
   actionName: string;
   metadata: AppNodeMetadata;
   actionKey: string;
+  outputSchema?: OutputSchemaField[];
 }
 
 export interface AppToolNodeMindedMetadata {

package/src/utils/schemaUtils.ts

@@ -54,7 +54,7 @@ export function createZodSchemaFromFields(
 
       // Handle optional fields
       if (field.required === false) {
-        fieldSchema = fieldSchema.optional();
+        fieldSchema = fieldSchema.optional().nullable();
       }
 
       schemaFields[field.name] = fieldSchema;