@minded-ai/mindedjs 2.0.28 → 2.0.29-beta.1

package/docs/self-hosting/examples/trust-policy-cross-account.json

@@ -0,0 +1,17 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "AWS": "arn:aws:iam::MINDED_AWS_ACCOUNT:root"
+      },
+      "Action": "sts:AssumeRole",
+      "Condition": {
+        "StringEquals": {
+          "sts:ExternalId": "YOUR_EXTERNAL_ID"
+        }
+      }
+    }
+  ]
+}

package/docs/self-hosting/examples/trust-policy-irsa.json

@@ -0,0 +1,22 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": {
+        "Federated": "arn:aws:iam::YOUR_ACCOUNT_ID:oidc-provider/oidc.eks.REGION.amazonaws.com/id/OIDC_ID"
+      },
+      "Action": "sts:AssumeRoleWithWebIdentity",
+      "Condition": {
+        "StringEquals": {
+          "oidc.eks.REGION.amazonaws.com/id/OIDC_ID:sub": [
+            "system:serviceaccount:dev:onprem-agent",
+            "system:serviceaccount:stg:onprem-agent",
+            "system:serviceaccount:prod:onprem-agent"
+          ],
+          "oidc.eks.REGION.amazonaws.com/id/OIDC_ID:aud": "sts.amazonaws.com"
+        }
+      }
+    }
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@minded-ai/mindedjs",
-  "version": "2.0.28",
+  "version": "2.0.29-beta.1",
   "description": "MindedJS is a TypeScript library for building agents.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/browserTask/executeBrowserTask.py

@@ -83,9 +83,25 @@ class ScreenshotCapture:
             
             logger.info(f"📸 Capturing screenshot #{step_number}")
             
+            # Wait for page to be in a stable state before taking screenshot
+            # This prevents hanging when page is still loading
+            try:
+                await asyncio.wait_for(
+                    page.wait_for_load_state('domcontentloaded'),
+                    timeout=5
+                )
+            except asyncio.TimeoutError:
+                logger.warning(f"⏱️ Page load state wait timed out for screenshot #{step_number}, attempting anyway...")
+            except Exception as e:
+                logger.warning(f"⚠️ Could not wait for page load state for screenshot #{step_number}: {str(e)}")
+            
             # Take screenshot - browser-use takes viewport by default
             # Set timeout to prevent hanging (30 seconds)
-            screenshot_data = await page.screenshot(timeout=30000)
+            try:
+                screenshot_data = await asyncio.wait_for(page.screenshot(), timeout=30)
+            except asyncio.TimeoutError:
+                logger.warning(f"⏱️ Screenshot #{step_number} timed out after 30 seconds, skipping...")
+                return
             
             # Ensure we have bytes - browser-use might return base64 string or bytes
             if isinstance(screenshot_data, bytes):

package/src/platform/deployment/Dockerfile.agent-org

@@ -0,0 +1,73 @@
+# syntax=docker/dockerfile:1.4
+
+# Your heavy, shared base (playwright, browsers, python, uv, etc.)
+# e.g., 123456789012.dkr.ecr.us-east-1.amazonaws.com/minded/agent-base:2025-10-01
+FROM 608342862701.dkr.ecr.us-east-1.amazonaws.com/base-agent:latest AS base
+WORKDIR /app
+
+# --------------------------------------------------------------------
+# Stage 1: builder — clone agent repo, install deps, build
+# --------------------------------------------------------------------
+FROM base AS builder
+
+ARG GIT_URL
+ARG GIT_REF=main
+# Optional: path within repo if the app lives in a subdir
+ARG APP_DIR=.
+# Optional: GitHub PAT for private repos
+ARG GH_PAT=
+
+# Clone the git repo
+RUN --mount=type=cache,target=/root/.npm \
+    bash -lc 'set -euo pipefail; \
+      # 1) Clone the repo (with auth if GH_PAT is provided)
+      if [ -n "${GH_PAT}" ]; then \
+        # For HTTPS URLs without auth, inject the PAT
+        AUTH_URL=$(echo ${GIT_URL} | sed "s|https://|https://${GH_PAT}@|"); \
+        git clone --depth 1 --branch ${GIT_REF} $AUTH_URL /src || \
+        (git clone $AUTH_URL /src && cd /src && git checkout ${GIT_REF}); \
+      else \
+        # Use GIT_URL as-is (may contain embedded auth credentials or be public)
+        git clone --depth 1 --branch ${GIT_REF} ${GIT_URL} /src || \
+        (git clone ${GIT_URL} /src && cd /src && git checkout ${GIT_REF}); \
+      fi; \
+      cd /src/${APP_DIR}; \
+      # 2) install deps deterministically (uses npm cache mount)
+      npm ci --no-audit --no-fund; \
+      # 3) build (ts->js) using cached npm tarballs
+      npm run build; \
+      # 4) prune dev deps for runtime
+      npm prune --omit=dev; \
+      # 5) stage the entire repo for runtime
+      mkdir -p /out; \
+      cp -r /src/${APP_DIR} /out/app; \
+    '
+
+# --------------------------------------------------------------------
+# Stage 2: runtime — thin layer with built app
+# --------------------------------------------------------------------
+FROM base AS runtime
+WORKDIR /app
+
+# non-root user (already fine if present in base; safe to repeat)
+RUN useradd -m -s /bin/bash agent || true
+
+# bring in the built artifacts
+COPY --from=builder /out/app /app
+
+# perms
+RUN chown -R agent:agent /app
+USER agent
+
+# runtime env is handled in K8s; keep only safe defaults here
+ENV NODE_ENV=production
+ENV PATH="/opt/venv/bin:$PATH"
+ENV PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright
+
+# Labels help trace back to the repo/commit (optional)
+ARG GIT_URL
+ARG GIT_REF
+LABEL org.opencontainers.image.source=$GIT_URL \
+      org.opencontainers.image.revision=$GIT_REF
+
+CMD ["node", "dist/src/agent.js"]

package/src/toolsLibrary/browser/extraction.ts

@@ -0,0 +1,393 @@
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { Tool } from '../../types/Tools.types';
+import { logger } from '../../utils/logger';
+import { JsonOutputParser } from '@langchain/core/output_parsers';
+import { SystemMessage } from '@langchain/core/messages';
+import { BaseLanguageModel } from '@langchain/core/language_models/base';
+
+// Type guard for checking if LLM supports structured output
+interface StructuredOutputLLM extends BaseLanguageModel {
+  withStructuredOutput<T extends z.ZodType>(schema: T): BaseLanguageModel;
+}
+
+function supportsStructuredOutput(llm: BaseLanguageModel): llm is StructuredOutputLLM {
+  return 'withStructuredOutput' in llm && typeof (llm as any).withStructuredOutput === 'function';
+}
+
+// Type definitions for extraction configuration using Zod
+export type ExtractionSchema = z.ZodTypeAny;
+
+export interface ExtractionConfig {
+  schema: ExtractionSchema;
+  systemPrompt?: string;
+  examples?: Array<{ input: string; output?: any }>;
+  defaultValue?: any;
+  maxRetries?: number;
+  strictMode?: boolean; // If true, will validate against schema
+}
+
+export type ExtractionResult = z.infer<ExtractionSchema>;
+
+// Default configuration
+const DEFAULT_CONFIG: Partial<ExtractionConfig> = {
+  maxRetries: 3,
+  strictMode: true,
+};
+
+/**
+ * Convert Zod schema to JSON schema format for prompt
+ */
+function schemaToJsonFormat(schema: ExtractionSchema): string {
+  // Use zod-to-json-schema for robust conversion
+  const jsonSchema = zodToJsonSchema(schema, {
+    target: 'openApi3',
+    $refStrategy: 'none', // Inline all references for simpler prompts
+  });
+
+  // Remove the $schema property for cleaner output
+  if (typeof jsonSchema === 'object' && jsonSchema !== null && '$schema' in jsonSchema) {
+    delete (jsonSchema as any).$schema;
+  }
+
+  return JSON.stringify(jsonSchema, null, 2);
+}
+
+/**
+ * Build prompt with schema for JSON parsing mode
+ */
+function buildPromptWithSchema(
+  basePrompt: string,
+  schema: ExtractionSchema,
+  examples: Array<{ input: string; output?: any }> | undefined,
+  content: string,
+): string {
+  const schemaDescription = schemaToJsonFormat(schema);
+  let prompt = `${basePrompt}\n\nExpected JSON format:\n${schemaDescription}\n\n`;
+
+  // Add examples if provided
+  if (examples && examples.length > 0) {
+    prompt += 'Examples:\n';
+    for (const example of examples) {
+      prompt += `Input: ${example.input}\nOutput: ${JSON.stringify(example.output)}\n\n`;
+    }
+  }
+
+  prompt += 'Return ONLY valid JSON that matches the schema. No additional text or explanation.\n\n';
+  prompt += `Content to extract from:\n${content}`;
+
+  return prompt;
+}
+
+/**
+ * Validate extraction result against Zod schema
+ */
+function validateResult(result: any, schema: ExtractionSchema): { valid: boolean; errors: string[] } {
+  try {
+    // Use Zod's built-in validation
+    schema.parse(result);
+    return { valid: true, errors: [] };
+  } catch (error) {
+    if (error instanceof z.ZodError) {
+      // Extract error messages from Zod validation
+      const errors = error.errors.map((err) => {
+        const path = err.path.join('.');
+        return path ? `${path}: ${err.message}` : err.message;
+      });
+      return { valid: false, errors };
+    }
+    // Handle non-Zod errors
+    return { valid: false, errors: [error instanceof Error ? error.message : 'Unknown validation error'] };
+  }
+}
+
+/**
+ * Generic extraction utility that can be used standalone
+ *
+ * When strictMode is true:
+ * - If the LLM supports withStructuredOutput, it will use that for guaranteed schema compliance
+ * - Otherwise, it falls back to JSON parsing with Zod validation and retries
+ *
+ * When strictMode is false:
+ * - Uses JSON parsing without validation for more flexibility
+ *
+ * @param content The content to extract from
+ * @param config The extraction configuration
+ * @param llm The language model to use for extraction
+ * @param attemptNumber Current attempt number (used internally for retries)
+ * @returns The extraction result
+ */
+export async function extract(
+  content: string,
+  config: ExtractionConfig,
+  llm: BaseLanguageModel,
+  attemptNumber: number = 0,
+): Promise<ExtractionResult> {
+  const mergedConfig = { ...DEFAULT_CONFIG, ...config };
+
+  try {
+    const basePrompt = mergedConfig.systemPrompt || 'Extract structured information from the given content:';
+
+    let result: any;
+
+    if (mergedConfig.strictMode) {
+      // Use structured output when strict mode is enabled
+      try {
+        // Check if the LLM supports structured output
+        if (supportsStructuredOutput(llm)) {
+          // When using withStructuredOutput, we don't need to include the schema in the prompt
+          // The LLM already knows the structure from the withStructuredOutput method
+          let prompt = basePrompt;
+
+          // Add examples if provided (still useful for context)
+          if (mergedConfig.examples && mergedConfig.examples.length > 0) {
+            prompt += '\n\nExamples:\n';
+            for (const example of mergedConfig.examples) {
+              prompt += `Input: ${example.input}\nOutput: ${JSON.stringify(example.output)}\n\n`;
+            }
+          }
+
+          prompt += `\n\nContent to extract from:\n${content}`;
+
+          const structuredLLM = llm.withStructuredOutput(mergedConfig.schema);
+          const messages = [new SystemMessage(prompt)];
+          result = await structuredLLM.invoke(messages);
+
+          // The result should already be validated by withStructuredOutput
+          return result as ExtractionResult;
+        } else {
+          // Fallback to JSON parser with validation if withStructuredOutput is not available
+          logger.info({
+            message: 'LLM does not support withStructuredOutput, falling back to JSON parser with validation',
+          });
+
+          // For JSON parsing mode, we need to include the schema in the prompt
+          const prompt = buildPromptWithSchema(basePrompt, mergedConfig.schema, mergedConfig.examples, content);
+
+          const parser = new JsonOutputParser();
+          const messages = [new SystemMessage(prompt)];
+          result = await llm.pipe(parser).invoke(messages);
+
+          // Validate the result
+          const validation = validateResult(result, mergedConfig.schema);
+
+          if (!validation.valid && attemptNumber < (mergedConfig.maxRetries || 3)) {
+            logger.warn({
+              message: 'Extraction validation failed, retrying',
+              errors: validation.errors,
+              attempt: attemptNumber + 1,
+            });
+
+            // Add validation errors to prompt for retry
+            const retryConfig = {
+              ...config,
+              systemPrompt: `${basePrompt}\n\nPrevious attempt had these errors:\n${validation.errors.join(
+                '\n',
+              )}\n\nPlease correct these issues and provide valid JSON.`,
+            };
+
+            return extract(content, retryConfig, llm, attemptNumber + 1);
+          } else if (!validation.valid) {
+            logger.error({
+              message: 'Extraction validation failed after max retries',
+              errors: validation.errors,
+            });
+
+            if (mergedConfig.defaultValue) {
+              return mergedConfig.defaultValue;
+            }
+
+            throw new Error(`Extraction validation failed: ${validation.errors.join(', ')}`);
+          }
+        }
+      } catch (err) {
+        // If structured output fails, retry with adjusted config
+        if (attemptNumber < (mergedConfig.maxRetries || 3)) {
+          logger.warn({
+            message: 'Structured output failed, retrying',
+            error: err instanceof Error ? err.message : 'Unknown error',
+            attempt: attemptNumber + 1,
+          });
+
+          return extract(content, config, llm, attemptNumber + 1);
+        } else {
+          throw err;
+        }
+      }
+    } else {
+      // Non-strict mode: use JSON parser without validation
+      const prompt = buildPromptWithSchema(basePrompt, mergedConfig.schema, mergedConfig.examples, content);
+
+      const parser = new JsonOutputParser();
+      const messages = [new SystemMessage(prompt)];
+      result = await llm.pipe(parser).invoke(messages);
+    }
+
+    return result as ExtractionResult;
+  } catch (err) {
+    logger.error({ message: 'Extraction failed', err });
+
+    if (mergedConfig.defaultValue) {
+      return mergedConfig.defaultValue;
+    }
+
+    throw err;
+  }
+}
+
+/**
+ * Create a simple extractor for common use cases
+ * @param schemaOrPrompt Either a Zod schema or a simple prompt string
+ * @param options Additional configuration options
+ * @returns A configured extract function
+ */
+export function createExtractor(schemaOrPrompt: ExtractionSchema | string, options?: Partial<ExtractionConfig>) {
+  let config: ExtractionConfig;
+
+  if (typeof schemaOrPrompt === 'string') {
+    // Simple prompt-based extraction - create a Zod schema for a single string result
+    config = {
+      schema: z.object({
+        result: z.string().describe('The extracted result'),
+      }),
+      systemPrompt: schemaOrPrompt,
+      ...options,
+    };
+  } else {
+    // Schema-based extraction
+    config = {
+      schema: schemaOrPrompt,
+      ...options,
+    };
+  }
+
+  return (content: string, llm: BaseLanguageModel) => extract(content, config, llm);
+}
+
+// Schema for the extraction tool
+export const schema = z.object({
+  content: z.string().describe('The content to extract information from'),
+  schema: z.record(z.any()).optional().describe('Extraction schema defining the structure of data to extract'),
+  systemPrompt: z.string().optional().describe('Custom system prompt for extraction'),
+  examples: z
+    .array(
+      z.object({
+        input: z.string(),
+        output: z.any(),
+      }),
+    )
+    .optional()
+    .describe('Examples to guide the extraction'),
+  strictMode: z.boolean().optional().default(true).describe('Whether to validate the extraction against the schema'),
+  maxRetries: z.number().optional().default(3).describe('Maximum number of retries if validation fails'),
+  defaultValue: z.any().optional().describe('Default value to return if extraction fails'),
+});
+
+const extractionTool: Tool<typeof schema, any> = {
+  name: 'minded-extraction',
+  description:
+    'Extract structured data from unstructured text using AI. Supports custom Zod schemas, validation, examples, and retries. When strictMode is enabled, uses LLM structured output (if available) for guaranteed schema compliance. Perfect for extracting specific information like country codes, emails, dates, or any custom data structure.',
+  input: schema,
+  isGlobal: false,
+  execute: async ({ input, state, agent }) => {
+    const { content, schema: inputSchema, systemPrompt, examples, strictMode, maxRetries, defaultValue } = input;
+
+    logger.info({
+      message: 'Extracting structured data',
+      sessionId: state.sessionId,
+      contentLength: content.length,
+      hasSchema: !!inputSchema,
+    });
+
+    // Convert input schema to Zod schema if provided
+    let extractionSchema: ExtractionSchema;
+    if (inputSchema) {
+      // Build a Zod schema from the input schema object
+      const zodShape: Record<string, z.ZodType> = {};
+
+      for (const [key, value] of Object.entries(inputSchema)) {
+        if (typeof value === 'object' && value !== null) {
+          const fieldConfig = value as any;
+          let zodType: z.ZodType;
+
+          // Create appropriate Zod type based on the field configuration
+          if (fieldConfig.type === 'string') {
+            zodType = z.string();
+          } else if (fieldConfig.type === 'number') {
+            zodType = z.number();
+          } else if (fieldConfig.type === 'boolean') {
+            zodType = z.boolean();
+          } else if (fieldConfig.type === 'array') {
+            zodType = z.array(z.any());
+          } else if (fieldConfig.type === 'object') {
+            zodType = z.object({});
+          } else {
+            // Default to string if type not specified
+            zodType = z.string();
+          }
+
+          // Add description if available
+          if (fieldConfig.description) {
+            zodType = zodType.describe(fieldConfig.description);
+          }
+
+          // Handle optional fields
+          if (fieldConfig.required === false) {
+            zodType = zodType.optional();
+          }
+
+          zodShape[key] = zodType;
+        } else {
+          // Default to string for simple values
+          zodShape[key] = z.string();
+        }
+      }
+
+      extractionSchema = z.object(zodShape);
+    } else {
+      // Default schema if none provided
+      extractionSchema = z.object({
+        result: z.string().describe('The extracted result'),
+      });
+    }
+
+    const config: ExtractionConfig = {
+      schema: extractionSchema,
+      systemPrompt,
+      examples,
+      strictMode,
+      maxRetries,
+      defaultValue,
+    };
+
+    try {
+      const result = await extract(content, config, agent.llm);
+
+      logger.info({
+        message: 'Extraction completed',
+        sessionId: state.sessionId,
+        resultKeys: Object.keys(result),
+      });
+
+      state.memory.lastExtraction = {
+        content: content.substring(0, 100) + (content.length > 100 ? '...' : ''),
+        result,
+        timestamp: new Date().toISOString(),
+      };
+
+      return {
+        result,
+      };
+    } catch (err) {
+      logger.error({
+        message: 'Extraction failed',
+        sessionId: state.sessionId,
+        err,
+      });
+
+      throw err;
+    }
+  },
+};
+
+export default extractionTool;