n8n-nodes-crawl4ai-plus 2.0.8 → 3.0.0

package/dist/nodes/Crawl4aiPlusBasicCrawler/actions/crawlMultipleUrls.operation.js

@@ -6,18 +6,282 @@ const n8n_workflow_1 = require("n8n-workflow");
 const utils_1 = require("../helpers/utils");
 const formatters_1 = require("../helpers/formatters");
 exports.description = [
+    {
+        displayName: 'Crawl Mode',
+        name: 'crawlMode',
+        type: 'options',
+        options: [
+            {
+                name: 'Manual URL List',
+                value: 'manual',
+                description: 'Provide an explicit list of URLs to crawl',
+            },
+            {
+                name: 'Discover From Seed URL',
+                value: 'discover',
+                description: 'Start from a homepage and recursively follow links matching your keywords (e.g. find all product pages, documentation sections, or contact info)',
+            },
+        ],
+        default: 'manual',
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+            },
+        },
+    },
+    {
+        displayName: 'Discovery Options',
+        name: 'discoveryOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+                crawlMode: ['discover'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Crawl Strategy',
+                name: 'crawlStrategy',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Best-First (Recommended)',
+                        value: 'BestFirstCrawlingStrategy',
+                        description: 'Visit highest-scoring pages first, regardless of depth. Best for finding most relevant content quickly. Requires query keywords to score pages.',
+                    },
+                    {
+                        name: 'Breadth-First Search (BFS)',
+                        value: 'BFSDeepCrawlStrategy',
+                        description: 'Explore all pages at each depth level before going deeper. Best for comprehensive coverage of nearby pages.',
+                    },
+                    {
+                        name: 'Depth-First Search (DFS)',
+                        value: 'DFSDeepCrawlStrategy',
+                        description: 'Follow links as deep as possible on each branch before backtracking. Best for focused deep exploration of specific paths.',
+                    },
+                ],
+                default: 'BestFirstCrawlingStrategy',
+                description: 'How the crawler explores links. Best-First prioritises relevance, BFS ensures breadth, DFS goes deep. Note: Best-First and DFS are validated but not officially tested by Crawl4AI.',
+            },
+            {
+                displayName: 'Discovery Query',
+                name: 'query',
+                type: 'string',
+                default: '',
+                placeholder: 'pricing features documentation',
+                description: 'Keywords that guide which links to follow. Use spaces for AND logic, "OR" for alternatives (e.g. "api documentation" finds pages about APIs and docs; "pricing OR plans" finds either).',
+            },
+            {
+                displayName: 'Exclude Domains',
+                name: 'excludeDomains',
+                type: 'string',
+                default: '',
+                placeholder: 'social.example.com, cdn.example.com',
+                description: 'Block specific domains even if external crawling is enabled. Useful for excluding CDNs, social media subdomains, etc.',
+            },
+            {
+                displayName: 'Exclude Patterns',
+                name: 'excludePatterns',
+                type: 'string',
+                default: '',
+                placeholder: '*/careers/*, */legal/*',
+                description: 'Skip URLs matching these patterns. Useful for avoiding careers, legal, login pages, etc.',
+            },
+            {
+                displayName: 'Include External Domains',
+                name: 'includeExternal',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to follow links to other domains. Leave OFF to stay on the same site, turn ON to follow external references.',
+            },
+            {
+                displayName: 'Include Patterns',
+                name: 'includePatterns',
+                type: 'string',
+                default: '',
+                placeholder: '*/products/*, */pricing/*',
+                description: 'Only follow URLs matching these patterns (wildcards supported). Example: "*/blog/*, */docs/*" only crawls blog and docs sections.',
+            },
+            {
+                displayName: 'Limit Returned Results',
+                name: 'resultLimit',
+                type: 'number',
+                default: 0,
+                description: 'Cap the number of results returned to n8n (0 = return all discovered pages). Useful for sampling or performance.',
+            },
+            {
+                displayName: 'Maximum Depth',
+                name: 'maxDepth',
+                type: 'number',
+                default: 2,
+                description: 'How many link levels deep to crawl. Depth 1 = only pages linked from seed; Depth 2 = seed + 1 more hop; etc. Range: 1-5. Lower is faster but less comprehensive.',
+            },
+            {
+                displayName: 'Maximum Pages',
+                name: 'maxPages',
+                type: 'number',
+                default: 50,
+                description: 'Hard limit on total pages crawled to prevent runaway crawls. Range: 1-200. Tip: Start with 20-50 for testing, increase if needed.',
+            },
+            {
+                displayName: 'Respect robots.txt',
+                name: 'respectRobotsTxt',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to check and respect robots.txt directives during discovery',
+            },
+            {
+                displayName: 'Score Threshold',
+                name: 'scoreThreshold',
+                type: 'number',
+                default: 0,
+                description: 'Minimum relevance score (0-1) for pages to be crawled. 0 = no threshold. Only used with BFS/DFS strategies; Best-First automatically prioritises high scores.',
+                displayOptions: {
+                    show: {
+                        crawlStrategy: ['BFSDeepCrawlStrategy', 'DFSDeepCrawlStrategy'],
+                    },
+                },
+            },
+            {
+                displayName: 'Seed URL',
+                name: 'seedUrl',
+                type: 'string',
+                default: '',
+                placeholder: 'https://example.com',
+                description: 'Starting point URL (usually homepage or main section). The crawler will follow links from here matching your query.',
+                required: true,
+            },
+        ],
+    },
+    {
+        displayName: 'Extraction Options',
+        name: 'extractionOptions',
+        type: 'collection',
+        placeholder: 'Add Extraction',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+                crawlMode: ['discover'],
+            },
+        },
+        description: 'Apply extraction strategies to discovered pages (shallow crawl with extraction)',
+        options: [
+            {
+                displayName: 'Base Selector',
+                name: 'baseSelector',
+                type: 'string',
+                default: '',
+                placeholder: 'article, .product-card',
+                description: 'CSS selector for the repeating container element',
+                displayOptions: {
+                    show: {
+                        enableExtraction: [true],
+                        extractionType: ['css'],
+                    },
+                },
+            },
+            {
+                displayName: 'Enable Extraction',
+                name: 'enableExtraction',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to apply an extraction strategy to each discovered page',
+            },
+            {
+                displayName: 'Extraction Instructions',
+                name: 'llmInstruction',
+                type: 'string',
+                typeOptions: {
+                    rows: 4,
+                },
+                default: '',
+                placeholder: 'Extract the main content, title, and any contact information from each page.',
+                description: 'Natural language instructions for the LLM on what to extract',
+                displayOptions: {
+                    show: {
+                        enableExtraction: [true],
+                        extractionType: ['llm'],
+                    },
+                },
+            },
+            {
+                displayName: 'Extraction Type',
+                name: 'extractionType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'CSS Selector Schema',
+                        value: 'css',
+                        description: 'Extract structured data using CSS selectors',
+                    },
+                    {
+                        name: 'LLM Extraction',
+                        value: 'llm',
+                        description: 'Extract data using LLM with natural language instructions (requires LLM credentials)',
+                    },
+                ],
+                default: 'css',
+                description: 'Type of extraction to apply to discovered pages',
+                displayOptions: {
+                    show: {
+                        enableExtraction: [true],
+                    },
+                },
+            },
+            {
+                displayName: 'Field Selectors (JSON)',
+                name: 'fieldSelectors',
+                type: 'string',
+                typeOptions: {
+                    rows: 6,
+                },
+                default: '[\n  {"name": "title", "selector": "h1, h2", "type": "text"},\n  {"name": "content", "selector": "p, .content", "type": "text"}\n]',
+                placeholder: '[{"name": "title", "selector": "h1", "type": "text"}]',
+                description: 'JSON array of field definitions: [{name, selector, type, attribute?}]. Type can be "text", "html", or "attribute".',
+                displayOptions: {
+                    show: {
+                        enableExtraction: [true],
+                        extractionType: ['css'],
+                    },
+                },
+            },
+            {
+                displayName: 'Schema (JSON)',
+                name: 'llmSchema',
+                type: 'string',
+                typeOptions: {
+                    rows: 8,
+                },
+                default: '{\n  "type": "object",\n  "properties": {\n    "title": {"type": "string"},\n    "summary": {"type": "string"}\n  }\n}',
+                placeholder: '{"type": "object", "properties": {"title": {"type": "string"}}}',
+                description: 'JSON schema defining the structure of data to extract',
+                displayOptions: {
+                    show: {
+                        enableExtraction: [true],
+                        extractionType: ['llm'],
+                    },
+                },
+            },
+        ],
+    },
     {
         displayName: 'URLs',
         name: 'urls',
         type: 'string',
-        required: true,
         default: '',
         placeholder: 'https://example.com, https://example.org',
-        description: 'Comma-separated list of URLs to crawl',
+        description: 'Comma-separated list of URLs to crawl. Required when using Manual URL List mode.',
         displayOptions: {
             show: {
                 operation: ['crawlMultipleUrls'],
             },
+            hide: {
+                crawlMode: ['discover'],
+            },
         },
     },
     {
@@ -46,6 +310,32 @@ exports.description = [
                 default: false,
                 description: 'Whether to enable stealth mode to bypass basic bot detection (hides webdriver properties and modifies browser fingerprints)',
             },
+            {
+                displayName: 'Extra Browser Arguments',
+                name: 'extraArgs',
+                type: 'fixedCollection',
+                typeOptions: {
+                    multipleValues: true,
+                },
+                default: {},
+                description: 'Additional command-line arguments to pass to the browser (advanced users only)',
+                options: [
+                    {
+                        name: 'args',
+                        displayName: 'Arguments',
+                        values: [
+                            {
+                                displayName: 'Argument',
+                                name: 'value',
+                                type: 'string',
+                                default: '',
+                                placeholder: '--disable-blink-features=AutomationControlled',
+                                description: 'Browser command-line argument (e.g., --disable-blink-features=AutomationControlled)',
+                            },
+                        ],
+                    },
+                ],
+            },
             {
                 displayName: 'Headless Mode',
                 name: 'headless',
@@ -84,6 +374,71 @@ exports.description = [
             },
         ],
     },
+    {
+        displayName: 'Session & Authentication',
+        name: 'sessionOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Cookies',
+                name: 'cookies',
+                type: 'json',
+                default: '',
+                placeholder: '[{"name": "session_id", "value": "abc123", "domain": ".example.com", "path": "/"}]',
+                description: 'Array of cookie objects to inject. Alternative to storage state for simple cookie-based auth.',
+            },
+            {
+                displayName: 'Storage State (JSON)',
+                name: 'storageState',
+                type: 'string',
+                typeOptions: {
+                    rows: 6,
+                },
+                default: '',
+                placeholder: '{"cookies": [...], "origins": [...]}',
+                description: 'Browser storage state as JSON (cookies, localStorage, sessionStorage). Captures authenticated session state. Works in all n8n environments.',
+            },
+            {
+                displayName: 'Use Managed Browser',
+                name: 'useManagedBrowser',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to use managed browser mode (required for persistent contexts). Advanced option.',
+                displayOptions: {
+                    show: {
+                        usePersistentContext: [true],
+                    },
+                },
+            },
+            {
+                displayName: 'Use Persistent Browser Context',
+                name: 'usePersistentContext',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to use a persistent browser context (requires user data directory). Only use in self-hosted environments with persistent storage.',
+            },
+            {
+                displayName: 'User Data Directory',
+                name: 'userDataDir',
+                type: 'string',
+                default: '',
+                placeholder: '/data/browser-profiles/profile1',
+                description: 'Path to browser profile directory for persistent sessions. Advanced: Only works in self-hosted n8n with persistent volumes. Use Storage State for cloud deployments.',
+                displayOptions: {
+                    show: {
+                        usePersistentContext: [true],
+                    },
+                },
+            },
+        ],
+    },
     {
         displayName: 'Crawler Options',
         name: 'crawlerOptions',
@@ -206,8 +561,8 @@ exports.description = [
         ],
     },
     {
-        displayName: 'Options',
-        name: 'options',
+        displayName: 'Output Options',
+        name: 'outputOptions',
         type: 'collection',
         placeholder: 'Add Option',
         default: {},
@@ -217,12 +572,78 @@ exports.description = [
             },
         },
         options: [
+            {
+                displayName: 'Capture Screenshot',
+                name: 'screenshot',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to capture a screenshot of each page (returned as base64)',
+            },
+            {
+                displayName: 'Fetch SSL Certificate',
+                name: 'fetchSslCertificate',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to retrieve SSL certificate information from each server',
+            },
+            {
+                displayName: 'Generate PDF',
+                name: 'pdf',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to generate a PDF of each page (returned as base64 or binary)',
+            },
+            {
+                displayName: 'Include Links',
+                name: 'includeLinks',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to include structured internal/external links data in output',
+            },
             {
                 displayName: 'Include Media Data',
                 name: 'includeMedia',
                 type: 'boolean',
                 default: false,
-                description: 'Whether to include media data in output (images, videos)',
+                description: 'Whether to include media data in output (images, videos, audios)',
+            },
+            {
+                displayName: 'Include Tables',
+                name: 'includeTables',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to include extracted tables in the output (if table extraction is enabled)',
+            },
+            {
+                displayName: 'Markdown Output',
+                name: 'markdownOutput',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Raw Markdown',
+                        value: 'raw',
+                        description: 'Return raw markdown (default, full content)',
+                    },
+                    {
+                        name: 'Filtered Markdown',
+                        value: 'fit',
+                        description: 'Return content-filtered markdown (cleaner, main content only)',
+                    },
+                    {
+                        name: 'Both',
+                        value: 'both',
+                        description: 'Return both raw and filtered markdown variants',
+                    },
+                ],
+                default: 'raw',
+                description: 'Which markdown variant(s) to return in the output',
+            },
+            {
+                displayName: 'Max Concurrent Crawls',
+                name: 'maxConcurrent',
+                type: 'number',
+                default: 5,
+                description: 'Maximum number of concurrent crawls',
             },
             {
                 displayName: 'Verbose Response',
@@ -231,49 +652,828 @@ exports.description = [
                 default: false,
                 description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
             },
+        ],
+    },
+    {
+        displayName: 'Content Filter',
+        name: 'contentFilter',
+        type: 'collection',
+        placeholder: 'Add Filter',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+            },
+        },
+        options: [
             {
-                displayName: 'Max Concurrent Crawls',
-                name: 'maxConcurrent',
+                displayName: 'BM25 Threshold',
+                name: 'bm25Threshold',
+                type: 'number',
+                default: 1.0,
+                displayOptions: {
+                    show: {
+                        filterType: ['bm25'],
+                    },
+                },
+                description: 'Minimum BM25 score threshold for content inclusion (default: 1.0)',
+            },
+            {
+                displayName: 'Chunk Token Threshold',
+                name: 'chunkTokenThreshold',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        filterType: ['llm'],
+                    },
+                },
+                default: 8192,
+                description: 'Maximum tokens per chunk for LLM processing (default: 8192, recommended: 4096-16384)',
+            },
+            {
+                displayName: 'Filter Type',
+                name: 'filterType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'None',
+                        value: 'none',
+                        description: 'No content filtering (return all content)',
+                    },
+                    {
+                        name: 'Pruning Filter',
+                        value: 'pruning',
+                        description: 'Remove low-value content using relevance thresholds',
+                    },
+                    {
+                        name: 'BM25 Filter',
+                        value: 'bm25',
+                        description: 'Filter content based on query relevance using BM25 algorithm',
+                    },
+                    {
+                        name: 'LLM Filter',
+                        value: 'llm',
+                        description: 'Intelligent content filtering using LLM (requires LLM credentials)',
+                    },
+                ],
+                default: 'none',
+                description: 'Type of content filtering to apply',
+            },
+            {
+                displayName: 'Ignore Cache',
+                name: 'ignoreCache',
+                type: 'boolean',
+                displayOptions: {
+                    show: {
+                        filterType: ['llm'],
+                    },
+                },
+                default: false,
+                description: 'Whether to skip cache and always generate fresh filtered content',
+            },
+            {
+                displayName: 'Ignore Links',
+                name: 'ignoreLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude links from markdown output',
+            },
+            {
+                displayName: 'LLM Instruction',
+                name: 'llmInstruction',
+                type: 'string',
+                typeOptions: {
+                    rows: 8,
+                },
+                displayOptions: {
+                    show: {
+                        filterType: ['llm'],
+                    },
+                },
+                default: `Extract the main content while preserving its original wording and substance completely.
+Remove only clearly irrelevant elements like:
+- Navigation menus
+- Advertisement sections
+- Cookie notices
+- Footers with site information
+- Sidebars with external links
+- Any UI elements that don't contribute to the content
+
+Keep all valuable educational or informational content intact.`,
+                description: 'Instructions for the LLM on how to filter and clean the content',
+                required: true,
+            },
+            {
+                displayName: 'LLM Verbose',
+                name: 'llmVerbose',
+                type: 'boolean',
+                displayOptions: {
+                    show: {
+                        filterType: ['llm'],
+                    },
+                },
+                default: false,
+                description: 'Whether to enable verbose logging for LLM content filtering',
+            },
+            {
+                displayName: 'Min Word Threshold',
+                name: 'minWordThreshold',
+                type: 'number',
+                default: 0,
+                displayOptions: {
+                    show: {
+                        filterType: ['pruning'],
+                    },
+                },
+                description: 'Minimum word count for content blocks to be considered (0 = no minimum)',
+            },
+            {
+                displayName: 'Threshold',
+                name: 'threshold',
+                type: 'number',
+                default: 0.48,
+                displayOptions: {
+                    show: {
+                        filterType: ['pruning'],
+                    },
+                },
+                description: 'Relevance threshold for pruning (0.0-1.0, default: 0.48). Higher values = more aggressive filtering.',
+            },
+            {
+                displayName: 'Threshold Type',
+                name: 'thresholdType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Fixed',
+                        value: 'fixed',
+                        description: 'Use fixed threshold value',
+                    },
+                    {
+                        name: 'Dynamic',
+                        value: 'dynamic',
+                        description: 'Calculate threshold dynamically based on content',
+                    },
+                ],
+                default: 'fixed',
+                displayOptions: {
+                    show: {
+                        filterType: ['pruning'],
+                    },
+                },
+                description: 'How to apply the pruning threshold',
+            },
+            {
+                displayName: 'User Query',
+                name: 'userQuery',
+                type: 'string',
+                default: '',
+                placeholder: 'main content topics keywords',
+                displayOptions: {
+                    show: {
+                        filterType: ['bm25'],
+                    },
+                },
+                description: 'Query string to filter relevant content (BM25 will rank content by relevance to this query)',
+            },
+        ],
+    },
+    {
+        displayName: 'Advanced Options',
+        name: 'advancedOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Anti-Bot Features',
+                name: 'antiBotFeatures',
+                type: 'fixedCollection',
+                default: {},
+                options: [
+                    {
+                        name: 'features',
+                        displayName: 'Features',
+                        values: [
+                            {
+                                displayName: 'Magic Mode',
+                                name: 'magic',
+                                type: 'boolean',
+                                default: false,
+                                description: 'Whether to enable anti-detection techniques (stealth++)',
+                            },
+                            {
+                                displayName: 'Simulate User Behavior',
+                                name: 'simulateUser',
+                                type: 'boolean',
+                                default: false,
+                                description: 'Whether to simulate human-like browsing behavior',
+                            },
+                            {
+                                displayName: 'Override Navigator',
+                                name: 'overrideNavigator',
+                                type: 'boolean',
+                                default: false,
+                                description: 'Whether to override navigator properties to avoid detection',
+                            },
+                        ],
+                    },
+                ],
+            },
+            {
+                displayName: 'Delay Before Return (Ms)',
+                name: 'delayBeforeReturnHtml',
                 type: 'number',
+                default: 0,
+                description: 'Milliseconds to wait before returning HTML (useful for dynamic content)',
+            },
+            {
+                displayName: 'Exclude External Images',
+                name: 'excludeExternalImages',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude images hosted on external domains',
+            },
+            {
+                displayName: 'Exclude Social Media Links',
+                name: 'excludeSocialMediaLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude links to social media platforms',
+            },
+            {
+                displayName: 'Verbose Mode',
+                name: 'verbose',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to enable verbose logging (debug mode)',
+            },
+            {
+                displayName: 'Wait Until',
+                name: 'waitUntil',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Load',
+                        value: 'load',
+                        description: 'Wait for the load event',
+                    },
+                    {
+                        name: 'DOM Content Loaded',
+                        value: 'domcontentloaded',
+                        description: 'Wait for DOMContentLoaded event',
+                    },
+                    {
+                        name: 'Network Idle',
+                        value: 'networkidle',
+                        description: 'Wait for network to be idle (no requests for 500ms)',
+                    },
+                    {
+                        name: 'Network Idle 2',
+                        value: 'networkidle2',
+                        description: 'Wait for network to be idle (no more than 2 requests for 500ms)',
+                    },
+                ],
+                default: 'load',
+                description: 'When to consider page load complete',
+            },
+        ],
+    },
+    {
+        displayName: 'Table Extraction',
+        name: 'tableExtraction',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlMultipleUrls'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Chunk Token Threshold',
+                name: 'chunkTokenThreshold',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                        enableChunking: [true],
+                    },
+                },
+                default: 10000,
+                description: 'Maximum tokens per chunk when processing large tables (default: 10000)',
+            },
+            {
+                displayName: 'CSS Selector',
+                name: 'cssSelector',
+                type: 'string',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                    },
+                },
+                default: '',
+                placeholder: '.main-content',
+                description: 'CSS selector to focus table extraction on specific page area (optional)',
+            },
+            {
+                displayName: 'Enable Chunking',
+                name: 'enableChunking',
+                type: 'boolean',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                    },
+                },
+                default: false,
+                description: 'Whether to enable chunking for large tables (recommended for tables with 100+ rows)',
+            },
+            {
+                displayName: 'Max Parallel Chunks',
+                name: 'maxParallelChunks',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                        enableChunking: [true],
+                    },
+                },
                 default: 5,
-                description: 'Maximum number of concurrent crawls',
+                description: 'Maximum number of chunks to process in parallel (default: 5)',
+            },
+            {
+                displayName: 'Max Tries',
+                name: 'maxTries',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                    },
+                },
+                default: 3,
+                description: 'Maximum number of retry attempts for LLM extraction (default: 3)',
+            },
+            {
+                displayName: 'Min Rows Per Chunk',
+                name: 'minRowsPerChunk',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm'],
+                        enableChunking: [true],
+                    },
+                },
+                default: 20,
+                description: 'Minimum number of rows per chunk (default: 20)',
+            },
+            {
+                displayName: 'Strategy Type',
+                name: 'strategyType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'None',
+                        value: 'none',
+                        description: 'No table extraction',
+                    },
+                    {
+                        name: 'LLM Table Extraction',
+                        value: 'llm',
+                        description: 'Extract tables using LLM (handles complex tables with rowspan/colspan)',
+                    },
+                    {
+                        name: 'Default Table Extraction',
+                        value: 'default',
+                        description: 'Extract tables using heuristics (faster, simpler tables only)',
+                    },
+                ],
+                default: 'none',
+                description: 'Table extraction strategy to use',
+            },
+            {
+                displayName: 'Table Score Threshold',
+                name: 'tableScoreThreshold',
+                type: 'number',
+                displayOptions: {
+                    show: {
+                        strategyType: ['default'],
+                    },
+                },
+                default: 5,
+                description: 'Minimum score for table to be included in results (default: 5, range: 0-10)',
+            },
+            {
+                displayName: 'Verbose',
+                name: 'verbose',
+                type: 'boolean',
+                displayOptions: {
+                    show: {
+                        strategyType: ['llm', 'default'],
+                    },
+                },
+                default: false,
+                description: 'Whether to enable verbose logging for table extraction',
             },
         ],
     },
 ];
 async function execute(items, nodeOptions) {
-    var _a;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
     const allResults = [];
     for (let i = 0; i < items.length; i++) {
         try {
-            const urlsString = this.getNodeParameter('urls', i, '');
             const browserOptions = this.getNodeParameter('browserOptions', i, {});
+            const sessionOptions = this.getNodeParameter('sessionOptions', i, {});
             const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
-            const options = this.getNodeParameter('options', i, {});
-            if (!urlsString) {
-                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URLs cannot be empty.', { itemIndex: i });
-            }
-            const urls = urlsString
-                .split(',')
-                .map(url => url.trim())
-                .filter(url => url);
-            if (urls.length === 0) {
-                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No valid URLs provided.', { itemIndex: i });
+            const outputOptions = this.getNodeParameter('outputOptions', i, {});
+            const contentFilter = this.getNodeParameter('contentFilter', i, {});
+            const advancedOptions = this.getNodeParameter('advancedOptions', i, {});
+            let mergedBrowserOptions = { ...browserOptions, ...sessionOptions };
+            if (browserOptions.extraArgs && typeof browserOptions.extraArgs === 'object') {
+                const extraArgsCollection = browserOptions.extraArgs;
+                if (extraArgsCollection.args && Array.isArray(extraArgsCollection.args)) {
+                    mergedBrowserOptions.extraArgs = extraArgsCollection.args.map((arg) => arg.value).filter((v) => v);
+                }
             }
-            const invalidUrls = urls.filter(url => !(0, utils_1.isValidUrl)(url));
-            if (invalidUrls.length > 0) {
-                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URLs: ${invalidUrls.join(', ')}`, { itemIndex: i });
+            const crawlMode = this.getNodeParameter('crawlMode', i, 'manual');
+            let urls = [];
+            if (crawlMode === 'manual') {
+                const urlsString = this.getNodeParameter('urls', i, '');
+                if (!urlsString) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URLs cannot be empty.', { itemIndex: i });
+                }
+                urls = urlsString
+                    .split(',')
+                    .map(url => url.trim())
+                    .filter(url => url);
+                if (urls.length === 0) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No valid URLs provided.', { itemIndex: i });
+                }
+                const invalidUrls = urls.filter(url => !(0, utils_1.isValidUrl)(url));
+                if (invalidUrls.length > 0) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URLs: ${invalidUrls.join(', ')}`, { itemIndex: i });
+                }
             }
-            const browserConfig = (0, utils_1.createBrowserConfig)(browserOptions);
-            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)({
+            const browserConfig = (0, utils_1.createBrowserConfig)(mergedBrowserOptions);
+            const antiBotFeatures = ((_a = advancedOptions.antiBotFeatures) === null || _a === void 0 ? void 0 : _a.features) || {};
+            const mergedCrawlerOptions = {
                 ...crawlerOptions,
                 ...browserConfig,
-                maxConcurrent: options.maxConcurrent ? Number(options.maxConcurrent) : 5,
-            });
+                maxConcurrent: outputOptions.maxConcurrent ? Number(outputOptions.maxConcurrent) : 5,
+                screenshot: outputOptions.screenshot,
+                pdf: outputOptions.pdf,
+                fetchSslCertificate: outputOptions.fetchSslCertificate,
+                magic: antiBotFeatures.magic,
+                simulateUser: antiBotFeatures.simulateUser,
+                overrideNavigator: antiBotFeatures.overrideNavigator,
+                excludeSocialMediaLinks: advancedOptions.excludeSocialMediaLinks,
+                excludeExternalImages: advancedOptions.excludeExternalImages,
+                delayBeforeReturnHtml: advancedOptions.delayBeforeReturnHtml,
+                waitUntil: advancedOptions.waitUntil,
+                verbose: advancedOptions.verbose,
+            };
+            if (contentFilter.filterType && contentFilter.filterType !== 'none') {
+                const enrichedFilterConfig = { ...contentFilter };
+                if (contentFilter.filterType === 'llm') {
+                    const credentials = await this.getCredentials('crawl4aiPlusApi');
+                    if (!credentials.enableLlm) {
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'LLM features must be enabled in Crawl4AI credentials to use LLM content filtering.', { itemIndex: i });
+                    }
+                    let provider = 'openai/gpt-4o';
+                    let apiKey = '';
+                    if (credentials.llmProvider === 'openai') {
+                        const model = credentials.llmModel || 'gpt-4o';
+                        provider = `openai/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'anthropic') {
+                        const model = credentials.llmModel || 'claude-3-haiku-20240307';
+                        provider = `anthropic/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'groq') {
+                        const model = credentials.llmModel || 'llama3-70b-8192';
+                        provider = `groq/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'ollama') {
+                        const model = credentials.ollamaModel || 'llama3';
+                        provider = `ollama/${model}`;
+                    }
+                    else if (credentials.llmProvider === 'other') {
+                        provider = credentials.customProvider || 'custom/model';
+                        apiKey = credentials.customApiKey || '';
+                    }
+                    if (!apiKey && credentials.llmProvider !== 'ollama') {
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), `API key is required for ${credentials.llmProvider} provider. Please configure it in the Crawl4AI credentials.`, { itemIndex: i });
+                    }
+                    enrichedFilterConfig.llmConfig = {
+                        type: 'LLMConfig',
+                        params: {
+                            provider,
+                            api_token: apiKey,
+                            ...(credentials.llmProvider === 'other' && credentials.customBaseUrl ?
+                                { api_base: credentials.customBaseUrl } : {}),
+                            ...(credentials.llmProvider === 'ollama' && credentials.ollamaUrl ?
+                                { api_base: credentials.ollamaUrl } : {})
+                        }
+                    };
+                }
+                mergedCrawlerOptions.markdownGenerator = (0, utils_1.createMarkdownGenerator)(enrichedFilterConfig);
+            }
+            const tableExtractionConfig = this.getNodeParameter('tableExtraction', i, {});
+            if (tableExtractionConfig.strategyType && tableExtractionConfig.strategyType !== 'none') {
+                const enrichedTableConfig = { ...tableExtractionConfig };
+                if (tableExtractionConfig.strategyType === 'llm') {
+                    const credentials = await this.getCredentials('crawl4aiPlusApi');
+                    if (!credentials.enableLlm) {
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'LLM features must be enabled in Crawl4AI credentials to use LLM table extraction.', { itemIndex: i });
+                    }
+                    let provider = 'openai/gpt-4o';
+                    let apiKey = '';
+                    if (credentials.llmProvider === 'openai') {
+                        const model = credentials.llmModel || 'gpt-4o';
+                        provider = `openai/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'anthropic') {
+                        const model = credentials.llmModel || 'claude-3-haiku-20240307';
+                        provider = `anthropic/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'groq') {
+                        const model = credentials.llmModel || 'llama3-70b-8192';
+                        provider = `groq/${model}`;
+                        apiKey = credentials.apiKey || '';
+                    }
+                    else if (credentials.llmProvider === 'ollama') {
+                        const model = credentials.ollamaModel || 'llama3';
+                        provider = `ollama/${model}`;
+                    }
+                    else if (credentials.llmProvider === 'other') {
+                        provider = credentials.customProvider || 'custom/model';
+                        apiKey = credentials.customApiKey || '';
+                    }
+                    if (!apiKey && credentials.llmProvider !== 'ollama') {
+                        throw new n8n_workflow_1.NodeOperationError(this.getNode(), `API key is required for ${credentials.llmProvider} provider. Please configure it in the Crawl4AI credentials.`, { itemIndex: i });
+                    }
+                    enrichedTableConfig.llmConfig = {
+                        type: 'LLMConfig',
+                        params: {
+                            provider,
+                            api_token: apiKey,
+                            ...(credentials.llmProvider === 'other' && credentials.customBaseUrl ?
+                                { api_base: credentials.customBaseUrl } : {}),
+                            ...(credentials.llmProvider === 'ollama' && credentials.ollamaUrl ?
+                                { api_base: credentials.ollamaUrl } : {})
+                        }
+                    };
+                }
+                mergedCrawlerOptions.tableExtraction = (0, utils_1.createTableExtractionStrategy)(enrichedTableConfig);
+            }
+            if (crawlMode === 'discover') {
+                const discoveryOptions = this.getNodeParameter('discoveryOptions', i, {});
+                const seedUrl = String((_b = discoveryOptions.seedUrl) !== null && _b !== void 0 ? _b : '').trim();
+                const query = String((_c = discoveryOptions.query) !== null && _c !== void 0 ? _c : '').trim();
+                if (!seedUrl) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Seed URL is required when discovery mode is enabled.', { itemIndex: i });
+                }
+                if (!(0, utils_1.isValidUrl)(seedUrl)) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid Seed URL: ${seedUrl}`, { itemIndex: i });
+                }
+                if (!query) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Discovery query cannot be empty.', { itemIndex: i });
+                }
+                const maxDepthRaw = (_d = discoveryOptions.maxDepth) !== null && _d !== void 0 ? _d : 2;
+                const maxDepth = Math.min(Math.max(Number(maxDepthRaw), 1), 5);
+                if (Number(maxDepthRaw) !== maxDepth) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Maximum Depth must be between 1 and 5.', { itemIndex: i });
+                }
+                const maxPagesRaw = (_e = discoveryOptions.maxPages) !== null && _e !== void 0 ? _e : 50;
+                const maxPages = Math.min(Math.max(Number(maxPagesRaw), 1), 200);
+                if (Number(maxPagesRaw) !== maxPages) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Maximum Pages must be between 1 and 200.', { itemIndex: i });
+                }
+                const includeExternal = discoveryOptions.includeExternal === true;
+                const respectRobotsTxt = discoveryOptions.respectRobotsTxt !== false;
+                const includePatternsRaw = discoveryOptions.includePatterns;
+                const includePatterns = Array.isArray(includePatternsRaw)
+                    ? includePatternsRaw
+                    : typeof includePatternsRaw === 'string'
+                        ? includePatternsRaw
+                            .split(',')
+                            .map(value => value.trim())
+                            .filter(value => value.length > 0)
+                        : [];
+                const excludePatternsRaw = discoveryOptions.excludePatterns;
+                const excludePatterns = Array.isArray(excludePatternsRaw)
+                    ? excludePatternsRaw
+                    : typeof excludePatternsRaw === 'string'
+                        ? excludePatternsRaw
+                            .split(',')
+                            .map(value => value.trim())
+                            .filter(value => value.length > 0)
+                        : [];
+                const excludeDomainsRaw = discoveryOptions.excludeDomains;
+                const excludeDomains = Array.isArray(excludeDomainsRaw)
+                    ? excludeDomainsRaw
+                    : typeof excludeDomainsRaw === 'string'
+                        ? excludeDomainsRaw
+                            .split(',')
+                            .map(value => value.trim())
+                            .filter(value => value.length > 0)
+                        : [];
+                const resultLimitRaw = (_f = discoveryOptions.resultLimit) !== null && _f !== void 0 ? _f : 0;
+                const resultLimit = Math.max(Number(resultLimitRaw), 0);
+                const filters = [];
+                if (excludeDomains.length > 0) {
+                    filters.push({
+                        type: 'DomainFilter',
+                        params: {
+                            blocked_domains: excludeDomains,
+                        },
+                    });
+                }
+                if (excludePatterns.length > 0) {
+                    filters.push({
+                        type: 'URLPatternFilter',
+                        params: {
+                            patterns: excludePatterns,
+                            reverse: true,
+                        },
+                    });
+                }
+                if (includePatterns.length > 0) {
+                    filters.push({
+                        type: 'URLPatternFilter',
+                        params: {
+                            patterns: includePatterns,
+                            reverse: false,
+                        },
+                    });
+                }
+                const urlScorer = query
+                    ? {
+                        type: 'KeywordRelevanceScorer',
+                        params: {
+                            keywords: query.split(/\s+OR\s+|\s+/).filter(k => k.trim()),
+                            weight: 1.0,
+                        },
+                    }
+                    : undefined;
+                const strategyType = String((_g = discoveryOptions.crawlStrategy) !== null && _g !== void 0 ? _g : 'BestFirstCrawlingStrategy');
+                const strategyParams = {
+                    max_depth: maxDepth,
+                    max_pages: maxPages,
+                    include_external: includeExternal,
+                    ...(filters.length > 0 ? {
+                        filter_chain: {
+                            type: 'FilterChain',
+                            params: {
+                                filters,
+                            },
+                        },
+                    } : {}),
+                    ...(urlScorer ? { url_scorer: urlScorer } : {}),
+                };
+                if (strategyType !== 'BestFirstCrawlingStrategy') {
+                    const scoreThreshold = Number((_h = discoveryOptions.scoreThreshold) !== null && _h !== void 0 ? _h : 0);
+                    if (scoreThreshold > 0) {
+                        strategyParams.score_threshold = scoreThreshold;
+                    }
+                }
+                const deepCrawlStrategy = {
+                    type: strategyType,
+                    params: strategyParams,
+                };
+                mergedCrawlerOptions.deepCrawlStrategy = deepCrawlStrategy;
+                if (respectRobotsTxt) {
+                    mergedCrawlerOptions.checkRobotsTxt = true;
+                }
+                urls = [seedUrl];
+                mergedCrawlerOptions.__resultLimit = resultLimit;
+                const extractionOptions = this.getNodeParameter('extractionOptions', i, {});
+                if (extractionOptions.enableExtraction === true) {
+                    const extractionType = extractionOptions.extractionType || 'css';
+                    if (extractionType === 'css') {
+                        const baseSelector = (extractionOptions.baseSelector || '').trim();
+                        const fieldSelectorsJson = (extractionOptions.fieldSelectors || '[]').trim();
+                        if (!baseSelector) {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Base selector is required for CSS extraction.', { itemIndex: i });
+                        }
+                        let fields = [];
+                        try {
+                            fields = JSON.parse(fieldSelectorsJson);
+                            if (!Array.isArray(fields) || fields.length === 0) {
+                                throw new Error('Field selectors must be a non-empty array');
+                            }
+                        }
+                        catch (e) {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid field selectors JSON: ${e.message}`, { itemIndex: i });
+                        }
+                        mergedCrawlerOptions.extractionStrategy = {
+                            type: 'JsonCssExtractionStrategy',
+                            params: {
+                                schema: {
+                                    type: 'dict',
+                                    value: {
+                                        name: 'ExtractedData',
+                                        baseSelector,
+                                        fields,
+                                    },
+                                },
+                            },
+                        };
+                    }
+                    else if (extractionType === 'llm') {
+                        const llmInstruction = (extractionOptions.llmInstruction || '').trim();
+                        const llmSchemaJson = (extractionOptions.llmSchema || '{}').trim();
+                        if (!llmInstruction) {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Extraction instructions are required for LLM extraction.', { itemIndex: i });
+                        }
+                        let schema = {};
+                        try {
+                            schema = JSON.parse(llmSchemaJson);
+                        }
+                        catch (e) {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid LLM schema JSON: ${e.message}`, { itemIndex: i });
+                        }
+                        const credentials = await this.getCredentials('crawl4aiPlusApi');
+                        if (!credentials.enableLlm) {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'LLM features must be enabled in Crawl4AI credentials to use LLM extraction.', { itemIndex: i });
+                        }
+                        let provider = 'openai/gpt-4o';
+                        let apiKey = '';
+                        if (credentials.llmProvider === 'openai') {
+                            const model = credentials.llmModel || 'gpt-4o';
+                            provider = `openai/${model}`;
+                            apiKey = credentials.apiKey || '';
+                        }
+                        else if (credentials.llmProvider === 'anthropic') {
+                            const model = credentials.llmModel || 'claude-3-haiku-20240307';
+                            provider = `anthropic/${model}`;
+                            apiKey = credentials.apiKey || '';
+                        }
+                        else if (credentials.llmProvider === 'groq') {
+                            const model = credentials.llmModel || 'llama3-70b-8192';
+                            provider = `groq/${model}`;
+                            apiKey = credentials.apiKey || '';
+                        }
+                        else if (credentials.llmProvider === 'ollama') {
+                            const model = credentials.ollamaModel || 'llama3';
+                            provider = `ollama/${model}`;
+                        }
+                        else if (credentials.llmProvider === 'other') {
+                            provider = credentials.customProvider || 'custom/model';
+                            apiKey = credentials.customApiKey || '';
+                        }
+                        if (!apiKey && credentials.llmProvider !== 'ollama') {
+                            throw new n8n_workflow_1.NodeOperationError(this.getNode(), `API key is required for ${credentials.llmProvider} provider.`, { itemIndex: i });
+                        }
+                        mergedCrawlerOptions.extractionStrategy = {
+                            type: 'LLMExtractionStrategy',
+                            params: {
+                                llm_config: {
+                                    type: 'LLMConfig',
+                                    params: {
+                                        provider,
+                                        api_token: apiKey,
+                                        ...(credentials.llmProvider === 'other' && credentials.customBaseUrl ?
+                                            { api_base: credentials.customBaseUrl } : {}),
+                                        ...(credentials.llmProvider === 'ollama' && credentials.ollamaUrl ?
+                                            { api_base: credentials.ollamaUrl } : {})
+                                    }
+                                },
+                                schema: { type: 'dict', value: schema },
+                                instruction: llmInstruction,
+                                extraction_type: 'schema',
+                            },
+                        };
+                    }
+                }
+            }
+            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)(mergedCrawlerOptions);
             const crawler = await (0, utils_1.getCrawl4aiClient)(this);
             const results = await crawler.crawlMultipleUrls(urls, crawlerConfig);
-            for (const result of results) {
-                const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse);
+            const resultLimit = crawlMode === 'discover'
+                ? Number((_j = mergedCrawlerOptions.__resultLimit) !== null && _j !== void 0 ? _j : 0)
+                : 0;
+            const limitedResults = resultLimit > 0 ? results.slice(0, resultLimit) : results;
+            for (const result of limitedResults) {
+                const formattedResult = (0, formatters_1.formatCrawlResult)(result, outputOptions.includeMedia, outputOptions.verboseResponse, {
+                    markdownOutput: outputOptions.markdownOutput,
+                    includeLinks: outputOptions.includeLinks,
+                    includeScreenshot: outputOptions.screenshot,
+                    includePdf: outputOptions.pdf,
+                    includeSslCertificate: outputOptions.fetchSslCertificate,
+                    includeTables: outputOptions.includeTables,
+                });
                 allResults.push({
                     json: formattedResult,
                     pairedItem: { item: i },
@@ -283,7 +1483,7 @@ async function execute(items, nodeOptions) {
         catch (error) {
             if (this.continueOnFail()) {
                 const node = this.getNode();
-                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                const errorItemIndex = (_k = error.itemIndex) !== null && _k !== void 0 ? _k : i;
                 allResults.push({
                     json: items[i].json,
                     error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),