@mendable/n8n-nodes-firecrawl 1.0.5 → 1.2.1

package/dist/nodes/Firecrawl/api/common.js

@@ -1,7 +1,111 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildApiProperties = exports.createOperationOption = exports.createScrapeOptionsProperty = exports.createExcludeTagsProperty = exports.createIncludeTagsProperty = exports.createLocationProperty = exports.createActionsProperty = exports.createUrlProperty = exports.createOperationNotice = exports.formatOperationName = void 0;
+exports.buildApiProperties = exports.createOperationOption = exports.createScrapeOptionsProperty = exports.createExcludeTagsProperty = exports.createBatchUrlsProperty = exports.createIncludeTagsProperty = exports.createLocationProperty = exports.createActionsProperty = exports.createUrlProperty = exports.createOperationNotice = exports.formatOperationName = exports.convertToSchema = exports.extractUrls = void 0;
 const helpers_1 = require("../helpers");
+function normalizeUrl(url) {
+    const trimmed = url.trim();
+    if (!trimmed)
+        return '';
+    if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
+        return trimmed;
+    }
+    return `https://${trimmed}`;
+}
+function looksLikeUrl(str) {
+    const trimmed = str.trim();
+    if (!trimmed)
+        return false;
+    if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) {
+        return true;
+    }
+    return trimmed.includes('.') && !trimmed.includes(' ');
+}
+function extractUrls(val) {
+    if (Array.isArray(val)) {
+        return val.flatMap(extractUrls);
+    }
+    if (typeof val === 'string') {
+        const trimmed = val.trim();
+        if (!trimmed)
+            return [];
+        if (trimmed.startsWith('[')) {
+            try {
+                const parsed = JSON.parse(trimmed);
+                return extractUrls(parsed);
+            }
+            catch {
+            }
+        }
+        if (trimmed.includes('\n')) {
+            return trimmed
+                .split('\n')
+                .map((u) => u.trim())
+                .filter(looksLikeUrl)
+                .map(normalizeUrl);
+        }
+        if (trimmed.includes(',')) {
+            return trimmed
+                .split(',')
+                .map((u) => u.trim())
+                .filter(looksLikeUrl)
+                .map(normalizeUrl);
+        }
+        if (trimmed.includes(' http')) {
+            return trimmed
+                .split(/\s+/)
+                .map((u) => u.trim())
+                .filter(looksLikeUrl)
+                .map(normalizeUrl);
+        }
+        if (looksLikeUrl(trimmed)) {
+            return [normalizeUrl(trimmed)];
+        }
+        return [];
+    }
+    if (typeof val === 'object' && val !== null && 'url' in val) {
+        const urlVal = val.url;
+        if (typeof urlVal === 'string' && looksLikeUrl(urlVal)) {
+            return [normalizeUrl(urlVal)];
+        }
+    }
+    return [];
+}
+exports.extractUrls = extractUrls;
+function convertToSchema(value) {
+    if (value === null) {
+        return { type: 'null' };
+    }
+    if (Array.isArray(value)) {
+        if (value.length === 0) {
+            return { type: 'array', items: {} };
+        }
+        return {
+            type: 'array',
+            items: convertToSchema(value[0]),
+        };
+    }
+    switch (typeof value) {
+        case 'string':
+            return { type: 'string' };
+        case 'number':
+            return Number.isInteger(value) ? { type: 'integer' } : { type: 'number' };
+        case 'boolean':
+            return { type: 'boolean' };
+        case 'object': {
+            const properties = {};
+            for (const [key, val] of Object.entries(value)) {
+                properties[key] = convertToSchema(val);
+            }
+            return {
+                type: 'object',
+                properties,
+            };
+        }
+        default:
+            return {};
+    }
+}
+exports.convertToSchema = convertToSchema;
 function formatOperationName(name) {
     return name
         .split(/(?=[A-Z])/)
@@ -29,10 +133,12 @@ function createOperationNotice(resourceName, operationName, method = 'POST') {
 exports.createOperationNotice = createOperationNotice;
 function createUrlProperty(operationName, defaultUrl = 'https://firecrawl.dev', resourceName = 'Default') {
     return {
-        displayName: 'Url',
+        displayName: 'URL',
         name: 'url',
         type: 'string',
         default: defaultUrl,
+        required: true,
+        description: 'The target webpage URL to scrape or crawl. Must be a valid HTTP/HTTPS URL (e.g., https://example.com). The URL should be publicly accessible or behind authentication configured in headers.',
         routing: {
             request: {
                 body: {
@@ -49,7 +155,7 @@ function createUrlProperty(operationName, defaultUrl = 'https://firecrawl.dev',
     };
 }
 exports.createUrlProperty = createUrlProperty;
-function createActionsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true) {
+function createActionsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true, resourceName = 'Scraping') {
     return {
         displayName: 'Actions',
         name: 'actions',
@@ -58,8 +164,8 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
         typeOptions: {
             multipleValues: true,
         },
-        description: 'List of actions to interact with dynamic content before scraping',
-        placeholder: 'Add item',
+        description: 'Pre-scraping automation actions to interact with dynamic content. Use these to handle JavaScript-rendered pages, click buttons, fill forms, scroll to load more content, or wait for elements. Actions execute in order before content extraction.',
+        placeholder: 'Add action',
         options: [
             {
                 displayName: 'Items',
@@ -101,7 +207,7 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
                         displayName: 'Selector',
                         type: 'string',
                         default: '',
-                        description: 'The CSS selector for `click` and `write` actions',
+                        description: 'CSS selector to target an element (e.g., "#submit-btn", ".load-more", "[data-testid=login]"). Used for click, write, and scroll actions.',
                         name: 'selector',
                         displayOptions: {
                             show: {
@@ -113,7 +219,7 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
                         displayName: 'Milliseconds',
                         type: 'number',
                         default: 1000,
-                        description: 'Milliseconds to wait for `wait` action',
+                        description: 'Time to wait in milliseconds. Use this to allow JavaScript content to load or animations to complete. Typical values: 1000-3000ms for most pages, 5000ms+ for heavy SPAs.',
                         name: 'milliseconds',
                         displayOptions: {
                             show: {
@@ -137,7 +243,7 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
                         displayName: 'Text',
                         type: 'string',
                         default: '',
-                        description: 'Text for `write` action',
+                        description: 'Text to type into an input field. Use with selector to target the input element. Useful for search boxes, login forms, or any text input.',
                         name: 'text',
                         displayOptions: {
                             show: {
@@ -149,7 +255,7 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
                         displayName: 'Key',
                         type: 'string',
                         default: '',
-                        description: 'Key for `press` action',
+                        description: 'Keyboard key to press (e.g., "Enter", "Tab", "Escape", "ArrowDown"). Use for form submission, navigation, or triggering keyboard shortcuts.',
                         name: 'key',
                         displayOptions: {
                             show: {
@@ -202,20 +308,20 @@ function createActionsProperty(operationName, omitDisplayOptions = false, useNes
                     useCustomBody: [true],
                 },
                 show: {
-                    resource: ['Default'],
+                    resource: [resourceName],
                     operation: [operationName],
                 },
             },
     };
 }
 exports.createActionsProperty = createActionsProperty;
-function createLocationProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true) {
+function createLocationProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true, resourceName = 'Scraping') {
     return {
         displayName: 'Location',
         name: 'location',
         type: 'fixedCollection',
         default: {},
-        description: 'Location settings for the request',
+        description: 'Geographic location settings for the request. Use this to access geo-restricted content or see region-specific versions of websites. Affects IP geolocation and Accept-Language headers.',
         options: [
             {
                 displayName: 'Location Settings',
@@ -294,14 +400,14 @@ function createLocationProperty(operationName, omitDisplayOptions = false, useNe
                     useCustomBody: [true],
                 },
                 show: {
-                    resource: ['Default'],
+                    resource: [resourceName],
                     operation: [operationName],
                 },
             },
     };
 }
 exports.createLocationProperty = createLocationProperty;
-function createIncludeTagsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true) {
+function createIncludeTagsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true, resourceName = 'Scraping') {
     return {
         displayName: 'Include Tags',
         name: 'includeTags',
@@ -310,7 +416,7 @@ function createIncludeTagsProperty(operationName, omitDisplayOptions = false, us
         typeOptions: {
             multipleValues: true,
         },
-        description: 'Specifies tags to include in the output',
+        description: 'HTML tags to retain in the output. Only content within these tags will be included. Useful for extracting specific sections like article content, product details, or main content areas.',
         placeholder: 'Add tag to include',
         options: [
             {
@@ -348,14 +454,62 @@ function createIncludeTagsProperty(operationName, omitDisplayOptions = false, us
                     useCustomBody: [true],
                 },
                 show: {
-                    resource: ['Default'],
+                    resource: [resourceName],
                     operation: [operationName],
                 },
             },
     };
 }
 exports.createIncludeTagsProperty = createIncludeTagsProperty;
-function createExcludeTagsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true) {
+function createBatchUrlsProperty(operationName, defaultUrl = 'https://firecrawl.dev', resourceName = 'Default') {
+    return {
+        displayName: 'URLs',
+        name: 'urls',
+        type: 'string',
+        typeOptions: {
+            rows: 4,
+        },
+        default: defaultUrl,
+        required: true,
+        description: 'URLs to scrape in batch. Accepts multiple formats: a single URL, multiple URLs separated by commas or newlines, a JSON array like ["url1", "url2"], or an array expression from a previous node. Each URL is scraped independently with the same settings. Example: "https://example.com/page1, https://example.com/page2" or paste one URL per line.',
+        placeholder: 'https://example.com/page1\nhttps://example.com/page2',
+        routing: {
+            request: {
+                body: {
+                    urls: '={{ $value }}',
+                },
+            },
+            send: {
+                preSend: [
+                    async function (requestOptions) {
+                        if (typeof requestOptions.body !== 'object' || !requestOptions.body) {
+                            return requestOptions;
+                        }
+                        const body = requestOptions.body;
+                        if (body.urls !== undefined) {
+                            const rawValue = body.urls;
+                            const urlsArray = extractUrls(rawValue);
+                            const uniqueUrls = [...new Set(urlsArray)].filter(Boolean);
+                            if (uniqueUrls.length === 0) {
+                                throw new Error('At least one valid URL is required. Provide URLs as a single URL, comma-separated list, one per line, or a JSON array.');
+                            }
+                            body.urls = uniqueUrls;
+                        }
+                        return requestOptions;
+                    },
+                ],
+            },
+        },
+        displayOptions: {
+            show: {
+                resource: [resourceName],
+                operation: [operationName],
+            },
+        },
+    };
+}
+exports.createBatchUrlsProperty = createBatchUrlsProperty;
+function createExcludeTagsProperty(operationName, omitDisplayOptions = false, useNestedScrapeOptions = true, resourceName = 'Scraping') {
     return {
         displayName: 'Exclude Tags',
         name: 'excludeTags',
@@ -364,7 +518,7 @@ function createExcludeTagsProperty(operationName, omitDisplayOptions = false, us
         typeOptions: {
             multipleValues: true,
         },
-        description: 'Specifies tags to exclude from the output',
+        description: 'HTML tags to remove from the output. Content within these tags will be excluded. Common uses: removing navigation (nav), sidebars (aside), footers, ads, or cookie banners.',
         placeholder: 'Add tag to exclude',
         options: [
             {
@@ -402,23 +556,149 @@ function createExcludeTagsProperty(operationName, omitDisplayOptions = false, us
                     useCustomBody: [true],
                 },
                 show: {
-                    resource: ['Default'],
+                    resource: [resourceName],
                     operation: [operationName],
                 },
             },
     };
 }
 exports.createExcludeTagsProperty = createExcludeTagsProperty;
-function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = true) {
+function createBatchSpecificProperties() {
+    return [
+        {
+            displayName: 'Webhook',
+            name: 'webhook',
+            type: 'fixedCollection',
+            default: {},
+            description: 'Configure webhook notifications for batch job progress. Receive real-time updates when scraping starts, completes each page, finishes, or fails. Ideal for monitoring long-running jobs.',
+            options: [
+                {
+                    displayName: 'Webhook Settings',
+                    name: 'settings',
+                    values: [
+                        {
+                            displayName: 'URL',
+                            name: 'url',
+                            type: 'string',
+                            required: true,
+                            default: '',
+                            description: 'The URL to send the webhook to. Triggers for batchScrape.started, batchScrape.page, batchScrape.completed, or batchScrape.failed events.',
+                        },
+                        {
+                            displayName: 'Headers',
+                            name: 'headers',
+                            type: 'fixedCollection',
+                            default: {},
+                            typeOptions: {
+                                multipleValues: true,
+                            },
+                            description: 'Headers to send to the webhook URL',
+                            options: [
+                                {
+                                    displayName: 'Header',
+                                    name: 'header',
+                                    values: [
+                                        {
+                                            displayName: 'Key',
+                                            name: 'key',
+                                            type: 'string',
+                                            default: '',
+                                            description: 'Header key',
+                                        },
+                                        {
+                                            displayName: 'Value',
+                                            name: 'value',
+                                            type: 'string',
+                                            default: '',
+                                            description: 'Header value',
+                                        },
+                                    ],
+                                },
+                            ],
+                        },
+                        {
+                            displayName: 'Metadata',
+                            name: 'metadata',
+                            type: 'json',
+                            default: '{}',
+                            description: 'Custom metadata that will be included in all webhook payloads for this crawl',
+                        },
+                        {
+                            displayName: 'Events',
+                            name: 'events',
+                            type: 'multiOptions',
+                            options: [
+                                {
+                                    name: 'Started',
+                                    value: 'started',
+                                },
+                                {
+                                    name: 'Page',
+                                    value: 'page',
+                                },
+                                {
+                                    name: 'Completed',
+                                    value: 'completed',
+                                },
+                                {
+                                    name: 'Failed',
+                                    value: 'failed',
+                                },
+                            ],
+                            default: ['completed', 'page', 'failed', 'started'],
+                            description: 'Type of events that should be sent to the webhook URL',
+                        },
+                    ],
+                },
+            ],
+            routing: {
+                request: {
+                    body: {
+                        webhook: '={{$value.settings ? { url: $value.settings.url, headers: $value.settings.headers?.header?.reduce((acc, h) => ({ ...acc, [h.key]: h.value }), {}) || {}, metadata: $value.settings.metadata ? JSON.parse($value.settings.metadata) : {}, events: $value.settings.events || [] } : undefined}}',
+                    },
+                },
+            },
+        },
+        {
+            displayName: 'Max Concurrency',
+            name: 'maxConcurrency',
+            type: 'number',
+            default: 100,
+            description: 'Maximum number of URLs to scrape simultaneously. Higher values speed up batch jobs but may trigger rate limits. Leave at default to use your team\'s plan limit.',
+        },
+        {
+            displayName: 'Ignore Invalid URLs',
+            name: 'ignoreInvalidURLs',
+            type: 'boolean',
+            default: true,
+            description: 'Whether to continue processing valid URLs even if some are malformed or inaccessible. When disabled, the entire batch fails if any URL is invalid.',
+        },
+        {
+            displayName: 'Store In Cache',
+            name: 'storeInCache',
+            type: 'boolean',
+            default: true,
+            description: 'Whether to cache scraped pages for faster subsequent requests. Disable for privacy-sensitive scraping or when you need fresh data on every request.',
+        },
+        {
+            displayName: 'Zero Data Retention',
+            name: 'zeroDataRetention',
+            type: 'boolean',
+            default: false,
+            description: 'Whether to delete all scraped data from Firecrawl servers immediately after returning results. Enable for maximum privacy compliance (GDPR, HIPAA). Requires special account access.',
+        },
+    ];
+}
+function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = true, batchMode = false, resourceName = 'Scraping') {
     const scrapeOptionsBody = useNestedScrapeOptions
         ? { scrapeOptions: '={{$value.options}}' }
-        : '={{$value.options}}';
+        : '={{Object.fromEntries(Object.entries($value.options || {}).filter(([k]) => k !== "webhook"))}}';
     return {
         displayName: 'Scrape Options',
         name: 'scrapeOptions',
         type: 'fixedCollection',
         default: {},
-        description: 'Options for scraping content during the crawl',
+        description: 'Configure how content is extracted from pages. Control output formats (markdown, HTML, JSON), content filtering, timing, caching, and browser emulation settings.',
         options: [
             {
                 displayName: 'Options',
@@ -432,7 +712,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                         typeOptions: {
                             multipleValues: true,
                         },
-                        description: 'Output format(s) for the scraped data',
+                        description: 'Output format(s) for scraped content. Use "Summary" when user wants a brief overview or summary of the page. Use "Markdown" for full content (best for AI/LLM). HTML preserves structure. JSON enables structured data extraction. Screenshot captures visual representation.',
                         placeholder: 'Add format',
                         options: [
                             {
@@ -444,39 +724,47 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                                         name: 'type',
                                         type: 'options',
                                         default: 'markdown',
-                                        description: 'The type of format',
+                                        description: 'Output format for scraped content. Use "Summary" when user asks for a summary, overview, or condensed version of page content. Use "Markdown" for full content in AI-readable format.',
                                         options: [
                                             {
                                                 name: 'Change Tracking',
                                                 value: 'changeTracking',
+                                                description: 'Track changes between page versions over time',
                                             },
                                             {
                                                 name: 'HTML',
                                                 value: 'html',
+                                                description: 'Clean HTML with scripts and styles removed',
                                             },
                                             {
                                                 name: 'JSON',
                                                 value: 'json',
+                                                description: 'Extract structured data using AI with a schema',
                                             },
                                             {
                                                 name: 'Links',
                                                 value: 'links',
+                                                description: 'Extract all links from the page',
                                             },
                                             {
                                                 name: 'Markdown',
                                                 value: 'markdown',
+                                                description: 'Full page content as markdown - best for AI/LLM processing and reading full content',
                                             },
                                             {
                                                 name: 'Raw HTML',
                                                 value: 'rawHtml',
+                                                description: 'Original HTML including scripts and styles',
                                             },
                                             {
                                                 name: 'Screenshot',
                                                 value: 'screenshot',
+                                                description: 'Visual screenshot of the page as base64 image',
                                             },
                                             {
                                                 name: 'Summary',
                                                 value: 'summary',
+                                                description: 'AI-generated concise summary of page content - USE THIS when user asks for a summary, overview, brief description, or wants to quickly understand what a page is about',
                                             },
                                         ],
                                     },
@@ -636,7 +924,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                         name: 'onlyMainContent',
                         type: 'boolean',
                         default: true,
-                        description: 'Whether to only return the main content of the page excluding headers, navs, footers, etc',
+                        description: 'Whether to automatically remove navigation, headers, footers, sidebars, and other boilerplate content. Best for extracting article text or primary page content. Disable to capture the full page.',
                     },
                     createIncludeTagsProperty(operationName, true, useNestedScrapeOptions),
                     createExcludeTagsProperty(operationName, true, useNestedScrapeOptions),
@@ -645,7 +933,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                         name: 'headers',
                         type: 'collection',
                         default: {},
-                        description: 'Headers to send with the request',
+                        description: 'Custom HTTP headers to send with the request. Use for authentication tokens, cookies, custom user-agents, or API keys required by the target website.',
                         options: [
                             {
                                 displayName: 'Key',
@@ -668,28 +956,28 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                         name: 'waitFor',
                         type: 'number',
                         default: 0,
-                        description: 'Wait milliseconds for the page to load before fetching content',
+                        description: 'Additional wait time in milliseconds after page load before extracting content. Use for JavaScript-heavy sites where content renders dynamically. 0 uses intelligent auto-detection.',
                     },
                     {
                         displayName: 'Mobile',
                         name: 'mobile',
                         type: 'boolean',
                         default: false,
-                        description: 'Whether to emulate scraping from a mobile device',
+                        description: 'Whether to emulate a mobile device when scraping. Useful for sites with mobile-specific content, responsive layouts, or mobile-only features. Changes viewport and user-agent.',
                     },
                     {
                         displayName: 'Skip TLS Verification',
                         name: 'skipTlsVerification',
                         type: 'boolean',
                         default: false,
-                        description: 'Whether to skip TLS certificate verification when making requests',
+                        description: 'Whether to bypass SSL/TLS certificate validation. Enable for sites with self-signed or expired certificates. Use with caution as it reduces security.',
                     },
                     {
                         displayName: 'Timeout (Ms)',
                         name: 'timeout',
                         type: 'number',
                         default: 30000,
-                        description: 'Timeout in milliseconds for the request',
+                        description: 'Maximum time in milliseconds to wait for the page to load. Increase for slow sites or complex pages. Default 30000ms (30 seconds).',
                     },
                     createActionsProperty(operationName, true, useNestedScrapeOptions),
                     createLocationProperty(operationName, true, useNestedScrapeOptions),
@@ -698,28 +986,28 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                         name: 'removeBase64Images',
                         type: 'boolean',
                         default: true,
-                        description: 'Whether to remove base64 encoded images from the output',
+                        description: 'Whether to strip embedded base64 images from output while preserving alt text. Reduces output size significantly. Disable if you need inline image data.',
                     },
                     {
                         displayName: 'Block Ads',
                         name: 'blockAds',
                         type: 'boolean',
                         default: true,
-                        description: 'Whether to enables ad-blocking and cookie popup blocking',
+                        description: 'Whether to enable ad-blocking and cookie consent popup blocking for cleaner content extraction. Recommended for most scraping tasks.',
                     },
                     {
                         displayName: 'Store In Cache',
                         name: 'storeInCache',
                         type: 'boolean',
                         default: true,
-                        description: 'Whether to store the page in the Firecrawl index and cache. Disable for data protection concerns or when using sensitive scraping parameters.',
+                        description: 'Whether to cache the scraped page for faster subsequent requests. Disable for real-time data needs, sensitive content, or when privacy is a concern.',
                     },
                     {
                         displayName: 'Proxy',
                         name: 'proxy',
                         type: 'options',
                         default: 'basic',
-                        description: 'Specifies the type of proxy to use',
+                        description: 'Proxy strategy for bypassing anti-bot measures. Basic is faster and cheaper. Stealth uses residential proxies for better success on protected sites.',
                         options: [
                             {
                                 name: 'Basic',
@@ -731,6 +1019,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                             },
                         ],
                     },
+                    ...(batchMode ? createBatchSpecificProperties() : []),
                 ],
             },
         ],
@@ -744,7 +1033,7 @@ function createScrapeOptionsProperty(operationName, useNestedScrapeOptions = tru
                 useCustomBody: [true],
             },
             show: {
-                resource: ['Default'],
+                resource: [resourceName],
                 operation: [operationName],
             },
         },