n8n-nodes-firecrawl-latest 1.0.0

package/dist/nodes/Firecrawl/resources/crawler/crawler.methods.js

@@ -0,0 +1,281 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.crawlerMethods = void 0;
+const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
+const n8n_workflow_1 = require("n8n-workflow");
+// Helper function to generate schema from JSON example
+function generateSchemaFromExample(jsonExample) {
+    if (jsonExample === null) {
+        return { type: 'null' };
+    }
+    if (typeof jsonExample === 'string') {
+        return { type: 'string' };
+    }
+    if (typeof jsonExample === 'number') {
+        return { type: 'number' };
+    }
+    if (typeof jsonExample === 'boolean') {
+        return { type: 'boolean' };
+    }
+    if (Array.isArray(jsonExample)) {
+        if (jsonExample.length === 0) {
+            return {
+                type: 'array',
+                items: { type: 'string' }, // Default to string items for empty arrays
+            };
+        }
+        // Use the first item as a sample for the items schema
+        const itemSchema = generateSchemaFromExample(jsonExample[0]);
+        return {
+            type: 'array',
+            items: itemSchema,
+        };
+    }
+    if (typeof jsonExample === 'object') {
+        const properties = {};
+        for (const [key, value] of Object.entries(jsonExample)) {
+            properties[key] = generateSchemaFromExample(value);
+        }
+        return {
+            type: 'object',
+            properties,
+            required: Object.keys(properties),
+        };
+    }
+    // Default fallback
+    return { type: 'string' };
+}
+exports.crawlerMethods = {
+    async execute() {
+        const items = this.getInputData();
+        const returnData = [];
+        // Get credentials
+        const credentials = await this.getCredentials('firecrawlApi');
+        const apiKey = credentials.apiKey;
+        // Initialize Firecrawl app
+        const firecrawl = new firecrawl_js_1.default({ apiKey });
+        // Process each item
+        for (let i = 0; i < items.length; i++) {
+            try {
+                // Get parameters
+                const url = this.getNodeParameter('url', i);
+                const limit = this.getNodeParameter('limit', i);
+                const operationMode = this.getNodeParameter('operationMode', i);
+                const outputFormats = this.getNodeParameter('outputFormats', i, ['markdown']);
+                const enableDebugLogs = this.getNodeParameter('enableDebugLogs', i, false);
+                const trackChanges = this.getNodeParameter('trackChanges', i, false);
+                const enableLlmExtraction = this.getNodeParameter('enableLlmExtraction', i, false);
+                // Optional path parameters
+                const includePath = this.getNodeParameter('includePath', i, '');
+                const excludePath = this.getNodeParameter('excludePath', i, '');
+                const maxDepth = this.getNodeParameter('maxDepth', i, 5);
+                // Prepare crawl parameters
+                const crawlParams = {
+                    limit,
+                    maxDepth,
+                };
+                // Add output formats
+                crawlParams.scrapeOptions = {
+                    formats: outputFormats,
+                };
+                // Add path filters if provided
+                if (includePath) {
+                    // Convert to array if it's not already
+                    crawlParams.includePaths = includePath.includes(',')
+                        ? includePath.split(',').map(path => path.trim())
+                        : [includePath.trim()];
+                }
+                if (excludePath) {
+                    // Convert to array if it's not already
+                    crawlParams.excludePaths = excludePath.includes(',')
+                        ? excludePath.split(',').map(path => path.trim())
+                        : [excludePath.trim()];
+                }
+                // Add change tracking if enabled
+                if (trackChanges) {
+                    const changeTrackingModes = this.getNodeParameter('changeTrackingMode', i, ['git-diff']);
+                    if (!crawlParams.scrapeOptions.formats.includes('changeTracking')) {
+                        crawlParams.scrapeOptions.formats.push('changeTracking');
+                    }
+                    crawlParams.scrapeOptions.changeTrackingOptions = {
+                        modes: changeTrackingModes,
+                    };
+                    // Add JSON schema for change tracking if JSON mode is selected
+                    if (changeTrackingModes.includes('json')) {
+                        const changeTrackingSchema = JSON.parse(this.getNodeParameter('changeTrackingSchema', i, '{}'));
+                        crawlParams.scrapeOptions.changeTrackingOptions.schema = changeTrackingSchema;
+                    }
+                }
+                // Add LLM extraction if enabled
+                if (enableLlmExtraction) {
+                    const extractionPrompt = this.getNodeParameter('extractionPrompt', i);
+                    const schemaDefinitionType = this.getNodeParameter('schemaDefinitionType', i);
+                    // Add 'json' to the formats if not already included
+                    if (!crawlParams.scrapeOptions.formats.includes('json')) {
+                        crawlParams.scrapeOptions.formats.push('json');
+                    }
+                    crawlParams.scrapeOptions.jsonOptions = {
+                        prompt: extractionPrompt,
+                    };
+                    // Add schema for extraction
+                    let schema;
+                    if (schemaDefinitionType === 'example') {
+                        const jsonExample = JSON.parse(this.getNodeParameter('jsonExample', i));
+                        schema = generateSchemaFromExample(jsonExample);
+                    }
+                    else {
+                        // Manual schema definition
+                        schema = JSON.parse(this.getNodeParameter('schemaDefinition', i));
+                    }
+                    crawlParams.scrapeOptions.jsonOptions.schema = schema;
+                }
+                // Log the crawler parameters if debug is enabled
+                if (enableDebugLogs) {
+                    console.log('URL:', url);
+                    console.log('Crawl parameters:', JSON.stringify(crawlParams, null, 2));
+                }
+                // Handle different operation modes
+                if (operationMode === 'sync') {
+                    // Synchronous mode - wait for the crawling to complete
+                    const results = await firecrawl.crawlUrl(url, crawlParams);
+                    // Log the results if debug is enabled
+                    if (enableDebugLogs) {
+                        console.log('Crawl results:', JSON.stringify(results, null, 2));
+                    }
+                    if (!results.success && 'error' in results) {
+                        returnData.push({
+                            json: {
+                                success: false,
+                                status: 'error',
+                                error: results.error,
+                                debug: enableDebugLogs
+                                    ? {
+                                        url,
+                                        params: crawlParams,
+                                    }
+                                    : undefined,
+                            },
+                        });
+                    }
+                    else {
+                        returnData.push({
+                            json: {
+                                success: true,
+                                status: 'status' in results ? results.status : 'completed',
+                                data: 'data' in results ? results.data : undefined,
+                                completed: 'completed' in results ? results.completed : undefined,
+                                total: 'total' in results ? results.total : undefined,
+                                creditsUsed: 'creditsUsed' in results ? results.creditsUsed : undefined,
+                                expiresAt: 'expiresAt' in results ? results.expiresAt : undefined,
+                                debug: enableDebugLogs
+                                    ? {
+                                        url,
+                                        params: crawlParams,
+                                    }
+                                    : undefined,
+                            },
+                        });
+                    }
+                }
+                else {
+                    // Asynchronous mode - start the process or check status
+                    const jobId = this.getNodeParameter('jobId', i, '');
+                    if (jobId && jobId.trim() !== '') {
+                        // Check status of an existing job
+                        const status = await firecrawl.checkCrawlStatus(jobId);
+                        // Log the status if debug is enabled
+                        if (enableDebugLogs) {
+                            console.log('Crawl status:', JSON.stringify(status, null, 2));
+                        }
+                        if (!status.success && 'error' in status) {
+                            returnData.push({
+                                json: {
+                                    success: false,
+                                    status: 'error',
+                                    error: status.error,
+                                    jobId,
+                                    debug: enableDebugLogs ? { jobId } : undefined,
+                                },
+                            });
+                        }
+                        else {
+                            returnData.push({
+                                json: {
+                                    success: true,
+                                    status: 'status' in status ? status.status : 'unknown',
+                                    jobId,
+                                    completed: 'completed' in status ? status.completed : undefined,
+                                    total: 'total' in status ? status.total : undefined,
+                                    creditsUsed: 'creditsUsed' in status ? status.creditsUsed : undefined,
+                                    expiresAt: 'expiresAt' in status ? status.expiresAt : undefined,
+                                    data: 'data' in status ? status.data : [],
+                                    next: 'next' in status ? status.next : undefined,
+                                    debug: enableDebugLogs ? { jobId } : undefined,
+                                },
+                            });
+                        }
+                    }
+                    else {
+                        // Start a new asynchronous job
+                        const job = await firecrawl.asyncCrawlUrl(url, crawlParams);
+                        // Log the job if debug is enabled
+                        if (enableDebugLogs) {
+                            console.log('Crawl job:', JSON.stringify(job, null, 2));
+                        }
+                        if (!job.success && 'error' in job) {
+                            returnData.push({
+                                json: {
+                                    success: false,
+                                    status: 'error',
+                                    error: job.error,
+                                    debug: enableDebugLogs
+                                        ? {
+                                            url,
+                                            params: crawlParams,
+                                        }
+                                        : undefined,
+                                },
+                            });
+                        }
+                        else {
+                            returnData.push({
+                                json: {
+                                    success: true,
+                                    status: 'started',
+                                    jobId: 'id' in job ? job.id : undefined,
+                                    message: 'Crawling started successfully. Use the job ID to check status.',
+                                    debug: enableDebugLogs
+                                        ? {
+                                            url,
+                                            params: crawlParams,
+                                        }
+                                        : undefined,
+                                },
+                            });
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                console.error('Crawler error:', errorMessage);
+                if (this.continueOnFail()) {
+                    returnData.push({
+                        json: {
+                            success: false,
+                            error: errorMessage,
+                        },
+                    });
+                    continue;
+                }
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
+                    itemIndex: i,
+                });
+            }
+        }
+        return [returnData];
+    },
+};

package/dist/nodes/Firecrawl/resources/crawler/crawler.properties.js

@@ -0,0 +1,313 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.crawlerProperties = void 0;
+// Fields for the Crawler resource
+const crawlerFields = [
+    {
+        displayName: 'URL',
+        name: 'url',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The starting URL to crawl',
+        placeholder: 'https://example.com',
+    },
+    {
+        displayName: 'Limit',
+        name: 'limit',
+        type: 'number',
+        typeOptions: {
+            minValue: 1,
+        },
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: 50,
+        description: 'Max number of results to return',
+    },
+    {
+        displayName: 'Output Formats',
+        name: 'outputFormats',
+        type: 'multiOptions',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        options: [
+            {
+                name: 'Full Page Screenshot',
+                value: 'screenshot@fullPage',
+                description: 'Return a screenshot of the entire page',
+            },
+            {
+                name: 'HTML',
+                value: 'html',
+                description: 'Return the content in HTML format (with some cleaning)',
+            },
+            {
+                name: 'Links',
+                value: 'links',
+                description: 'Return a list of links found on the page',
+            },
+            {
+                name: 'Markdown',
+                value: 'markdown',
+                description: 'Return the content in Markdown format',
+            },
+            {
+                name: 'Raw HTML',
+                value: 'rawHtml',
+                description: 'Return the raw HTML content with no modifications',
+            },
+            {
+                name: 'Screenshot',
+                value: 'screenshot',
+                description: 'Return a screenshot of the visible part of the page',
+            },
+        ],
+        default: ['markdown'],
+        description: 'The formats in which to return the crawled content',
+    },
+    {
+        displayName: 'Track Changes',
+        name: 'trackChanges',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: false,
+        description: 'Whether to track changes between the current crawl and previous ones',
+    },
+    {
+        displayName: 'Change Tracking Mode',
+        name: 'changeTrackingMode',
+        type: 'multiOptions',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                trackChanges: [true],
+            },
+        },
+        options: [
+            {
+                name: 'Git-Diff',
+                value: 'git-diff',
+                description: 'Track changes using Git-style diff format',
+            },
+            {
+                name: 'JSON',
+                value: 'json',
+                description: 'Track changes in structured JSON format',
+            },
+        ],
+        default: ['git-diff'],
+        description: 'The modes to use for change tracking',
+    },
+    {
+        displayName: 'JSON Schema for Change Tracking',
+        name: 'changeTrackingSchema',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 8,
+        },
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                trackChanges: [true],
+                changeTrackingMode: ['json'],
+            },
+        },
+        default: '{\n  "type": "object",\n  "properties": {\n    "title": {\n      "type": "string",\n      "description": "The title of the page"\n    },\n    "content": {\n      "type": "string",\n      "description": "The main content of the page"\n    }\n  }\n}',
+        description: 'Schema for JSON-based change tracking',
+    },
+    {
+        displayName: 'Include Path Pattern',
+        name: 'includePath',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: '',
+        description: 'Regex pattern for paths to include (e.g., /blog/.*)',
+    },
+    {
+        displayName: 'Exclude Path Pattern',
+        name: 'excludePath',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: '',
+        description: 'Regex pattern for paths to exclude (e.g., /wp-admin/.*)',
+    },
+    {
+        displayName: 'Maximum Depth',
+        name: 'maxDepth',
+        type: 'number',
+        typeOptions: {
+            minValue: 1,
+            maxValue: 10,
+        },
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: 5,
+        description: 'Maximum depth of links to follow (1-10)',
+    },
+    {
+        displayName: 'Operation Mode',
+        name: 'operationMode',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        options: [
+            {
+                name: 'Synchronous',
+                value: 'sync',
+                description: 'Wait for the crawling to complete (suitable for smaller sites)',
+            },
+            {
+                name: 'Asynchronous',
+                value: 'async',
+                description: 'Start the crawling process and return a job ID (suitable for larger sites)',
+            },
+        ],
+        default: 'sync',
+        description: 'Whether to wait for the crawling to complete or just start the process',
+    },
+    {
+        displayName: 'Job ID',
+        name: 'jobId',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                operationMode: ['async'],
+            },
+        },
+        default: '',
+        description: 'Job ID for checking the status of an existing crawl (leave empty to start a new job)',
+    },
+    {
+        displayName: 'Enable LLM Extraction',
+        name: 'enableLlmExtraction',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: false,
+        description: 'Whether to extract structured data from crawled pages using LLMs',
+    },
+    {
+        displayName: 'Extraction Prompt',
+        name: 'extractionPrompt',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                enableLlmExtraction: [true],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The prompt to guide the extraction process',
+        placeholder: 'Extract the title and main content from each page',
+    },
+    {
+        displayName: 'Schema Definition Type',
+        name: 'schemaDefinitionType',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                enableLlmExtraction: [true],
+            },
+        },
+        options: [
+            {
+                name: 'Generate From JSON Example',
+                value: 'example',
+                description: 'Generate schema from a JSON example',
+            },
+            {
+                name: 'Define Below',
+                value: 'manual',
+                description: 'Define schema manually in JSON Schema format',
+            },
+        ],
+        default: 'manual',
+        description: 'How to define the schema for extraction',
+    },
+    {
+        displayName: 'JSON Example',
+        name: 'jsonExample',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 8,
+        },
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                enableLlmExtraction: [true],
+                schemaDefinitionType: ['example'],
+            },
+        },
+        default: '{\n  "title": "Example Page Title",\n  "content": "This is the main content of the page"\n}',
+        description: 'A JSON example that represents the data structure you want to extract',
+    },
+    {
+        displayName: 'Schema Definition',
+        name: 'schemaDefinition',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 10,
+        },
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+                enableLlmExtraction: [true],
+                schemaDefinitionType: ['manual'],
+            },
+        },
+        default: '{\n  "type": "object",\n  "properties": {\n    "title": {\n      "type": "string",\n      "description": "The title of the page"\n    },\n    "content": {\n      "type": "string",\n      "description": "The main content of the page"\n    }\n  },\n  "required": ["title", "content"]\n}',
+        description: 'The schema definition in standard JSON Schema format',
+    },
+    {
+        displayName: 'Enable Debug Logs',
+        name: 'enableDebugLogs',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['crawler'],
+            },
+        },
+        default: false,
+        description: 'Whether to enable debug logs in the output',
+    },
+];
+// Export all properties for the Crawler resource
+exports.crawlerProperties = [...crawlerFields];