n8n-nodes-firecrawl-latest 1.0.0

package/dist/nodes/Firecrawl/resources/batchScrape/batchScrape.methods.js

@@ -0,0 +1,253 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.batchScrapeMethods = void 0;
+const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
+const n8n_workflow_1 = require("n8n-workflow");
+// Helper function to parse URLs input
+function parseUrlsInput(urlsInput) {
+    if (!urlsInput)
+        return [];
+    // Check if input is already a JSON array
+    if (urlsInput.trim().startsWith('[') && urlsInput.trim().endsWith(']')) {
+        try {
+            const parsed = JSON.parse(urlsInput);
+            if (Array.isArray(parsed)) {
+                return parsed.map((url) => url.toString().trim());
+            }
+        }
+        catch (e) {
+            // If JSON parsing fails, continue with other methods
+        }
+    }
+    // Split by comma if it's a comma-separated list
+    if (urlsInput.includes(',')) {
+        return urlsInput.split(',').map((url) => url.trim());
+    }
+    // Single URL case
+    return [urlsInput.trim()];
+}
+// Helper function to generate schema from JSON example
+function generateSchemaFromExample(jsonExample) {
+    if (jsonExample === null) {
+        return { type: 'null' };
+    }
+    if (typeof jsonExample === 'string') {
+        return { type: 'string' };
+    }
+    if (typeof jsonExample === 'number') {
+        return { type: 'number' };
+    }
+    if (typeof jsonExample === 'boolean') {
+        return { type: 'boolean' };
+    }
+    if (Array.isArray(jsonExample)) {
+        if (jsonExample.length === 0) {
+            return {
+                type: 'array',
+                items: { type: 'string' }, // Default to string items for empty arrays
+            };
+        }
+        // Use the first item as a sample for the items schema
+        const itemSchema = generateSchemaFromExample(jsonExample[0]);
+        return {
+            type: 'array',
+            items: itemSchema,
+        };
+    }
+    if (typeof jsonExample === 'object') {
+        const properties = {};
+        for (const [key, value] of Object.entries(jsonExample)) {
+            properties[key] = generateSchemaFromExample(value);
+        }
+        return {
+            type: 'object',
+            properties,
+            required: Object.keys(properties),
+        };
+    }
+    // Default fallback
+    return { type: 'string' };
+}
+exports.batchScrapeMethods = {
+    async execute() {
+        const items = this.getInputData();
+        const returnData = [];
+        // Get credentials
+        const credentials = await this.getCredentials('firecrawlApi');
+        const apiKey = credentials.apiKey;
+        // Initialize Firecrawl app
+        const firecrawl = new firecrawl_js_1.default({ apiKey });
+        // Process each item
+        for (let i = 0; i < items.length; i++) {
+            try {
+                // Get parameters
+                const urlsInput = this.getNodeParameter('urls', i);
+                const urls = parseUrlsInput(urlsInput);
+                const outputFormats = this.getNodeParameter('outputFormats', i);
+                const operationMode = this.getNodeParameter('operationMode', i);
+                const includeExtract = this.getNodeParameter('includeExtract', i, false);
+                const enableDebugLogs = this.getNodeParameter('enableDebugLogs', i, false);
+                if (urls.length === 0) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'No valid URLs provided', {
+                        itemIndex: i,
+                    });
+                }
+                // Prepare the options for batch scraping
+                const options = {
+                    formats: outputFormats,
+                };
+                // Add extraction if needed
+                if (includeExtract) {
+                    const extractionPrompt = this.getNodeParameter('extractionPrompt', i);
+                    const schemaDefinitionType = this.getNodeParameter('schemaDefinitionType', i);
+                    options.formats.push('extract');
+                    options.extract = {
+                        prompt: extractionPrompt,
+                    };
+                    // Add schema if selected
+                    if (schemaDefinitionType === 'example') {
+                        const jsonExample = JSON.parse(this.getNodeParameter('jsonExample', i));
+                        const schema = generateSchemaFromExample(jsonExample);
+                        options.extract.schema = schema;
+                    }
+                    else if (schemaDefinitionType === 'manual') {
+                        const schema = JSON.parse(this.getNodeParameter('schemaDefinition', i));
+                        options.extract.schema = schema;
+                    }
+                }
+                // Log the batch scrape parameters if debug is enabled
+                if (enableDebugLogs) {
+                    console.log('URLs:', urls);
+                    console.log('Options:', JSON.stringify(options, null, 2));
+                }
+                // Handle different operation modes
+                if (operationMode === 'sync') {
+                    // Synchronous mode - wait for the scraping to complete
+                    const results = await firecrawl.batchScrapeUrls(urls, options);
+                    // Log the results if debug is enabled
+                    if (enableDebugLogs) {
+                        console.log('Batch Scrape Results:', JSON.stringify(results, null, 2));
+                    }
+                    if (!results.success && 'error' in results) {
+                        returnData.push({
+                            json: {
+                                success: results.success,
+                                status: 'error',
+                                error: results.error,
+                                debug: enableDebugLogs ? { urls, options } : undefined,
+                            },
+                        });
+                    }
+                    else {
+                        returnData.push({
+                            json: {
+                                success: results.success,
+                                status: 'completed',
+                                data: 'data' in results ? results.data : undefined,
+                                creditsUsed: 'creditsUsed' in results ? results.creditsUsed : undefined,
+                                total: 'total' in results ? results.total : undefined,
+                                completed: 'completed' in results ? results.completed : undefined,
+                                error: 'error' in results ? results.error : undefined,
+                                debug: enableDebugLogs ? { urls, options } : undefined,
+                            },
+                        });
+                    }
+                }
+                else {
+                    // Asynchronous mode - start the process or check status
+                    const jobId = this.getNodeParameter('jobId', i, '');
+                    if (jobId && jobId.trim() !== '') {
+                        // Check status of an existing job
+                        const status = await firecrawl.checkBatchScrapeStatus(jobId);
+                        // Log the status if debug is enabled
+                        if (enableDebugLogs) {
+                            console.log('Batch Scrape Status:', JSON.stringify(status, null, 2));
+                        }
+                        if (!status.success && 'error' in status) {
+                            // It's an error response
+                            returnData.push({
+                                json: {
+                                    success: status.success,
+                                    status: 'error',
+                                    error: status.error,
+                                    jobId,
+                                    debug: enableDebugLogs ? { jobId } : undefined,
+                                },
+                            });
+                        }
+                        else {
+                            // It's a successful status response
+                            returnData.push({
+                                json: {
+                                    success: status.success,
+                                    status: 'status' in status ? status.status : 'unknown',
+                                    data: 'data' in status ? status.data : undefined,
+                                    creditsUsed: 'creditsUsed' in status ? status.creditsUsed : undefined,
+                                    total: 'total' in status ? status.total : undefined,
+                                    completed: 'completed' in status ? status.completed : undefined,
+                                    expiresAt: 'expiresAt' in status ? status.expiresAt : undefined,
+                                    next: 'next' in status ? status.next : undefined,
+                                    error: 'error' in status ? status.error : undefined,
+                                    jobId,
+                                    debug: enableDebugLogs ? { jobId } : undefined,
+                                },
+                            });
+                        }
+                    }
+                    else {
+                        // Start a new asynchronous job
+                        const job = await firecrawl.asyncBatchScrapeUrls(urls, options);
+                        // Log the job if debug is enabled
+                        if (enableDebugLogs) {
+                            console.log('Batch Scrape Job:', JSON.stringify(job, null, 2));
+                        }
+                        if (!job.success && 'error' in job) {
+                            // It's an error response
+                            returnData.push({
+                                json: {
+                                    success: job.success,
+                                    status: 'error',
+                                    error: job.error,
+                                    debug: enableDebugLogs ? { urls, options } : undefined,
+                                },
+                            });
+                        }
+                        else {
+                            // It's a successful job response
+                            returnData.push({
+                                json: {
+                                    success: job.success,
+                                    status: 'started',
+                                    jobId: 'id' in job ? job.id : undefined,
+                                    url: 'url' in job ? job.url : undefined,
+                                    message: 'Batch scraping started successfully. Use the job ID to check status.',
+                                    debug: enableDebugLogs ? { urls, options } : undefined,
+                                },
+                            });
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                console.error('Batch Scrape error:', errorMessage);
+                if (this.continueOnFail()) {
+                    returnData.push({
+                        json: {
+                            success: false,
+                            error: errorMessage,
+                        },
+                    });
+                    continue;
+                }
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
+                    itemIndex: i,
+                });
+            }
+        }
+        return [returnData];
+    },
+};

package/dist/nodes/Firecrawl/resources/batchScrape/batchScrape.properties.js

@@ -0,0 +1,205 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.batchScrapeProperties = void 0;
+// Fields for the Batch Scrape resource
+const batchScrapeFields = [
+    {
+        displayName: 'URLs',
+        name: 'urls',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The URLs to scrape. For multiple URLs, provide a comma-separated list or a JSON array in string format.',
+        placeholder: 'https://example.com, https://example.org or ["https://example.com", "https://example.org"]',
+    },
+    {
+        displayName: 'Output Formats',
+        name: 'outputFormats',
+        type: 'multiOptions',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+            },
+        },
+        options: [
+            {
+                name: 'Full Page Screenshot',
+                value: 'screenshot@fullPage',
+                description: 'Return a screenshot of the entire page',
+            },
+            {
+                name: 'HTML',
+                value: 'html',
+                description: 'Return the content in HTML format (with some cleaning)',
+            },
+            {
+                name: 'Links',
+                value: 'links',
+                description: 'Return a list of links found on the page',
+            },
+            {
+                name: 'Markdown',
+                value: 'markdown',
+                description: 'Return the content in Markdown format',
+            },
+            {
+                name: 'Raw HTML',
+                value: 'rawHtml',
+                description: 'Return the raw HTML content with no modifications',
+            },
+            {
+                name: 'Screenshot',
+                value: 'screenshot',
+                description: 'Return a screenshot of the visible part of the page',
+            },
+        ],
+        default: ['markdown'],
+        description: 'The formats in which to return the scraped content',
+    },
+    {
+        displayName: 'Include Extract',
+        name: 'includeExtract',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to include structured data extraction with the batch scrape',
+    },
+    {
+        displayName: 'Extraction Prompt',
+        name: 'extractionPrompt',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+                includeExtract: [true],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The prompt to guide the extraction process',
+        placeholder: 'Extract the title and description from each page',
+    },
+    {
+        displayName: 'Schema Definition Type',
+        name: 'schemaDefinitionType',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+                includeExtract: [true],
+            },
+        },
+        options: [
+            {
+                name: 'Generate From JSON Example',
+                value: 'example',
+                description: 'Generate schema from a JSON example',
+            },
+            {
+                name: 'Define Below',
+                value: 'manual',
+                description: 'Define schema manually in JSON Schema format',
+            },
+        ],
+        default: 'manual',
+        description: 'How to define the schema for extraction',
+    },
+    {
+        displayName: 'JSON Example',
+        name: 'jsonExample',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 8,
+        },
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+                includeExtract: [true],
+                schemaDefinitionType: ['example'],
+            },
+        },
+        default: '{\n  "title": "Example Page Title",\n  "description": "This is an example page description"\n}',
+        description: 'A JSON example that represents the data structure you want to extract',
+        hint: 'Provide a JSON object that represents the structure you want to extract',
+    },
+    {
+        displayName: 'Schema Definition',
+        name: 'schemaDefinition',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 12,
+        },
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+                includeExtract: [true],
+                schemaDefinitionType: ['manual'],
+            },
+        },
+        default: '{\n  "type": "object",\n  "properties": {\n    "title": {\n      "type": "string",\n      "description": "The title of the page"\n    },\n    "description": {\n      "type": "string",\n      "description": "The meta description or summary of the page"\n    }\n  },\n  "required": ["title", "description"]\n}',
+        description: 'The schema definition in standard JSON Schema format. Define the structure you want to extract.',
+        hint: 'Use standard JSON Schema format with "type", "properties", and optional "required" fields',
+    },
+    {
+        displayName: 'Operation Mode',
+        name: 'operationMode',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+            },
+        },
+        options: [
+            {
+                name: 'Synchronous',
+                value: 'sync',
+                description: 'Wait for the scraping to complete (suitable for smaller batches)',
+            },
+            {
+                name: 'Asynchronous',
+                value: 'async',
+                description: 'Start the scraping process and return a job ID (suitable for larger batches)',
+            },
+        ],
+        default: 'sync',
+        description: 'Whether to wait for the scraping to complete or just start the process',
+    },
+    {
+        displayName: 'Job ID',
+        name: 'jobId',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+                operationMode: ['async'],
+            },
+        },
+        default: '',
+        description: 'Job ID for checking the status of an existing batch scrape (leave empty to start a new job)',
+    },
+    {
+        displayName: 'Enable Debug Logs',
+        name: 'enableDebugLogs',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['batchScrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to enable debug logs in the output',
+    },
+];
+// Export all properties for the Batch Scrape resource
+exports.batchScrapeProperties = [...batchScrapeFields];