n8n-nodes-firecrawl-latest 1.0.0

package/dist/nodes/Firecrawl/resources/scrape/scrape.methods.js

@@ -0,0 +1,203 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapeMethods = void 0;
+const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
+const n8n_workflow_1 = require("n8n-workflow");
+// Helper function to generate schema from JSON example
+function generateSchemaFromExample(jsonExample) {
+    if (jsonExample === null) {
+        return { type: 'null' };
+    }
+    if (typeof jsonExample === 'string') {
+        return { type: 'string' };
+    }
+    if (typeof jsonExample === 'number') {
+        return { type: 'number' };
+    }
+    if (typeof jsonExample === 'boolean') {
+        return { type: 'boolean' };
+    }
+    if (Array.isArray(jsonExample)) {
+        if (jsonExample.length === 0) {
+            return {
+                type: 'array',
+                items: { type: 'string' }, // Default to string items for empty arrays
+            };
+        }
+        // Use the first item as a sample for the items schema
+        const itemSchema = generateSchemaFromExample(jsonExample[0]);
+        return {
+            type: 'array',
+            items: itemSchema,
+        };
+    }
+    if (typeof jsonExample === 'object') {
+        const properties = {};
+        for (const [key, value] of Object.entries(jsonExample)) {
+            properties[key] = generateSchemaFromExample(value);
+        }
+        return {
+            type: 'object',
+            properties,
+            required: Object.keys(properties),
+        };
+    }
+    // Default fallback
+    return { type: 'string' };
+}
+exports.scrapeMethods = {
+    async execute() {
+        const items = this.getInputData();
+        const returnData = [];
+        // Get credentials
+        const credentials = await this.getCredentials('firecrawlApi');
+        const apiKey = credentials.apiKey;
+        // Initialize Firecrawl app
+        const firecrawl = new firecrawl_js_1.default({ apiKey });
+        // Process each item
+        for (let i = 0; i < items.length; i++) {
+            try {
+                // Get parameters
+                const url = this.getNodeParameter('url', i);
+                const outputFormats = this.getNodeParameter('outputFormats', i, ['markdown']);
+                const enableDebugLogs = this.getNodeParameter('enableDebugLogs', i, false);
+                const includeExtract = this.getNodeParameter('includeExtract', i, false);
+                const includeActions = this.getNodeParameter('includeActions', i, false);
+                const includeLocation = this.getNodeParameter('includeLocation', i, false);
+                const trackChanges = this.getNodeParameter('trackChanges', i, false);
+                const useFire1Agent = this.getNodeParameter('useFire1Agent', i, false);
+                // Create scrape options
+                const scrapeOptions = {
+                    formats: outputFormats,
+                };
+                // Add change tracking if enabled
+                if (trackChanges) {
+                    const changeTrackingModes = this.getNodeParameter('changeTrackingMode', i, ['git-diff']);
+                    if (!scrapeOptions.formats.includes('changeTracking')) {
+                        scrapeOptions.formats.push('changeTracking');
+                    }
+                    scrapeOptions.changeTrackingOptions = {
+                        modes: changeTrackingModes,
+                    };
+                    // Add JSON schema for change tracking if JSON mode is selected
+                    if (changeTrackingModes.includes('json')) {
+                        const changeTrackingSchema = JSON.parse(this.getNodeParameter('changeTrackingSchema', i, '{}'));
+                        scrapeOptions.changeTrackingOptions.schema = changeTrackingSchema;
+                    }
+                }
+                // Add extraction if needed
+                if (includeExtract) {
+                    const extractionMethod = this.getNodeParameter('extractionMethod', i);
+                    // We need to use 'json' as the format name for extraction
+                    if (!scrapeOptions.formats.includes('json')) {
+                        scrapeOptions.formats.push('json');
+                    }
+                    scrapeOptions.jsonOptions = {};
+                    if (extractionMethod === 'simple') {
+                        const extractionPrompt = this.getNodeParameter('extractionPrompt', i);
+                        scrapeOptions.jsonOptions.prompt = extractionPrompt;
+                    }
+                    else {
+                        // Schema-based extraction
+                        const schemaDefinitionType = this.getNodeParameter('schemaDefinitionType', i);
+                        let schema;
+                        if (schemaDefinitionType === 'example') {
+                            const jsonExample = JSON.parse(this.getNodeParameter('jsonExample', i));
+                            schema = generateSchemaFromExample(jsonExample);
+                        }
+                        else {
+                            // Manual schema definition
+                            schema = JSON.parse(this.getNodeParameter('schemaDefinition', i));
+                        }
+                        scrapeOptions.jsonOptions.schema = schema;
+                    }
+                    // Add system prompt if provided
+                    const systemPrompt = this.getNodeParameter('systemPrompt', i, '');
+                    if (systemPrompt) {
+                        scrapeOptions.jsonOptions.systemPrompt = systemPrompt;
+                    }
+                }
+                // Add FIRE-1 agent if enabled
+                if (useFire1Agent) {
+                    scrapeOptions.agent = {
+                        model: 'FIRE-1',
+                    };
+                }
+                // Add page actions if needed
+                if (includeActions) {
+                    const pageActionsString = this.getNodeParameter('pageActions', i);
+                    try {
+                        const pageActions = JSON.parse(pageActionsString);
+                        if (useFire1Agent) {
+                            // If using FIRE-1, add actions to the agent config
+                            scrapeOptions.agent.actions = pageActions;
+                        }
+                        else {
+                            // Otherwise add them as regular actions
+                            scrapeOptions.actions = pageActions;
+                        }
+                    }
+                    catch (e) {
+                        throw new Error(`Invalid page actions JSON: ${e instanceof Error ? e.message : String(e)}`);
+                    }
+                }
+                // Add location settings if needed
+                if (includeLocation) {
+                    const country = this.getNodeParameter('country', i, 'US');
+                    const languagesString = this.getNodeParameter('languages', i, '');
+                    scrapeOptions.location = {
+                        country,
+                    };
+                    if (languagesString) {
+                        const languages = languagesString.split(',').map(lang => lang.trim());
+                        scrapeOptions.location.languages = languages;
+                    }
+                }
+                // Log the scrape parameters if debug is enabled
+                if (enableDebugLogs) {
+                    console.log('URL:', url);
+                    console.log('Scrape options:', JSON.stringify(scrapeOptions, null, 2));
+                }
+                // Scrape URL
+                const scrapeResponse = await firecrawl.scrapeUrl(url, scrapeOptions);
+                // Log the result if debug is enabled
+                if (enableDebugLogs) {
+                    console.log('Scrape result:', JSON.stringify(scrapeResponse, null, 2));
+                }
+                // Add result to return data
+                returnData.push({
+                    json: {
+                        success: true,
+                        data: scrapeResponse,
+                        debug: enableDebugLogs
+                            ? {
+                                url,
+                                options: scrapeOptions,
+                            }
+                            : undefined,
+                    },
+                });
+            }
+            catch (error) {
+                const errorMessage = error instanceof Error ? error.message : String(error);
+                console.error('Scrape error:', errorMessage);
+                if (this.continueOnFail()) {
+                    returnData.push({
+                        json: {
+                            success: false,
+                            error: errorMessage,
+                        },
+                    });
+                    continue;
+                }
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), error, {
+                    itemIndex: i,
+                });
+            }
+        }
+        return [returnData];
+    },
+};

package/dist/nodes/Firecrawl/resources/scrape/scrape.properties.js

@@ -0,0 +1,348 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scrapeProperties = void 0;
+// Fields for the Scrape resource
+const scrapeFields = [
+    {
+        displayName: 'URL',
+        name: 'url',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The URL to scrape',
+        placeholder: 'https://example.com',
+    },
+    {
+        displayName: 'Output Formats',
+        name: 'outputFormats',
+        type: 'multiOptions',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        options: [
+            {
+                name: 'Full Page Screenshot',
+                value: 'screenshot@fullPage',
+                description: 'Return a screenshot of the entire page',
+            },
+            {
+                name: 'HTML',
+                value: 'html',
+                description: 'Return the content in HTML format (with some cleaning)',
+            },
+            {
+                name: 'Links',
+                value: 'links',
+                description: 'Return a list of links found on the page',
+            },
+            {
+                name: 'Markdown',
+                value: 'markdown',
+                description: 'Return the content in Markdown format',
+            },
+            {
+                name: 'Raw HTML',
+                value: 'rawHtml',
+                description: 'Return the raw HTML content with no modifications',
+            },
+            {
+                name: 'Screenshot',
+                value: 'screenshot',
+                description: 'Return a screenshot of the visible part of the page',
+            },
+        ],
+        default: ['markdown'],
+        description: 'The formats in which to return the scraped content',
+    },
+    {
+        displayName: 'Track Changes',
+        name: 'trackChanges',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to track changes between the current and previous scrapes',
+    },
+    {
+        displayName: 'Change Tracking Mode',
+        name: 'changeTrackingMode',
+        type: 'multiOptions',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                trackChanges: [true],
+            },
+        },
+        options: [
+            {
+                name: 'Git-Diff',
+                value: 'git-diff',
+                description: 'Track changes using Git-style diff format',
+            },
+            {
+                name: 'JSON',
+                value: 'json',
+                description: 'Track changes in structured JSON format',
+            },
+        ],
+        default: ['git-diff'],
+        description: 'The modes to use for change tracking',
+    },
+    {
+        displayName: 'JSON Schema for Change Tracking',
+        name: 'changeTrackingSchema',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 8,
+        },
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                trackChanges: [true],
+                changeTrackingMode: ['json'],
+            },
+        },
+        default: '{\n  "type": "object",\n  "properties": {\n    "title": {\n      "type": "string",\n      "description": "The title of the page"\n    },\n    "content": {\n      "type": "string",\n      "description": "The main content of the page"\n    }\n  }\n}',
+        description: 'Schema for JSON-based change tracking',
+    },
+    {
+        displayName: 'Include Extract',
+        name: 'includeExtract',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to include structured data extraction with the scrape',
+    },
+    {
+        displayName: 'Extraction Method',
+        name: 'extractionMethod',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+            },
+        },
+        options: [
+            {
+                name: 'Simple Extraction',
+                value: 'simple',
+                description: 'Extract data using a prompt',
+            },
+            {
+                name: 'Schema Based Extraction',
+                value: 'schema',
+                description: 'Extract data using a defined schema',
+            },
+        ],
+        default: 'simple',
+        description: 'The method to use for extraction',
+    },
+    {
+        displayName: 'Extraction Prompt',
+        name: 'extractionPrompt',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+                extractionMethod: ['simple'],
+            },
+        },
+        default: '',
+        required: true,
+        description: 'The prompt to guide the extraction process',
+        placeholder: 'Extract the company mission from the page',
+    },
+    {
+        displayName: 'System Prompt',
+        name: 'systemPrompt',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+            },
+        },
+        default: '',
+        description: 'Optional system prompt to provide additional context for the extraction',
+        placeholder: 'You are a data extraction assistant that extracts specific information from web content.',
+    },
+    {
+        displayName: 'Schema Definition Type',
+        name: 'schemaDefinitionType',
+        type: 'options',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+                extractionMethod: ['schema'],
+            },
+        },
+        options: [
+            {
+                name: 'Generate From JSON Example',
+                value: 'example',
+                description: 'Generate schema from a JSON example',
+            },
+            {
+                name: 'Define Below',
+                value: 'manual',
+                description: 'Define schema manually in JSON Schema format',
+            },
+        ],
+        default: 'manual',
+        description: 'How to define the schema for extraction',
+    },
+    {
+        displayName: 'JSON Example',
+        name: 'jsonExample',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 8,
+        },
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+                extractionMethod: ['schema'],
+                schemaDefinitionType: ['example'],
+            },
+        },
+        default: '{\n  "company_mission": "Our mission is to...",\n  "supports_sso": true,\n  "is_open_source": false\n}',
+        description: 'A JSON example that represents the data structure you want to extract',
+        hint: 'Provide a JSON object that represents the structure you want to extract',
+    },
+    {
+        displayName: 'Schema Definition',
+        name: 'schemaDefinition',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 12,
+        },
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeExtract: [true],
+                extractionMethod: ['schema'],
+                schemaDefinitionType: ['manual'],
+            },
+        },
+        default: '{\n  "type": "object",\n  "properties": {\n    "company_mission": {\n      "type": "string",\n      "description": "The mission statement of the company"\n    },\n    "supports_sso": {\n      "type": "boolean",\n      "description": "Whether the product supports Single Sign-On"\n    },\n    "is_open_source": {\n      "type": "boolean",\n      "description": "Whether the product is open source"\n    }\n  },\n  "required": ["company_mission"]\n}',
+        description: 'The schema definition in standard JSON Schema format. Define the structure you want to extract.',
+        hint: 'Use standard JSON Schema format with "type", "properties", and optional "required" fields',
+    },
+    {
+        displayName: 'Include Page Actions',
+        name: 'includeActions',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to perform actions on the page before scraping (for dynamic content)',
+    },
+    {
+        displayName: 'Page Actions',
+        name: 'pageActions',
+        type: 'json',
+        typeOptions: {
+            alwaysOpenEditWindow: true,
+            rows: 12,
+        },
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeActions: [true],
+            },
+        },
+        default: '[\n  { "type": "wait", "milliseconds": 2000 },\n  { "type": "click", "selector": "button.accept-cookies" },\n  { "type": "wait", "milliseconds": 1000 },\n  { "type": "scrape" }\n]',
+        description: 'Actions to perform on the page before scraping (JSON array of action objects)',
+        hint: 'Common actions: wait, click, write, press, scrape, screenshot. See docs for full list.',
+    },
+    {
+        displayName: 'Use FIRE-1 Agent',
+        name: 'useFire1Agent',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to use the FIRE-1 agent for advanced scraping capabilities',
+    },
+    {
+        displayName: 'Location Settings',
+        name: 'includeLocation',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to specify location and language settings for the request',
+    },
+    {
+        displayName: 'Country Code',
+        name: 'country',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeLocation: [true],
+            },
+        },
+        default: 'US',
+        description: 'ISO 3166-1 alpha-2 country code (e.g., US, DE, JP, BR)',
+        placeholder: 'US',
+    },
+    {
+        displayName: 'Languages',
+        name: 'languages',
+        type: 'string',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+                includeLocation: [true],
+            },
+        },
+        default: '',
+        description: 'Comma-separated list of preferred languages (e.g., en-US,en,fr)',
+        placeholder: 'en-US,en',
+    },
+    {
+        displayName: 'Enable Debug Logs',
+        name: 'enableDebugLogs',
+        type: 'boolean',
+        displayOptions: {
+            show: {
+                resource: ['scrape'],
+            },
+        },
+        default: false,
+        description: 'Whether to enable debug logs in the output',
+    },
+];
+// Export all properties for the Scrape resource
+exports.scrapeProperties = [...scrapeFields];

package/dist/nodes/HttpBin/HttpBin.node.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HttpBin = void 0;
+const HttpVerbDescription_1 = require("./HttpVerbDescription");
+class HttpBin {
+    constructor() {
+        this.description = {
+            displayName: 'HttpBin',
+            name: 'httpBin',
+            icon: 'file:httpbin.svg',
+            group: ['transform'],
+            version: 1,
+            subtitle: '={{$parameter["operation"] + ": " + $parameter["resource"]}}',
+            description: 'Interact with HttpBin API',
+            defaults: {
+                name: 'HttpBin',
+            },
+            inputs: ['main'],
+            outputs: ['main'],
+            requestDefaults: {
+                baseURL: 'https://httpbin.org',
+                url: '',
+                headers: {
+                    Accept: 'application/json',
+                    'Content-Type': 'application/json',
+                },
+            },
+            /**
+             * In the properties array we have two mandatory options objects required
+             *
+             * [Resource & Operation]
+             *
+             * https://docs.n8n.io/integrations/creating-nodes/code/create-first-node/#resources-and-operations
+             *
+             * In our example, the operations are separated into their own file (HTTPVerbDescription.ts)
+             * to keep this class easy to read.
+             *
+             */
+            properties: [
+                {
+                    displayName: 'Resource',
+                    name: 'resource',
+                    type: 'options',
+                    noDataExpression: true,
+                    options: [
+                        {
+                            name: 'HTTP Verb',
+                            value: 'httpVerb',
+                        },
+                    ],
+                    default: 'httpVerb',
+                },
+                ...HttpVerbDescription_1.httpVerbOperations,
+                ...HttpVerbDescription_1.httpVerbFields,
+            ],
+        };
+    }
+}
+exports.HttpBin = HttpBin;