n8n-nodes-crawl4ai-plus 2.0.0

package/dist/nodes/Crawl4aiContentExtractor/actions/llmExtractor.operation.js

@@ -0,0 +1,786 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = void 0;
+exports.execute = execute;
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../helpers/utils");
+const formatters_1 = require("../../Crawl4aiBasicCrawler/helpers/formatters");
+exports.description = [
+    {
+        displayName: 'URL',
+        name: 'url',
+        type: 'string',
+        required: true,
+        default: '',
+        placeholder: 'https://example.com',
+        description: 'The URL to extract content from',
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+    },
+    {
+        displayName: 'Extraction Instructions',
+        name: 'instruction',
+        type: 'string',
+        typeOptions: {
+            rows: 4,
+        },
+        required: true,
+        default: '',
+        placeholder: 'Extract the product name, price, and description from this page.',
+        description: 'Instructions for the LLM on what to extract from the page',
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+    },
+    {
+        displayName: 'Schema Input Mode',
+        name: 'schemaMode',
+        type: 'options',
+        options: [
+            {
+                name: 'Simple Fields',
+                value: 'simple',
+                description: 'Define schema using individual field inputs',
+            },
+            {
+                name: 'Advanced JSON',
+                value: 'advanced',
+                description: 'Define schema using JSON editor',
+            },
+        ],
+        default: 'simple',
+        description: 'Choose how to define the extraction schema',
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+    },
+    {
+        displayName: 'Schema Fields',
+        name: 'schemaFields',
+        placeholder: 'Add Schema Field',
+        type: 'fixedCollection',
+        typeOptions: {
+            multipleValues: true,
+        },
+        default: {},
+        required: true,
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+                schemaMode: ['simple'],
+            },
+        },
+        options: [
+            {
+                name: 'fieldsValues',
+                displayName: 'Fields',
+                values: [
+                    {
+                        displayName: 'Field Name',
+                        name: 'name',
+                        type: 'string',
+                        required: true,
+                        default: '',
+                        placeholder: 'title',
+                        description: 'Name of the field to extract',
+                    },
+                    {
+                        displayName: 'Field Type',
+                        name: 'fieldType',
+                        type: 'options',
+                        options: [
+                            {
+                                name: 'String',
+                                value: 'string',
+                                description: 'Plain text string',
+                            },
+                            {
+                                name: 'Number',
+                                value: 'number',
+                                description: 'Numeric value',
+                            },
+                            {
+                                name: 'Boolean',
+                                value: 'boolean',
+                                description: 'True/false value',
+                            },
+                            {
+                                name: 'Array',
+                                value: 'array',
+                                description: 'Array of values',
+                            },
+                        ],
+                        default: 'string',
+                        description: 'Type of the field',
+                    },
+                    {
+                        displayName: 'Description',
+                        name: 'description',
+                        type: 'string',
+                        default: '',
+                        placeholder: 'The main title of the product',
+                        description: 'Description of the field to help the LLM understand what to extract',
+                    },
+                    {
+                        displayName: 'Required',
+                        name: 'required',
+                        type: 'boolean',
+                        default: true,
+                        description: 'Whether this field is required',
+                    },
+                ],
+            },
+        ],
+    },
+    {
+        displayName: 'JSON Schema',
+        name: 'jsonSchema',
+        type: 'string',
+        default: `{
+  "type": "object",
+  "properties": {
+    "title": {
+      "type": "string",
+      "description": "Main page title"
+    },
+    "description": {
+      "type": "string",
+      "description": "Page description or summary"
+    }
+  },
+  "required": ["title"]
+}`,
+        placeholder: `{
+  "type": "object",
+  "properties": {
+    "title": {
+      "type": "string",
+      "description": "Main page title"
+    },
+    "price": {
+      "type": "number",
+      "description": "Product price"
+    },
+    "features": {
+      "type": "array",
+      "items": {"type": "string"},
+      "description": "List of product features"
+    }
+  },
+  "required": ["title", "price"]
+}`,
+        description: 'JSON schema defining the structure of data to extract. Must be valid JSON format.',
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+                schemaMode: ['advanced'],
+            },
+        },
+        typeOptions: {
+            rows: 12,
+            alwaysOpenEditWindow: false,
+        },
+    },
+    {
+        displayName: 'Browser Options',
+        name: 'browserOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Browser Type',
+                name: 'browserType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Chromium',
+                        value: 'chromium',
+                        description: 'Use Chromium browser (default, most compatible)',
+                    },
+                    {
+                        name: 'Firefox',
+                        value: 'firefox',
+                        description: 'Use Firefox browser',
+                    },
+                    {
+                        name: 'Webkit',
+                        value: 'webkit',
+                        description: 'Use Webkit browser (Safari engine)',
+                    },
+                ],
+                default: 'chromium',
+                description: 'Which browser engine to use for crawling',
+            },
+            {
+                displayName: 'Enable JavaScript',
+                name: 'javaScriptEnabled',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to enable JavaScript execution',
+            },
+            {
+                displayName: 'Enable Stealth Mode',
+                name: 'enableStealth',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to enable stealth mode to bypass basic bot detection (hides webdriver properties and modifies browser fingerprints)',
+            },
+            {
+                displayName: 'Headless Mode',
+                name: 'headless',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to run browser in headless mode',
+            },
+            {
+                displayName: 'JavaScript Code',
+                name: 'jsCode',
+                type: 'string',
+                typeOptions: {
+                    rows: 4,
+                },
+                default: '',
+                placeholder: 'document.querySelector("button.load-more").click();',
+                description: 'JavaScript code to execute before extraction (e.g., to click buttons, scroll)',
+            },
+            {
+                displayName: 'Timeout (MS)',
+                name: 'timeout',
+                type: 'number',
+                default: 60000,
+                description: 'Maximum time to wait for the browser to load the page',
+            },
+            {
+                displayName: 'Viewport Height',
+                name: 'viewportHeight',
+                type: 'number',
+                default: 800,
+                description: 'The height of the browser viewport',
+            },
+            {
+                displayName: 'Viewport Width',
+                name: 'viewportWidth',
+                type: 'number',
+                default: 1280,
+                description: 'The width of the browser viewport',
+            },
+        ],
+    },
+    {
+        displayName: 'LLM Options',
+        name: 'llmOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+        options: [
+            {
+                displayName: 'LLM Provider',
+                name: 'llmProvider',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Anthropic Claude 3 Haiku',
+                        value: 'anthropic/claude-3-haiku-20240307',
+                        description: 'Claude 3 Haiku (Fast)',
+                    },
+                    {
+                        name: 'Anthropic Claude 3 Opus',
+                        value: 'anthropic/claude-3-opus-20240229',
+                        description: 'Claude 3 Opus (Most Capable)',
+                    },
+                    {
+                        name: 'Anthropic Claude 3 Sonnet',
+                        value: 'anthropic/claude-3-sonnet-20240229',
+                    },
+                    {
+                        name: 'Anthropic Claude 3.5 Sonnet',
+                        value: 'anthropic/claude-3-5-sonnet-20241022',
+                    },
+                    {
+                        name: 'Anthropic Claude 3.7 Sonnet',
+                        value: 'anthropic/claude-3-7-sonnet-20250219',
+                        description: 'Claude 3.7 Sonnet (Latest, Best)',
+                    },
+                    {
+                        name: 'DeepSeek Chat',
+                        value: 'deepseek/deepseek-chat',
+                        description: 'DeepSeek Chat (Affordable)',
+                    },
+                    {
+                        name: 'DeepSeek Coder',
+                        value: 'deepseek/deepseek-coder',
+                        description: 'DeepSeek Coder (Code-Focused)',
+                    },
+                    {
+                        name: 'Google Gemini 1.5 Flash',
+                        value: 'gemini/gemini-1.5-flash',
+                        description: 'Gemini 1.5 Flash (Fast)',
+                    },
+                    {
+                        name: 'Google Gemini 1.5 Pro',
+                        value: 'gemini/gemini-1.5-pro',
+                        description: 'Gemini 1.5 Pro (Large Context)',
+                    },
+                    {
+                        name: 'Google Gemini Pro',
+                        value: 'gemini/gemini-pro',
+                    },
+                    {
+                        name: 'Groq Llama 3 70B',
+                        value: 'groq/llama3-70b-8192',
+                        description: 'Groq Llama 3 70B (Fast)',
+                    },
+                    {
+                        name: 'Groq Llama 3.1 70B',
+                        value: 'groq/llama-3.1-70b-versatile',
+                        description: 'Groq Llama 3.1 70B (Fast)',
+                    },
+                    {
+                        name: 'Groq Llama 3.3 70B',
+                        value: 'groq/llama-3.3-70b-versatile',
+                        description: 'Groq Llama 3.3 70B (Fast)',
+                    },
+                    {
+                        name: 'Groq Mixtral 8x7B',
+                        value: 'groq/mixtral-8x7b-32768',
+                        description: 'Groq Mixtral 8x7B (Fast)',
+                    },
+                    {
+                        name: 'Ollama Llama 3',
+                        value: 'ollama/llama3',
+                        description: 'Ollama Llama 3 (Local)',
+                    },
+                    {
+                        name: 'Ollama Llama 3.3',
+                        value: 'ollama/llama3.3',
+                        description: 'Ollama Llama 3.3 (Local)',
+                    },
+                    {
+                        name: 'Ollama Mistral',
+                        value: 'ollama/mistral',
+                        description: 'Ollama Mistral (Local)',
+                    },
+                    {
+                        name: 'Ollama Qwen 2.5',
+                        value: 'ollama/qwen2.5',
+                        description: 'Ollama Qwen 2.5 (Local)',
+                    },
+                    {
+                        name: 'OpenAI GPT-3.5 Turbo',
+                        value: 'openai/gpt-3.5-turbo',
+                        description: 'OpenAI GPT-3.5 Turbo (Fast)',
+                    },
+                    {
+                        name: 'OpenAI GPT-4 Turbo',
+                        value: 'openai/gpt-4-turbo',
+                    },
+                    {
+                        name: 'OpenAI GPT-4o',
+                        value: 'openai/gpt-4o',
+                        description: 'OpenAI GPT-4o (Recommended)',
+                    },
+                    {
+                        name: 'OpenAI GPT-4o Mini',
+                        value: 'openai/gpt-4o-mini',
+                        description: 'OpenAI GPT-4o Mini (Fast & Affordable)',
+                    },
+                ],
+                default: 'openai/gpt-4o-mini',
+                description: 'LLM provider to use for extraction. Supports 100+ models via LiteLLM.',
+                displayOptions: {
+                    show: {
+                        overrideProvider: [true],
+                    },
+                },
+            },
+            {
+                displayName: 'Max Tokens',
+                name: 'maxTokens',
+                type: 'number',
+                default: 2000,
+                description: 'Maximum number of tokens for the LLM response',
+            },
+            {
+                displayName: 'Override LLM Provider',
+                name: 'overrideProvider',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to override the LLM provider from credentials',
+            },
+            {
+                displayName: 'Provider API Key',
+                name: 'apiKey',
+                type: 'string',
+                typeOptions: {
+                    password: true,
+                },
+                default: '',
+                description: 'API key for the LLM provider (leave empty to use API key from credentials)',
+                displayOptions: {
+                    show: {
+                        overrideProvider: [true],
+                    },
+                },
+            },
+            {
+                displayName: 'Temperature',
+                name: 'temperature',
+                type: 'number',
+                typeOptions: {
+                    minValue: 0,
+                    maxValue: 1,
+                    numberPrecision: 1,
+                },
+                default: 0,
+                description: 'Controls randomness: 0 for deterministic results, higher for more creativity',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['llmExtractor'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Array Handling',
+                name: 'arrayHandling',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Keep As Object (Default)',
+                        value: 'none',
+                        description: 'Maintain current behavior - arrays become indexed properties',
+                    },
+                    {
+                        name: 'Split Top-Level Arrays',
+                        value: 'topLevel',
+                        description: 'Create separate items only for arrays at root level',
+                    },
+                    {
+                        name: 'Split All Object Arrays',
+                        value: 'allObjects',
+                        description: 'Split any array containing objects, preserve primitive arrays',
+                    },
+                    {
+                        name: 'Smart Split',
+                        value: 'smart',
+                        description: 'Automatically detect main content arrays and split intelligently',
+                    },
+                ],
+                default: 'none',
+                description: 'How to handle arrays in the extracted data',
+            },
+            {
+                displayName: 'Cache Mode',
+                name: 'cacheMode',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Bypass (Skip Cache)',
+                        value: 'BYPASS',
+                        description: 'Skip cache for this operation, fetch fresh content',
+                    },
+                    {
+                        name: 'Disabled (No Cache)',
+                        value: 'DISABLED',
+                        description: 'No caching at all',
+                    },
+                    {
+                        name: 'Enabled (Read/Write)',
+                        value: 'ENABLED',
+                        description: 'Use cache if available, save new results to cache',
+                    },
+                    {
+                        name: 'Read Only',
+                        value: 'READ_ONLY',
+                        description: 'Only read from cache, do not write new results',
+                    },
+                    {
+                        name: 'Write Only',
+                        value: 'WRITE_ONLY',
+                        description: 'Only write to cache, do not read existing cache',
+                    },
+                ],
+                default: 'ENABLED',
+                description: 'How to use the cache when crawling',
+            },
+            {
+                displayName: 'CSS Selector',
+                name: 'cssSelector',
+                type: 'string',
+                default: '',
+                placeholder: 'article.content',
+                description: 'CSS selector to focus extraction on a specific part of the page (leave empty for full page)',
+            },
+            {
+                displayName: 'Include Metadata in Split Items',
+                name: 'includeMetadataInSplitItems',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include URL, success, and other metadata in each split item (reduces redundancy when disabled)',
+                displayOptions: {
+                    show: {
+                        arrayHandling: ['topLevel', 'allObjects', 'smart'],
+                    },
+                },
+            },
+            {
+                displayName: 'Include Original Text',
+                name: 'includeFullText',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include the original webpage text in output',
+            },
+        ],
+    },
+];
+function getNumericKeys(obj) {
+    return Object.keys(obj).filter(key => /^\d+$/.test(key)).sort((a, b) => parseInt(a) - parseInt(b));
+}
+function getMetadataKeys(obj) {
+    return Object.keys(obj).filter(key => !/^\d+$/.test(key));
+}
+function detectMainArray(obj) {
+    const numericKeys = getNumericKeys(obj);
+    if (numericKeys.length === 0)
+        return null;
+    const firstItem = obj[numericKeys[0]];
+    if (typeof firstItem === 'object' && firstItem !== null) {
+        const keyCount = Object.keys(firstItem).length;
+        if (keyCount >= 2) {
+            return 'numeric';
+        }
+    }
+    return null;
+}
+function processArrayHandling(data, strategy, baseMetadata, includeMetadata = true) {
+    if (strategy === 'none') {
+        return [data];
+    }
+    const numericKeys = getNumericKeys(data);
+    const baseData = {};
+    if (includeMetadata) {
+        const metadata = getMetadataKeys(data);
+        metadata.forEach(key => {
+            baseData[key] = data[key];
+        });
+        Object.entries(baseMetadata).forEach(([key, value]) => {
+            baseData[key] = value;
+        });
+    }
+    if (numericKeys.length === 0) {
+        return [data];
+    }
+    switch (strategy) {
+        case 'topLevel':
+            return numericKeys.map(key => {
+                const itemData = data[key];
+                if (typeof itemData === 'object' && itemData !== null) {
+                    return {
+                        ...baseData,
+                        ...itemData
+                    };
+                }
+                return {
+                    ...baseData,
+                    value: itemData
+                };
+            });
+        case 'allObjects':
+            const firstItem = data[numericKeys[0]];
+            if (typeof firstItem === 'object' && firstItem !== null) {
+                return numericKeys.map(key => {
+                    const itemData = data[key];
+                    return {
+                        ...baseData,
+                        ...(typeof itemData === 'object' && itemData !== null ? itemData : { value: itemData })
+                    };
+                });
+            }
+            return [data];
+        case 'smart':
+            const mainArray = detectMainArray(data);
+            if (mainArray === 'numeric') {
+                return numericKeys.map(key => {
+                    const itemData = data[key];
+                    return {
+                        ...baseData,
+                        ...(typeof itemData === 'object' && itemData !== null ? itemData : { value: itemData })
+                    };
+                });
+            }
+            return [data];
+        default:
+            return [data];
+    }
+}
+async function execute(items, nodeOptions) {
+    var _a;
+    const allResults = [];
+    const credentials = (await this.getCredentials('crawl4aiApi'));
+    if (!credentials.enableLlm) {
+        throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'LLM features are not enabled in Crawl4AI credentials. Please enable them and configure an LLM provider.', { itemIndex: 0 });
+    }
+    for (let i = 0; i < items.length; i++) {
+        try {
+            const url = this.getNodeParameter('url', i, '');
+            const instruction = this.getNodeParameter('instruction', i, '');
+            const schemaMode = this.getNodeParameter('schemaMode', i, 'simple');
+            const schemaFieldsValues = this.getNodeParameter('schemaFields.fieldsValues', i, []);
+            const jsonSchema = this.getNodeParameter('jsonSchema', i, {});
+            const browserOptions = this.getNodeParameter('browserOptions', i, {});
+            const llmOptions = this.getNodeParameter('llmOptions', i, {});
+            const options = this.getNodeParameter('options', i, {});
+            const arrayHandling = options.arrayHandling || 'none';
+            const includeMetadataInSplitItems = options.includeMetadataInSplitItems || false;
+            if (!url) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URL cannot be empty.', { itemIndex: i });
+            }
+            if (!(0, utils_1.isValidUrl)(url)) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${url}`, { itemIndex: i });
+            }
+            if (!instruction) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'Extraction instructions cannot be empty.', { itemIndex: i });
+            }
+            if (schemaMode === 'simple') {
+                if (!schemaFieldsValues || schemaFieldsValues.length === 0) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'At least one schema field must be defined.', { itemIndex: i });
+                }
+            }
+            else if (schemaMode === 'advanced') {
+                if (!jsonSchema || jsonSchema.trim() === '') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'JSON schema cannot be empty.', { itemIndex: i });
+                }
+            }
+            let schema;
+            if (schemaMode === 'simple') {
+                const schemaProperties = {};
+                const requiredFields = [];
+                schemaFieldsValues.forEach(field => {
+                    const fieldName = field.name;
+                    schemaProperties[fieldName] = {
+                        name: fieldName,
+                        type: field.fieldType,
+                        description: field.description || undefined,
+                    };
+                    if (field.required === true) {
+                        requiredFields.push(fieldName);
+                    }
+                });
+                schema = {
+                    title: 'ExtractedData',
+                    type: 'object',
+                    properties: schemaProperties,
+                    required: requiredFields.length > 0 ? requiredFields : undefined,
+                };
+            }
+            else {
+                const jsonSchemaString = jsonSchema;
+                if (!jsonSchemaString || jsonSchemaString.trim() === '') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'JSON schema cannot be empty in advanced mode.', { itemIndex: i });
+                }
+                let parsedSchema;
+                try {
+                    parsedSchema = JSON.parse(jsonSchemaString.trim());
+                }
+                catch (error) {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid JSON schema: ${error.message}`, { itemIndex: i });
+                }
+                if (!parsedSchema || typeof parsedSchema !== 'object') {
+                    throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'JSON schema must be a valid object', { itemIndex: i });
+                }
+                schema = parsedSchema;
+                if (!schema.type) {
+                    schema.type = 'object';
+                }
+                if (!schema.title) {
+                    schema.title = 'ExtractedData';
+                }
+            }
+            let provider;
+            let apiKey;
+            let baseUrl;
+            if (credentials.llmProvider === 'other') {
+                provider = credentials.customProvider || 'openai/gpt-4o';
+                apiKey = credentials.customApiKey || '';
+                baseUrl = credentials.customBaseUrl || undefined;
+            }
+            else {
+                provider = credentials.llmProvider || 'openai/gpt-4o';
+                apiKey = credentials.apiKey || '';
+                baseUrl = undefined;
+            }
+            if (llmOptions.overrideProvider === true) {
+                provider = llmOptions.llmProvider || provider;
+                apiKey = llmOptions.apiKey || apiKey;
+            }
+            const browserConfig = (0, utils_1.createBrowserConfig)(browserOptions);
+            const extractionStrategy = (0, utils_1.createLlmExtractionStrategy)(schema, instruction, provider, apiKey, baseUrl);
+            const crawler = await (0, utils_1.getCrawl4aiClient)(this);
+            const extraArgs = {};
+            if (llmOptions.temperature !== undefined) {
+                extraArgs.temperature = llmOptions.temperature;
+            }
+            if (llmOptions.maxTokens !== undefined) {
+                extraArgs.max_tokens = llmOptions.maxTokens;
+            }
+            const result = await crawler.arun(url, {
+                browserConfig,
+                extractionStrategy,
+                cacheMode: options.cacheMode || 'enabled',
+                jsCode: browserOptions.jsCode,
+                cssSelector: options.cssSelector,
+                extraArgs,
+            });
+            const extractedData = (0, formatters_1.parseExtractedJson)(result);
+            const formattedResult = (0, formatters_1.formatExtractionResult)(result, extractedData, options.includeFullText);
+            const processedResults = processArrayHandling(formattedResult, arrayHandling, {}, includeMetadataInSplitItems);
+            processedResults.forEach(processedResult => {
+                allResults.push({
+                    json: processedResult,
+                    pairedItem: { item: i },
+                });
+            });
+        }
+        catch (error) {
+            if (this.continueOnFail()) {
+                const node = this.getNode();
+                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                allResults.push({
+                    json: items[i].json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
+                    pairedItem: { item: i },
+                });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return allResults;
+}
+//# sourceMappingURL=llmExtractor.operation.js.map