n8n-nodes-crawl4ai-onuro 0.2.0

package/dist/nodes/Crawl4aiBasicCrawler/actions/crawlSingleUrl.operation.js

@@ -0,0 +1,422 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = void 0;
+exports.execute = execute;
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../helpers/utils");
+const formatters_1 = require("../helpers/formatters");
+exports.description = [
+    {
+        displayName: 'URL',
+        name: 'url',
+        type: 'string',
+        required: true,
+        default: '',
+        placeholder: 'https://example.com',
+        description: 'The URL to crawl',
+        displayOptions: {
+            show: {
+                operation: ['crawlSingleUrl'],
+            },
+        },
+    },
+    {
+        displayName: 'Browser Options',
+        name: 'browserOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlSingleUrl'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Enable JavaScript',
+                name: 'javaScriptEnabled',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to enable JavaScript execution',
+            },
+            {
+                displayName: 'Headless Mode',
+                name: 'headless',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to run browser in headless mode',
+            },
+            {
+                displayName: 'Timeout (Ms)',
+                name: 'timeout',
+                type: 'number',
+                default: 30000,
+                description: 'Maximum time to wait for the browser to load the page',
+            },
+            {
+                displayName: 'User Agent',
+                name: 'userAgent',
+                type: 'string',
+                default: '',
+                placeholder: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ...',
+                description: 'The user agent to use (leave empty for default)',
+            },
+            {
+                displayName: 'Viewport Height',
+                name: 'viewportHeight',
+                type: 'number',
+                default: 800,
+                description: 'The height of the browser viewport',
+            },
+            {
+                displayName: 'Viewport Width',
+                name: 'viewportWidth',
+                type: 'number',
+                default: 1280,
+                description: 'The width of the browser viewport',
+            },
+        ],
+    },
+    {
+        displayName: 'Crawler Options',
+        name: 'crawlerOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlSingleUrl'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Cache Mode',
+                name: 'cacheMode',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Enabled (Read/Write)',
+                        value: 'enabled',
+                        description: 'Use cache if available, save new results to cache',
+                    },
+                    {
+                        name: 'Bypass (Force Fresh)',
+                        value: 'bypass',
+                        description: 'Ignore cache, always fetch fresh content',
+                    },
+                    {
+                        name: 'Only (Read Only)',
+                        value: 'only',
+                        description: 'Only use cache, do not make new requests',
+                    },
+                ],
+                default: 'enabled',
+                description: 'How to use the cache when crawling',
+            },
+            {
+                displayName: 'Check Robots.txt',
+                name: 'checkRobotsTxt',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to respect robots.txt rules',
+            },
+            {
+                displayName: 'CSS Selector',
+                name: 'cssSelector',
+                type: 'string',
+                default: '',
+                placeholder: 'article.content',
+                description: 'CSS selector to focus on specific content (leave empty for full page)',
+            },
+            {
+                displayName: 'Exclude External Links',
+                name: 'excludeExternalLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude external links from the result',
+            },
+            {
+                displayName: 'Excluded Tags',
+                name: 'excludedTags',
+                type: 'string',
+                default: '',
+                placeholder: 'nav,footer,aside',
+                description: 'Comma-separated list of HTML tags to exclude from processing',
+            },
+            {
+                displayName: 'JavaScript Code',
+                name: 'jsCode',
+                type: 'string',
+                typeOptions: {
+                    rows: 4,
+                },
+                default: '',
+                placeholder: 'document.querySelector("button.load-more").click();',
+                description: 'JavaScript code to execute on the page after load',
+            },
+            {
+                displayName: 'JavaScript Only Mode',
+                name: 'jsOnly',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to only execute JavaScript without crawling',
+            },
+            {
+                displayName: 'Max Retries',
+                name: 'maxRetries',
+                type: 'number',
+                default: 3,
+                description: 'Maximum number of retries for failed requests',
+            },
+            {
+                displayName: 'Page Timeout (Ms)',
+                name: 'pageTimeout',
+                type: 'number',
+                default: 30000,
+                description: 'Maximum time to wait for the page to load',
+            },
+            {
+                displayName: 'Request Timeout (Ms)',
+                name: 'requestTimeout',
+                type: 'number',
+                default: 30000,
+                description: 'Maximum time to wait for network requests',
+            },
+            {
+                displayName: 'Session ID',
+                name: 'sessionId',
+                type: 'string',
+                default: '',
+                placeholder: 'my-session-ID',
+                description: 'ID to maintain browser state across multiple crawls (for multi-step crawling)',
+            },
+            {
+                displayName: 'Word Count Threshold',
+                name: 'wordCountThreshold',
+                type: 'number',
+                default: 0,
+                description: 'Minimum number of words for content to be included',
+            },
+            {
+                displayName: 'Exclude Domains',
+                name: 'excludeDomains',
+                type: 'string',
+                default: '',
+                placeholder: 'ads.example.com,tracker.example.com',
+                description: 'Comma-separated list of domains to exclude from links',
+            },
+            {
+                displayName: 'Exclude External Images',
+                name: 'excludeExternalImages',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude images not hosted on the same domain',
+            },
+            {
+                displayName: 'Exclude Internal Links',
+                name: 'excludeInternalLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude links pointing to the same domain',
+            },
+            {
+                displayName: 'Exclude Social Media Links',
+                name: 'excludeSocialMediaLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to remove links to social media platforms',
+            },
+            {
+                displayName: 'Process Iframes',
+                name: 'processIframes',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to extract content from embedded iframes',
+            },
+            {
+                displayName: 'Remove Overlay Elements',
+                name: 'removeOverlayElements',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to remove popups, modals, and cookie banners',
+            },
+            {
+                displayName: 'Target Elements',
+                name: 'targetElements',
+                type: 'string',
+                default: '',
+                placeholder: 'article.content,div.main',
+                description: 'Comma-separated CSS selectors for focused extraction',
+            },
+        ],
+    },
+    {
+        displayName: 'LLM Extraction',
+        name: 'llmExtraction',
+        type: 'collection',
+        placeholder: 'Add LLM Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlSingleUrl'],
+            },
+        },
+        options: [
+            {
+                displayName: 'LLM Provider',
+                name: 'provider',
+                type: 'options',
+                options: [
+                    { name: 'DeepSeek', value: 'deepseek' },
+                    { name: 'OpenAI', value: 'openai' },
+                    { name: 'Anthropic', value: 'anthropic' },
+                    { name: 'Ollama (Local)', value: 'ollama' },
+                ],
+                default: 'deepseek',
+                description: 'The LLM provider to use for extraction',
+            },
+            {
+                displayName: 'API Key',
+                name: 'apiKey',
+                type: 'string',
+                typeOptions: { password: true },
+                default: '',
+                description: 'API key for the LLM provider',
+            },
+            {
+                displayName: 'Base URL',
+                name: 'baseUrl',
+                type: 'string',
+                default: '',
+                placeholder: 'https://api.deepseek.com/v1',
+                description: 'Base URL for the LLM API (required for DeepSeek/Ollama)',
+            },
+            {
+                displayName: 'Extraction Instruction',
+                name: 'instruction',
+                type: 'string',
+                typeOptions: { rows: 3 },
+                default: '',
+                placeholder: 'Extract all product names and prices from this page',
+                description: 'Instructions for the LLM on what to extract',
+            },
+            {
+                displayName: 'Schema (JSON)',
+                name: 'schema',
+                type: 'string',
+                typeOptions: { rows: 4 },
+                default: '',
+                placeholder: '{"type": "object", "properties": {"title": {"type": "string"}}}',
+                description: 'Optional JSON schema for structured output',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['crawlSingleUrl'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Output Format',
+                name: 'outputFormat',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Markdown',
+                        value: 'markdown',
+                        description: 'Clean markdown format (default, best for LLMs)',
+                    },
+                    {
+                        name: 'Fit Markdown',
+                        value: 'fitMarkdown',
+                        description: 'Shorter markdown optimized for LLM token limits',
+                    },
+                    {
+                        name: 'Plain Text',
+                        value: 'text',
+                        description: 'Plain text with no formatting',
+                    },
+                    {
+                        name: 'Cleaned HTML',
+                        value: 'cleanedHtml',
+                        description: 'Cleaned HTML with unnecessary elements removed',
+                    },
+                    {
+                        name: 'Raw HTML',
+                        value: 'html',
+                        description: 'Original raw HTML from the page',
+                    },
+                ],
+                default: 'markdown',
+                description: 'Format for the content output',
+            },
+            {
+                displayName: 'Include Media Data',
+                name: 'includeMedia',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include media data in output (images, videos)',
+            },
+            {
+                displayName: 'Verbose Response',
+                name: 'verboseResponse',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
+            },
+        ],
+    },
+];
+async function execute(items, nodeOptions) {
+    var _a;
+    const allResults = [];
+    for (let i = 0; i < items.length; i++) {
+        try {
+            const url = this.getNodeParameter('url', i, '');
+            const browserOptions = this.getNodeParameter('browserOptions', i, {});
+            const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
+            const llmExtraction = this.getNodeParameter('llmExtraction', i, {});
+            const options = this.getNodeParameter('options', i, {});
+            if (!url) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URL cannot be empty.', { itemIndex: i });
+            }
+            if (!(0, utils_1.isValidUrl)(url)) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${url}`, { itemIndex: i });
+            }
+            const browserConfig = (0, utils_1.createBrowserConfig)(browserOptions);
+            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)({
+                ...crawlerOptions,
+                ...browserConfig,
+            });
+            const crawler = await (0, utils_1.getCrawl4aiClient)(this);
+            const result = await crawler.crawlUrl(url, crawlerConfig, llmExtraction);
+            const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse, options.outputFormat || 'markdown');
+            allResults.push({
+                json: formattedResult,
+                pairedItem: { item: i },
+            });
+        }
+        catch (error) {
+            if (this.continueOnFail()) {
+                const node = this.getNode();
+                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                allResults.push({
+                    json: items[i].json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
+                    pairedItem: { item: i },
+                });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return allResults;
+}
+//# sourceMappingURL=crawlSingleUrl.operation.js.map

package/dist/nodes/Crawl4aiBasicCrawler/actions/crawlSingleUrl.operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawlSingleUrl.operation.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/actions/crawlSingleUrl.operation.ts"],"names":[],"mappings":";;;AAsYA,0BAqEC;AArcD,+CAAkD;AAIlD,4CAK0B;AAC1B,sDAA0D;AAG7C,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,KAAK;QAClB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,qBAAqB;QAClC,WAAW,EAAE,kBAAkB;QAC/B,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;KACF;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,mBAAmB;gBAChC,IAAI,EAAE,mBAAmB;gBACzB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,wCAAwC;aACtD;YACD;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,yCAAyC;aACvD;YACD;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,uDAAuD;aACrE;YACD;gBACE,WAAW,EAAE,YAAY;gBACzB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,+CAA+C;gBAC5D,WAAW,EAAE,iDAAiD;aAC/D;YACD;gBACE,WAAW,EAAE,iBAAiB;gBAC9B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,GAAG;gBACZ,WAAW,EAAE,oCAAoC;aAClD;YACD;gBACE,WAAW,EAAE,gBAAgB;gBAC7B,IAAI,EAAE,eAAe;gBACrB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,mCAAmC;aACjD;SACF;KACF;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,YAAY;gBACzB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,mDAAmD;qBACjE;oBACD;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,KAAK,EAAE,QAAQ;wBACf,WAAW,EAAE,0CAA0C;qBACxD;oBACD;wBACE,IAAI,EAAE,kBAAkB;wBACxB,KAAK,EAAE,MAAM;wBACb,WAAW,EAAE,0CAA0C;qBACxD;iBACF;gBACD,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,oCAAoC;aAClD;YACD;gBACE,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,qCAAqC;aACnD;YACD;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,iBAAiB;gBAC9B,WAAW,EAAE,uEAAuE;aACrF;YACD;gBACE,WAAW,EAAE,wBAAwB;gBACrC,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,kBAAkB;gBAC/B,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,WAAW,EAAE,iBAAiB;gBAC9B,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE;oBACX,IAAI,EAAE,CAAC;iBACR;gBACD,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,qDAAqD;gBAClE,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,WAAW,EAAE,sBAAsB;gBACnC,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,qDAAqD;aACnE;YACD;gBACE,WAAW,EAAE,aAAa;gBAC1B,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC;gBACV,WAAW,EAAE,+CAA+C;aAC7D;YACD;gBACE,WAAW,EAAE,mBAAmB;gBAChC,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,2CAA2C;aACzD;YACD;gBACE,WAAW,EAAE,sBAAsB;gBACnC,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,2CAA2C;aACzD;YACD;gBACE,WAAW,EAAE,YAAY;gBACzB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,eAAe;gBAC5B,WAAW,EAAE,+EAA+E;aAC7F;YACD;gBACE,WAAW,EAAE,sBAAsB;gBACnC,IAAI,EAAE,oBAAoB;gBAC1B,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC;gBACV,WAAW,EAAE,oDAAoD;aAClE;YACD;gBACE,WAAW,EAAE,iBAAiB;gBAC9B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,qCAAqC;gBAClD,WAAW,EAAE,uDAAuD;aACrE;YACD;gBACE,WAAW,EAAE,yBAAyB;gBACtC,IAAI,EAAE,uBAAuB;gBAC7B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,yDAAyD;aACvE;YACD;gBACE,WAAW,EAAE,wBAAwB;gBACrC,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,sDAAsD;aACpE;YACD;gBACE,WAAW,EAAE,4BAA4B;gBACzC,IAAI,EAAE,yBAAyB;gBAC/B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,WAAW,EAAE,iBAAiB;gBAC9B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,kDAAkD;aAChE;YACD;gBACE,WAAW,EAAE,yBAAyB;gBACtC,IAAI,EAAE,uBAAuB;gBAC7B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,sDAAsD;aACpE;YACD;gBACE,WAAW,EAAE,iBAAiB;gBAC9B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,0BAA0B;gBACvC,WAAW,EAAE,sDAAsD;aACpE;SACF;KACF;IACD;QACE,WAAW,EAAE,gBAAgB;QAC7B,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,gBAAgB;QAC7B,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;oBACP,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,EAAE;oBACvC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE;oBACnC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE;oBACzC,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,EAAE;iBAC5C;gBACD,OAAO,EAAE,UAAU;gBACnB,WAAW,EAAE,wCAAwC;aACtD;YACD;gBACE,WAAW,EAAE,SAAS;gBACtB,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;gBAC/B,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,8BAA8B;aAC5C;YACD;gBACE,WAAW,EAAE,UAAU;gBACvB,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,6BAA6B;gBAC1C,WAAW,EAAE,yDAAyD;aACvE;YACD;gBACE,WAAW,EAAE,wBAAwB;gBACrC,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;gBACxB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,qDAAqD;gBAClE,WAAW,EAAE,6CAA6C;aAC3D;YACD;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,QAAQ;gBACd,WAAW,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;gBACxB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,iEAAiE;gBAC9E,WAAW,EAAE,4CAA4C;aAC1D;SACF;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,UAAU;wBACjB,WAAW,EAAE,gDAAgD;qBAC9D;oBACD;wBACE,IAAI,EAAE,cAAc;wBACpB,KAAK,EAAE,aAAa;wBACpB,WAAW,EAAE,iDAAiD;qBAC/D;oBACD;wBACE,IAAI,EAAE,YAAY;wBAClB,KAAK,EAAE,MAAM;wBACb,WAAW,EAAE,+BAA+B;qBAC7C;oBACD;wBACE,IAAI,EAAE,cAAc;wBACpB,KAAK,EAAE,aAAa;wBACpB,WAAW,EAAE,gDAAgD;qBAC9D;oBACD;wBACE,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,MAAM;wBACb,WAAW,EAAE,iCAAiC;qBAC/C;iBACF;gBACD,OAAO,EAAE,UAAU;gBACnB,WAAW,EAAE,+BAA+B;aAC7C;YACD;gBACE,WAAW,EAAE,oBAAoB;gBACjC,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,0DAA0D;aACxE;YACD;gBACE,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,iBAAiB;gBACvB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,uEAAuE;aACrF;SACF;KACF;CACF,CAAC;AAGK,KAAK,UAAU,OAAO,CAE3B,KAA2B,EAC3B,WAAgC;;IAEhC,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC;YAEH,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAW,CAAC;YAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YACrF,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YACrF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YACnF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YAEvE,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,sBAAsB,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,IAAI,CAAC,IAAA,kBAAU,EAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,gBAAgB,GAAG,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACxF,CAAC;YAGD,MAAM,aAAa,GAAG,IAAA,2BAAmB,EAAC,cAAc,CAAC,CAAC;YAC1D,MAAM,aAAa,GAAG,IAAA,8BAAsB,EAAC;gBAC3C,GAAG,cAAc;gBACjB,GAAG,aAAa;aACjB,CAAC,CAAC;YAGH,MAAM,OAAO,GAAG,MAAM,IAAA,yBAAiB,EAAC,IAAI,CAAC,CAAC;YAG9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE,aAAa,EAAE,aAAa,CAAC,CAAC;YAGzE,MAAM,eAAe,GAAG,IAAA,8BAAiB,EACvC,MAAM,EACN,OAAO,CAAC,YAAuB,EAC/B,OAAO,CAAC,eAA0B,EACjC,OAAO,CAAC,YAAuB,IAAI,UAAU,CAC/C,CAAC;YAGF,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;aACxB,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,cAAc,GAAG,MAAC,KAAa,CAAC,SAAS,mCAAI,CAAC,CAAC;gBACrD,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;oBACnB,KAAK,EAAE,IAAI,iCAAkB,CAAC,IAAI,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;oBAC5F,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/actions/operations.d.ts

@@ -0,0 +1,8 @@
+import type { INodeProperties, IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
+import type { Crawl4aiNodeOptions } from '../helpers/interfaces';
+type OperationExecuteFunction = (this: IExecuteFunctions, items: INodeExecutionData[], nodeOptions: Crawl4aiNodeOptions) => Promise<INodeExecutionData[]>;
+export declare const operations: {
+    [key: string]: OperationExecuteFunction;
+};
+export declare const description: INodeProperties[];
+export {};

package/dist/nodes/Crawl4aiBasicCrawler/actions/operations.js

@@ -0,0 +1,67 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = exports.operations = void 0;
+const crawlSingleUrl = __importStar(require("./crawlSingleUrl.operation"));
+const crawlMultipleUrls = __importStar(require("./crawlMultipleUrls.operation"));
+const processRawHtml = __importStar(require("./processRawHtml.operation"));
+exports.operations = {
+    crawlSingleUrl: crawlSingleUrl.execute,
+    crawlMultipleUrls: crawlMultipleUrls.execute,
+    processRawHtml: processRawHtml.execute,
+};
+exports.description = [
+    {
+        displayName: 'Operation',
+        name: 'operation',
+        type: 'options',
+        noDataExpression: true,
+        options: [
+            {
+                name: 'Crawl Single URL',
+                value: 'crawlSingleUrl',
+                description: 'Crawl a single URL and extract content',
+                action: 'Crawl a single URL',
+            },
+            {
+                name: 'Crawl Multiple URLs',
+                value: 'crawlMultipleUrls',
+                description: 'Crawl multiple URLs and extract content',
+                action: 'Crawl multiple ur ls',
+            },
+            {
+                name: 'Process Raw HTML',
+                value: 'processRawHtml',
+                description: 'Process provided HTML content without crawling',
+                action: 'Process raw html',
+            },
+        ],
+        default: 'crawlSingleUrl',
+    },
+    ...crawlSingleUrl.description,
+    ...crawlMultipleUrls.description,
+    ...processRawHtml.description,
+];
+//# sourceMappingURL=operations.js.map

package/dist/nodes/Crawl4aiBasicCrawler/actions/operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/actions/operations.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,2EAA6D;AAC7D,iFAAmE;AACnE,2EAA6D;AAWhD,QAAA,UAAU,GAAgD;IACrE,cAAc,EAAE,cAAc,CAAC,OAAO;IACtC,iBAAiB,EAAE,iBAAiB,CAAC,OAAO;IAC5C,cAAc,EAAE,cAAc,CAAC,OAAO;CACvC,CAAC;AAGW,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,gBAAgB;gBACvB,WAAW,EAAE,wCAAwC;gBACrD,MAAM,EAAE,oBAAoB;aAC7B;YACD;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,mBAAmB;gBAC1B,WAAW,EAAE,yCAAyC;gBACtD,MAAM,EAAE,sBAAsB;aAC/B;YACD;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,gBAAgB;gBACvB,WAAW,EAAE,gDAAgD;gBAC7D,MAAM,EAAE,kBAAkB;aAC3B;SACF;QACD,OAAO,EAAE,gBAAgB;KAC1B;IAGD,GAAG,cAAc,CAAC,WAAW;IAC7B,GAAG,iBAAiB,CAAC,WAAW;IAChC,GAAG,cAAc,CAAC,WAAW;CAC9B,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.d.ts

@@ -0,0 +1,4 @@
+import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
+import type { Crawl4aiNodeOptions } from '../helpers/interfaces';
+export declare const description: INodeProperties[];
+export declare function execute(this: IExecuteFunctions, items: INodeExecutionData[], nodeOptions: Crawl4aiNodeOptions): Promise<INodeExecutionData[]>;

package/dist/nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.js

@@ -0,0 +1,148 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = void 0;
+exports.execute = execute;
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../helpers/utils");
+const formatters_1 = require("../helpers/formatters");
+exports.description = [
+    {
+        displayName: 'HTML Content',
+        name: 'html',
+        type: 'string',
+        typeOptions: {
+            rows: 8,
+        },
+        required: true,
+        default: '',
+        placeholder: '<html><body><h1>Example</h1><p>Content</p></body></html>',
+        description: 'The raw HTML content to process',
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+    },
+    {
+        displayName: 'Base URL',
+        name: 'baseUrl',
+        type: 'string',
+        default: 'https://example.com',
+        description: 'The base URL to use for resolving relative links',
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+    },
+    {
+        displayName: 'Crawler Options',
+        name: 'crawlerOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+        options: [
+            {
+                displayName: 'CSS Selector',
+                name: 'cssSelector',
+                type: 'string',
+                default: '',
+                placeholder: 'article.content',
+                description: 'CSS selector to focus on specific content (leave empty for full page)',
+            },
+            {
+                displayName: 'Exclude External Links',
+                name: 'excludeExternalLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude external links from the result',
+            },
+            {
+                displayName: 'Excluded Tags',
+                name: 'excludedTags',
+                type: 'string',
+                default: '',
+                placeholder: 'nav,footer,aside',
+                description: 'Comma-separated list of HTML tags to exclude from processing',
+            },
+            {
+                displayName: 'Word Count Threshold',
+                name: 'wordCountThreshold',
+                type: 'number',
+                default: 0,
+                description: 'Minimum number of words for content to be included',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Include Media Data',
+                name: 'includeMedia',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include media data in output (images, videos)',
+            },
+            {
+                displayName: 'Verbose Response',
+                name: 'verboseResponse',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
+            },
+        ],
+    },
+];
+async function execute(items, nodeOptions) {
+    var _a;
+    const allResults = [];
+    for (let i = 0; i < items.length; i++) {
+        try {
+            const html = this.getNodeParameter('html', i, '');
+            const baseUrl = this.getNodeParameter('baseUrl', i, 'https://example.com');
+            const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
+            const options = this.getNodeParameter('options', i, {});
+            if (!html) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'HTML content cannot be empty.', { itemIndex: i });
+            }
+            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)(crawlerOptions);
+            const crawler = await (0, utils_1.getCrawl4aiClient)(this);
+            const result = await crawler.processRawHtml(html, baseUrl, crawlerConfig);
+            const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse);
+            allResults.push({
+                json: formattedResult,
+                pairedItem: { item: i },
+            });
+        }
+        catch (error) {
+            if (this.continueOnFail()) {
+                const node = this.getNode();
+                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                allResults.push({
+                    json: items[i].json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
+                    pairedItem: { item: i },
+                });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return allResults;
+}
+//# sourceMappingURL=processRawHtml.operation.js.map