n8n-nodes-crawl4ai-plus 2.0.9 → 3.0.0

package/dist/nodes/Crawl4aiPlusContentExtractor/actions/seoExtractor.operation.js

@@ -0,0 +1,376 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = void 0;
+exports.execute = execute;
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../helpers/utils");
+exports.description = [
+    {
+        displayName: 'URL',
+        name: 'url',
+        type: 'string',
+        required: true,
+        default: '',
+        placeholder: 'https://example.com',
+        description: 'The URL to extract SEO metadata from',
+        displayOptions: {
+            show: {
+                operation: ['seoExtractor'],
+            },
+        },
+    },
+    {
+        displayName: 'Metadata Types',
+        name: 'metadataTypes',
+        type: 'multiOptions',
+        options: [
+            {
+                name: 'Basic Meta Tags',
+                value: 'basic',
+                description: 'Title, description, keywords, canonical URL',
+            },
+            {
+                name: 'JSON-LD Structured Data',
+                value: 'jsonLd',
+                description: 'Schema.org structured data in JSON-LD format',
+            },
+            {
+                name: 'Language & Locale',
+                value: 'language',
+                description: 'HTML lang, hreflang tags, locale settings',
+            },
+            {
+                name: 'Open Graph (OG) Tags',
+                value: 'openGraph',
+                description: 'OG title, description, image, type, URL',
+            },
+            {
+                name: 'Robots & Indexing',
+                value: 'robots',
+                description: 'Robots meta, noindex, nofollow directives',
+            },
+            {
+                name: 'Twitter Cards',
+                value: 'twitter',
+                description: 'Twitter card metadata',
+            },
+        ],
+        default: ['basic', 'openGraph', 'jsonLd'],
+        description: 'Select which types of SEO metadata to extract',
+        displayOptions: {
+            show: {
+                operation: ['seoExtractor'],
+            },
+        },
+    },
+    {
+        displayName: 'Browser Options',
+        name: 'browserOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['seoExtractor'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Browser Type',
+                name: 'browserType',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Chromium',
+                        value: 'chromium',
+                        description: 'Use Chromium browser (default, most compatible)',
+                    },
+                    {
+                        name: 'Firefox',
+                        value: 'firefox',
+                        description: 'Use Firefox browser',
+                    },
+                    {
+                        name: 'Webkit',
+                        value: 'webkit',
+                        description: 'Use Webkit browser (Safari engine)',
+                    },
+                ],
+                default: 'chromium',
+                description: 'Which browser engine to use for crawling',
+            },
+            {
+                displayName: 'Enable JavaScript',
+                name: 'java_script_enabled',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to enable JavaScript execution (recommended for dynamic SEO tags)',
+            },
+            {
+                displayName: 'Headless Mode',
+                name: 'headless',
+                type: 'boolean',
+                default: true,
+                description: 'Whether to run browser in headless mode',
+            },
+            {
+                displayName: 'Timeout (MS)',
+                name: 'timeout',
+                type: 'number',
+                default: 30000,
+                description: 'Maximum time to wait for the browser to load the page',
+            },
+            {
+                displayName: 'Wait For',
+                name: 'waitFor',
+                type: 'string',
+                default: '',
+                placeholder: 'head',
+                description: 'CSS selector to wait for before extracting (useful for dynamically injected meta tags)',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['seoExtractor'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Cache Mode',
+                name: 'cacheMode',
+                type: 'options',
+                options: [
+                    {
+                        name: 'Bypass (Skip Cache)',
+                        value: 'BYPASS',
+                        description: 'Skip cache for this operation, fetch fresh content',
+                    },
+                    {
+                        name: 'Disabled (No Cache)',
+                        value: 'DISABLED',
+                        description: 'No caching at all',
+                    },
+                    {
+                        name: 'Enabled (Read/Write)',
+                        value: 'ENABLED',
+                        description: 'Use cache if available, save new results to cache',
+                    },
+                    {
+                        name: 'Read Only',
+                        value: 'READ_ONLY',
+                        description: 'Only read from cache, do not write new results',
+                    },
+                    {
+                        name: 'Write Only',
+                        value: 'WRITE_ONLY',
+                        description: 'Only write to cache, do not read existing cache',
+                    },
+                ],
+                default: 'ENABLED',
+                description: 'How to use the cache when crawling',
+            },
+            {
+                displayName: 'Include Raw HTML',
+                name: 'includeRawHtml',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include the raw HTML head section in output',
+            },
+        ],
+    },
+];
+const SEO_FIELDS = {
+    basic: [
+        { name: 'title', selector: 'title', type: 'text' },
+        { name: 'metaDescription', selector: 'meta[name="description"]', type: 'attribute', attribute: 'content' },
+        { name: 'metaKeywords', selector: 'meta[name="keywords"]', type: 'attribute', attribute: 'content' },
+        { name: 'canonicalUrl', selector: 'link[rel="canonical"]', type: 'attribute', attribute: 'href' },
+        { name: 'author', selector: 'meta[name="author"]', type: 'attribute', attribute: 'content' },
+        { name: 'viewport', selector: 'meta[name="viewport"]', type: 'attribute', attribute: 'content' },
+    ],
+    openGraph: [
+        { name: 'ogTitle', selector: 'meta[property="og:title"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogDescription', selector: 'meta[property="og:description"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogImage', selector: 'meta[property="og:image"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogType', selector: 'meta[property="og:type"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogUrl', selector: 'meta[property="og:url"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogSiteName', selector: 'meta[property="og:site_name"]', type: 'attribute', attribute: 'content' },
+        { name: 'ogLocale', selector: 'meta[property="og:locale"]', type: 'attribute', attribute: 'content' },
+    ],
+    twitter: [
+        { name: 'twitterCard', selector: 'meta[name="twitter:card"]', type: 'attribute', attribute: 'content' },
+        { name: 'twitterTitle', selector: 'meta[name="twitter:title"]', type: 'attribute', attribute: 'content' },
+        { name: 'twitterDescription', selector: 'meta[name="twitter:description"]', type: 'attribute', attribute: 'content' },
+        { name: 'twitterImage', selector: 'meta[name="twitter:image"]', type: 'attribute', attribute: 'content' },
+        { name: 'twitterSite', selector: 'meta[name="twitter:site"]', type: 'attribute', attribute: 'content' },
+        { name: 'twitterCreator', selector: 'meta[name="twitter:creator"]', type: 'attribute', attribute: 'content' },
+    ],
+    robots: [
+        { name: 'robots', selector: 'meta[name="robots"]', type: 'attribute', attribute: 'content' },
+        { name: 'googlebot', selector: 'meta[name="googlebot"]', type: 'attribute', attribute: 'content' },
+        { name: 'bingbot', selector: 'meta[name="bingbot"]', type: 'attribute', attribute: 'content' },
+    ],
+    language: [
+        { name: 'htmlLang', selector: 'html', type: 'attribute', attribute: 'lang' },
+        { name: 'contentLanguage', selector: 'meta[http-equiv="content-language"]', type: 'attribute', attribute: 'content' },
+    ],
+};
+async function execute(items, nodeOptions) {
+    var _a;
+    const allResults = [];
+    for (let i = 0; i < items.length; i++) {
+        try {
+            const url = this.getNodeParameter('url', i, '');
+            const metadataTypes = this.getNodeParameter('metadataTypes', i, ['basic', 'openGraph', 'jsonLd']);
+            const browserOptions = this.getNodeParameter('browserOptions', i, {});
+            const options = this.getNodeParameter('options', i, {});
+            if (!url) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'URL cannot be empty.', { itemIndex: i });
+            }
+            if (!(0, utils_1.isValidUrl)(url)) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Invalid URL: ${url}`, { itemIndex: i });
+            }
+            if (!metadataTypes || metadataTypes.length === 0) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'At least one metadata type must be selected.', { itemIndex: i });
+            }
+            const fields = [];
+            for (const metaType of metadataTypes) {
+                if (metaType !== 'jsonLd' && SEO_FIELDS[metaType]) {
+                    fields.push(...SEO_FIELDS[metaType]);
+                }
+            }
+            const extractionStrategy = fields.length > 0 ? {
+                type: 'JsonCssExtractionStrategy',
+                params: {
+                    schema: {
+                        type: 'dict',
+                        value: {
+                            name: 'SEO_Metadata',
+                            baseSelector: 'html',
+                            fields: fields.map(field => ({
+                                name: field.name,
+                                selector: field.selector,
+                                type: field.type,
+                                ...(field.attribute ? { attribute: field.attribute } : {}),
+                            })),
+                        },
+                    },
+                },
+            } : null;
+            const crawlerOptions = {
+                ...browserOptions,
+                cacheMode: options.cacheMode || 'ENABLED',
+                waitFor: browserOptions.waitFor,
+            };
+            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)(crawlerOptions);
+            if (extractionStrategy) {
+                crawlerConfig.extractionStrategy = extractionStrategy;
+            }
+            const crawler = await (0, utils_1.getCrawl4aiClient)(this);
+            const result = await crawler.arun(url, crawlerConfig);
+            if (!result.success) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Failed to crawl URL: ${result.error_message || 'Unknown error'}`, { itemIndex: i });
+            }
+            let seoData = {};
+            if (result.extracted_content) {
+                try {
+                    const parsed = JSON.parse(result.extracted_content);
+                    if (Array.isArray(parsed) && parsed.length > 0) {
+                        seoData = { ...seoData, ...parsed[0] };
+                    }
+                    else if (typeof parsed === 'object') {
+                        seoData = { ...seoData, ...parsed };
+                    }
+                }
+                catch (e) {
+                }
+            }
+            if (metadataTypes.includes('jsonLd')) {
+                const jsonLdData = extractJsonLd(result.html || result.cleaned_html || '');
+                if (jsonLdData.length > 0) {
+                    seoData.jsonLd = jsonLdData;
+                }
+            }
+            if (metadataTypes.includes('language')) {
+                const hreflangTags = extractHreflang(result.html || result.cleaned_html || '');
+                if (hreflangTags.length > 0) {
+                    seoData.hreflang = hreflangTags;
+                }
+            }
+            const formattedResult = {
+                url,
+                success: true,
+                seo: seoData,
+                ...(options.includeRawHtml ? { rawHtml: extractHead(result.html || '') } : {}),
+            };
+            allResults.push({
+                json: formattedResult,
+                pairedItem: { item: i },
+            });
+        }
+        catch (error) {
+            if (this.continueOnFail()) {
+                const node = this.getNode();
+                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                allResults.push({
+                    json: items[i].json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
+                    pairedItem: { item: i },
+                });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return allResults;
+}
+function extractJsonLd(html) {
+    const jsonLdData = [];
+    const regex = /<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+    let match;
+    while ((match = regex.exec(html)) !== null) {
+        try {
+            const data = JSON.parse(match[1].trim());
+            jsonLdData.push(data);
+        }
+        catch (e) {
+        }
+    }
+    return jsonLdData;
+}
+function extractHreflang(html) {
+    const hreflangTags = [];
+    const regex = /<link[^>]*rel=["']alternate["'][^>]*hreflang=["']([^"']+)["'][^>]*href=["']([^"']+)["'][^>]*\/?>/gi;
+    const regex2 = /<link[^>]*hreflang=["']([^"']+)["'][^>]*rel=["']alternate["'][^>]*href=["']([^"']+)["'][^>]*\/?>/gi;
+    const regex3 = /<link[^>]*href=["']([^"']+)["'][^>]*hreflang=["']([^"']+)["'][^>]*rel=["']alternate["'][^>]*\/?>/gi;
+    let match;
+    while ((match = regex.exec(html)) !== null) {
+        hreflangTags.push({ lang: match[1], href: match[2] });
+    }
+    while ((match = regex2.exec(html)) !== null) {
+        hreflangTags.push({ lang: match[1], href: match[2] });
+    }
+    while ((match = regex3.exec(html)) !== null) {
+        hreflangTags.push({ lang: match[2], href: match[1] });
+    }
+    const seen = new Set();
+    return hreflangTags.filter(tag => {
+        const key = `${tag.lang}:${tag.href}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+function extractHead(html) {
+    const match = html.match(/<head[^>]*>([\s\S]*?)<\/head>/i);
+    return match ? match[1] : '';
+}
+//# sourceMappingURL=seoExtractor.operation.js.map

package/dist/nodes/Crawl4aiPlusContentExtractor/actions/seoExtractor.operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"seoExtractor.operation.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiPlusContentExtractor/actions/seoExtractor.operation.ts"],"names":[],"mappings":";;;AAuPA,0BAoJC;AArYD,+CAAkD;AAIlD,4CAI0B;AAGb,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,KAAK;QAClB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,qBAAqB;QAClC,WAAW,EAAE,sCAAsC;QACnD,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,cAAc,CAAC;aAC5B;SACF;KACF;IACD;QACE,WAAW,EAAE,gBAAgB;QAC7B,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,cAAc;QACpB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,iBAAiB;gBACvB,KAAK,EAAE,OAAO;gBACd,WAAW,EAAE,6CAA6C;aAC3D;YACD;gBACE,IAAI,EAAE,yBAAyB;gBAC/B,KAAK,EAAE,QAAQ;gBACf,WAAW,EAAE,8CAA8C;aAC5D;YACD;gBACE,IAAI,EAAE,mBAAmB;gBACzB,KAAK,EAAE,UAAU;gBACjB,WAAW,EAAE,2CAA2C;aACzD;YACJ;gBACK,IAAI,EAAE,sBAAsB;gBAC5B,KAAK,EAAE,WAAW;gBAClB,WAAW,EAAE,yCAAyC;aACvD;YACD;gBACE,IAAI,EAAE,mBAAmB;gBACzB,KAAK,EAAE,QAAQ;gBACf,WAAW,EAAE,2CAA2C;aACzD;YACJ;gBACK,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,SAAS;gBAChB,WAAW,EAAE,uBAAuB;aACrC;SACF;QACD,OAAO,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC;QACzC,WAAW,EAAE,+CAA+C;QAC5D,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,cAAc,CAAC;aAC5B;SACF;KACF;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,cAAc,CAAC;aAC5B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,UAAU;wBACjB,WAAW,EAAE,iDAAiD;qBAC/D;oBACD;wBACE,IAAI,EAAE,SAAS;wBACf,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,qBAAqB;qBACnC;oBACD;wBACE,IAAI,EAAE,QAAQ;wBACd,KAAK,EAAE,QAAQ;wBACf,WAAW,EAAE,oCAAoC;qBAClD;iBACF;gBACD,OAAO,EAAE,UAAU;gBACnB,WAAW,EAAE,0CAA0C;aACxD;YACD;gBACE,WAAW,EAAE,mBAAmB;gBAChC,IAAI,EAAE,qBAAqB;gBAC3B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,2EAA2E;aACzF;YACD;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,IAAI;gBACb,WAAW,EAAE,yCAAyC;aACvD;YACD;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,uDAAuD;aACrE;YACD;gBACE,WAAW,EAAE,UAAU;gBACvB,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,MAAM;gBACnB,WAAW,EAAE,wFAAwF;aACtG;SACF;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,cAAc,CAAC;aAC5B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,YAAY;gBACzB,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,qBAAqB;wBAC3B,KAAK,EAAE,QAAQ;wBACf,WAAW,EAAE,oDAAoD;qBAClE;oBACD;wBACE,IAAI,EAAE,qBAAqB;wBAC3B,KAAK,EAAE,UAAU;wBACjB,WAAW,EAAE,mBAAmB;qBACjC;oBACD;wBACE,IAAI,EAAE,sBAAsB;wBAC5B,KAAK,EAAE,SAAS;wBAChB,WAAW,EAAE,mDAAmD;qBACjE;oBACD;wBACE,IAAI,EAAE,WAAW;wBACjB,KAAK,EAAE,WAAW;wBAClB,WAAW,EAAE,gDAAgD;qBAC9D;oBACD;wBACE,IAAI,EAAE,YAAY;wBAClB,KAAK,EAAE,YAAY;wBACnB,WAAW,EAAE,iDAAiD;qBAC/D;iBACF;gBACD,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,oCAAoC;aAClD;YACD;gBACE,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,wDAAwD;aACtE;SACF;KACF;CACF,CAAC;AAWF,MAAM,UAAU,GAA+B;IAC7C,KAAK,EAAE;QACL,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE;QAClD,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC1G,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACpG,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE;QACjG,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC5F,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,uBAAuB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;KACjG;IACD,SAAS,EAAE;QACT,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACnG,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,iCAAiC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC/G,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACnG,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,0BAA0B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACjG,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,yBAAyB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC/F,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,+BAA+B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC1G,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,4BAA4B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;KACtG;IACD,OAAO,EAAE;QACP,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACvG,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,4BAA4B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACzG,EAAE,IAAI,EAAE,oBAAoB,EAAE,QAAQ,EAAE,kCAAkC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACrH,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,4BAA4B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACzG,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,2BAA2B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QACvG,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,8BAA8B,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;KAC9G;IACD,MAAM,EAAE;QACN,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAC5F,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,wBAAwB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;QAClG,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,sBAAsB,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;KAC/F;IACD,QAAQ,EAAE;QACR,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE;QAC5E,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,qCAAqC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE;KACtH;CACF,CAAC;AAGK,KAAK,UAAU,OAAO,CAE3B,KAA2B,EAC3B,WAAgC;;IAEhC,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC;YAEH,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAW,CAAC;YAC1D,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAa,CAAC;YAC9G,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YACrF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YAEvE,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,sBAAsB,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACzF,CAAC;YAED,IAAI,CAAC,IAAA,kBAAU,EAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,gBAAgB,GAAG,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACxF,CAAC;YAED,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,8CAA8C,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YACjH,CAAC;YAGD,MAAM,MAAM,GAAe,EAAE,CAAC;YAC9B,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;gBACrC,IAAI,QAAQ,KAAK,QAAQ,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAClD,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACvC,CAAC;YACH,CAAC;YAGD,MAAM,kBAAkB,GAAQ,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClD,IAAI,EAAE,2BAA2B;gBACjC,MAAM,EAAE;oBACN,MAAM,EAAE;wBACN,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL,IAAI,EAAE,cAAc;4BACpB,YAAY,EAAE,MAAM;4BACpB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gCAC3B,IAAI,EAAE,KAAK,CAAC,IAAI;gCAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gCACxB,IAAI,EAAE,KAAK,CAAC,IAAI;gCAChB,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;6BAC3D,CAAC,CAAC;yBACJ;qBACF;iBACF;aACF,CAAC,CAAC,CAAC,IAAI,CAAC;YAGT,MAAM,cAAc,GAAQ;gBAC1B,GAAG,cAAc;gBACjB,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,SAAS;gBACzC,OAAO,EAAE,cAAc,CAAC,OAAO;aAChC,CAAC;YAEF,MAAM,aAAa,GAAG,IAAA,8BAAsB,EAAC,cAAc,CAAC,CAAC;YAG7D,IAAI,kBAAkB,EAAE,CAAC;gBACvB,aAAa,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;YACxD,CAAC;YAGD,MAAM,OAAO,GAAG,MAAM,IAAA,yBAAiB,EAAC,IAAI,CAAC,CAAC;YAG9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;YAEtD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,iCAAkB,CAC1B,IAAI,CAAC,OAAO,EAAE,EACd,wBAAwB,MAAM,CAAC,aAAa,IAAI,eAAe,EAAE,EACjE,EAAE,SAAS,EAAE,CAAC,EAAE,CACjB,CAAC;YACJ,CAAC;YAGD,IAAI,OAAO,GAAgB,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;oBAEpD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC/C,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;oBACzC,CAAC;yBAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;wBACtC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,MAAM,EAAE,CAAC;oBACtC,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;gBAEb,CAAC;YACH,CAAC;YAGD,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;gBAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC;gBAC9B,CAAC;YACH,CAAC;YAGD,IAAI,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACvC,MAAM,YAAY,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;gBAC/E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC5B,OAAO,CAAC,QAAQ,GAAG,YAAY,CAAC;gBAClC,CAAC;YACH,CAAC;YAGD,MAAM,eAAe,GAAgB;gBACnC,GAAG;gBACH,OAAO,EAAE,IAAI;gBACb,GAAG,EAAE,OAAO;gBACZ,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/E,CAAC;YAGF,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;aACxB,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,cAAc,GAAG,MAAC,KAAa,CAAC,SAAS,mCAAI,CAAC,CAAC;gBACrD,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;oBACnB,KAAK,EAAE,IAAI,iCAAkB,CAAC,IAAI,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;oBAC5F,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAOD,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,UAAU,GAAU,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,4EAA4E,CAAC;IAC3F,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;QAEb,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAKD,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,YAAY,GAA0C,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,oGAAoG,CAAC;IACnH,MAAM,MAAM,GAAG,oGAAoG,CAAC;IACpH,MAAM,MAAM,GAAG,oGAAoG,CAAC;IAEpH,IAAI,KAAK,CAAC;IAEV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3C,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5C,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5C,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,CAAC;IAGD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;QAC/B,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAKD,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC3D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/B,CAAC"}

package/dist/nodes/Crawl4aiPlusContentExtractor/helpers/utils.d.ts

@@ -1,6 +1,8 @@
-export { getCrawl4aiClient, createBrowserConfig, createCrawlerRunConfig, safeJsonParse, cleanText, isValidUrl } from '../../Crawl4aiPlusBasicCrawler/helpers/utils';
+export { getCrawl4aiClient, createBrowserConfig, createCrawlerRunConfig, safeJsonParse, cleanText, isValidUrl, buildLlmConfig, validateLlmCredentials, } from '../../Crawl4aiPlusBasicCrawler/helpers/utils';
+export type { LlmConfigResult } from '../../Crawl4aiPlusBasicCrawler/helpers/utils';
 import { IDataObject } from 'n8n-workflow';
 import { CssSelectorSchema, LlmSchema } from './interfaces';
 export declare function createCssSelectorExtractionStrategy(schema: CssSelectorSchema): any;
-export declare function createLlmExtractionStrategy(schema: LlmSchema, instruction: string, provider: string, apiKey?: string, baseUrl?: string): any;
+export declare function createLlmExtractionStrategy(schema: LlmSchema, instruction: string, provider: string, apiKey?: string, baseUrl?: string, inputFormat?: 'markdown' | 'html' | 'fit_markdown'): any;
+export declare function createCosineExtractionStrategy(semanticFilter: string, options?: IDataObject): any;
 export declare function cleanExtractedData(data: IDataObject): IDataObject;

package/dist/nodes/Crawl4aiPlusContentExtractor/helpers/utils.js

@@ -1,8 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isValidUrl = exports.cleanText = exports.safeJsonParse = exports.createCrawlerRunConfig = exports.createBrowserConfig = exports.getCrawl4aiClient = void 0;
+exports.validateLlmCredentials = exports.buildLlmConfig = exports.isValidUrl = exports.cleanText = exports.safeJsonParse = exports.createCrawlerRunConfig = exports.createBrowserConfig = exports.getCrawl4aiClient = void 0;
 exports.createCssSelectorExtractionStrategy = createCssSelectorExtractionStrategy;
 exports.createLlmExtractionStrategy = createLlmExtractionStrategy;
+exports.createCosineExtractionStrategy = createCosineExtractionStrategy;
 exports.cleanExtractedData = cleanExtractedData;
 var utils_1 = require("../../Crawl4aiPlusBasicCrawler/helpers/utils");
 Object.defineProperty(exports, "getCrawl4aiClient", { enumerable: true, get: function () { return utils_1.getCrawl4aiClient; } });
@@ -11,6 +12,8 @@ Object.defineProperty(exports, "createCrawlerRunConfig", { enumerable: true, get
 Object.defineProperty(exports, "safeJsonParse", { enumerable: true, get: function () { return utils_1.safeJsonParse; } });
 Object.defineProperty(exports, "cleanText", { enumerable: true, get: function () { return utils_1.cleanText; } });
 Object.defineProperty(exports, "isValidUrl", { enumerable: true, get: function () { return utils_1.isValidUrl; } });
+Object.defineProperty(exports, "buildLlmConfig", { enumerable: true, get: function () { return utils_1.buildLlmConfig; } });
+Object.defineProperty(exports, "validateLlmCredentials", { enumerable: true, get: function () { return utils_1.validateLlmCredentials; } });
 const utils_2 = require("../../Crawl4aiPlusBasicCrawler/helpers/utils");
 function createCssSelectorExtractionStrategy(schema) {
     return {
@@ -32,7 +35,7 @@ function createCssSelectorExtractionStrategy(schema) {
         },
     };
 }
-function createLlmExtractionStrategy(schema, instruction, provider, apiKey, baseUrl) {
+function createLlmExtractionStrategy(schema, instruction, provider, apiKey, baseUrl, inputFormat) {
     const llmConfigParams = {
         provider: provider || 'openai/gpt-4o',
         api_token: apiKey,
@@ -40,22 +43,56 @@ function createLlmExtractionStrategy(schema, instruction, provider, apiKey, base
     if (baseUrl && baseUrl.trim() !== '') {
         llmConfigParams.api_base = baseUrl;
     }
+    const strategyParams = {
+        llm_config: {
+            type: 'LLMConfig',
+            params: llmConfigParams,
+        },
+        instruction,
+        schema: {
+            type: 'dict',
+            value: schema,
+        },
+        extraction_type: 'schema',
+        apply_chunking: false,
+        force_json_response: true,
+    };
+    if (inputFormat && inputFormat !== 'markdown') {
+        strategyParams.input_format = inputFormat;
+    }
     return {
         type: 'LLMExtractionStrategy',
-        params: {
-            llm_config: {
-                type: 'LLMConfig',
-                params: llmConfigParams,
-            },
-            instruction,
-            schema: {
-                type: 'dict',
-                value: schema,
-            },
-            extraction_type: 'schema',
-            apply_chunking: false,
-            force_json_response: true,
-        },
+        params: strategyParams,
+    };
+}
+function createCosineExtractionStrategy(semanticFilter, options = {}) {
+    const strategyParams = {
+        semantic_filter: semanticFilter,
+    };
+    if (options.wordCountThreshold !== undefined) {
+        strategyParams.word_count_threshold = Number(options.wordCountThreshold);
+    }
+    if (options.simThreshold !== undefined) {
+        strategyParams.sim_threshold = Number(options.simThreshold);
+    }
+    if (options.maxDist !== undefined) {
+        strategyParams.max_dist = Number(options.maxDist);
+    }
+    if (options.linkageMethod !== undefined && options.linkageMethod !== '') {
+        strategyParams.linkage_method = String(options.linkageMethod);
+    }
+    if (options.topK !== undefined) {
+        strategyParams.top_k = Number(options.topK);
+    }
+    if (options.modelName !== undefined && options.modelName !== '') {
+        strategyParams.model_name = String(options.modelName);
+    }
+    if (options.verbose === true) {
+        strategyParams.verbose = true;
+    }
+    return {
+        type: 'CosineStrategy',
+        params: strategyParams,
     };
 }
 function cleanExtractedData(data) {

package/dist/nodes/Crawl4aiPlusContentExtractor/helpers/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiPlusContentExtractor/helpers/utils.ts"],"names":[],"mappings":";;;AAmBA,kFAmBC;AAWD,kEAkCC;AAKD,gDAyBC;AAhHD,sEAOsD;AANpD,0GAAA,iBAAiB,OAAA;AACjB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AACtB,sGAAA,aAAa,OAAA;AACb,kGAAA,SAAS,OAAA;AACT,mGAAA,UAAU,OAAA;AAIZ,wEAAyE;AAQzE,SAAgB,mCAAmC,CAAC,MAAyB;IAC3E,OAAO;QACL,IAAI,EAAE,2BAA2B;QACjC,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;iBACJ;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAWD,SAAgB,2BAA2B,CACzC,MAAiB,EACjB,WAAmB,EACnB,QAAgB,EAChB,MAAe,EACf,OAAgB;IAEhB,MAAM,eAAe,GAAQ;QAC3B,QAAQ,EAAE,QAAQ,IAAI,eAAe;QACrC,SAAS,EAAE,MAAM;KAClB,CAAC;IAGF,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,eAAe,CAAC,QAAQ,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,MAAM,EAAE;YACN,UAAU,EAAE;gBACV,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,eAAe;aACxB;YACD,WAAW;YACX,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,MAAM;aACd;YACD,eAAe,EAAE,QAAQ;YACzB,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,IAAI;SAC1B;KACF,CAAC;AACJ,CAAC;AAKD,SAAgB,kBAAkB,CAAC,IAAiB;IAClD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,WAAW,GAAgB,EAAE,CAAC;IAEpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,iBAAS,EAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;qBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrD,OAAO,kBAAkB,CAAC,IAAmB,CAAC,CAAC;gBACjD,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,KAAoB,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiPlusContentExtractor/helpers/utils.ts"],"names":[],"mappings":";;;AAwBA,kFAmBC;AAYD,kEA0CC;AAQD,wEAyCC;AAKD,gDAyBC;AA/KD,sEASsD;AARpD,0GAAA,iBAAiB,OAAA;AACjB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AACtB,sGAAA,aAAa,OAAA;AACb,kGAAA,SAAS,OAAA;AACT,mGAAA,UAAU,OAAA;AACV,uGAAA,cAAc,OAAA;AACd,+GAAA,sBAAsB,OAAA;AAOxB,wEAAyE;AAQzE,SAAgB,mCAAmC,CAAC,MAAyB;IAC3E,OAAO;QACL,IAAI,EAAE,2BAA2B;QACjC,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;iBACJ;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAYD,SAAgB,2BAA2B,CACzC,MAAiB,EACjB,WAAmB,EACnB,QAAgB,EAChB,MAAe,EACf,OAAgB,EAChB,WAAkD;IAElD,MAAM,eAAe,GAAQ;QAC3B,QAAQ,EAAE,QAAQ,IAAI,eAAe;QACrC,SAAS,EAAE,MAAM;KAClB,CAAC;IAGF,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,eAAe,CAAC,QAAQ,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,cAAc,GAAQ;QAC1B,UAAU,EAAE;YACV,IAAI,EAAE,WAAW;YACjB,MAAM,EAAE,eAAe;SACxB;QACD,WAAW;QACX,MAAM,EAAE;YACN,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,MAAM;SACd;QACD,eAAe,EAAE,QAAQ;QACzB,cAAc,EAAE,KAAK;QACrB,mBAAmB,EAAE,IAAI;KAC1B,CAAC;IAGF,IAAI,WAAW,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;QAC9C,cAAc,CAAC,YAAY,GAAG,WAAW,CAAC;IAC5C,CAAC;IAED,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,MAAM,EAAE,cAAc;KACvB,CAAC;AACJ,CAAC;AAQD,SAAgB,8BAA8B,CAC5C,cAAsB,EACtB,UAAuB,EAAE;IAEzB,MAAM,cAAc,GAAQ;QAC1B,eAAe,EAAE,cAAc;KAChC,CAAC;IAGF,IAAI,OAAO,CAAC,kBAAkB,KAAK,SAAS,EAAE,CAAC;QAC7C,cAAc,CAAC,oBAAoB,GAAG,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACvC,cAAc,CAAC,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAClC,cAAc,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,CAAC,aAAa,KAAK,EAAE,EAAE,CAAC;QACxE,cAAc,CAAC,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,cAAc,CAAC,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,SAAS,KAAK,EAAE,EAAE,CAAC;QAChE,cAAc,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7B,cAAc,CAAC,OAAO,GAAG,IAAI,CAAC;IAChC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,gBAAgB;QACtB,MAAM,EAAE,cAAc;KACvB,CAAC;AACJ,CAAC;AAKD,SAAgB,kBAAkB,CAAC,IAAiB;IAClD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,WAAW,GAAgB,EAAE,CAAC;IAEpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,iBAAS,EAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;qBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrD,OAAO,kBAAkB,CAAC,IAAmB,CAAC,CAAC;gBACjD,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,KAAoB,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}