n8n-nodes-crawl4ai-plus 2.0.0

package/dist/nodes/Crawl4aiBasicCrawler/actions/operations.js

@@ -0,0 +1,67 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = exports.operations = void 0;
+const crawlSingleUrl = __importStar(require("./crawlSingleUrl.operation"));
+const crawlMultipleUrls = __importStar(require("./crawlMultipleUrls.operation"));
+const processRawHtml = __importStar(require("./processRawHtml.operation"));
+exports.operations = {
+    crawlSingleUrl: crawlSingleUrl.execute,
+    crawlMultipleUrls: crawlMultipleUrls.execute,
+    processRawHtml: processRawHtml.execute,
+};
+exports.description = [
+    {
+        displayName: 'Operation',
+        name: 'operation',
+        type: 'options',
+        noDataExpression: true,
+        options: [
+            {
+                name: 'Crawl Single URL',
+                value: 'crawlSingleUrl',
+                description: 'Crawl a single URL and extract content',
+                action: 'Crawl a single URL',
+            },
+            {
+                name: 'Crawl Multiple URLs',
+                value: 'crawlMultipleUrls',
+                description: 'Crawl multiple URLs and extract content',
+                action: 'Crawl multiple ur ls',
+            },
+            {
+                name: 'Process Raw HTML',
+                value: 'processRawHtml',
+                description: 'Process provided HTML content without crawling',
+                action: 'Process raw html',
+            },
+        ],
+        default: 'crawlSingleUrl',
+    },
+    ...crawlSingleUrl.description,
+    ...crawlMultipleUrls.description,
+    ...processRawHtml.description,
+];
+//# sourceMappingURL=operations.js.map

package/dist/nodes/Crawl4aiBasicCrawler/actions/operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/actions/operations.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,2EAA6D;AAC7D,iFAAmE;AACnE,2EAA6D;AAWhD,QAAA,UAAU,GAAgD;IACrE,cAAc,EAAE,cAAc,CAAC,OAAO;IACtC,iBAAiB,EAAE,iBAAiB,CAAC,OAAO;IAC5C,cAAc,EAAE,cAAc,CAAC,OAAO;CACvC,CAAC;AAGW,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,gBAAgB;gBACvB,WAAW,EAAE,wCAAwC;gBACrD,MAAM,EAAE,oBAAoB;aAC7B;YACD;gBACE,IAAI,EAAE,qBAAqB;gBAC3B,KAAK,EAAE,mBAAmB;gBAC1B,WAAW,EAAE,yCAAyC;gBACtD,MAAM,EAAE,sBAAsB;aAC/B;YACD;gBACE,IAAI,EAAE,kBAAkB;gBACxB,KAAK,EAAE,gBAAgB;gBACvB,WAAW,EAAE,gDAAgD;gBAC7D,MAAM,EAAE,kBAAkB;aAC3B;SACF;QACD,OAAO,EAAE,gBAAgB;KAC1B;IAGD,GAAG,cAAc,CAAC,WAAW;IAC7B,GAAG,iBAAiB,CAAC,WAAW;IAChC,GAAG,cAAc,CAAC,WAAW;CAC9B,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.d.ts

@@ -0,0 +1,4 @@
+import type { IExecuteFunctions, INodeExecutionData, INodeProperties } from 'n8n-workflow';
+import type { Crawl4aiNodeOptions } from '../helpers/interfaces';
+export declare const description: INodeProperties[];
+export declare function execute(this: IExecuteFunctions, items: INodeExecutionData[], nodeOptions: Crawl4aiNodeOptions): Promise<INodeExecutionData[]>;

package/dist/nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.js

@@ -0,0 +1,148 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.description = void 0;
+exports.execute = execute;
+const n8n_workflow_1 = require("n8n-workflow");
+const utils_1 = require("../helpers/utils");
+const formatters_1 = require("../helpers/formatters");
+exports.description = [
+    {
+        displayName: 'HTML Content',
+        name: 'html',
+        type: 'string',
+        typeOptions: {
+            rows: 8,
+        },
+        required: true,
+        default: '',
+        placeholder: '<html><body><h1>Example</h1><p>Content</p></body></html>',
+        description: 'The raw HTML content to process',
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+    },
+    {
+        displayName: 'Base URL',
+        name: 'baseUrl',
+        type: 'string',
+        default: 'https://example.com',
+        description: 'The base URL to use for resolving relative links',
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+    },
+    {
+        displayName: 'Crawler Options',
+        name: 'crawlerOptions',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+        options: [
+            {
+                displayName: 'CSS Selector',
+                name: 'cssSelector',
+                type: 'string',
+                default: '',
+                placeholder: 'article.content',
+                description: 'CSS selector to focus on specific content (leave empty for full page)',
+            },
+            {
+                displayName: 'Exclude External Links',
+                name: 'excludeExternalLinks',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to exclude external links from the result',
+            },
+            {
+                displayName: 'Excluded Tags',
+                name: 'excludedTags',
+                type: 'string',
+                default: '',
+                placeholder: 'nav,footer,aside',
+                description: 'Comma-separated list of HTML tags to exclude from processing',
+            },
+            {
+                displayName: 'Word Count Threshold',
+                name: 'wordCountThreshold',
+                type: 'number',
+                default: 0,
+                description: 'Minimum number of words for content to be included',
+            },
+        ],
+    },
+    {
+        displayName: 'Options',
+        name: 'options',
+        type: 'collection',
+        placeholder: 'Add Option',
+        default: {},
+        displayOptions: {
+            show: {
+                operation: ['processRawHtml'],
+            },
+        },
+        options: [
+            {
+                displayName: 'Include Media Data',
+                name: 'includeMedia',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include media data in output (images, videos)',
+            },
+            {
+                displayName: 'Verbose Response',
+                name: 'verboseResponse',
+                type: 'boolean',
+                default: false,
+                description: 'Whether to include detailed data in output (HTML, status codes, etc.)',
+            },
+        ],
+    },
+];
+async function execute(items, nodeOptions) {
+    var _a;
+    const allResults = [];
+    for (let i = 0; i < items.length; i++) {
+        try {
+            const html = this.getNodeParameter('html', i, '');
+            const baseUrl = this.getNodeParameter('baseUrl', i, 'https://example.com');
+            const crawlerOptions = this.getNodeParameter('crawlerOptions', i, {});
+            const options = this.getNodeParameter('options', i, {});
+            if (!html) {
+                throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'HTML content cannot be empty.', { itemIndex: i });
+            }
+            const crawlerConfig = (0, utils_1.createCrawlerRunConfig)(crawlerOptions);
+            const crawler = await (0, utils_1.getCrawl4aiClient)(this);
+            const result = await crawler.processRawHtml(html, baseUrl, crawlerConfig);
+            const formattedResult = (0, formatters_1.formatCrawlResult)(result, options.includeMedia, options.verboseResponse);
+            allResults.push({
+                json: formattedResult,
+                pairedItem: { item: i },
+            });
+        }
+        catch (error) {
+            if (this.continueOnFail()) {
+                const node = this.getNode();
+                const errorItemIndex = (_a = error.itemIndex) !== null && _a !== void 0 ? _a : i;
+                allResults.push({
+                    json: items[i].json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: errorItemIndex }),
+                    pairedItem: { item: i },
+                });
+                continue;
+            }
+            throw error;
+        }
+    }
+    return allResults;
+}
+//# sourceMappingURL=processRawHtml.operation.js.map

package/dist/nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"processRawHtml.operation.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/actions/processRawHtml.operation.ts"],"names":[],"mappings":";;;AA0HA,0BA2DC;AA/KD,+CAAkD;AAIlD,4CAG0B;AAC1B,sDAA0D;AAG7C,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,cAAc;QAC3B,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE;YACX,IAAI,EAAE,CAAC;SACR;QACD,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,0DAA0D;QACvE,WAAW,EAAE,iCAAiC;QAC9C,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;KACF;IACD;QACE,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,qBAAqB;QAC9B,WAAW,EAAE,kDAAkD;QAC/D,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;KACF;IACD;QACE,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,cAAc;gBAC3B,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,iBAAiB;gBAC9B,WAAW,EAAE,uEAAuE;aACrF;YACD;gBACE,WAAW,EAAE,wBAAwB;gBACrC,IAAI,EAAE,sBAAsB;gBAC5B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,WAAW,EAAE,eAAe;gBAC5B,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,kBAAkB;gBAC/B,WAAW,EAAE,8DAA8D;aAC5E;YACD;gBACE,WAAW,EAAE,sBAAsB;gBACnC,IAAI,EAAE,oBAAoB;gBAC1B,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE,CAAC;gBACV,WAAW,EAAE,oDAAoD;aAClE;SACF;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,SAAS,EAAE,CAAC,gBAAgB,CAAC;aAC9B;SACF;QACD,OAAO,EAAE;YACP;gBACE,WAAW,EAAE,oBAAoB;gBACjC,IAAI,EAAE,cAAc;gBACpB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,0DAA0D;aACxE;YACD;gBACE,WAAW,EAAE,kBAAkB;gBAC/B,IAAI,EAAE,iBAAiB;gBACvB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,KAAK;gBACd,WAAW,EAAE,uEAAuE;aACrF;SACF;KACF;CACF,CAAC;AAGK,KAAK,UAAU,OAAO,CAE3B,KAA2B,EAC3B,WAAgC;;IAEhC,MAAM,UAAU,GAAyB,EAAE,CAAC;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC;YAEH,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAW,CAAC;YAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,qBAAqB,CAAW,CAAC;YACrF,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YACrF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAgB,CAAC;YAEvE,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,+BAA+B,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;YAClG,CAAC;YAGD,MAAM,aAAa,GAAG,IAAA,8BAAsB,EAAC,cAAc,CAAC,CAAC;YAG7D,MAAM,OAAO,GAAG,MAAM,IAAA,yBAAiB,EAAC,IAAI,CAAC,CAAC;YAG9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;YAG1E,MAAM,eAAe,GAAG,IAAA,8BAAiB,EACvC,MAAM,EACN,OAAO,CAAC,YAAuB,EAC/B,OAAO,CAAC,eAA0B,CACnC,CAAC;YAGF,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI,EAAE,eAAe;gBACrB,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;aACxB,CAAC,CAAC;QAEL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAEf,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,cAAc,GAAG,MAAC,KAAa,CAAC,SAAS,mCAAI,CAAC,CAAC;gBACrD,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;oBACnB,KAAK,EAAE,IAAI,iCAAkB,CAAC,IAAI,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC;oBAC5F,UAAU,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;iBACxB,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/actions/router.d.ts

@@ -0,0 +1,2 @@
+import type { IExecuteFunctions, INodeExecutionData } from 'n8n-workflow';
+export declare function router(this: IExecuteFunctions): Promise<INodeExecutionData[][]>;

package/dist/nodes/Crawl4aiBasicCrawler/actions/router.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.router = router;
+const n8n_workflow_1 = require("n8n-workflow");
+const operations_1 = require("./operations");
+async function router() {
+    const items = this.getInputData();
+    let returnData = [];
+    const node = this.getNode();
+    const operation = this.getNodeParameter('operation', 0);
+    const nodeOptions = this.getNodeParameter('options', 0, {});
+    const continueOnFail = this.continueOnFail();
+    try {
+        const executeFunction = operations_1.operations[operation];
+        if (!executeFunction) {
+            throw new n8n_workflow_1.NodeOperationError(node, `The operation "${operation}" is not supported!`, { itemIndex: 0 });
+        }
+        returnData = await executeFunction.call(this, items, nodeOptions);
+    }
+    catch (error) {
+        if (continueOnFail) {
+            returnData = items.map((item, index) => {
+                var _a;
+                return ({
+                    json: item.json,
+                    error: new n8n_workflow_1.NodeOperationError(node, error.message, { itemIndex: (_a = error.itemIndex) !== null && _a !== void 0 ? _a : index }),
+                    pairedItem: { item: index },
+                });
+            });
+        }
+        else {
+            throw error;
+        }
+    }
+    return this.prepareOutputData(returnData);
+}
+//# sourceMappingURL=router.js.map

package/dist/nodes/Crawl4aiBasicCrawler/actions/router.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"router.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/actions/router.ts"],"names":[],"mappings":";;AAYA,wBA0CC;AAlDD,+CAAkD;AAMlD,6CAA0C;AAEnC,KAAK,UAAU,MAAM;IAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,IAAI,UAAU,GAAyB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAG5B,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,CAAC,CAAW,CAAC;IAElE,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAwB,CAAC;IAEnF,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;IAE7C,IAAI,CAAC;QAIH,MAAM,eAAe,GAAG,uBAAU,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,IAAI,iCAAkB,CAAC,IAAI,EAAE,kBAAkB,SAAS,qBAAqB,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;QAGD,UAAU,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;IAGpE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,cAAc,EAAE,CAAC;YAEnB,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;;gBAAC,OAAA,CAAC;oBACvC,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,KAAK,EAAE,IAAI,iCAAkB,CAAC,IAAI,EAAG,KAAe,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,MAAC,KAAa,CAAC,SAAS,mCAAI,KAAK,EAAE,CAAC;oBAC/G,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE;iBAC5B,CAAC,CAAA;aAAA,CAAC,CAAC;QACN,CAAC;aAAM,CAAC;YAEN,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAGD,OAAO,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;AAC5C,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/crawl4ai.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="32px" height="32px" viewBox="0 0 32 32" version="1.1" xmlns="http://www.w3.org/2000/svg">
+    <title>Crawl4AI Icon</title>
+    <g id="Crawl4AI" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <path d="M16,2 C8.268,2 2,8.268 2,16 C2,23.732 8.268,30 16,30 C23.732,30 30,23.732 30,16 C30,8.268 23.732,2 16,2 Z" id="Spider-Body" fill="#3498DB"></path>
+        <path d="M16,6 C10.477,6 6,10.477 6,16 C6,21.523 10.477,26 16,26 C21.523,26 26,21.523 26,16 C26,10.477 21.523,6 16,6 Z M16,22 C12.686,22 10,19.314 10,16 C10,12.686 12.686,10 16,10 C19.314,10 22,12.686 22,16 C22,19.314 19.314,22 16,22 Z" id="Web" fill="#FFFFFF"></path>
+        <circle id="Eye-1" fill="#FFFFFF" cx="13" cy="14" r="1.5"></circle>
+        <circle id="Eye-2" fill="#FFFFFF" cx="19" cy="14" r="1.5"></circle>
+        <!-- Spider legs -->
+        <path d="M4,8 C2,6 2,4 3,3 C4,2 6,2 8,4" id="Leg-1" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+        <path d="M28,8 C30,6 30,4 29,3 C28,2 26,2 24,4" id="Leg-2" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+        <path d="M4,24 C2,26 2,28 3,29 C4,30 6,30 8,28" id="Leg-3" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+        <path d="M28,24 C30,26 30,28 29,29 C28,30 26,30 24,28" id="Leg-4" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+        <path d="M4,16 C2,16 0,15 0,14 C0,13 2,12 4,12" id="Leg-5" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+        <path d="M28,16 C30,16 32,15 32,14 C32,13 30,12 28,12" id="Leg-6" stroke="#2980B9" stroke-width="1.5" stroke-linecap="round"></path>
+    </g>
+</svg>

package/dist/nodes/Crawl4aiBasicCrawler/helpers/apiClient.d.ts

@@ -0,0 +1,15 @@
+import { Crawl4aiApiCredentials, CrawlerRunConfig, CrawlResult } from './interfaces';
+export declare class Crawl4aiClient {
+    private apiClient;
+    private readonly credentials;
+    constructor(credentials: Crawl4aiApiCredentials);
+    private createApiClient;
+    crawlUrl(url: string, config: CrawlerRunConfig): Promise<CrawlResult>;
+    crawlMultipleUrls(urls: string[], config: CrawlerRunConfig): Promise<CrawlResult[]>;
+    processRawHtml(html: string, baseUrl: string, config: CrawlerRunConfig): Promise<CrawlResult>;
+    arun(url: string, options: any): Promise<CrawlResult>;
+    private formatBrowserConfig;
+    private formatCrawlerConfig;
+    close(): Promise<void>;
+}
+export declare function createCrawlerInstance(credentials: Crawl4aiApiCredentials): Promise<Crawl4aiClient>;

package/dist/nodes/Crawl4aiBasicCrawler/helpers/apiClient.js

@@ -0,0 +1,226 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Crawl4aiClient = void 0;
+exports.createCrawlerInstance = createCrawlerInstance;
+const axios_1 = __importDefault(require("axios"));
+class Crawl4aiClient {
+    constructor(credentials) {
+        this.credentials = credentials;
+        this.apiClient = this.createApiClient();
+    }
+    createApiClient() {
+        const baseURL = this.credentials.connectionMode === 'docker'
+            ? this.credentials.dockerUrl
+            : 'http://localhost:11235';
+        const client = axios_1.default.create({
+            baseURL,
+            timeout: 60000,
+        });
+        if (this.credentials.connectionMode === 'docker' && this.credentials.authenticationType) {
+            if (this.credentials.authenticationType === 'token' && this.credentials.apiToken) {
+                client.defaults.headers.common['Authorization'] = `Bearer ${this.credentials.apiToken}`;
+            }
+            else if (this.credentials.authenticationType === 'basic' && this.credentials.username && this.credentials.password) {
+                client.defaults.auth = {
+                    username: this.credentials.username,
+                    password: this.credentials.password
+                };
+            }
+        }
+        return client;
+    }
+    async crawlUrl(url, config) {
+        try {
+            const response = await this.apiClient.post('/crawl', {
+                urls: [url],
+                browser_config: this.formatBrowserConfig(config),
+                crawler_config: this.formatCrawlerConfig(config),
+            });
+            if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
+                return response.data.results[0];
+            }
+            return {
+                url,
+                success: false,
+                error_message: 'No result returned from Crawl4AI API',
+            };
+        }
+        catch (error) {
+            return {
+                url,
+                success: false,
+                error_message: error instanceof Error ? error.message : 'Unknown error occurred',
+            };
+        }
+    }
+    async crawlMultipleUrls(urls, config) {
+        try {
+            const response = await this.apiClient.post('/crawl', {
+                urls,
+                browser_config: this.formatBrowserConfig(config),
+                crawler_config: this.formatCrawlerConfig(config),
+            });
+            if (response.data && Array.isArray(response.data.results)) {
+                return response.data.results;
+            }
+            return urls.map(url => ({
+                url,
+                success: false,
+                error_message: 'No results returned from Crawl4AI API',
+            }));
+        }
+        catch (error) {
+            return urls.map(url => ({
+                url,
+                success: false,
+                error_message: error instanceof Error ? error.message : 'Unknown error occurred',
+            }));
+        }
+    }
+    async processRawHtml(html, baseUrl, config) {
+        try {
+            const rawUrl = `raw://${encodeURIComponent(html)}`;
+            const response = await this.apiClient.post('/crawl', {
+                urls: [rawUrl],
+                browser_config: this.formatBrowserConfig(config),
+                crawler_config: {
+                    ...this.formatCrawlerConfig(config),
+                    base_url: baseUrl,
+                },
+            });
+            if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
+                return response.data.results[0];
+            }
+            return {
+                url: baseUrl,
+                success: false,
+                error_message: 'No result returned from Crawl4AI API',
+            };
+        }
+        catch (error) {
+            return {
+                url: baseUrl,
+                success: false,
+                error_message: error instanceof Error ? error.message : 'Unknown error occurred',
+            };
+        }
+    }
+    async arun(url, options) {
+        var _a, _b;
+        try {
+            const crawlerConfig = {
+                cache_mode: options.cacheMode || 'ENABLED',
+                js_code: options.jsCode,
+                css_selector: options.cssSelector,
+            };
+            if (options.extractionStrategy) {
+                crawlerConfig.extraction_strategy = options.extractionStrategy;
+            }
+            if (options.extraArgs) {
+                Object.assign(crawlerConfig, options.extraArgs);
+            }
+            const requestBody = {
+                urls: [url],
+                browser_config: this.formatBrowserConfig(options.browserConfig || {}),
+                crawler_config: {
+                    type: 'CrawlerRunConfig',
+                    params: crawlerConfig,
+                },
+            };
+            const response = await this.apiClient.post('/crawl', requestBody);
+            if (response.data && Array.isArray(response.data.results) && response.data.results.length > 0) {
+                return response.data.results[0];
+            }
+            return {
+                url,
+                success: false,
+                error_message: 'No result returned from Crawl4AI API',
+            };
+        }
+        catch (error) {
+            return {
+                url,
+                success: false,
+                error_message: ((_b = (_a = error.response) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.detail) || error.message || 'Unknown error occurred',
+            };
+        }
+    }
+    formatBrowserConfig(config) {
+        var _a, _b;
+        return {
+            type: 'BrowserConfig',
+            params: {
+                browser_type: config.browserType || 'chromium',
+                headless: config.headless !== false,
+                browser_mode: 'dedicated',
+                use_managed_browser: false,
+                cdp_url: null,
+                debugging_port: 9222,
+                use_persistent_context: false,
+                user_data_dir: null,
+                chrome_channel: 'chromium',
+                channel: 'chromium',
+                proxy: null,
+                viewport_width: ((_a = config.viewport) === null || _a === void 0 ? void 0 : _a.width) || 1080,
+                viewport_height: ((_b = config.viewport) === null || _b === void 0 ? void 0 : _b.height) || 600,
+                accept_downloads: false,
+                downloads_path: null,
+                storage_state: null,
+                ignore_https_errors: true,
+                java_script_enabled: config.javaScriptEnabled !== false,
+                sleep_on_close: false,
+                verbose: true,
+                cookies: [],
+                headers: {},
+                user_agent: config.userAgent || "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36",
+                user_agent_mode: "",
+                user_agent_generator_config: {},
+                text_mode: false,
+                light_mode: false,
+                extra_args: [],
+                host: "localhost",
+                enable_stealth: config.enable_stealth || false,
+            },
+        };
+    }
+    formatCrawlerConfig(config) {
+        const params = {
+            cache_mode: config.cacheMode || 'ENABLED',
+            stream: config.streamEnabled || false,
+            page_timeout: config.pageTimeout || config.timeout || 30000,
+            wait_until: 'domcontentloaded',
+            wait_for: config.waitFor || null,
+            js_code: config.jsCode,
+            js_only: config.jsOnly || false,
+            css_selector: config.cssSelector,
+            excluded_tags: config.excludedTags || [],
+            exclude_external_links: config.excludeExternalLinks || false,
+            check_robots_txt: config.checkRobotsTxt || false,
+            word_count_threshold: config.wordCountThreshold || 0,
+            session_id: config.sessionId,
+            only_text: false,
+            scan_full_page: false,
+            remove_overlay_elements: false,
+            simulate_user: false,
+            override_navigator: false,
+            magic: false,
+        };
+        if (config.extractionStrategy) {
+            params.extraction_strategy = config.extractionStrategy;
+        }
+        return {
+            type: 'CrawlerRunConfig',
+            params,
+        };
+    }
+    async close() {
+    }
+}
+exports.Crawl4aiClient = Crawl4aiClient;
+async function createCrawlerInstance(credentials) {
+    return new Crawl4aiClient(credentials);
+}
+//# sourceMappingURL=apiClient.js.map

package/dist/nodes/Crawl4aiBasicCrawler/helpers/apiClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apiClient.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/helpers/apiClient.ts"],"names":[],"mappings":";;;;;;AA6RA,sDAIC;AAjSD,kDAA6C;AAM7C,MAAa,cAAc;IAIzB,YAAY,WAAmC;QAC7C,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;IAC1C,CAAC;IAKO,eAAe;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,KAAK,QAAQ;YAC1D,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS;YAC5B,CAAC,CAAC,wBAAwB,CAAC;QAE7B,MAAM,MAAM,GAAG,eAAK,CAAC,MAAM,CAAC;YAC1B,OAAO;YACP,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QAGH,IAAI,IAAI,CAAC,WAAW,CAAC,cAAc,KAAK,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,EAAE,CAAC;YACxF,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,KAAK,OAAO,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;gBACjF,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,UAAU,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;YAC1F,CAAC;iBAAM,IAAI,IAAI,CAAC,WAAW,CAAC,kBAAkB,KAAK,OAAO,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;gBAErH,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG;oBACrB,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ;oBACnC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ;iBACpC,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAKD,KAAK,CAAC,QAAQ,CAAC,GAAW,EAAE,MAAwB;QAClD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACnD,IAAI,EAAE,CAAC,GAAG,CAAC;gBACX,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAChD,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;aACjD,CAAC,CAAC;YAGH,IAAI,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9F,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,sCAAsC;aACtD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;aACjF,CAAC;QACJ,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,iBAAiB,CAAC,IAAc,EAAE,MAAwB;QAC9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACnD,IAAI;gBACJ,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAChD,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;aACjD,CAAC,CAAC;YAGH,IAAI,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1D,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;YAC/B,CAAC;YAED,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtB,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,uCAAuC;aACvD,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtB,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;aACjF,CAAC,CAAC,CAAC;QACN,CAAC;IACH,CAAC;IAKD,KAAK,CAAC,cAAc,CAAC,IAAY,EAAE,OAAe,EAAE,MAAwB;QAC1E,IAAI,CAAC;YAGH,MAAM,MAAM,GAAG,SAAS,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE;gBACnD,IAAI,EAAE,CAAC,MAAM,CAAC;gBACd,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;gBAChD,cAAc,EAAE;oBACd,GAAG,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC;oBACnC,QAAQ,EAAE,OAAO;iBAClB;aACF,CAAC,CAAC;YAGH,IAAI,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9F,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,OAAO;gBACL,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,sCAAsC;aACtD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,GAAG,EAAE,OAAO;gBACZ,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;aACjF,CAAC;QACJ,CAAC;IACH,CAAC;IAQD,KAAK,CAAC,IAAI,CAAC,GAAW,EAAE,OAAY;;QAClC,IAAI,CAAC;YAEH,MAAM,aAAa,GAAQ;gBACzB,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,SAAS;gBAC1C,OAAO,EAAE,OAAO,CAAC,MAAM;gBACvB,YAAY,EAAE,OAAO,CAAC,WAAW;aAClC,CAAC;YAGF,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBAC/B,aAAa,CAAC,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;YACjE,CAAC;YAGD,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBACtB,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAClD,CAAC;YAGD,MAAM,WAAW,GAAG;gBAClB,IAAI,EAAE,CAAC,GAAG,CAAC;gBACX,cAAc,EAAE,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;gBACrE,cAAc,EAAE;oBACd,IAAI,EAAE,kBAAkB;oBACxB,MAAM,EAAE,aAAa;iBACtB;aACF,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;YAGlE,IAAI,QAAQ,CAAC,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9F,OAAO,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC;YAED,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,sCAAsC;aACtD,CAAC;QACJ,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,KAAK;gBACd,aAAa,EAAE,CAAA,MAAA,MAAA,KAAK,CAAC,QAAQ,0CAAE,IAAI,0CAAE,MAAM,KAAI,KAAK,CAAC,OAAO,IAAI,wBAAwB;aACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAKO,mBAAmB,CAAC,MAAwB;;QAClD,OAAO;YACL,IAAI,EAAE,eAAe;YACrB,MAAM,EAAE;gBACN,YAAY,EAAG,MAAc,CAAC,WAAW,IAAI,UAAU;gBACvD,QAAQ,EAAE,MAAM,CAAC,QAAQ,KAAK,KAAK;gBACnC,YAAY,EAAE,WAAW;gBACzB,mBAAmB,EAAE,KAAK;gBAC1B,OAAO,EAAE,IAAI;gBACb,cAAc,EAAE,IAAI;gBACpB,sBAAsB,EAAE,KAAK;gBAC7B,aAAa,EAAE,IAAI;gBACnB,cAAc,EAAE,UAAU;gBAC1B,OAAO,EAAE,UAAU;gBACnB,KAAK,EAAE,IAAI;gBACX,cAAc,EAAE,CAAA,MAAA,MAAM,CAAC,QAAQ,0CAAE,KAAK,KAAI,IAAI;gBAC9C,eAAe,EAAE,CAAA,MAAA,MAAM,CAAC,QAAQ,0CAAE,MAAM,KAAI,GAAG;gBAC/C,gBAAgB,EAAE,KAAK;gBACvB,cAAc,EAAE,IAAI;gBACpB,aAAa,EAAE,IAAI;gBACnB,mBAAmB,EAAE,IAAI;gBACzB,mBAAmB,EAAE,MAAM,CAAC,iBAAiB,KAAK,KAAK;gBACvD,cAAc,EAAE,KAAK;gBACrB,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE,EAAE;gBACX,UAAU,EAAE,MAAM,CAAC,SAAS,IAAI,mFAAmF;gBACnH,eAAe,EAAE,EAAE;gBACnB,2BAA2B,EAAE,EAAE;gBAC/B,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,KAAK;gBACjB,UAAU,EAAE,EAAE;gBACd,IAAI,EAAE,WAAW;gBACjB,cAAc,EAAG,MAAc,CAAC,cAAc,IAAI,KAAK;aACxD;SACF,CAAC;IACJ,CAAC;IAKO,mBAAmB,CAAC,MAAwB;QAClD,MAAM,MAAM,GAAQ;YAClB,UAAU,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;YACzC,MAAM,EAAE,MAAM,CAAC,aAAa,IAAI,KAAK;YACrC,YAAY,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK;YAC3D,UAAU,EAAE,kBAAkB;YAC9B,QAAQ,EAAG,MAAc,CAAC,OAAO,IAAI,IAAI;YACzC,OAAO,EAAE,MAAM,CAAC,MAAM;YACtB,OAAO,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC/B,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,aAAa,EAAE,MAAM,CAAC,YAAY,IAAI,EAAE;YACxC,sBAAsB,EAAE,MAAM,CAAC,oBAAoB,IAAI,KAAK;YAC5D,gBAAgB,EAAE,MAAM,CAAC,cAAc,IAAI,KAAK;YAChD,oBAAoB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC;YACpD,UAAU,EAAE,MAAM,CAAC,SAAS;YAC5B,SAAS,EAAE,KAAK;YAChB,cAAc,EAAE,KAAK;YACrB,uBAAuB,EAAE,KAAK;YAC9B,aAAa,EAAE,KAAK;YACpB,kBAAkB,EAAE,KAAK;YACzB,KAAK,EAAE,KAAK;SACb,CAAC;QAGF,IAAI,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC9B,MAAM,CAAC,mBAAmB,GAAG,MAAM,CAAC,kBAAkB,CAAC;QACzD,CAAC;QAED,OAAO;YACL,IAAI,EAAE,kBAAkB;YACxB,MAAM;SACP,CAAC;IACJ,CAAC;IAKD,KAAK,CAAC,KAAK;IAEX,CAAC;CACF;AAlRD,wCAkRC;AAKM,KAAK,UAAU,qBAAqB,CACzC,WAAmC;IAEnC,OAAO,IAAI,cAAc,CAAC,WAAW,CAAC,CAAC;AACzC,CAAC"}

package/dist/nodes/Crawl4aiBasicCrawler/helpers/formatters.d.ts

@@ -0,0 +1,5 @@
+import { IDataObject } from 'n8n-workflow';
+import { CrawlResult } from './interfaces';
+export declare function formatCrawlResult(result: CrawlResult, includeMedia?: boolean, verboseResponse?: boolean): IDataObject;
+export declare function parseExtractedJson(result: CrawlResult): IDataObject | null;
+export declare function formatExtractionResult(result: CrawlResult, extractedData: IDataObject | null, includeFullText?: boolean): IDataObject;

package/dist/nodes/Crawl4aiBasicCrawler/helpers/formatters.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatCrawlResult = formatCrawlResult;
+exports.parseExtractedJson = parseExtractedJson;
+exports.formatExtractionResult = formatExtractionResult;
+function formatCrawlResult(result, includeMedia = false, verboseResponse = false) {
+    const formattedResult = {
+        url: result.url,
+        success: result.success,
+        statusCode: result.statusCode || result.status_code || null,
+        title: result.title || '',
+        content: result.markdown || '',
+        text: result.text || '',
+    };
+    if (!result.success && result.error_message) {
+        formattedResult.error = result.error_message;
+    }
+    if (result.extracted_content) {
+        try {
+            formattedResult.extractedContent = JSON.parse(result.extracted_content);
+        }
+        catch (e) {
+            formattedResult.extractedContent = result.extracted_content;
+        }
+    }
+    if (result.links) {
+        formattedResult.links = {
+            internal: result.links.internal,
+            external: result.links.external,
+        };
+    }
+    if (includeMedia && result.media) {
+        formattedResult.media = {
+            images: result.media.images,
+            videos: result.media.videos,
+        };
+    }
+    if (verboseResponse) {
+        formattedResult.html = result.html || '';
+        formattedResult.cleanedHtml = result.cleaned_html || '';
+        formattedResult.crawlTime = result.crawl_time;
+        formattedResult.metadata = result.metadata || {};
+    }
+    return formattedResult;
+}
+function parseExtractedJson(result) {
+    if (!result.extracted_content) {
+        return null;
+    }
+    try {
+        return JSON.parse(result.extracted_content);
+    }
+    catch (error) {
+        return null;
+    }
+}
+function formatExtractionResult(result, extractedData, includeFullText = false) {
+    const formattedResult = {
+        url: result.url,
+        success: result.success,
+        statusCode: result.statusCode || result.status_code || null,
+    };
+    if (!result.success && result.error_message) {
+        formattedResult.error = result.error_message;
+        return formattedResult;
+    }
+    if (extractedData) {
+        Object.entries(extractedData).forEach(([key, value]) => {
+            formattedResult[key] = value;
+        });
+    }
+    else {
+        formattedResult.extractionFailed = true;
+    }
+    if (includeFullText) {
+        formattedResult.originalText = result.text || '';
+        formattedResult.title = result.title || '';
+    }
+    return formattedResult;
+}
+//# sourceMappingURL=formatters.js.map

package/dist/nodes/Crawl4aiBasicCrawler/helpers/formatters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatters.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiBasicCrawler/helpers/formatters.ts"],"names":[],"mappings":";;AAUA,8CAwDC;AAOD,gDAUC;AASD,wDAkCC;AApHD,SAAgB,iBAAiB,CAC/B,MAAmB,EACnB,eAAwB,KAAK,EAC7B,kBAA2B,KAAK;IAGhC,MAAM,eAAe,GAAgB;QACnC,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI;QAC3D,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;QACzB,OAAO,EAAE,MAAM,CAAC,QAAQ,IAAI,EAAE;QAC9B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE;KACxB,CAAC;IAGF,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QAC5C,eAAe,CAAC,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;IAC/C,CAAC;IAGD,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC7B,IAAI,CAAC;YAEH,eAAe,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC1E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YAEX,eAAe,CAAC,gBAAgB,GAAG,MAAM,CAAC,iBAAiB,CAAC;QAC9D,CAAC;IACH,CAAC;IAGD,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,eAAe,CAAC,KAAK,GAAG;YACtB,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;YAC/B,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;SAChC,CAAC;IACJ,CAAC;IAGD,IAAI,YAAY,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjC,eAAe,CAAC,KAAK,GAAG;YACtB,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;YAC3B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;SAC5B,CAAC;IACJ,CAAC;IAGD,IAAI,eAAe,EAAE,CAAC;QACpB,eAAe,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACzC,eAAe,CAAC,WAAW,GAAG,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC;QACxD,eAAe,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC;QAC9C,eAAe,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAOD,SAAgB,kBAAkB,CAAC,MAAmB;IACpD,IAAI,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAgB,CAAC;IAC7D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AASD,SAAgB,sBAAsB,CACpC,MAAmB,EACnB,aAAiC,EACjC,kBAA2B,KAAK;IAGhC,MAAM,eAAe,GAAgB;QACnC,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,WAAW,IAAI,IAAI;KAC5D,CAAC;IAGF,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QAC5C,eAAe,CAAC,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC;QAC7C,OAAO,eAAe,CAAC;IACzB,CAAC;IAGD,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YACrD,eAAe,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC/B,CAAC,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,eAAe,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAC1C,CAAC;IAGD,IAAI,eAAe,EAAE,CAAC;QACpB,eAAe,CAAC,YAAY,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACjD,eAAe,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC"}