n8n-nodes-crawl4ai-plus 2.0.0

package/dist/nodes/Crawl4aiContentExtractor/helpers/apiClient.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCrawlerInstance = exports.Crawl4aiClient = void 0;
+var apiClient_1 = require("../../Crawl4aiBasicCrawler/helpers/apiClient");
+Object.defineProperty(exports, "Crawl4aiClient", { enumerable: true, get: function () { return apiClient_1.Crawl4aiClient; } });
+Object.defineProperty(exports, "createCrawlerInstance", { enumerable: true, get: function () { return apiClient_1.createCrawlerInstance; } });
+//# sourceMappingURL=apiClient.js.map

package/dist/nodes/Crawl4aiContentExtractor/helpers/apiClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apiClient.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiContentExtractor/helpers/apiClient.ts"],"names":[],"mappings":";;;AACA,0EAGsD;AAFpD,2GAAA,cAAc,OAAA;AACd,kHAAA,qBAAqB,OAAA"}

package/dist/nodes/Crawl4aiContentExtractor/helpers/formatters.d.ts

@@ -0,0 +1 @@
+export { formatCrawlResult, parseExtractedJson, formatExtractionResult } from '../../Crawl4aiBasicCrawler/helpers/formatters';

package/dist/nodes/Crawl4aiContentExtractor/helpers/formatters.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatExtractionResult = exports.parseExtractedJson = exports.formatCrawlResult = void 0;
+var formatters_1 = require("../../Crawl4aiBasicCrawler/helpers/formatters");
+Object.defineProperty(exports, "formatCrawlResult", { enumerable: true, get: function () { return formatters_1.formatCrawlResult; } });
+Object.defineProperty(exports, "parseExtractedJson", { enumerable: true, get: function () { return formatters_1.parseExtractedJson; } });
+Object.defineProperty(exports, "formatExtractionResult", { enumerable: true, get: function () { return formatters_1.formatExtractionResult; } });
+//# sourceMappingURL=formatters.js.map

package/dist/nodes/Crawl4aiContentExtractor/helpers/formatters.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"formatters.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiContentExtractor/helpers/formatters.ts"],"names":[],"mappings":";;;AACA,4EAIuD;AAHrD,+GAAA,iBAAiB,OAAA;AACjB,gHAAA,kBAAkB,OAAA;AAClB,oHAAA,sBAAsB,OAAA"}

package/dist/nodes/Crawl4aiContentExtractor/helpers/interfaces.d.ts

@@ -0,0 +1 @@
+export { Crawl4aiApiCredentials, Crawl4aiNodeOptions, BrowserConfig, CrawlerRunConfig, CrawlResult, Link, Media, CssSelectorField, CssSelectorSchema, LlmSchemaField, LlmSchema } from '../../Crawl4aiBasicCrawler/helpers/interfaces';

package/dist/nodes/Crawl4aiContentExtractor/helpers/interfaces.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=interfaces.js.map

package/dist/nodes/Crawl4aiContentExtractor/helpers/interfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"interfaces.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiContentExtractor/helpers/interfaces.ts"],"names":[],"mappings":""}

package/dist/nodes/Crawl4aiContentExtractor/helpers/utils.d.ts

@@ -0,0 +1,6 @@
+export { getCrawl4aiClient, createBrowserConfig, createCrawlerRunConfig, safeJsonParse, cleanText, isValidUrl } from '../../Crawl4aiBasicCrawler/helpers/utils';
+import { IDataObject } from 'n8n-workflow';
+import { CssSelectorSchema, LlmSchema } from './interfaces';
+export declare function createCssSelectorExtractionStrategy(schema: CssSelectorSchema): any;
+export declare function createLlmExtractionStrategy(schema: LlmSchema, instruction: string, provider: string, apiKey?: string, baseUrl?: string): any;
+export declare function cleanExtractedData(data: IDataObject): IDataObject;

package/dist/nodes/Crawl4aiContentExtractor/helpers/utils.js

@@ -0,0 +1,89 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isValidUrl = exports.cleanText = exports.safeJsonParse = exports.createCrawlerRunConfig = exports.createBrowserConfig = exports.getCrawl4aiClient = void 0;
+exports.createCssSelectorExtractionStrategy = createCssSelectorExtractionStrategy;
+exports.createLlmExtractionStrategy = createLlmExtractionStrategy;
+exports.cleanExtractedData = cleanExtractedData;
+var utils_1 = require("../../Crawl4aiBasicCrawler/helpers/utils");
+Object.defineProperty(exports, "getCrawl4aiClient", { enumerable: true, get: function () { return utils_1.getCrawl4aiClient; } });
+Object.defineProperty(exports, "createBrowserConfig", { enumerable: true, get: function () { return utils_1.createBrowserConfig; } });
+Object.defineProperty(exports, "createCrawlerRunConfig", { enumerable: true, get: function () { return utils_1.createCrawlerRunConfig; } });
+Object.defineProperty(exports, "safeJsonParse", { enumerable: true, get: function () { return utils_1.safeJsonParse; } });
+Object.defineProperty(exports, "cleanText", { enumerable: true, get: function () { return utils_1.cleanText; } });
+Object.defineProperty(exports, "isValidUrl", { enumerable: true, get: function () { return utils_1.isValidUrl; } });
+const utils_2 = require("../../Crawl4aiBasicCrawler/helpers/utils");
+function createCssSelectorExtractionStrategy(schema) {
+    return {
+        type: 'JsonCssExtractionStrategy',
+        params: {
+            schema: {
+                type: 'dict',
+                value: {
+                    name: schema.name,
+                    baseSelector: schema.baseSelector,
+                    fields: schema.fields.map(field => ({
+                        name: field.name,
+                        selector: field.selector,
+                        type: field.type,
+                        attribute: field.attribute,
+                    })),
+                },
+            },
+        },
+    };
+}
+function createLlmExtractionStrategy(schema, instruction, provider, apiKey, baseUrl) {
+    const llmConfigParams = {
+        provider: provider || 'openai/gpt-4o',
+        api_token: apiKey,
+    };
+    if (baseUrl && baseUrl.trim() !== '') {
+        llmConfigParams.api_base = baseUrl;
+    }
+    return {
+        type: 'LLMExtractionStrategy',
+        params: {
+            llm_config: {
+                type: 'LLMConfig',
+                params: llmConfigParams,
+            },
+            instruction,
+            schema: {
+                type: 'dict',
+                value: schema,
+            },
+            extraction_type: 'schema',
+            apply_chunking: false,
+            force_json_response: true,
+        },
+    };
+}
+function cleanExtractedData(data) {
+    if (!data)
+        return {};
+    const cleanedData = {};
+    Object.entries(data).forEach(([key, value]) => {
+        if (typeof value === 'string') {
+            cleanedData[key] = (0, utils_2.cleanText)(value);
+        }
+        else if (Array.isArray(value)) {
+            cleanedData[key] = value.map(item => {
+                if (typeof item === 'string') {
+                    return (0, utils_2.cleanText)(item);
+                }
+                else if (typeof item === 'object' && item !== null) {
+                    return cleanExtractedData(item);
+                }
+                return item;
+            });
+        }
+        else if (typeof value === 'object' && value !== null) {
+            cleanedData[key] = cleanExtractedData(value);
+        }
+        else {
+            cleanedData[key] = value;
+        }
+    });
+    return cleanedData;
+}
+//# sourceMappingURL=utils.js.map

package/dist/nodes/Crawl4aiContentExtractor/helpers/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiContentExtractor/helpers/utils.ts"],"names":[],"mappings":";;;AAmBA,kFAmBC;AAWD,kEAkCC;AAKD,gDAyBC;AAhHD,kEAOkD;AANhD,0GAAA,iBAAiB,OAAA;AACjB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AACtB,sGAAA,aAAa,OAAA;AACb,kGAAA,SAAS,OAAA;AACT,mGAAA,UAAU,OAAA;AAIZ,oEAAqE;AAQrE,SAAgB,mCAAmC,CAAC,MAAyB;IAC3E,OAAO;QACL,IAAI,EAAE,2BAA2B;QACjC,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;iBACJ;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAWD,SAAgB,2BAA2B,CACzC,MAAiB,EACjB,WAAmB,EACnB,QAAgB,EAChB,MAAe,EACf,OAAgB;IAEhB,MAAM,eAAe,GAAQ;QAC3B,QAAQ,EAAE,QAAQ,IAAI,eAAe;QACrC,SAAS,EAAE,MAAM;KAClB,CAAC;IAGF,IAAI,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,eAAe,CAAC,QAAQ,GAAG,OAAO,CAAC;IACrC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,MAAM,EAAE;YACN,UAAU,EAAE;gBACV,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE,eAAe;aACxB;YACD,WAAW;YACX,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,MAAM;aACd;YACD,eAAe,EAAE,QAAQ;YACzB,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,IAAI;SAC1B;KACF,CAAC;AACJ,CAAC;AAKD,SAAgB,kBAAkB,CAAC,IAAiB;IAClD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,WAAW,GAAgB,EAAE,CAAC;IAEpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,iBAAS,EAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;qBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrD,OAAO,kBAAkB,CAAC,IAAmB,CAAC,CAAC;gBACjD,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,KAAoB,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}