n8n-nodes-crawl4ai-onuro 0.2.0

package/dist/nodes/Crawl4aiContentExtractor/helpers/utils.js

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isValidUrl = exports.cleanText = exports.safeJsonParse = exports.createCrawlerRunConfig = exports.createBrowserConfig = exports.getCrawl4aiClient = void 0;
+exports.createCssSelectorExtractionStrategy = createCssSelectorExtractionStrategy;
+exports.createLlmExtractionStrategy = createLlmExtractionStrategy;
+exports.cleanExtractedData = cleanExtractedData;
+var utils_1 = require("../../Crawl4aiBasicCrawler/helpers/utils");
+Object.defineProperty(exports, "getCrawl4aiClient", { enumerable: true, get: function () { return utils_1.getCrawl4aiClient; } });
+Object.defineProperty(exports, "createBrowserConfig", { enumerable: true, get: function () { return utils_1.createBrowserConfig; } });
+Object.defineProperty(exports, "createCrawlerRunConfig", { enumerable: true, get: function () { return utils_1.createCrawlerRunConfig; } });
+Object.defineProperty(exports, "safeJsonParse", { enumerable: true, get: function () { return utils_1.safeJsonParse; } });
+Object.defineProperty(exports, "cleanText", { enumerable: true, get: function () { return utils_1.cleanText; } });
+Object.defineProperty(exports, "isValidUrl", { enumerable: true, get: function () { return utils_1.isValidUrl; } });
+const utils_2 = require("../../Crawl4aiBasicCrawler/helpers/utils");
+function createCssSelectorExtractionStrategy(schema) {
+    return {
+        type: 'JsonCssExtractionStrategy',
+        params: {
+            schema: {
+                type: 'dict',
+                value: {
+                    name: schema.name,
+                    baseSelector: schema.baseSelector,
+                    fields: schema.fields.map(field => ({
+                        name: field.name,
+                        selector: field.selector,
+                        type: field.type,
+                        attribute: field.attribute,
+                    })),
+                },
+            },
+        },
+    };
+}
+function createLlmExtractionStrategy(schema, instruction, provider, apiKey, extraArgs) {
+    const providerMap = {
+        'openai': 'openai/gpt-4o',
+        'anthropic': 'anthropic/claude-3-sonnet',
+        'groq': 'groq/llama3-70b-8192',
+        'ollama': 'ollama/llama2',
+    };
+    const fullProvider = providerMap[provider] || provider || 'openai/gpt-4o';
+    return {
+        type: 'LLMExtractionStrategy',
+        params: {
+            llm_config: {
+                type: 'LLMConfig',
+                params: {
+                    provider: fullProvider,
+                    api_token: apiKey,
+                },
+            },
+            instruction,
+            schema: {
+                type: 'dict',
+                value: schema,
+            },
+            extraction_type: 'schema',
+            apply_chunking: false,
+            force_json_response: true,
+            extra_args: extraArgs || { temperature: 0, max_tokens: 4000 },
+        },
+    };
+}
+function cleanExtractedData(data) {
+    if (!data)
+        return {};
+    const cleanedData = {};
+    Object.entries(data).forEach(([key, value]) => {
+        if (typeof value === 'string') {
+            cleanedData[key] = (0, utils_2.cleanText)(value);
+        }
+        else if (Array.isArray(value)) {
+            cleanedData[key] = value.map(item => {
+                if (typeof item === 'string') {
+                    return (0, utils_2.cleanText)(item);
+                }
+                else if (typeof item === 'object' && item !== null) {
+                    return cleanExtractedData(item);
+                }
+                return item;
+            });
+        }
+        else if (typeof value === 'object' && value !== null) {
+            cleanedData[key] = cleanExtractedData(value);
+        }
+        else {
+            cleanedData[key] = value;
+        }
+    });
+    return cleanedData;
+}
+//# sourceMappingURL=utils.js.map

package/dist/nodes/Crawl4aiContentExtractor/helpers/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../../nodes/Crawl4aiContentExtractor/helpers/utils.ts"],"names":[],"mappings":";;;AAqBA,kFAmBC;AAWD,kEAqCC;AAKD,gDAyBC;AArHD,kEAOkD;AANhD,0GAAA,iBAAiB,OAAA;AACjB,4GAAA,mBAAmB,OAAA;AACnB,+GAAA,sBAAsB,OAAA;AACtB,sGAAA,aAAa,OAAA;AACb,kGAAA,SAAS,OAAA;AACT,mGAAA,UAAU,OAAA;AAIZ,oEAAqE;AAUrE,SAAgB,mCAAmC,CAAC,MAAyB;IAC3E,OAAO;QACL,IAAI,EAAE,2BAA2B;QACjC,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE;oBACL,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,YAAY,EAAE,MAAM,CAAC,YAAY;oBACjC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAClC,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,QAAQ,EAAE,KAAK,CAAC,QAAQ;wBACxB,IAAI,EAAE,KAAK,CAAC,IAAI;wBAChB,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC,CAAC;iBACJ;aACF;SACF;KACF,CAAC;AACJ,CAAC;AAWD,SAAgB,2BAA2B,CACzC,MAAiB,EACjB,WAAmB,EACnB,QAAgB,EAChB,MAAe,EACf,SAAyD;IAGzD,MAAM,WAAW,GAA8B;QAC7C,QAAQ,EAAE,eAAe;QACzB,WAAW,EAAE,2BAA2B;QACxC,MAAM,EAAE,sBAAsB;QAC9B,QAAQ,EAAE,eAAe;KAC1B,CAAC;IACF,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,eAAe,CAAC;IAE1E,OAAO;QACL,IAAI,EAAE,uBAAuB;QAC7B,MAAM,EAAE;YACN,UAAU,EAAE;gBACV,IAAI,EAAE,WAAW;gBACjB,MAAM,EAAE;oBACN,QAAQ,EAAE,YAAY;oBACtB,SAAS,EAAE,MAAM;iBAClB;aACF;YACD,WAAW;YACX,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,MAAM;aACd;YACD,eAAe,EAAE,QAAQ;YACzB,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,IAAI;YACzB,UAAU,EAAE,SAAS,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE;SAC9D;KACF,CAAC;AACJ,CAAC;AAKD,SAAgB,kBAAkB,CAAC,IAAiB;IAClD,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IAErB,MAAM,WAAW,GAAgB,EAAE,CAAC;IAEpC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,CAAC,GAAG,CAAC,GAAG,IAAA,iBAAS,EAAC,KAAK,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;gBAClC,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;oBAC7B,OAAO,IAAA,iBAAS,EAAC,IAAI,CAAC,CAAC;gBACzB,CAAC;qBAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oBACrD,OAAO,kBAAkB,CAAC,IAAmB,CAAC,CAAC;gBACjD,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,GAAG,kBAAkB,CAAC,KAAoB,CAAC,CAAC;QAC9D,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC;AACrB,CAAC"}