n8n-nodes-duckduckgo-search 32.1.0 → 32.2.0

package/README.md

@@ -1,6 +1,6 @@
 # DuckDuckGo Search Node for n8n
 
-[![npm version](https://img.shields.io/npm/v/n8n-nodes-duckduckgo-search.svg?v=32.1.0)](https://www.npmjs.com/package/n8n-nodes-duckduckgo-search)
+[![npm version](https://img.shields.io/npm/v/n8n-nodes-duckduckgo-search.svg?v=32.2.0)](https://www.npmjs.com/package/n8n-nodes-duckduckgo-search)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
 A professional-grade n8n community node for DuckDuckGo search with enterprise-level error handling, validation, and quality assurance. Search the web, find images, discover news, and explore videos with privacy-focused, reliable results.

package/dist/nodes/DuckDuckGo/DuckDuckGo.node.js

@@ -9,6 +9,7 @@ const constants_1 = require("./constants");
 const errors_1 = require("./errors");
 const validation_1 = require("./validation");
 const reliability_1 = require("./reliability");
+const adaptiveRateLimiter_1 = require("./adaptiveRateLimiter");
 const LOCALE_OPTIONS = [
     { name: 'English (US)', value: 'en-us' },
     { name: 'English (UK)', value: 'uk-en' },
@@ -207,10 +208,11 @@ class DuckDuckGo {
     async execute() {
         const items = this.getInputData();
         const returnData = [];
-        const requestQueue = (0, reliability_1.getRequestQueue)(1000);
         for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
             try {
                 const operation = this.getNodeParameter('operation', itemIndex);
+                const adaptiveRateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
+                await adaptiveRateLimiter.wait();
                 const rawQuery = this.getNodeParameter('query', itemIndex);
                 const rawLocale = this.getNodeParameter('locale', itemIndex);
                 const rawMaxResults = this.getNodeParameter('maxResults', itemIndex);
@@ -240,13 +242,13 @@ class DuckDuckGo {
                 let results = [];
                 const performSearch = async () => {
                     var _a, _b, _c, _d;
-                    const timeout = 30000;
+                    const timeout = 45000;
                     if (operation === types_1.DuckDuckGoOperation.Search) {
-                        const searchResult = await requestQueue.enqueue(() => (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.search)(query, {
+                        const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.search)(query, {
                             safeSearch: getSafeSearchType(safeSearch),
                             locale: region || locale,
                             time: getSearchTimeType(timePeriod),
-                        }), timeout, 'Web Search'), `Web Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG));
+                        }), timeout, 'Web Search'), `Web Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
                         if ((_a = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _a === void 0 ? void 0 : _a.length) {
                             results = (0, processors_1.processWebSearchResults)(searchResult.results, itemIndex, searchResult).slice(0, maxResults);
                         }
@@ -264,10 +266,10 @@ class DuckDuckGo {
                         }
                     }
                     else if (operation === types_1.DuckDuckGoOperation.SearchImages) {
-                        const searchResult = await requestQueue.enqueue(() => (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchImages)(query, {
+                        const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchImages)(query, {
                             safeSearch: getSafeSearchType(safeSearch),
                             locale: region || locale,
-                        }), timeout, 'Image Search'), `Image Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG));
+                        }), timeout, 'Image Search'), `Image Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
                         if ((_b = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _b === void 0 ? void 0 : _b.length) {
                             results = (0, processors_1.processImageSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
                         }
@@ -285,11 +287,11 @@ class DuckDuckGo {
                         }
                     }
                     else if (operation === types_1.DuckDuckGoOperation.SearchNews) {
-                        const searchResult = await requestQueue.enqueue(() => (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchNews)(query, {
+                        const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchNews)(query, {
                             safeSearch: getSafeSearchType(safeSearch),
                             locale: region || locale,
                             time: getSearchTimeType(timePeriod),
-                        }), timeout, 'News Search'), `News Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG));
+                        }), timeout, 'News Search'), `News Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
                         if ((_c = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _c === void 0 ? void 0 : _c.length) {
                             results = (0, processors_1.processNewsSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
                         }
@@ -307,10 +309,10 @@ class DuckDuckGo {
                         }
                     }
                     else if (operation === types_1.DuckDuckGoOperation.SearchVideos) {
-                        const searchResult = await requestQueue.enqueue(() => (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchVideos)(query, {
+                        const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchVideos)(query, {
                             safeSearch: getSafeSearchType(safeSearch),
                             locale: region || locale,
-                        }), timeout, 'Video Search'), `Video Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG));
+                        }), timeout, 'Video Search'), `Video Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
                         if ((_d = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _d === void 0 ? void 0 : _d.length) {
                             results = (0, processors_1.processVideoSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
                         }
@@ -333,12 +335,22 @@ class DuckDuckGo {
                     return results;
                 };
                 const searchResults = await performSearch();
+                const rateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
+                rateLimiter.reportSuccess();
                 returnData.push(...searchResults);
             }
             catch (error) {
+                const operation = this.getNodeParameter('operation', itemIndex);
+                const rateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
                 const classifiedError = error instanceof errors_1.DuckDuckGoError
                     ? error
                     : (0, errors_1.classifyError)(error);
+                if (classifiedError.category === 'rate_limit') {
+                    rateLimiter.reportRateLimitFailure();
+                }
+                else {
+                    rateLimiter.reportOtherFailure();
+                }
                 if (this.continueOnFail()) {
                     returnData.push({
                         json: {

package/dist/nodes/DuckDuckGo/adaptiveRateLimiter.js

@@ -0,0 +1,108 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resetOperationRateLimiter = exports.resetAdaptiveRateLimiter = exports.getAdaptiveRateLimiter = exports.AdaptiveRateLimiter = exports.OPERATION_RATE_LIMITS = void 0;
+exports.OPERATION_RATE_LIMITS = {
+    search: {
+        initialDelay: 3000,
+        minDelay: 2500,
+        maxDelay: 10000,
+    },
+    searchImages: {
+        initialDelay: 3000,
+        minDelay: 2500,
+        maxDelay: 10000,
+    },
+    searchNews: {
+        initialDelay: 2000,
+        minDelay: 1500,
+        maxDelay: 8000,
+    },
+    searchVideos: {
+        initialDelay: 2000,
+        minDelay: 1500,
+        maxDelay: 8000,
+    },
+};
+class AdaptiveRateLimiter {
+    constructor(operationType = 'search', initialDelay) {
+        this.increaseMultiplier = 1.5;
+        this.decreaseMultiplier = 0.9;
+        this.consecutiveSuccesses = 0;
+        this.lastRequestTime = 0;
+        this.operationType = operationType;
+        const config = exports.OPERATION_RATE_LIMITS[operationType] || exports.OPERATION_RATE_LIMITS.search;
+        this.minDelay = config.minDelay;
+        this.maxDelay = config.maxDelay;
+        this.currentDelay = initialDelay || config.initialDelay;
+        this.currentDelay = Math.max(this.minDelay, Math.min(this.currentDelay, this.maxDelay));
+    }
+    async wait() {
+        const now = Date.now();
+        const timeSinceLastRequest = now - this.lastRequestTime;
+        if (timeSinceLastRequest < this.currentDelay) {
+            const waitTime = this.currentDelay - timeSinceLastRequest;
+            const jitter = Math.random() * 500;
+            await this.sleep(waitTime + jitter);
+        }
+        else {
+            const jitter = Math.random() * 300;
+            await this.sleep(jitter);
+        }
+        this.lastRequestTime = Date.now();
+    }
+    reportSuccess() {
+        this.consecutiveSuccesses++;
+        if (this.consecutiveSuccesses >= 3) {
+            const newDelay = this.currentDelay * this.decreaseMultiplier;
+            this.currentDelay = Math.max(this.minDelay, newDelay);
+            this.consecutiveSuccesses = 0;
+            if (process.env.NODE_ENV === 'development') {
+                console.log(`[AdaptiveRateLimiter] Decreased delay to ${Math.round(this.currentDelay)}ms`);
+            }
+        }
+    }
+    reportRateLimitFailure() {
+        this.consecutiveSuccesses = 0;
+        const newDelay = this.currentDelay * this.increaseMultiplier;
+        this.currentDelay = Math.min(this.maxDelay, newDelay);
+        if (process.env.NODE_ENV === 'development') {
+            console.log(`[AdaptiveRateLimiter] Increased delay to ${Math.round(this.currentDelay)}ms after rate limit`);
+        }
+    }
+    reportOtherFailure() {
+        this.consecutiveSuccesses = 0;
+    }
+    getCurrentDelay() {
+        return this.currentDelay;
+    }
+    reset(initialDelay) {
+        const config = exports.OPERATION_RATE_LIMITS[this.operationType] || exports.OPERATION_RATE_LIMITS.search;
+        this.currentDelay = initialDelay || config.initialDelay;
+        this.currentDelay = Math.max(this.minDelay, Math.min(this.currentDelay, this.maxDelay));
+        this.consecutiveSuccesses = 0;
+        this.lastRequestTime = 0;
+    }
+    getOperationType() {
+        return this.operationType;
+    }
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+}
+exports.AdaptiveRateLimiter = AdaptiveRateLimiter;
+const globalAdaptiveRateLimiters = new Map();
+function getAdaptiveRateLimiter(operationType = 'search', initialDelay) {
+    if (!globalAdaptiveRateLimiters.has(operationType)) {
+        globalAdaptiveRateLimiters.set(operationType, new AdaptiveRateLimiter(operationType, initialDelay));
+    }
+    return globalAdaptiveRateLimiters.get(operationType);
+}
+exports.getAdaptiveRateLimiter = getAdaptiveRateLimiter;
+function resetAdaptiveRateLimiter() {
+    globalAdaptiveRateLimiters.clear();
+}
+exports.resetAdaptiveRateLimiter = resetAdaptiveRateLimiter;
+function resetOperationRateLimiter(operationType) {
+    globalAdaptiveRateLimiters.delete(operationType);
+}
+exports.resetOperationRateLimiter = resetOperationRateLimiter;

package/dist/nodes/DuckDuckGo/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BROWSER_USER_AGENT = exports.NODE_INFO = exports.DEFAULT_PARAMETERS = exports.REGIONS = void 0;
+exports.getRandomUserAgent = exports.USER_AGENTS = exports.NODE_INFO = exports.DEFAULT_PARAMETERS = exports.REGIONS = void 0;
 exports.REGIONS = [
     { name: 'Argentina', value: 'ar-es' },
     { name: 'Australia', value: 'au-en' },
@@ -63,4 +63,23 @@ exports.NODE_INFO = {
     VERSION: 1,
     DESCRIPTION: 'Search using DuckDuckGo',
 };
-exports.BROWSER_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
+exports.USER_AGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
+];
+function getRandomUserAgent() {
+    return exports.USER_AGENTS[Math.floor(Math.random() * exports.USER_AGENTS.length)];
+}
+exports.getRandomUserAgent = getRandomUserAgent;

package/dist/nodes/DuckDuckGo/errors.js

@@ -156,7 +156,9 @@ function classifyError(error) {
         lowerMessage.includes('502') ||
         lowerMessage.includes('503') ||
         lowerMessage.includes('504') ||
-        lowerMessage.includes('server error')) {
+        lowerMessage.includes('server error') ||
+        lowerMessage.includes('temporarily unavailable') ||
+        lowerMessage.includes('internal server error')) {
         const statusMatch = message.match(/(\d{3})/);
         const statusCode = statusMatch ? parseInt(statusMatch[1], 10) : 500;
         return new ServerError(statusCode, error instanceof Error ? error : undefined);

package/dist/nodes/DuckDuckGo/reliability.js

@@ -7,10 +7,10 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 exports.sleep = sleep;
 exports.DEFAULT_RETRY_CONFIG = {
     maxRetries: 3,
-    initialDelay: 1000,
-    maxDelay: 8000,
+    initialDelay: 2000,
+    maxDelay: 15000,
     backoffMultiplier: 2,
-    jitterFactor: 0.3,
+    jitterFactor: 0.5,
     retryableCategories: new Set(['network', 'rate_limit', 'timeout']),
 };
 async function executeWithRetry(fn, context = 'operation', config = exports.DEFAULT_RETRY_CONFIG) {
@@ -62,7 +62,7 @@ async function withTimeout(promise, timeoutMs, operation) {
 }
 exports.withTimeout = withTimeout;
 class RequestQueue {
-    constructor(minInterval = 1000, maxConcurrent = 1) {
+    constructor(minInterval = 3000, maxConcurrent = 1) {
         this.queue = [];
         this.processing = false;
         this.lastRequestTime = 0;
@@ -97,7 +97,12 @@ class RequestQueue {
             const timeSinceLastRequest = now - this.lastRequestTime;
             if (timeSinceLastRequest < this.minInterval) {
                 const delay = this.minInterval - timeSinceLastRequest;
-                await (0, exports.sleep)(delay);
+                const jitter = Math.random() * 1000;
+                await (0, exports.sleep)(delay + jitter);
+            }
+            else {
+                const smallJitter = Math.random() * 500;
+                await (0, exports.sleep)(smallJitter);
             }
             const fn = this.queue.shift();
             if (fn) {
@@ -126,7 +131,7 @@ class RequestQueue {
 }
 exports.RequestQueue = RequestQueue;
 let globalRequestQueue = null;
-function getRequestQueue(minInterval = 1000) {
+function getRequestQueue(minInterval = 3000) {
     if (!globalRequestQueue) {
         globalRequestQueue = new RequestQueue(minInterval);
     }

package/dist/package.json

@@ -1,6 +1,6 @@
 {
     "name": "n8n-nodes-duckduckgo-search",
-    "version": "32.1.0",
+    "version": "32.2.0",
     "description": "Professional-grade n8n community node for DuckDuckGo search with comprehensive error handling, validation, and quality filtering.",
     "keywords": [
         "n8n-community-node-package",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "n8n-nodes-duckduckgo-search",
-  "version": "32.1.0",
+  "version": "32.2.0",
   "description": "Professional-grade n8n community node for DuckDuckGo search with comprehensive error handling, validation, and quality filtering.",
   "keywords": [
     "n8n-community-node-package",