n8n-nodes-duckduckgo-search 32.1.0 → 32.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/nodes/DuckDuckGo/DuckDuckGo.node.js +23 -10
- package/dist/nodes/DuckDuckGo/__tests__/DuckDuckGo.test.js +36 -5
- package/dist/nodes/DuckDuckGo/adaptiveRateLimiter.js +108 -0
- package/dist/nodes/DuckDuckGo/constants.js +48 -2
- package/dist/nodes/DuckDuckGo/errors.js +3 -1
- package/dist/nodes/DuckDuckGo/reliability.js +11 -6
- package/dist/package.json +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# DuckDuckGo Search Node for n8n
|
|
2
2
|
|
|
3
|
-
[](https://www.npmjs.com/package/n8n-nodes-duckduckgo-search)
|
|
4
4
|
[](https://opensource.org/licenses/MIT)
|
|
5
5
|
|
|
6
6
|
A professional-grade n8n community node for DuckDuckGo search with enterprise-level error handling, validation, and quality assurance. Search the web, find images, discover news, and explore videos with privacy-focused, reliable results.
|
|
@@ -9,6 +9,7 @@ const constants_1 = require("./constants");
|
|
|
9
9
|
const errors_1 = require("./errors");
|
|
10
10
|
const validation_1 = require("./validation");
|
|
11
11
|
const reliability_1 = require("./reliability");
|
|
12
|
+
const adaptiveRateLimiter_1 = require("./adaptiveRateLimiter");
|
|
12
13
|
const LOCALE_OPTIONS = [
|
|
13
14
|
{ name: 'English (US)', value: 'en-us' },
|
|
14
15
|
{ name: 'English (UK)', value: 'uk-en' },
|
|
@@ -207,10 +208,11 @@ class DuckDuckGo {
|
|
|
207
208
|
async execute() {
|
|
208
209
|
const items = this.getInputData();
|
|
209
210
|
const returnData = [];
|
|
210
|
-
const requestQueue = (0, reliability_1.getRequestQueue)(1000);
|
|
211
211
|
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
212
212
|
try {
|
|
213
213
|
const operation = this.getNodeParameter('operation', itemIndex);
|
|
214
|
+
const adaptiveRateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
|
|
215
|
+
await adaptiveRateLimiter.wait();
|
|
214
216
|
const rawQuery = this.getNodeParameter('query', itemIndex);
|
|
215
217
|
const rawLocale = this.getNodeParameter('locale', itemIndex);
|
|
216
218
|
const rawMaxResults = this.getNodeParameter('maxResults', itemIndex);
|
|
@@ -240,13 +242,14 @@ class DuckDuckGo {
|
|
|
240
242
|
let results = [];
|
|
241
243
|
const performSearch = async () => {
|
|
242
244
|
var _a, _b, _c, _d;
|
|
243
|
-
const timeout =
|
|
245
|
+
const timeout = 45000;
|
|
246
|
+
const needleOptions = (0, constants_1.getNeedleOptions)();
|
|
244
247
|
if (operation === types_1.DuckDuckGoOperation.Search) {
|
|
245
|
-
const searchResult = await
|
|
248
|
+
const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.search)(query, {
|
|
246
249
|
safeSearch: getSafeSearchType(safeSearch),
|
|
247
250
|
locale: region || locale,
|
|
248
251
|
time: getSearchTimeType(timePeriod),
|
|
249
|
-
}), timeout, 'Web Search'), `Web Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG)
|
|
252
|
+
}, needleOptions), timeout, 'Web Search'), `Web Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
|
|
250
253
|
if ((_a = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _a === void 0 ? void 0 : _a.length) {
|
|
251
254
|
results = (0, processors_1.processWebSearchResults)(searchResult.results, itemIndex, searchResult).slice(0, maxResults);
|
|
252
255
|
}
|
|
@@ -264,10 +267,10 @@ class DuckDuckGo {
|
|
|
264
267
|
}
|
|
265
268
|
}
|
|
266
269
|
else if (operation === types_1.DuckDuckGoOperation.SearchImages) {
|
|
267
|
-
const searchResult = await
|
|
270
|
+
const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchImages)(query, {
|
|
268
271
|
safeSearch: getSafeSearchType(safeSearch),
|
|
269
272
|
locale: region || locale,
|
|
270
|
-
}), timeout, 'Image Search'), `Image Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG)
|
|
273
|
+
}, needleOptions), timeout, 'Image Search'), `Image Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
|
|
271
274
|
if ((_b = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _b === void 0 ? void 0 : _b.length) {
|
|
272
275
|
results = (0, processors_1.processImageSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
|
|
273
276
|
}
|
|
@@ -285,11 +288,11 @@ class DuckDuckGo {
|
|
|
285
288
|
}
|
|
286
289
|
}
|
|
287
290
|
else if (operation === types_1.DuckDuckGoOperation.SearchNews) {
|
|
288
|
-
const searchResult = await
|
|
291
|
+
const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchNews)(query, {
|
|
289
292
|
safeSearch: getSafeSearchType(safeSearch),
|
|
290
293
|
locale: region || locale,
|
|
291
294
|
time: getSearchTimeType(timePeriod),
|
|
292
|
-
}), timeout, 'News Search'), `News Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG)
|
|
295
|
+
}, needleOptions), timeout, 'News Search'), `News Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
|
|
293
296
|
if ((_c = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _c === void 0 ? void 0 : _c.length) {
|
|
294
297
|
results = (0, processors_1.processNewsSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
|
|
295
298
|
}
|
|
@@ -307,10 +310,10 @@ class DuckDuckGo {
|
|
|
307
310
|
}
|
|
308
311
|
}
|
|
309
312
|
else if (operation === types_1.DuckDuckGoOperation.SearchVideos) {
|
|
310
|
-
const searchResult = await
|
|
313
|
+
const searchResult = await (0, reliability_1.executeWithRetry)(() => (0, reliability_1.withTimeout)((0, duck_duck_scrape_1.searchVideos)(query, {
|
|
311
314
|
safeSearch: getSafeSearchType(safeSearch),
|
|
312
315
|
locale: region || locale,
|
|
313
|
-
}), timeout, 'Video Search'), `Video Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG)
|
|
316
|
+
}, needleOptions), timeout, 'Video Search'), `Video Search: "${query}"`, reliability_1.DEFAULT_RETRY_CONFIG);
|
|
314
317
|
if ((_d = searchResult === null || searchResult === void 0 ? void 0 : searchResult.results) === null || _d === void 0 ? void 0 : _d.length) {
|
|
315
318
|
results = (0, processors_1.processVideoSearchResults)(searchResult.results, itemIndex).slice(0, maxResults);
|
|
316
319
|
}
|
|
@@ -333,12 +336,22 @@ class DuckDuckGo {
|
|
|
333
336
|
return results;
|
|
334
337
|
};
|
|
335
338
|
const searchResults = await performSearch();
|
|
339
|
+
const rateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
|
|
340
|
+
rateLimiter.reportSuccess();
|
|
336
341
|
returnData.push(...searchResults);
|
|
337
342
|
}
|
|
338
343
|
catch (error) {
|
|
344
|
+
const operation = this.getNodeParameter('operation', itemIndex);
|
|
345
|
+
const rateLimiter = (0, adaptiveRateLimiter_1.getAdaptiveRateLimiter)(operation);
|
|
339
346
|
const classifiedError = error instanceof errors_1.DuckDuckGoError
|
|
340
347
|
? error
|
|
341
348
|
: (0, errors_1.classifyError)(error);
|
|
349
|
+
if (classifiedError.category === 'rate_limit') {
|
|
350
|
+
rateLimiter.reportRateLimitFailure();
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
rateLimiter.reportOtherFailure();
|
|
354
|
+
}
|
|
342
355
|
if (this.continueOnFail()) {
|
|
343
356
|
returnData.push({
|
|
344
357
|
json: {
|
|
@@ -25,6 +25,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
const DuckDuckGo_node_1 = require("../DuckDuckGo.node");
|
|
27
27
|
const duckDuckScrape = __importStar(require("duck-duck-scrape"));
|
|
28
|
+
jest.mock('../adaptiveRateLimiter', () => ({
|
|
29
|
+
getAdaptiveRateLimiter: jest.fn().mockReturnValue({
|
|
30
|
+
wait: jest.fn().mockResolvedValue(undefined),
|
|
31
|
+
reportSuccess: jest.fn(),
|
|
32
|
+
reportRateLimitFailure: jest.fn(),
|
|
33
|
+
reportOtherFailure: jest.fn(),
|
|
34
|
+
}),
|
|
35
|
+
resetAdaptiveRateLimiter: jest.fn(),
|
|
36
|
+
}));
|
|
28
37
|
jest.mock('duck-duck-scrape', () => ({
|
|
29
38
|
search: jest.fn(),
|
|
30
39
|
searchNews: jest.fn(),
|
|
@@ -95,7 +104,9 @@ describe('DuckDuckGo Node', () => {
|
|
|
95
104
|
setupNodeParameters('search', 'test query');
|
|
96
105
|
duckDuckScrape.search.mockResolvedValue(mockResults);
|
|
97
106
|
const result = await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
98
|
-
expect(duckDuckScrape.search).toHaveBeenCalledWith('test query', expect.any(Object)
|
|
107
|
+
expect(duckDuckScrape.search).toHaveBeenCalledWith('test query', expect.any(Object), expect.objectContaining({
|
|
108
|
+
headers: expect.any(Object),
|
|
109
|
+
}));
|
|
99
110
|
expect(result).toHaveLength(1);
|
|
100
111
|
expect(result[0]).toHaveLength(2);
|
|
101
112
|
expect(result[0][0].json).toHaveProperty('title', 'Result 1');
|
|
@@ -131,7 +142,9 @@ describe('DuckDuckGo Node', () => {
|
|
|
131
142
|
setupNodeParameters('searchImages', 'cat');
|
|
132
143
|
duckDuckScrape.searchImages.mockResolvedValue(mockResults);
|
|
133
144
|
const result = await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
134
|
-
expect(duckDuckScrape.searchImages).toHaveBeenCalledWith('cat', expect.any(Object)
|
|
145
|
+
expect(duckDuckScrape.searchImages).toHaveBeenCalledWith('cat', expect.any(Object), expect.objectContaining({
|
|
146
|
+
headers: expect.any(Object),
|
|
147
|
+
}));
|
|
135
148
|
expect(result[0][0].json).toHaveProperty('sourceType', 'image');
|
|
136
149
|
expect(result[0][0].json).toHaveProperty('imageUrl', 'https://example.com/img1.jpg');
|
|
137
150
|
});
|
|
@@ -146,7 +159,9 @@ describe('DuckDuckGo Node', () => {
|
|
|
146
159
|
setupNodeParameters('searchNews', 'tech news');
|
|
147
160
|
duckDuckScrape.searchNews.mockResolvedValue(mockResults);
|
|
148
161
|
const result = await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
149
|
-
expect(duckDuckScrape.searchNews).toHaveBeenCalledWith('tech news', expect.any(Object)
|
|
162
|
+
expect(duckDuckScrape.searchNews).toHaveBeenCalledWith('tech news', expect.any(Object), expect.objectContaining({
|
|
163
|
+
headers: expect.any(Object),
|
|
164
|
+
}));
|
|
150
165
|
expect(result[0][0].json).toHaveProperty('sourceType', 'news');
|
|
151
166
|
});
|
|
152
167
|
});
|
|
@@ -160,7 +175,9 @@ describe('DuckDuckGo Node', () => {
|
|
|
160
175
|
setupNodeParameters('searchVideos', 'tutorial');
|
|
161
176
|
duckDuckScrape.searchVideos.mockResolvedValue(mockResults);
|
|
162
177
|
const result = await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
163
|
-
expect(duckDuckScrape.searchVideos).toHaveBeenCalledWith('tutorial', expect.any(Object)
|
|
178
|
+
expect(duckDuckScrape.searchVideos).toHaveBeenCalledWith('tutorial', expect.any(Object), expect.objectContaining({
|
|
179
|
+
headers: expect.any(Object),
|
|
180
|
+
}));
|
|
164
181
|
expect(result[0][0].json).toHaveProperty('sourceType', 'video');
|
|
165
182
|
});
|
|
166
183
|
});
|
|
@@ -171,7 +188,7 @@ describe('DuckDuckGo Node', () => {
|
|
|
171
188
|
await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
172
189
|
expect(duckDuckScrape.search).toHaveBeenCalledWith('test', expect.objectContaining({
|
|
173
190
|
locale: 'fr-fr',
|
|
174
|
-
}));
|
|
191
|
+
}), expect.any(Object));
|
|
175
192
|
});
|
|
176
193
|
it('should limit results to maxResults', async () => {
|
|
177
194
|
setupNodeParameters('search', 'test', { maxResults: 2 });
|
|
@@ -185,4 +202,18 @@ describe('DuckDuckGo Node', () => {
|
|
|
185
202
|
expect(result[0]).toHaveLength(2);
|
|
186
203
|
});
|
|
187
204
|
});
|
|
205
|
+
describe('Needle Options', () => {
|
|
206
|
+
it('should pass custom headers with User-Agent to search functions', async () => {
|
|
207
|
+
setupNodeParameters('search', 'test');
|
|
208
|
+
duckDuckScrape.search.mockResolvedValue({ results: [] });
|
|
209
|
+
await duckDuckGoNode.execute.call(mockExecuteFunction);
|
|
210
|
+
const callArgs = duckDuckScrape.search.mock.calls[0];
|
|
211
|
+
const needleOptions = callArgs[2];
|
|
212
|
+
expect(needleOptions).toHaveProperty('headers');
|
|
213
|
+
expect(needleOptions.headers).toHaveProperty('user-agent');
|
|
214
|
+
expect(needleOptions.headers).toHaveProperty('sec-ch-ua');
|
|
215
|
+
expect(needleOptions.headers).toHaveProperty('accept');
|
|
216
|
+
expect(needleOptions).toHaveProperty('response_timeout', 30000);
|
|
217
|
+
});
|
|
218
|
+
});
|
|
188
219
|
});
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resetOperationRateLimiter = exports.resetAdaptiveRateLimiter = exports.getAdaptiveRateLimiter = exports.AdaptiveRateLimiter = exports.OPERATION_RATE_LIMITS = void 0;
|
|
4
|
+
exports.OPERATION_RATE_LIMITS = {
|
|
5
|
+
search: {
|
|
6
|
+
initialDelay: 5000,
|
|
7
|
+
minDelay: 4000,
|
|
8
|
+
maxDelay: 15000,
|
|
9
|
+
},
|
|
10
|
+
searchImages: {
|
|
11
|
+
initialDelay: 5000,
|
|
12
|
+
minDelay: 4000,
|
|
13
|
+
maxDelay: 15000,
|
|
14
|
+
},
|
|
15
|
+
searchNews: {
|
|
16
|
+
initialDelay: 2000,
|
|
17
|
+
minDelay: 1500,
|
|
18
|
+
maxDelay: 8000,
|
|
19
|
+
},
|
|
20
|
+
searchVideos: {
|
|
21
|
+
initialDelay: 2000,
|
|
22
|
+
minDelay: 1500,
|
|
23
|
+
maxDelay: 8000,
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
class AdaptiveRateLimiter {
|
|
27
|
+
constructor(operationType = 'search', initialDelay) {
|
|
28
|
+
this.increaseMultiplier = 1.5;
|
|
29
|
+
this.decreaseMultiplier = 0.9;
|
|
30
|
+
this.consecutiveSuccesses = 0;
|
|
31
|
+
this.lastRequestTime = 0;
|
|
32
|
+
this.operationType = operationType;
|
|
33
|
+
const config = exports.OPERATION_RATE_LIMITS[operationType] || exports.OPERATION_RATE_LIMITS.search;
|
|
34
|
+
this.minDelay = config.minDelay;
|
|
35
|
+
this.maxDelay = config.maxDelay;
|
|
36
|
+
this.currentDelay = initialDelay || config.initialDelay;
|
|
37
|
+
this.currentDelay = Math.max(this.minDelay, Math.min(this.currentDelay, this.maxDelay));
|
|
38
|
+
}
|
|
39
|
+
async wait() {
|
|
40
|
+
const now = Date.now();
|
|
41
|
+
const timeSinceLastRequest = now - this.lastRequestTime;
|
|
42
|
+
if (timeSinceLastRequest < this.currentDelay) {
|
|
43
|
+
const waitTime = this.currentDelay - timeSinceLastRequest;
|
|
44
|
+
const jitter = Math.random() * 500;
|
|
45
|
+
await this.sleep(waitTime + jitter);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const jitter = Math.random() * 300;
|
|
49
|
+
await this.sleep(jitter);
|
|
50
|
+
}
|
|
51
|
+
this.lastRequestTime = Date.now();
|
|
52
|
+
}
|
|
53
|
+
reportSuccess() {
|
|
54
|
+
this.consecutiveSuccesses++;
|
|
55
|
+
if (this.consecutiveSuccesses >= 3) {
|
|
56
|
+
const newDelay = this.currentDelay * this.decreaseMultiplier;
|
|
57
|
+
this.currentDelay = Math.max(this.minDelay, newDelay);
|
|
58
|
+
this.consecutiveSuccesses = 0;
|
|
59
|
+
if (process.env.NODE_ENV === 'development') {
|
|
60
|
+
console.log(`[AdaptiveRateLimiter] Decreased delay to ${Math.round(this.currentDelay)}ms`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
reportRateLimitFailure() {
|
|
65
|
+
this.consecutiveSuccesses = 0;
|
|
66
|
+
const newDelay = this.currentDelay * this.increaseMultiplier;
|
|
67
|
+
this.currentDelay = Math.min(this.maxDelay, newDelay);
|
|
68
|
+
if (process.env.NODE_ENV === 'development') {
|
|
69
|
+
console.log(`[AdaptiveRateLimiter] Increased delay to ${Math.round(this.currentDelay)}ms after rate limit`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
reportOtherFailure() {
|
|
73
|
+
this.consecutiveSuccesses = 0;
|
|
74
|
+
}
|
|
75
|
+
getCurrentDelay() {
|
|
76
|
+
return this.currentDelay;
|
|
77
|
+
}
|
|
78
|
+
reset(initialDelay) {
|
|
79
|
+
const config = exports.OPERATION_RATE_LIMITS[this.operationType] || exports.OPERATION_RATE_LIMITS.search;
|
|
80
|
+
this.currentDelay = initialDelay || config.initialDelay;
|
|
81
|
+
this.currentDelay = Math.max(this.minDelay, Math.min(this.currentDelay, this.maxDelay));
|
|
82
|
+
this.consecutiveSuccesses = 0;
|
|
83
|
+
this.lastRequestTime = 0;
|
|
84
|
+
}
|
|
85
|
+
getOperationType() {
|
|
86
|
+
return this.operationType;
|
|
87
|
+
}
|
|
88
|
+
sleep(ms) {
|
|
89
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
exports.AdaptiveRateLimiter = AdaptiveRateLimiter;
|
|
93
|
+
const globalAdaptiveRateLimiters = new Map();
|
|
94
|
+
function getAdaptiveRateLimiter(operationType = 'search', initialDelay) {
|
|
95
|
+
if (!globalAdaptiveRateLimiters.has(operationType)) {
|
|
96
|
+
globalAdaptiveRateLimiters.set(operationType, new AdaptiveRateLimiter(operationType, initialDelay));
|
|
97
|
+
}
|
|
98
|
+
return globalAdaptiveRateLimiters.get(operationType);
|
|
99
|
+
}
|
|
100
|
+
exports.getAdaptiveRateLimiter = getAdaptiveRateLimiter;
|
|
101
|
+
function resetAdaptiveRateLimiter() {
|
|
102
|
+
globalAdaptiveRateLimiters.clear();
|
|
103
|
+
}
|
|
104
|
+
exports.resetAdaptiveRateLimiter = resetAdaptiveRateLimiter;
|
|
105
|
+
function resetOperationRateLimiter(operationType) {
|
|
106
|
+
globalAdaptiveRateLimiters.delete(operationType);
|
|
107
|
+
}
|
|
108
|
+
exports.resetOperationRateLimiter = resetOperationRateLimiter;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.getNeedleOptions = exports.getRandomUserAgent = exports.USER_AGENTS = exports.NODE_INFO = exports.DEFAULT_PARAMETERS = exports.REGIONS = void 0;
|
|
4
4
|
exports.REGIONS = [
|
|
5
5
|
{ name: 'Argentina', value: 'ar-es' },
|
|
6
6
|
{ name: 'Australia', value: 'au-en' },
|
|
@@ -63,4 +63,50 @@ exports.NODE_INFO = {
|
|
|
63
63
|
VERSION: 1,
|
|
64
64
|
DESCRIPTION: 'Search using DuckDuckGo',
|
|
65
65
|
};
|
|
66
|
-
exports.
|
|
66
|
+
exports.USER_AGENTS = [
|
|
67
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
68
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
69
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
70
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
71
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
|
|
72
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
73
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0',
|
|
74
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
75
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:122.0) Gecko/20100101 Firefox/122.0',
|
|
76
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|
77
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15',
|
|
78
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
|
79
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0',
|
|
80
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
|
|
81
|
+
];
|
|
82
|
+
function getRandomUserAgent() {
|
|
83
|
+
return exports.USER_AGENTS[Math.floor(Math.random() * exports.USER_AGENTS.length)];
|
|
84
|
+
}
|
|
85
|
+
exports.getRandomUserAgent = getRandomUserAgent;
|
|
86
|
+
function getNeedleOptions() {
|
|
87
|
+
const userAgent = getRandomUserAgent();
|
|
88
|
+
const chromeVersionMatch = userAgent.match(/Chrome\/(\d+)/);
|
|
89
|
+
const chromeVersion = chromeVersionMatch ? chromeVersionMatch[1] : '121';
|
|
90
|
+
return {
|
|
91
|
+
headers: {
|
|
92
|
+
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
|
93
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
94
|
+
'cache-control': 'no-cache',
|
|
95
|
+
'pragma': 'no-cache',
|
|
96
|
+
'sec-ch-ua': `"Not A(Brand";v="8", "Chromium";v="${chromeVersion}", "Google Chrome";v="${chromeVersion}"`,
|
|
97
|
+
'sec-ch-ua-mobile': '?0',
|
|
98
|
+
'sec-ch-ua-platform': '"Windows"',
|
|
99
|
+
'sec-fetch-dest': 'document',
|
|
100
|
+
'sec-fetch-mode': 'navigate',
|
|
101
|
+
'sec-fetch-site': 'none',
|
|
102
|
+
'sec-fetch-user': '?1',
|
|
103
|
+
'sec-gpc': '1',
|
|
104
|
+
'upgrade-insecure-requests': '1',
|
|
105
|
+
'user-agent': userAgent,
|
|
106
|
+
},
|
|
107
|
+
follow_max: 5,
|
|
108
|
+
response_timeout: 30000,
|
|
109
|
+
read_timeout: 30000,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
exports.getNeedleOptions = getNeedleOptions;
|
|
@@ -156,7 +156,9 @@ function classifyError(error) {
|
|
|
156
156
|
lowerMessage.includes('502') ||
|
|
157
157
|
lowerMessage.includes('503') ||
|
|
158
158
|
lowerMessage.includes('504') ||
|
|
159
|
-
lowerMessage.includes('server error')
|
|
159
|
+
lowerMessage.includes('server error') ||
|
|
160
|
+
lowerMessage.includes('temporarily unavailable') ||
|
|
161
|
+
lowerMessage.includes('internal server error')) {
|
|
160
162
|
const statusMatch = message.match(/(\d{3})/);
|
|
161
163
|
const statusCode = statusMatch ? parseInt(statusMatch[1], 10) : 500;
|
|
162
164
|
return new ServerError(statusCode, error instanceof Error ? error : undefined);
|
|
@@ -7,10 +7,10 @@ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
|
7
7
|
exports.sleep = sleep;
|
|
8
8
|
exports.DEFAULT_RETRY_CONFIG = {
|
|
9
9
|
maxRetries: 3,
|
|
10
|
-
initialDelay:
|
|
11
|
-
maxDelay:
|
|
10
|
+
initialDelay: 2000,
|
|
11
|
+
maxDelay: 15000,
|
|
12
12
|
backoffMultiplier: 2,
|
|
13
|
-
jitterFactor: 0.
|
|
13
|
+
jitterFactor: 0.5,
|
|
14
14
|
retryableCategories: new Set(['network', 'rate_limit', 'timeout']),
|
|
15
15
|
};
|
|
16
16
|
async function executeWithRetry(fn, context = 'operation', config = exports.DEFAULT_RETRY_CONFIG) {
|
|
@@ -62,7 +62,7 @@ async function withTimeout(promise, timeoutMs, operation) {
|
|
|
62
62
|
}
|
|
63
63
|
exports.withTimeout = withTimeout;
|
|
64
64
|
class RequestQueue {
|
|
65
|
-
constructor(minInterval =
|
|
65
|
+
constructor(minInterval = 3000, maxConcurrent = 1) {
|
|
66
66
|
this.queue = [];
|
|
67
67
|
this.processing = false;
|
|
68
68
|
this.lastRequestTime = 0;
|
|
@@ -97,7 +97,12 @@ class RequestQueue {
|
|
|
97
97
|
const timeSinceLastRequest = now - this.lastRequestTime;
|
|
98
98
|
if (timeSinceLastRequest < this.minInterval) {
|
|
99
99
|
const delay = this.minInterval - timeSinceLastRequest;
|
|
100
|
-
|
|
100
|
+
const jitter = Math.random() * 1000;
|
|
101
|
+
await (0, exports.sleep)(delay + jitter);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
const smallJitter = Math.random() * 500;
|
|
105
|
+
await (0, exports.sleep)(smallJitter);
|
|
101
106
|
}
|
|
102
107
|
const fn = this.queue.shift();
|
|
103
108
|
if (fn) {
|
|
@@ -126,7 +131,7 @@ class RequestQueue {
|
|
|
126
131
|
}
|
|
127
132
|
exports.RequestQueue = RequestQueue;
|
|
128
133
|
let globalRequestQueue = null;
|
|
129
|
-
function getRequestQueue(minInterval =
|
|
134
|
+
function getRequestQueue(minInterval = 3000) {
|
|
130
135
|
if (!globalRequestQueue) {
|
|
131
136
|
globalRequestQueue = new RequestQueue(minInterval);
|
|
132
137
|
}
|
package/dist/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-duckduckgo-search",
|
|
3
|
-
"version": "32.
|
|
3
|
+
"version": "32.3.0",
|
|
4
4
|
"description": "Professional-grade n8n community node for DuckDuckGo search with comprehensive error handling, validation, and quality filtering.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"n8n-community-node-package",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "n8n-nodes-duckduckgo-search",
|
|
3
|
-
"version": "32.
|
|
3
|
+
"version": "32.3.0",
|
|
4
4
|
"description": "Professional-grade n8n community node for DuckDuckGo search with comprehensive error handling, validation, and quality filtering.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"n8n-community-node-package",
|