clean-web-scraper 3.6.2 → 3.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/main.js +4 -4
- package/package.json +1 -1
package/main.js
CHANGED
|
@@ -49,7 +49,7 @@ class WebScraper
|
|
|
49
49
|
this.strictBaseURL = strictBaseURL || true;
|
|
50
50
|
this.maxDepth = maxDepth || Infinity;
|
|
51
51
|
this.maxArticles = maxArticles || Infinity;
|
|
52
|
-
this.concurrencyLimit = concurrencyLimit ||
|
|
52
|
+
this.concurrencyLimit = concurrencyLimit || 7;
|
|
53
53
|
|
|
54
54
|
// Output paths setup
|
|
55
55
|
this.scrapResultPath = scrapResultPath;
|
|
@@ -551,7 +551,7 @@ class WebScraper
|
|
|
551
551
|
...this.axiosOptions,
|
|
552
552
|
};
|
|
553
553
|
|
|
554
|
-
let maxRetries =
|
|
554
|
+
let maxRetries = 11;
|
|
555
555
|
for ( let attempt = 1; attempt <= maxRetries; attempt++ )
|
|
556
556
|
{
|
|
557
557
|
try
|
|
@@ -561,8 +561,8 @@ class WebScraper
|
|
|
561
561
|
catch ( error )
|
|
562
562
|
{
|
|
563
563
|
if ( attempt === maxRetries ) throw error;
|
|
564
|
-
await WebScraper.sleep(
|
|
565
|
-
console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${maxRetries})
|
|
564
|
+
await WebScraper.sleep( 5000 * attempt );
|
|
565
|
+
console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${maxRetries})`, error.message, error.code );
|
|
566
566
|
}
|
|
567
567
|
}
|
|
568
568
|
}
|