clean-web-scraper 3.8.2 → 3.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/example-usage.js +2 -0
  2. package/main.js +1 -1
  3. package/package.json +1 -1
package/example-usage.js CHANGED
@@ -128,6 +128,7 @@ async function electronicintifada ( enable )
128
128
  metadataFields: ["author", "title", "description", "dateScrapedDate"],
129
129
  maxDepth: 10,
130
130
  maxArticles: 2000,
131
+ concurrencyLimit: 4,
131
132
  axiosHeaders: headers,
132
133
  retryDelay: 10000
133
134
  });
@@ -189,6 +190,7 @@ async function mondoweiss ( enable )
189
190
  csvOutputPath: "./dataset/mondoweiss/train.csv",
190
191
  maxArticles: 2500,
191
192
  maxRetries: 2,
193
+ concurrencyLimit: 4,
192
194
  axiosHeaders: headers,
193
195
  axiosProxy: {
194
196
  host: "localhost",
package/main.js CHANGED
@@ -585,7 +585,7 @@ class WebScraper
585
585
  {
586
586
  if ( this.hasReachedMax( ) )
587
587
  {
588
- throw new Error( "Max reached" );
588
+ break;
589
589
  }
590
590
  return await axios.get( url, options );
591
591
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "clean-web-scraper",
3
- "version": "3.8.2",
3
+ "version": "3.8.3",
4
4
  "main": "main.js",
5
5
  "scripts": {
6
6
  "start": "node main.js",