clean-web-scraper 4.3.3 → 4.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example-usage.js +6 -2
- package/package.json +1 -1
package/example-usage.js
CHANGED
|
@@ -261,13 +261,17 @@ async function bdsmovement ( enable )
|
|
|
261
261
|
"https://bdsmovement.net/contact-us",
|
|
262
262
|
"https://bdsmovement.net/taxonomy",
|
|
263
263
|
"https://bdsmovement.net/news-type",
|
|
264
|
-
"https://bdsmovement.net/cdn-cgi"
|
|
264
|
+
"https://bdsmovement.net/cdn-cgi",
|
|
265
|
+
"https://bdsmovement.net/es/",
|
|
266
|
+
"https://bdsmovement.net/ar/",
|
|
267
|
+
"https://bdsmovement.net/resource-type/",
|
|
265
268
|
],
|
|
266
269
|
exactExcludeList: [
|
|
267
270
|
"https://bdsmovement.net/",
|
|
268
271
|
"https://bdsmovement.net/shutdownnation",
|
|
269
272
|
"https://bdsmovement.net/campaigns",
|
|
270
273
|
"https://bdsmovement.net/resources",
|
|
274
|
+
"https://bdsmovement.net/news",
|
|
271
275
|
/^https:\/\/bdsmovement\.net\/resources\?page=\d+$/,
|
|
272
276
|
/^https:\/\/bdsmovement\.net\/resources\?campaign=\d+$/,
|
|
273
277
|
/^https:\/\/bdsmovement\.net\/resources\?type=\d+$/,
|
|
@@ -283,7 +287,7 @@ async function bdsmovement ( enable )
|
|
|
283
287
|
metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
|
|
284
288
|
maxArticles: 2000,
|
|
285
289
|
maxDepth: 16,
|
|
286
|
-
batchSize:
|
|
290
|
+
batchSize: 100,
|
|
287
291
|
axiosHeaders: headers,
|
|
288
292
|
axiosMaxRetries: 2,
|
|
289
293
|
axiosRetryDelay: 8000,
|