clean-web-scraper 4.3.0 → 4.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example-usage.js +7 -5
- package/package.json +2 -2
package/example-usage.js
CHANGED
|
@@ -134,9 +134,11 @@ async function electronicintifada ( enable )
|
|
|
134
134
|
"https://electronicintifada.net/review",
|
|
135
135
|
"https://electronicintifada.net/artmusicculture",
|
|
136
136
|
"https://electronicintifada.net/blog/editors",
|
|
137
|
-
"https://electronicintifada.net/blog"
|
|
138
137
|
],
|
|
139
138
|
exactExcludeList: [
|
|
139
|
+
"https://electronicintifada.net/blog",
|
|
140
|
+
/^https:\/\/electronicintifada\.net\/blog\/.*/,
|
|
141
|
+
/^https:\/\/electronicintifada\.net\/blog\?page=\d+$/,
|
|
140
142
|
"https://electronicintifada.net",
|
|
141
143
|
"https://electronicintifada.net/blogs",
|
|
142
144
|
"https://electronicintifada.net/review",
|
|
@@ -149,10 +151,10 @@ async function electronicintifada ( enable )
|
|
|
149
151
|
metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
|
|
150
152
|
maxArticles: 2000,
|
|
151
153
|
maxDepth: 16,
|
|
152
|
-
batchSize:
|
|
154
|
+
batchSize: 40,
|
|
153
155
|
axiosHeaders: headers,
|
|
154
|
-
axiosMaxRetries:
|
|
155
|
-
axiosRetryDelay:
|
|
156
|
+
axiosMaxRetries: 2,
|
|
157
|
+
axiosRetryDelay: 8000,
|
|
156
158
|
axiosProxy: {
|
|
157
159
|
host: "localhost",
|
|
158
160
|
port: 2080,
|
|
@@ -230,7 +232,7 @@ async function mondoweiss ( enable )
|
|
|
230
232
|
maxDepth: 15,
|
|
231
233
|
batchSize: 20,
|
|
232
234
|
axiosHeaders: headers,
|
|
233
|
-
axiosMaxRetries:
|
|
235
|
+
axiosMaxRetries: 2,
|
|
234
236
|
axiosRetryDelay: 10000,
|
|
235
237
|
axiosProxy: {
|
|
236
238
|
host: "localhost",
|
package/package.json
CHANGED