clean-web-scraper 3.3.7 → 3.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example-usage.js +2 -4
- package/package.json +1 -1
- package/src/WebScraper.js +1 -1
package/example-usage.js
CHANGED
|
@@ -100,7 +100,8 @@ async function electronicintifada ()
|
|
|
100
100
|
"https://electronicintifada.net/tags/",
|
|
101
101
|
"https://electronicintifada.net/blog",
|
|
102
102
|
"https://electronicintifada.net/people",
|
|
103
|
-
"https://electronicintifada.net/location"
|
|
103
|
+
"https://electronicintifada.net/location",
|
|
104
|
+
"https://electronicintifada.net/file"
|
|
104
105
|
],
|
|
105
106
|
exactExcludeList: [
|
|
106
107
|
"https://electronicintifada.net",
|
|
@@ -133,9 +134,6 @@ void async function main ()
|
|
|
133
134
|
electronicintifadaScraper
|
|
134
135
|
] );
|
|
135
136
|
|
|
136
|
-
// 4
|
|
137
|
-
// https://electronicintifada.net/
|
|
138
|
-
|
|
139
137
|
// 5
|
|
140
138
|
// https://www.palestineremembered.com/ZionistFAQ.html
|
|
141
139
|
|
package/package.json
CHANGED
package/src/WebScraper.js
CHANGED
|
@@ -20,7 +20,7 @@ class WebScraper
|
|
|
20
20
|
textOutputPath,
|
|
21
21
|
csvOutputPath,
|
|
22
22
|
includeMetadata = false,
|
|
23
|
-
metadataFields = [], // ['title', 'description', 'author',
|
|
23
|
+
metadataFields = [], // ['title', 'description', 'author', etc.]
|
|
24
24
|
headers,
|
|
25
25
|
usePuppeteer,
|
|
26
26
|
puppeteerProxy, // e.g. http://127.0.0.1:2080
|