html-get 2.13.14 → 2.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -8
- package/scripts/postinstall +5 -4
- package/src/auto-domains.json +1 -260
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "html-get",
|
|
3
3
|
"description": "Get the HTML from any website, using prerendering when is necessary.",
|
|
4
4
|
"homepage": "https://nicedoc.com/microlinkhq/html-get",
|
|
5
|
-
"version": "2.
|
|
5
|
+
"version": "2.14.0",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"bin": {
|
|
8
8
|
"html-get": "bin/index.js"
|
|
@@ -29,23 +29,22 @@
|
|
|
29
29
|
"request"
|
|
30
30
|
],
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@kikobeats/time-span": "~1.0.
|
|
33
|
-
"@metascraper/helpers": "~5.
|
|
32
|
+
"@kikobeats/time-span": "~1.0.3",
|
|
33
|
+
"@metascraper/helpers": "~5.43.0",
|
|
34
34
|
"cheerio": "~1.0.0-rc.12",
|
|
35
35
|
"css-url-regex": "~4.0.0",
|
|
36
36
|
"debug-logfmt": "~1.2.0",
|
|
37
37
|
"execall": "~2.0.0",
|
|
38
38
|
"got": "~11.8.6",
|
|
39
39
|
"html-encode": "~2.1.6",
|
|
40
|
-
"html-urls": "~2.4.
|
|
40
|
+
"html-urls": "~2.4.55",
|
|
41
41
|
"is-html-content": "~1.0.0",
|
|
42
42
|
"lodash": "~4.17.21",
|
|
43
43
|
"mri": "~1.2.0",
|
|
44
44
|
"p-cancelable": "~2.1.0",
|
|
45
45
|
"p-retry": "~4.6.0",
|
|
46
46
|
"replace-string": "~3.1.0",
|
|
47
|
-
"top-sites": "~1.1.
|
|
48
|
-
"write-json-file": "~4.3.0"
|
|
47
|
+
"top-sites": "~1.1.202"
|
|
49
48
|
},
|
|
50
49
|
"devDependencies": {
|
|
51
50
|
"@commitlint/cli": "latest",
|
|
@@ -55,9 +54,9 @@
|
|
|
55
54
|
"browserless": "latest",
|
|
56
55
|
"c8": "latest",
|
|
57
56
|
"ci-publish": "latest",
|
|
58
|
-
"conventional-github-releaser": "latest",
|
|
59
57
|
"finepack": "latest",
|
|
60
58
|
"git-authors-cli": "latest",
|
|
59
|
+
"github-generate-release": "latest",
|
|
61
60
|
"nano-staged": "latest",
|
|
62
61
|
"npm-check-updates": "latest",
|
|
63
62
|
"pretty": "latest",
|
|
@@ -85,7 +84,7 @@
|
|
|
85
84
|
"prerelease": "npm run update:check && npm run contributors",
|
|
86
85
|
"pretest": "npm run lint",
|
|
87
86
|
"release": "standard-version -a",
|
|
88
|
-
"release:github": "
|
|
87
|
+
"release:github": "github-generate-release",
|
|
89
88
|
"release:tags": "git push --follow-tags origin HEAD:master",
|
|
90
89
|
"test": "c8 ava",
|
|
91
90
|
"update": "ncu -u",
|
package/scripts/postinstall
CHANGED
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
|
|
5
5
|
const { compact, reduce, findIndex } = require('lodash')
|
|
6
6
|
const { parseUrl } = require('@metascraper/helpers')
|
|
7
|
-
const
|
|
7
|
+
const { writeFile } = require('fs').promises
|
|
8
8
|
const topsites = require('top-sites')
|
|
9
9
|
|
|
10
10
|
const domains = [
|
|
11
|
+
[['domain', 'abc.net.au']],
|
|
11
12
|
[['domainWithoutSuffix', 'apple']],
|
|
12
13
|
[['domainWithoutSuffix', 'bbc']],
|
|
13
14
|
[['domainWithoutSuffix', 'blogspot']],
|
|
@@ -43,14 +44,14 @@ const domains = [
|
|
|
43
44
|
[['domainWithoutSuffix', 'theguardian']],
|
|
44
45
|
[['domainWithoutSuffix', 'theverge']],
|
|
45
46
|
[['domainWithoutSuffix', 'tumblr']],
|
|
47
|
+
[['domainWithoutSuffix', 'twitter']],
|
|
46
48
|
[['domainWithoutSuffix', 'vimeo']],
|
|
47
49
|
[['domainWithoutSuffix', 'wikipedia']],
|
|
48
50
|
[['domainWithoutSuffix', 'wordpress']],
|
|
49
51
|
[['domainWithoutSuffix', 'ycombinator']],
|
|
50
52
|
[['domainWithoutSuffix', 'yelp']],
|
|
51
53
|
[['domainWithoutSuffix', 'youtube']],
|
|
52
|
-
[['domainWithoutSuffix', 'zoom']]
|
|
53
|
-
[['domain', 'abc.net.au']]
|
|
54
|
+
[['domainWithoutSuffix', 'zoom']]
|
|
54
55
|
]
|
|
55
56
|
|
|
56
57
|
const { top, rest } = reduce(
|
|
@@ -74,6 +75,6 @@ const { top, rest } = reduce(
|
|
|
74
75
|
{ top: new Array(topsites.length), rest: [] }
|
|
75
76
|
)
|
|
76
77
|
|
|
77
|
-
|
|
78
|
+
writeFile('./src/auto-domains.json', JSON.stringify(compact(top).concat(rest)), null, 2).catch(
|
|
78
79
|
error => console.log(error)
|
|
79
80
|
)
|
package/src/auto-domains.json
CHANGED
|
@@ -1,260 +1 @@
|
|
|
1
|
-
[
|
|
2
|
-
[
|
|
3
|
-
[
|
|
4
|
-
"domainWithoutSuffix",
|
|
5
|
-
"google"
|
|
6
|
-
]
|
|
7
|
-
],
|
|
8
|
-
[
|
|
9
|
-
[
|
|
10
|
-
"domainWithoutSuffix",
|
|
11
|
-
"youtube"
|
|
12
|
-
]
|
|
13
|
-
],
|
|
14
|
-
[
|
|
15
|
-
[
|
|
16
|
-
"domainWithoutSuffix",
|
|
17
|
-
"microsoft"
|
|
18
|
-
]
|
|
19
|
-
],
|
|
20
|
-
[
|
|
21
|
-
[
|
|
22
|
-
"domainWithoutSuffix",
|
|
23
|
-
"apple"
|
|
24
|
-
]
|
|
25
|
-
],
|
|
26
|
-
[
|
|
27
|
-
[
|
|
28
|
-
"domainWithoutSuffix",
|
|
29
|
-
"wordpress"
|
|
30
|
-
]
|
|
31
|
-
],
|
|
32
|
-
[
|
|
33
|
-
[
|
|
34
|
-
"domainWithoutSuffix",
|
|
35
|
-
"wikipedia"
|
|
36
|
-
]
|
|
37
|
-
],
|
|
38
|
-
[
|
|
39
|
-
[
|
|
40
|
-
"domainWithoutSuffix",
|
|
41
|
-
"blogspot"
|
|
42
|
-
]
|
|
43
|
-
],
|
|
44
|
-
[
|
|
45
|
-
[
|
|
46
|
-
"domainWithoutSuffix",
|
|
47
|
-
"vimeo"
|
|
48
|
-
]
|
|
49
|
-
],
|
|
50
|
-
[
|
|
51
|
-
[
|
|
52
|
-
"domainWithoutSuffix",
|
|
53
|
-
"github"
|
|
54
|
-
]
|
|
55
|
-
],
|
|
56
|
-
[
|
|
57
|
-
[
|
|
58
|
-
"domainWithoutSuffix",
|
|
59
|
-
"bbc"
|
|
60
|
-
]
|
|
61
|
-
],
|
|
62
|
-
[
|
|
63
|
-
[
|
|
64
|
-
"domainWithoutSuffix",
|
|
65
|
-
"imdb"
|
|
66
|
-
]
|
|
67
|
-
],
|
|
68
|
-
[
|
|
69
|
-
[
|
|
70
|
-
"domainWithoutSuffix",
|
|
71
|
-
"nytimes"
|
|
72
|
-
]
|
|
73
|
-
],
|
|
74
|
-
[
|
|
75
|
-
[
|
|
76
|
-
"domainWithoutSuffix",
|
|
77
|
-
"slideshare"
|
|
78
|
-
]
|
|
79
|
-
],
|
|
80
|
-
[
|
|
81
|
-
[
|
|
82
|
-
"domainWithoutSuffix",
|
|
83
|
-
"theguardian"
|
|
84
|
-
]
|
|
85
|
-
],
|
|
86
|
-
[
|
|
87
|
-
[
|
|
88
|
-
"domainWithoutSuffix",
|
|
89
|
-
"spotify"
|
|
90
|
-
]
|
|
91
|
-
],
|
|
92
|
-
[
|
|
93
|
-
[
|
|
94
|
-
"domainWithoutSuffix",
|
|
95
|
-
"telegraph"
|
|
96
|
-
]
|
|
97
|
-
],
|
|
98
|
-
[
|
|
99
|
-
[
|
|
100
|
-
"domainWithoutSuffix",
|
|
101
|
-
"pinterest"
|
|
102
|
-
]
|
|
103
|
-
],
|
|
104
|
-
[
|
|
105
|
-
[
|
|
106
|
-
"domainWithoutSuffix",
|
|
107
|
-
"huffingtonpost"
|
|
108
|
-
]
|
|
109
|
-
],
|
|
110
|
-
[
|
|
111
|
-
[
|
|
112
|
-
"domainWithoutSuffix",
|
|
113
|
-
"soundcloud"
|
|
114
|
-
]
|
|
115
|
-
],
|
|
116
|
-
[
|
|
117
|
-
[
|
|
118
|
-
"domainWithoutSuffix",
|
|
119
|
-
"stackoverflow"
|
|
120
|
-
]
|
|
121
|
-
],
|
|
122
|
-
[
|
|
123
|
-
[
|
|
124
|
-
"domain",
|
|
125
|
-
"abc.net.au"
|
|
126
|
-
]
|
|
127
|
-
],
|
|
128
|
-
[
|
|
129
|
-
[
|
|
130
|
-
"domainWithoutSuffix",
|
|
131
|
-
"engadget"
|
|
132
|
-
]
|
|
133
|
-
],
|
|
134
|
-
[
|
|
135
|
-
[
|
|
136
|
-
"domainWithoutSuffix",
|
|
137
|
-
"yelp"
|
|
138
|
-
]
|
|
139
|
-
],
|
|
140
|
-
[
|
|
141
|
-
[
|
|
142
|
-
"domainWithoutSuffix",
|
|
143
|
-
"zoom"
|
|
144
|
-
]
|
|
145
|
-
],
|
|
146
|
-
[
|
|
147
|
-
[
|
|
148
|
-
"domainWithoutSuffix",
|
|
149
|
-
"techcrunch"
|
|
150
|
-
]
|
|
151
|
-
],
|
|
152
|
-
[
|
|
153
|
-
[
|
|
154
|
-
"domainWithoutSuffix",
|
|
155
|
-
"eventbrite"
|
|
156
|
-
]
|
|
157
|
-
],
|
|
158
|
-
[
|
|
159
|
-
[
|
|
160
|
-
"domainWithoutSuffix",
|
|
161
|
-
"theverge"
|
|
162
|
-
]
|
|
163
|
-
],
|
|
164
|
-
[
|
|
165
|
-
[
|
|
166
|
-
"domainWithoutSuffix",
|
|
167
|
-
"flickr"
|
|
168
|
-
]
|
|
169
|
-
],
|
|
170
|
-
[
|
|
171
|
-
[
|
|
172
|
-
"domainWithoutSuffix",
|
|
173
|
-
"csdn"
|
|
174
|
-
]
|
|
175
|
-
],
|
|
176
|
-
[
|
|
177
|
-
[
|
|
178
|
-
"domainWithoutSuffix",
|
|
179
|
-
"deviantart"
|
|
180
|
-
]
|
|
181
|
-
],
|
|
182
|
-
[
|
|
183
|
-
[
|
|
184
|
-
"domainWithoutSuffix",
|
|
185
|
-
"digg"
|
|
186
|
-
]
|
|
187
|
-
],
|
|
188
|
-
[
|
|
189
|
-
[
|
|
190
|
-
"domainWithoutSuffix",
|
|
191
|
-
"dribbble"
|
|
192
|
-
]
|
|
193
|
-
],
|
|
194
|
-
[
|
|
195
|
-
[
|
|
196
|
-
"domainWithoutSuffix",
|
|
197
|
-
"etsy"
|
|
198
|
-
]
|
|
199
|
-
],
|
|
200
|
-
[
|
|
201
|
-
[
|
|
202
|
-
"domainWithoutSuffix",
|
|
203
|
-
"ghost"
|
|
204
|
-
]
|
|
205
|
-
],
|
|
206
|
-
[
|
|
207
|
-
[
|
|
208
|
-
"domainWithoutSuffix",
|
|
209
|
-
"giphy"
|
|
210
|
-
]
|
|
211
|
-
],
|
|
212
|
-
[
|
|
213
|
-
[
|
|
214
|
-
"domainWithoutSuffix",
|
|
215
|
-
"gitlab"
|
|
216
|
-
]
|
|
217
|
-
],
|
|
218
|
-
[
|
|
219
|
-
[
|
|
220
|
-
"domainWithoutSuffix",
|
|
221
|
-
"imgur"
|
|
222
|
-
]
|
|
223
|
-
],
|
|
224
|
-
[
|
|
225
|
-
[
|
|
226
|
-
"domainWithoutSuffix",
|
|
227
|
-
"meetup"
|
|
228
|
-
]
|
|
229
|
-
],
|
|
230
|
-
[
|
|
231
|
-
[
|
|
232
|
-
"domainWithoutSuffix",
|
|
233
|
-
"producthunt"
|
|
234
|
-
]
|
|
235
|
-
],
|
|
236
|
-
[
|
|
237
|
-
[
|
|
238
|
-
"domainWithoutSuffix",
|
|
239
|
-
"sourceforge"
|
|
240
|
-
]
|
|
241
|
-
],
|
|
242
|
-
[
|
|
243
|
-
[
|
|
244
|
-
"domainWithoutSuffix",
|
|
245
|
-
"substack"
|
|
246
|
-
]
|
|
247
|
-
],
|
|
248
|
-
[
|
|
249
|
-
[
|
|
250
|
-
"domainWithoutSuffix",
|
|
251
|
-
"tumblr"
|
|
252
|
-
]
|
|
253
|
-
],
|
|
254
|
-
[
|
|
255
|
-
[
|
|
256
|
-
"domainWithoutSuffix",
|
|
257
|
-
"ycombinator"
|
|
258
|
-
]
|
|
259
|
-
]
|
|
260
|
-
]
|
|
1
|
+
[[["domainWithoutSuffix","google"]],[["domainWithoutSuffix","youtube"]],[["domainWithoutSuffix","microsoft"]],[["domainWithoutSuffix","apple"]],[["domainWithoutSuffix","wordpress"]],[["domainWithoutSuffix","wikipedia"]],[["domainWithoutSuffix","blogspot"]],[["domainWithoutSuffix","vimeo"]],[["domainWithoutSuffix","github"]],[["domainWithoutSuffix","bbc"]],[["domainWithoutSuffix","imdb"]],[["domainWithoutSuffix","nytimes"]],[["domainWithoutSuffix","slideshare"]],[["domainWithoutSuffix","theguardian"]],[["domainWithoutSuffix","spotify"]],[["domainWithoutSuffix","telegraph"]],[["domainWithoutSuffix","pinterest"]],[["domainWithoutSuffix","huffingtonpost"]],[["domainWithoutSuffix","soundcloud"]],[["domainWithoutSuffix","twitter"]],[["domainWithoutSuffix","yelp"]],[["domainWithoutSuffix","eventbrite"]],[["domainWithoutSuffix","engadget"]],[["domain","abc.net.au"]],[["domainWithoutSuffix","stackoverflow"]],[["domainWithoutSuffix","zoom"]],[["domainWithoutSuffix","techcrunch"]],[["domainWithoutSuffix","theverge"]],[["domainWithoutSuffix","digg"]],[["domainWithoutSuffix","giphy"]],[["domainWithoutSuffix","flickr"]],[["domainWithoutSuffix","substack"]],[["domainWithoutSuffix","csdn"]],[["domainWithoutSuffix","deviantart"]],[["domainWithoutSuffix","dribbble"]],[["domainWithoutSuffix","etsy"]],[["domainWithoutSuffix","ghost"]],[["domainWithoutSuffix","gitlab"]],[["domainWithoutSuffix","imgur"]],[["domainWithoutSuffix","meetup"]],[["domainWithoutSuffix","producthunt"]],[["domainWithoutSuffix","sourceforge"]],[["domainWithoutSuffix","tumblr"]],[["domainWithoutSuffix","ycombinator"]]]
|