html-get 2.11.1 → 2.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/package.json +1 -1
  2. package/scripts/postinstall +57 -49
  3. package/src/auto-domains.json +258 -42
  4. package/src/index.js +7 -2
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "html-get",
3
3
  "description": "Get the HTML from any website, using prerendering when is necessary.",
4
4
  "homepage": "https://nicedoc.com/microlinkhq/html-get",
5
- "version": "2.11.1",
5
+ "version": "2.11.3",
6
6
  "main": "src/index.js",
7
7
  "bin": {
8
8
  "html-get": "bin/index.js"
package/scripts/postinstall CHANGED
@@ -8,59 +8,67 @@ const writeJsonFile = require('write-json-file')
8
8
  const topsites = require('top-sites')
9
9
 
10
10
  const domains = [
11
- 'apple',
12
- 'bbc',
13
- 'blogspot',
14
- 'csdn',
15
- 'deviantart',
16
- 'digg',
17
- 'dribbble',
18
- 'engadget',
19
- 'etsy',
20
- 'eventbrite',
21
- 'flickr',
22
- 'ghost',
23
- 'giphy',
24
- 'github',
25
- 'gitlab',
26
- 'google',
27
- 'huffingtonpost',
28
- 'imdb',
29
- 'imgur',
30
- 'meetup',
31
- 'microsoft',
32
- 'nytimes',
33
- 'pinterest',
34
- 'producthunt',
35
- 'slideshare',
36
- 'soundcloud',
37
- 'sourceforge',
38
- 'spotify',
39
- 'stackoverflow',
40
- 'substack',
41
- 'techcrunch',
42
- 'telegraph',
43
- 'theguardian',
44
- 'theverge',
45
- 'tumblr',
46
- 'vimeo',
47
- 'wikipedia',
48
- 'wordpress',
49
- 'ycombinator',
50
- 'yelp',
51
- 'youtube',
52
- 'zoom'
11
+ [['domainWithoutSuffix', 'apple']],
12
+ [['domainWithoutSuffix', 'bbc']],
13
+ [['domainWithoutSuffix', 'blogspot']],
14
+ [['domainWithoutSuffix', 'csdn']],
15
+ [['domainWithoutSuffix', 'deviantart']],
16
+ [['domainWithoutSuffix', 'digg']],
17
+ [['domainWithoutSuffix', 'dribbble']],
18
+ [['domainWithoutSuffix', 'engadget']],
19
+ [['domainWithoutSuffix', 'etsy']],
20
+ [['domainWithoutSuffix', 'eventbrite']],
21
+ [['domainWithoutSuffix', 'flickr']],
22
+ [['domainWithoutSuffix', 'ghost']],
23
+ [['domainWithoutSuffix', 'giphy']],
24
+ [['domainWithoutSuffix', 'github']],
25
+ [['domainWithoutSuffix', 'gitlab']],
26
+ [['domainWithoutSuffix', 'google']],
27
+ [['domainWithoutSuffix', 'huffingtonpost']],
28
+ [['domainWithoutSuffix', 'imdb']],
29
+ [['domainWithoutSuffix', 'imgur']],
30
+ [['domainWithoutSuffix', 'meetup']],
31
+ [['domainWithoutSuffix', 'microsoft']],
32
+ [['domainWithoutSuffix', 'nytimes']],
33
+ [['domainWithoutSuffix', 'pinterest']],
34
+ [['domainWithoutSuffix', 'producthunt']],
35
+ [['domainWithoutSuffix', 'slideshare']],
36
+ [['domainWithoutSuffix', 'soundcloud']],
37
+ [['domainWithoutSuffix', 'sourceforge']],
38
+ [['domainWithoutSuffix', 'spotify']],
39
+ [['domainWithoutSuffix', 'stackoverflow']],
40
+ [['domainWithoutSuffix', 'substack']],
41
+ [['domainWithoutSuffix', 'techcrunch']],
42
+ [['domainWithoutSuffix', 'telegraph']],
43
+ [['domainWithoutSuffix', 'theguardian']],
44
+ [['domainWithoutSuffix', 'theverge']],
45
+ [['domainWithoutSuffix', 'tumblr']],
46
+ [['domainWithoutSuffix', 'vimeo']],
47
+ [['domainWithoutSuffix', 'wikipedia']],
48
+ [['domainWithoutSuffix', 'wordpress']],
49
+ [['domainWithoutSuffix', 'ycombinator']],
50
+ [['domainWithoutSuffix', 'yelp']],
51
+ [['domainWithoutSuffix', 'youtube']],
52
+ [['domainWithoutSuffix', 'zoom']],
53
+ [['domain', 'abc.net.au']]
53
54
  ]
54
55
 
55
56
  const { top, rest } = reduce(
56
57
  domains,
57
- (acc, domain) => {
58
- const index = findIndex(
59
- topsites,
60
- ({ rootDomain }) => parseUrl(rootDomain).domainWithoutSuffix === domain
61
- )
62
- if (index !== -1) acc.top[index] = domain
63
- else acc.rest.push(domain)
58
+ (acc, conditions) => {
59
+ for (const [key, value] of conditions) {
60
+ const index = findIndex(topsites, ({ rootDomain }) => {
61
+ const parsedUrl = parseUrl(rootDomain)
62
+ return parsedUrl[key] === value
63
+ })
64
+
65
+ if (index !== -1) {
66
+ acc.top[index] = conditions
67
+ return acc
68
+ }
69
+ }
70
+
71
+ acc.rest.push(conditions)
64
72
  return acc
65
73
  },
66
74
  { top: new Array(topsites.length), rest: [] }
package/src/auto-domains.json CHANGED
@@ -1,44 +1,260 @@
1
1
  [
2
- "google",
3
- "youtube",
4
- "apple",
5
- "microsoft",
6
- "wordpress",
7
- "wikipedia",
8
- "blogspot",
9
- "vimeo",
10
- "github",
11
- "theguardian",
12
- "nytimes",
13
- "slideshare",
14
- "bbc",
15
- "imdb",
16
- "pinterest",
17
- "telegraph",
18
- "spotify",
19
- "huffingtonpost",
20
- "soundcloud",
21
- "engadget",
22
- "zoom",
23
- "techcrunch",
24
- "yelp",
25
- "eventbrite",
26
- "theverge",
27
- "flickr",
28
- "digg",
29
- "csdn",
30
- "deviantart",
31
- "dribbble",
32
- "etsy",
33
- "ghost",
34
- "giphy",
35
- "gitlab",
36
- "imgur",
37
- "meetup",
38
- "producthunt",
39
- "sourceforge",
40
- "stackoverflow",
41
- "substack",
42
- "tumblr",
43
- "ycombinator"
2
+ [
3
+ [
4
+ "domainWithoutSuffix",
5
+ "google"
6
+ ]
7
+ ],
8
+ [
9
+ [
10
+ "domainWithoutSuffix",
11
+ "youtube"
12
+ ]
13
+ ],
14
+ [
15
+ [
16
+ "domainWithoutSuffix",
17
+ "wordpress"
18
+ ]
19
+ ],
20
+ [
21
+ [
22
+ "domainWithoutSuffix",
23
+ "apple"
24
+ ]
25
+ ],
26
+ [
27
+ [
28
+ "domainWithoutSuffix",
29
+ "wikipedia"
30
+ ]
31
+ ],
32
+ [
33
+ [
34
+ "domainWithoutSuffix",
35
+ "microsoft"
36
+ ]
37
+ ],
38
+ [
39
+ [
40
+ "domainWithoutSuffix",
41
+ "blogspot"
42
+ ]
43
+ ],
44
+ [
45
+ [
46
+ "domainWithoutSuffix",
47
+ "vimeo"
48
+ ]
49
+ ],
50
+ [
51
+ [
52
+ "domainWithoutSuffix",
53
+ "github"
54
+ ]
55
+ ],
56
+ [
57
+ [
58
+ "domainWithoutSuffix",
59
+ "imdb"
60
+ ]
61
+ ],
62
+ [
63
+ [
64
+ "domainWithoutSuffix",
65
+ "bbc"
66
+ ]
67
+ ],
68
+ [
69
+ [
70
+ "domainWithoutSuffix",
71
+ "slideshare"
72
+ ]
73
+ ],
74
+ [
75
+ [
76
+ "domainWithoutSuffix",
77
+ "theguardian"
78
+ ]
79
+ ],
80
+ [
81
+ [
82
+ "domainWithoutSuffix",
83
+ "nytimes"
84
+ ]
85
+ ],
86
+ [
87
+ [
88
+ "domainWithoutSuffix",
89
+ "spotify"
90
+ ]
91
+ ],
92
+ [
93
+ [
94
+ "domainWithoutSuffix",
95
+ "huffingtonpost"
96
+ ]
97
+ ],
98
+ [
99
+ [
100
+ "domainWithoutSuffix",
101
+ "soundcloud"
102
+ ]
103
+ ],
104
+ [
105
+ [
106
+ "domainWithoutSuffix",
107
+ "telegraph"
108
+ ]
109
+ ],
110
+ [
111
+ [
112
+ "domainWithoutSuffix",
113
+ "pinterest"
114
+ ]
115
+ ],
116
+ [
117
+ [
118
+ "domainWithoutSuffix",
119
+ "yelp"
120
+ ]
121
+ ],
122
+ [
123
+ [
124
+ "domainWithoutSuffix",
125
+ "techcrunch"
126
+ ]
127
+ ],
128
+ [
129
+ [
130
+ "domainWithoutSuffix",
131
+ "zoom"
132
+ ]
133
+ ],
134
+ [
135
+ [
136
+ "domain",
137
+ "abc.net.au"
138
+ ]
139
+ ],
140
+ [
141
+ [
142
+ "domainWithoutSuffix",
143
+ "engadget"
144
+ ]
145
+ ],
146
+ [
147
+ [
148
+ "domainWithoutSuffix",
149
+ "eventbrite"
150
+ ]
151
+ ],
152
+ [
153
+ [
154
+ "domainWithoutSuffix",
155
+ "theverge"
156
+ ]
157
+ ],
158
+ [
159
+ [
160
+ "domainWithoutSuffix",
161
+ "flickr"
162
+ ]
163
+ ],
164
+ [
165
+ [
166
+ "domainWithoutSuffix",
167
+ "stackoverflow"
168
+ ]
169
+ ],
170
+ [
171
+ [
172
+ "domainWithoutSuffix",
173
+ "digg"
174
+ ]
175
+ ],
176
+ [
177
+ [
178
+ "domainWithoutSuffix",
179
+ "etsy"
180
+ ]
181
+ ],
182
+ [
183
+ [
184
+ "domainWithoutSuffix",
185
+ "csdn"
186
+ ]
187
+ ],
188
+ [
189
+ [
190
+ "domainWithoutSuffix",
191
+ "deviantart"
192
+ ]
193
+ ],
194
+ [
195
+ [
196
+ "domainWithoutSuffix",
197
+ "dribbble"
198
+ ]
199
+ ],
200
+ [
201
+ [
202
+ "domainWithoutSuffix",
203
+ "ghost"
204
+ ]
205
+ ],
206
+ [
207
+ [
208
+ "domainWithoutSuffix",
209
+ "giphy"
210
+ ]
211
+ ],
212
+ [
213
+ [
214
+ "domainWithoutSuffix",
215
+ "gitlab"
216
+ ]
217
+ ],
218
+ [
219
+ [
220
+ "domainWithoutSuffix",
221
+ "imgur"
222
+ ]
223
+ ],
224
+ [
225
+ [
226
+ "domainWithoutSuffix",
227
+ "meetup"
228
+ ]
229
+ ],
230
+ [
231
+ [
232
+ "domainWithoutSuffix",
233
+ "producthunt"
234
+ ]
235
+ ],
236
+ [
237
+ [
238
+ "domainWithoutSuffix",
239
+ "sourceforge"
240
+ ]
241
+ ],
242
+ [
243
+ [
244
+ "domainWithoutSuffix",
245
+ "substack"
246
+ ]
247
+ ],
248
+ [
249
+ [
250
+ "domainWithoutSuffix",
251
+ "tumblr"
252
+ ]
253
+ ],
254
+ [
255
+ [
256
+ "domainWithoutSuffix",
257
+ "ycombinator"
258
+ ]
259
+ ]
44
260
  ]
package/src/index.js CHANGED
@@ -138,8 +138,12 @@ const prerender = PCancelable.fn(
138
138
 
139
139
  const modes = { fetch, prerender }
140
140
 
141
- const isFetchMode = url =>
142
- autoDomains.includes(parseUrl(url).domainWithoutSuffix)
141
+ const isFetchMode = url => {
142
+ const parsedUrl = parseUrl(url)
143
+ return autoDomains.some(conditions =>
144
+ conditions.every(([prop, value]) => parsedUrl[prop] === value)
145
+ )
146
+ }
143
147
 
144
148
  const determinateMode = (url, { prerender }) => {
145
149
  if (prerender === false || isMediaUrl(url)) return 'fetch'
@@ -210,3 +214,4 @@ module.exports = PCancelable.fn(
210
214
  )
211
215
 
212
216
  module.exports.REQ_TIMEOUT = REQ_TIMEOUT
217
+ module.exports.isFetchMode = isFetchMode