html-get 2.11.0 → 2.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/package.json +1 -1
  2. package/scripts/postinstall +57 -49
  3. package/src/auto-domains.json +258 -42
  4. package/src/index.js +7 -7
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "html-get",
3
3
  "description": "Get the HTML from any website, using prerendering when is necessary.",
4
4
  "homepage": "https://nicedoc.com/microlinkhq/html-get",
5
- "version": "2.11.0",
5
+ "version": "2.11.2",
6
6
  "main": "src/index.js",
7
7
  "bin": {
8
8
  "html-get": "bin/index.js"
package/scripts/postinstall CHANGED
@@ -8,59 +8,67 @@ const writeJsonFile = require('write-json-file')
8
8
  const topsites = require('top-sites')
9
9
 
10
10
  const domains = [
11
- 'apple',
12
- 'bbc',
13
- 'blogspot',
14
- 'csdn',
15
- 'deviantart',
16
- 'digg',
17
- 'dribbble',
18
- 'engadget',
19
- 'etsy',
20
- 'eventbrite',
21
- 'flickr',
22
- 'ghost',
23
- 'giphy',
24
- 'github',
25
- 'gitlab',
26
- 'google',
27
- 'huffingtonpost',
28
- 'imdb',
29
- 'imgur',
30
- 'meetup',
31
- 'microsoft',
32
- 'nytimes',
33
- 'pinterest',
34
- 'producthunt',
35
- 'slideshare',
36
- 'soundcloud',
37
- 'sourceforge',
38
- 'spotify',
39
- 'stackoverflow',
40
- 'substack',
41
- 'techcrunch',
42
- 'telegraph',
43
- 'theguardian',
44
- 'theverge',
45
- 'tumblr',
46
- 'vimeo',
47
- 'wikipedia',
48
- 'wordpress',
49
- 'ycombinator',
50
- 'yelp',
51
- 'youtube',
52
- 'zoom'
11
+ [['domainWithoutSuffix', 'apple']],
12
+ [['domainWithoutSuffix', 'bbc']],
13
+ [['domainWithoutSuffix', 'blogspot']],
14
+ [['domainWithoutSuffix', 'csdn']],
15
+ [['domainWithoutSuffix', 'deviantart']],
16
+ [['domainWithoutSuffix', 'digg']],
17
+ [['domainWithoutSuffix', 'dribbble']],
18
+ [['domainWithoutSuffix', 'engadget']],
19
+ [['domainWithoutSuffix', 'etsy']],
20
+ [['domainWithoutSuffix', 'eventbrite']],
21
+ [['domainWithoutSuffix', 'flickr']],
22
+ [['domainWithoutSuffix', 'ghost']],
23
+ [['domainWithoutSuffix', 'giphy']],
24
+ [['domainWithoutSuffix', 'github']],
25
+ [['domainWithoutSuffix', 'gitlab']],
26
+ [['domainWithoutSuffix', 'google']],
27
+ [['domainWithoutSuffix', 'huffingtonpost']],
28
+ [['domainWithoutSuffix', 'imdb']],
29
+ [['domainWithoutSuffix', 'imgur']],
30
+ [['domainWithoutSuffix', 'meetup']],
31
+ [['domainWithoutSuffix', 'microsoft']],
32
+ [['domainWithoutSuffix', 'nytimes']],
33
+ [['domainWithoutSuffix', 'pinterest']],
34
+ [['domainWithoutSuffix', 'producthunt']],
35
+ [['domainWithoutSuffix', 'slideshare']],
36
+ [['domainWithoutSuffix', 'soundcloud']],
37
+ [['domainWithoutSuffix', 'sourceforge']],
38
+ [['domainWithoutSuffix', 'spotify']],
39
+ [['domainWithoutSuffix', 'stackoverflow']],
40
+ [['domainWithoutSuffix', 'substack']],
41
+ [['domainWithoutSuffix', 'techcrunch']],
42
+ [['domainWithoutSuffix', 'telegraph']],
43
+ [['domainWithoutSuffix', 'theguardian']],
44
+ [['domainWithoutSuffix', 'theverge']],
45
+ [['domainWithoutSuffix', 'tumblr']],
46
+ [['domainWithoutSuffix', 'vimeo']],
47
+ [['domainWithoutSuffix', 'wikipedia']],
48
+ [['domainWithoutSuffix', 'wordpress']],
49
+ [['domainWithoutSuffix', 'ycombinator']],
50
+ [['domainWithoutSuffix', 'yelp']],
51
+ [['domainWithoutSuffix', 'youtube']],
52
+ [['domainWithoutSuffix', 'zoom']],
53
+ [['domain', 'abc.net.au']]
53
54
  ]
54
55
 
55
56
  const { top, rest } = reduce(
56
57
  domains,
57
- (acc, domain) => {
58
- const index = findIndex(
59
- topsites,
60
- ({ rootDomain }) => parseUrl(rootDomain).domainWithoutSuffix === domain
61
- )
62
- if (index !== -1) acc.top[index] = domain
63
- else acc.rest.push(domain)
58
+ (acc, conditions) => {
59
+ for (const [key, value] of conditions) {
60
+ const index = findIndex(topsites, ({ rootDomain }) => {
61
+ const parsedUrl = parseUrl(rootDomain)
62
+ return parsedUrl[key] === value
63
+ })
64
+
65
+ if (index !== -1) {
66
+ acc.top[index] = conditions
67
+ return acc
68
+ }
69
+ }
70
+
71
+ acc.rest.push(conditions)
64
72
  return acc
65
73
  },
66
74
  { top: new Array(topsites.length), rest: [] }
package/src/auto-domains.json CHANGED
@@ -1,44 +1,260 @@
1
1
  [
2
- "google",
3
- "youtube",
4
- "microsoft",
5
- "apple",
6
- "wikipedia",
7
- "wordpress",
8
- "blogspot",
9
- "vimeo",
10
- "github",
11
- "theguardian",
12
- "imdb",
13
- "bbc",
14
- "nytimes",
15
- "slideshare",
16
- "soundcloud",
17
- "pinterest",
18
- "telegraph",
19
- "huffingtonpost",
20
- "engadget",
21
- "techcrunch",
22
- "zoom",
23
- "yelp",
24
- "eventbrite",
25
- "spotify",
26
- "theverge",
27
- "dribbble",
28
- "giphy",
29
- "imgur",
30
- "csdn",
31
- "deviantart",
32
- "digg",
33
- "etsy",
34
- "flickr",
35
- "ghost",
36
- "gitlab",
37
- "meetup",
38
- "producthunt",
39
- "sourceforge",
40
- "stackoverflow",
41
- "substack",
42
- "tumblr",
43
- "ycombinator"
2
+ [
3
+ [
4
+ "domainWithoutSuffix",
5
+ "youtube"
6
+ ]
7
+ ],
8
+ [
9
+ [
10
+ "domainWithoutSuffix",
11
+ "google"
12
+ ]
13
+ ],
14
+ [
15
+ [
16
+ "domainWithoutSuffix",
17
+ "wordpress"
18
+ ]
19
+ ],
20
+ [
21
+ [
22
+ "domainWithoutSuffix",
23
+ "apple"
24
+ ]
25
+ ],
26
+ [
27
+ [
28
+ "domainWithoutSuffix",
29
+ "microsoft"
30
+ ]
31
+ ],
32
+ [
33
+ [
34
+ "domainWithoutSuffix",
35
+ "wikipedia"
36
+ ]
37
+ ],
38
+ [
39
+ [
40
+ "domainWithoutSuffix",
41
+ "blogspot"
42
+ ]
43
+ ],
44
+ [
45
+ [
46
+ "domainWithoutSuffix",
47
+ "vimeo"
48
+ ]
49
+ ],
50
+ [
51
+ [
52
+ "domainWithoutSuffix",
53
+ "github"
54
+ ]
55
+ ],
56
+ [
57
+ [
58
+ "domainWithoutSuffix",
59
+ "imdb"
60
+ ]
61
+ ],
62
+ [
63
+ [
64
+ "domainWithoutSuffix",
65
+ "theguardian"
66
+ ]
67
+ ],
68
+ [
69
+ [
70
+ "domainWithoutSuffix",
71
+ "nytimes"
72
+ ]
73
+ ],
74
+ [
75
+ [
76
+ "domainWithoutSuffix",
77
+ "bbc"
78
+ ]
79
+ ],
80
+ [
81
+ [
82
+ "domainWithoutSuffix",
83
+ "slideshare"
84
+ ]
85
+ ],
86
+ [
87
+ [
88
+ "domainWithoutSuffix",
89
+ "pinterest"
90
+ ]
91
+ ],
92
+ [
93
+ [
94
+ "domainWithoutSuffix",
95
+ "huffingtonpost"
96
+ ]
97
+ ],
98
+ [
99
+ [
100
+ "domainWithoutSuffix",
101
+ "spotify"
102
+ ]
103
+ ],
104
+ [
105
+ [
106
+ "domainWithoutSuffix",
107
+ "soundcloud"
108
+ ]
109
+ ],
110
+ [
111
+ [
112
+ "domainWithoutSuffix",
113
+ "telegraph"
114
+ ]
115
+ ],
116
+ [
117
+ [
118
+ "domainWithoutSuffix",
119
+ "yelp"
120
+ ]
121
+ ],
122
+ [
123
+ [
124
+ "domainWithoutSuffix",
125
+ "eventbrite"
126
+ ]
127
+ ],
128
+ [
129
+ [
130
+ "domain",
131
+ "abc.net.au"
132
+ ]
133
+ ],
134
+ [
135
+ [
136
+ "domainWithoutSuffix",
137
+ "zoom"
138
+ ]
139
+ ],
140
+ [
141
+ [
142
+ "domainWithoutSuffix",
143
+ "techcrunch"
144
+ ]
145
+ ],
146
+ [
147
+ [
148
+ "domainWithoutSuffix",
149
+ "theverge"
150
+ ]
151
+ ],
152
+ [
153
+ [
154
+ "domainWithoutSuffix",
155
+ "engadget"
156
+ ]
157
+ ],
158
+ [
159
+ [
160
+ "domainWithoutSuffix",
161
+ "digg"
162
+ ]
163
+ ],
164
+ [
165
+ [
166
+ "domainWithoutSuffix",
167
+ "csdn"
168
+ ]
169
+ ],
170
+ [
171
+ [
172
+ "domainWithoutSuffix",
173
+ "deviantart"
174
+ ]
175
+ ],
176
+ [
177
+ [
178
+ "domainWithoutSuffix",
179
+ "dribbble"
180
+ ]
181
+ ],
182
+ [
183
+ [
184
+ "domainWithoutSuffix",
185
+ "etsy"
186
+ ]
187
+ ],
188
+ [
189
+ [
190
+ "domainWithoutSuffix",
191
+ "flickr"
192
+ ]
193
+ ],
194
+ [
195
+ [
196
+ "domainWithoutSuffix",
197
+ "ghost"
198
+ ]
199
+ ],
200
+ [
201
+ [
202
+ "domainWithoutSuffix",
203
+ "giphy"
204
+ ]
205
+ ],
206
+ [
207
+ [
208
+ "domainWithoutSuffix",
209
+ "gitlab"
210
+ ]
211
+ ],
212
+ [
213
+ [
214
+ "domainWithoutSuffix",
215
+ "imgur"
216
+ ]
217
+ ],
218
+ [
219
+ [
220
+ "domainWithoutSuffix",
221
+ "meetup"
222
+ ]
223
+ ],
224
+ [
225
+ [
226
+ "domainWithoutSuffix",
227
+ "producthunt"
228
+ ]
229
+ ],
230
+ [
231
+ [
232
+ "domainWithoutSuffix",
233
+ "sourceforge"
234
+ ]
235
+ ],
236
+ [
237
+ [
238
+ "domainWithoutSuffix",
239
+ "stackoverflow"
240
+ ]
241
+ ],
242
+ [
243
+ [
244
+ "domainWithoutSuffix",
245
+ "substack"
246
+ ]
247
+ ],
248
+ [
249
+ [
250
+ "domainWithoutSuffix",
251
+ "tumblr"
252
+ ]
253
+ ],
254
+ [
255
+ [
256
+ "domainWithoutSuffix",
257
+ "ycombinator"
258
+ ]
259
+ ]
44
260
  ]
package/src/index.js CHANGED
@@ -108,11 +108,6 @@ const prerender = PCancelable.fn(
108
108
  }
109
109
  )
110
110
 
111
- onCancel(() => {
112
- debug('prerender:cancel', { url })
113
- getPayload.cancel()
114
- })
115
-
116
111
  const payload = await getPayload(url, opts)
117
112
  await fetchRes.cancel()
118
113
  debug('prerender', { url, state: 'success' })
@@ -143,8 +138,12 @@ const prerender = PCancelable.fn(
143
138
 
144
139
  const modes = { fetch, prerender }
145
140
 
146
- const isFetchMode = url =>
147
- autoDomains.includes(parseUrl(url).domainWithoutSuffix)
141
+ const isFetchMode = url => {
142
+ const parsedUrl = parseUrl(url)
143
+ return autoDomains.some(conditions =>
144
+ conditions.every(([prop, value]) => parsedUrl[prop] === value)
145
+ )
146
+ }
148
147
 
149
148
  const determinateMode = (url, { prerender }) => {
150
149
  if (prerender === false || isMediaUrl(url)) return 'fetch'
@@ -215,3 +214,4 @@ module.exports = PCancelable.fn(
215
214
  )
216
215
 
217
216
  module.exports.REQ_TIMEOUT = REQ_TIMEOUT
217
+ module.exports.isFetchMode = isFetchMode