es6-crawler-detect 3.3.0 → 4.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (383) hide show
  1. package/.babelrc +1 -1
  2. package/.mocharc.json +5 -0
  3. package/README.md +1 -0
  4. package/dist/index.d.ts +5 -0
  5. package/dist/index.d.ts.map +1 -0
  6. package/dist/index.js +18 -0
  7. package/dist/index.js.map +1 -0
  8. package/dist/lib/crawler/crawlers.d.ts +5 -0
  9. package/dist/lib/crawler/crawlers.d.ts.map +1 -0
  10. package/dist/lib/crawler/crawlers.js +1400 -0
  11. package/dist/lib/crawler/crawlers.js.map +1 -0
  12. package/dist/lib/crawler/exclusions.d.ts +5 -0
  13. package/dist/lib/crawler/exclusions.d.ts.map +1 -0
  14. package/dist/lib/crawler/exclusions.js +59 -0
  15. package/dist/lib/crawler/exclusions.js.map +1 -0
  16. package/dist/lib/crawler/headers.d.ts +5 -0
  17. package/dist/lib/crawler/headers.d.ts.map +1 -0
  18. package/dist/lib/crawler/headers.js +21 -0
  19. package/dist/lib/crawler/headers.js.map +1 -0
  20. package/dist/lib/crawler/provider.d.ts +4 -0
  21. package/dist/lib/crawler/provider.d.ts.map +1 -0
  22. package/dist/lib/crawler/provider.js +3 -0
  23. package/dist/lib/crawler/provider.js.map +1 -0
  24. package/dist/lib/crawler.d.ts +21 -0
  25. package/dist/lib/crawler.d.ts.map +1 -0
  26. package/dist/lib/crawler.js +98 -0
  27. package/dist/lib/crawler.js.map +1 -0
  28. package/dist/types.d.ts +9 -0
  29. package/dist/types.d.ts.map +1 -0
  30. package/dist/types.js +3 -0
  31. package/dist/types.js.map +1 -0
  32. package/eslint.config.mjs +62 -0
  33. package/example/node/package.json +1 -1
  34. package/example/node/yarn.lock +222 -99
  35. package/package.json +30 -24
  36. package/src/index.ts +18 -0
  37. package/src/lib/crawler/{crawlers.js → crawlers.ts} +1397 -1403
  38. package/src/lib/crawler/{exclusions.js → exclusions.ts} +56 -62
  39. package/src/lib/crawler/{headers.js → headers.ts} +4 -10
  40. package/src/lib/crawler/provider.ts +3 -0
  41. package/src/lib/crawler.ts +128 -0
  42. package/src/types.ts +15 -0
  43. package/test/lib/crawler.spec.ts +116 -0
  44. package/tsconfig.json +28 -0
  45. package/.eslintignore +0 -6
  46. package/.eslintrc.json +0 -30
  47. package/dist/main.bundle.js +0 -1
  48. package/example/node/dist/crawler.min.js +0 -1
  49. package/example/node/node_modules/.bin/mime +0 -15
  50. package/example/node/node_modules/.bin/mime.cmd +0 -7
  51. package/example/node/node_modules/.yarn-integrity +0 -67
  52. package/example/node/node_modules/accepts/HISTORY.md +0 -236
  53. package/example/node/node_modules/accepts/LICENSE +0 -23
  54. package/example/node/node_modules/accepts/README.md +0 -142
  55. package/example/node/node_modules/accepts/index.js +0 -238
  56. package/example/node/node_modules/accepts/package.json +0 -47
  57. package/example/node/node_modules/array-flatten/LICENSE +0 -21
  58. package/example/node/node_modules/array-flatten/README.md +0 -43
  59. package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
  60. package/example/node/node_modules/array-flatten/package.json +0 -39
  61. package/example/node/node_modules/body-parser/HISTORY.md +0 -624
  62. package/example/node/node_modules/body-parser/LICENSE +0 -23
  63. package/example/node/node_modules/body-parser/README.md +0 -456
  64. package/example/node/node_modules/body-parser/index.js +0 -157
  65. package/example/node/node_modules/body-parser/lib/read.js +0 -181
  66. package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
  67. package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
  68. package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
  69. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
  70. package/example/node/node_modules/body-parser/package.json +0 -52
  71. package/example/node/node_modules/bytes/History.md +0 -92
  72. package/example/node/node_modules/bytes/LICENSE +0 -23
  73. package/example/node/node_modules/bytes/Readme.md +0 -152
  74. package/example/node/node_modules/bytes/index.js +0 -166
  75. package/example/node/node_modules/bytes/package.json +0 -42
  76. package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
  77. package/example/node/node_modules/content-disposition/LICENSE +0 -22
  78. package/example/node/node_modules/content-disposition/README.md +0 -142
  79. package/example/node/node_modules/content-disposition/index.js +0 -458
  80. package/example/node/node_modules/content-disposition/package.json +0 -44
  81. package/example/node/node_modules/content-type/HISTORY.md +0 -24
  82. package/example/node/node_modules/content-type/LICENSE +0 -22
  83. package/example/node/node_modules/content-type/README.md +0 -92
  84. package/example/node/node_modules/content-type/index.js +0 -222
  85. package/example/node/node_modules/content-type/package.json +0 -40
  86. package/example/node/node_modules/cookie/HISTORY.md +0 -128
  87. package/example/node/node_modules/cookie/LICENSE +0 -24
  88. package/example/node/node_modules/cookie/README.md +0 -257
  89. package/example/node/node_modules/cookie/index.js +0 -202
  90. package/example/node/node_modules/cookie/package.json +0 -40
  91. package/example/node/node_modules/cookie-signature/History.md +0 -38
  92. package/example/node/node_modules/cookie-signature/Readme.md +0 -42
  93. package/example/node/node_modules/cookie-signature/index.js +0 -51
  94. package/example/node/node_modules/cookie-signature/package.json +0 -18
  95. package/example/node/node_modules/debug/.coveralls.yml +0 -1
  96. package/example/node/node_modules/debug/.eslintrc +0 -11
  97. package/example/node/node_modules/debug/.travis.yml +0 -14
  98. package/example/node/node_modules/debug/CHANGELOG.md +0 -362
  99. package/example/node/node_modules/debug/LICENSE +0 -19
  100. package/example/node/node_modules/debug/Makefile +0 -50
  101. package/example/node/node_modules/debug/README.md +0 -312
  102. package/example/node/node_modules/debug/component.json +0 -19
  103. package/example/node/node_modules/debug/karma.conf.js +0 -70
  104. package/example/node/node_modules/debug/node.js +0 -1
  105. package/example/node/node_modules/debug/package.json +0 -49
  106. package/example/node/node_modules/debug/src/browser.js +0 -185
  107. package/example/node/node_modules/debug/src/debug.js +0 -202
  108. package/example/node/node_modules/debug/src/index.js +0 -10
  109. package/example/node/node_modules/debug/src/inspector-log.js +0 -15
  110. package/example/node/node_modules/debug/src/node.js +0 -248
  111. package/example/node/node_modules/depd/History.md +0 -96
  112. package/example/node/node_modules/depd/LICENSE +0 -22
  113. package/example/node/node_modules/depd/Readme.md +0 -280
  114. package/example/node/node_modules/depd/index.js +0 -522
  115. package/example/node/node_modules/depd/lib/browser/index.js +0 -77
  116. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
  117. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
  118. package/example/node/node_modules/depd/lib/compat/index.js +0 -79
  119. package/example/node/node_modules/depd/package.json +0 -41
  120. package/example/node/node_modules/destroy/LICENSE +0 -22
  121. package/example/node/node_modules/destroy/README.md +0 -60
  122. package/example/node/node_modules/destroy/index.js +0 -75
  123. package/example/node/node_modules/destroy/package.json +0 -37
  124. package/example/node/node_modules/ee-first/LICENSE +0 -22
  125. package/example/node/node_modules/ee-first/README.md +0 -80
  126. package/example/node/node_modules/ee-first/index.js +0 -95
  127. package/example/node/node_modules/ee-first/package.json +0 -29
  128. package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
  129. package/example/node/node_modules/encodeurl/LICENSE +0 -22
  130. package/example/node/node_modules/encodeurl/README.md +0 -128
  131. package/example/node/node_modules/encodeurl/index.js +0 -60
  132. package/example/node/node_modules/encodeurl/package.json +0 -40
  133. package/example/node/node_modules/escape-html/LICENSE +0 -24
  134. package/example/node/node_modules/escape-html/Readme.md +0 -43
  135. package/example/node/node_modules/escape-html/index.js +0 -78
  136. package/example/node/node_modules/escape-html/package.json +0 -24
  137. package/example/node/node_modules/etag/HISTORY.md +0 -83
  138. package/example/node/node_modules/etag/LICENSE +0 -22
  139. package/example/node/node_modules/etag/README.md +0 -159
  140. package/example/node/node_modules/etag/index.js +0 -131
  141. package/example/node/node_modules/etag/package.json +0 -47
  142. package/example/node/node_modules/express/History.md +0 -3510
  143. package/example/node/node_modules/express/LICENSE +0 -24
  144. package/example/node/node_modules/express/Readme.md +0 -158
  145. package/example/node/node_modules/express/index.js +0 -11
  146. package/example/node/node_modules/express/lib/application.js +0 -644
  147. package/example/node/node_modules/express/lib/express.js +0 -116
  148. package/example/node/node_modules/express/lib/middleware/init.js +0 -43
  149. package/example/node/node_modules/express/lib/middleware/query.js +0 -47
  150. package/example/node/node_modules/express/lib/request.js +0 -525
  151. package/example/node/node_modules/express/lib/response.js +0 -1147
  152. package/example/node/node_modules/express/lib/router/index.js +0 -668
  153. package/example/node/node_modules/express/lib/router/layer.js +0 -181
  154. package/example/node/node_modules/express/lib/router/route.js +0 -216
  155. package/example/node/node_modules/express/lib/utils.js +0 -302
  156. package/example/node/node_modules/express/lib/view.js +0 -182
  157. package/example/node/node_modules/express/package.json +0 -99
  158. package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
  159. package/example/node/node_modules/finalhandler/LICENSE +0 -22
  160. package/example/node/node_modules/finalhandler/README.md +0 -148
  161. package/example/node/node_modules/finalhandler/index.js +0 -331
  162. package/example/node/node_modules/finalhandler/package.json +0 -45
  163. package/example/node/node_modules/forwarded/HISTORY.md +0 -21
  164. package/example/node/node_modules/forwarded/LICENSE +0 -22
  165. package/example/node/node_modules/forwarded/README.md +0 -57
  166. package/example/node/node_modules/forwarded/index.js +0 -90
  167. package/example/node/node_modules/forwarded/package.json +0 -45
  168. package/example/node/node_modules/fresh/HISTORY.md +0 -70
  169. package/example/node/node_modules/fresh/LICENSE +0 -23
  170. package/example/node/node_modules/fresh/README.md +0 -119
  171. package/example/node/node_modules/fresh/index.js +0 -137
  172. package/example/node/node_modules/fresh/package.json +0 -46
  173. package/example/node/node_modules/http-errors/HISTORY.md +0 -165
  174. package/example/node/node_modules/http-errors/LICENSE +0 -23
  175. package/example/node/node_modules/http-errors/README.md +0 -169
  176. package/example/node/node_modules/http-errors/index.js +0 -299
  177. package/example/node/node_modules/http-errors/package.json +0 -49
  178. package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
  179. package/example/node/node_modules/iconv-lite/LICENSE +0 -21
  180. package/example/node/node_modules/iconv-lite/README.md +0 -156
  181. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
  182. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
  183. package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
  184. package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
  185. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
  186. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
  187. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
  188. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
  189. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
  190. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
  191. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
  192. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
  193. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
  194. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
  195. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
  196. package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
  197. package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
  198. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
  199. package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
  200. package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
  201. package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
  202. package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
  203. package/example/node/node_modules/iconv-lite/package.json +0 -46
  204. package/example/node/node_modules/inherits/LICENSE +0 -16
  205. package/example/node/node_modules/inherits/README.md +0 -42
  206. package/example/node/node_modules/inherits/inherits.js +0 -9
  207. package/example/node/node_modules/inherits/inherits_browser.js +0 -27
  208. package/example/node/node_modules/inherits/package.json +0 -29
  209. package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
  210. package/example/node/node_modules/ipaddr.js/README.md +0 -233
  211. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
  212. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
  213. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
  214. package/example/node/node_modules/ipaddr.js/package.json +0 -35
  215. package/example/node/node_modules/media-typer/HISTORY.md +0 -22
  216. package/example/node/node_modules/media-typer/LICENSE +0 -22
  217. package/example/node/node_modules/media-typer/README.md +0 -81
  218. package/example/node/node_modules/media-typer/index.js +0 -270
  219. package/example/node/node_modules/media-typer/package.json +0 -26
  220. package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
  221. package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
  222. package/example/node/node_modules/merge-descriptors/README.md +0 -48
  223. package/example/node/node_modules/merge-descriptors/index.js +0 -60
  224. package/example/node/node_modules/merge-descriptors/package.json +0 -32
  225. package/example/node/node_modules/methods/HISTORY.md +0 -29
  226. package/example/node/node_modules/methods/LICENSE +0 -24
  227. package/example/node/node_modules/methods/README.md +0 -51
  228. package/example/node/node_modules/methods/index.js +0 -69
  229. package/example/node/node_modules/methods/package.json +0 -36
  230. package/example/node/node_modules/mime/CHANGELOG.md +0 -164
  231. package/example/node/node_modules/mime/LICENSE +0 -21
  232. package/example/node/node_modules/mime/README.md +0 -90
  233. package/example/node/node_modules/mime/cli.js +0 -8
  234. package/example/node/node_modules/mime/mime.js +0 -108
  235. package/example/node/node_modules/mime/package.json +0 -44
  236. package/example/node/node_modules/mime/src/build.js +0 -53
  237. package/example/node/node_modules/mime/src/test.js +0 -60
  238. package/example/node/node_modules/mime/types.json +0 -1
  239. package/example/node/node_modules/mime-db/HISTORY.md +0 -499
  240. package/example/node/node_modules/mime-db/LICENSE +0 -22
  241. package/example/node/node_modules/mime-db/README.md +0 -100
  242. package/example/node/node_modules/mime-db/db.json +0 -8471
  243. package/example/node/node_modules/mime-db/index.js +0 -11
  244. package/example/node/node_modules/mime-db/package.json +0 -59
  245. package/example/node/node_modules/mime-types/HISTORY.md +0 -388
  246. package/example/node/node_modules/mime-types/LICENSE +0 -23
  247. package/example/node/node_modules/mime-types/README.md +0 -113
  248. package/example/node/node_modules/mime-types/index.js +0 -188
  249. package/example/node/node_modules/mime-types/package.json +0 -44
  250. package/example/node/node_modules/ms/index.js +0 -152
  251. package/example/node/node_modules/ms/license.md +0 -21
  252. package/example/node/node_modules/ms/package.json +0 -37
  253. package/example/node/node_modules/ms/readme.md +0 -51
  254. package/example/node/node_modules/negotiator/HISTORY.md +0 -103
  255. package/example/node/node_modules/negotiator/LICENSE +0 -24
  256. package/example/node/node_modules/negotiator/README.md +0 -203
  257. package/example/node/node_modules/negotiator/index.js +0 -124
  258. package/example/node/node_modules/negotiator/lib/charset.js +0 -169
  259. package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
  260. package/example/node/node_modules/negotiator/lib/language.js +0 -179
  261. package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
  262. package/example/node/node_modules/negotiator/package.json +0 -42
  263. package/example/node/node_modules/on-finished/HISTORY.md +0 -88
  264. package/example/node/node_modules/on-finished/LICENSE +0 -23
  265. package/example/node/node_modules/on-finished/README.md +0 -154
  266. package/example/node/node_modules/on-finished/index.js +0 -196
  267. package/example/node/node_modules/on-finished/package.json +0 -31
  268. package/example/node/node_modules/parseurl/HISTORY.md +0 -58
  269. package/example/node/node_modules/parseurl/LICENSE +0 -24
  270. package/example/node/node_modules/parseurl/README.md +0 -133
  271. package/example/node/node_modules/parseurl/index.js +0 -158
  272. package/example/node/node_modules/parseurl/package.json +0 -40
  273. package/example/node/node_modules/path-to-regexp/History.md +0 -36
  274. package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
  275. package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
  276. package/example/node/node_modules/path-to-regexp/index.js +0 -129
  277. package/example/node/node_modules/path-to-regexp/package.json +0 -30
  278. package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
  279. package/example/node/node_modules/proxy-addr/LICENSE +0 -22
  280. package/example/node/node_modules/proxy-addr/README.md +0 -139
  281. package/example/node/node_modules/proxy-addr/index.js +0 -327
  282. package/example/node/node_modules/proxy-addr/package.json +0 -47
  283. package/example/node/node_modules/qs/.editorconfig +0 -39
  284. package/example/node/node_modules/qs/.eslintignore +0 -2
  285. package/example/node/node_modules/qs/.eslintrc +0 -35
  286. package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
  287. package/example/node/node_modules/qs/.nycrc +0 -13
  288. package/example/node/node_modules/qs/CHANGELOG.md +0 -359
  289. package/example/node/node_modules/qs/LICENSE.md +0 -29
  290. package/example/node/node_modules/qs/README.md +0 -604
  291. package/example/node/node_modules/qs/dist/qs.js +0 -832
  292. package/example/node/node_modules/qs/lib/formats.js +0 -23
  293. package/example/node/node_modules/qs/lib/index.js +0 -11
  294. package/example/node/node_modules/qs/lib/parse.js +0 -257
  295. package/example/node/node_modules/qs/lib/stringify.js +0 -278
  296. package/example/node/node_modules/qs/lib/utils.js +0 -251
  297. package/example/node/node_modules/qs/package.json +0 -69
  298. package/example/node/node_modules/qs/test/parse.js +0 -772
  299. package/example/node/node_modules/qs/test/stringify.js +0 -793
  300. package/example/node/node_modules/qs/test/utils.js +0 -136
  301. package/example/node/node_modules/range-parser/HISTORY.md +0 -56
  302. package/example/node/node_modules/range-parser/LICENSE +0 -23
  303. package/example/node/node_modules/range-parser/README.md +0 -84
  304. package/example/node/node_modules/range-parser/index.js +0 -162
  305. package/example/node/node_modules/range-parser/package.json +0 -44
  306. package/example/node/node_modules/raw-body/HISTORY.md +0 -284
  307. package/example/node/node_modules/raw-body/LICENSE +0 -22
  308. package/example/node/node_modules/raw-body/README.md +0 -217
  309. package/example/node/node_modules/raw-body/index.d.ts +0 -87
  310. package/example/node/node_modules/raw-body/index.js +0 -286
  311. package/example/node/node_modules/raw-body/package.json +0 -48
  312. package/example/node/node_modules/safe-buffer/LICENSE +0 -21
  313. package/example/node/node_modules/safe-buffer/README.md +0 -584
  314. package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
  315. package/example/node/node_modules/safe-buffer/index.js +0 -65
  316. package/example/node/node_modules/safe-buffer/package.json +0 -51
  317. package/example/node/node_modules/safer-buffer/LICENSE +0 -21
  318. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
  319. package/example/node/node_modules/safer-buffer/Readme.md +0 -156
  320. package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
  321. package/example/node/node_modules/safer-buffer/package.json +0 -34
  322. package/example/node/node_modules/safer-buffer/safer.js +0 -77
  323. package/example/node/node_modules/safer-buffer/tests.js +0 -406
  324. package/example/node/node_modules/send/HISTORY.md +0 -506
  325. package/example/node/node_modules/send/LICENSE +0 -23
  326. package/example/node/node_modules/send/README.md +0 -327
  327. package/example/node/node_modules/send/index.js +0 -1133
  328. package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
  329. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
  330. package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
  331. package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
  332. package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
  333. package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
  334. package/example/node/node_modules/send/package.json +0 -61
  335. package/example/node/node_modules/serve-static/HISTORY.md +0 -459
  336. package/example/node/node_modules/serve-static/LICENSE +0 -25
  337. package/example/node/node_modules/serve-static/README.md +0 -257
  338. package/example/node/node_modules/serve-static/index.js +0 -210
  339. package/example/node/node_modules/serve-static/package.json +0 -42
  340. package/example/node/node_modules/setprototypeof/LICENSE +0 -13
  341. package/example/node/node_modules/setprototypeof/README.md +0 -31
  342. package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
  343. package/example/node/node_modules/setprototypeof/index.js +0 -17
  344. package/example/node/node_modules/setprototypeof/package.json +0 -38
  345. package/example/node/node_modules/setprototypeof/test/index.js +0 -24
  346. package/example/node/node_modules/statuses/HISTORY.md +0 -65
  347. package/example/node/node_modules/statuses/LICENSE +0 -23
  348. package/example/node/node_modules/statuses/README.md +0 -127
  349. package/example/node/node_modules/statuses/codes.json +0 -66
  350. package/example/node/node_modules/statuses/index.js +0 -113
  351. package/example/node/node_modules/statuses/package.json +0 -48
  352. package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
  353. package/example/node/node_modules/toidentifier/LICENSE +0 -21
  354. package/example/node/node_modules/toidentifier/README.md +0 -61
  355. package/example/node/node_modules/toidentifier/index.js +0 -32
  356. package/example/node/node_modules/toidentifier/package.json +0 -38
  357. package/example/node/node_modules/type-is/HISTORY.md +0 -259
  358. package/example/node/node_modules/type-is/LICENSE +0 -23
  359. package/example/node/node_modules/type-is/README.md +0 -170
  360. package/example/node/node_modules/type-is/index.js +0 -266
  361. package/example/node/node_modules/type-is/package.json +0 -45
  362. package/example/node/node_modules/unpipe/HISTORY.md +0 -4
  363. package/example/node/node_modules/unpipe/LICENSE +0 -22
  364. package/example/node/node_modules/unpipe/README.md +0 -43
  365. package/example/node/node_modules/unpipe/index.js +0 -69
  366. package/example/node/node_modules/unpipe/package.json +0 -27
  367. package/example/node/node_modules/utils-merge/LICENSE +0 -20
  368. package/example/node/node_modules/utils-merge/README.md +0 -34
  369. package/example/node/node_modules/utils-merge/index.js +0 -23
  370. package/example/node/node_modules/utils-merge/package.json +0 -40
  371. package/example/node/node_modules/vary/HISTORY.md +0 -39
  372. package/example/node/node_modules/vary/LICENSE +0 -22
  373. package/example/node/node_modules/vary/README.md +0 -101
  374. package/example/node/node_modules/vary/index.js +0 -149
  375. package/example/node/node_modules/vary/package.json +0 -43
  376. package/example/node/server.js +0 -38
  377. package/src/index.js +0 -16
  378. package/src/lib/crawler/provider.js +0 -11
  379. package/src/lib/crawler.js +0 -130
  380. package/test/lib/crawler.test.js +0 -94
  381. package/webpack.common.js +0 -20
  382. package/webpack.dev.js +0 -7
  383. package/webpack.prod.js +0 -14
@@ -1,23 +0,0 @@
1
- 'use strict';
2
-
3
- var replace = String.prototype.replace;
4
- var percentTwenties = /%20/g;
5
-
6
- var Format = {
7
- RFC1738: 'RFC1738',
8
- RFC3986: 'RFC3986'
9
- };
10
-
11
- module.exports = {
12
- 'default': Format.RFC3986,
13
- formatters: {
14
- RFC1738: function (value) {
15
- return replace.call(value, percentTwenties, '+');
16
- },
17
- RFC3986: function (value) {
18
- return String(value);
19
- }
20
- },
21
- RFC1738: Format.RFC1738,
22
- RFC3986: Format.RFC3986
23
- };
@@ -1,11 +0,0 @@
1
- 'use strict';
2
-
3
- var stringify = require('./stringify');
4
- var parse = require('./parse');
5
- var formats = require('./formats');
6
-
7
- module.exports = {
8
- formats: formats,
9
- parse: parse,
10
- stringify: stringify
11
- };
@@ -1,257 +0,0 @@
1
- 'use strict';
2
-
3
- var utils = require('./utils');
4
-
5
- var has = Object.prototype.hasOwnProperty;
6
- var isArray = Array.isArray;
7
-
8
- var defaults = {
9
- allowDots: false,
10
- allowPrototypes: false,
11
- arrayLimit: 20,
12
- charset: 'utf-8',
13
- charsetSentinel: false,
14
- comma: false,
15
- decoder: utils.decode,
16
- delimiter: '&',
17
- depth: 5,
18
- ignoreQueryPrefix: false,
19
- interpretNumericEntities: false,
20
- parameterLimit: 1000,
21
- parseArrays: true,
22
- plainObjects: false,
23
- strictNullHandling: false
24
- };
25
-
26
- var interpretNumericEntities = function (str) {
27
- return str.replace(/&#(\d+);/g, function ($0, numberStr) {
28
- return String.fromCharCode(parseInt(numberStr, 10));
29
- });
30
- };
31
-
32
- var parseArrayValue = function (val, options) {
33
- if (val && typeof val === 'string' && options.comma && val.indexOf(',') > -1) {
34
- return val.split(',');
35
- }
36
-
37
- return val;
38
- };
39
-
40
- // This is what browsers will submit when the ✓ character occurs in an
41
- // application/x-www-form-urlencoded body and the encoding of the page containing
42
- // the form is iso-8859-1, or when the submitted form has an accept-charset
43
- // attribute of iso-8859-1. Presumably also with other charsets that do not contain
44
- // the ✓ character, such as us-ascii.
45
- var isoSentinel = 'utf8=%26%2310003%3B'; // encodeURIComponent('✓')
46
-
47
- // These are the percent-encoded utf-8 octets representing a checkmark, indicating that the request actually is utf-8 encoded.
48
- var charsetSentinel = 'utf8=%E2%9C%93'; // encodeURIComponent('✓')
49
-
50
- var parseValues = function parseQueryStringValues(str, options) {
51
- var obj = {};
52
- var cleanStr = options.ignoreQueryPrefix ? str.replace(/^\?/, '') : str;
53
- var limit = options.parameterLimit === Infinity ? undefined : options.parameterLimit;
54
- var parts = cleanStr.split(options.delimiter, limit);
55
- var skipIndex = -1; // Keep track of where the utf8 sentinel was found
56
- var i;
57
-
58
- var charset = options.charset;
59
- if (options.charsetSentinel) {
60
- for (i = 0; i < parts.length; ++i) {
61
- if (parts[i].indexOf('utf8=') === 0) {
62
- if (parts[i] === charsetSentinel) {
63
- charset = 'utf-8';
64
- } else if (parts[i] === isoSentinel) {
65
- charset = 'iso-8859-1';
66
- }
67
- skipIndex = i;
68
- i = parts.length; // The eslint settings do not allow break;
69
- }
70
- }
71
- }
72
-
73
- for (i = 0; i < parts.length; ++i) {
74
- if (i === skipIndex) {
75
- continue;
76
- }
77
- var part = parts[i];
78
-
79
- var bracketEqualsPos = part.indexOf(']=');
80
- var pos = bracketEqualsPos === -1 ? part.indexOf('=') : bracketEqualsPos + 1;
81
-
82
- var key, val;
83
- if (pos === -1) {
84
- key = options.decoder(part, defaults.decoder, charset, 'key');
85
- val = options.strictNullHandling ? null : '';
86
- } else {
87
- key = options.decoder(part.slice(0, pos), defaults.decoder, charset, 'key');
88
- val = utils.maybeMap(
89
- parseArrayValue(part.slice(pos + 1), options),
90
- function (encodedVal) {
91
- return options.decoder(encodedVal, defaults.decoder, charset, 'value');
92
- }
93
- );
94
- }
95
-
96
- if (val && options.interpretNumericEntities && charset === 'iso-8859-1') {
97
- val = interpretNumericEntities(val);
98
- }
99
-
100
- if (part.indexOf('[]=') > -1) {
101
- val = isArray(val) ? [val] : val;
102
- }
103
-
104
- if (has.call(obj, key)) {
105
- obj[key] = utils.combine(obj[key], val);
106
- } else {
107
- obj[key] = val;
108
- }
109
- }
110
-
111
- return obj;
112
- };
113
-
114
- var parseObject = function (chain, val, options, valuesParsed) {
115
- var leaf = valuesParsed ? val : parseArrayValue(val, options);
116
-
117
- for (var i = chain.length - 1; i >= 0; --i) {
118
- var obj;
119
- var root = chain[i];
120
-
121
- if (root === '[]' && options.parseArrays) {
122
- obj = [].concat(leaf);
123
- } else {
124
- obj = options.plainObjects ? Object.create(null) : {};
125
- var cleanRoot = root.charAt(0) === '[' && root.charAt(root.length - 1) === ']' ? root.slice(1, -1) : root;
126
- var index = parseInt(cleanRoot, 10);
127
- if (!options.parseArrays && cleanRoot === '') {
128
- obj = { 0: leaf };
129
- } else if (
130
- !isNaN(index)
131
- && root !== cleanRoot
132
- && String(index) === cleanRoot
133
- && index >= 0
134
- && (options.parseArrays && index <= options.arrayLimit)
135
- ) {
136
- obj = [];
137
- obj[index] = leaf;
138
- } else {
139
- obj[cleanRoot] = leaf;
140
- }
141
- }
142
-
143
- leaf = obj;
144
- }
145
-
146
- return leaf;
147
- };
148
-
149
- var parseKeys = function parseQueryStringKeys(givenKey, val, options, valuesParsed) {
150
- if (!givenKey) {
151
- return;
152
- }
153
-
154
- // Transform dot notation to bracket notation
155
- var key = options.allowDots ? givenKey.replace(/\.([^.[]+)/g, '[$1]') : givenKey;
156
-
157
- // The regex chunks
158
-
159
- var brackets = /(\[[^[\]]*])/;
160
- var child = /(\[[^[\]]*])/g;
161
-
162
- // Get the parent
163
-
164
- var segment = options.depth > 0 && brackets.exec(key);
165
- var parent = segment ? key.slice(0, segment.index) : key;
166
-
167
- // Stash the parent if it exists
168
-
169
- var keys = [];
170
- if (parent) {
171
- // If we aren't using plain objects, optionally prefix keys that would overwrite object prototype properties
172
- if (!options.plainObjects && has.call(Object.prototype, parent)) {
173
- if (!options.allowPrototypes) {
174
- return;
175
- }
176
- }
177
-
178
- keys.push(parent);
179
- }
180
-
181
- // Loop through children appending to the array until we hit depth
182
-
183
- var i = 0;
184
- while (options.depth > 0 && (segment = child.exec(key)) !== null && i < options.depth) {
185
- i += 1;
186
- if (!options.plainObjects && has.call(Object.prototype, segment[1].slice(1, -1))) {
187
- if (!options.allowPrototypes) {
188
- return;
189
- }
190
- }
191
- keys.push(segment[1]);
192
- }
193
-
194
- // If there's a remainder, just add whatever is left
195
-
196
- if (segment) {
197
- keys.push('[' + key.slice(segment.index) + ']');
198
- }
199
-
200
- return parseObject(keys, val, options, valuesParsed);
201
- };
202
-
203
- var normalizeParseOptions = function normalizeParseOptions(opts) {
204
- if (!opts) {
205
- return defaults;
206
- }
207
-
208
- if (opts.decoder !== null && opts.decoder !== undefined && typeof opts.decoder !== 'function') {
209
- throw new TypeError('Decoder has to be a function.');
210
- }
211
-
212
- if (typeof opts.charset !== 'undefined' && opts.charset !== 'utf-8' && opts.charset !== 'iso-8859-1') {
213
- throw new TypeError('The charset option must be either utf-8, iso-8859-1, or undefined');
214
- }
215
- var charset = typeof opts.charset === 'undefined' ? defaults.charset : opts.charset;
216
-
217
- return {
218
- allowDots: typeof opts.allowDots === 'undefined' ? defaults.allowDots : !!opts.allowDots,
219
- allowPrototypes: typeof opts.allowPrototypes === 'boolean' ? opts.allowPrototypes : defaults.allowPrototypes,
220
- arrayLimit: typeof opts.arrayLimit === 'number' ? opts.arrayLimit : defaults.arrayLimit,
221
- charset: charset,
222
- charsetSentinel: typeof opts.charsetSentinel === 'boolean' ? opts.charsetSentinel : defaults.charsetSentinel,
223
- comma: typeof opts.comma === 'boolean' ? opts.comma : defaults.comma,
224
- decoder: typeof opts.decoder === 'function' ? opts.decoder : defaults.decoder,
225
- delimiter: typeof opts.delimiter === 'string' || utils.isRegExp(opts.delimiter) ? opts.delimiter : defaults.delimiter,
226
- // eslint-disable-next-line no-implicit-coercion, no-extra-parens
227
- depth: (typeof opts.depth === 'number' || opts.depth === false) ? +opts.depth : defaults.depth,
228
- ignoreQueryPrefix: opts.ignoreQueryPrefix === true,
229
- interpretNumericEntities: typeof opts.interpretNumericEntities === 'boolean' ? opts.interpretNumericEntities : defaults.interpretNumericEntities,
230
- parameterLimit: typeof opts.parameterLimit === 'number' ? opts.parameterLimit : defaults.parameterLimit,
231
- parseArrays: opts.parseArrays !== false,
232
- plainObjects: typeof opts.plainObjects === 'boolean' ? opts.plainObjects : defaults.plainObjects,
233
- strictNullHandling: typeof opts.strictNullHandling === 'boolean' ? opts.strictNullHandling : defaults.strictNullHandling
234
- };
235
- };
236
-
237
- module.exports = function (str, opts) {
238
- var options = normalizeParseOptions(opts);
239
-
240
- if (str === '' || str === null || typeof str === 'undefined') {
241
- return options.plainObjects ? Object.create(null) : {};
242
- }
243
-
244
- var tempObj = typeof str === 'string' ? parseValues(str, options) : str;
245
- var obj = options.plainObjects ? Object.create(null) : {};
246
-
247
- // Iterate over the keys and setup the new object
248
-
249
- var keys = Object.keys(tempObj);
250
- for (var i = 0; i < keys.length; ++i) {
251
- var key = keys[i];
252
- var newObj = parseKeys(key, tempObj[key], options, typeof str === 'string');
253
- obj = utils.merge(obj, newObj, options);
254
- }
255
-
256
- return utils.compact(obj);
257
- };
@@ -1,278 +0,0 @@
1
- 'use strict';
2
-
3
- var utils = require('./utils');
4
- var formats = require('./formats');
5
- var has = Object.prototype.hasOwnProperty;
6
-
7
- var arrayPrefixGenerators = {
8
- brackets: function brackets(prefix) {
9
- return prefix + '[]';
10
- },
11
- comma: 'comma',
12
- indices: function indices(prefix, key) {
13
- return prefix + '[' + key + ']';
14
- },
15
- repeat: function repeat(prefix) {
16
- return prefix;
17
- }
18
- };
19
-
20
- var isArray = Array.isArray;
21
- var push = Array.prototype.push;
22
- var pushToArray = function (arr, valueOrArray) {
23
- push.apply(arr, isArray(valueOrArray) ? valueOrArray : [valueOrArray]);
24
- };
25
-
26
- var toISO = Date.prototype.toISOString;
27
-
28
- var defaultFormat = formats['default'];
29
- var defaults = {
30
- addQueryPrefix: false,
31
- allowDots: false,
32
- charset: 'utf-8',
33
- charsetSentinel: false,
34
- delimiter: '&',
35
- encode: true,
36
- encoder: utils.encode,
37
- encodeValuesOnly: false,
38
- format: defaultFormat,
39
- formatter: formats.formatters[defaultFormat],
40
- // deprecated
41
- indices: false,
42
- serializeDate: function serializeDate(date) {
43
- return toISO.call(date);
44
- },
45
- skipNulls: false,
46
- strictNullHandling: false
47
- };
48
-
49
- var isNonNullishPrimitive = function isNonNullishPrimitive(v) {
50
- return typeof v === 'string'
51
- || typeof v === 'number'
52
- || typeof v === 'boolean'
53
- || typeof v === 'symbol'
54
- || typeof v === 'bigint';
55
- };
56
-
57
- var stringify = function stringify(
58
- object,
59
- prefix,
60
- generateArrayPrefix,
61
- strictNullHandling,
62
- skipNulls,
63
- encoder,
64
- filter,
65
- sort,
66
- allowDots,
67
- serializeDate,
68
- format,
69
- formatter,
70
- encodeValuesOnly,
71
- charset
72
- ) {
73
- var obj = object;
74
- if (typeof filter === 'function') {
75
- obj = filter(prefix, obj);
76
- } else if (obj instanceof Date) {
77
- obj = serializeDate(obj);
78
- } else if (generateArrayPrefix === 'comma' && isArray(obj)) {
79
- obj = utils.maybeMap(obj, function (value) {
80
- if (value instanceof Date) {
81
- return serializeDate(value);
82
- }
83
- return value;
84
- });
85
- }
86
-
87
- if (obj === null) {
88
- if (strictNullHandling) {
89
- return encoder && !encodeValuesOnly ? encoder(prefix, defaults.encoder, charset, 'key', format) : prefix;
90
- }
91
-
92
- obj = '';
93
- }
94
-
95
- if (isNonNullishPrimitive(obj) || utils.isBuffer(obj)) {
96
- if (encoder) {
97
- var keyValue = encodeValuesOnly ? prefix : encoder(prefix, defaults.encoder, charset, 'key', format);
98
- return [formatter(keyValue) + '=' + formatter(encoder(obj, defaults.encoder, charset, 'value', format))];
99
- }
100
- return [formatter(prefix) + '=' + formatter(String(obj))];
101
- }
102
-
103
- var values = [];
104
-
105
- if (typeof obj === 'undefined') {
106
- return values;
107
- }
108
-
109
- var objKeys;
110
- if (generateArrayPrefix === 'comma' && isArray(obj)) {
111
- // we need to join elements in
112
- objKeys = [{ value: obj.length > 0 ? obj.join(',') || null : undefined }];
113
- } else if (isArray(filter)) {
114
- objKeys = filter;
115
- } else {
116
- var keys = Object.keys(obj);
117
- objKeys = sort ? keys.sort(sort) : keys;
118
- }
119
-
120
- for (var i = 0; i < objKeys.length; ++i) {
121
- var key = objKeys[i];
122
- var value = typeof key === 'object' && key.value !== undefined ? key.value : obj[key];
123
-
124
- if (skipNulls && value === null) {
125
- continue;
126
- }
127
-
128
- var keyPrefix = isArray(obj)
129
- ? typeof generateArrayPrefix === 'function' ? generateArrayPrefix(prefix, key) : prefix
130
- : prefix + (allowDots ? '.' + key : '[' + key + ']');
131
-
132
- pushToArray(values, stringify(
133
- value,
134
- keyPrefix,
135
- generateArrayPrefix,
136
- strictNullHandling,
137
- skipNulls,
138
- encoder,
139
- filter,
140
- sort,
141
- allowDots,
142
- serializeDate,
143
- format,
144
- formatter,
145
- encodeValuesOnly,
146
- charset
147
- ));
148
- }
149
-
150
- return values;
151
- };
152
-
153
- var normalizeStringifyOptions = function normalizeStringifyOptions(opts) {
154
- if (!opts) {
155
- return defaults;
156
- }
157
-
158
- if (opts.encoder !== null && opts.encoder !== undefined && typeof opts.encoder !== 'function') {
159
- throw new TypeError('Encoder has to be a function.');
160
- }
161
-
162
- var charset = opts.charset || defaults.charset;
163
- if (typeof opts.charset !== 'undefined' && opts.charset !== 'utf-8' && opts.charset !== 'iso-8859-1') {
164
- throw new TypeError('The charset option must be either utf-8, iso-8859-1, or undefined');
165
- }
166
-
167
- var format = formats['default'];
168
- if (typeof opts.format !== 'undefined') {
169
- if (!has.call(formats.formatters, opts.format)) {
170
- throw new TypeError('Unknown format option provided.');
171
- }
172
- format = opts.format;
173
- }
174
- var formatter = formats.formatters[format];
175
-
176
- var filter = defaults.filter;
177
- if (typeof opts.filter === 'function' || isArray(opts.filter)) {
178
- filter = opts.filter;
179
- }
180
-
181
- return {
182
- addQueryPrefix: typeof opts.addQueryPrefix === 'boolean' ? opts.addQueryPrefix : defaults.addQueryPrefix,
183
- allowDots: typeof opts.allowDots === 'undefined' ? defaults.allowDots : !!opts.allowDots,
184
- charset: charset,
185
- charsetSentinel: typeof opts.charsetSentinel === 'boolean' ? opts.charsetSentinel : defaults.charsetSentinel,
186
- delimiter: typeof opts.delimiter === 'undefined' ? defaults.delimiter : opts.delimiter,
187
- encode: typeof opts.encode === 'boolean' ? opts.encode : defaults.encode,
188
- encoder: typeof opts.encoder === 'function' ? opts.encoder : defaults.encoder,
189
- encodeValuesOnly: typeof opts.encodeValuesOnly === 'boolean' ? opts.encodeValuesOnly : defaults.encodeValuesOnly,
190
- filter: filter,
191
- format: format,
192
- formatter: formatter,
193
- serializeDate: typeof opts.serializeDate === 'function' ? opts.serializeDate : defaults.serializeDate,
194
- skipNulls: typeof opts.skipNulls === 'boolean' ? opts.skipNulls : defaults.skipNulls,
195
- sort: typeof opts.sort === 'function' ? opts.sort : null,
196
- strictNullHandling: typeof opts.strictNullHandling === 'boolean' ? opts.strictNullHandling : defaults.strictNullHandling
197
- };
198
- };
199
-
200
- module.exports = function (object, opts) {
201
- var obj = object;
202
- var options = normalizeStringifyOptions(opts);
203
-
204
- var objKeys;
205
- var filter;
206
-
207
- if (typeof options.filter === 'function') {
208
- filter = options.filter;
209
- obj = filter('', obj);
210
- } else if (isArray(options.filter)) {
211
- filter = options.filter;
212
- objKeys = filter;
213
- }
214
-
215
- var keys = [];
216
-
217
- if (typeof obj !== 'object' || obj === null) {
218
- return '';
219
- }
220
-
221
- var arrayFormat;
222
- if (opts && opts.arrayFormat in arrayPrefixGenerators) {
223
- arrayFormat = opts.arrayFormat;
224
- } else if (opts && 'indices' in opts) {
225
- arrayFormat = opts.indices ? 'indices' : 'repeat';
226
- } else {
227
- arrayFormat = 'indices';
228
- }
229
-
230
- var generateArrayPrefix = arrayPrefixGenerators[arrayFormat];
231
-
232
- if (!objKeys) {
233
- objKeys = Object.keys(obj);
234
- }
235
-
236
- if (options.sort) {
237
- objKeys.sort(options.sort);
238
- }
239
-
240
- for (var i = 0; i < objKeys.length; ++i) {
241
- var key = objKeys[i];
242
-
243
- if (options.skipNulls && obj[key] === null) {
244
- continue;
245
- }
246
- pushToArray(keys, stringify(
247
- obj[key],
248
- key,
249
- generateArrayPrefix,
250
- options.strictNullHandling,
251
- options.skipNulls,
252
- options.encode ? options.encoder : null,
253
- options.filter,
254
- options.sort,
255
- options.allowDots,
256
- options.serializeDate,
257
- options.format,
258
- options.formatter,
259
- options.encodeValuesOnly,
260
- options.charset
261
- ));
262
- }
263
-
264
- var joined = keys.join(options.delimiter);
265
- var prefix = options.addQueryPrefix === true ? '?' : '';
266
-
267
- if (options.charsetSentinel) {
268
- if (options.charset === 'iso-8859-1') {
269
- // encodeURIComponent('&#10003;'), the "numeric entity" representation of a checkmark
270
- prefix += 'utf8=%26%2310003%3B&';
271
- } else {
272
- // encodeURIComponent('✓')
273
- prefix += 'utf8=%E2%9C%93&';
274
- }
275
- }
276
-
277
- return joined.length > 0 ? prefix + joined : '';
278
- };