es6-crawler-detect 3.1.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +27 -15
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +29 -18
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1399 -11
  344. package/src/lib/crawler/exclusions.js +56 -9
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +123 -132
  348. package/test/lib/crawler.test.js +83 -24
  349. package/test/lib/database/crawlers.txt +3651 -0
  350. package/test/lib/database/devices.txt +165636 -0
  351. package/webpack.common.js +20 -20
  352. package/webpack.dev.js +7 -7
  353. package/webpack.prod.js +14 -15
@@ -0,0 +1,259 @@
1
+ 1.6.18 / 2019-04-26
2
+ ===================
3
+
4
+ * Fix regression passing request object to `typeis.is`
5
+
6
+ 1.6.17 / 2019-04-25
7
+ ===================
8
+
9
+ * deps: mime-types@~2.1.24
10
+ - Add Apple file extensions from IANA
11
+ - Add extension `.csl` to `application/vnd.citationstyles.style+xml`
12
+ - Add extension `.es` to `application/ecmascript`
13
+ - Add extension `.nq` to `application/n-quads`
14
+ - Add extension `.nt` to `application/n-triples`
15
+ - Add extension `.owl` to `application/rdf+xml`
16
+ - Add extensions `.siv` and `.sieve` to `application/sieve`
17
+ - Add extensions from IANA for `image/*` types
18
+ - Add extensions from IANA for `model/*` types
19
+ - Add extensions to HEIC image types
20
+ - Add new mime types
21
+ - Add `text/mdx` with extension `.mdx`
22
+ * perf: prevent internal `throw` on invalid type
23
+
24
+ 1.6.16 / 2018-02-16
25
+ ===================
26
+
27
+ * deps: mime-types@~2.1.18
28
+ - Add `application/raml+yaml` with extension `.raml`
29
+ - Add `application/wasm` with extension `.wasm`
30
+ - Add `text/shex` with extension `.shex`
31
+ - Add extensions for JPEG-2000 images
32
+ - Add extensions from IANA for `message/*` types
33
+ - Add extension `.mjs` to `application/javascript`
34
+ - Add extension `.wadl` to `application/vnd.sun.wadl+xml`
35
+ - Add extension `.gz` to `application/gzip`
36
+ - Add glTF types and extensions
37
+ - Add new mime types
38
+ - Update extensions `.md` and `.markdown` to be `text/markdown`
39
+ - Update font MIME types
40
+ - Update `text/hjson` to registered `application/hjson`
41
+
42
+ 1.6.15 / 2017-03-31
43
+ ===================
44
+
45
+ * deps: mime-types@~2.1.15
46
+ - Add new mime types
47
+
48
+ 1.6.14 / 2016-11-18
49
+ ===================
50
+
51
+ * deps: mime-types@~2.1.13
52
+ - Add new mime types
53
+
54
+ 1.6.13 / 2016-05-18
55
+ ===================
56
+
57
+ * deps: mime-types@~2.1.11
58
+ - Add new mime types
59
+
60
+ 1.6.12 / 2016-02-28
61
+ ===================
62
+
63
+ * deps: mime-types@~2.1.10
64
+ - Add new mime types
65
+ - Fix extension of `application/dash+xml`
66
+ - Update primary extension for `audio/mp4`
67
+
68
+ 1.6.11 / 2016-01-29
69
+ ===================
70
+
71
+ * deps: mime-types@~2.1.9
72
+ - Add new mime types
73
+
74
+ 1.6.10 / 2015-12-01
75
+ ===================
76
+
77
+ * deps: mime-types@~2.1.8
78
+ - Add new mime types
79
+
80
+ 1.6.9 / 2015-09-27
81
+ ==================
82
+
83
+ * deps: mime-types@~2.1.7
84
+ - Add new mime types
85
+
86
+ 1.6.8 / 2015-09-04
87
+ ==================
88
+
89
+ * deps: mime-types@~2.1.6
90
+ - Add new mime types
91
+
92
+ 1.6.7 / 2015-08-20
93
+ ==================
94
+
95
+ * Fix type error when given invalid type to match against
96
+ * deps: mime-types@~2.1.5
97
+ - Add new mime types
98
+
99
+ 1.6.6 / 2015-07-31
100
+ ==================
101
+
102
+ * deps: mime-types@~2.1.4
103
+ - Add new mime types
104
+
105
+ 1.6.5 / 2015-07-16
106
+ ==================
107
+
108
+ * deps: mime-types@~2.1.3
109
+ - Add new mime types
110
+
111
+ 1.6.4 / 2015-07-01
112
+ ==================
113
+
114
+ * deps: mime-types@~2.1.2
115
+ - Add new mime types
116
+ * perf: enable strict mode
117
+ * perf: remove argument reassignment
118
+
119
+ 1.6.3 / 2015-06-08
120
+ ==================
121
+
122
+ * deps: mime-types@~2.1.1
123
+ - Add new mime types
124
+ * perf: reduce try block size
125
+ * perf: remove bitwise operations
126
+
127
+ 1.6.2 / 2015-05-10
128
+ ==================
129
+
130
+ * deps: mime-types@~2.0.11
131
+ - Add new mime types
132
+
133
+ 1.6.1 / 2015-03-13
134
+ ==================
135
+
136
+ * deps: mime-types@~2.0.10
137
+ - Add new mime types
138
+
139
+ 1.6.0 / 2015-02-12
140
+ ==================
141
+
142
+ * fix false-positives in `hasBody` `Transfer-Encoding` check
143
+ * support wildcard for both type and subtype (`*/*`)
144
+
145
+ 1.5.7 / 2015-02-09
146
+ ==================
147
+
148
+ * fix argument reassignment
149
+ * deps: mime-types@~2.0.9
150
+ - Add new mime types
151
+
152
+ 1.5.6 / 2015-01-29
153
+ ==================
154
+
155
+ * deps: mime-types@~2.0.8
156
+ - Add new mime types
157
+
158
+ 1.5.5 / 2014-12-30
159
+ ==================
160
+
161
+ * deps: mime-types@~2.0.7
162
+ - Add new mime types
163
+ - Fix missing extensions
164
+ - Fix various invalid MIME type entries
165
+ - Remove example template MIME types
166
+ - deps: mime-db@~1.5.0
167
+
168
+ 1.5.4 / 2014-12-10
169
+ ==================
170
+
171
+ * deps: mime-types@~2.0.4
172
+ - Add new mime types
173
+ - deps: mime-db@~1.3.0
174
+
175
+ 1.5.3 / 2014-11-09
176
+ ==================
177
+
178
+ * deps: mime-types@~2.0.3
179
+ - Add new mime types
180
+ - deps: mime-db@~1.2.0
181
+
182
+ 1.5.2 / 2014-09-28
183
+ ==================
184
+
185
+ * deps: mime-types@~2.0.2
186
+ - Add new mime types
187
+ - deps: mime-db@~1.1.0
188
+
189
+ 1.5.1 / 2014-09-07
190
+ ==================
191
+
192
+ * Support Node.js 0.6
193
+ * deps: media-typer@0.3.0
194
+ * deps: mime-types@~2.0.1
195
+ - Support Node.js 0.6
196
+
197
+ 1.5.0 / 2014-09-05
198
+ ==================
199
+
200
+ * fix `hasbody` to be true for `content-length: 0`
201
+
202
+ 1.4.0 / 2014-09-02
203
+ ==================
204
+
205
+ * update mime-types
206
+
207
+ 1.3.2 / 2014-06-24
208
+ ==================
209
+
210
+ * use `~` range on mime-types
211
+
212
+ 1.3.1 / 2014-06-19
213
+ ==================
214
+
215
+ * fix global variable leak
216
+
217
+ 1.3.0 / 2014-06-19
218
+ ==================
219
+
220
+ * improve type parsing
221
+
222
+ - invalid media type never matches
223
+ - media type not case-sensitive
224
+ - extra LWS does not affect results
225
+
226
+ 1.2.2 / 2014-06-19
227
+ ==================
228
+
229
+ * fix behavior on unknown type argument
230
+
231
+ 1.2.1 / 2014-06-03
232
+ ==================
233
+
234
+ * switch dependency from `mime` to `mime-types@1.0.0`
235
+
236
+ 1.2.0 / 2014-05-11
237
+ ==================
238
+
239
+ * support suffix matching:
240
+
241
+ - `+json` matches `application/vnd+json`
242
+ - `*/vnd+json` matches `application/vnd+json`
243
+ - `application/*+json` matches `application/vnd+json`
244
+
245
+ 1.1.0 / 2014-04-12
246
+ ==================
247
+
248
+ * add non-array values support
249
+ * expose internal utilities:
250
+
251
+ - `.is()`
252
+ - `.hasBody()`
253
+ - `.normalize()`
254
+ - `.match()`
255
+
256
+ 1.0.1 / 2014-03-30
257
+ ==================
258
+
259
+ * add `multipart` as a shorthand
@@ -0,0 +1,23 @@
1
+ (The MIT License)
2
+
3
+ Copyright (c) 2014 Jonathan Ong <me@jongleberry.com>
4
+ Copyright (c) 2014-2015 Douglas Christopher Wilson <doug@somethingdoug.com>
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining
7
+ a copy of this software and associated documentation files (the
8
+ 'Software'), to deal in the Software without restriction, including
9
+ without limitation the rights to use, copy, modify, merge, publish,
10
+ distribute, sublicense, and/or sell copies of the Software, and to
11
+ permit persons to whom the Software is furnished to do so, subject to
12
+ the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be
15
+ included in all copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
18
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
21
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,170 @@
1
+ # type-is
2
+
3
+ [![NPM Version][npm-version-image]][npm-url]
4
+ [![NPM Downloads][npm-downloads-image]][npm-url]
5
+ [![Node.js Version][node-version-image]][node-version-url]
6
+ [![Build Status][travis-image]][travis-url]
7
+ [![Test Coverage][coveralls-image]][coveralls-url]
8
+
9
+ Infer the content-type of a request.
10
+
11
+ ### Install
12
+
13
+ This is a [Node.js](https://nodejs.org/en/) module available through the
14
+ [npm registry](https://www.npmjs.com/). Installation is done using the
15
+ [`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
16
+
17
+ ```sh
18
+ $ npm install type-is
19
+ ```
20
+
21
+ ## API
22
+
23
+ ```js
24
+ var http = require('http')
25
+ var typeis = require('type-is')
26
+
27
+ http.createServer(function (req, res) {
28
+ var istext = typeis(req, ['text/*'])
29
+ res.end('you ' + (istext ? 'sent' : 'did not send') + ' me text')
30
+ })
31
+ ```
32
+
33
+ ### typeis(request, types)
34
+
35
+ Checks if the `request` is one of the `types`. If the request has no body,
36
+ even if there is a `Content-Type` header, then `null` is returned. If the
37
+ `Content-Type` header is invalid or does not matches any of the `types`, then
38
+ `false` is returned. Otherwise, a string of the type that matched is returned.
39
+
40
+ The `request` argument is expected to be a Node.js HTTP request. The `types`
41
+ argument is an array of type strings.
42
+
43
+ Each type in the `types` array can be one of the following:
44
+
45
+ - A file extension name such as `json`. This name will be returned if matched.
46
+ - A mime type such as `application/json`.
47
+ - A mime type with a wildcard such as `*/*` or `*/json` or `application/*`.
48
+ The full mime type will be returned if matched.
49
+ - A suffix such as `+json`. This can be combined with a wildcard such as
50
+ `*/vnd+json` or `application/*+json`. The full mime type will be returned
51
+ if matched.
52
+
53
+ Some examples to illustrate the inputs and returned value:
54
+
55
+ <!-- eslint-disable no-undef -->
56
+
57
+ ```js
58
+ // req.headers.content-type = 'application/json'
59
+
60
+ typeis(req, ['json']) // => 'json'
61
+ typeis(req, ['html', 'json']) // => 'json'
62
+ typeis(req, ['application/*']) // => 'application/json'
63
+ typeis(req, ['application/json']) // => 'application/json'
64
+
65
+ typeis(req, ['html']) // => false
66
+ ```
67
+
68
+ ### typeis.hasBody(request)
69
+
70
+ Returns a Boolean if the given `request` has a body, regardless of the
71
+ `Content-Type` header.
72
+
73
+ Having a body has no relation to how large the body is (it may be 0 bytes).
74
+ This is similar to how file existence works. If a body does exist, then this
75
+ indicates that there is data to read from the Node.js request stream.
76
+
77
+ <!-- eslint-disable no-undef -->
78
+
79
+ ```js
80
+ if (typeis.hasBody(req)) {
81
+ // read the body, since there is one
82
+
83
+ req.on('data', function (chunk) {
84
+ // ...
85
+ })
86
+ }
87
+ ```
88
+
89
+ ### typeis.is(mediaType, types)
90
+
91
+ Checks if the `mediaType` is one of the `types`. If the `mediaType` is invalid
92
+ or does not matches any of the `types`, then `false` is returned. Otherwise, a
93
+ string of the type that matched is returned.
94
+
95
+ The `mediaType` argument is expected to be a
96
+ [media type](https://tools.ietf.org/html/rfc6838) string. The `types` argument
97
+ is an array of type strings.
98
+
99
+ Each type in the `types` array can be one of the following:
100
+
101
+ - A file extension name such as `json`. This name will be returned if matched.
102
+ - A mime type such as `application/json`.
103
+ - A mime type with a wildcard such as `*/*` or `*/json` or `application/*`.
104
+ The full mime type will be returned if matched.
105
+ - A suffix such as `+json`. This can be combined with a wildcard such as
106
+ `*/vnd+json` or `application/*+json`. The full mime type will be returned
107
+ if matched.
108
+
109
+ Some examples to illustrate the inputs and returned value:
110
+
111
+ <!-- eslint-disable no-undef -->
112
+
113
+ ```js
114
+ var mediaType = 'application/json'
115
+
116
+ typeis.is(mediaType, ['json']) // => 'json'
117
+ typeis.is(mediaType, ['html', 'json']) // => 'json'
118
+ typeis.is(mediaType, ['application/*']) // => 'application/json'
119
+ typeis.is(mediaType, ['application/json']) // => 'application/json'
120
+
121
+ typeis.is(mediaType, ['html']) // => false
122
+ ```
123
+
124
+ ## Examples
125
+
126
+ ### Example body parser
127
+
128
+ ```js
129
+ var express = require('express')
130
+ var typeis = require('type-is')
131
+
132
+ var app = express()
133
+
134
+ app.use(function bodyParser (req, res, next) {
135
+ if (!typeis.hasBody(req)) {
136
+ return next()
137
+ }
138
+
139
+ switch (typeis(req, ['urlencoded', 'json', 'multipart'])) {
140
+ case 'urlencoded':
141
+ // parse urlencoded body
142
+ throw new Error('implement urlencoded body parsing')
143
+ case 'json':
144
+ // parse json body
145
+ throw new Error('implement json body parsing')
146
+ case 'multipart':
147
+ // parse multipart body
148
+ throw new Error('implement multipart body parsing')
149
+ default:
150
+ // 415 error code
151
+ res.statusCode = 415
152
+ res.end()
153
+ break
154
+ }
155
+ })
156
+ ```
157
+
158
+ ## License
159
+
160
+ [MIT](LICENSE)
161
+
162
+ [coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/type-is/master
163
+ [coveralls-url]: https://coveralls.io/r/jshttp/type-is?branch=master
164
+ [node-version-image]: https://badgen.net/npm/node/type-is
165
+ [node-version-url]: https://nodejs.org/en/download
166
+ [npm-downloads-image]: https://badgen.net/npm/dm/type-is
167
+ [npm-url]: https://npmjs.org/package/type-is
168
+ [npm-version-image]: https://badgen.net/npm/v/type-is
169
+ [travis-image]: https://badgen.net/travis/jshttp/type-is/master
170
+ [travis-url]: https://travis-ci.org/jshttp/type-is
@@ -0,0 +1,266 @@
1
+ /*!
2
+ * type-is
3
+ * Copyright(c) 2014 Jonathan Ong
4
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
5
+ * MIT Licensed
6
+ */
7
+
8
+ 'use strict'
9
+
10
+ /**
11
+ * Module dependencies.
12
+ * @private
13
+ */
14
+
15
+ var typer = require('media-typer')
16
+ var mime = require('mime-types')
17
+
18
+ /**
19
+ * Module exports.
20
+ * @public
21
+ */
22
+
23
+ module.exports = typeofrequest
24
+ module.exports.is = typeis
25
+ module.exports.hasBody = hasbody
26
+ module.exports.normalize = normalize
27
+ module.exports.match = mimeMatch
28
+
29
+ /**
30
+ * Compare a `value` content-type with `types`.
31
+ * Each `type` can be an extension like `html`,
32
+ * a special shortcut like `multipart` or `urlencoded`,
33
+ * or a mime type.
34
+ *
35
+ * If no types match, `false` is returned.
36
+ * Otherwise, the first `type` that matches is returned.
37
+ *
38
+ * @param {String} value
39
+ * @param {Array} types
40
+ * @public
41
+ */
42
+
43
+ function typeis (value, types_) {
44
+ var i
45
+ var types = types_
46
+
47
+ // remove parameters and normalize
48
+ var val = tryNormalizeType(value)
49
+
50
+ // no type or invalid
51
+ if (!val) {
52
+ return false
53
+ }
54
+
55
+ // support flattened arguments
56
+ if (types && !Array.isArray(types)) {
57
+ types = new Array(arguments.length - 1)
58
+ for (i = 0; i < types.length; i++) {
59
+ types[i] = arguments[i + 1]
60
+ }
61
+ }
62
+
63
+ // no types, return the content type
64
+ if (!types || !types.length) {
65
+ return val
66
+ }
67
+
68
+ var type
69
+ for (i = 0; i < types.length; i++) {
70
+ if (mimeMatch(normalize(type = types[i]), val)) {
71
+ return type[0] === '+' || type.indexOf('*') !== -1
72
+ ? val
73
+ : type
74
+ }
75
+ }
76
+
77
+ // no matches
78
+ return false
79
+ }
80
+
81
+ /**
82
+ * Check if a request has a request body.
83
+ * A request with a body __must__ either have `transfer-encoding`
84
+ * or `content-length` headers set.
85
+ * http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.3
86
+ *
87
+ * @param {Object} request
88
+ * @return {Boolean}
89
+ * @public
90
+ */
91
+
92
+ function hasbody (req) {
93
+ return req.headers['transfer-encoding'] !== undefined ||
94
+ !isNaN(req.headers['content-length'])
95
+ }
96
+
97
+ /**
98
+ * Check if the incoming request contains the "Content-Type"
99
+ * header field, and it contains any of the give mime `type`s.
100
+ * If there is no request body, `null` is returned.
101
+ * If there is no content type, `false` is returned.
102
+ * Otherwise, it returns the first `type` that matches.
103
+ *
104
+ * Examples:
105
+ *
106
+ * // With Content-Type: text/html; charset=utf-8
107
+ * this.is('html'); // => 'html'
108
+ * this.is('text/html'); // => 'text/html'
109
+ * this.is('text/*', 'application/json'); // => 'text/html'
110
+ *
111
+ * // When Content-Type is application/json
112
+ * this.is('json', 'urlencoded'); // => 'json'
113
+ * this.is('application/json'); // => 'application/json'
114
+ * this.is('html', 'application/*'); // => 'application/json'
115
+ *
116
+ * this.is('html'); // => false
117
+ *
118
+ * @param {String|Array} types...
119
+ * @return {String|false|null}
120
+ * @public
121
+ */
122
+
123
+ function typeofrequest (req, types_) {
124
+ var types = types_
125
+
126
+ // no body
127
+ if (!hasbody(req)) {
128
+ return null
129
+ }
130
+
131
+ // support flattened arguments
132
+ if (arguments.length > 2) {
133
+ types = new Array(arguments.length - 1)
134
+ for (var i = 0; i < types.length; i++) {
135
+ types[i] = arguments[i + 1]
136
+ }
137
+ }
138
+
139
+ // request content type
140
+ var value = req.headers['content-type']
141
+
142
+ return typeis(value, types)
143
+ }
144
+
145
+ /**
146
+ * Normalize a mime type.
147
+ * If it's a shorthand, expand it to a valid mime type.
148
+ *
149
+ * In general, you probably want:
150
+ *
151
+ * var type = is(req, ['urlencoded', 'json', 'multipart']);
152
+ *
153
+ * Then use the appropriate body parsers.
154
+ * These three are the most common request body types
155
+ * and are thus ensured to work.
156
+ *
157
+ * @param {String} type
158
+ * @private
159
+ */
160
+
161
+ function normalize (type) {
162
+ if (typeof type !== 'string') {
163
+ // invalid type
164
+ return false
165
+ }
166
+
167
+ switch (type) {
168
+ case 'urlencoded':
169
+ return 'application/x-www-form-urlencoded'
170
+ case 'multipart':
171
+ return 'multipart/*'
172
+ }
173
+
174
+ if (type[0] === '+') {
175
+ // "+json" -> "*/*+json" expando
176
+ return '*/*' + type
177
+ }
178
+
179
+ return type.indexOf('/') === -1
180
+ ? mime.lookup(type)
181
+ : type
182
+ }
183
+
184
+ /**
185
+ * Check if `expected` mime type
186
+ * matches `actual` mime type with
187
+ * wildcard and +suffix support.
188
+ *
189
+ * @param {String} expected
190
+ * @param {String} actual
191
+ * @return {Boolean}
192
+ * @private
193
+ */
194
+
195
+ function mimeMatch (expected, actual) {
196
+ // invalid type
197
+ if (expected === false) {
198
+ return false
199
+ }
200
+
201
+ // split types
202
+ var actualParts = actual.split('/')
203
+ var expectedParts = expected.split('/')
204
+
205
+ // invalid format
206
+ if (actualParts.length !== 2 || expectedParts.length !== 2) {
207
+ return false
208
+ }
209
+
210
+ // validate type
211
+ if (expectedParts[0] !== '*' && expectedParts[0] !== actualParts[0]) {
212
+ return false
213
+ }
214
+
215
+ // validate suffix wildcard
216
+ if (expectedParts[1].substr(0, 2) === '*+') {
217
+ return expectedParts[1].length <= actualParts[1].length + 1 &&
218
+ expectedParts[1].substr(1) === actualParts[1].substr(1 - expectedParts[1].length)
219
+ }
220
+
221
+ // validate subtype
222
+ if (expectedParts[1] !== '*' && expectedParts[1] !== actualParts[1]) {
223
+ return false
224
+ }
225
+
226
+ return true
227
+ }
228
+
229
+ /**
230
+ * Normalize a type and remove parameters.
231
+ *
232
+ * @param {string} value
233
+ * @return {string}
234
+ * @private
235
+ */
236
+
237
+ function normalizeType (value) {
238
+ // parse the type
239
+ var type = typer.parse(value)
240
+
241
+ // remove the parameters
242
+ type.parameters = undefined
243
+
244
+ // reformat it
245
+ return typer.format(type)
246
+ }
247
+
248
+ /**
249
+ * Try to normalize a type and remove parameters.
250
+ *
251
+ * @param {string} value
252
+ * @return {string}
253
+ * @private
254
+ */
255
+
256
+ function tryNormalizeType (value) {
257
+ if (!value) {
258
+ return null
259
+ }
260
+
261
+ try {
262
+ return normalizeType(value)
263
+ } catch (err) {
264
+ return null
265
+ }
266
+ }