es6-crawler-detect 3.1.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +27 -15
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +29 -18
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1399 -11
  344. package/src/lib/crawler/exclusions.js +56 -9
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +123 -132
  348. package/test/lib/crawler.test.js +83 -24
  349. package/test/lib/database/crawlers.txt +3651 -0
  350. package/test/lib/database/devices.txt +165636 -0
  351. package/webpack.common.js +20 -20
  352. package/webpack.dev.js +7 -7
  353. package/webpack.prod.js +14 -15
@@ -0,0 +1,23 @@
1
+ 'use strict';
2
+
3
+ var replace = String.prototype.replace;
4
+ var percentTwenties = /%20/g;
5
+
6
+ var Format = {
7
+ RFC1738: 'RFC1738',
8
+ RFC3986: 'RFC3986'
9
+ };
10
+
11
+ module.exports = {
12
+ 'default': Format.RFC3986,
13
+ formatters: {
14
+ RFC1738: function (value) {
15
+ return replace.call(value, percentTwenties, '+');
16
+ },
17
+ RFC3986: function (value) {
18
+ return String(value);
19
+ }
20
+ },
21
+ RFC1738: Format.RFC1738,
22
+ RFC3986: Format.RFC3986
23
+ };
@@ -0,0 +1,11 @@
1
+ 'use strict';
2
+
3
+ var stringify = require('./stringify');
4
+ var parse = require('./parse');
5
+ var formats = require('./formats');
6
+
7
+ module.exports = {
8
+ formats: formats,
9
+ parse: parse,
10
+ stringify: stringify
11
+ };
@@ -0,0 +1,257 @@
1
+ 'use strict';
2
+
3
+ var utils = require('./utils');
4
+
5
+ var has = Object.prototype.hasOwnProperty;
6
+ var isArray = Array.isArray;
7
+
8
+ var defaults = {
9
+ allowDots: false,
10
+ allowPrototypes: false,
11
+ arrayLimit: 20,
12
+ charset: 'utf-8',
13
+ charsetSentinel: false,
14
+ comma: false,
15
+ decoder: utils.decode,
16
+ delimiter: '&',
17
+ depth: 5,
18
+ ignoreQueryPrefix: false,
19
+ interpretNumericEntities: false,
20
+ parameterLimit: 1000,
21
+ parseArrays: true,
22
+ plainObjects: false,
23
+ strictNullHandling: false
24
+ };
25
+
26
+ var interpretNumericEntities = function (str) {
27
+ return str.replace(/&#(\d+);/g, function ($0, numberStr) {
28
+ return String.fromCharCode(parseInt(numberStr, 10));
29
+ });
30
+ };
31
+
32
+ var parseArrayValue = function (val, options) {
33
+ if (val && typeof val === 'string' && options.comma && val.indexOf(',') > -1) {
34
+ return val.split(',');
35
+ }
36
+
37
+ return val;
38
+ };
39
+
40
+ // This is what browsers will submit when the ✓ character occurs in an
41
+ // application/x-www-form-urlencoded body and the encoding of the page containing
42
+ // the form is iso-8859-1, or when the submitted form has an accept-charset
43
+ // attribute of iso-8859-1. Presumably also with other charsets that do not contain
44
+ // the ✓ character, such as us-ascii.
45
+ var isoSentinel = 'utf8=%26%2310003%3B'; // encodeURIComponent('✓')
46
+
47
+ // These are the percent-encoded utf-8 octets representing a checkmark, indicating that the request actually is utf-8 encoded.
48
+ var charsetSentinel = 'utf8=%E2%9C%93'; // encodeURIComponent('✓')
49
+
50
+ var parseValues = function parseQueryStringValues(str, options) {
51
+ var obj = {};
52
+ var cleanStr = options.ignoreQueryPrefix ? str.replace(/^\?/, '') : str;
53
+ var limit = options.parameterLimit === Infinity ? undefined : options.parameterLimit;
54
+ var parts = cleanStr.split(options.delimiter, limit);
55
+ var skipIndex = -1; // Keep track of where the utf8 sentinel was found
56
+ var i;
57
+
58
+ var charset = options.charset;
59
+ if (options.charsetSentinel) {
60
+ for (i = 0; i < parts.length; ++i) {
61
+ if (parts[i].indexOf('utf8=') === 0) {
62
+ if (parts[i] === charsetSentinel) {
63
+ charset = 'utf-8';
64
+ } else if (parts[i] === isoSentinel) {
65
+ charset = 'iso-8859-1';
66
+ }
67
+ skipIndex = i;
68
+ i = parts.length; // The eslint settings do not allow break;
69
+ }
70
+ }
71
+ }
72
+
73
+ for (i = 0; i < parts.length; ++i) {
74
+ if (i === skipIndex) {
75
+ continue;
76
+ }
77
+ var part = parts[i];
78
+
79
+ var bracketEqualsPos = part.indexOf(']=');
80
+ var pos = bracketEqualsPos === -1 ? part.indexOf('=') : bracketEqualsPos + 1;
81
+
82
+ var key, val;
83
+ if (pos === -1) {
84
+ key = options.decoder(part, defaults.decoder, charset, 'key');
85
+ val = options.strictNullHandling ? null : '';
86
+ } else {
87
+ key = options.decoder(part.slice(0, pos), defaults.decoder, charset, 'key');
88
+ val = utils.maybeMap(
89
+ parseArrayValue(part.slice(pos + 1), options),
90
+ function (encodedVal) {
91
+ return options.decoder(encodedVal, defaults.decoder, charset, 'value');
92
+ }
93
+ );
94
+ }
95
+
96
+ if (val && options.interpretNumericEntities && charset === 'iso-8859-1') {
97
+ val = interpretNumericEntities(val);
98
+ }
99
+
100
+ if (part.indexOf('[]=') > -1) {
101
+ val = isArray(val) ? [val] : val;
102
+ }
103
+
104
+ if (has.call(obj, key)) {
105
+ obj[key] = utils.combine(obj[key], val);
106
+ } else {
107
+ obj[key] = val;
108
+ }
109
+ }
110
+
111
+ return obj;
112
+ };
113
+
114
+ var parseObject = function (chain, val, options, valuesParsed) {
115
+ var leaf = valuesParsed ? val : parseArrayValue(val, options);
116
+
117
+ for (var i = chain.length - 1; i >= 0; --i) {
118
+ var obj;
119
+ var root = chain[i];
120
+
121
+ if (root === '[]' && options.parseArrays) {
122
+ obj = [].concat(leaf);
123
+ } else {
124
+ obj = options.plainObjects ? Object.create(null) : {};
125
+ var cleanRoot = root.charAt(0) === '[' && root.charAt(root.length - 1) === ']' ? root.slice(1, -1) : root;
126
+ var index = parseInt(cleanRoot, 10);
127
+ if (!options.parseArrays && cleanRoot === '') {
128
+ obj = { 0: leaf };
129
+ } else if (
130
+ !isNaN(index)
131
+ && root !== cleanRoot
132
+ && String(index) === cleanRoot
133
+ && index >= 0
134
+ && (options.parseArrays && index <= options.arrayLimit)
135
+ ) {
136
+ obj = [];
137
+ obj[index] = leaf;
138
+ } else {
139
+ obj[cleanRoot] = leaf;
140
+ }
141
+ }
142
+
143
+ leaf = obj;
144
+ }
145
+
146
+ return leaf;
147
+ };
148
+
149
+ var parseKeys = function parseQueryStringKeys(givenKey, val, options, valuesParsed) {
150
+ if (!givenKey) {
151
+ return;
152
+ }
153
+
154
+ // Transform dot notation to bracket notation
155
+ var key = options.allowDots ? givenKey.replace(/\.([^.[]+)/g, '[$1]') : givenKey;
156
+
157
+ // The regex chunks
158
+
159
+ var brackets = /(\[[^[\]]*])/;
160
+ var child = /(\[[^[\]]*])/g;
161
+
162
+ // Get the parent
163
+
164
+ var segment = options.depth > 0 && brackets.exec(key);
165
+ var parent = segment ? key.slice(0, segment.index) : key;
166
+
167
+ // Stash the parent if it exists
168
+
169
+ var keys = [];
170
+ if (parent) {
171
+ // If we aren't using plain objects, optionally prefix keys that would overwrite object prototype properties
172
+ if (!options.plainObjects && has.call(Object.prototype, parent)) {
173
+ if (!options.allowPrototypes) {
174
+ return;
175
+ }
176
+ }
177
+
178
+ keys.push(parent);
179
+ }
180
+
181
+ // Loop through children appending to the array until we hit depth
182
+
183
+ var i = 0;
184
+ while (options.depth > 0 && (segment = child.exec(key)) !== null && i < options.depth) {
185
+ i += 1;
186
+ if (!options.plainObjects && has.call(Object.prototype, segment[1].slice(1, -1))) {
187
+ if (!options.allowPrototypes) {
188
+ return;
189
+ }
190
+ }
191
+ keys.push(segment[1]);
192
+ }
193
+
194
+ // If there's a remainder, just add whatever is left
195
+
196
+ if (segment) {
197
+ keys.push('[' + key.slice(segment.index) + ']');
198
+ }
199
+
200
+ return parseObject(keys, val, options, valuesParsed);
201
+ };
202
+
203
+ var normalizeParseOptions = function normalizeParseOptions(opts) {
204
+ if (!opts) {
205
+ return defaults;
206
+ }
207
+
208
+ if (opts.decoder !== null && opts.decoder !== undefined && typeof opts.decoder !== 'function') {
209
+ throw new TypeError('Decoder has to be a function.');
210
+ }
211
+
212
+ if (typeof opts.charset !== 'undefined' && opts.charset !== 'utf-8' && opts.charset !== 'iso-8859-1') {
213
+ throw new TypeError('The charset option must be either utf-8, iso-8859-1, or undefined');
214
+ }
215
+ var charset = typeof opts.charset === 'undefined' ? defaults.charset : opts.charset;
216
+
217
+ return {
218
+ allowDots: typeof opts.allowDots === 'undefined' ? defaults.allowDots : !!opts.allowDots,
219
+ allowPrototypes: typeof opts.allowPrototypes === 'boolean' ? opts.allowPrototypes : defaults.allowPrototypes,
220
+ arrayLimit: typeof opts.arrayLimit === 'number' ? opts.arrayLimit : defaults.arrayLimit,
221
+ charset: charset,
222
+ charsetSentinel: typeof opts.charsetSentinel === 'boolean' ? opts.charsetSentinel : defaults.charsetSentinel,
223
+ comma: typeof opts.comma === 'boolean' ? opts.comma : defaults.comma,
224
+ decoder: typeof opts.decoder === 'function' ? opts.decoder : defaults.decoder,
225
+ delimiter: typeof opts.delimiter === 'string' || utils.isRegExp(opts.delimiter) ? opts.delimiter : defaults.delimiter,
226
+ // eslint-disable-next-line no-implicit-coercion, no-extra-parens
227
+ depth: (typeof opts.depth === 'number' || opts.depth === false) ? +opts.depth : defaults.depth,
228
+ ignoreQueryPrefix: opts.ignoreQueryPrefix === true,
229
+ interpretNumericEntities: typeof opts.interpretNumericEntities === 'boolean' ? opts.interpretNumericEntities : defaults.interpretNumericEntities,
230
+ parameterLimit: typeof opts.parameterLimit === 'number' ? opts.parameterLimit : defaults.parameterLimit,
231
+ parseArrays: opts.parseArrays !== false,
232
+ plainObjects: typeof opts.plainObjects === 'boolean' ? opts.plainObjects : defaults.plainObjects,
233
+ strictNullHandling: typeof opts.strictNullHandling === 'boolean' ? opts.strictNullHandling : defaults.strictNullHandling
234
+ };
235
+ };
236
+
237
+ module.exports = function (str, opts) {
238
+ var options = normalizeParseOptions(opts);
239
+
240
+ if (str === '' || str === null || typeof str === 'undefined') {
241
+ return options.plainObjects ? Object.create(null) : {};
242
+ }
243
+
244
+ var tempObj = typeof str === 'string' ? parseValues(str, options) : str;
245
+ var obj = options.plainObjects ? Object.create(null) : {};
246
+
247
+ // Iterate over the keys and setup the new object
248
+
249
+ var keys = Object.keys(tempObj);
250
+ for (var i = 0; i < keys.length; ++i) {
251
+ var key = keys[i];
252
+ var newObj = parseKeys(key, tempObj[key], options, typeof str === 'string');
253
+ obj = utils.merge(obj, newObj, options);
254
+ }
255
+
256
+ return utils.compact(obj);
257
+ };
@@ -0,0 +1,278 @@
1
+ 'use strict';
2
+
3
+ var utils = require('./utils');
4
+ var formats = require('./formats');
5
+ var has = Object.prototype.hasOwnProperty;
6
+
7
+ var arrayPrefixGenerators = {
8
+ brackets: function brackets(prefix) {
9
+ return prefix + '[]';
10
+ },
11
+ comma: 'comma',
12
+ indices: function indices(prefix, key) {
13
+ return prefix + '[' + key + ']';
14
+ },
15
+ repeat: function repeat(prefix) {
16
+ return prefix;
17
+ }
18
+ };
19
+
20
+ var isArray = Array.isArray;
21
+ var push = Array.prototype.push;
22
+ var pushToArray = function (arr, valueOrArray) {
23
+ push.apply(arr, isArray(valueOrArray) ? valueOrArray : [valueOrArray]);
24
+ };
25
+
26
+ var toISO = Date.prototype.toISOString;
27
+
28
+ var defaultFormat = formats['default'];
29
+ var defaults = {
30
+ addQueryPrefix: false,
31
+ allowDots: false,
32
+ charset: 'utf-8',
33
+ charsetSentinel: false,
34
+ delimiter: '&',
35
+ encode: true,
36
+ encoder: utils.encode,
37
+ encodeValuesOnly: false,
38
+ format: defaultFormat,
39
+ formatter: formats.formatters[defaultFormat],
40
+ // deprecated
41
+ indices: false,
42
+ serializeDate: function serializeDate(date) {
43
+ return toISO.call(date);
44
+ },
45
+ skipNulls: false,
46
+ strictNullHandling: false
47
+ };
48
+
49
+ var isNonNullishPrimitive = function isNonNullishPrimitive(v) {
50
+ return typeof v === 'string'
51
+ || typeof v === 'number'
52
+ || typeof v === 'boolean'
53
+ || typeof v === 'symbol'
54
+ || typeof v === 'bigint';
55
+ };
56
+
57
+ var stringify = function stringify(
58
+ object,
59
+ prefix,
60
+ generateArrayPrefix,
61
+ strictNullHandling,
62
+ skipNulls,
63
+ encoder,
64
+ filter,
65
+ sort,
66
+ allowDots,
67
+ serializeDate,
68
+ format,
69
+ formatter,
70
+ encodeValuesOnly,
71
+ charset
72
+ ) {
73
+ var obj = object;
74
+ if (typeof filter === 'function') {
75
+ obj = filter(prefix, obj);
76
+ } else if (obj instanceof Date) {
77
+ obj = serializeDate(obj);
78
+ } else if (generateArrayPrefix === 'comma' && isArray(obj)) {
79
+ obj = utils.maybeMap(obj, function (value) {
80
+ if (value instanceof Date) {
81
+ return serializeDate(value);
82
+ }
83
+ return value;
84
+ });
85
+ }
86
+
87
+ if (obj === null) {
88
+ if (strictNullHandling) {
89
+ return encoder && !encodeValuesOnly ? encoder(prefix, defaults.encoder, charset, 'key', format) : prefix;
90
+ }
91
+
92
+ obj = '';
93
+ }
94
+
95
+ if (isNonNullishPrimitive(obj) || utils.isBuffer(obj)) {
96
+ if (encoder) {
97
+ var keyValue = encodeValuesOnly ? prefix : encoder(prefix, defaults.encoder, charset, 'key', format);
98
+ return [formatter(keyValue) + '=' + formatter(encoder(obj, defaults.encoder, charset, 'value', format))];
99
+ }
100
+ return [formatter(prefix) + '=' + formatter(String(obj))];
101
+ }
102
+
103
+ var values = [];
104
+
105
+ if (typeof obj === 'undefined') {
106
+ return values;
107
+ }
108
+
109
+ var objKeys;
110
+ if (generateArrayPrefix === 'comma' && isArray(obj)) {
111
+ // we need to join elements in
112
+ objKeys = [{ value: obj.length > 0 ? obj.join(',') || null : undefined }];
113
+ } else if (isArray(filter)) {
114
+ objKeys = filter;
115
+ } else {
116
+ var keys = Object.keys(obj);
117
+ objKeys = sort ? keys.sort(sort) : keys;
118
+ }
119
+
120
+ for (var i = 0; i < objKeys.length; ++i) {
121
+ var key = objKeys[i];
122
+ var value = typeof key === 'object' && key.value !== undefined ? key.value : obj[key];
123
+
124
+ if (skipNulls && value === null) {
125
+ continue;
126
+ }
127
+
128
+ var keyPrefix = isArray(obj)
129
+ ? typeof generateArrayPrefix === 'function' ? generateArrayPrefix(prefix, key) : prefix
130
+ : prefix + (allowDots ? '.' + key : '[' + key + ']');
131
+
132
+ pushToArray(values, stringify(
133
+ value,
134
+ keyPrefix,
135
+ generateArrayPrefix,
136
+ strictNullHandling,
137
+ skipNulls,
138
+ encoder,
139
+ filter,
140
+ sort,
141
+ allowDots,
142
+ serializeDate,
143
+ format,
144
+ formatter,
145
+ encodeValuesOnly,
146
+ charset
147
+ ));
148
+ }
149
+
150
+ return values;
151
+ };
152
+
153
+ var normalizeStringifyOptions = function normalizeStringifyOptions(opts) {
154
+ if (!opts) {
155
+ return defaults;
156
+ }
157
+
158
+ if (opts.encoder !== null && opts.encoder !== undefined && typeof opts.encoder !== 'function') {
159
+ throw new TypeError('Encoder has to be a function.');
160
+ }
161
+
162
+ var charset = opts.charset || defaults.charset;
163
+ if (typeof opts.charset !== 'undefined' && opts.charset !== 'utf-8' && opts.charset !== 'iso-8859-1') {
164
+ throw new TypeError('The charset option must be either utf-8, iso-8859-1, or undefined');
165
+ }
166
+
167
+ var format = formats['default'];
168
+ if (typeof opts.format !== 'undefined') {
169
+ if (!has.call(formats.formatters, opts.format)) {
170
+ throw new TypeError('Unknown format option provided.');
171
+ }
172
+ format = opts.format;
173
+ }
174
+ var formatter = formats.formatters[format];
175
+
176
+ var filter = defaults.filter;
177
+ if (typeof opts.filter === 'function' || isArray(opts.filter)) {
178
+ filter = opts.filter;
179
+ }
180
+
181
+ return {
182
+ addQueryPrefix: typeof opts.addQueryPrefix === 'boolean' ? opts.addQueryPrefix : defaults.addQueryPrefix,
183
+ allowDots: typeof opts.allowDots === 'undefined' ? defaults.allowDots : !!opts.allowDots,
184
+ charset: charset,
185
+ charsetSentinel: typeof opts.charsetSentinel === 'boolean' ? opts.charsetSentinel : defaults.charsetSentinel,
186
+ delimiter: typeof opts.delimiter === 'undefined' ? defaults.delimiter : opts.delimiter,
187
+ encode: typeof opts.encode === 'boolean' ? opts.encode : defaults.encode,
188
+ encoder: typeof opts.encoder === 'function' ? opts.encoder : defaults.encoder,
189
+ encodeValuesOnly: typeof opts.encodeValuesOnly === 'boolean' ? opts.encodeValuesOnly : defaults.encodeValuesOnly,
190
+ filter: filter,
191
+ format: format,
192
+ formatter: formatter,
193
+ serializeDate: typeof opts.serializeDate === 'function' ? opts.serializeDate : defaults.serializeDate,
194
+ skipNulls: typeof opts.skipNulls === 'boolean' ? opts.skipNulls : defaults.skipNulls,
195
+ sort: typeof opts.sort === 'function' ? opts.sort : null,
196
+ strictNullHandling: typeof opts.strictNullHandling === 'boolean' ? opts.strictNullHandling : defaults.strictNullHandling
197
+ };
198
+ };
199
+
200
+ module.exports = function (object, opts) {
201
+ var obj = object;
202
+ var options = normalizeStringifyOptions(opts);
203
+
204
+ var objKeys;
205
+ var filter;
206
+
207
+ if (typeof options.filter === 'function') {
208
+ filter = options.filter;
209
+ obj = filter('', obj);
210
+ } else if (isArray(options.filter)) {
211
+ filter = options.filter;
212
+ objKeys = filter;
213
+ }
214
+
215
+ var keys = [];
216
+
217
+ if (typeof obj !== 'object' || obj === null) {
218
+ return '';
219
+ }
220
+
221
+ var arrayFormat;
222
+ if (opts && opts.arrayFormat in arrayPrefixGenerators) {
223
+ arrayFormat = opts.arrayFormat;
224
+ } else if (opts && 'indices' in opts) {
225
+ arrayFormat = opts.indices ? 'indices' : 'repeat';
226
+ } else {
227
+ arrayFormat = 'indices';
228
+ }
229
+
230
+ var generateArrayPrefix = arrayPrefixGenerators[arrayFormat];
231
+
232
+ if (!objKeys) {
233
+ objKeys = Object.keys(obj);
234
+ }
235
+
236
+ if (options.sort) {
237
+ objKeys.sort(options.sort);
238
+ }
239
+
240
+ for (var i = 0; i < objKeys.length; ++i) {
241
+ var key = objKeys[i];
242
+
243
+ if (options.skipNulls && obj[key] === null) {
244
+ continue;
245
+ }
246
+ pushToArray(keys, stringify(
247
+ obj[key],
248
+ key,
249
+ generateArrayPrefix,
250
+ options.strictNullHandling,
251
+ options.skipNulls,
252
+ options.encode ? options.encoder : null,
253
+ options.filter,
254
+ options.sort,
255
+ options.allowDots,
256
+ options.serializeDate,
257
+ options.format,
258
+ options.formatter,
259
+ options.encodeValuesOnly,
260
+ options.charset
261
+ ));
262
+ }
263
+
264
+ var joined = keys.join(options.delimiter);
265
+ var prefix = options.addQueryPrefix === true ? '?' : '';
266
+
267
+ if (options.charsetSentinel) {
268
+ if (options.charset === 'iso-8859-1') {
269
+ // encodeURIComponent('&#10003;'), the "numeric entity" representation of a checkmark
270
+ prefix += 'utf8=%26%2310003%3B&';
271
+ } else {
272
+ // encodeURIComponent('✓')
273
+ prefix += 'utf8=%E2%9C%93&';
274
+ }
275
+ }
276
+
277
+ return joined.length > 0 ? prefix + joined : '';
278
+ };