es6-crawler-detect 3.1.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +27 -15
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +29 -18
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1399 -11
  344. package/src/lib/crawler/exclusions.js +56 -9
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +123 -132
  348. package/test/lib/crawler.test.js +83 -24
  349. package/test/lib/database/crawlers.txt +3651 -0
  350. package/test/lib/database/devices.txt +165636 -0
  351. package/webpack.common.js +20 -20
  352. package/webpack.dev.js +7 -7
  353. package/webpack.prod.js +14 -15
@@ -0,0 +1,139 @@
1
+ # proxy-addr
2
+
3
+ [![NPM Version][npm-version-image]][npm-url]
4
+ [![NPM Downloads][npm-downloads-image]][npm-url]
5
+ [![Node.js Version][node-image]][node-url]
6
+ [![Build Status][ci-image]][ci-url]
7
+ [![Test Coverage][coveralls-image]][coveralls-url]
8
+
9
+ Determine address of proxied request
10
+
11
+ ## Install
12
+
13
+ This is a [Node.js](https://nodejs.org/en/) module available through the
14
+ [npm registry](https://www.npmjs.com/). Installation is done using the
15
+ [`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
16
+
17
+ ```sh
18
+ $ npm install proxy-addr
19
+ ```
20
+
21
+ ## API
22
+
23
+ ```js
24
+ var proxyaddr = require('proxy-addr')
25
+ ```
26
+
27
+ ### proxyaddr(req, trust)
28
+
29
+ Return the address of the request, using the given `trust` parameter.
30
+
31
+ The `trust` argument is a function that returns `true` if you trust
32
+ the address, `false` if you don't. The closest untrusted address is
33
+ returned.
34
+
35
+ ```js
36
+ proxyaddr(req, function (addr) { return addr === '127.0.0.1' })
37
+ proxyaddr(req, function (addr, i) { return i < 1 })
38
+ ```
39
+
40
+ The `trust` arugment may also be a single IP address string or an
41
+ array of trusted addresses, as plain IP addresses, CIDR-formatted
42
+ strings, or IP/netmask strings.
43
+
44
+ ```js
45
+ proxyaddr(req, '127.0.0.1')
46
+ proxyaddr(req, ['127.0.0.0/8', '10.0.0.0/8'])
47
+ proxyaddr(req, ['127.0.0.0/255.0.0.0', '192.168.0.0/255.255.0.0'])
48
+ ```
49
+
50
+ This module also supports IPv6. Your IPv6 addresses will be normalized
51
+ automatically (i.e. `fe80::00ed:1` equals `fe80:0:0:0:0:0:ed:1`).
52
+
53
+ ```js
54
+ proxyaddr(req, '::1')
55
+ proxyaddr(req, ['::1/128', 'fe80::/10'])
56
+ ```
57
+
58
+ This module will automatically work with IPv4-mapped IPv6 addresses
59
+ as well to support node.js in IPv6-only mode. This means that you do
60
+ not have to specify both `::ffff:a00:1` and `10.0.0.1`.
61
+
62
+ As a convenience, this module also takes certain pre-defined names
63
+ in addition to IP addresses, which expand into IP addresses:
64
+
65
+ ```js
66
+ proxyaddr(req, 'loopback')
67
+ proxyaddr(req, ['loopback', 'fc00:ac:1ab5:fff::1/64'])
68
+ ```
69
+
70
+ * `loopback`: IPv4 and IPv6 loopback addresses (like `::1` and
71
+ `127.0.0.1`).
72
+ * `linklocal`: IPv4 and IPv6 link-local addresses (like
73
+ `fe80::1:1:1:1` and `169.254.0.1`).
74
+ * `uniquelocal`: IPv4 private addresses and IPv6 unique-local
75
+ addresses (like `fc00:ac:1ab5:fff::1` and `192.168.0.1`).
76
+
77
+ When `trust` is specified as a function, it will be called for each
78
+ address to determine if it is a trusted address. The function is
79
+ given two arguments: `addr` and `i`, where `addr` is a string of
80
+ the address to check and `i` is a number that represents the distance
81
+ from the socket address.
82
+
83
+ ### proxyaddr.all(req, [trust])
84
+
85
+ Return all the addresses of the request, optionally stopping at the
86
+ first untrusted. This array is ordered from closest to furthest
87
+ (i.e. `arr[0] === req.connection.remoteAddress`).
88
+
89
+ ```js
90
+ proxyaddr.all(req)
91
+ ```
92
+
93
+ The optional `trust` argument takes the same arguments as `trust`
94
+ does in `proxyaddr(req, trust)`.
95
+
96
+ ```js
97
+ proxyaddr.all(req, 'loopback')
98
+ ```
99
+
100
+ ### proxyaddr.compile(val)
101
+
102
+ Compiles argument `val` into a `trust` function. This function takes
103
+ the same arguments as `trust` does in `proxyaddr(req, trust)` and
104
+ returns a function suitable for `proxyaddr(req, trust)`.
105
+
106
+ ```js
107
+ var trust = proxyaddr.compile('loopback')
108
+ var addr = proxyaddr(req, trust)
109
+ ```
110
+
111
+ This function is meant to be optimized for use against every request.
112
+ It is recommend to compile a trust function up-front for the trusted
113
+ configuration and pass that to `proxyaddr(req, trust)` for each request.
114
+
115
+ ## Testing
116
+
117
+ ```sh
118
+ $ npm test
119
+ ```
120
+
121
+ ## Benchmarks
122
+
123
+ ```sh
124
+ $ npm run-script bench
125
+ ```
126
+
127
+ ## License
128
+
129
+ [MIT](LICENSE)
130
+
131
+ [ci-image]: https://badgen.net/github/checks/jshttp/proxy-addr/master?label=ci
132
+ [ci-url]: https://github.com/jshttp/proxy-addr/actions?query=workflow%3Aci
133
+ [coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/proxy-addr/master
134
+ [coveralls-url]: https://coveralls.io/r/jshttp/proxy-addr?branch=master
135
+ [node-image]: https://badgen.net/npm/node/proxy-addr
136
+ [node-url]: https://nodejs.org/en/download
137
+ [npm-downloads-image]: https://badgen.net/npm/dm/proxy-addr
138
+ [npm-url]: https://npmjs.org/package/proxy-addr
139
+ [npm-version-image]: https://badgen.net/npm/v/proxy-addr
@@ -0,0 +1,327 @@
1
+ /*!
2
+ * proxy-addr
3
+ * Copyright(c) 2014-2016 Douglas Christopher Wilson
4
+ * MIT Licensed
5
+ */
6
+
7
+ 'use strict'
8
+
9
+ /**
10
+ * Module exports.
11
+ * @public
12
+ */
13
+
14
+ module.exports = proxyaddr
15
+ module.exports.all = alladdrs
16
+ module.exports.compile = compile
17
+
18
+ /**
19
+ * Module dependencies.
20
+ * @private
21
+ */
22
+
23
+ var forwarded = require('forwarded')
24
+ var ipaddr = require('ipaddr.js')
25
+
26
+ /**
27
+ * Variables.
28
+ * @private
29
+ */
30
+
31
+ var DIGIT_REGEXP = /^[0-9]+$/
32
+ var isip = ipaddr.isValid
33
+ var parseip = ipaddr.parse
34
+
35
+ /**
36
+ * Pre-defined IP ranges.
37
+ * @private
38
+ */
39
+
40
+ var IP_RANGES = {
41
+ linklocal: ['169.254.0.0/16', 'fe80::/10'],
42
+ loopback: ['127.0.0.1/8', '::1/128'],
43
+ uniquelocal: ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', 'fc00::/7']
44
+ }
45
+
46
+ /**
47
+ * Get all addresses in the request, optionally stopping
48
+ * at the first untrusted.
49
+ *
50
+ * @param {Object} request
51
+ * @param {Function|Array|String} [trust]
52
+ * @public
53
+ */
54
+
55
+ function alladdrs (req, trust) {
56
+ // get addresses
57
+ var addrs = forwarded(req)
58
+
59
+ if (!trust) {
60
+ // Return all addresses
61
+ return addrs
62
+ }
63
+
64
+ if (typeof trust !== 'function') {
65
+ trust = compile(trust)
66
+ }
67
+
68
+ for (var i = 0; i < addrs.length - 1; i++) {
69
+ if (trust(addrs[i], i)) continue
70
+
71
+ addrs.length = i + 1
72
+ }
73
+
74
+ return addrs
75
+ }
76
+
77
+ /**
78
+ * Compile argument into trust function.
79
+ *
80
+ * @param {Array|String} val
81
+ * @private
82
+ */
83
+
84
+ function compile (val) {
85
+ if (!val) {
86
+ throw new TypeError('argument is required')
87
+ }
88
+
89
+ var trust
90
+
91
+ if (typeof val === 'string') {
92
+ trust = [val]
93
+ } else if (Array.isArray(val)) {
94
+ trust = val.slice()
95
+ } else {
96
+ throw new TypeError('unsupported trust argument')
97
+ }
98
+
99
+ for (var i = 0; i < trust.length; i++) {
100
+ val = trust[i]
101
+
102
+ if (!Object.prototype.hasOwnProperty.call(IP_RANGES, val)) {
103
+ continue
104
+ }
105
+
106
+ // Splice in pre-defined range
107
+ val = IP_RANGES[val]
108
+ trust.splice.apply(trust, [i, 1].concat(val))
109
+ i += val.length - 1
110
+ }
111
+
112
+ return compileTrust(compileRangeSubnets(trust))
113
+ }
114
+
115
+ /**
116
+ * Compile `arr` elements into range subnets.
117
+ *
118
+ * @param {Array} arr
119
+ * @private
120
+ */
121
+
122
+ function compileRangeSubnets (arr) {
123
+ var rangeSubnets = new Array(arr.length)
124
+
125
+ for (var i = 0; i < arr.length; i++) {
126
+ rangeSubnets[i] = parseipNotation(arr[i])
127
+ }
128
+
129
+ return rangeSubnets
130
+ }
131
+
132
+ /**
133
+ * Compile range subnet array into trust function.
134
+ *
135
+ * @param {Array} rangeSubnets
136
+ * @private
137
+ */
138
+
139
+ function compileTrust (rangeSubnets) {
140
+ // Return optimized function based on length
141
+ var len = rangeSubnets.length
142
+ return len === 0
143
+ ? trustNone
144
+ : len === 1
145
+ ? trustSingle(rangeSubnets[0])
146
+ : trustMulti(rangeSubnets)
147
+ }
148
+
149
+ /**
150
+ * Parse IP notation string into range subnet.
151
+ *
152
+ * @param {String} note
153
+ * @private
154
+ */
155
+
156
+ function parseipNotation (note) {
157
+ var pos = note.lastIndexOf('/')
158
+ var str = pos !== -1
159
+ ? note.substring(0, pos)
160
+ : note
161
+
162
+ if (!isip(str)) {
163
+ throw new TypeError('invalid IP address: ' + str)
164
+ }
165
+
166
+ var ip = parseip(str)
167
+
168
+ if (pos === -1 && ip.kind() === 'ipv6' && ip.isIPv4MappedAddress()) {
169
+ // Store as IPv4
170
+ ip = ip.toIPv4Address()
171
+ }
172
+
173
+ var max = ip.kind() === 'ipv6'
174
+ ? 128
175
+ : 32
176
+
177
+ var range = pos !== -1
178
+ ? note.substring(pos + 1, note.length)
179
+ : null
180
+
181
+ if (range === null) {
182
+ range = max
183
+ } else if (DIGIT_REGEXP.test(range)) {
184
+ range = parseInt(range, 10)
185
+ } else if (ip.kind() === 'ipv4' && isip(range)) {
186
+ range = parseNetmask(range)
187
+ } else {
188
+ range = null
189
+ }
190
+
191
+ if (range <= 0 || range > max) {
192
+ throw new TypeError('invalid range on address: ' + note)
193
+ }
194
+
195
+ return [ip, range]
196
+ }
197
+
198
+ /**
199
+ * Parse netmask string into CIDR range.
200
+ *
201
+ * @param {String} netmask
202
+ * @private
203
+ */
204
+
205
+ function parseNetmask (netmask) {
206
+ var ip = parseip(netmask)
207
+ var kind = ip.kind()
208
+
209
+ return kind === 'ipv4'
210
+ ? ip.prefixLengthFromSubnetMask()
211
+ : null
212
+ }
213
+
214
+ /**
215
+ * Determine address of proxied request.
216
+ *
217
+ * @param {Object} request
218
+ * @param {Function|Array|String} trust
219
+ * @public
220
+ */
221
+
222
+ function proxyaddr (req, trust) {
223
+ if (!req) {
224
+ throw new TypeError('req argument is required')
225
+ }
226
+
227
+ if (!trust) {
228
+ throw new TypeError('trust argument is required')
229
+ }
230
+
231
+ var addrs = alladdrs(req, trust)
232
+ var addr = addrs[addrs.length - 1]
233
+
234
+ return addr
235
+ }
236
+
237
+ /**
238
+ * Static trust function to trust nothing.
239
+ *
240
+ * @private
241
+ */
242
+
243
+ function trustNone () {
244
+ return false
245
+ }
246
+
247
+ /**
248
+ * Compile trust function for multiple subnets.
249
+ *
250
+ * @param {Array} subnets
251
+ * @private
252
+ */
253
+
254
+ function trustMulti (subnets) {
255
+ return function trust (addr) {
256
+ if (!isip(addr)) return false
257
+
258
+ var ip = parseip(addr)
259
+ var ipconv
260
+ var kind = ip.kind()
261
+
262
+ for (var i = 0; i < subnets.length; i++) {
263
+ var subnet = subnets[i]
264
+ var subnetip = subnet[0]
265
+ var subnetkind = subnetip.kind()
266
+ var subnetrange = subnet[1]
267
+ var trusted = ip
268
+
269
+ if (kind !== subnetkind) {
270
+ if (subnetkind === 'ipv4' && !ip.isIPv4MappedAddress()) {
271
+ // Incompatible IP addresses
272
+ continue
273
+ }
274
+
275
+ if (!ipconv) {
276
+ // Convert IP to match subnet IP kind
277
+ ipconv = subnetkind === 'ipv4'
278
+ ? ip.toIPv4Address()
279
+ : ip.toIPv4MappedAddress()
280
+ }
281
+
282
+ trusted = ipconv
283
+ }
284
+
285
+ if (trusted.match(subnetip, subnetrange)) {
286
+ return true
287
+ }
288
+ }
289
+
290
+ return false
291
+ }
292
+ }
293
+
294
+ /**
295
+ * Compile trust function for single subnet.
296
+ *
297
+ * @param {Object} subnet
298
+ * @private
299
+ */
300
+
301
+ function trustSingle (subnet) {
302
+ var subnetip = subnet[0]
303
+ var subnetkind = subnetip.kind()
304
+ var subnetisipv4 = subnetkind === 'ipv4'
305
+ var subnetrange = subnet[1]
306
+
307
+ return function trust (addr) {
308
+ if (!isip(addr)) return false
309
+
310
+ var ip = parseip(addr)
311
+ var kind = ip.kind()
312
+
313
+ if (kind !== subnetkind) {
314
+ if (subnetisipv4 && !ip.isIPv4MappedAddress()) {
315
+ // Incompatible IP addresses
316
+ return false
317
+ }
318
+
319
+ // Convert IP to match subnet IP kind
320
+ ip = subnetisipv4
321
+ ? ip.toIPv4Address()
322
+ : ip.toIPv4MappedAddress()
323
+ }
324
+
325
+ return ip.match(subnetip, subnetrange)
326
+ }
327
+ }
@@ -0,0 +1,47 @@
1
+ {
2
+ "name": "proxy-addr",
3
+ "description": "Determine address of proxied request",
4
+ "version": "2.0.7",
5
+ "author": "Douglas Christopher Wilson <doug@somethingdoug.com>",
6
+ "license": "MIT",
7
+ "keywords": [
8
+ "ip",
9
+ "proxy",
10
+ "x-forwarded-for"
11
+ ],
12
+ "repository": "jshttp/proxy-addr",
13
+ "dependencies": {
14
+ "forwarded": "0.2.0",
15
+ "ipaddr.js": "1.9.1"
16
+ },
17
+ "devDependencies": {
18
+ "benchmark": "2.1.4",
19
+ "beautify-benchmark": "0.2.4",
20
+ "deep-equal": "1.0.1",
21
+ "eslint": "7.26.0",
22
+ "eslint-config-standard": "14.1.1",
23
+ "eslint-plugin-import": "2.23.4",
24
+ "eslint-plugin-markdown": "2.2.0",
25
+ "eslint-plugin-node": "11.1.0",
26
+ "eslint-plugin-promise": "4.3.1",
27
+ "eslint-plugin-standard": "4.1.0",
28
+ "mocha": "8.4.0",
29
+ "nyc": "15.1.0"
30
+ },
31
+ "files": [
32
+ "LICENSE",
33
+ "HISTORY.md",
34
+ "README.md",
35
+ "index.js"
36
+ ],
37
+ "engines": {
38
+ "node": ">= 0.10"
39
+ },
40
+ "scripts": {
41
+ "bench": "node benchmark/index.js",
42
+ "lint": "eslint .",
43
+ "test": "mocha --reporter spec --bail --check-leaks test/",
44
+ "test-ci": "nyc --reporter=lcov --reporter=text npm test",
45
+ "test-cov": "nyc --reporter=html --reporter=text npm test"
46
+ }
47
+ }
@@ -0,0 +1,39 @@
1
+ root = true
2
+
3
+ [*]
4
+ indent_style = space
5
+ indent_size = 4
6
+ end_of_line = lf
7
+ charset = utf-8
8
+ trim_trailing_whitespace = true
9
+ insert_final_newline = true
10
+ max_line_length = 160
11
+
12
+ [test/*]
13
+ max_line_length = off
14
+
15
+ [LICENSE.md]
16
+ indent_size = off
17
+
18
+ [*.md]
19
+ max_line_length = off
20
+
21
+ [*.json]
22
+ max_line_length = off
23
+
24
+ [Makefile]
25
+ max_line_length = off
26
+
27
+ [CHANGELOG.md]
28
+ indent_style = space
29
+ indent_size = 2
30
+
31
+ [LICENSE]
32
+ indent_size = 2
33
+ max_line_length = off
34
+
35
+ [coverage/**/*]
36
+ indent_size = off
37
+ indent_style = off
38
+ indent = off
39
+ max_line_length = off
@@ -0,0 +1,2 @@
1
+ dist/
2
+ coverage/
@@ -0,0 +1,35 @@
1
+ {
2
+ "root": true,
3
+
4
+ "extends": "@ljharb",
5
+
6
+ "rules": {
7
+ "complexity": 0,
8
+ "consistent-return": 1,
9
+ "func-name-matching": 0,
10
+ "id-length": [2, { "min": 1, "max": 25, "properties": "never" }],
11
+ "indent": [2, 4],
12
+ "max-lines-per-function": [2, { "max": 150 }],
13
+ "max-params": [2, 14],
14
+ "max-statements": [2, 52],
15
+ "multiline-comment-style": 0,
16
+ "no-continue": 1,
17
+ "no-magic-numbers": 0,
18
+ "no-restricted-syntax": [2, "BreakStatement", "DebuggerStatement", "ForInStatement", "LabeledStatement", "WithStatement"],
19
+ "operator-linebreak": [2, "before"],
20
+ },
21
+
22
+ "overrides": [
23
+ {
24
+ "files": "test/**",
25
+ "rules": {
26
+ "function-paren-newline": 0,
27
+ "max-lines-per-function": 0,
28
+ "max-statements": 0,
29
+ "no-buffer-constructor": 0,
30
+ "no-extend-native": 0,
31
+ "no-throw-literal": 0,
32
+ }
33
+ }
34
+ ]
35
+ }
@@ -0,0 +1,12 @@
1
+ # These are supported funding model platforms
2
+
3
+ github: [ljharb]
4
+ patreon: # Replace with a single Patreon username
5
+ open_collective: # Replace with a single Open Collective username
6
+ ko_fi: # Replace with a single Ko-fi username
7
+ tidelift: npm/qs
8
+ community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9
+ liberapay: # Replace with a single Liberapay username
10
+ issuehunt: # Replace with a single IssueHunt username
11
+ otechie: # Replace with a single Otechie username
12
+ custom: # Replace with a single custom sponsorship URL
@@ -0,0 +1,13 @@
1
+ {
2
+ "all": true,
3
+ "check-coverage": false,
4
+ "reporter": ["text-summary", "text", "html", "json"],
5
+ "lines": 86,
6
+ "statements": 85.93,
7
+ "functions": 82.43,
8
+ "branches": 76.06,
9
+ "exclude": [
10
+ "coverage",
11
+ "dist"
12
+ ]
13
+ }