es6-crawler-detect 3.1.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +27 -15
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +29 -18
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1399 -11
  344. package/src/lib/crawler/exclusions.js +56 -9
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +123 -132
  348. package/test/lib/crawler.test.js +83 -24
  349. package/test/lib/database/crawlers.txt +3651 -0
  350. package/test/lib/database/devices.txt +165636 -0
  351. package/webpack.common.js +20 -20
  352. package/webpack.dev.js +7 -7
  353. package/webpack.prod.js +14 -15
@@ -0,0 +1,456 @@
1
+ # body-parser
2
+
3
+ [![NPM Version][npm-image]][npm-url]
4
+ [![NPM Downloads][downloads-image]][downloads-url]
5
+ [![Build Status][github-actions-ci-image]][github-actions-ci-url]
6
+ [![Test Coverage][coveralls-image]][coveralls-url]
7
+
8
+ Node.js body parsing middleware.
9
+
10
+ Parse incoming request bodies in a middleware before your handlers, available
11
+ under the `req.body` property.
12
+
13
+ **Note** As `req.body`'s shape is based on user-controlled input, all
14
+ properties and values in this object are untrusted and should be validated
15
+ before trusting. For example, `req.body.foo.toString()` may fail in multiple
16
+ ways, for example the `foo` property may not be there or may not be a string,
17
+ and `toString` may not be a function and instead a string or other user input.
18
+
19
+ [Learn about the anatomy of an HTTP transaction in Node.js](https://nodejs.org/en/docs/guides/anatomy-of-an-http-transaction/).
20
+
21
+ _This does not handle multipart bodies_, due to their complex and typically
22
+ large nature. For multipart bodies, you may be interested in the following
23
+ modules:
24
+
25
+ * [busboy](https://www.npmjs.org/package/busboy#readme) and
26
+ [connect-busboy](https://www.npmjs.org/package/connect-busboy#readme)
27
+ * [multiparty](https://www.npmjs.org/package/multiparty#readme) and
28
+ [connect-multiparty](https://www.npmjs.org/package/connect-multiparty#readme)
29
+ * [formidable](https://www.npmjs.org/package/formidable#readme)
30
+ * [multer](https://www.npmjs.org/package/multer#readme)
31
+
32
+ This module provides the following parsers:
33
+
34
+ * [JSON body parser](#bodyparserjsonoptions)
35
+ * [Raw body parser](#bodyparserrawoptions)
36
+ * [Text body parser](#bodyparsertextoptions)
37
+ * [URL-encoded form body parser](#bodyparserurlencodedoptions)
38
+
39
+ Other body parsers you might be interested in:
40
+
41
+ - [body](https://www.npmjs.org/package/body#readme)
42
+ - [co-body](https://www.npmjs.org/package/co-body#readme)
43
+
44
+ ## Installation
45
+
46
+ ```sh
47
+ $ npm install body-parser
48
+ ```
49
+
50
+ ## API
51
+
52
+ ```js
53
+ var bodyParser = require('body-parser')
54
+ ```
55
+
56
+ The `bodyParser` object exposes various factories to create middlewares. All
57
+ middlewares will populate the `req.body` property with the parsed body when
58
+ the `Content-Type` request header matches the `type` option, or an empty
59
+ object (`{}`) if there was no body to parse, the `Content-Type` was not matched,
60
+ or an error occurred.
61
+
62
+ The various errors returned by this module are described in the
63
+ [errors section](#errors).
64
+
65
+ ### bodyParser.json([options])
66
+
67
+ Returns middleware that only parses `json` and only looks at requests where
68
+ the `Content-Type` header matches the `type` option. This parser accepts any
69
+ Unicode encoding of the body and supports automatic inflation of `gzip` and
70
+ `deflate` encodings.
71
+
72
+ A new `body` object containing the parsed data is populated on the `request`
73
+ object after the middleware (i.e. `req.body`).
74
+
75
+ #### Options
76
+
77
+ The `json` function takes an optional `options` object that may contain any of
78
+ the following keys:
79
+
80
+ ##### inflate
81
+
82
+ When set to `true`, then deflated (compressed) bodies will be inflated; when
83
+ `false`, deflated bodies are rejected. Defaults to `true`.
84
+
85
+ ##### limit
86
+
87
+ Controls the maximum request body size. If this is a number, then the value
88
+ specifies the number of bytes; if it is a string, the value is passed to the
89
+ [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
90
+ to `'100kb'`.
91
+
92
+ ##### reviver
93
+
94
+ The `reviver` option is passed directly to `JSON.parse` as the second
95
+ argument. You can find more information on this argument
96
+ [in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter).
97
+
98
+ ##### strict
99
+
100
+ When set to `true`, will only accept arrays and objects; when `false` will
101
+ accept anything `JSON.parse` accepts. Defaults to `true`.
102
+
103
+ ##### type
104
+
105
+ The `type` option is used to determine what media type the middleware will
106
+ parse. This option can be a string, array of strings, or a function. If not a
107
+ function, `type` option is passed directly to the
108
+ [type-is](https://www.npmjs.org/package/type-is#readme) library and this can
109
+ be an extension name (like `json`), a mime type (like `application/json`), or
110
+ a mime type with a wildcard (like `*/*` or `*/json`). If a function, the `type`
111
+ option is called as `fn(req)` and the request is parsed if it returns a truthy
112
+ value. Defaults to `application/json`.
113
+
114
+ ##### verify
115
+
116
+ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
117
+ where `buf` is a `Buffer` of the raw request body and `encoding` is the
118
+ encoding of the request. The parsing can be aborted by throwing an error.
119
+
120
+ ### bodyParser.raw([options])
121
+
122
+ Returns middleware that parses all bodies as a `Buffer` and only looks at
123
+ requests where the `Content-Type` header matches the `type` option. This
124
+ parser supports automatic inflation of `gzip` and `deflate` encodings.
125
+
126
+ A new `body` object containing the parsed data is populated on the `request`
127
+ object after the middleware (i.e. `req.body`). This will be a `Buffer` object
128
+ of the body.
129
+
130
+ #### Options
131
+
132
+ The `raw` function takes an optional `options` object that may contain any of
133
+ the following keys:
134
+
135
+ ##### inflate
136
+
137
+ When set to `true`, then deflated (compressed) bodies will be inflated; when
138
+ `false`, deflated bodies are rejected. Defaults to `true`.
139
+
140
+ ##### limit
141
+
142
+ Controls the maximum request body size. If this is a number, then the value
143
+ specifies the number of bytes; if it is a string, the value is passed to the
144
+ [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
145
+ to `'100kb'`.
146
+
147
+ ##### type
148
+
149
+ The `type` option is used to determine what media type the middleware will
150
+ parse. This option can be a string, array of strings, or a function.
151
+ If not a function, `type` option is passed directly to the
152
+ [type-is](https://www.npmjs.org/package/type-is#readme) library and this
153
+ can be an extension name (like `bin`), a mime type (like
154
+ `application/octet-stream`), or a mime type with a wildcard (like `*/*` or
155
+ `application/*`). If a function, the `type` option is called as `fn(req)`
156
+ and the request is parsed if it returns a truthy value. Defaults to
157
+ `application/octet-stream`.
158
+
159
+ ##### verify
160
+
161
+ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
162
+ where `buf` is a `Buffer` of the raw request body and `encoding` is the
163
+ encoding of the request. The parsing can be aborted by throwing an error.
164
+
165
+ ### bodyParser.text([options])
166
+
167
+ Returns middleware that parses all bodies as a string and only looks at
168
+ requests where the `Content-Type` header matches the `type` option. This
169
+ parser supports automatic inflation of `gzip` and `deflate` encodings.
170
+
171
+ A new `body` string containing the parsed data is populated on the `request`
172
+ object after the middleware (i.e. `req.body`). This will be a string of the
173
+ body.
174
+
175
+ #### Options
176
+
177
+ The `text` function takes an optional `options` object that may contain any of
178
+ the following keys:
179
+
180
+ ##### defaultCharset
181
+
182
+ Specify the default character set for the text content if the charset is not
183
+ specified in the `Content-Type` header of the request. Defaults to `utf-8`.
184
+
185
+ ##### inflate
186
+
187
+ When set to `true`, then deflated (compressed) bodies will be inflated; when
188
+ `false`, deflated bodies are rejected. Defaults to `true`.
189
+
190
+ ##### limit
191
+
192
+ Controls the maximum request body size. If this is a number, then the value
193
+ specifies the number of bytes; if it is a string, the value is passed to the
194
+ [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
195
+ to `'100kb'`.
196
+
197
+ ##### type
198
+
199
+ The `type` option is used to determine what media type the middleware will
200
+ parse. This option can be a string, array of strings, or a function. If not
201
+ a function, `type` option is passed directly to the
202
+ [type-is](https://www.npmjs.org/package/type-is#readme) library and this can
203
+ be an extension name (like `txt`), a mime type (like `text/plain`), or a mime
204
+ type with a wildcard (like `*/*` or `text/*`). If a function, the `type`
205
+ option is called as `fn(req)` and the request is parsed if it returns a
206
+ truthy value. Defaults to `text/plain`.
207
+
208
+ ##### verify
209
+
210
+ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
211
+ where `buf` is a `Buffer` of the raw request body and `encoding` is the
212
+ encoding of the request. The parsing can be aborted by throwing an error.
213
+
214
+ ### bodyParser.urlencoded([options])
215
+
216
+ Returns middleware that only parses `urlencoded` bodies and only looks at
217
+ requests where the `Content-Type` header matches the `type` option. This
218
+ parser accepts only UTF-8 encoding of the body and supports automatic
219
+ inflation of `gzip` and `deflate` encodings.
220
+
221
+ A new `body` object containing the parsed data is populated on the `request`
222
+ object after the middleware (i.e. `req.body`). This object will contain
223
+ key-value pairs, where the value can be a string or array (when `extended` is
224
+ `false`), or any type (when `extended` is `true`).
225
+
226
+ #### Options
227
+
228
+ The `urlencoded` function takes an optional `options` object that may contain
229
+ any of the following keys:
230
+
231
+ ##### extended
232
+
233
+ The `extended` option allows to choose between parsing the URL-encoded data
234
+ with the `querystring` library (when `false`) or the `qs` library (when
235
+ `true`). The "extended" syntax allows for rich objects and arrays to be
236
+ encoded into the URL-encoded format, allowing for a JSON-like experience
237
+ with URL-encoded. For more information, please
238
+ [see the qs library](https://www.npmjs.org/package/qs#readme).
239
+
240
+ Defaults to `true`, but using the default has been deprecated. Please
241
+ research into the difference between `qs` and `querystring` and choose the
242
+ appropriate setting.
243
+
244
+ ##### inflate
245
+
246
+ When set to `true`, then deflated (compressed) bodies will be inflated; when
247
+ `false`, deflated bodies are rejected. Defaults to `true`.
248
+
249
+ ##### limit
250
+
251
+ Controls the maximum request body size. If this is a number, then the value
252
+ specifies the number of bytes; if it is a string, the value is passed to the
253
+ [bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
254
+ to `'100kb'`.
255
+
256
+ ##### parameterLimit
257
+
258
+ The `parameterLimit` option controls the maximum number of parameters that
259
+ are allowed in the URL-encoded data. If a request contains more parameters
260
+ than this value, a 413 will be returned to the client. Defaults to `1000`.
261
+
262
+ ##### type
263
+
264
+ The `type` option is used to determine what media type the middleware will
265
+ parse. This option can be a string, array of strings, or a function. If not
266
+ a function, `type` option is passed directly to the
267
+ [type-is](https://www.npmjs.org/package/type-is#readme) library and this can
268
+ be an extension name (like `urlencoded`), a mime type (like
269
+ `application/x-www-form-urlencoded`), or a mime type with a wildcard (like
270
+ `*/x-www-form-urlencoded`). If a function, the `type` option is called as
271
+ `fn(req)` and the request is parsed if it returns a truthy value. Defaults
272
+ to `application/x-www-form-urlencoded`.
273
+
274
+ ##### verify
275
+
276
+ The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
277
+ where `buf` is a `Buffer` of the raw request body and `encoding` is the
278
+ encoding of the request. The parsing can be aborted by throwing an error.
279
+
280
+ ## Errors
281
+
282
+ The middlewares provided by this module create errors using the
283
+ [`http-errors` module](https://www.npmjs.com/package/http-errors). The errors
284
+ will typically have a `status`/`statusCode` property that contains the suggested
285
+ HTTP response code, an `expose` property to determine if the `message` property
286
+ should be displayed to the client, a `type` property to determine the type of
287
+ error without matching against the `message`, and a `body` property containing
288
+ the read body, if available.
289
+
290
+ The following are the common errors created, though any error can come through
291
+ for various reasons.
292
+
293
+ ### content encoding unsupported
294
+
295
+ This error will occur when the request had a `Content-Encoding` header that
296
+ contained an encoding but the "inflation" option was set to `false`. The
297
+ `status` property is set to `415`, the `type` property is set to
298
+ `'encoding.unsupported'`, and the `charset` property will be set to the
299
+ encoding that is unsupported.
300
+
301
+ ### entity parse failed
302
+
303
+ This error will occur when the request contained an entity that could not be
304
+ parsed by the middleware. The `status` property is set to `400`, the `type`
305
+ property is set to `'entity.parse.failed'`, and the `body` property is set to
306
+ the entity value that failed parsing.
307
+
308
+ ### entity verify failed
309
+
310
+ This error will occur when the request contained an entity that could not be
311
+ failed verification by the defined `verify` option. The `status` property is
312
+ set to `403`, the `type` property is set to `'entity.verify.failed'`, and the
313
+ `body` property is set to the entity value that failed verification.
314
+
315
+ ### request aborted
316
+
317
+ This error will occur when the request is aborted by the client before reading
318
+ the body has finished. The `received` property will be set to the number of
319
+ bytes received before the request was aborted and the `expected` property is
320
+ set to the number of expected bytes. The `status` property is set to `400`
321
+ and `type` property is set to `'request.aborted'`.
322
+
323
+ ### request entity too large
324
+
325
+ This error will occur when the request body's size is larger than the "limit"
326
+ option. The `limit` property will be set to the byte limit and the `length`
327
+ property will be set to the request body's length. The `status` property is
328
+ set to `413` and the `type` property is set to `'entity.too.large'`.
329
+
330
+ ### request size did not match content length
331
+
332
+ This error will occur when the request's length did not match the length from
333
+ the `Content-Length` header. This typically occurs when the request is malformed,
334
+ typically when the `Content-Length` header was calculated based on characters
335
+ instead of bytes. The `status` property is set to `400` and the `type` property
336
+ is set to `'request.size.invalid'`.
337
+
338
+ ### stream encoding should not be set
339
+
340
+ This error will occur when something called the `req.setEncoding` method prior
341
+ to this middleware. This module operates directly on bytes only and you cannot
342
+ call `req.setEncoding` when using this module. The `status` property is set to
343
+ `500` and the `type` property is set to `'stream.encoding.set'`.
344
+
345
+ ### too many parameters
346
+
347
+ This error will occur when the content of the request exceeds the configured
348
+ `parameterLimit` for the `urlencoded` parser. The `status` property is set to
349
+ `413` and the `type` property is set to `'parameters.too.many'`.
350
+
351
+ ### unsupported charset "BOGUS"
352
+
353
+ This error will occur when the request had a charset parameter in the
354
+ `Content-Type` header, but the `iconv-lite` module does not support it OR the
355
+ parser does not support it. The charset is contained in the message as well
356
+ as in the `charset` property. The `status` property is set to `415`, the
357
+ `type` property is set to `'charset.unsupported'`, and the `charset` property
358
+ is set to the charset that is unsupported.
359
+
360
+ ### unsupported content encoding "bogus"
361
+
362
+ This error will occur when the request had a `Content-Encoding` header that
363
+ contained an unsupported encoding. The encoding is contained in the message
364
+ as well as in the `encoding` property. The `status` property is set to `415`,
365
+ the `type` property is set to `'encoding.unsupported'`, and the `encoding`
366
+ property is set to the encoding that is unsupported.
367
+
368
+ ## Examples
369
+
370
+ ### Express/Connect top-level generic
371
+
372
+ This example demonstrates adding a generic JSON and URL-encoded parser as a
373
+ top-level middleware, which will parse the bodies of all incoming requests.
374
+ This is the simplest setup.
375
+
376
+ ```js
377
+ var express = require('express')
378
+ var bodyParser = require('body-parser')
379
+
380
+ var app = express()
381
+
382
+ // parse application/x-www-form-urlencoded
383
+ app.use(bodyParser.urlencoded({ extended: false }))
384
+
385
+ // parse application/json
386
+ app.use(bodyParser.json())
387
+
388
+ app.use(function (req, res) {
389
+ res.setHeader('Content-Type', 'text/plain')
390
+ res.write('you posted:\n')
391
+ res.end(JSON.stringify(req.body, null, 2))
392
+ })
393
+ ```
394
+
395
+ ### Express route-specific
396
+
397
+ This example demonstrates adding body parsers specifically to the routes that
398
+ need them. In general, this is the most recommended way to use body-parser with
399
+ Express.
400
+
401
+ ```js
402
+ var express = require('express')
403
+ var bodyParser = require('body-parser')
404
+
405
+ var app = express()
406
+
407
+ // create application/json parser
408
+ var jsonParser = bodyParser.json()
409
+
410
+ // create application/x-www-form-urlencoded parser
411
+ var urlencodedParser = bodyParser.urlencoded({ extended: false })
412
+
413
+ // POST /login gets urlencoded bodies
414
+ app.post('/login', urlencodedParser, function (req, res) {
415
+ res.send('welcome, ' + req.body.username)
416
+ })
417
+
418
+ // POST /api/users gets JSON bodies
419
+ app.post('/api/users', jsonParser, function (req, res) {
420
+ // create user in req.body
421
+ })
422
+ ```
423
+
424
+ ### Change accepted type for parsers
425
+
426
+ All the parsers accept a `type` option which allows you to change the
427
+ `Content-Type` that the middleware will parse.
428
+
429
+ ```js
430
+ var express = require('express')
431
+ var bodyParser = require('body-parser')
432
+
433
+ var app = express()
434
+
435
+ // parse various different custom JSON types as JSON
436
+ app.use(bodyParser.json({ type: 'application/*+json' }))
437
+
438
+ // parse some custom thing into a Buffer
439
+ app.use(bodyParser.raw({ type: 'application/vnd.custom-type' }))
440
+
441
+ // parse an HTML body into a string
442
+ app.use(bodyParser.text({ type: 'text/html' }))
443
+ ```
444
+
445
+ ## License
446
+
447
+ [MIT](LICENSE)
448
+
449
+ [npm-image]: https://img.shields.io/npm/v/body-parser.svg
450
+ [npm-url]: https://npmjs.org/package/body-parser
451
+ [coveralls-image]: https://img.shields.io/coveralls/expressjs/body-parser/master.svg
452
+ [coveralls-url]: https://coveralls.io/r/expressjs/body-parser?branch=master
453
+ [downloads-image]: https://img.shields.io/npm/dm/body-parser.svg
454
+ [downloads-url]: https://npmjs.org/package/body-parser
455
+ [github-actions-ci-image]: https://img.shields.io/github/workflow/status/expressjs/body-parser/ci/master?label=ci
456
+ [github-actions-ci-url]: https://github.com/expressjs/body-parser?query=workflow%3Aci
@@ -0,0 +1,157 @@
1
+ /*!
2
+ * body-parser
3
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
+ * MIT Licensed
5
+ */
6
+
7
+ 'use strict'
8
+
9
+ /**
10
+ * Module dependencies.
11
+ * @private
12
+ */
13
+
14
+ var deprecate = require('depd')('body-parser')
15
+
16
+ /**
17
+ * Cache of loaded parsers.
18
+ * @private
19
+ */
20
+
21
+ var parsers = Object.create(null)
22
+
23
+ /**
24
+ * @typedef Parsers
25
+ * @type {function}
26
+ * @property {function} json
27
+ * @property {function} raw
28
+ * @property {function} text
29
+ * @property {function} urlencoded
30
+ */
31
+
32
+ /**
33
+ * Module exports.
34
+ * @type {Parsers}
35
+ */
36
+
37
+ exports = module.exports = deprecate.function(bodyParser,
38
+ 'bodyParser: use individual json/urlencoded middlewares')
39
+
40
+ /**
41
+ * JSON parser.
42
+ * @public
43
+ */
44
+
45
+ Object.defineProperty(exports, 'json', {
46
+ configurable: true,
47
+ enumerable: true,
48
+ get: createParserGetter('json')
49
+ })
50
+
51
+ /**
52
+ * Raw parser.
53
+ * @public
54
+ */
55
+
56
+ Object.defineProperty(exports, 'raw', {
57
+ configurable: true,
58
+ enumerable: true,
59
+ get: createParserGetter('raw')
60
+ })
61
+
62
+ /**
63
+ * Text parser.
64
+ * @public
65
+ */
66
+
67
+ Object.defineProperty(exports, 'text', {
68
+ configurable: true,
69
+ enumerable: true,
70
+ get: createParserGetter('text')
71
+ })
72
+
73
+ /**
74
+ * URL-encoded parser.
75
+ * @public
76
+ */
77
+
78
+ Object.defineProperty(exports, 'urlencoded', {
79
+ configurable: true,
80
+ enumerable: true,
81
+ get: createParserGetter('urlencoded')
82
+ })
83
+
84
+ /**
85
+ * Create a middleware to parse json and urlencoded bodies.
86
+ *
87
+ * @param {object} [options]
88
+ * @return {function}
89
+ * @deprecated
90
+ * @public
91
+ */
92
+
93
+ function bodyParser (options) {
94
+ var opts = {}
95
+
96
+ // exclude type option
97
+ if (options) {
98
+ for (var prop in options) {
99
+ if (prop !== 'type') {
100
+ opts[prop] = options[prop]
101
+ }
102
+ }
103
+ }
104
+
105
+ var _urlencoded = exports.urlencoded(opts)
106
+ var _json = exports.json(opts)
107
+
108
+ return function bodyParser (req, res, next) {
109
+ _json(req, res, function (err) {
110
+ if (err) return next(err)
111
+ _urlencoded(req, res, next)
112
+ })
113
+ }
114
+ }
115
+
116
+ /**
117
+ * Create a getter for loading a parser.
118
+ * @private
119
+ */
120
+
121
+ function createParserGetter (name) {
122
+ return function get () {
123
+ return loadParser(name)
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Load a parser module.
129
+ * @private
130
+ */
131
+
132
+ function loadParser (parserName) {
133
+ var parser = parsers[parserName]
134
+
135
+ if (parser !== undefined) {
136
+ return parser
137
+ }
138
+
139
+ // this uses a switch for static require analysis
140
+ switch (parserName) {
141
+ case 'json':
142
+ parser = require('./lib/types/json')
143
+ break
144
+ case 'raw':
145
+ parser = require('./lib/types/raw')
146
+ break
147
+ case 'text':
148
+ parser = require('./lib/types/text')
149
+ break
150
+ case 'urlencoded':
151
+ parser = require('./lib/types/urlencoded')
152
+ break
153
+ }
154
+
155
+ // store to prevent invoking require()
156
+ return (parsers[parserName] = parser)
157
+ }