es6-crawler-detect 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (351) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +32 -26
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +30 -19
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1402 -15
  344. package/src/lib/crawler/exclusions.js +62 -15
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +135 -139
  348. package/test/lib/crawler.test.js +52 -35
  349. package/webpack.common.js +20 -20
  350. package/webpack.dev.js +7 -7
  351. package/webpack.prod.js +14 -15
@@ -0,0 +1,284 @@
1
+ 2.4.2 / 2021-11-16
2
+ ==================
3
+
4
+ * deps: bytes@3.1.1
5
+ * deps: http-errors@1.8.1
6
+ - deps: setprototypeof@1.2.0
7
+ - deps: toidentifier@1.0.1
8
+
9
+ 2.4.1 / 2019-06-25
10
+ ==================
11
+
12
+ * deps: http-errors@1.7.3
13
+ - deps: inherits@2.0.4
14
+
15
+ 2.4.0 / 2019-04-17
16
+ ==================
17
+
18
+ * deps: bytes@3.1.0
19
+ - Add petabyte (`pb`) support
20
+ * deps: http-errors@1.7.2
21
+ - Set constructor name when possible
22
+ - deps: setprototypeof@1.1.1
23
+ - deps: statuses@'>= 1.5.0 < 2'
24
+ * deps: iconv-lite@0.4.24
25
+ - Added encoding MIK
26
+
27
+ 2.3.3 / 2018-05-08
28
+ ==================
29
+
30
+ * deps: http-errors@1.6.3
31
+ - deps: depd@~1.1.2
32
+ - deps: setprototypeof@1.1.0
33
+ - deps: statuses@'>= 1.3.1 < 2'
34
+ * deps: iconv-lite@0.4.23
35
+ - Fix loading encoding with year appended
36
+ - Fix deprecation warnings on Node.js 10+
37
+
38
+ 2.3.2 / 2017-09-09
39
+ ==================
40
+
41
+ * deps: iconv-lite@0.4.19
42
+ - Fix ISO-8859-1 regression
43
+ - Update Windows-1255
44
+
45
+ 2.3.1 / 2017-09-07
46
+ ==================
47
+
48
+ * deps: bytes@3.0.0
49
+ * deps: http-errors@1.6.2
50
+ - deps: depd@1.1.1
51
+ * perf: skip buffer decoding on overage chunk
52
+
53
+ 2.3.0 / 2017-08-04
54
+ ==================
55
+
56
+ * Add TypeScript definitions
57
+ * Use `http-errors` for standard emitted errors
58
+ * deps: bytes@2.5.0
59
+ * deps: iconv-lite@0.4.18
60
+ - Add support for React Native
61
+ - Add a warning if not loaded as utf-8
62
+ - Fix CESU-8 decoding in Node.js 8
63
+ - Improve speed of ISO-8859-1 encoding
64
+
65
+ 2.2.0 / 2017-01-02
66
+ ==================
67
+
68
+ * deps: iconv-lite@0.4.15
69
+ - Added encoding MS-31J
70
+ - Added encoding MS-932
71
+ - Added encoding MS-936
72
+ - Added encoding MS-949
73
+ - Added encoding MS-950
74
+ - Fix GBK/GB18030 handling of Euro character
75
+
76
+ 2.1.7 / 2016-06-19
77
+ ==================
78
+
79
+ * deps: bytes@2.4.0
80
+ * perf: remove double-cleanup on happy path
81
+
82
+ 2.1.6 / 2016-03-07
83
+ ==================
84
+
85
+ * deps: bytes@2.3.0
86
+ - Drop partial bytes on all parsed units
87
+ - Fix parsing byte string that looks like hex
88
+
89
+ 2.1.5 / 2015-11-30
90
+ ==================
91
+
92
+ * deps: bytes@2.2.0
93
+ * deps: iconv-lite@0.4.13
94
+
95
+ 2.1.4 / 2015-09-27
96
+ ==================
97
+
98
+ * Fix masking critical errors from `iconv-lite`
99
+ * deps: iconv-lite@0.4.12
100
+ - Fix CESU-8 decoding in Node.js 4.x
101
+
102
+ 2.1.3 / 2015-09-12
103
+ ==================
104
+
105
+ * Fix sync callback when attaching data listener causes sync read
106
+ - Node.js 0.10 compatibility issue
107
+
108
+ 2.1.2 / 2015-07-05
109
+ ==================
110
+
111
+ * Fix error stack traces to skip `makeError`
112
+ * deps: iconv-lite@0.4.11
113
+ - Add encoding CESU-8
114
+
115
+ 2.1.1 / 2015-06-14
116
+ ==================
117
+
118
+ * Use `unpipe` module for unpiping requests
119
+
120
+ 2.1.0 / 2015-05-28
121
+ ==================
122
+
123
+ * deps: iconv-lite@0.4.10
124
+ - Improved UTF-16 endianness detection
125
+ - Leading BOM is now removed when decoding
126
+ - The encoding UTF-16 without BOM now defaults to UTF-16LE when detection fails
127
+
128
+ 2.0.2 / 2015-05-21
129
+ ==================
130
+
131
+ * deps: bytes@2.1.0
132
+ - Slight optimizations
133
+
134
+ 2.0.1 / 2015-05-10
135
+ ==================
136
+
137
+ * Fix a false-positive when unpiping in Node.js 0.8
138
+
139
+ 2.0.0 / 2015-05-08
140
+ ==================
141
+
142
+ * Return a promise without callback instead of thunk
143
+ * deps: bytes@2.0.1
144
+ - units no longer case sensitive when parsing
145
+
146
+ 1.3.4 / 2015-04-15
147
+ ==================
148
+
149
+ * Fix hanging callback if request aborts during read
150
+ * deps: iconv-lite@0.4.8
151
+ - Add encoding alias UNICODE-1-1-UTF-7
152
+
153
+ 1.3.3 / 2015-02-08
154
+ ==================
155
+
156
+ * deps: iconv-lite@0.4.7
157
+ - Gracefully support enumerables on `Object.prototype`
158
+
159
+ 1.3.2 / 2015-01-20
160
+ ==================
161
+
162
+ * deps: iconv-lite@0.4.6
163
+ - Fix rare aliases of single-byte encodings
164
+
165
+ 1.3.1 / 2014-11-21
166
+ ==================
167
+
168
+ * deps: iconv-lite@0.4.5
169
+ - Fix Windows-31J and X-SJIS encoding support
170
+
171
+ 1.3.0 / 2014-07-20
172
+ ==================
173
+
174
+ * Fully unpipe the stream on error
175
+ - Fixes `Cannot switch to old mode now` error on Node.js 0.10+
176
+
177
+ 1.2.3 / 2014-07-20
178
+ ==================
179
+
180
+ * deps: iconv-lite@0.4.4
181
+ - Added encoding UTF-7
182
+
183
+ 1.2.2 / 2014-06-19
184
+ ==================
185
+
186
+ * Send invalid encoding error to callback
187
+
188
+ 1.2.1 / 2014-06-15
189
+ ==================
190
+
191
+ * deps: iconv-lite@0.4.3
192
+ - Added encodings UTF-16BE and UTF-16 with BOM
193
+
194
+ 1.2.0 / 2014-06-13
195
+ ==================
196
+
197
+ * Passing string as `options` interpreted as encoding
198
+ * Support all encodings from `iconv-lite`
199
+
200
+ 1.1.7 / 2014-06-12
201
+ ==================
202
+
203
+ * use `string_decoder` module from npm
204
+
205
+ 1.1.6 / 2014-05-27
206
+ ==================
207
+
208
+ * check encoding for old streams1
209
+ * support node.js < 0.10.6
210
+
211
+ 1.1.5 / 2014-05-14
212
+ ==================
213
+
214
+ * bump bytes
215
+
216
+ 1.1.4 / 2014-04-19
217
+ ==================
218
+
219
+ * allow true as an option
220
+ * bump bytes
221
+
222
+ 1.1.3 / 2014-03-02
223
+ ==================
224
+
225
+ * fix case when length=null
226
+
227
+ 1.1.2 / 2013-12-01
228
+ ==================
229
+
230
+ * be less strict on state.encoding check
231
+
232
+ 1.1.1 / 2013-11-27
233
+ ==================
234
+
235
+ * add engines
236
+
237
+ 1.1.0 / 2013-11-27
238
+ ==================
239
+
240
+ * add err.statusCode and err.type
241
+ * allow for encoding option to be true
242
+ * pause the stream instead of dumping on error
243
+ * throw if the stream's encoding is set
244
+
245
+ 1.0.1 / 2013-11-19
246
+ ==================
247
+
248
+ * dont support streams1, throw if dev set encoding
249
+
250
+ 1.0.0 / 2013-11-17
251
+ ==================
252
+
253
+ * rename `expected` option to `length`
254
+
255
+ 0.2.0 / 2013-11-15
256
+ ==================
257
+
258
+ * republish
259
+
260
+ 0.1.1 / 2013-11-15
261
+ ==================
262
+
263
+ * use bytes
264
+
265
+ 0.1.0 / 2013-11-11
266
+ ==================
267
+
268
+ * generator support
269
+
270
+ 0.0.3 / 2013-10-10
271
+ ==================
272
+
273
+ * update repo
274
+
275
+ 0.0.2 / 2013-09-14
276
+ ==================
277
+
278
+ * dump stream on bad headers
279
+ * listen to events after defining received and buffers
280
+
281
+ 0.0.1 / 2013-09-14
282
+ ==================
283
+
284
+ * Initial release
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2013-2014 Jonathan Ong <me@jongleberry.com>
4
+ Copyright (c) 2014-2015 Douglas Christopher Wilson <doug@somethingdoug.com>
5
+
6
+ Permission is hereby granted, free of charge, to any person obtaining a copy
7
+ of this software and associated documentation files (the "Software"), to deal
8
+ in the Software without restriction, including without limitation the rights
9
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
+ copies of the Software, and to permit persons to whom the Software is
11
+ furnished to do so, subject to the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be included in
14
+ all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
+ THE SOFTWARE.
@@ -0,0 +1,217 @@
1
+ # raw-body
2
+
3
+ [![NPM Version][npm-image]][npm-url]
4
+ [![NPM Downloads][downloads-image]][downloads-url]
5
+ [![Node.js Version][node-version-image]][node-version-url]
6
+ [![Build status][github-actions-ci-image]][github-actions-ci-url]
7
+ [![Test coverage][coveralls-image]][coveralls-url]
8
+
9
+ Gets the entire buffer of a stream either as a `Buffer` or a string.
10
+ Validates the stream's length against an expected length and maximum limit.
11
+ Ideal for parsing request bodies.
12
+
13
+ ## Install
14
+
15
+ This is a [Node.js](https://nodejs.org/en/) module available through the
16
+ [npm registry](https://www.npmjs.com/). Installation is done using the
17
+ [`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
18
+
19
+ ```sh
20
+ $ npm install raw-body
21
+ ```
22
+
23
+ ### TypeScript
24
+
25
+ This module includes a [TypeScript](https://www.typescriptlang.org/)
26
+ declaration file to enable auto complete in compatible editors and type
27
+ information for TypeScript projects. This module depends on the Node.js
28
+ types, so install `@types/node`:
29
+
30
+ ```sh
31
+ $ npm install @types/node
32
+ ```
33
+
34
+ ## API
35
+
36
+ ```js
37
+ var getRawBody = require('raw-body')
38
+ ```
39
+
40
+ ### getRawBody(stream, [options], [callback])
41
+
42
+ **Returns a promise if no callback specified and global `Promise` exists.**
43
+
44
+ Options:
45
+
46
+ - `length` - The length of the stream.
47
+ If the contents of the stream do not add up to this length,
48
+ an `400` error code is returned.
49
+ - `limit` - The byte limit of the body.
50
+ This is the number of bytes or any string format supported by
51
+ [bytes](https://www.npmjs.com/package/bytes),
52
+ for example `1000`, `'500kb'` or `'3mb'`.
53
+ If the body ends up being larger than this limit,
54
+ a `413` error code is returned.
55
+ - `encoding` - The encoding to use to decode the body into a string.
56
+ By default, a `Buffer` instance will be returned when no encoding is specified.
57
+ Most likely, you want `utf-8`, so setting `encoding` to `true` will decode as `utf-8`.
58
+ You can use any type of encoding supported by [iconv-lite](https://www.npmjs.org/package/iconv-lite#readme).
59
+
60
+ You can also pass a string in place of options to just specify the encoding.
61
+
62
+ If an error occurs, the stream will be paused, everything unpiped,
63
+ and you are responsible for correctly disposing the stream.
64
+ For HTTP requests, no handling is required if you send a response.
65
+ For streams that use file descriptors, you should `stream.destroy()` or `stream.close()` to prevent leaks.
66
+
67
+ ## Errors
68
+
69
+ This module creates errors depending on the error condition during reading.
70
+ The error may be an error from the underlying Node.js implementation, but is
71
+ otherwise an error created by this module, which has the following attributes:
72
+
73
+ * `limit` - the limit in bytes
74
+ * `length` and `expected` - the expected length of the stream
75
+ * `received` - the received bytes
76
+ * `encoding` - the invalid encoding
77
+ * `status` and `statusCode` - the corresponding status code for the error
78
+ * `type` - the error type
79
+
80
+ ### Types
81
+
82
+ The errors from this module have a `type` property which allows for the progamatic
83
+ determination of the type of error returned.
84
+
85
+ #### encoding.unsupported
86
+
87
+ This error will occur when the `encoding` option is specified, but the value does
88
+ not map to an encoding supported by the [iconv-lite](https://www.npmjs.org/package/iconv-lite#readme)
89
+ module.
90
+
91
+ #### entity.too.large
92
+
93
+ This error will occur when the `limit` option is specified, but the stream has
94
+ an entity that is larger.
95
+
96
+ #### request.aborted
97
+
98
+ This error will occur when the request stream is aborted by the client before
99
+ reading the body has finished.
100
+
101
+ #### request.size.invalid
102
+
103
+ This error will occur when the `length` option is specified, but the stream has
104
+ emitted more bytes.
105
+
106
+ #### stream.encoding.set
107
+
108
+ This error will occur when the given stream has an encoding set on it, making it
109
+ a decoded stream. The stream should not have an encoding set and is expected to
110
+ emit `Buffer` objects.
111
+
112
+ ## Examples
113
+
114
+ ### Simple Express example
115
+
116
+ ```js
117
+ var contentType = require('content-type')
118
+ var express = require('express')
119
+ var getRawBody = require('raw-body')
120
+
121
+ var app = express()
122
+
123
+ app.use(function (req, res, next) {
124
+ getRawBody(req, {
125
+ length: req.headers['content-length'],
126
+ limit: '1mb',
127
+ encoding: contentType.parse(req).parameters.charset
128
+ }, function (err, string) {
129
+ if (err) return next(err)
130
+ req.text = string
131
+ next()
132
+ })
133
+ })
134
+
135
+ // now access req.text
136
+ ```
137
+
138
+ ### Simple Koa example
139
+
140
+ ```js
141
+ var contentType = require('content-type')
142
+ var getRawBody = require('raw-body')
143
+ var koa = require('koa')
144
+
145
+ var app = koa()
146
+
147
+ app.use(function * (next) {
148
+ this.text = yield getRawBody(this.req, {
149
+ length: this.req.headers['content-length'],
150
+ limit: '1mb',
151
+ encoding: contentType.parse(this.req).parameters.charset
152
+ })
153
+ yield next
154
+ })
155
+
156
+ // now access this.text
157
+ ```
158
+
159
+ ### Using as a promise
160
+
161
+ To use this library as a promise, simply omit the `callback` and a promise is
162
+ returned, provided that a global `Promise` is defined.
163
+
164
+ ```js
165
+ var getRawBody = require('raw-body')
166
+ var http = require('http')
167
+
168
+ var server = http.createServer(function (req, res) {
169
+ getRawBody(req)
170
+ .then(function (buf) {
171
+ res.statusCode = 200
172
+ res.end(buf.length + ' bytes submitted')
173
+ })
174
+ .catch(function (err) {
175
+ res.statusCode = 500
176
+ res.end(err.message)
177
+ })
178
+ })
179
+
180
+ server.listen(3000)
181
+ ```
182
+
183
+ ### Using with TypeScript
184
+
185
+ ```ts
186
+ import * as getRawBody from 'raw-body';
187
+ import * as http from 'http';
188
+
189
+ const server = http.createServer((req, res) => {
190
+ getRawBody(req)
191
+ .then((buf) => {
192
+ res.statusCode = 200;
193
+ res.end(buf.length + ' bytes submitted');
194
+ })
195
+ .catch((err) => {
196
+ res.statusCode = err.statusCode;
197
+ res.end(err.message);
198
+ });
199
+ });
200
+
201
+ server.listen(3000);
202
+ ```
203
+
204
+ ## License
205
+
206
+ [MIT](LICENSE)
207
+
208
+ [npm-image]: https://img.shields.io/npm/v/raw-body.svg
209
+ [npm-url]: https://npmjs.org/package/raw-body
210
+ [node-version-image]: https://img.shields.io/node/v/raw-body.svg
211
+ [node-version-url]: https://nodejs.org/en/download/
212
+ [coveralls-image]: https://img.shields.io/coveralls/stream-utils/raw-body/master.svg
213
+ [coveralls-url]: https://coveralls.io/r/stream-utils/raw-body?branch=master
214
+ [downloads-image]: https://img.shields.io/npm/dm/raw-body.svg
215
+ [downloads-url]: https://npmjs.org/package/raw-body
216
+ [github-actions-ci-image]: https://img.shields.io/github/workflow/status/stream-utils/raw-body/ci/master?label=ci
217
+ [github-actions-ci-url]: https://github.com/jshttp/stream-utils/raw-body?query=workflow%3Aci
@@ -0,0 +1,87 @@
1
+ import { Readable } from 'stream';
2
+
3
+ declare namespace getRawBody {
4
+ export type Encoding = string | true;
5
+
6
+ export interface Options {
7
+ /**
8
+ * The expected length of the stream.
9
+ */
10
+ length?: number | string | null;
11
+ /**
12
+ * The byte limit of the body. This is the number of bytes or any string
13
+ * format supported by `bytes`, for example `1000`, `'500kb'` or `'3mb'`.
14
+ */
15
+ limit?: number | string | null;
16
+ /**
17
+ * The encoding to use to decode the body into a string. By default, a
18
+ * `Buffer` instance will be returned when no encoding is specified. Most
19
+ * likely, you want `utf-8`, so setting encoding to `true` will decode as
20
+ * `utf-8`. You can use any type of encoding supported by `iconv-lite`.
21
+ */
22
+ encoding?: Encoding | null;
23
+ }
24
+
25
+ export interface RawBodyError extends Error {
26
+ /**
27
+ * The limit in bytes.
28
+ */
29
+ limit?: number;
30
+ /**
31
+ * The expected length of the stream.
32
+ */
33
+ length?: number;
34
+ expected?: number;
35
+ /**
36
+ * The received bytes.
37
+ */
38
+ received?: number;
39
+ /**
40
+ * The encoding.
41
+ */
42
+ encoding?: string;
43
+ /**
44
+ * The corresponding status code for the error.
45
+ */
46
+ status: number;
47
+ statusCode: number;
48
+ /**
49
+ * The error type.
50
+ */
51
+ type: string;
52
+ }
53
+ }
54
+
55
+ /**
56
+ * Gets the entire buffer of a stream either as a `Buffer` or a string.
57
+ * Validates the stream's length against an expected length and maximum
58
+ * limit. Ideal for parsing request bodies.
59
+ */
60
+ declare function getRawBody(
61
+ stream: Readable,
62
+ callback: (err: getRawBody.RawBodyError, body: Buffer) => void
63
+ ): void;
64
+
65
+ declare function getRawBody(
66
+ stream: Readable,
67
+ options: (getRawBody.Options & { encoding: getRawBody.Encoding }) | getRawBody.Encoding,
68
+ callback: (err: getRawBody.RawBodyError, body: string) => void
69
+ ): void;
70
+
71
+ declare function getRawBody(
72
+ stream: Readable,
73
+ options: getRawBody.Options,
74
+ callback: (err: getRawBody.RawBodyError, body: Buffer) => void
75
+ ): void;
76
+
77
+ declare function getRawBody(
78
+ stream: Readable,
79
+ options: (getRawBody.Options & { encoding: getRawBody.Encoding }) | getRawBody.Encoding
80
+ ): Promise<string>;
81
+
82
+ declare function getRawBody(
83
+ stream: Readable,
84
+ options?: getRawBody.Options
85
+ ): Promise<Buffer>;
86
+
87
+ export = getRawBody;