es6-crawler-detect 3.1.1 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (353) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +27 -15
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +29 -18
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1399 -11
  344. package/src/lib/crawler/exclusions.js +56 -9
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +123 -132
  348. package/test/lib/crawler.test.js +83 -24
  349. package/test/lib/database/crawlers.txt +3651 -0
  350. package/test/lib/database/devices.txt +165636 -0
  351. package/webpack.common.js +20 -20
  352. package/webpack.dev.js +7 -7
  353. package/webpack.prod.js +14 -15
@@ -0,0 +1,286 @@
1
+ /*!
2
+ * raw-body
3
+ * Copyright(c) 2013-2014 Jonathan Ong
4
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
5
+ * MIT Licensed
6
+ */
7
+
8
+ 'use strict'
9
+
10
+ /**
11
+ * Module dependencies.
12
+ * @private
13
+ */
14
+
15
+ var bytes = require('bytes')
16
+ var createError = require('http-errors')
17
+ var iconv = require('iconv-lite')
18
+ var unpipe = require('unpipe')
19
+
20
+ /**
21
+ * Module exports.
22
+ * @public
23
+ */
24
+
25
+ module.exports = getRawBody
26
+
27
+ /**
28
+ * Module variables.
29
+ * @private
30
+ */
31
+
32
+ var ICONV_ENCODING_MESSAGE_REGEXP = /^Encoding not recognized: /
33
+
34
+ /**
35
+ * Get the decoder for a given encoding.
36
+ *
37
+ * @param {string} encoding
38
+ * @private
39
+ */
40
+
41
+ function getDecoder (encoding) {
42
+ if (!encoding) return null
43
+
44
+ try {
45
+ return iconv.getDecoder(encoding)
46
+ } catch (e) {
47
+ // error getting decoder
48
+ if (!ICONV_ENCODING_MESSAGE_REGEXP.test(e.message)) throw e
49
+
50
+ // the encoding was not found
51
+ throw createError(415, 'specified encoding unsupported', {
52
+ encoding: encoding,
53
+ type: 'encoding.unsupported'
54
+ })
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Get the raw body of a stream (typically HTTP).
60
+ *
61
+ * @param {object} stream
62
+ * @param {object|string|function} [options]
63
+ * @param {function} [callback]
64
+ * @public
65
+ */
66
+
67
+ function getRawBody (stream, options, callback) {
68
+ var done = callback
69
+ var opts = options || {}
70
+
71
+ if (options === true || typeof options === 'string') {
72
+ // short cut for encoding
73
+ opts = {
74
+ encoding: options
75
+ }
76
+ }
77
+
78
+ if (typeof options === 'function') {
79
+ done = options
80
+ opts = {}
81
+ }
82
+
83
+ // validate callback is a function, if provided
84
+ if (done !== undefined && typeof done !== 'function') {
85
+ throw new TypeError('argument callback must be a function')
86
+ }
87
+
88
+ // require the callback without promises
89
+ if (!done && !global.Promise) {
90
+ throw new TypeError('argument callback is required')
91
+ }
92
+
93
+ // get encoding
94
+ var encoding = opts.encoding !== true
95
+ ? opts.encoding
96
+ : 'utf-8'
97
+
98
+ // convert the limit to an integer
99
+ var limit = bytes.parse(opts.limit)
100
+
101
+ // convert the expected length to an integer
102
+ var length = opts.length != null && !isNaN(opts.length)
103
+ ? parseInt(opts.length, 10)
104
+ : null
105
+
106
+ if (done) {
107
+ // classic callback style
108
+ return readStream(stream, encoding, length, limit, done)
109
+ }
110
+
111
+ return new Promise(function executor (resolve, reject) {
112
+ readStream(stream, encoding, length, limit, function onRead (err, buf) {
113
+ if (err) return reject(err)
114
+ resolve(buf)
115
+ })
116
+ })
117
+ }
118
+
119
+ /**
120
+ * Halt a stream.
121
+ *
122
+ * @param {Object} stream
123
+ * @private
124
+ */
125
+
126
+ function halt (stream) {
127
+ // unpipe everything from the stream
128
+ unpipe(stream)
129
+
130
+ // pause stream
131
+ if (typeof stream.pause === 'function') {
132
+ stream.pause()
133
+ }
134
+ }
135
+
136
+ /**
137
+ * Read the data from the stream.
138
+ *
139
+ * @param {object} stream
140
+ * @param {string} encoding
141
+ * @param {number} length
142
+ * @param {number} limit
143
+ * @param {function} callback
144
+ * @public
145
+ */
146
+
147
+ function readStream (stream, encoding, length, limit, callback) {
148
+ var complete = false
149
+ var sync = true
150
+
151
+ // check the length and limit options.
152
+ // note: we intentionally leave the stream paused,
153
+ // so users should handle the stream themselves.
154
+ if (limit !== null && length !== null && length > limit) {
155
+ return done(createError(413, 'request entity too large', {
156
+ expected: length,
157
+ length: length,
158
+ limit: limit,
159
+ type: 'entity.too.large'
160
+ }))
161
+ }
162
+
163
+ // streams1: assert request encoding is buffer.
164
+ // streams2+: assert the stream encoding is buffer.
165
+ // stream._decoder: streams1
166
+ // state.encoding: streams2
167
+ // state.decoder: streams2, specifically < 0.10.6
168
+ var state = stream._readableState
169
+ if (stream._decoder || (state && (state.encoding || state.decoder))) {
170
+ // developer error
171
+ return done(createError(500, 'stream encoding should not be set', {
172
+ type: 'stream.encoding.set'
173
+ }))
174
+ }
175
+
176
+ var received = 0
177
+ var decoder
178
+
179
+ try {
180
+ decoder = getDecoder(encoding)
181
+ } catch (err) {
182
+ return done(err)
183
+ }
184
+
185
+ var buffer = decoder
186
+ ? ''
187
+ : []
188
+
189
+ // attach listeners
190
+ stream.on('aborted', onAborted)
191
+ stream.on('close', cleanup)
192
+ stream.on('data', onData)
193
+ stream.on('end', onEnd)
194
+ stream.on('error', onEnd)
195
+
196
+ // mark sync section complete
197
+ sync = false
198
+
199
+ function done () {
200
+ var args = new Array(arguments.length)
201
+
202
+ // copy arguments
203
+ for (var i = 0; i < args.length; i++) {
204
+ args[i] = arguments[i]
205
+ }
206
+
207
+ // mark complete
208
+ complete = true
209
+
210
+ if (sync) {
211
+ process.nextTick(invokeCallback)
212
+ } else {
213
+ invokeCallback()
214
+ }
215
+
216
+ function invokeCallback () {
217
+ cleanup()
218
+
219
+ if (args[0]) {
220
+ // halt the stream on error
221
+ halt(stream)
222
+ }
223
+
224
+ callback.apply(null, args)
225
+ }
226
+ }
227
+
228
+ function onAborted () {
229
+ if (complete) return
230
+
231
+ done(createError(400, 'request aborted', {
232
+ code: 'ECONNABORTED',
233
+ expected: length,
234
+ length: length,
235
+ received: received,
236
+ type: 'request.aborted'
237
+ }))
238
+ }
239
+
240
+ function onData (chunk) {
241
+ if (complete) return
242
+
243
+ received += chunk.length
244
+
245
+ if (limit !== null && received > limit) {
246
+ done(createError(413, 'request entity too large', {
247
+ limit: limit,
248
+ received: received,
249
+ type: 'entity.too.large'
250
+ }))
251
+ } else if (decoder) {
252
+ buffer += decoder.write(chunk)
253
+ } else {
254
+ buffer.push(chunk)
255
+ }
256
+ }
257
+
258
+ function onEnd (err) {
259
+ if (complete) return
260
+ if (err) return done(err)
261
+
262
+ if (length !== null && received !== length) {
263
+ done(createError(400, 'request size did not match content length', {
264
+ expected: length,
265
+ length: length,
266
+ received: received,
267
+ type: 'request.size.invalid'
268
+ }))
269
+ } else {
270
+ var string = decoder
271
+ ? buffer + (decoder.end() || '')
272
+ : Buffer.concat(buffer)
273
+ done(null, string)
274
+ }
275
+ }
276
+
277
+ function cleanup () {
278
+ buffer = null
279
+
280
+ stream.removeListener('aborted', onAborted)
281
+ stream.removeListener('data', onData)
282
+ stream.removeListener('end', onEnd)
283
+ stream.removeListener('error', onEnd)
284
+ stream.removeListener('close', cleanup)
285
+ }
286
+ }
@@ -0,0 +1,48 @@
1
+ {
2
+ "name": "raw-body",
3
+ "description": "Get and validate the raw body of a readable stream.",
4
+ "version": "2.4.2",
5
+ "author": "Jonathan Ong <me@jongleberry.com> (http://jongleberry.com)",
6
+ "contributors": [
7
+ "Douglas Christopher Wilson <doug@somethingdoug.com>",
8
+ "Raynos <raynos2@gmail.com>"
9
+ ],
10
+ "license": "MIT",
11
+ "repository": "stream-utils/raw-body",
12
+ "dependencies": {
13
+ "bytes": "3.1.1",
14
+ "http-errors": "1.8.1",
15
+ "iconv-lite": "0.4.24",
16
+ "unpipe": "1.0.0"
17
+ },
18
+ "devDependencies": {
19
+ "bluebird": "3.7.2",
20
+ "eslint": "7.32.0",
21
+ "eslint-config-standard": "14.1.1",
22
+ "eslint-plugin-import": "2.25.3",
23
+ "eslint-plugin-markdown": "2.2.1",
24
+ "eslint-plugin-node": "11.1.0",
25
+ "eslint-plugin-promise": "5.1.1",
26
+ "eslint-plugin-standard": "4.1.0",
27
+ "mocha": "9.1.3",
28
+ "nyc": "15.1.0",
29
+ "readable-stream": "2.3.7",
30
+ "safe-buffer": "5.2.1"
31
+ },
32
+ "engines": {
33
+ "node": ">= 0.8"
34
+ },
35
+ "files": [
36
+ "HISTORY.md",
37
+ "LICENSE",
38
+ "README.md",
39
+ "index.d.ts",
40
+ "index.js"
41
+ ],
42
+ "scripts": {
43
+ "lint": "eslint .",
44
+ "test": "mocha --trace-deprecation --reporter spec --bail --check-leaks test/",
45
+ "test-ci": "nyc --reporter=lcov --reporter=text npm test",
46
+ "test-cov": "nyc --reporter=html --reporter=text npm test"
47
+ }
48
+ }
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) Feross Aboukhadijeh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.