es6-crawler-detect 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (351) hide show
  1. package/.babelrc +2 -1
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/ISSUE_TEMPLATE/bug_report.md +11 -8
  5. package/.github/ISSUE_TEMPLATE/feature_request.md +1 -1
  6. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  7. package/.prettierrc.json +5 -0
  8. package/README.md +27 -17
  9. package/dist/main.bundle.js +1 -1
  10. package/example/node/dist/crawler.min.js +1 -1
  11. package/example/node/node_modules/.bin/mime +15 -0
  12. package/example/node/node_modules/.bin/mime.cmd +7 -0
  13. package/example/node/node_modules/.yarn-integrity +67 -0
  14. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  15. package/example/node/node_modules/accepts/LICENSE +23 -0
  16. package/example/node/node_modules/accepts/README.md +142 -0
  17. package/example/node/node_modules/accepts/index.js +238 -0
  18. package/example/node/node_modules/accepts/package.json +47 -0
  19. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  20. package/example/node/node_modules/array-flatten/README.md +43 -0
  21. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  22. package/example/node/node_modules/array-flatten/package.json +39 -0
  23. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  24. package/example/node/node_modules/body-parser/LICENSE +23 -0
  25. package/example/node/node_modules/body-parser/README.md +456 -0
  26. package/example/node/node_modules/body-parser/index.js +157 -0
  27. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  28. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  29. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  30. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  31. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  32. package/example/node/node_modules/body-parser/package.json +52 -0
  33. package/example/node/node_modules/bytes/History.md +92 -0
  34. package/example/node/node_modules/bytes/LICENSE +23 -0
  35. package/example/node/node_modules/bytes/Readme.md +152 -0
  36. package/example/node/node_modules/bytes/index.js +166 -0
  37. package/example/node/node_modules/bytes/package.json +42 -0
  38. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  39. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  40. package/example/node/node_modules/content-disposition/README.md +142 -0
  41. package/example/node/node_modules/content-disposition/index.js +458 -0
  42. package/example/node/node_modules/content-disposition/package.json +44 -0
  43. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  44. package/example/node/node_modules/content-type/LICENSE +22 -0
  45. package/example/node/node_modules/content-type/README.md +92 -0
  46. package/example/node/node_modules/content-type/index.js +222 -0
  47. package/example/node/node_modules/content-type/package.json +40 -0
  48. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  49. package/example/node/node_modules/cookie/LICENSE +24 -0
  50. package/example/node/node_modules/cookie/README.md +257 -0
  51. package/example/node/node_modules/cookie/index.js +202 -0
  52. package/example/node/node_modules/cookie/package.json +40 -0
  53. package/example/node/node_modules/cookie-signature/History.md +38 -0
  54. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  55. package/example/node/node_modules/cookie-signature/index.js +51 -0
  56. package/example/node/node_modules/cookie-signature/package.json +18 -0
  57. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  58. package/example/node/node_modules/debug/.eslintrc +11 -0
  59. package/example/node/node_modules/debug/.travis.yml +14 -0
  60. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  61. package/example/node/node_modules/debug/LICENSE +19 -0
  62. package/example/node/node_modules/debug/Makefile +50 -0
  63. package/example/node/node_modules/debug/README.md +312 -0
  64. package/example/node/node_modules/debug/component.json +19 -0
  65. package/example/node/node_modules/debug/karma.conf.js +70 -0
  66. package/example/node/node_modules/debug/node.js +1 -0
  67. package/example/node/node_modules/debug/package.json +49 -0
  68. package/example/node/node_modules/debug/src/browser.js +185 -0
  69. package/example/node/node_modules/debug/src/debug.js +202 -0
  70. package/example/node/node_modules/debug/src/index.js +10 -0
  71. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  72. package/example/node/node_modules/debug/src/node.js +248 -0
  73. package/example/node/node_modules/depd/History.md +96 -0
  74. package/example/node/node_modules/depd/LICENSE +22 -0
  75. package/example/node/node_modules/depd/Readme.md +280 -0
  76. package/example/node/node_modules/depd/index.js +522 -0
  77. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  78. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  79. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  80. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  81. package/example/node/node_modules/depd/package.json +41 -0
  82. package/example/node/node_modules/destroy/LICENSE +22 -0
  83. package/example/node/node_modules/destroy/README.md +60 -0
  84. package/example/node/node_modules/destroy/index.js +75 -0
  85. package/example/node/node_modules/destroy/package.json +37 -0
  86. package/example/node/node_modules/ee-first/LICENSE +22 -0
  87. package/example/node/node_modules/ee-first/README.md +80 -0
  88. package/example/node/node_modules/ee-first/index.js +95 -0
  89. package/example/node/node_modules/ee-first/package.json +29 -0
  90. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  91. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  92. package/example/node/node_modules/encodeurl/README.md +128 -0
  93. package/example/node/node_modules/encodeurl/index.js +60 -0
  94. package/example/node/node_modules/encodeurl/package.json +40 -0
  95. package/example/node/node_modules/escape-html/LICENSE +24 -0
  96. package/example/node/node_modules/escape-html/Readme.md +43 -0
  97. package/example/node/node_modules/escape-html/index.js +78 -0
  98. package/example/node/node_modules/escape-html/package.json +24 -0
  99. package/example/node/node_modules/etag/HISTORY.md +83 -0
  100. package/example/node/node_modules/etag/LICENSE +22 -0
  101. package/example/node/node_modules/etag/README.md +159 -0
  102. package/example/node/node_modules/etag/index.js +131 -0
  103. package/example/node/node_modules/etag/package.json +47 -0
  104. package/example/node/node_modules/express/History.md +3510 -0
  105. package/example/node/node_modules/express/LICENSE +24 -0
  106. package/example/node/node_modules/express/Readme.md +158 -0
  107. package/example/node/node_modules/express/index.js +11 -0
  108. package/example/node/node_modules/express/lib/application.js +644 -0
  109. package/example/node/node_modules/express/lib/express.js +116 -0
  110. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  111. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  112. package/example/node/node_modules/express/lib/request.js +525 -0
  113. package/example/node/node_modules/express/lib/response.js +1147 -0
  114. package/example/node/node_modules/express/lib/router/index.js +668 -0
  115. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  116. package/example/node/node_modules/express/lib/router/route.js +216 -0
  117. package/example/node/node_modules/express/lib/utils.js +302 -0
  118. package/example/node/node_modules/express/lib/view.js +182 -0
  119. package/example/node/node_modules/express/package.json +99 -0
  120. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  121. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  122. package/example/node/node_modules/finalhandler/README.md +148 -0
  123. package/example/node/node_modules/finalhandler/index.js +331 -0
  124. package/example/node/node_modules/finalhandler/package.json +45 -0
  125. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  126. package/example/node/node_modules/forwarded/LICENSE +22 -0
  127. package/example/node/node_modules/forwarded/README.md +57 -0
  128. package/example/node/node_modules/forwarded/index.js +90 -0
  129. package/example/node/node_modules/forwarded/package.json +45 -0
  130. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  131. package/example/node/node_modules/fresh/LICENSE +23 -0
  132. package/example/node/node_modules/fresh/README.md +119 -0
  133. package/example/node/node_modules/fresh/index.js +137 -0
  134. package/example/node/node_modules/fresh/package.json +46 -0
  135. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  136. package/example/node/node_modules/http-errors/LICENSE +23 -0
  137. package/example/node/node_modules/http-errors/README.md +169 -0
  138. package/example/node/node_modules/http-errors/index.js +299 -0
  139. package/example/node/node_modules/http-errors/package.json +49 -0
  140. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  141. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  142. package/example/node/node_modules/iconv-lite/README.md +156 -0
  143. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  144. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  145. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  146. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  147. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  148. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  149. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  150. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  151. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  152. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  153. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  158. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  159. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  160. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  161. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  162. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  163. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  164. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  165. package/example/node/node_modules/iconv-lite/package.json +46 -0
  166. package/example/node/node_modules/inherits/LICENSE +16 -0
  167. package/example/node/node_modules/inherits/README.md +42 -0
  168. package/example/node/node_modules/inherits/inherits.js +9 -0
  169. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  170. package/example/node/node_modules/inherits/package.json +29 -0
  171. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  172. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  173. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  174. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  175. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  176. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  177. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  178. package/example/node/node_modules/media-typer/LICENSE +22 -0
  179. package/example/node/node_modules/media-typer/README.md +81 -0
  180. package/example/node/node_modules/media-typer/index.js +270 -0
  181. package/example/node/node_modules/media-typer/package.json +26 -0
  182. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  183. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  184. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  185. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  186. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  187. package/example/node/node_modules/methods/HISTORY.md +29 -0
  188. package/example/node/node_modules/methods/LICENSE +24 -0
  189. package/example/node/node_modules/methods/README.md +51 -0
  190. package/example/node/node_modules/methods/index.js +69 -0
  191. package/example/node/node_modules/methods/package.json +36 -0
  192. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  193. package/example/node/node_modules/mime/LICENSE +21 -0
  194. package/example/node/node_modules/mime/README.md +90 -0
  195. package/example/node/node_modules/mime/cli.js +8 -0
  196. package/example/node/node_modules/mime/mime.js +108 -0
  197. package/example/node/node_modules/mime/package.json +44 -0
  198. package/example/node/node_modules/mime/src/build.js +53 -0
  199. package/example/node/node_modules/mime/src/test.js +60 -0
  200. package/example/node/node_modules/mime/types.json +1 -0
  201. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  202. package/example/node/node_modules/mime-db/LICENSE +22 -0
  203. package/example/node/node_modules/mime-db/README.md +100 -0
  204. package/example/node/node_modules/mime-db/db.json +8471 -0
  205. package/example/node/node_modules/mime-db/index.js +11 -0
  206. package/example/node/node_modules/mime-db/package.json +59 -0
  207. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  208. package/example/node/node_modules/mime-types/LICENSE +23 -0
  209. package/example/node/node_modules/mime-types/README.md +113 -0
  210. package/example/node/node_modules/mime-types/index.js +188 -0
  211. package/example/node/node_modules/mime-types/package.json +44 -0
  212. package/example/node/node_modules/ms/index.js +152 -0
  213. package/example/node/node_modules/ms/license.md +21 -0
  214. package/example/node/node_modules/ms/package.json +37 -0
  215. package/example/node/node_modules/ms/readme.md +51 -0
  216. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  217. package/example/node/node_modules/negotiator/LICENSE +24 -0
  218. package/example/node/node_modules/negotiator/README.md +203 -0
  219. package/example/node/node_modules/negotiator/index.js +124 -0
  220. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  221. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  222. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  223. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  224. package/example/node/node_modules/negotiator/package.json +42 -0
  225. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  226. package/example/node/node_modules/on-finished/LICENSE +23 -0
  227. package/example/node/node_modules/on-finished/README.md +154 -0
  228. package/example/node/node_modules/on-finished/index.js +196 -0
  229. package/example/node/node_modules/on-finished/package.json +31 -0
  230. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  231. package/example/node/node_modules/parseurl/LICENSE +24 -0
  232. package/example/node/node_modules/parseurl/README.md +133 -0
  233. package/example/node/node_modules/parseurl/index.js +158 -0
  234. package/example/node/node_modules/parseurl/package.json +40 -0
  235. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  236. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  237. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  238. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  239. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  240. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  241. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  242. package/example/node/node_modules/proxy-addr/README.md +139 -0
  243. package/example/node/node_modules/proxy-addr/index.js +327 -0
  244. package/example/node/node_modules/proxy-addr/package.json +47 -0
  245. package/example/node/node_modules/qs/.editorconfig +39 -0
  246. package/example/node/node_modules/qs/.eslintignore +2 -0
  247. package/example/node/node_modules/qs/.eslintrc +35 -0
  248. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  249. package/example/node/node_modules/qs/.nycrc +13 -0
  250. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  251. package/example/node/node_modules/qs/LICENSE.md +29 -0
  252. package/example/node/node_modules/qs/README.md +604 -0
  253. package/example/node/node_modules/qs/dist/qs.js +832 -0
  254. package/example/node/node_modules/qs/lib/formats.js +23 -0
  255. package/example/node/node_modules/qs/lib/index.js +11 -0
  256. package/example/node/node_modules/qs/lib/parse.js +257 -0
  257. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  258. package/example/node/node_modules/qs/lib/utils.js +251 -0
  259. package/example/node/node_modules/qs/package.json +69 -0
  260. package/example/node/node_modules/qs/test/parse.js +772 -0
  261. package/example/node/node_modules/qs/test/stringify.js +793 -0
  262. package/example/node/node_modules/qs/test/utils.js +136 -0
  263. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  264. package/example/node/node_modules/range-parser/LICENSE +23 -0
  265. package/example/node/node_modules/range-parser/README.md +84 -0
  266. package/example/node/node_modules/range-parser/index.js +162 -0
  267. package/example/node/node_modules/range-parser/package.json +44 -0
  268. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  269. package/example/node/node_modules/raw-body/LICENSE +22 -0
  270. package/example/node/node_modules/raw-body/README.md +217 -0
  271. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  272. package/example/node/node_modules/raw-body/index.js +286 -0
  273. package/example/node/node_modules/raw-body/package.json +48 -0
  274. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  275. package/example/node/node_modules/safe-buffer/README.md +584 -0
  276. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  277. package/example/node/node_modules/safe-buffer/index.js +65 -0
  278. package/example/node/node_modules/safe-buffer/package.json +51 -0
  279. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  280. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  281. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  282. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  283. package/example/node/node_modules/safer-buffer/package.json +34 -0
  284. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  285. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  286. package/example/node/node_modules/send/HISTORY.md +506 -0
  287. package/example/node/node_modules/send/LICENSE +23 -0
  288. package/example/node/node_modules/send/README.md +327 -0
  289. package/example/node/node_modules/send/index.js +1133 -0
  290. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  291. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  292. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  293. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  294. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  295. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  296. package/example/node/node_modules/send/package.json +61 -0
  297. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  298. package/example/node/node_modules/serve-static/LICENSE +25 -0
  299. package/example/node/node_modules/serve-static/README.md +257 -0
  300. package/example/node/node_modules/serve-static/index.js +210 -0
  301. package/example/node/node_modules/serve-static/package.json +42 -0
  302. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  303. package/example/node/node_modules/setprototypeof/README.md +31 -0
  304. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  305. package/example/node/node_modules/setprototypeof/index.js +17 -0
  306. package/example/node/node_modules/setprototypeof/package.json +38 -0
  307. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  308. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  309. package/example/node/node_modules/statuses/LICENSE +23 -0
  310. package/example/node/node_modules/statuses/README.md +127 -0
  311. package/example/node/node_modules/statuses/codes.json +66 -0
  312. package/example/node/node_modules/statuses/index.js +113 -0
  313. package/example/node/node_modules/statuses/package.json +48 -0
  314. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  315. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  316. package/example/node/node_modules/toidentifier/README.md +61 -0
  317. package/example/node/node_modules/toidentifier/index.js +32 -0
  318. package/example/node/node_modules/toidentifier/package.json +38 -0
  319. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  320. package/example/node/node_modules/type-is/LICENSE +23 -0
  321. package/example/node/node_modules/type-is/README.md +170 -0
  322. package/example/node/node_modules/type-is/index.js +266 -0
  323. package/example/node/node_modules/type-is/package.json +45 -0
  324. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  325. package/example/node/node_modules/unpipe/LICENSE +22 -0
  326. package/example/node/node_modules/unpipe/README.md +43 -0
  327. package/example/node/node_modules/unpipe/index.js +69 -0
  328. package/example/node/node_modules/unpipe/package.json +27 -0
  329. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  330. package/example/node/node_modules/utils-merge/README.md +34 -0
  331. package/example/node/node_modules/utils-merge/index.js +23 -0
  332. package/example/node/node_modules/utils-merge/package.json +40 -0
  333. package/example/node/node_modules/vary/HISTORY.md +39 -0
  334. package/example/node/node_modules/vary/LICENSE +22 -0
  335. package/example/node/node_modules/vary/README.md +101 -0
  336. package/example/node/node_modules/vary/index.js +149 -0
  337. package/example/node/node_modules/vary/package.json +43 -0
  338. package/example/node/package.json +11 -0
  339. package/example/node/server.js +32 -26
  340. package/example/node/yarn.lock +352 -0
  341. package/package.json +30 -19
  342. package/src/index.js +16 -10
  343. package/src/lib/crawler/crawlers.js +1402 -15
  344. package/src/lib/crawler/exclusions.js +62 -15
  345. package/src/lib/crawler/headers.js +24 -15
  346. package/src/lib/crawler/provider.js +11 -13
  347. package/src/lib/crawler.js +135 -139
  348. package/test/lib/crawler.test.js +52 -35
  349. package/webpack.common.js +20 -20
  350. package/webpack.dev.js +7 -7
  351. package/webpack.prod.js +14 -15
@@ -0,0 +1,406 @@
1
+ /* eslint-disable node/no-deprecated-api */
2
+
3
+ 'use strict'
4
+
5
+ var test = require('tape')
6
+
7
+ var buffer = require('buffer')
8
+
9
+ var index = require('./')
10
+ var safer = require('./safer')
11
+ var dangerous = require('./dangerous')
12
+
13
+ /* Inheritance tests */
14
+
15
+ test('Default is Safer', function (t) {
16
+ t.equal(index, safer)
17
+ t.notEqual(safer, dangerous)
18
+ t.notEqual(index, dangerous)
19
+ t.end()
20
+ })
21
+
22
+ test('Is not a function', function (t) {
23
+ [index, safer, dangerous].forEach(function (impl) {
24
+ t.equal(typeof impl, 'object')
25
+ t.equal(typeof impl.Buffer, 'object')
26
+ });
27
+ [buffer].forEach(function (impl) {
28
+ t.equal(typeof impl, 'object')
29
+ t.equal(typeof impl.Buffer, 'function')
30
+ })
31
+ t.end()
32
+ })
33
+
34
+ test('Constructor throws', function (t) {
35
+ [index, safer, dangerous].forEach(function (impl) {
36
+ t.throws(function () { impl.Buffer() })
37
+ t.throws(function () { impl.Buffer(0) })
38
+ t.throws(function () { impl.Buffer('a') })
39
+ t.throws(function () { impl.Buffer('a', 'utf-8') })
40
+ t.throws(function () { return new impl.Buffer() })
41
+ t.throws(function () { return new impl.Buffer(0) })
42
+ t.throws(function () { return new impl.Buffer('a') })
43
+ t.throws(function () { return new impl.Buffer('a', 'utf-8') })
44
+ })
45
+ t.end()
46
+ })
47
+
48
+ test('Safe methods exist', function (t) {
49
+ [index, safer, dangerous].forEach(function (impl) {
50
+ t.equal(typeof impl.Buffer.alloc, 'function', 'alloc')
51
+ t.equal(typeof impl.Buffer.from, 'function', 'from')
52
+ })
53
+ t.end()
54
+ })
55
+
56
+ test('Unsafe methods exist only in Dangerous', function (t) {
57
+ [index, safer].forEach(function (impl) {
58
+ t.equal(typeof impl.Buffer.allocUnsafe, 'undefined')
59
+ t.equal(typeof impl.Buffer.allocUnsafeSlow, 'undefined')
60
+ });
61
+ [dangerous].forEach(function (impl) {
62
+ t.equal(typeof impl.Buffer.allocUnsafe, 'function')
63
+ t.equal(typeof impl.Buffer.allocUnsafeSlow, 'function')
64
+ })
65
+ t.end()
66
+ })
67
+
68
+ test('Generic methods/properties are defined and equal', function (t) {
69
+ ['poolSize', 'isBuffer', 'concat', 'byteLength'].forEach(function (method) {
70
+ [index, safer, dangerous].forEach(function (impl) {
71
+ t.equal(impl.Buffer[method], buffer.Buffer[method], method)
72
+ t.notEqual(typeof impl.Buffer[method], 'undefined', method)
73
+ })
74
+ })
75
+ t.end()
76
+ })
77
+
78
+ test('Built-in buffer static methods/properties are inherited', function (t) {
79
+ Object.keys(buffer).forEach(function (method) {
80
+ if (method === 'SlowBuffer' || method === 'Buffer') return;
81
+ [index, safer, dangerous].forEach(function (impl) {
82
+ t.equal(impl[method], buffer[method], method)
83
+ t.notEqual(typeof impl[method], 'undefined', method)
84
+ })
85
+ })
86
+ t.end()
87
+ })
88
+
89
+ test('Built-in Buffer static methods/properties are inherited', function (t) {
90
+ Object.keys(buffer.Buffer).forEach(function (method) {
91
+ if (method === 'allocUnsafe' || method === 'allocUnsafeSlow') return;
92
+ [index, safer, dangerous].forEach(function (impl) {
93
+ t.equal(impl.Buffer[method], buffer.Buffer[method], method)
94
+ t.notEqual(typeof impl.Buffer[method], 'undefined', method)
95
+ })
96
+ })
97
+ t.end()
98
+ })
99
+
100
+ test('.prototype property of Buffer is inherited', function (t) {
101
+ [index, safer, dangerous].forEach(function (impl) {
102
+ t.equal(impl.Buffer.prototype, buffer.Buffer.prototype, 'prototype')
103
+ t.notEqual(typeof impl.Buffer.prototype, 'undefined', 'prototype')
104
+ })
105
+ t.end()
106
+ })
107
+
108
+ test('All Safer methods are present in Dangerous', function (t) {
109
+ Object.keys(safer).forEach(function (method) {
110
+ if (method === 'Buffer') return;
111
+ [index, safer, dangerous].forEach(function (impl) {
112
+ t.equal(impl[method], safer[method], method)
113
+ if (method !== 'kStringMaxLength') {
114
+ t.notEqual(typeof impl[method], 'undefined', method)
115
+ }
116
+ })
117
+ })
118
+ Object.keys(safer.Buffer).forEach(function (method) {
119
+ [index, safer, dangerous].forEach(function (impl) {
120
+ t.equal(impl.Buffer[method], safer.Buffer[method], method)
121
+ t.notEqual(typeof impl.Buffer[method], 'undefined', method)
122
+ })
123
+ })
124
+ t.end()
125
+ })
126
+
127
+ test('Safe methods from Dangerous methods are present in Safer', function (t) {
128
+ Object.keys(dangerous).forEach(function (method) {
129
+ if (method === 'Buffer') return;
130
+ [index, safer, dangerous].forEach(function (impl) {
131
+ t.equal(impl[method], dangerous[method], method)
132
+ if (method !== 'kStringMaxLength') {
133
+ t.notEqual(typeof impl[method], 'undefined', method)
134
+ }
135
+ })
136
+ })
137
+ Object.keys(dangerous.Buffer).forEach(function (method) {
138
+ if (method === 'allocUnsafe' || method === 'allocUnsafeSlow') return;
139
+ [index, safer, dangerous].forEach(function (impl) {
140
+ t.equal(impl.Buffer[method], dangerous.Buffer[method], method)
141
+ t.notEqual(typeof impl.Buffer[method], 'undefined', method)
142
+ })
143
+ })
144
+ t.end()
145
+ })
146
+
147
+ /* Behaviour tests */
148
+
149
+ test('Methods return Buffers', function (t) {
150
+ [index, safer, dangerous].forEach(function (impl) {
151
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(0)))
152
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(0, 10)))
153
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(0, 'a')))
154
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(10)))
155
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(10, 'x')))
156
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.alloc(9, 'ab')))
157
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from('')))
158
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from('string')))
159
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from('string', 'utf-8')))
160
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from('b25ldHdvdGhyZWU=', 'base64')))
161
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from([0, 42, 3])))
162
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from(new Uint8Array([0, 42, 3]))))
163
+ t.ok(buffer.Buffer.isBuffer(impl.Buffer.from([])))
164
+ });
165
+ ['allocUnsafe', 'allocUnsafeSlow'].forEach(function (method) {
166
+ t.ok(buffer.Buffer.isBuffer(dangerous.Buffer[method](0)))
167
+ t.ok(buffer.Buffer.isBuffer(dangerous.Buffer[method](10)))
168
+ })
169
+ t.end()
170
+ })
171
+
172
+ test('Constructor is buffer.Buffer', function (t) {
173
+ [index, safer, dangerous].forEach(function (impl) {
174
+ t.equal(impl.Buffer.alloc(0).constructor, buffer.Buffer)
175
+ t.equal(impl.Buffer.alloc(0, 10).constructor, buffer.Buffer)
176
+ t.equal(impl.Buffer.alloc(0, 'a').constructor, buffer.Buffer)
177
+ t.equal(impl.Buffer.alloc(10).constructor, buffer.Buffer)
178
+ t.equal(impl.Buffer.alloc(10, 'x').constructor, buffer.Buffer)
179
+ t.equal(impl.Buffer.alloc(9, 'ab').constructor, buffer.Buffer)
180
+ t.equal(impl.Buffer.from('').constructor, buffer.Buffer)
181
+ t.equal(impl.Buffer.from('string').constructor, buffer.Buffer)
182
+ t.equal(impl.Buffer.from('string', 'utf-8').constructor, buffer.Buffer)
183
+ t.equal(impl.Buffer.from('b25ldHdvdGhyZWU=', 'base64').constructor, buffer.Buffer)
184
+ t.equal(impl.Buffer.from([0, 42, 3]).constructor, buffer.Buffer)
185
+ t.equal(impl.Buffer.from(new Uint8Array([0, 42, 3])).constructor, buffer.Buffer)
186
+ t.equal(impl.Buffer.from([]).constructor, buffer.Buffer)
187
+ });
188
+ [0, 10, 100].forEach(function (arg) {
189
+ t.equal(dangerous.Buffer.allocUnsafe(arg).constructor, buffer.Buffer)
190
+ t.equal(dangerous.Buffer.allocUnsafeSlow(arg).constructor, buffer.SlowBuffer(0).constructor)
191
+ })
192
+ t.end()
193
+ })
194
+
195
+ test('Invalid calls throw', function (t) {
196
+ [index, safer, dangerous].forEach(function (impl) {
197
+ t.throws(function () { impl.Buffer.from(0) })
198
+ t.throws(function () { impl.Buffer.from(10) })
199
+ t.throws(function () { impl.Buffer.from(10, 'utf-8') })
200
+ t.throws(function () { impl.Buffer.from('string', 'invalid encoding') })
201
+ t.throws(function () { impl.Buffer.from(-10) })
202
+ t.throws(function () { impl.Buffer.from(1e90) })
203
+ t.throws(function () { impl.Buffer.from(Infinity) })
204
+ t.throws(function () { impl.Buffer.from(-Infinity) })
205
+ t.throws(function () { impl.Buffer.from(NaN) })
206
+ t.throws(function () { impl.Buffer.from(null) })
207
+ t.throws(function () { impl.Buffer.from(undefined) })
208
+ t.throws(function () { impl.Buffer.from() })
209
+ t.throws(function () { impl.Buffer.from({}) })
210
+ t.throws(function () { impl.Buffer.alloc('') })
211
+ t.throws(function () { impl.Buffer.alloc('string') })
212
+ t.throws(function () { impl.Buffer.alloc('string', 'utf-8') })
213
+ t.throws(function () { impl.Buffer.alloc('b25ldHdvdGhyZWU=', 'base64') })
214
+ t.throws(function () { impl.Buffer.alloc(-10) })
215
+ t.throws(function () { impl.Buffer.alloc(1e90) })
216
+ t.throws(function () { impl.Buffer.alloc(2 * (1 << 30)) })
217
+ t.throws(function () { impl.Buffer.alloc(Infinity) })
218
+ t.throws(function () { impl.Buffer.alloc(-Infinity) })
219
+ t.throws(function () { impl.Buffer.alloc(null) })
220
+ t.throws(function () { impl.Buffer.alloc(undefined) })
221
+ t.throws(function () { impl.Buffer.alloc() })
222
+ t.throws(function () { impl.Buffer.alloc([]) })
223
+ t.throws(function () { impl.Buffer.alloc([0, 42, 3]) })
224
+ t.throws(function () { impl.Buffer.alloc({}) })
225
+ });
226
+ ['allocUnsafe', 'allocUnsafeSlow'].forEach(function (method) {
227
+ t.throws(function () { dangerous.Buffer[method]('') })
228
+ t.throws(function () { dangerous.Buffer[method]('string') })
229
+ t.throws(function () { dangerous.Buffer[method]('string', 'utf-8') })
230
+ t.throws(function () { dangerous.Buffer[method](2 * (1 << 30)) })
231
+ t.throws(function () { dangerous.Buffer[method](Infinity) })
232
+ if (dangerous.Buffer[method] === buffer.Buffer.allocUnsafe) {
233
+ t.skip('Skipping, older impl of allocUnsafe coerced negative sizes to 0')
234
+ } else {
235
+ t.throws(function () { dangerous.Buffer[method](-10) })
236
+ t.throws(function () { dangerous.Buffer[method](-1e90) })
237
+ t.throws(function () { dangerous.Buffer[method](-Infinity) })
238
+ }
239
+ t.throws(function () { dangerous.Buffer[method](null) })
240
+ t.throws(function () { dangerous.Buffer[method](undefined) })
241
+ t.throws(function () { dangerous.Buffer[method]() })
242
+ t.throws(function () { dangerous.Buffer[method]([]) })
243
+ t.throws(function () { dangerous.Buffer[method]([0, 42, 3]) })
244
+ t.throws(function () { dangerous.Buffer[method]({}) })
245
+ })
246
+ t.end()
247
+ })
248
+
249
+ test('Buffers have appropriate lengths', function (t) {
250
+ [index, safer, dangerous].forEach(function (impl) {
251
+ t.equal(impl.Buffer.alloc(0).length, 0)
252
+ t.equal(impl.Buffer.alloc(10).length, 10)
253
+ t.equal(impl.Buffer.from('').length, 0)
254
+ t.equal(impl.Buffer.from('string').length, 6)
255
+ t.equal(impl.Buffer.from('string', 'utf-8').length, 6)
256
+ t.equal(impl.Buffer.from('b25ldHdvdGhyZWU=', 'base64').length, 11)
257
+ t.equal(impl.Buffer.from([0, 42, 3]).length, 3)
258
+ t.equal(impl.Buffer.from(new Uint8Array([0, 42, 3])).length, 3)
259
+ t.equal(impl.Buffer.from([]).length, 0)
260
+ });
261
+ ['allocUnsafe', 'allocUnsafeSlow'].forEach(function (method) {
262
+ t.equal(dangerous.Buffer[method](0).length, 0)
263
+ t.equal(dangerous.Buffer[method](10).length, 10)
264
+ })
265
+ t.end()
266
+ })
267
+
268
+ test('Buffers have appropriate lengths (2)', function (t) {
269
+ t.equal(index.Buffer.alloc, safer.Buffer.alloc)
270
+ t.equal(index.Buffer.alloc, dangerous.Buffer.alloc)
271
+ var ok = true;
272
+ [ safer.Buffer.alloc,
273
+ dangerous.Buffer.allocUnsafe,
274
+ dangerous.Buffer.allocUnsafeSlow
275
+ ].forEach(function (method) {
276
+ for (var i = 0; i < 1e2; i++) {
277
+ var length = Math.round(Math.random() * 1e5)
278
+ var buf = method(length)
279
+ if (!buffer.Buffer.isBuffer(buf)) ok = false
280
+ if (buf.length !== length) ok = false
281
+ }
282
+ })
283
+ t.ok(ok)
284
+ t.end()
285
+ })
286
+
287
+ test('.alloc(size) is zero-filled and has correct length', function (t) {
288
+ t.equal(index.Buffer.alloc, safer.Buffer.alloc)
289
+ t.equal(index.Buffer.alloc, dangerous.Buffer.alloc)
290
+ var ok = true
291
+ for (var i = 0; i < 1e2; i++) {
292
+ var length = Math.round(Math.random() * 2e6)
293
+ var buf = index.Buffer.alloc(length)
294
+ if (!buffer.Buffer.isBuffer(buf)) ok = false
295
+ if (buf.length !== length) ok = false
296
+ var j
297
+ for (j = 0; j < length; j++) {
298
+ if (buf[j] !== 0) ok = false
299
+ }
300
+ buf.fill(1)
301
+ for (j = 0; j < length; j++) {
302
+ if (buf[j] !== 1) ok = false
303
+ }
304
+ }
305
+ t.ok(ok)
306
+ t.end()
307
+ })
308
+
309
+ test('.allocUnsafe / .allocUnsafeSlow are fillable and have correct lengths', function (t) {
310
+ ['allocUnsafe', 'allocUnsafeSlow'].forEach(function (method) {
311
+ var ok = true
312
+ for (var i = 0; i < 1e2; i++) {
313
+ var length = Math.round(Math.random() * 2e6)
314
+ var buf = dangerous.Buffer[method](length)
315
+ if (!buffer.Buffer.isBuffer(buf)) ok = false
316
+ if (buf.length !== length) ok = false
317
+ buf.fill(0, 0, length)
318
+ var j
319
+ for (j = 0; j < length; j++) {
320
+ if (buf[j] !== 0) ok = false
321
+ }
322
+ buf.fill(1, 0, length)
323
+ for (j = 0; j < length; j++) {
324
+ if (buf[j] !== 1) ok = false
325
+ }
326
+ }
327
+ t.ok(ok, method)
328
+ })
329
+ t.end()
330
+ })
331
+
332
+ test('.alloc(size, fill) is `fill`-filled', function (t) {
333
+ t.equal(index.Buffer.alloc, safer.Buffer.alloc)
334
+ t.equal(index.Buffer.alloc, dangerous.Buffer.alloc)
335
+ var ok = true
336
+ for (var i = 0; i < 1e2; i++) {
337
+ var length = Math.round(Math.random() * 2e6)
338
+ var fill = Math.round(Math.random() * 255)
339
+ var buf = index.Buffer.alloc(length, fill)
340
+ if (!buffer.Buffer.isBuffer(buf)) ok = false
341
+ if (buf.length !== length) ok = false
342
+ for (var j = 0; j < length; j++) {
343
+ if (buf[j] !== fill) ok = false
344
+ }
345
+ }
346
+ t.ok(ok)
347
+ t.end()
348
+ })
349
+
350
+ test('.alloc(size, fill) is `fill`-filled', function (t) {
351
+ t.equal(index.Buffer.alloc, safer.Buffer.alloc)
352
+ t.equal(index.Buffer.alloc, dangerous.Buffer.alloc)
353
+ var ok = true
354
+ for (var i = 0; i < 1e2; i++) {
355
+ var length = Math.round(Math.random() * 2e6)
356
+ var fill = Math.round(Math.random() * 255)
357
+ var buf = index.Buffer.alloc(length, fill)
358
+ if (!buffer.Buffer.isBuffer(buf)) ok = false
359
+ if (buf.length !== length) ok = false
360
+ for (var j = 0; j < length; j++) {
361
+ if (buf[j] !== fill) ok = false
362
+ }
363
+ }
364
+ t.ok(ok)
365
+ t.deepEqual(index.Buffer.alloc(9, 'a'), index.Buffer.alloc(9, 97))
366
+ t.notDeepEqual(index.Buffer.alloc(9, 'a'), index.Buffer.alloc(9, 98))
367
+
368
+ var tmp = new buffer.Buffer(2)
369
+ tmp.fill('ok')
370
+ if (tmp[1] === tmp[0]) {
371
+ // Outdated Node.js
372
+ t.deepEqual(index.Buffer.alloc(5, 'ok'), index.Buffer.from('ooooo'))
373
+ } else {
374
+ t.deepEqual(index.Buffer.alloc(5, 'ok'), index.Buffer.from('okoko'))
375
+ }
376
+ t.notDeepEqual(index.Buffer.alloc(5, 'ok'), index.Buffer.from('kokok'))
377
+
378
+ t.end()
379
+ })
380
+
381
+ test('safer.Buffer.from returns results same as Buffer constructor', function (t) {
382
+ [index, safer, dangerous].forEach(function (impl) {
383
+ t.deepEqual(impl.Buffer.from(''), new buffer.Buffer(''))
384
+ t.deepEqual(impl.Buffer.from('string'), new buffer.Buffer('string'))
385
+ t.deepEqual(impl.Buffer.from('string', 'utf-8'), new buffer.Buffer('string', 'utf-8'))
386
+ t.deepEqual(impl.Buffer.from('b25ldHdvdGhyZWU=', 'base64'), new buffer.Buffer('b25ldHdvdGhyZWU=', 'base64'))
387
+ t.deepEqual(impl.Buffer.from([0, 42, 3]), new buffer.Buffer([0, 42, 3]))
388
+ t.deepEqual(impl.Buffer.from(new Uint8Array([0, 42, 3])), new buffer.Buffer(new Uint8Array([0, 42, 3])))
389
+ t.deepEqual(impl.Buffer.from([]), new buffer.Buffer([]))
390
+ })
391
+ t.end()
392
+ })
393
+
394
+ test('safer.Buffer.from returns consistent results', function (t) {
395
+ [index, safer, dangerous].forEach(function (impl) {
396
+ t.deepEqual(impl.Buffer.from(''), impl.Buffer.alloc(0))
397
+ t.deepEqual(impl.Buffer.from([]), impl.Buffer.alloc(0))
398
+ t.deepEqual(impl.Buffer.from(new Uint8Array([])), impl.Buffer.alloc(0))
399
+ t.deepEqual(impl.Buffer.from('string', 'utf-8'), impl.Buffer.from('string'))
400
+ t.deepEqual(impl.Buffer.from('string'), impl.Buffer.from([115, 116, 114, 105, 110, 103]))
401
+ t.deepEqual(impl.Buffer.from('string'), impl.Buffer.from(impl.Buffer.from('string')))
402
+ t.deepEqual(impl.Buffer.from('b25ldHdvdGhyZWU=', 'base64'), impl.Buffer.from('onetwothree'))
403
+ t.notDeepEqual(impl.Buffer.from('b25ldHdvdGhyZWU='), impl.Buffer.from('onetwothree'))
404
+ })
405
+ t.end()
406
+ })