es6-crawler-detect 3.1.3 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (355) hide show
  1. package/.babelrc +18 -17
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/FUNDING.yml +12 -12
  5. package/.github/ISSUE_TEMPLATE/bug_report.md +41 -38
  6. package/.github/ISSUE_TEMPLATE/feature_request.md +20 -20
  7. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  8. package/.prettierrc.json +5 -0
  9. package/CODE_OF_CONDUCT.md +76 -76
  10. package/LICENSE +21 -21
  11. package/README.md +103 -93
  12. package/dist/main.bundle.js +1 -1
  13. package/example/node/dist/crawler.min.js +1 -1
  14. package/example/node/index.html +21 -21
  15. package/example/node/node_modules/.bin/mime +15 -0
  16. package/example/node/node_modules/.bin/mime.cmd +7 -0
  17. package/example/node/node_modules/.yarn-integrity +67 -0
  18. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  19. package/example/node/node_modules/accepts/LICENSE +23 -0
  20. package/example/node/node_modules/accepts/README.md +142 -0
  21. package/example/node/node_modules/accepts/index.js +238 -0
  22. package/example/node/node_modules/accepts/package.json +47 -0
  23. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  24. package/example/node/node_modules/array-flatten/README.md +43 -0
  25. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  26. package/example/node/node_modules/array-flatten/package.json +39 -0
  27. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  28. package/example/node/node_modules/body-parser/LICENSE +23 -0
  29. package/example/node/node_modules/body-parser/README.md +456 -0
  30. package/example/node/node_modules/body-parser/index.js +157 -0
  31. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  32. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  33. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  34. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  35. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  36. package/example/node/node_modules/body-parser/package.json +52 -0
  37. package/example/node/node_modules/bytes/History.md +92 -0
  38. package/example/node/node_modules/bytes/LICENSE +23 -0
  39. package/example/node/node_modules/bytes/Readme.md +152 -0
  40. package/example/node/node_modules/bytes/index.js +166 -0
  41. package/example/node/node_modules/bytes/package.json +42 -0
  42. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  43. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  44. package/example/node/node_modules/content-disposition/README.md +142 -0
  45. package/example/node/node_modules/content-disposition/index.js +458 -0
  46. package/example/node/node_modules/content-disposition/package.json +44 -0
  47. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  48. package/example/node/node_modules/content-type/LICENSE +22 -0
  49. package/example/node/node_modules/content-type/README.md +92 -0
  50. package/example/node/node_modules/content-type/index.js +222 -0
  51. package/example/node/node_modules/content-type/package.json +40 -0
  52. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  53. package/example/node/node_modules/cookie/LICENSE +24 -0
  54. package/example/node/node_modules/cookie/README.md +257 -0
  55. package/example/node/node_modules/cookie/index.js +202 -0
  56. package/example/node/node_modules/cookie/package.json +40 -0
  57. package/example/node/node_modules/cookie-signature/History.md +38 -0
  58. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  59. package/example/node/node_modules/cookie-signature/index.js +51 -0
  60. package/example/node/node_modules/cookie-signature/package.json +18 -0
  61. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  62. package/example/node/node_modules/debug/.eslintrc +11 -0
  63. package/example/node/node_modules/debug/.travis.yml +14 -0
  64. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  65. package/example/node/node_modules/debug/LICENSE +19 -0
  66. package/example/node/node_modules/debug/Makefile +50 -0
  67. package/example/node/node_modules/debug/README.md +312 -0
  68. package/example/node/node_modules/debug/component.json +19 -0
  69. package/example/node/node_modules/debug/karma.conf.js +70 -0
  70. package/example/node/node_modules/debug/node.js +1 -0
  71. package/example/node/node_modules/debug/package.json +49 -0
  72. package/example/node/node_modules/debug/src/browser.js +185 -0
  73. package/example/node/node_modules/debug/src/debug.js +202 -0
  74. package/example/node/node_modules/debug/src/index.js +10 -0
  75. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  76. package/example/node/node_modules/debug/src/node.js +248 -0
  77. package/example/node/node_modules/depd/History.md +96 -0
  78. package/example/node/node_modules/depd/LICENSE +22 -0
  79. package/example/node/node_modules/depd/Readme.md +280 -0
  80. package/example/node/node_modules/depd/index.js +522 -0
  81. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  82. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  83. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  84. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  85. package/example/node/node_modules/depd/package.json +41 -0
  86. package/example/node/node_modules/destroy/LICENSE +22 -0
  87. package/example/node/node_modules/destroy/README.md +60 -0
  88. package/example/node/node_modules/destroy/index.js +75 -0
  89. package/example/node/node_modules/destroy/package.json +37 -0
  90. package/example/node/node_modules/ee-first/LICENSE +22 -0
  91. package/example/node/node_modules/ee-first/README.md +80 -0
  92. package/example/node/node_modules/ee-first/index.js +95 -0
  93. package/example/node/node_modules/ee-first/package.json +29 -0
  94. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  95. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  96. package/example/node/node_modules/encodeurl/README.md +128 -0
  97. package/example/node/node_modules/encodeurl/index.js +60 -0
  98. package/example/node/node_modules/encodeurl/package.json +40 -0
  99. package/example/node/node_modules/escape-html/LICENSE +24 -0
  100. package/example/node/node_modules/escape-html/Readme.md +43 -0
  101. package/example/node/node_modules/escape-html/index.js +78 -0
  102. package/example/node/node_modules/escape-html/package.json +24 -0
  103. package/example/node/node_modules/etag/HISTORY.md +83 -0
  104. package/example/node/node_modules/etag/LICENSE +22 -0
  105. package/example/node/node_modules/etag/README.md +159 -0
  106. package/example/node/node_modules/etag/index.js +131 -0
  107. package/example/node/node_modules/etag/package.json +47 -0
  108. package/example/node/node_modules/express/History.md +3510 -0
  109. package/example/node/node_modules/express/LICENSE +24 -0
  110. package/example/node/node_modules/express/Readme.md +158 -0
  111. package/example/node/node_modules/express/index.js +11 -0
  112. package/example/node/node_modules/express/lib/application.js +644 -0
  113. package/example/node/node_modules/express/lib/express.js +116 -0
  114. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  115. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  116. package/example/node/node_modules/express/lib/request.js +525 -0
  117. package/example/node/node_modules/express/lib/response.js +1147 -0
  118. package/example/node/node_modules/express/lib/router/index.js +668 -0
  119. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  120. package/example/node/node_modules/express/lib/router/route.js +216 -0
  121. package/example/node/node_modules/express/lib/utils.js +302 -0
  122. package/example/node/node_modules/express/lib/view.js +182 -0
  123. package/example/node/node_modules/express/package.json +99 -0
  124. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  125. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  126. package/example/node/node_modules/finalhandler/README.md +148 -0
  127. package/example/node/node_modules/finalhandler/index.js +331 -0
  128. package/example/node/node_modules/finalhandler/package.json +45 -0
  129. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  130. package/example/node/node_modules/forwarded/LICENSE +22 -0
  131. package/example/node/node_modules/forwarded/README.md +57 -0
  132. package/example/node/node_modules/forwarded/index.js +90 -0
  133. package/example/node/node_modules/forwarded/package.json +45 -0
  134. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  135. package/example/node/node_modules/fresh/LICENSE +23 -0
  136. package/example/node/node_modules/fresh/README.md +119 -0
  137. package/example/node/node_modules/fresh/index.js +137 -0
  138. package/example/node/node_modules/fresh/package.json +46 -0
  139. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  140. package/example/node/node_modules/http-errors/LICENSE +23 -0
  141. package/example/node/node_modules/http-errors/README.md +169 -0
  142. package/example/node/node_modules/http-errors/index.js +299 -0
  143. package/example/node/node_modules/http-errors/package.json +49 -0
  144. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  145. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  146. package/example/node/node_modules/iconv-lite/README.md +156 -0
  147. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  148. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  149. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  150. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  151. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  152. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  153. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  158. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  159. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  160. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  161. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  162. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  163. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  164. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  165. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  166. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  167. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  168. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  169. package/example/node/node_modules/iconv-lite/package.json +46 -0
  170. package/example/node/node_modules/inherits/LICENSE +16 -0
  171. package/example/node/node_modules/inherits/README.md +42 -0
  172. package/example/node/node_modules/inherits/inherits.js +9 -0
  173. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  174. package/example/node/node_modules/inherits/package.json +29 -0
  175. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  176. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  177. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  178. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  179. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  180. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  181. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  182. package/example/node/node_modules/media-typer/LICENSE +22 -0
  183. package/example/node/node_modules/media-typer/README.md +81 -0
  184. package/example/node/node_modules/media-typer/index.js +270 -0
  185. package/example/node/node_modules/media-typer/package.json +26 -0
  186. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  187. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  188. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  189. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  190. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  191. package/example/node/node_modules/methods/HISTORY.md +29 -0
  192. package/example/node/node_modules/methods/LICENSE +24 -0
  193. package/example/node/node_modules/methods/README.md +51 -0
  194. package/example/node/node_modules/methods/index.js +69 -0
  195. package/example/node/node_modules/methods/package.json +36 -0
  196. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  197. package/example/node/node_modules/mime/LICENSE +21 -0
  198. package/example/node/node_modules/mime/README.md +90 -0
  199. package/example/node/node_modules/mime/cli.js +8 -0
  200. package/example/node/node_modules/mime/mime.js +108 -0
  201. package/example/node/node_modules/mime/package.json +44 -0
  202. package/example/node/node_modules/mime/src/build.js +53 -0
  203. package/example/node/node_modules/mime/src/test.js +60 -0
  204. package/example/node/node_modules/mime/types.json +1 -0
  205. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  206. package/example/node/node_modules/mime-db/LICENSE +22 -0
  207. package/example/node/node_modules/mime-db/README.md +100 -0
  208. package/example/node/node_modules/mime-db/db.json +8471 -0
  209. package/example/node/node_modules/mime-db/index.js +11 -0
  210. package/example/node/node_modules/mime-db/package.json +59 -0
  211. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  212. package/example/node/node_modules/mime-types/LICENSE +23 -0
  213. package/example/node/node_modules/mime-types/README.md +113 -0
  214. package/example/node/node_modules/mime-types/index.js +188 -0
  215. package/example/node/node_modules/mime-types/package.json +44 -0
  216. package/example/node/node_modules/ms/index.js +152 -0
  217. package/example/node/node_modules/ms/license.md +21 -0
  218. package/example/node/node_modules/ms/package.json +37 -0
  219. package/example/node/node_modules/ms/readme.md +51 -0
  220. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  221. package/example/node/node_modules/negotiator/LICENSE +24 -0
  222. package/example/node/node_modules/negotiator/README.md +203 -0
  223. package/example/node/node_modules/negotiator/index.js +124 -0
  224. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  225. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  226. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  227. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  228. package/example/node/node_modules/negotiator/package.json +42 -0
  229. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  230. package/example/node/node_modules/on-finished/LICENSE +23 -0
  231. package/example/node/node_modules/on-finished/README.md +154 -0
  232. package/example/node/node_modules/on-finished/index.js +196 -0
  233. package/example/node/node_modules/on-finished/package.json +31 -0
  234. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  235. package/example/node/node_modules/parseurl/LICENSE +24 -0
  236. package/example/node/node_modules/parseurl/README.md +133 -0
  237. package/example/node/node_modules/parseurl/index.js +158 -0
  238. package/example/node/node_modules/parseurl/package.json +40 -0
  239. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  240. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  241. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  242. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  243. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  244. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  245. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  246. package/example/node/node_modules/proxy-addr/README.md +139 -0
  247. package/example/node/node_modules/proxy-addr/index.js +327 -0
  248. package/example/node/node_modules/proxy-addr/package.json +47 -0
  249. package/example/node/node_modules/qs/.editorconfig +39 -0
  250. package/example/node/node_modules/qs/.eslintignore +2 -0
  251. package/example/node/node_modules/qs/.eslintrc +35 -0
  252. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  253. package/example/node/node_modules/qs/.nycrc +13 -0
  254. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  255. package/example/node/node_modules/qs/LICENSE.md +29 -0
  256. package/example/node/node_modules/qs/README.md +604 -0
  257. package/example/node/node_modules/qs/dist/qs.js +832 -0
  258. package/example/node/node_modules/qs/lib/formats.js +23 -0
  259. package/example/node/node_modules/qs/lib/index.js +11 -0
  260. package/example/node/node_modules/qs/lib/parse.js +257 -0
  261. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  262. package/example/node/node_modules/qs/lib/utils.js +251 -0
  263. package/example/node/node_modules/qs/package.json +69 -0
  264. package/example/node/node_modules/qs/test/parse.js +772 -0
  265. package/example/node/node_modules/qs/test/stringify.js +793 -0
  266. package/example/node/node_modules/qs/test/utils.js +136 -0
  267. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  268. package/example/node/node_modules/range-parser/LICENSE +23 -0
  269. package/example/node/node_modules/range-parser/README.md +84 -0
  270. package/example/node/node_modules/range-parser/index.js +162 -0
  271. package/example/node/node_modules/range-parser/package.json +44 -0
  272. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  273. package/example/node/node_modules/raw-body/LICENSE +22 -0
  274. package/example/node/node_modules/raw-body/README.md +217 -0
  275. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  276. package/example/node/node_modules/raw-body/index.js +286 -0
  277. package/example/node/node_modules/raw-body/package.json +48 -0
  278. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  279. package/example/node/node_modules/safe-buffer/README.md +584 -0
  280. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  281. package/example/node/node_modules/safe-buffer/index.js +65 -0
  282. package/example/node/node_modules/safe-buffer/package.json +51 -0
  283. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  284. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  285. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  286. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  287. package/example/node/node_modules/safer-buffer/package.json +34 -0
  288. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  289. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  290. package/example/node/node_modules/send/HISTORY.md +506 -0
  291. package/example/node/node_modules/send/LICENSE +23 -0
  292. package/example/node/node_modules/send/README.md +327 -0
  293. package/example/node/node_modules/send/index.js +1133 -0
  294. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  295. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  296. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  297. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  298. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  299. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  300. package/example/node/node_modules/send/package.json +61 -0
  301. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  302. package/example/node/node_modules/serve-static/LICENSE +25 -0
  303. package/example/node/node_modules/serve-static/README.md +257 -0
  304. package/example/node/node_modules/serve-static/index.js +210 -0
  305. package/example/node/node_modules/serve-static/package.json +42 -0
  306. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  307. package/example/node/node_modules/setprototypeof/README.md +31 -0
  308. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  309. package/example/node/node_modules/setprototypeof/index.js +17 -0
  310. package/example/node/node_modules/setprototypeof/package.json +38 -0
  311. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  312. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  313. package/example/node/node_modules/statuses/LICENSE +23 -0
  314. package/example/node/node_modules/statuses/README.md +127 -0
  315. package/example/node/node_modules/statuses/codes.json +66 -0
  316. package/example/node/node_modules/statuses/index.js +113 -0
  317. package/example/node/node_modules/statuses/package.json +48 -0
  318. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  319. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  320. package/example/node/node_modules/toidentifier/README.md +61 -0
  321. package/example/node/node_modules/toidentifier/index.js +32 -0
  322. package/example/node/node_modules/toidentifier/package.json +38 -0
  323. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  324. package/example/node/node_modules/type-is/LICENSE +23 -0
  325. package/example/node/node_modules/type-is/README.md +170 -0
  326. package/example/node/node_modules/type-is/index.js +266 -0
  327. package/example/node/node_modules/type-is/package.json +45 -0
  328. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  329. package/example/node/node_modules/unpipe/LICENSE +22 -0
  330. package/example/node/node_modules/unpipe/README.md +43 -0
  331. package/example/node/node_modules/unpipe/index.js +69 -0
  332. package/example/node/node_modules/unpipe/package.json +27 -0
  333. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  334. package/example/node/node_modules/utils-merge/README.md +34 -0
  335. package/example/node/node_modules/utils-merge/index.js +23 -0
  336. package/example/node/node_modules/utils-merge/package.json +40 -0
  337. package/example/node/node_modules/vary/HISTORY.md +39 -0
  338. package/example/node/node_modules/vary/LICENSE +22 -0
  339. package/example/node/node_modules/vary/README.md +101 -0
  340. package/example/node/node_modules/vary/index.js +149 -0
  341. package/example/node/node_modules/vary/package.json +43 -0
  342. package/example/node/package.json +11 -0
  343. package/example/node/server.js +21 -15
  344. package/example/node/yarn.lock +352 -0
  345. package/package.json +59 -47
  346. package/src/index.js +15 -10
  347. package/src/lib/crawler/crawlers.js +1398 -11
  348. package/src/lib/crawler/exclusions.js +55 -8
  349. package/src/lib/crawler/headers.js +18 -9
  350. package/src/lib/crawler/provider.js +6 -8
  351. package/src/lib/crawler.js +101 -90
  352. package/test/lib/crawler.test.js +43 -26
  353. package/webpack.common.js +7 -7
  354. package/webpack.dev.js +4 -4
  355. package/webpack.prod.js +8 -9
@@ -1,15 +1,62 @@
1
1
  'use strict';
2
2
 
3
- const Provider = require("./provider");
3
+ const Provider = require('./provider');
4
4
 
5
- class Exclusions extends Provider
6
- {
7
- constructor()
8
- {
9
- super();
5
+ class Exclusions extends Provider {
6
+ constructor() {
7
+ super();
10
8
 
11
- this.data = ["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","kube-probe.[\\d\\.]*",";"];
12
- }
9
+ this.data = [
10
+ 'Safari.[\\d\\.]*',
11
+ 'Firefox.[\\d\\.]*',
12
+ ' Chrome.[\\d\\.]*',
13
+ 'Chromium.[\\d\\.]*',
14
+ 'MSIE.[\\d\\.]',
15
+ 'Opera\\/[\\d\\.]*',
16
+ 'Mozilla.[\\d\\.]*',
17
+ 'AppleWebKit.[\\d\\.]*',
18
+ 'Trident.[\\d\\.]*',
19
+ 'Windows NT.[\\d\\.]*',
20
+ 'Android [\\d\\.]*',
21
+ 'Macintosh.',
22
+ 'Ubuntu',
23
+ 'Linux',
24
+ '[ ]Intel',
25
+ 'Mac OS X [\\d_]*',
26
+ '(like )?Gecko(.[\\d\\.]*)?',
27
+ 'KHTML,',
28
+ 'CriOS.[\\d\\.]*',
29
+ 'CPU iPhone OS ([0-9_])* like Mac OS X',
30
+ 'CPU OS ([0-9_])* like Mac OS X',
31
+ 'iPod',
32
+ 'compatible',
33
+ 'x86_..',
34
+ 'i686',
35
+ 'x64',
36
+ 'X11',
37
+ 'rv:[\\d\\.]*',
38
+ 'Version.[\\d\\.]*',
39
+ 'WOW64',
40
+ 'Win64',
41
+ 'Dalvik.[\\d\\.]*',
42
+ ' \\.NET CLR [\\d\\.]*',
43
+ 'Presto.[\\d\\.]*',
44
+ 'Media Center PC',
45
+ 'BlackBerry',
46
+ 'Build',
47
+ 'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
48
+ 'Opera',
49
+ ' \\.NET[\\d\\.]*',
50
+ 'cubot',
51
+ '; M bot',
52
+ '; CRONO',
53
+ '; B bot',
54
+ '; IDbot',
55
+ '; ID bot',
56
+ '; POWER BOT',
57
+ ';',
58
+ ];
59
+ }
13
60
  }
14
61
 
15
62
  module.exports = Exclusions;
@@ -1,15 +1,24 @@
1
1
  'use strict';
2
2
 
3
- const Provider = require("./provider");
3
+ const Provider = require('./provider');
4
4
 
5
- class Headers extends Provider
6
- {
7
- constructor()
8
- {
9
- super();
5
+ class Headers extends Provider {
6
+ constructor() {
7
+ super();
10
8
 
11
- this.data = ["USER-AGENT","X-OPERAMINI-PHONE-UA","X-DEVICE-USER-AGENT","X-ORIGINAL-USER-AGENT","X-SKYFIRE-PHONE","X-BOLT-PHONE-UA","DEVICE-STOCK-UA","X-UCBROWSER-DEVICE-UA","FROM","X-SCANNER"];
12
- }
9
+ this.data = [
10
+ 'USER-AGENT',
11
+ 'X-OPERAMINI-PHONE-UA',
12
+ 'X-DEVICE-USER-AGENT',
13
+ 'X-ORIGINAL-USER-AGENT',
14
+ 'X-SKYFIRE-PHONE',
15
+ 'X-BOLT-PHONE-UA',
16
+ 'DEVICE-STOCK-UA',
17
+ 'X-UCBROWSER-DEVICE-UA',
18
+ 'FROM',
19
+ 'X-SCANNER',
20
+ ];
21
+ }
13
22
  }
14
23
 
15
- module.exports = Headers;
24
+ module.exports = Headers;
@@ -1,13 +1,11 @@
1
1
  'use strict';
2
2
 
3
- class Provider
4
- {
5
- constructor(){}
3
+ class Provider {
4
+ constructor() {}
6
5
 
7
- getAll()
8
- {
9
- return this.data;
10
- }
6
+ getAll() {
7
+ return this.data;
8
+ }
11
9
  }
12
10
 
13
- module.exports = Provider;
11
+ module.exports = Provider;
@@ -5,120 +5,131 @@ const Exclusions = require('./crawler/exclusions');
5
5
  const Headers = require('./crawler/headers');
6
6
 
7
7
  class Crawler {
8
- constructor(request, headers, userAgent) {
9
- /**
10
- * Init classes
11
- */
12
- this._init();
13
-
14
- /**
15
- * This request must be an object
16
- */
17
- this.request = typeof request === 'object' ? request : {};
18
-
19
- // The regex-list must not be used with g-flag!
20
- // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
21
- this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
22
-
23
- // The exclusions should be used with g-flag in order to remove each value.
24
- this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'g');
25
-
26
- /**
27
- * Set http headers
28
- */
29
- this.setHttpHeaders(headers);
30
-
31
- /**
32
- * Set userAgent
33
- */
34
- this.userAgent = this.setUserAgent(userAgent);
35
- }
36
-
8
+ constructor(request, headers, userAgent) {
37
9
  /**
38
- * Init Classes Instances
10
+ * Init classes
39
11
  */
40
- _init() {
41
- this.crawlers = new Crawlers();
42
- this.headers = new Headers();
43
- this.exclusions = new Exclusions();
44
- }
45
-
46
- compileRegex(patterns, flags) {
47
- return new RegExp(patterns.join('|').trim(), flags);
48
- }
12
+ this._init();
49
13
 
50
14
  /**
51
- * Set HTTP headers.
15
+ * This request must be an object
52
16
  */
53
- setHttpHeaders(headers) {
54
- // Use the Request headers if httpHeaders is not defined
55
- if (typeof headers === 'undefined' || Object.keys(headers).length === 0) {
56
- headers = Object.keys(this.request).length ? this.request.headers : {};
57
- }
17
+ this.request = typeof request === 'object' ? request : {};
58
18
 
59
- // Clear existing headers.
60
- this.httpHeaders = [];
19
+ // The regex-list must not be used with g-flag!
20
+ // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
21
+ this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
61
22
 
62
- // Only save HTTP headers.
63
- for (const key in headers) {
64
- this.httpHeaders[key] = headers[key];
65
- }
66
- }
23
+ // The exclusions should be used with g-flag in order to remove each value.
24
+ this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'g');
67
25
 
68
26
  /**
69
- * Set user agent
27
+ * Set http headers
70
28
  */
71
- setUserAgent(userAgent) {
72
- if (typeof userAgent === 'undefined' || userAgent === null || !userAgent.length) {
73
- for (const header of this.getUaHttpHeaders()) {
74
- if (Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) >= 0) {
75
- userAgent += this.httpHeaders[header] + ' ';
76
- }
77
- }
78
- }
79
-
80
- return userAgent;
81
- }
29
+ this.setHttpHeaders(headers);
82
30
 
83
31
  /**
84
- * Get user agent headers
32
+ * Set userAgent
85
33
  */
86
- getUaHttpHeaders() {
87
- return this.headers.getAll();
34
+ this.userAgent = this.setUserAgent(userAgent);
35
+ }
36
+
37
+ /**
38
+ * Init Classes Instances
39
+ */
40
+ _init() {
41
+ this.crawlers = new Crawlers();
42
+ this.headers = new Headers();
43
+ this.exclusions = new Exclusions();
44
+ }
45
+
46
+ compileRegex(patterns, flags) {
47
+ return new RegExp(patterns.join('|').trim(), flags);
48
+ }
49
+
50
+ /**
51
+ * Set HTTP headers.
52
+ */
53
+ setHttpHeaders(headers) {
54
+ // Use the Request headers if httpHeaders is not defined
55
+ if (typeof headers === 'undefined' || Object.keys(headers).length === 0) {
56
+ headers = Object.keys(this.request).length ? this.request.headers : {};
88
57
  }
89
58
 
90
- /**
91
- * Check user agent string against the regex.
92
- */
93
- isCrawler(userAgent = undefined) {
94
- if (Buffer.byteLength(userAgent || "", 'utf8') > 4096) {
95
- return false;
96
- }
97
- var agent = (typeof userAgent === 'undefined' || userAgent === null ? this.userAgent : userAgent);
98
-
99
- // test on compiled regx
100
- agent = agent.replace(this.compiledExclusions, '');
59
+ // Clear existing headers.
60
+ this.httpHeaders = [];
101
61
 
102
- if (agent.trim().length === 0) {
103
- return false;
62
+ // Only save HTTP headers.
63
+ for (const key in headers) {
64
+ this.httpHeaders[key] = headers[key];
65
+ }
66
+ }
67
+
68
+ /**
69
+ * Set user agent
70
+ */
71
+ setUserAgent(userAgent) {
72
+ if (
73
+ typeof userAgent === 'undefined' ||
74
+ userAgent === null ||
75
+ !userAgent.length
76
+ ) {
77
+ for (const header of this.getUaHttpHeaders()) {
78
+ if (Object.keys(this.httpHeaders).indexOf(header.toLowerCase()) >= 0) {
79
+ userAgent += this.httpHeaders[header] + ' ';
104
80
  }
81
+ }
82
+ }
105
83
 
84
+ return userAgent;
85
+ }
86
+
87
+ /**
88
+ * Get user agent headers
89
+ */
90
+ getUaHttpHeaders() {
91
+ return this.headers.getAll();
92
+ }
93
+
94
+ /**
95
+ * Check user agent string against the regex.
96
+ */
97
+ isCrawler(userAgent = undefined) {
98
+ if (Buffer.byteLength(userAgent || '', 'utf8') > 4096) {
99
+ return false;
100
+ }
106
101
 
107
- var matches = this.compiledRegexList.exec(agent.trim());
102
+ var agent =
103
+ typeof userAgent === 'undefined' || userAgent === null
104
+ ? this.userAgent
105
+ : userAgent;
108
106
 
109
- if (matches) {
110
- this.matches = matches;
111
- }
107
+ // test on compiled regx
108
+ agent = agent.replace(this.compiledExclusions, '');
112
109
 
113
- return matches !== null ? (matches.length ? true : false) : false;
110
+ if (agent.trim().length === 0) {
111
+ return false;
114
112
  }
115
113
 
116
- /**
117
- * Return the matches.
118
- */
119
- getMatches() {
120
- return this.matches !== undefined ? (this.matches.length ? this.matches[0] : null) : {};
114
+ var matches = this.compiledRegexList.exec(agent.trim());
115
+
116
+ if (matches) {
117
+ this.matches = matches;
121
118
  }
119
+
120
+ return matches !== null ? (matches.length ? true : false) : false;
121
+ }
122
+
123
+ /**
124
+ * Return the matches.
125
+ */
126
+ getMatches() {
127
+ return this.matches !== undefined
128
+ ? this.matches.length
129
+ ? this.matches[0]
130
+ : null
131
+ : {};
132
+ }
122
133
  }
123
134
 
124
135
  module.exports = Crawler;
@@ -1,35 +1,52 @@
1
- var assert = require("assert");
1
+ var assert = require('assert');
2
2
 
3
- const Crawler = require("../../src/lib/crawler");
3
+ const Crawler = require('../../src/lib/crawler');
4
4
 
5
- describe("crawler", () => {
6
- var crawler = new Crawler();
5
+ describe('crawler', () => {
6
+ var crawler = new Crawler();
7
7
 
8
- it("will identify crawlers correctly on subsequent calls", () => {
9
- assert.strictEqual(crawler.isCrawler("Zombie.js"), true);
10
- assert.strictEqual(crawler.isCrawler("Zombie.js"), true, "crawler was not identified on subsequent call");
11
- });
8
+ it('will identify crawlers correctly on subsequent calls', () => {
9
+ assert.strictEqual(crawler.isCrawler('Zombie.js'), true);
10
+ assert.strictEqual(
11
+ crawler.isCrawler('Zombie.js'),
12
+ true,
13
+ 'crawler was not identified on subsequent call'
14
+ );
15
+ });
12
16
 
13
- it("will identify telegram bot", () => {
14
- assert.strictEqual(crawler.isCrawler("TelegramBot (like TwitterBot)"), true);
15
- });
17
+ it('will identify telegram bot', () => {
18
+ assert.strictEqual(
19
+ crawler.isCrawler('TelegramBot (like TwitterBot)'),
20
+ true
21
+ );
22
+ });
16
23
 
17
- describe("regex-compilation", () => {
18
- it("will join list of patterns with pipes", () => {
19
- assert.strictEqual(crawler.compileRegex(["some", "patterns"]).source, "some|patterns");
20
- assert.strictEqual(crawler.compileRegex(["single"]).source, "single");
21
- assert.strictEqual(crawler.compileRegex([" remove-whitespaces "]).source, "remove-whitespaces");
22
- });
24
+ describe('regex-compilation', () => {
25
+ it('will join list of patterns with pipes', () => {
26
+ assert.strictEqual(
27
+ crawler.compileRegex(['some', 'patterns']).source,
28
+ 'some|patterns'
29
+ );
30
+ assert.strictEqual(crawler.compileRegex(['single']).source, 'single');
31
+ assert.strictEqual(
32
+ crawler.compileRegex([' remove-whitespaces ']).source,
33
+ 'remove-whitespaces'
34
+ );
35
+ });
23
36
 
24
- it("will accept regex-flags for compilation", () => {
25
- var patterns = ['some', 'patterns'];
26
- assert.strictEqual(crawler.compileRegex(patterns, "g").flags, "g");
27
- assert.strictEqual(crawler.compileRegex(patterns, "i").flags, "i");
28
- });
37
+ it('will accept regex-flags for compilation', () => {
38
+ var patterns = ['some', 'patterns'];
39
+ assert.strictEqual(crawler.compileRegex(patterns, 'g').flags, 'g');
40
+ assert.strictEqual(crawler.compileRegex(patterns, 'i').flags, 'i');
41
+ });
29
42
 
30
- it("should be case insensitive", ()=>{
31
- assert.strictEqual(crawler.isCrawler("Facebot\1.0"), true);
32
- assert.strictEqual(crawler.getMatches("Facebot\1.0"), "Facebot", "Crawler was not able to indentify crawler correctly");
33
- })
43
+ it('should be case insensitive', () => {
44
+ assert.strictEqual(crawler.isCrawler('Facebot\\1.0'), true);
45
+ assert.strictEqual(
46
+ crawler.getMatches('Facebot\\1.0'),
47
+ 'Facebot',
48
+ 'Crawler was not able to indentify crawler correctly'
49
+ );
34
50
  });
51
+ });
35
52
  });
package/webpack.common.js CHANGED
@@ -1,4 +1,4 @@
1
- const path = require('path')
1
+ const path = require('path');
2
2
 
3
3
  module.exports = {
4
4
  context: path.resolve(__dirname, 'src'),
@@ -6,15 +6,15 @@ module.exports = {
6
6
  output: {
7
7
  path: path.resolve(__dirname, 'dist'),
8
8
  filename: '[name].bundle.js',
9
- library: 'Crawler'
9
+ library: 'Crawler',
10
10
  },
11
11
  module: {
12
12
  rules: [
13
13
  {
14
14
  test: /\.js$/,
15
15
  use: 'babel-loader',
16
- exclude: /node_modules/
17
- }
18
- ]
19
- }
20
- }
16
+ exclude: /node_modules/,
17
+ },
18
+ ],
19
+ },
20
+ };
package/webpack.dev.js CHANGED
@@ -1,7 +1,7 @@
1
- const merge = require('webpack-merge')
2
- const common = require('./webpack.common')
1
+ const { merge } = require('webpack-merge');
2
+ const common = require('./webpack.common');
3
3
 
4
4
  module.exports = merge(common, {
5
5
  mode: 'development',
6
- devtool: 'inline-source-map'
7
- })
6
+ devtool: 'inline-source-map',
7
+ });
package/webpack.prod.js CHANGED
@@ -1,15 +1,14 @@
1
- const merge = require('webpack-merge')
2
- const TerserPlugin = require('terser-webpack-plugin')
3
- const common = require('./webpack.common')
1
+ const { merge } = require('webpack-merge');
2
+ const TerserPlugin = require('terser-webpack-plugin');
3
+ const common = require('./webpack.common');
4
4
 
5
5
  module.exports = merge(common, {
6
6
  mode: 'production',
7
7
  optimization: {
8
8
  minimizer: [
9
9
  new TerserPlugin({
10
- cache: true,
11
- parallel: true
12
- })
13
- ]
14
- }
15
- })
10
+ parallel: true,
11
+ }),
12
+ ],
13
+ },
14
+ });