es6-crawler-detect 3.1.3 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (355) hide show
  1. package/.babelrc +18 -17
  2. package/.eslintignore +6 -0
  3. package/.eslintrc.json +30 -0
  4. package/.github/FUNDING.yml +12 -12
  5. package/.github/ISSUE_TEMPLATE/bug_report.md +41 -38
  6. package/.github/ISSUE_TEMPLATE/feature_request.md +20 -20
  7. package/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +29 -0
  8. package/.prettierrc.json +5 -0
  9. package/CODE_OF_CONDUCT.md +76 -76
  10. package/LICENSE +21 -21
  11. package/README.md +103 -93
  12. package/dist/main.bundle.js +1 -1
  13. package/example/node/dist/crawler.min.js +1 -1
  14. package/example/node/index.html +21 -21
  15. package/example/node/node_modules/.bin/mime +15 -0
  16. package/example/node/node_modules/.bin/mime.cmd +7 -0
  17. package/example/node/node_modules/.yarn-integrity +67 -0
  18. package/example/node/node_modules/accepts/HISTORY.md +236 -0
  19. package/example/node/node_modules/accepts/LICENSE +23 -0
  20. package/example/node/node_modules/accepts/README.md +142 -0
  21. package/example/node/node_modules/accepts/index.js +238 -0
  22. package/example/node/node_modules/accepts/package.json +47 -0
  23. package/example/node/node_modules/array-flatten/LICENSE +21 -0
  24. package/example/node/node_modules/array-flatten/README.md +43 -0
  25. package/example/node/node_modules/array-flatten/array-flatten.js +64 -0
  26. package/example/node/node_modules/array-flatten/package.json +39 -0
  27. package/example/node/node_modules/body-parser/HISTORY.md +624 -0
  28. package/example/node/node_modules/body-parser/LICENSE +23 -0
  29. package/example/node/node_modules/body-parser/README.md +456 -0
  30. package/example/node/node_modules/body-parser/index.js +157 -0
  31. package/example/node/node_modules/body-parser/lib/read.js +181 -0
  32. package/example/node/node_modules/body-parser/lib/types/json.js +230 -0
  33. package/example/node/node_modules/body-parser/lib/types/raw.js +101 -0
  34. package/example/node/node_modules/body-parser/lib/types/text.js +121 -0
  35. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +284 -0
  36. package/example/node/node_modules/body-parser/package.json +52 -0
  37. package/example/node/node_modules/bytes/History.md +92 -0
  38. package/example/node/node_modules/bytes/LICENSE +23 -0
  39. package/example/node/node_modules/bytes/Readme.md +152 -0
  40. package/example/node/node_modules/bytes/index.js +166 -0
  41. package/example/node/node_modules/bytes/package.json +42 -0
  42. package/example/node/node_modules/content-disposition/HISTORY.md +60 -0
  43. package/example/node/node_modules/content-disposition/LICENSE +22 -0
  44. package/example/node/node_modules/content-disposition/README.md +142 -0
  45. package/example/node/node_modules/content-disposition/index.js +458 -0
  46. package/example/node/node_modules/content-disposition/package.json +44 -0
  47. package/example/node/node_modules/content-type/HISTORY.md +24 -0
  48. package/example/node/node_modules/content-type/LICENSE +22 -0
  49. package/example/node/node_modules/content-type/README.md +92 -0
  50. package/example/node/node_modules/content-type/index.js +222 -0
  51. package/example/node/node_modules/content-type/package.json +40 -0
  52. package/example/node/node_modules/cookie/HISTORY.md +128 -0
  53. package/example/node/node_modules/cookie/LICENSE +24 -0
  54. package/example/node/node_modules/cookie/README.md +257 -0
  55. package/example/node/node_modules/cookie/index.js +202 -0
  56. package/example/node/node_modules/cookie/package.json +40 -0
  57. package/example/node/node_modules/cookie-signature/History.md +38 -0
  58. package/example/node/node_modules/cookie-signature/Readme.md +42 -0
  59. package/example/node/node_modules/cookie-signature/index.js +51 -0
  60. package/example/node/node_modules/cookie-signature/package.json +18 -0
  61. package/example/node/node_modules/debug/.coveralls.yml +1 -0
  62. package/example/node/node_modules/debug/.eslintrc +11 -0
  63. package/example/node/node_modules/debug/.travis.yml +14 -0
  64. package/example/node/node_modules/debug/CHANGELOG.md +362 -0
  65. package/example/node/node_modules/debug/LICENSE +19 -0
  66. package/example/node/node_modules/debug/Makefile +50 -0
  67. package/example/node/node_modules/debug/README.md +312 -0
  68. package/example/node/node_modules/debug/component.json +19 -0
  69. package/example/node/node_modules/debug/karma.conf.js +70 -0
  70. package/example/node/node_modules/debug/node.js +1 -0
  71. package/example/node/node_modules/debug/package.json +49 -0
  72. package/example/node/node_modules/debug/src/browser.js +185 -0
  73. package/example/node/node_modules/debug/src/debug.js +202 -0
  74. package/example/node/node_modules/debug/src/index.js +10 -0
  75. package/example/node/node_modules/debug/src/inspector-log.js +15 -0
  76. package/example/node/node_modules/debug/src/node.js +248 -0
  77. package/example/node/node_modules/depd/History.md +96 -0
  78. package/example/node/node_modules/depd/LICENSE +22 -0
  79. package/example/node/node_modules/depd/Readme.md +280 -0
  80. package/example/node/node_modules/depd/index.js +522 -0
  81. package/example/node/node_modules/depd/lib/browser/index.js +77 -0
  82. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +103 -0
  83. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +22 -0
  84. package/example/node/node_modules/depd/lib/compat/index.js +79 -0
  85. package/example/node/node_modules/depd/package.json +41 -0
  86. package/example/node/node_modules/destroy/LICENSE +22 -0
  87. package/example/node/node_modules/destroy/README.md +60 -0
  88. package/example/node/node_modules/destroy/index.js +75 -0
  89. package/example/node/node_modules/destroy/package.json +37 -0
  90. package/example/node/node_modules/ee-first/LICENSE +22 -0
  91. package/example/node/node_modules/ee-first/README.md +80 -0
  92. package/example/node/node_modules/ee-first/index.js +95 -0
  93. package/example/node/node_modules/ee-first/package.json +29 -0
  94. package/example/node/node_modules/encodeurl/HISTORY.md +14 -0
  95. package/example/node/node_modules/encodeurl/LICENSE +22 -0
  96. package/example/node/node_modules/encodeurl/README.md +128 -0
  97. package/example/node/node_modules/encodeurl/index.js +60 -0
  98. package/example/node/node_modules/encodeurl/package.json +40 -0
  99. package/example/node/node_modules/escape-html/LICENSE +24 -0
  100. package/example/node/node_modules/escape-html/Readme.md +43 -0
  101. package/example/node/node_modules/escape-html/index.js +78 -0
  102. package/example/node/node_modules/escape-html/package.json +24 -0
  103. package/example/node/node_modules/etag/HISTORY.md +83 -0
  104. package/example/node/node_modules/etag/LICENSE +22 -0
  105. package/example/node/node_modules/etag/README.md +159 -0
  106. package/example/node/node_modules/etag/index.js +131 -0
  107. package/example/node/node_modules/etag/package.json +47 -0
  108. package/example/node/node_modules/express/History.md +3510 -0
  109. package/example/node/node_modules/express/LICENSE +24 -0
  110. package/example/node/node_modules/express/Readme.md +158 -0
  111. package/example/node/node_modules/express/index.js +11 -0
  112. package/example/node/node_modules/express/lib/application.js +644 -0
  113. package/example/node/node_modules/express/lib/express.js +116 -0
  114. package/example/node/node_modules/express/lib/middleware/init.js +43 -0
  115. package/example/node/node_modules/express/lib/middleware/query.js +47 -0
  116. package/example/node/node_modules/express/lib/request.js +525 -0
  117. package/example/node/node_modules/express/lib/response.js +1147 -0
  118. package/example/node/node_modules/express/lib/router/index.js +668 -0
  119. package/example/node/node_modules/express/lib/router/layer.js +181 -0
  120. package/example/node/node_modules/express/lib/router/route.js +216 -0
  121. package/example/node/node_modules/express/lib/utils.js +302 -0
  122. package/example/node/node_modules/express/lib/view.js +182 -0
  123. package/example/node/node_modules/express/package.json +99 -0
  124. package/example/node/node_modules/finalhandler/HISTORY.md +187 -0
  125. package/example/node/node_modules/finalhandler/LICENSE +22 -0
  126. package/example/node/node_modules/finalhandler/README.md +148 -0
  127. package/example/node/node_modules/finalhandler/index.js +331 -0
  128. package/example/node/node_modules/finalhandler/package.json +45 -0
  129. package/example/node/node_modules/forwarded/HISTORY.md +21 -0
  130. package/example/node/node_modules/forwarded/LICENSE +22 -0
  131. package/example/node/node_modules/forwarded/README.md +57 -0
  132. package/example/node/node_modules/forwarded/index.js +90 -0
  133. package/example/node/node_modules/forwarded/package.json +45 -0
  134. package/example/node/node_modules/fresh/HISTORY.md +70 -0
  135. package/example/node/node_modules/fresh/LICENSE +23 -0
  136. package/example/node/node_modules/fresh/README.md +119 -0
  137. package/example/node/node_modules/fresh/index.js +137 -0
  138. package/example/node/node_modules/fresh/package.json +46 -0
  139. package/example/node/node_modules/http-errors/HISTORY.md +165 -0
  140. package/example/node/node_modules/http-errors/LICENSE +23 -0
  141. package/example/node/node_modules/http-errors/README.md +169 -0
  142. package/example/node/node_modules/http-errors/index.js +299 -0
  143. package/example/node/node_modules/http-errors/package.json +49 -0
  144. package/example/node/node_modules/iconv-lite/Changelog.md +162 -0
  145. package/example/node/node_modules/iconv-lite/LICENSE +21 -0
  146. package/example/node/node_modules/iconv-lite/README.md +156 -0
  147. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +555 -0
  148. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +176 -0
  149. package/example/node/node_modules/iconv-lite/encodings/index.js +22 -0
  150. package/example/node/node_modules/iconv-lite/encodings/internal.js +188 -0
  151. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +72 -0
  152. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +451 -0
  153. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +174 -0
  154. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +122 -0
  155. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +264 -0
  156. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +273 -0
  157. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +177 -0
  158. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +182 -0
  159. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +1 -0
  160. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +55 -0
  161. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +125 -0
  162. package/example/node/node_modules/iconv-lite/encodings/utf16.js +177 -0
  163. package/example/node/node_modules/iconv-lite/encodings/utf7.js +290 -0
  164. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +52 -0
  165. package/example/node/node_modules/iconv-lite/lib/extend-node.js +217 -0
  166. package/example/node/node_modules/iconv-lite/lib/index.d.ts +24 -0
  167. package/example/node/node_modules/iconv-lite/lib/index.js +153 -0
  168. package/example/node/node_modules/iconv-lite/lib/streams.js +121 -0
  169. package/example/node/node_modules/iconv-lite/package.json +46 -0
  170. package/example/node/node_modules/inherits/LICENSE +16 -0
  171. package/example/node/node_modules/inherits/README.md +42 -0
  172. package/example/node/node_modules/inherits/inherits.js +9 -0
  173. package/example/node/node_modules/inherits/inherits_browser.js +27 -0
  174. package/example/node/node_modules/inherits/package.json +29 -0
  175. package/example/node/node_modules/ipaddr.js/LICENSE +19 -0
  176. package/example/node/node_modules/ipaddr.js/README.md +233 -0
  177. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +1 -0
  178. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +673 -0
  179. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +68 -0
  180. package/example/node/node_modules/ipaddr.js/package.json +35 -0
  181. package/example/node/node_modules/media-typer/HISTORY.md +22 -0
  182. package/example/node/node_modules/media-typer/LICENSE +22 -0
  183. package/example/node/node_modules/media-typer/README.md +81 -0
  184. package/example/node/node_modules/media-typer/index.js +270 -0
  185. package/example/node/node_modules/media-typer/package.json +26 -0
  186. package/example/node/node_modules/merge-descriptors/HISTORY.md +21 -0
  187. package/example/node/node_modules/merge-descriptors/LICENSE +23 -0
  188. package/example/node/node_modules/merge-descriptors/README.md +48 -0
  189. package/example/node/node_modules/merge-descriptors/index.js +60 -0
  190. package/example/node/node_modules/merge-descriptors/package.json +32 -0
  191. package/example/node/node_modules/methods/HISTORY.md +29 -0
  192. package/example/node/node_modules/methods/LICENSE +24 -0
  193. package/example/node/node_modules/methods/README.md +51 -0
  194. package/example/node/node_modules/methods/index.js +69 -0
  195. package/example/node/node_modules/methods/package.json +36 -0
  196. package/example/node/node_modules/mime/CHANGELOG.md +164 -0
  197. package/example/node/node_modules/mime/LICENSE +21 -0
  198. package/example/node/node_modules/mime/README.md +90 -0
  199. package/example/node/node_modules/mime/cli.js +8 -0
  200. package/example/node/node_modules/mime/mime.js +108 -0
  201. package/example/node/node_modules/mime/package.json +44 -0
  202. package/example/node/node_modules/mime/src/build.js +53 -0
  203. package/example/node/node_modules/mime/src/test.js +60 -0
  204. package/example/node/node_modules/mime/types.json +1 -0
  205. package/example/node/node_modules/mime-db/HISTORY.md +499 -0
  206. package/example/node/node_modules/mime-db/LICENSE +22 -0
  207. package/example/node/node_modules/mime-db/README.md +100 -0
  208. package/example/node/node_modules/mime-db/db.json +8471 -0
  209. package/example/node/node_modules/mime-db/index.js +11 -0
  210. package/example/node/node_modules/mime-db/package.json +59 -0
  211. package/example/node/node_modules/mime-types/HISTORY.md +388 -0
  212. package/example/node/node_modules/mime-types/LICENSE +23 -0
  213. package/example/node/node_modules/mime-types/README.md +113 -0
  214. package/example/node/node_modules/mime-types/index.js +188 -0
  215. package/example/node/node_modules/mime-types/package.json +44 -0
  216. package/example/node/node_modules/ms/index.js +152 -0
  217. package/example/node/node_modules/ms/license.md +21 -0
  218. package/example/node/node_modules/ms/package.json +37 -0
  219. package/example/node/node_modules/ms/readme.md +51 -0
  220. package/example/node/node_modules/negotiator/HISTORY.md +103 -0
  221. package/example/node/node_modules/negotiator/LICENSE +24 -0
  222. package/example/node/node_modules/negotiator/README.md +203 -0
  223. package/example/node/node_modules/negotiator/index.js +124 -0
  224. package/example/node/node_modules/negotiator/lib/charset.js +169 -0
  225. package/example/node/node_modules/negotiator/lib/encoding.js +184 -0
  226. package/example/node/node_modules/negotiator/lib/language.js +179 -0
  227. package/example/node/node_modules/negotiator/lib/mediaType.js +294 -0
  228. package/example/node/node_modules/negotiator/package.json +42 -0
  229. package/example/node/node_modules/on-finished/HISTORY.md +88 -0
  230. package/example/node/node_modules/on-finished/LICENSE +23 -0
  231. package/example/node/node_modules/on-finished/README.md +154 -0
  232. package/example/node/node_modules/on-finished/index.js +196 -0
  233. package/example/node/node_modules/on-finished/package.json +31 -0
  234. package/example/node/node_modules/parseurl/HISTORY.md +58 -0
  235. package/example/node/node_modules/parseurl/LICENSE +24 -0
  236. package/example/node/node_modules/parseurl/README.md +133 -0
  237. package/example/node/node_modules/parseurl/index.js +158 -0
  238. package/example/node/node_modules/parseurl/package.json +40 -0
  239. package/example/node/node_modules/path-to-regexp/History.md +36 -0
  240. package/example/node/node_modules/path-to-regexp/LICENSE +21 -0
  241. package/example/node/node_modules/path-to-regexp/Readme.md +35 -0
  242. package/example/node/node_modules/path-to-regexp/index.js +129 -0
  243. package/example/node/node_modules/path-to-regexp/package.json +30 -0
  244. package/example/node/node_modules/proxy-addr/HISTORY.md +161 -0
  245. package/example/node/node_modules/proxy-addr/LICENSE +22 -0
  246. package/example/node/node_modules/proxy-addr/README.md +139 -0
  247. package/example/node/node_modules/proxy-addr/index.js +327 -0
  248. package/example/node/node_modules/proxy-addr/package.json +47 -0
  249. package/example/node/node_modules/qs/.editorconfig +39 -0
  250. package/example/node/node_modules/qs/.eslintignore +2 -0
  251. package/example/node/node_modules/qs/.eslintrc +35 -0
  252. package/example/node/node_modules/qs/.github/FUNDING.yml +12 -0
  253. package/example/node/node_modules/qs/.nycrc +13 -0
  254. package/example/node/node_modules/qs/CHANGELOG.md +359 -0
  255. package/example/node/node_modules/qs/LICENSE.md +29 -0
  256. package/example/node/node_modules/qs/README.md +604 -0
  257. package/example/node/node_modules/qs/dist/qs.js +832 -0
  258. package/example/node/node_modules/qs/lib/formats.js +23 -0
  259. package/example/node/node_modules/qs/lib/index.js +11 -0
  260. package/example/node/node_modules/qs/lib/parse.js +257 -0
  261. package/example/node/node_modules/qs/lib/stringify.js +278 -0
  262. package/example/node/node_modules/qs/lib/utils.js +251 -0
  263. package/example/node/node_modules/qs/package.json +69 -0
  264. package/example/node/node_modules/qs/test/parse.js +772 -0
  265. package/example/node/node_modules/qs/test/stringify.js +793 -0
  266. package/example/node/node_modules/qs/test/utils.js +136 -0
  267. package/example/node/node_modules/range-parser/HISTORY.md +56 -0
  268. package/example/node/node_modules/range-parser/LICENSE +23 -0
  269. package/example/node/node_modules/range-parser/README.md +84 -0
  270. package/example/node/node_modules/range-parser/index.js +162 -0
  271. package/example/node/node_modules/range-parser/package.json +44 -0
  272. package/example/node/node_modules/raw-body/HISTORY.md +284 -0
  273. package/example/node/node_modules/raw-body/LICENSE +22 -0
  274. package/example/node/node_modules/raw-body/README.md +217 -0
  275. package/example/node/node_modules/raw-body/index.d.ts +87 -0
  276. package/example/node/node_modules/raw-body/index.js +286 -0
  277. package/example/node/node_modules/raw-body/package.json +48 -0
  278. package/example/node/node_modules/safe-buffer/LICENSE +21 -0
  279. package/example/node/node_modules/safe-buffer/README.md +584 -0
  280. package/example/node/node_modules/safe-buffer/index.d.ts +187 -0
  281. package/example/node/node_modules/safe-buffer/index.js +65 -0
  282. package/example/node/node_modules/safe-buffer/package.json +51 -0
  283. package/example/node/node_modules/safer-buffer/LICENSE +21 -0
  284. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +268 -0
  285. package/example/node/node_modules/safer-buffer/Readme.md +156 -0
  286. package/example/node/node_modules/safer-buffer/dangerous.js +58 -0
  287. package/example/node/node_modules/safer-buffer/package.json +34 -0
  288. package/example/node/node_modules/safer-buffer/safer.js +77 -0
  289. package/example/node/node_modules/safer-buffer/tests.js +406 -0
  290. package/example/node/node_modules/send/HISTORY.md +506 -0
  291. package/example/node/node_modules/send/LICENSE +23 -0
  292. package/example/node/node_modules/send/README.md +327 -0
  293. package/example/node/node_modules/send/index.js +1133 -0
  294. package/example/node/node_modules/send/node_modules/.bin/mime +15 -0
  295. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +7 -0
  296. package/example/node/node_modules/send/node_modules/ms/index.js +162 -0
  297. package/example/node/node_modules/send/node_modules/ms/license.md +21 -0
  298. package/example/node/node_modules/send/node_modules/ms/package.json +38 -0
  299. package/example/node/node_modules/send/node_modules/ms/readme.md +59 -0
  300. package/example/node/node_modules/send/package.json +61 -0
  301. package/example/node/node_modules/serve-static/HISTORY.md +459 -0
  302. package/example/node/node_modules/serve-static/LICENSE +25 -0
  303. package/example/node/node_modules/serve-static/README.md +257 -0
  304. package/example/node/node_modules/serve-static/index.js +210 -0
  305. package/example/node/node_modules/serve-static/package.json +42 -0
  306. package/example/node/node_modules/setprototypeof/LICENSE +13 -0
  307. package/example/node/node_modules/setprototypeof/README.md +31 -0
  308. package/example/node/node_modules/setprototypeof/index.d.ts +2 -0
  309. package/example/node/node_modules/setprototypeof/index.js +17 -0
  310. package/example/node/node_modules/setprototypeof/package.json +38 -0
  311. package/example/node/node_modules/setprototypeof/test/index.js +24 -0
  312. package/example/node/node_modules/statuses/HISTORY.md +65 -0
  313. package/example/node/node_modules/statuses/LICENSE +23 -0
  314. package/example/node/node_modules/statuses/README.md +127 -0
  315. package/example/node/node_modules/statuses/codes.json +66 -0
  316. package/example/node/node_modules/statuses/index.js +113 -0
  317. package/example/node/node_modules/statuses/package.json +48 -0
  318. package/example/node/node_modules/toidentifier/HISTORY.md +9 -0
  319. package/example/node/node_modules/toidentifier/LICENSE +21 -0
  320. package/example/node/node_modules/toidentifier/README.md +61 -0
  321. package/example/node/node_modules/toidentifier/index.js +32 -0
  322. package/example/node/node_modules/toidentifier/package.json +38 -0
  323. package/example/node/node_modules/type-is/HISTORY.md +259 -0
  324. package/example/node/node_modules/type-is/LICENSE +23 -0
  325. package/example/node/node_modules/type-is/README.md +170 -0
  326. package/example/node/node_modules/type-is/index.js +266 -0
  327. package/example/node/node_modules/type-is/package.json +45 -0
  328. package/example/node/node_modules/unpipe/HISTORY.md +4 -0
  329. package/example/node/node_modules/unpipe/LICENSE +22 -0
  330. package/example/node/node_modules/unpipe/README.md +43 -0
  331. package/example/node/node_modules/unpipe/index.js +69 -0
  332. package/example/node/node_modules/unpipe/package.json +27 -0
  333. package/example/node/node_modules/utils-merge/LICENSE +20 -0
  334. package/example/node/node_modules/utils-merge/README.md +34 -0
  335. package/example/node/node_modules/utils-merge/index.js +23 -0
  336. package/example/node/node_modules/utils-merge/package.json +40 -0
  337. package/example/node/node_modules/vary/HISTORY.md +39 -0
  338. package/example/node/node_modules/vary/LICENSE +22 -0
  339. package/example/node/node_modules/vary/README.md +101 -0
  340. package/example/node/node_modules/vary/index.js +149 -0
  341. package/example/node/node_modules/vary/package.json +43 -0
  342. package/example/node/package.json +11 -0
  343. package/example/node/server.js +21 -15
  344. package/example/node/yarn.lock +352 -0
  345. package/package.json +59 -47
  346. package/src/index.js +15 -10
  347. package/src/lib/crawler/crawlers.js +1398 -11
  348. package/src/lib/crawler/exclusions.js +55 -8
  349. package/src/lib/crawler/headers.js +18 -9
  350. package/src/lib/crawler/provider.js +6 -8
  351. package/src/lib/crawler.js +101 -90
  352. package/test/lib/crawler.test.js +43 -26
  353. package/webpack.common.js +7 -7
  354. package/webpack.dev.js +4 -4
  355. package/webpack.prod.js +8 -9
@@ -0,0 +1,668 @@
1
+ /*!
2
+ * express
3
+ * Copyright(c) 2009-2013 TJ Holowaychuk
4
+ * Copyright(c) 2013 Roman Shtylman
5
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
6
+ * MIT Licensed
7
+ */
8
+
9
+ 'use strict';
10
+
11
+ /**
12
+ * Module dependencies.
13
+ * @private
14
+ */
15
+
16
+ var Route = require('./route');
17
+ var Layer = require('./layer');
18
+ var methods = require('methods');
19
+ var mixin = require('utils-merge');
20
+ var debug = require('debug')('express:router');
21
+ var deprecate = require('depd')('express');
22
+ var flatten = require('array-flatten');
23
+ var parseUrl = require('parseurl');
24
+ var setPrototypeOf = require('setprototypeof')
25
+
26
+ /**
27
+ * Module variables.
28
+ * @private
29
+ */
30
+
31
+ var objectRegExp = /^\[object (\S+)\]$/;
32
+ var slice = Array.prototype.slice;
33
+ var toString = Object.prototype.toString;
34
+
35
+ /**
36
+ * Initialize a new `Router` with the given `options`.
37
+ *
38
+ * @param {Object} [options]
39
+ * @return {Router} which is an callable function
40
+ * @public
41
+ */
42
+
43
+ var proto = module.exports = function(options) {
44
+ var opts = options || {};
45
+
46
+ function router(req, res, next) {
47
+ router.handle(req, res, next);
48
+ }
49
+
50
+ // mixin Router class functions
51
+ setPrototypeOf(router, proto)
52
+
53
+ router.params = {};
54
+ router._params = [];
55
+ router.caseSensitive = opts.caseSensitive;
56
+ router.mergeParams = opts.mergeParams;
57
+ router.strict = opts.strict;
58
+ router.stack = [];
59
+
60
+ return router;
61
+ };
62
+
63
+ /**
64
+ * Map the given param placeholder `name`(s) to the given callback.
65
+ *
66
+ * Parameter mapping is used to provide pre-conditions to routes
67
+ * which use normalized placeholders. For example a _:user_id_ parameter
68
+ * could automatically load a user's information from the database without
69
+ * any additional code,
70
+ *
71
+ * The callback uses the same signature as middleware, the only difference
72
+ * being that the value of the placeholder is passed, in this case the _id_
73
+ * of the user. Once the `next()` function is invoked, just like middleware
74
+ * it will continue on to execute the route, or subsequent parameter functions.
75
+ *
76
+ * Just like in middleware, you must either respond to the request or call next
77
+ * to avoid stalling the request.
78
+ *
79
+ * app.param('user_id', function(req, res, next, id){
80
+ * User.find(id, function(err, user){
81
+ * if (err) {
82
+ * return next(err);
83
+ * } else if (!user) {
84
+ * return next(new Error('failed to load user'));
85
+ * }
86
+ * req.user = user;
87
+ * next();
88
+ * });
89
+ * });
90
+ *
91
+ * @param {String} name
92
+ * @param {Function} fn
93
+ * @return {app} for chaining
94
+ * @public
95
+ */
96
+
97
+ proto.param = function param(name, fn) {
98
+ // param logic
99
+ if (typeof name === 'function') {
100
+ deprecate('router.param(fn): Refactor to use path params');
101
+ this._params.push(name);
102
+ return;
103
+ }
104
+
105
+ // apply param functions
106
+ var params = this._params;
107
+ var len = params.length;
108
+ var ret;
109
+
110
+ if (name[0] === ':') {
111
+ deprecate('router.param(' + JSON.stringify(name) + ', fn): Use router.param(' + JSON.stringify(name.substr(1)) + ', fn) instead');
112
+ name = name.substr(1);
113
+ }
114
+
115
+ for (var i = 0; i < len; ++i) {
116
+ if (ret = params[i](name, fn)) {
117
+ fn = ret;
118
+ }
119
+ }
120
+
121
+ // ensure we end up with a
122
+ // middleware function
123
+ if ('function' !== typeof fn) {
124
+ throw new Error('invalid param() call for ' + name + ', got ' + fn);
125
+ }
126
+
127
+ (this.params[name] = this.params[name] || []).push(fn);
128
+ return this;
129
+ };
130
+
131
+ /**
132
+ * Dispatch a req, res into the router.
133
+ * @private
134
+ */
135
+
136
+ proto.handle = function handle(req, res, out) {
137
+ var self = this;
138
+
139
+ debug('dispatching %s %s', req.method, req.url);
140
+
141
+ var idx = 0;
142
+ var protohost = getProtohost(req.url) || ''
143
+ var removed = '';
144
+ var slashAdded = false;
145
+ var paramcalled = {};
146
+
147
+ // store options for OPTIONS request
148
+ // only used if OPTIONS request
149
+ var options = [];
150
+
151
+ // middleware and routes
152
+ var stack = self.stack;
153
+
154
+ // manage inter-router variables
155
+ var parentParams = req.params;
156
+ var parentUrl = req.baseUrl || '';
157
+ var done = restore(out, req, 'baseUrl', 'next', 'params');
158
+
159
+ // setup next layer
160
+ req.next = next;
161
+
162
+ // for options requests, respond with a default if nothing else responds
163
+ if (req.method === 'OPTIONS') {
164
+ done = wrap(done, function(old, err) {
165
+ if (err || options.length === 0) return old(err);
166
+ sendOptionsResponse(res, options, old);
167
+ });
168
+ }
169
+
170
+ // setup basic req values
171
+ req.baseUrl = parentUrl;
172
+ req.originalUrl = req.originalUrl || req.url;
173
+
174
+ next();
175
+
176
+ function next(err) {
177
+ var layerError = err === 'route'
178
+ ? null
179
+ : err;
180
+
181
+ // remove added slash
182
+ if (slashAdded) {
183
+ req.url = req.url.substr(1);
184
+ slashAdded = false;
185
+ }
186
+
187
+ // restore altered req.url
188
+ if (removed.length !== 0) {
189
+ req.baseUrl = parentUrl;
190
+ req.url = protohost + removed + req.url.substr(protohost.length);
191
+ removed = '';
192
+ }
193
+
194
+ // signal to exit router
195
+ if (layerError === 'router') {
196
+ setImmediate(done, null)
197
+ return
198
+ }
199
+
200
+ // no more matching layers
201
+ if (idx >= stack.length) {
202
+ setImmediate(done, layerError);
203
+ return;
204
+ }
205
+
206
+ // get pathname of request
207
+ var path = getPathname(req);
208
+
209
+ if (path == null) {
210
+ return done(layerError);
211
+ }
212
+
213
+ // find next matching layer
214
+ var layer;
215
+ var match;
216
+ var route;
217
+
218
+ while (match !== true && idx < stack.length) {
219
+ layer = stack[idx++];
220
+ match = matchLayer(layer, path);
221
+ route = layer.route;
222
+
223
+ if (typeof match !== 'boolean') {
224
+ // hold on to layerError
225
+ layerError = layerError || match;
226
+ }
227
+
228
+ if (match !== true) {
229
+ continue;
230
+ }
231
+
232
+ if (!route) {
233
+ // process non-route handlers normally
234
+ continue;
235
+ }
236
+
237
+ if (layerError) {
238
+ // routes do not match with a pending error
239
+ match = false;
240
+ continue;
241
+ }
242
+
243
+ var method = req.method;
244
+ var has_method = route._handles_method(method);
245
+
246
+ // build up automatic options response
247
+ if (!has_method && method === 'OPTIONS') {
248
+ appendMethods(options, route._options());
249
+ }
250
+
251
+ // don't even bother matching route
252
+ if (!has_method && method !== 'HEAD') {
253
+ match = false;
254
+ continue;
255
+ }
256
+ }
257
+
258
+ // no match
259
+ if (match !== true) {
260
+ return done(layerError);
261
+ }
262
+
263
+ // store route for dispatch on change
264
+ if (route) {
265
+ req.route = route;
266
+ }
267
+
268
+ // Capture one-time layer values
269
+ req.params = self.mergeParams
270
+ ? mergeParams(layer.params, parentParams)
271
+ : layer.params;
272
+ var layerPath = layer.path;
273
+
274
+ // this should be done for the layer
275
+ self.process_params(layer, paramcalled, req, res, function (err) {
276
+ if (err) {
277
+ return next(layerError || err);
278
+ }
279
+
280
+ if (route) {
281
+ return layer.handle_request(req, res, next);
282
+ }
283
+
284
+ trim_prefix(layer, layerError, layerPath, path);
285
+ });
286
+ }
287
+
288
+ function trim_prefix(layer, layerError, layerPath, path) {
289
+ if (layerPath.length !== 0) {
290
+ // Validate path is a prefix match
291
+ if (layerPath !== path.substr(0, layerPath.length)) {
292
+ next(layerError)
293
+ return
294
+ }
295
+
296
+ // Validate path breaks on a path separator
297
+ var c = path[layerPath.length]
298
+ if (c && c !== '/' && c !== '.') return next(layerError)
299
+
300
+ // Trim off the part of the url that matches the route
301
+ // middleware (.use stuff) needs to have the path stripped
302
+ debug('trim prefix (%s) from url %s', layerPath, req.url);
303
+ removed = layerPath;
304
+ req.url = protohost + req.url.substr(protohost.length + removed.length);
305
+
306
+ // Ensure leading slash
307
+ if (!protohost && req.url[0] !== '/') {
308
+ req.url = '/' + req.url;
309
+ slashAdded = true;
310
+ }
311
+
312
+ // Setup base URL (no trailing slash)
313
+ req.baseUrl = parentUrl + (removed[removed.length - 1] === '/'
314
+ ? removed.substring(0, removed.length - 1)
315
+ : removed);
316
+ }
317
+
318
+ debug('%s %s : %s', layer.name, layerPath, req.originalUrl);
319
+
320
+ if (layerError) {
321
+ layer.handle_error(layerError, req, res, next);
322
+ } else {
323
+ layer.handle_request(req, res, next);
324
+ }
325
+ }
326
+ };
327
+
328
+ /**
329
+ * Process any parameters for the layer.
330
+ * @private
331
+ */
332
+
333
+ proto.process_params = function process_params(layer, called, req, res, done) {
334
+ var params = this.params;
335
+
336
+ // captured parameters from the layer, keys and values
337
+ var keys = layer.keys;
338
+
339
+ // fast track
340
+ if (!keys || keys.length === 0) {
341
+ return done();
342
+ }
343
+
344
+ var i = 0;
345
+ var name;
346
+ var paramIndex = 0;
347
+ var key;
348
+ var paramVal;
349
+ var paramCallbacks;
350
+ var paramCalled;
351
+
352
+ // process params in order
353
+ // param callbacks can be async
354
+ function param(err) {
355
+ if (err) {
356
+ return done(err);
357
+ }
358
+
359
+ if (i >= keys.length ) {
360
+ return done();
361
+ }
362
+
363
+ paramIndex = 0;
364
+ key = keys[i++];
365
+ name = key.name;
366
+ paramVal = req.params[name];
367
+ paramCallbacks = params[name];
368
+ paramCalled = called[name];
369
+
370
+ if (paramVal === undefined || !paramCallbacks) {
371
+ return param();
372
+ }
373
+
374
+ // param previously called with same value or error occurred
375
+ if (paramCalled && (paramCalled.match === paramVal
376
+ || (paramCalled.error && paramCalled.error !== 'route'))) {
377
+ // restore value
378
+ req.params[name] = paramCalled.value;
379
+
380
+ // next param
381
+ return param(paramCalled.error);
382
+ }
383
+
384
+ called[name] = paramCalled = {
385
+ error: null,
386
+ match: paramVal,
387
+ value: paramVal
388
+ };
389
+
390
+ paramCallback();
391
+ }
392
+
393
+ // single param callbacks
394
+ function paramCallback(err) {
395
+ var fn = paramCallbacks[paramIndex++];
396
+
397
+ // store updated value
398
+ paramCalled.value = req.params[key.name];
399
+
400
+ if (err) {
401
+ // store error
402
+ paramCalled.error = err;
403
+ param(err);
404
+ return;
405
+ }
406
+
407
+ if (!fn) return param();
408
+
409
+ try {
410
+ fn(req, res, paramCallback, paramVal, key.name);
411
+ } catch (e) {
412
+ paramCallback(e);
413
+ }
414
+ }
415
+
416
+ param();
417
+ };
418
+
419
+ /**
420
+ * Use the given middleware function, with optional path, defaulting to "/".
421
+ *
422
+ * Use (like `.all`) will run for any http METHOD, but it will not add
423
+ * handlers for those methods so OPTIONS requests will not consider `.use`
424
+ * functions even if they could respond.
425
+ *
426
+ * The other difference is that _route_ path is stripped and not visible
427
+ * to the handler function. The main effect of this feature is that mounted
428
+ * handlers can operate without any code changes regardless of the "prefix"
429
+ * pathname.
430
+ *
431
+ * @public
432
+ */
433
+
434
+ proto.use = function use(fn) {
435
+ var offset = 0;
436
+ var path = '/';
437
+
438
+ // default path to '/'
439
+ // disambiguate router.use([fn])
440
+ if (typeof fn !== 'function') {
441
+ var arg = fn;
442
+
443
+ while (Array.isArray(arg) && arg.length !== 0) {
444
+ arg = arg[0];
445
+ }
446
+
447
+ // first arg is the path
448
+ if (typeof arg !== 'function') {
449
+ offset = 1;
450
+ path = fn;
451
+ }
452
+ }
453
+
454
+ var callbacks = flatten(slice.call(arguments, offset));
455
+
456
+ if (callbacks.length === 0) {
457
+ throw new TypeError('Router.use() requires a middleware function')
458
+ }
459
+
460
+ for (var i = 0; i < callbacks.length; i++) {
461
+ var fn = callbacks[i];
462
+
463
+ if (typeof fn !== 'function') {
464
+ throw new TypeError('Router.use() requires a middleware function but got a ' + gettype(fn))
465
+ }
466
+
467
+ // add the middleware
468
+ debug('use %o %s', path, fn.name || '<anonymous>')
469
+
470
+ var layer = new Layer(path, {
471
+ sensitive: this.caseSensitive,
472
+ strict: false,
473
+ end: false
474
+ }, fn);
475
+
476
+ layer.route = undefined;
477
+
478
+ this.stack.push(layer);
479
+ }
480
+
481
+ return this;
482
+ };
483
+
484
+ /**
485
+ * Create a new Route for the given path.
486
+ *
487
+ * Each route contains a separate middleware stack and VERB handlers.
488
+ *
489
+ * See the Route api documentation for details on adding handlers
490
+ * and middleware to routes.
491
+ *
492
+ * @param {String} path
493
+ * @return {Route}
494
+ * @public
495
+ */
496
+
497
+ proto.route = function route(path) {
498
+ var route = new Route(path);
499
+
500
+ var layer = new Layer(path, {
501
+ sensitive: this.caseSensitive,
502
+ strict: this.strict,
503
+ end: true
504
+ }, route.dispatch.bind(route));
505
+
506
+ layer.route = route;
507
+
508
+ this.stack.push(layer);
509
+ return route;
510
+ };
511
+
512
+ // create Router#VERB functions
513
+ methods.concat('all').forEach(function(method){
514
+ proto[method] = function(path){
515
+ var route = this.route(path)
516
+ route[method].apply(route, slice.call(arguments, 1));
517
+ return this;
518
+ };
519
+ });
520
+
521
+ // append methods to a list of methods
522
+ function appendMethods(list, addition) {
523
+ for (var i = 0; i < addition.length; i++) {
524
+ var method = addition[i];
525
+ if (list.indexOf(method) === -1) {
526
+ list.push(method);
527
+ }
528
+ }
529
+ }
530
+
531
+ // get pathname of request
532
+ function getPathname(req) {
533
+ try {
534
+ return parseUrl(req).pathname;
535
+ } catch (err) {
536
+ return undefined;
537
+ }
538
+ }
539
+
540
+ // Get get protocol + host for a URL
541
+ function getProtohost(url) {
542
+ if (typeof url !== 'string' || url.length === 0 || url[0] === '/') {
543
+ return undefined
544
+ }
545
+
546
+ var searchIndex = url.indexOf('?')
547
+ var pathLength = searchIndex !== -1
548
+ ? searchIndex
549
+ : url.length
550
+ var fqdnIndex = url.substr(0, pathLength).indexOf('://')
551
+
552
+ return fqdnIndex !== -1
553
+ ? url.substr(0, url.indexOf('/', 3 + fqdnIndex))
554
+ : undefined
555
+ }
556
+
557
+ // get type for error message
558
+ function gettype(obj) {
559
+ var type = typeof obj;
560
+
561
+ if (type !== 'object') {
562
+ return type;
563
+ }
564
+
565
+ // inspect [[Class]] for objects
566
+ return toString.call(obj)
567
+ .replace(objectRegExp, '$1');
568
+ }
569
+
570
+ /**
571
+ * Match path to a layer.
572
+ *
573
+ * @param {Layer} layer
574
+ * @param {string} path
575
+ * @private
576
+ */
577
+
578
+ function matchLayer(layer, path) {
579
+ try {
580
+ return layer.match(path);
581
+ } catch (err) {
582
+ return err;
583
+ }
584
+ }
585
+
586
+ // merge params with parent params
587
+ function mergeParams(params, parent) {
588
+ if (typeof parent !== 'object' || !parent) {
589
+ return params;
590
+ }
591
+
592
+ // make copy of parent for base
593
+ var obj = mixin({}, parent);
594
+
595
+ // simple non-numeric merging
596
+ if (!(0 in params) || !(0 in parent)) {
597
+ return mixin(obj, params);
598
+ }
599
+
600
+ var i = 0;
601
+ var o = 0;
602
+
603
+ // determine numeric gaps
604
+ while (i in params) {
605
+ i++;
606
+ }
607
+
608
+ while (o in parent) {
609
+ o++;
610
+ }
611
+
612
+ // offset numeric indices in params before merge
613
+ for (i--; i >= 0; i--) {
614
+ params[i + o] = params[i];
615
+
616
+ // create holes for the merge when necessary
617
+ if (i < o) {
618
+ delete params[i];
619
+ }
620
+ }
621
+
622
+ return mixin(obj, params);
623
+ }
624
+
625
+ // restore obj props after function
626
+ function restore(fn, obj) {
627
+ var props = new Array(arguments.length - 2);
628
+ var vals = new Array(arguments.length - 2);
629
+
630
+ for (var i = 0; i < props.length; i++) {
631
+ props[i] = arguments[i + 2];
632
+ vals[i] = obj[props[i]];
633
+ }
634
+
635
+ return function () {
636
+ // restore vals
637
+ for (var i = 0; i < props.length; i++) {
638
+ obj[props[i]] = vals[i];
639
+ }
640
+
641
+ return fn.apply(this, arguments);
642
+ };
643
+ }
644
+
645
+ // send an OPTIONS response
646
+ function sendOptionsResponse(res, options, next) {
647
+ try {
648
+ var body = options.join(',');
649
+ res.set('Allow', body);
650
+ res.send(body);
651
+ } catch (err) {
652
+ next(err);
653
+ }
654
+ }
655
+
656
+ // wrap a function
657
+ function wrap(old, fn) {
658
+ return function proxy() {
659
+ var args = new Array(arguments.length + 1);
660
+
661
+ args[0] = old;
662
+ for (var i = 0, len = arguments.length; i < len; i++) {
663
+ args[i + 1] = arguments[i];
664
+ }
665
+
666
+ fn.apply(this, args);
667
+ };
668
+ }