es6-crawler-detect 3.3.0 → 4.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (358) hide show
  1. package/.babelrc +1 -1
  2. package/.mocharc.json +5 -0
  3. package/README.md +1 -0
  4. package/dist/main.bundle.js +1 -1
  5. package/eslint.config.mjs +62 -0
  6. package/example/node/package.json +1 -1
  7. package/example/node/yarn.lock +222 -99
  8. package/package.json +30 -25
  9. package/src/index.js +20 -14
  10. package/src/index.ts +18 -0
  11. package/src/lib/crawler/crawlers.js +1399 -1403
  12. package/src/lib/crawler/crawlers.ts +1397 -0
  13. package/src/lib/crawler/exclusions.js +58 -62
  14. package/src/lib/crawler/exclusions.ts +56 -0
  15. package/src/lib/crawler/headers.js +6 -10
  16. package/src/lib/crawler/headers.ts +18 -0
  17. package/src/lib/crawler/provider.js +1 -10
  18. package/src/lib/crawler/provider.ts +3 -0
  19. package/src/lib/crawler.js +98 -130
  20. package/src/lib/crawler.ts +128 -0
  21. package/src/types.js +2 -0
  22. package/src/types.ts +15 -0
  23. package/test/lib/crawler.spec.js +149 -0
  24. package/test/lib/crawler.spec.ts +116 -0
  25. package/tsconfig.json +16 -0
  26. package/webpack.common.js +20 -20
  27. package/webpack.dev.js +7 -7
  28. package/webpack.prod.js +14 -14
  29. package/.eslintignore +0 -6
  30. package/.eslintrc.json +0 -30
  31. package/example/node/node_modules/.bin/mime +0 -15
  32. package/example/node/node_modules/.bin/mime.cmd +0 -7
  33. package/example/node/node_modules/.yarn-integrity +0 -67
  34. package/example/node/node_modules/accepts/HISTORY.md +0 -236
  35. package/example/node/node_modules/accepts/LICENSE +0 -23
  36. package/example/node/node_modules/accepts/README.md +0 -142
  37. package/example/node/node_modules/accepts/index.js +0 -238
  38. package/example/node/node_modules/accepts/package.json +0 -47
  39. package/example/node/node_modules/array-flatten/LICENSE +0 -21
  40. package/example/node/node_modules/array-flatten/README.md +0 -43
  41. package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
  42. package/example/node/node_modules/array-flatten/package.json +0 -39
  43. package/example/node/node_modules/body-parser/HISTORY.md +0 -624
  44. package/example/node/node_modules/body-parser/LICENSE +0 -23
  45. package/example/node/node_modules/body-parser/README.md +0 -456
  46. package/example/node/node_modules/body-parser/index.js +0 -157
  47. package/example/node/node_modules/body-parser/lib/read.js +0 -181
  48. package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
  49. package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
  50. package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
  51. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
  52. package/example/node/node_modules/body-parser/package.json +0 -52
  53. package/example/node/node_modules/bytes/History.md +0 -92
  54. package/example/node/node_modules/bytes/LICENSE +0 -23
  55. package/example/node/node_modules/bytes/Readme.md +0 -152
  56. package/example/node/node_modules/bytes/index.js +0 -166
  57. package/example/node/node_modules/bytes/package.json +0 -42
  58. package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
  59. package/example/node/node_modules/content-disposition/LICENSE +0 -22
  60. package/example/node/node_modules/content-disposition/README.md +0 -142
  61. package/example/node/node_modules/content-disposition/index.js +0 -458
  62. package/example/node/node_modules/content-disposition/package.json +0 -44
  63. package/example/node/node_modules/content-type/HISTORY.md +0 -24
  64. package/example/node/node_modules/content-type/LICENSE +0 -22
  65. package/example/node/node_modules/content-type/README.md +0 -92
  66. package/example/node/node_modules/content-type/index.js +0 -222
  67. package/example/node/node_modules/content-type/package.json +0 -40
  68. package/example/node/node_modules/cookie/HISTORY.md +0 -128
  69. package/example/node/node_modules/cookie/LICENSE +0 -24
  70. package/example/node/node_modules/cookie/README.md +0 -257
  71. package/example/node/node_modules/cookie/index.js +0 -202
  72. package/example/node/node_modules/cookie/package.json +0 -40
  73. package/example/node/node_modules/cookie-signature/History.md +0 -38
  74. package/example/node/node_modules/cookie-signature/Readme.md +0 -42
  75. package/example/node/node_modules/cookie-signature/index.js +0 -51
  76. package/example/node/node_modules/cookie-signature/package.json +0 -18
  77. package/example/node/node_modules/debug/.coveralls.yml +0 -1
  78. package/example/node/node_modules/debug/.eslintrc +0 -11
  79. package/example/node/node_modules/debug/.travis.yml +0 -14
  80. package/example/node/node_modules/debug/CHANGELOG.md +0 -362
  81. package/example/node/node_modules/debug/LICENSE +0 -19
  82. package/example/node/node_modules/debug/Makefile +0 -50
  83. package/example/node/node_modules/debug/README.md +0 -312
  84. package/example/node/node_modules/debug/component.json +0 -19
  85. package/example/node/node_modules/debug/karma.conf.js +0 -70
  86. package/example/node/node_modules/debug/node.js +0 -1
  87. package/example/node/node_modules/debug/package.json +0 -49
  88. package/example/node/node_modules/debug/src/browser.js +0 -185
  89. package/example/node/node_modules/debug/src/debug.js +0 -202
  90. package/example/node/node_modules/debug/src/index.js +0 -10
  91. package/example/node/node_modules/debug/src/inspector-log.js +0 -15
  92. package/example/node/node_modules/debug/src/node.js +0 -248
  93. package/example/node/node_modules/depd/History.md +0 -96
  94. package/example/node/node_modules/depd/LICENSE +0 -22
  95. package/example/node/node_modules/depd/Readme.md +0 -280
  96. package/example/node/node_modules/depd/index.js +0 -522
  97. package/example/node/node_modules/depd/lib/browser/index.js +0 -77
  98. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
  99. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
  100. package/example/node/node_modules/depd/lib/compat/index.js +0 -79
  101. package/example/node/node_modules/depd/package.json +0 -41
  102. package/example/node/node_modules/destroy/LICENSE +0 -22
  103. package/example/node/node_modules/destroy/README.md +0 -60
  104. package/example/node/node_modules/destroy/index.js +0 -75
  105. package/example/node/node_modules/destroy/package.json +0 -37
  106. package/example/node/node_modules/ee-first/LICENSE +0 -22
  107. package/example/node/node_modules/ee-first/README.md +0 -80
  108. package/example/node/node_modules/ee-first/index.js +0 -95
  109. package/example/node/node_modules/ee-first/package.json +0 -29
  110. package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
  111. package/example/node/node_modules/encodeurl/LICENSE +0 -22
  112. package/example/node/node_modules/encodeurl/README.md +0 -128
  113. package/example/node/node_modules/encodeurl/index.js +0 -60
  114. package/example/node/node_modules/encodeurl/package.json +0 -40
  115. package/example/node/node_modules/escape-html/LICENSE +0 -24
  116. package/example/node/node_modules/escape-html/Readme.md +0 -43
  117. package/example/node/node_modules/escape-html/index.js +0 -78
  118. package/example/node/node_modules/escape-html/package.json +0 -24
  119. package/example/node/node_modules/etag/HISTORY.md +0 -83
  120. package/example/node/node_modules/etag/LICENSE +0 -22
  121. package/example/node/node_modules/etag/README.md +0 -159
  122. package/example/node/node_modules/etag/index.js +0 -131
  123. package/example/node/node_modules/etag/package.json +0 -47
  124. package/example/node/node_modules/express/History.md +0 -3510
  125. package/example/node/node_modules/express/LICENSE +0 -24
  126. package/example/node/node_modules/express/Readme.md +0 -158
  127. package/example/node/node_modules/express/index.js +0 -11
  128. package/example/node/node_modules/express/lib/application.js +0 -644
  129. package/example/node/node_modules/express/lib/express.js +0 -116
  130. package/example/node/node_modules/express/lib/middleware/init.js +0 -43
  131. package/example/node/node_modules/express/lib/middleware/query.js +0 -47
  132. package/example/node/node_modules/express/lib/request.js +0 -525
  133. package/example/node/node_modules/express/lib/response.js +0 -1147
  134. package/example/node/node_modules/express/lib/router/index.js +0 -668
  135. package/example/node/node_modules/express/lib/router/layer.js +0 -181
  136. package/example/node/node_modules/express/lib/router/route.js +0 -216
  137. package/example/node/node_modules/express/lib/utils.js +0 -302
  138. package/example/node/node_modules/express/lib/view.js +0 -182
  139. package/example/node/node_modules/express/package.json +0 -99
  140. package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
  141. package/example/node/node_modules/finalhandler/LICENSE +0 -22
  142. package/example/node/node_modules/finalhandler/README.md +0 -148
  143. package/example/node/node_modules/finalhandler/index.js +0 -331
  144. package/example/node/node_modules/finalhandler/package.json +0 -45
  145. package/example/node/node_modules/forwarded/HISTORY.md +0 -21
  146. package/example/node/node_modules/forwarded/LICENSE +0 -22
  147. package/example/node/node_modules/forwarded/README.md +0 -57
  148. package/example/node/node_modules/forwarded/index.js +0 -90
  149. package/example/node/node_modules/forwarded/package.json +0 -45
  150. package/example/node/node_modules/fresh/HISTORY.md +0 -70
  151. package/example/node/node_modules/fresh/LICENSE +0 -23
  152. package/example/node/node_modules/fresh/README.md +0 -119
  153. package/example/node/node_modules/fresh/index.js +0 -137
  154. package/example/node/node_modules/fresh/package.json +0 -46
  155. package/example/node/node_modules/http-errors/HISTORY.md +0 -165
  156. package/example/node/node_modules/http-errors/LICENSE +0 -23
  157. package/example/node/node_modules/http-errors/README.md +0 -169
  158. package/example/node/node_modules/http-errors/index.js +0 -299
  159. package/example/node/node_modules/http-errors/package.json +0 -49
  160. package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
  161. package/example/node/node_modules/iconv-lite/LICENSE +0 -21
  162. package/example/node/node_modules/iconv-lite/README.md +0 -156
  163. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
  164. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
  165. package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
  166. package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
  167. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
  168. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
  169. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
  170. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
  171. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
  172. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
  173. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
  174. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
  175. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
  176. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
  177. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
  178. package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
  179. package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
  180. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
  181. package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
  182. package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
  183. package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
  184. package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
  185. package/example/node/node_modules/iconv-lite/package.json +0 -46
  186. package/example/node/node_modules/inherits/LICENSE +0 -16
  187. package/example/node/node_modules/inherits/README.md +0 -42
  188. package/example/node/node_modules/inherits/inherits.js +0 -9
  189. package/example/node/node_modules/inherits/inherits_browser.js +0 -27
  190. package/example/node/node_modules/inherits/package.json +0 -29
  191. package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
  192. package/example/node/node_modules/ipaddr.js/README.md +0 -233
  193. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
  194. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
  195. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
  196. package/example/node/node_modules/ipaddr.js/package.json +0 -35
  197. package/example/node/node_modules/media-typer/HISTORY.md +0 -22
  198. package/example/node/node_modules/media-typer/LICENSE +0 -22
  199. package/example/node/node_modules/media-typer/README.md +0 -81
  200. package/example/node/node_modules/media-typer/index.js +0 -270
  201. package/example/node/node_modules/media-typer/package.json +0 -26
  202. package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
  203. package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
  204. package/example/node/node_modules/merge-descriptors/README.md +0 -48
  205. package/example/node/node_modules/merge-descriptors/index.js +0 -60
  206. package/example/node/node_modules/merge-descriptors/package.json +0 -32
  207. package/example/node/node_modules/methods/HISTORY.md +0 -29
  208. package/example/node/node_modules/methods/LICENSE +0 -24
  209. package/example/node/node_modules/methods/README.md +0 -51
  210. package/example/node/node_modules/methods/index.js +0 -69
  211. package/example/node/node_modules/methods/package.json +0 -36
  212. package/example/node/node_modules/mime/CHANGELOG.md +0 -164
  213. package/example/node/node_modules/mime/LICENSE +0 -21
  214. package/example/node/node_modules/mime/README.md +0 -90
  215. package/example/node/node_modules/mime/cli.js +0 -8
  216. package/example/node/node_modules/mime/mime.js +0 -108
  217. package/example/node/node_modules/mime/package.json +0 -44
  218. package/example/node/node_modules/mime/src/build.js +0 -53
  219. package/example/node/node_modules/mime/src/test.js +0 -60
  220. package/example/node/node_modules/mime/types.json +0 -1
  221. package/example/node/node_modules/mime-db/HISTORY.md +0 -499
  222. package/example/node/node_modules/mime-db/LICENSE +0 -22
  223. package/example/node/node_modules/mime-db/README.md +0 -100
  224. package/example/node/node_modules/mime-db/db.json +0 -8471
  225. package/example/node/node_modules/mime-db/index.js +0 -11
  226. package/example/node/node_modules/mime-db/package.json +0 -59
  227. package/example/node/node_modules/mime-types/HISTORY.md +0 -388
  228. package/example/node/node_modules/mime-types/LICENSE +0 -23
  229. package/example/node/node_modules/mime-types/README.md +0 -113
  230. package/example/node/node_modules/mime-types/index.js +0 -188
  231. package/example/node/node_modules/mime-types/package.json +0 -44
  232. package/example/node/node_modules/ms/index.js +0 -152
  233. package/example/node/node_modules/ms/license.md +0 -21
  234. package/example/node/node_modules/ms/package.json +0 -37
  235. package/example/node/node_modules/ms/readme.md +0 -51
  236. package/example/node/node_modules/negotiator/HISTORY.md +0 -103
  237. package/example/node/node_modules/negotiator/LICENSE +0 -24
  238. package/example/node/node_modules/negotiator/README.md +0 -203
  239. package/example/node/node_modules/negotiator/index.js +0 -124
  240. package/example/node/node_modules/negotiator/lib/charset.js +0 -169
  241. package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
  242. package/example/node/node_modules/negotiator/lib/language.js +0 -179
  243. package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
  244. package/example/node/node_modules/negotiator/package.json +0 -42
  245. package/example/node/node_modules/on-finished/HISTORY.md +0 -88
  246. package/example/node/node_modules/on-finished/LICENSE +0 -23
  247. package/example/node/node_modules/on-finished/README.md +0 -154
  248. package/example/node/node_modules/on-finished/index.js +0 -196
  249. package/example/node/node_modules/on-finished/package.json +0 -31
  250. package/example/node/node_modules/parseurl/HISTORY.md +0 -58
  251. package/example/node/node_modules/parseurl/LICENSE +0 -24
  252. package/example/node/node_modules/parseurl/README.md +0 -133
  253. package/example/node/node_modules/parseurl/index.js +0 -158
  254. package/example/node/node_modules/parseurl/package.json +0 -40
  255. package/example/node/node_modules/path-to-regexp/History.md +0 -36
  256. package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
  257. package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
  258. package/example/node/node_modules/path-to-regexp/index.js +0 -129
  259. package/example/node/node_modules/path-to-regexp/package.json +0 -30
  260. package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
  261. package/example/node/node_modules/proxy-addr/LICENSE +0 -22
  262. package/example/node/node_modules/proxy-addr/README.md +0 -139
  263. package/example/node/node_modules/proxy-addr/index.js +0 -327
  264. package/example/node/node_modules/proxy-addr/package.json +0 -47
  265. package/example/node/node_modules/qs/.editorconfig +0 -39
  266. package/example/node/node_modules/qs/.eslintignore +0 -2
  267. package/example/node/node_modules/qs/.eslintrc +0 -35
  268. package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
  269. package/example/node/node_modules/qs/.nycrc +0 -13
  270. package/example/node/node_modules/qs/CHANGELOG.md +0 -359
  271. package/example/node/node_modules/qs/LICENSE.md +0 -29
  272. package/example/node/node_modules/qs/README.md +0 -604
  273. package/example/node/node_modules/qs/dist/qs.js +0 -832
  274. package/example/node/node_modules/qs/lib/formats.js +0 -23
  275. package/example/node/node_modules/qs/lib/index.js +0 -11
  276. package/example/node/node_modules/qs/lib/parse.js +0 -257
  277. package/example/node/node_modules/qs/lib/stringify.js +0 -278
  278. package/example/node/node_modules/qs/lib/utils.js +0 -251
  279. package/example/node/node_modules/qs/package.json +0 -69
  280. package/example/node/node_modules/qs/test/parse.js +0 -772
  281. package/example/node/node_modules/qs/test/stringify.js +0 -793
  282. package/example/node/node_modules/qs/test/utils.js +0 -136
  283. package/example/node/node_modules/range-parser/HISTORY.md +0 -56
  284. package/example/node/node_modules/range-parser/LICENSE +0 -23
  285. package/example/node/node_modules/range-parser/README.md +0 -84
  286. package/example/node/node_modules/range-parser/index.js +0 -162
  287. package/example/node/node_modules/range-parser/package.json +0 -44
  288. package/example/node/node_modules/raw-body/HISTORY.md +0 -284
  289. package/example/node/node_modules/raw-body/LICENSE +0 -22
  290. package/example/node/node_modules/raw-body/README.md +0 -217
  291. package/example/node/node_modules/raw-body/index.d.ts +0 -87
  292. package/example/node/node_modules/raw-body/index.js +0 -286
  293. package/example/node/node_modules/raw-body/package.json +0 -48
  294. package/example/node/node_modules/safe-buffer/LICENSE +0 -21
  295. package/example/node/node_modules/safe-buffer/README.md +0 -584
  296. package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
  297. package/example/node/node_modules/safe-buffer/index.js +0 -65
  298. package/example/node/node_modules/safe-buffer/package.json +0 -51
  299. package/example/node/node_modules/safer-buffer/LICENSE +0 -21
  300. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
  301. package/example/node/node_modules/safer-buffer/Readme.md +0 -156
  302. package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
  303. package/example/node/node_modules/safer-buffer/package.json +0 -34
  304. package/example/node/node_modules/safer-buffer/safer.js +0 -77
  305. package/example/node/node_modules/safer-buffer/tests.js +0 -406
  306. package/example/node/node_modules/send/HISTORY.md +0 -506
  307. package/example/node/node_modules/send/LICENSE +0 -23
  308. package/example/node/node_modules/send/README.md +0 -327
  309. package/example/node/node_modules/send/index.js +0 -1133
  310. package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
  311. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
  312. package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
  313. package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
  314. package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
  315. package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
  316. package/example/node/node_modules/send/package.json +0 -61
  317. package/example/node/node_modules/serve-static/HISTORY.md +0 -459
  318. package/example/node/node_modules/serve-static/LICENSE +0 -25
  319. package/example/node/node_modules/serve-static/README.md +0 -257
  320. package/example/node/node_modules/serve-static/index.js +0 -210
  321. package/example/node/node_modules/serve-static/package.json +0 -42
  322. package/example/node/node_modules/setprototypeof/LICENSE +0 -13
  323. package/example/node/node_modules/setprototypeof/README.md +0 -31
  324. package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
  325. package/example/node/node_modules/setprototypeof/index.js +0 -17
  326. package/example/node/node_modules/setprototypeof/package.json +0 -38
  327. package/example/node/node_modules/setprototypeof/test/index.js +0 -24
  328. package/example/node/node_modules/statuses/HISTORY.md +0 -65
  329. package/example/node/node_modules/statuses/LICENSE +0 -23
  330. package/example/node/node_modules/statuses/README.md +0 -127
  331. package/example/node/node_modules/statuses/codes.json +0 -66
  332. package/example/node/node_modules/statuses/index.js +0 -113
  333. package/example/node/node_modules/statuses/package.json +0 -48
  334. package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
  335. package/example/node/node_modules/toidentifier/LICENSE +0 -21
  336. package/example/node/node_modules/toidentifier/README.md +0 -61
  337. package/example/node/node_modules/toidentifier/index.js +0 -32
  338. package/example/node/node_modules/toidentifier/package.json +0 -38
  339. package/example/node/node_modules/type-is/HISTORY.md +0 -259
  340. package/example/node/node_modules/type-is/LICENSE +0 -23
  341. package/example/node/node_modules/type-is/README.md +0 -170
  342. package/example/node/node_modules/type-is/index.js +0 -266
  343. package/example/node/node_modules/type-is/package.json +0 -45
  344. package/example/node/node_modules/unpipe/HISTORY.md +0 -4
  345. package/example/node/node_modules/unpipe/LICENSE +0 -22
  346. package/example/node/node_modules/unpipe/README.md +0 -43
  347. package/example/node/node_modules/unpipe/index.js +0 -69
  348. package/example/node/node_modules/unpipe/package.json +0 -27
  349. package/example/node/node_modules/utils-merge/LICENSE +0 -20
  350. package/example/node/node_modules/utils-merge/README.md +0 -34
  351. package/example/node/node_modules/utils-merge/index.js +0 -23
  352. package/example/node/node_modules/utils-merge/package.json +0 -40
  353. package/example/node/node_modules/vary/HISTORY.md +0 -39
  354. package/example/node/node_modules/vary/LICENSE +0 -22
  355. package/example/node/node_modules/vary/README.md +0 -101
  356. package/example/node/node_modules/vary/index.js +0 -149
  357. package/example/node/node_modules/vary/package.json +0 -43
  358. package/test/lib/crawler.test.js +0 -94
@@ -1,181 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- * @private
12
- */
13
-
14
- var createError = require('http-errors')
15
- var getBody = require('raw-body')
16
- var iconv = require('iconv-lite')
17
- var onFinished = require('on-finished')
18
- var zlib = require('zlib')
19
-
20
- /**
21
- * Module exports.
22
- */
23
-
24
- module.exports = read
25
-
26
- /**
27
- * Read a request into a buffer and parse.
28
- *
29
- * @param {object} req
30
- * @param {object} res
31
- * @param {function} next
32
- * @param {function} parse
33
- * @param {function} debug
34
- * @param {object} options
35
- * @private
36
- */
37
-
38
- function read (req, res, next, parse, debug, options) {
39
- var length
40
- var opts = options
41
- var stream
42
-
43
- // flag as parsed
44
- req._body = true
45
-
46
- // read options
47
- var encoding = opts.encoding !== null
48
- ? opts.encoding
49
- : null
50
- var verify = opts.verify
51
-
52
- try {
53
- // get the content stream
54
- stream = contentstream(req, debug, opts.inflate)
55
- length = stream.length
56
- stream.length = undefined
57
- } catch (err) {
58
- return next(err)
59
- }
60
-
61
- // set raw-body options
62
- opts.length = length
63
- opts.encoding = verify
64
- ? null
65
- : encoding
66
-
67
- // assert charset is supported
68
- if (opts.encoding === null && encoding !== null && !iconv.encodingExists(encoding)) {
69
- return next(createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
70
- charset: encoding.toLowerCase(),
71
- type: 'charset.unsupported'
72
- }))
73
- }
74
-
75
- // read body
76
- debug('read body')
77
- getBody(stream, opts, function (error, body) {
78
- if (error) {
79
- var _error
80
-
81
- if (error.type === 'encoding.unsupported') {
82
- // echo back charset
83
- _error = createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
84
- charset: encoding.toLowerCase(),
85
- type: 'charset.unsupported'
86
- })
87
- } else {
88
- // set status code on error
89
- _error = createError(400, error)
90
- }
91
-
92
- // read off entire request
93
- stream.resume()
94
- onFinished(req, function onfinished () {
95
- next(createError(400, _error))
96
- })
97
- return
98
- }
99
-
100
- // verify
101
- if (verify) {
102
- try {
103
- debug('verify body')
104
- verify(req, res, body, encoding)
105
- } catch (err) {
106
- next(createError(403, err, {
107
- body: body,
108
- type: err.type || 'entity.verify.failed'
109
- }))
110
- return
111
- }
112
- }
113
-
114
- // parse
115
- var str = body
116
- try {
117
- debug('parse body')
118
- str = typeof body !== 'string' && encoding !== null
119
- ? iconv.decode(body, encoding)
120
- : body
121
- req.body = parse(str)
122
- } catch (err) {
123
- next(createError(400, err, {
124
- body: str,
125
- type: err.type || 'entity.parse.failed'
126
- }))
127
- return
128
- }
129
-
130
- next()
131
- })
132
- }
133
-
134
- /**
135
- * Get the content stream of the request.
136
- *
137
- * @param {object} req
138
- * @param {function} debug
139
- * @param {boolean} [inflate=true]
140
- * @return {object}
141
- * @api private
142
- */
143
-
144
- function contentstream (req, debug, inflate) {
145
- var encoding = (req.headers['content-encoding'] || 'identity').toLowerCase()
146
- var length = req.headers['content-length']
147
- var stream
148
-
149
- debug('content-encoding "%s"', encoding)
150
-
151
- if (inflate === false && encoding !== 'identity') {
152
- throw createError(415, 'content encoding unsupported', {
153
- encoding: encoding,
154
- type: 'encoding.unsupported'
155
- })
156
- }
157
-
158
- switch (encoding) {
159
- case 'deflate':
160
- stream = zlib.createInflate()
161
- debug('inflate body')
162
- req.pipe(stream)
163
- break
164
- case 'gzip':
165
- stream = zlib.createGunzip()
166
- debug('gunzip body')
167
- req.pipe(stream)
168
- break
169
- case 'identity':
170
- stream = req
171
- stream.length = length
172
- break
173
- default:
174
- throw createError(415, 'unsupported content encoding "' + encoding + '"', {
175
- encoding: encoding,
176
- type: 'encoding.unsupported'
177
- })
178
- }
179
-
180
- return stream
181
- }
@@ -1,230 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014 Jonathan Ong
4
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
5
- * MIT Licensed
6
- */
7
-
8
- 'use strict'
9
-
10
- /**
11
- * Module dependencies.
12
- * @private
13
- */
14
-
15
- var bytes = require('bytes')
16
- var contentType = require('content-type')
17
- var createError = require('http-errors')
18
- var debug = require('debug')('body-parser:json')
19
- var read = require('../read')
20
- var typeis = require('type-is')
21
-
22
- /**
23
- * Module exports.
24
- */
25
-
26
- module.exports = json
27
-
28
- /**
29
- * RegExp to match the first non-space in a string.
30
- *
31
- * Allowed whitespace is defined in RFC 7159:
32
- *
33
- * ws = *(
34
- * %x20 / ; Space
35
- * %x09 / ; Horizontal tab
36
- * %x0A / ; Line feed or New line
37
- * %x0D ) ; Carriage return
38
- */
39
-
40
- var FIRST_CHAR_REGEXP = /^[\x20\x09\x0a\x0d]*(.)/ // eslint-disable-line no-control-regex
41
-
42
- /**
43
- * Create a middleware to parse JSON bodies.
44
- *
45
- * @param {object} [options]
46
- * @return {function}
47
- * @public
48
- */
49
-
50
- function json (options) {
51
- var opts = options || {}
52
-
53
- var limit = typeof opts.limit !== 'number'
54
- ? bytes.parse(opts.limit || '100kb')
55
- : opts.limit
56
- var inflate = opts.inflate !== false
57
- var reviver = opts.reviver
58
- var strict = opts.strict !== false
59
- var type = opts.type || 'application/json'
60
- var verify = opts.verify || false
61
-
62
- if (verify !== false && typeof verify !== 'function') {
63
- throw new TypeError('option verify must be function')
64
- }
65
-
66
- // create the appropriate type checking function
67
- var shouldParse = typeof type !== 'function'
68
- ? typeChecker(type)
69
- : type
70
-
71
- function parse (body) {
72
- if (body.length === 0) {
73
- // special-case empty json body, as it's a common client-side mistake
74
- // TODO: maybe make this configurable or part of "strict" option
75
- return {}
76
- }
77
-
78
- if (strict) {
79
- var first = firstchar(body)
80
-
81
- if (first !== '{' && first !== '[') {
82
- debug('strict violation')
83
- throw createStrictSyntaxError(body, first)
84
- }
85
- }
86
-
87
- try {
88
- debug('parse json')
89
- return JSON.parse(body, reviver)
90
- } catch (e) {
91
- throw normalizeJsonSyntaxError(e, {
92
- message: e.message,
93
- stack: e.stack
94
- })
95
- }
96
- }
97
-
98
- return function jsonParser (req, res, next) {
99
- if (req._body) {
100
- debug('body already parsed')
101
- next()
102
- return
103
- }
104
-
105
- req.body = req.body || {}
106
-
107
- // skip requests without bodies
108
- if (!typeis.hasBody(req)) {
109
- debug('skip empty body')
110
- next()
111
- return
112
- }
113
-
114
- debug('content-type %j', req.headers['content-type'])
115
-
116
- // determine if request should be parsed
117
- if (!shouldParse(req)) {
118
- debug('skip parsing')
119
- next()
120
- return
121
- }
122
-
123
- // assert charset per RFC 7159 sec 8.1
124
- var charset = getCharset(req) || 'utf-8'
125
- if (charset.substr(0, 4) !== 'utf-') {
126
- debug('invalid charset')
127
- next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
128
- charset: charset,
129
- type: 'charset.unsupported'
130
- }))
131
- return
132
- }
133
-
134
- // read
135
- read(req, res, next, parse, debug, {
136
- encoding: charset,
137
- inflate: inflate,
138
- limit: limit,
139
- verify: verify
140
- })
141
- }
142
- }
143
-
144
- /**
145
- * Create strict violation syntax error matching native error.
146
- *
147
- * @param {string} str
148
- * @param {string} char
149
- * @return {Error}
150
- * @private
151
- */
152
-
153
- function createStrictSyntaxError (str, char) {
154
- var index = str.indexOf(char)
155
- var partial = str.substring(0, index) + '#'
156
-
157
- try {
158
- JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation')
159
- } catch (e) {
160
- return normalizeJsonSyntaxError(e, {
161
- message: e.message.replace('#', char),
162
- stack: e.stack
163
- })
164
- }
165
- }
166
-
167
- /**
168
- * Get the first non-whitespace character in a string.
169
- *
170
- * @param {string} str
171
- * @return {function}
172
- * @private
173
- */
174
-
175
- function firstchar (str) {
176
- return FIRST_CHAR_REGEXP.exec(str)[1]
177
- }
178
-
179
- /**
180
- * Get the charset of a request.
181
- *
182
- * @param {object} req
183
- * @api private
184
- */
185
-
186
- function getCharset (req) {
187
- try {
188
- return (contentType.parse(req).parameters.charset || '').toLowerCase()
189
- } catch (e) {
190
- return undefined
191
- }
192
- }
193
-
194
- /**
195
- * Normalize a SyntaxError for JSON.parse.
196
- *
197
- * @param {SyntaxError} error
198
- * @param {object} obj
199
- * @return {SyntaxError}
200
- */
201
-
202
- function normalizeJsonSyntaxError (error, obj) {
203
- var keys = Object.getOwnPropertyNames(error)
204
-
205
- for (var i = 0; i < keys.length; i++) {
206
- var key = keys[i]
207
- if (key !== 'stack' && key !== 'message') {
208
- delete error[key]
209
- }
210
- }
211
-
212
- // replace stack before message for Node.js 0.10 and below
213
- error.stack = obj.stack.replace(error.message, obj.message)
214
- error.message = obj.message
215
-
216
- return error
217
- }
218
-
219
- /**
220
- * Get the simple type checker.
221
- *
222
- * @param {string} type
223
- * @return {function}
224
- */
225
-
226
- function typeChecker (type) {
227
- return function checkType (req) {
228
- return Boolean(typeis(req, type))
229
- }
230
- }
@@ -1,101 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- */
12
-
13
- var bytes = require('bytes')
14
- var debug = require('debug')('body-parser:raw')
15
- var read = require('../read')
16
- var typeis = require('type-is')
17
-
18
- /**
19
- * Module exports.
20
- */
21
-
22
- module.exports = raw
23
-
24
- /**
25
- * Create a middleware to parse raw bodies.
26
- *
27
- * @param {object} [options]
28
- * @return {function}
29
- * @api public
30
- */
31
-
32
- function raw (options) {
33
- var opts = options || {}
34
-
35
- var inflate = opts.inflate !== false
36
- var limit = typeof opts.limit !== 'number'
37
- ? bytes.parse(opts.limit || '100kb')
38
- : opts.limit
39
- var type = opts.type || 'application/octet-stream'
40
- var verify = opts.verify || false
41
-
42
- if (verify !== false && typeof verify !== 'function') {
43
- throw new TypeError('option verify must be function')
44
- }
45
-
46
- // create the appropriate type checking function
47
- var shouldParse = typeof type !== 'function'
48
- ? typeChecker(type)
49
- : type
50
-
51
- function parse (buf) {
52
- return buf
53
- }
54
-
55
- return function rawParser (req, res, next) {
56
- if (req._body) {
57
- debug('body already parsed')
58
- next()
59
- return
60
- }
61
-
62
- req.body = req.body || {}
63
-
64
- // skip requests without bodies
65
- if (!typeis.hasBody(req)) {
66
- debug('skip empty body')
67
- next()
68
- return
69
- }
70
-
71
- debug('content-type %j', req.headers['content-type'])
72
-
73
- // determine if request should be parsed
74
- if (!shouldParse(req)) {
75
- debug('skip parsing')
76
- next()
77
- return
78
- }
79
-
80
- // read
81
- read(req, res, next, parse, debug, {
82
- encoding: null,
83
- inflate: inflate,
84
- limit: limit,
85
- verify: verify
86
- })
87
- }
88
- }
89
-
90
- /**
91
- * Get the simple type checker.
92
- *
93
- * @param {string} type
94
- * @return {function}
95
- */
96
-
97
- function typeChecker (type) {
98
- return function checkType (req) {
99
- return Boolean(typeis(req, type))
100
- }
101
- }
@@ -1,121 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- */
12
-
13
- var bytes = require('bytes')
14
- var contentType = require('content-type')
15
- var debug = require('debug')('body-parser:text')
16
- var read = require('../read')
17
- var typeis = require('type-is')
18
-
19
- /**
20
- * Module exports.
21
- */
22
-
23
- module.exports = text
24
-
25
- /**
26
- * Create a middleware to parse text bodies.
27
- *
28
- * @param {object} [options]
29
- * @return {function}
30
- * @api public
31
- */
32
-
33
- function text (options) {
34
- var opts = options || {}
35
-
36
- var defaultCharset = opts.defaultCharset || 'utf-8'
37
- var inflate = opts.inflate !== false
38
- var limit = typeof opts.limit !== 'number'
39
- ? bytes.parse(opts.limit || '100kb')
40
- : opts.limit
41
- var type = opts.type || 'text/plain'
42
- var verify = opts.verify || false
43
-
44
- if (verify !== false && typeof verify !== 'function') {
45
- throw new TypeError('option verify must be function')
46
- }
47
-
48
- // create the appropriate type checking function
49
- var shouldParse = typeof type !== 'function'
50
- ? typeChecker(type)
51
- : type
52
-
53
- function parse (buf) {
54
- return buf
55
- }
56
-
57
- return function textParser (req, res, next) {
58
- if (req._body) {
59
- debug('body already parsed')
60
- next()
61
- return
62
- }
63
-
64
- req.body = req.body || {}
65
-
66
- // skip requests without bodies
67
- if (!typeis.hasBody(req)) {
68
- debug('skip empty body')
69
- next()
70
- return
71
- }
72
-
73
- debug('content-type %j', req.headers['content-type'])
74
-
75
- // determine if request should be parsed
76
- if (!shouldParse(req)) {
77
- debug('skip parsing')
78
- next()
79
- return
80
- }
81
-
82
- // get charset
83
- var charset = getCharset(req) || defaultCharset
84
-
85
- // read
86
- read(req, res, next, parse, debug, {
87
- encoding: charset,
88
- inflate: inflate,
89
- limit: limit,
90
- verify: verify
91
- })
92
- }
93
- }
94
-
95
- /**
96
- * Get the charset of a request.
97
- *
98
- * @param {object} req
99
- * @api private
100
- */
101
-
102
- function getCharset (req) {
103
- try {
104
- return (contentType.parse(req).parameters.charset || '').toLowerCase()
105
- } catch (e) {
106
- return undefined
107
- }
108
- }
109
-
110
- /**
111
- * Get the simple type checker.
112
- *
113
- * @param {string} type
114
- * @return {function}
115
- */
116
-
117
- function typeChecker (type) {
118
- return function checkType (req) {
119
- return Boolean(typeis(req, type))
120
- }
121
- }