es6-crawler-detect 3.2.0 → 4.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (361) hide show
  1. package/.babelrc +1 -1
  2. package/.mocharc.json +5 -0
  3. package/README.md +1 -0
  4. package/dist/main.bundle.js +1 -1
  5. package/eslint.config.mjs +62 -0
  6. package/example/node/package.json +1 -1
  7. package/example/node/server.js +38 -32
  8. package/example/node/yarn.lock +222 -99
  9. package/package.json +30 -25
  10. package/src/index.js +20 -14
  11. package/src/index.ts +18 -0
  12. package/src/lib/crawler/crawlers.js +9 -12
  13. package/src/lib/crawler/crawlers.ts +1397 -0
  14. package/src/lib/crawler/exclusions.js +7 -11
  15. package/src/lib/crawler/exclusions.ts +56 -0
  16. package/src/lib/crawler/headers.js +6 -10
  17. package/src/lib/crawler/headers.ts +18 -0
  18. package/src/lib/crawler/provider.js +1 -10
  19. package/src/lib/crawler/provider.ts +3 -0
  20. package/src/lib/crawler.js +61 -98
  21. package/src/lib/crawler.ts +128 -0
  22. package/src/types.js +2 -0
  23. package/src/types.ts +15 -0
  24. package/test/lib/crawler.spec.js +149 -0
  25. package/test/lib/crawler.spec.ts +116 -0
  26. package/test/lib/database/crawlers.txt +3651 -0
  27. package/test/lib/database/devices.txt +165636 -0
  28. package/tsconfig.json +16 -0
  29. package/webpack.common.js +20 -20
  30. package/webpack.dev.js +7 -7
  31. package/webpack.prod.js +14 -14
  32. package/.eslintignore +0 -6
  33. package/.eslintrc.json +0 -30
  34. package/example/node/node_modules/.bin/mime +0 -15
  35. package/example/node/node_modules/.bin/mime.cmd +0 -7
  36. package/example/node/node_modules/.yarn-integrity +0 -67
  37. package/example/node/node_modules/accepts/HISTORY.md +0 -236
  38. package/example/node/node_modules/accepts/LICENSE +0 -23
  39. package/example/node/node_modules/accepts/README.md +0 -142
  40. package/example/node/node_modules/accepts/index.js +0 -238
  41. package/example/node/node_modules/accepts/package.json +0 -47
  42. package/example/node/node_modules/array-flatten/LICENSE +0 -21
  43. package/example/node/node_modules/array-flatten/README.md +0 -43
  44. package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
  45. package/example/node/node_modules/array-flatten/package.json +0 -39
  46. package/example/node/node_modules/body-parser/HISTORY.md +0 -624
  47. package/example/node/node_modules/body-parser/LICENSE +0 -23
  48. package/example/node/node_modules/body-parser/README.md +0 -456
  49. package/example/node/node_modules/body-parser/index.js +0 -157
  50. package/example/node/node_modules/body-parser/lib/read.js +0 -181
  51. package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
  52. package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
  53. package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
  54. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
  55. package/example/node/node_modules/body-parser/package.json +0 -52
  56. package/example/node/node_modules/bytes/History.md +0 -92
  57. package/example/node/node_modules/bytes/LICENSE +0 -23
  58. package/example/node/node_modules/bytes/Readme.md +0 -152
  59. package/example/node/node_modules/bytes/index.js +0 -166
  60. package/example/node/node_modules/bytes/package.json +0 -42
  61. package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
  62. package/example/node/node_modules/content-disposition/LICENSE +0 -22
  63. package/example/node/node_modules/content-disposition/README.md +0 -142
  64. package/example/node/node_modules/content-disposition/index.js +0 -458
  65. package/example/node/node_modules/content-disposition/package.json +0 -44
  66. package/example/node/node_modules/content-type/HISTORY.md +0 -24
  67. package/example/node/node_modules/content-type/LICENSE +0 -22
  68. package/example/node/node_modules/content-type/README.md +0 -92
  69. package/example/node/node_modules/content-type/index.js +0 -222
  70. package/example/node/node_modules/content-type/package.json +0 -40
  71. package/example/node/node_modules/cookie/HISTORY.md +0 -128
  72. package/example/node/node_modules/cookie/LICENSE +0 -24
  73. package/example/node/node_modules/cookie/README.md +0 -257
  74. package/example/node/node_modules/cookie/index.js +0 -202
  75. package/example/node/node_modules/cookie/package.json +0 -40
  76. package/example/node/node_modules/cookie-signature/History.md +0 -38
  77. package/example/node/node_modules/cookie-signature/Readme.md +0 -42
  78. package/example/node/node_modules/cookie-signature/index.js +0 -51
  79. package/example/node/node_modules/cookie-signature/package.json +0 -18
  80. package/example/node/node_modules/debug/.coveralls.yml +0 -1
  81. package/example/node/node_modules/debug/.eslintrc +0 -11
  82. package/example/node/node_modules/debug/.travis.yml +0 -14
  83. package/example/node/node_modules/debug/CHANGELOG.md +0 -362
  84. package/example/node/node_modules/debug/LICENSE +0 -19
  85. package/example/node/node_modules/debug/Makefile +0 -50
  86. package/example/node/node_modules/debug/README.md +0 -312
  87. package/example/node/node_modules/debug/component.json +0 -19
  88. package/example/node/node_modules/debug/karma.conf.js +0 -70
  89. package/example/node/node_modules/debug/node.js +0 -1
  90. package/example/node/node_modules/debug/package.json +0 -49
  91. package/example/node/node_modules/debug/src/browser.js +0 -185
  92. package/example/node/node_modules/debug/src/debug.js +0 -202
  93. package/example/node/node_modules/debug/src/index.js +0 -10
  94. package/example/node/node_modules/debug/src/inspector-log.js +0 -15
  95. package/example/node/node_modules/debug/src/node.js +0 -248
  96. package/example/node/node_modules/depd/History.md +0 -96
  97. package/example/node/node_modules/depd/LICENSE +0 -22
  98. package/example/node/node_modules/depd/Readme.md +0 -280
  99. package/example/node/node_modules/depd/index.js +0 -522
  100. package/example/node/node_modules/depd/lib/browser/index.js +0 -77
  101. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
  102. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
  103. package/example/node/node_modules/depd/lib/compat/index.js +0 -79
  104. package/example/node/node_modules/depd/package.json +0 -41
  105. package/example/node/node_modules/destroy/LICENSE +0 -22
  106. package/example/node/node_modules/destroy/README.md +0 -60
  107. package/example/node/node_modules/destroy/index.js +0 -75
  108. package/example/node/node_modules/destroy/package.json +0 -37
  109. package/example/node/node_modules/ee-first/LICENSE +0 -22
  110. package/example/node/node_modules/ee-first/README.md +0 -80
  111. package/example/node/node_modules/ee-first/index.js +0 -95
  112. package/example/node/node_modules/ee-first/package.json +0 -29
  113. package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
  114. package/example/node/node_modules/encodeurl/LICENSE +0 -22
  115. package/example/node/node_modules/encodeurl/README.md +0 -128
  116. package/example/node/node_modules/encodeurl/index.js +0 -60
  117. package/example/node/node_modules/encodeurl/package.json +0 -40
  118. package/example/node/node_modules/escape-html/LICENSE +0 -24
  119. package/example/node/node_modules/escape-html/Readme.md +0 -43
  120. package/example/node/node_modules/escape-html/index.js +0 -78
  121. package/example/node/node_modules/escape-html/package.json +0 -24
  122. package/example/node/node_modules/etag/HISTORY.md +0 -83
  123. package/example/node/node_modules/etag/LICENSE +0 -22
  124. package/example/node/node_modules/etag/README.md +0 -159
  125. package/example/node/node_modules/etag/index.js +0 -131
  126. package/example/node/node_modules/etag/package.json +0 -47
  127. package/example/node/node_modules/express/History.md +0 -3510
  128. package/example/node/node_modules/express/LICENSE +0 -24
  129. package/example/node/node_modules/express/Readme.md +0 -158
  130. package/example/node/node_modules/express/index.js +0 -11
  131. package/example/node/node_modules/express/lib/application.js +0 -644
  132. package/example/node/node_modules/express/lib/express.js +0 -116
  133. package/example/node/node_modules/express/lib/middleware/init.js +0 -43
  134. package/example/node/node_modules/express/lib/middleware/query.js +0 -47
  135. package/example/node/node_modules/express/lib/request.js +0 -525
  136. package/example/node/node_modules/express/lib/response.js +0 -1147
  137. package/example/node/node_modules/express/lib/router/index.js +0 -668
  138. package/example/node/node_modules/express/lib/router/layer.js +0 -181
  139. package/example/node/node_modules/express/lib/router/route.js +0 -216
  140. package/example/node/node_modules/express/lib/utils.js +0 -302
  141. package/example/node/node_modules/express/lib/view.js +0 -182
  142. package/example/node/node_modules/express/package.json +0 -99
  143. package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
  144. package/example/node/node_modules/finalhandler/LICENSE +0 -22
  145. package/example/node/node_modules/finalhandler/README.md +0 -148
  146. package/example/node/node_modules/finalhandler/index.js +0 -331
  147. package/example/node/node_modules/finalhandler/package.json +0 -45
  148. package/example/node/node_modules/forwarded/HISTORY.md +0 -21
  149. package/example/node/node_modules/forwarded/LICENSE +0 -22
  150. package/example/node/node_modules/forwarded/README.md +0 -57
  151. package/example/node/node_modules/forwarded/index.js +0 -90
  152. package/example/node/node_modules/forwarded/package.json +0 -45
  153. package/example/node/node_modules/fresh/HISTORY.md +0 -70
  154. package/example/node/node_modules/fresh/LICENSE +0 -23
  155. package/example/node/node_modules/fresh/README.md +0 -119
  156. package/example/node/node_modules/fresh/index.js +0 -137
  157. package/example/node/node_modules/fresh/package.json +0 -46
  158. package/example/node/node_modules/http-errors/HISTORY.md +0 -165
  159. package/example/node/node_modules/http-errors/LICENSE +0 -23
  160. package/example/node/node_modules/http-errors/README.md +0 -169
  161. package/example/node/node_modules/http-errors/index.js +0 -299
  162. package/example/node/node_modules/http-errors/package.json +0 -49
  163. package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
  164. package/example/node/node_modules/iconv-lite/LICENSE +0 -21
  165. package/example/node/node_modules/iconv-lite/README.md +0 -156
  166. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
  167. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
  168. package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
  169. package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
  170. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
  171. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
  172. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
  173. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
  174. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
  175. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
  176. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
  177. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
  178. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
  179. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
  180. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
  181. package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
  182. package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
  183. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
  184. package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
  185. package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
  186. package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
  187. package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
  188. package/example/node/node_modules/iconv-lite/package.json +0 -46
  189. package/example/node/node_modules/inherits/LICENSE +0 -16
  190. package/example/node/node_modules/inherits/README.md +0 -42
  191. package/example/node/node_modules/inherits/inherits.js +0 -9
  192. package/example/node/node_modules/inherits/inherits_browser.js +0 -27
  193. package/example/node/node_modules/inherits/package.json +0 -29
  194. package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
  195. package/example/node/node_modules/ipaddr.js/README.md +0 -233
  196. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
  197. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
  198. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
  199. package/example/node/node_modules/ipaddr.js/package.json +0 -35
  200. package/example/node/node_modules/media-typer/HISTORY.md +0 -22
  201. package/example/node/node_modules/media-typer/LICENSE +0 -22
  202. package/example/node/node_modules/media-typer/README.md +0 -81
  203. package/example/node/node_modules/media-typer/index.js +0 -270
  204. package/example/node/node_modules/media-typer/package.json +0 -26
  205. package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
  206. package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
  207. package/example/node/node_modules/merge-descriptors/README.md +0 -48
  208. package/example/node/node_modules/merge-descriptors/index.js +0 -60
  209. package/example/node/node_modules/merge-descriptors/package.json +0 -32
  210. package/example/node/node_modules/methods/HISTORY.md +0 -29
  211. package/example/node/node_modules/methods/LICENSE +0 -24
  212. package/example/node/node_modules/methods/README.md +0 -51
  213. package/example/node/node_modules/methods/index.js +0 -69
  214. package/example/node/node_modules/methods/package.json +0 -36
  215. package/example/node/node_modules/mime/CHANGELOG.md +0 -164
  216. package/example/node/node_modules/mime/LICENSE +0 -21
  217. package/example/node/node_modules/mime/README.md +0 -90
  218. package/example/node/node_modules/mime/cli.js +0 -8
  219. package/example/node/node_modules/mime/mime.js +0 -108
  220. package/example/node/node_modules/mime/package.json +0 -44
  221. package/example/node/node_modules/mime/src/build.js +0 -53
  222. package/example/node/node_modules/mime/src/test.js +0 -60
  223. package/example/node/node_modules/mime/types.json +0 -1
  224. package/example/node/node_modules/mime-db/HISTORY.md +0 -499
  225. package/example/node/node_modules/mime-db/LICENSE +0 -22
  226. package/example/node/node_modules/mime-db/README.md +0 -100
  227. package/example/node/node_modules/mime-db/db.json +0 -8471
  228. package/example/node/node_modules/mime-db/index.js +0 -11
  229. package/example/node/node_modules/mime-db/package.json +0 -59
  230. package/example/node/node_modules/mime-types/HISTORY.md +0 -388
  231. package/example/node/node_modules/mime-types/LICENSE +0 -23
  232. package/example/node/node_modules/mime-types/README.md +0 -113
  233. package/example/node/node_modules/mime-types/index.js +0 -188
  234. package/example/node/node_modules/mime-types/package.json +0 -44
  235. package/example/node/node_modules/ms/index.js +0 -152
  236. package/example/node/node_modules/ms/license.md +0 -21
  237. package/example/node/node_modules/ms/package.json +0 -37
  238. package/example/node/node_modules/ms/readme.md +0 -51
  239. package/example/node/node_modules/negotiator/HISTORY.md +0 -103
  240. package/example/node/node_modules/negotiator/LICENSE +0 -24
  241. package/example/node/node_modules/negotiator/README.md +0 -203
  242. package/example/node/node_modules/negotiator/index.js +0 -124
  243. package/example/node/node_modules/negotiator/lib/charset.js +0 -169
  244. package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
  245. package/example/node/node_modules/negotiator/lib/language.js +0 -179
  246. package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
  247. package/example/node/node_modules/negotiator/package.json +0 -42
  248. package/example/node/node_modules/on-finished/HISTORY.md +0 -88
  249. package/example/node/node_modules/on-finished/LICENSE +0 -23
  250. package/example/node/node_modules/on-finished/README.md +0 -154
  251. package/example/node/node_modules/on-finished/index.js +0 -196
  252. package/example/node/node_modules/on-finished/package.json +0 -31
  253. package/example/node/node_modules/parseurl/HISTORY.md +0 -58
  254. package/example/node/node_modules/parseurl/LICENSE +0 -24
  255. package/example/node/node_modules/parseurl/README.md +0 -133
  256. package/example/node/node_modules/parseurl/index.js +0 -158
  257. package/example/node/node_modules/parseurl/package.json +0 -40
  258. package/example/node/node_modules/path-to-regexp/History.md +0 -36
  259. package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
  260. package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
  261. package/example/node/node_modules/path-to-regexp/index.js +0 -129
  262. package/example/node/node_modules/path-to-regexp/package.json +0 -30
  263. package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
  264. package/example/node/node_modules/proxy-addr/LICENSE +0 -22
  265. package/example/node/node_modules/proxy-addr/README.md +0 -139
  266. package/example/node/node_modules/proxy-addr/index.js +0 -327
  267. package/example/node/node_modules/proxy-addr/package.json +0 -47
  268. package/example/node/node_modules/qs/.editorconfig +0 -39
  269. package/example/node/node_modules/qs/.eslintignore +0 -2
  270. package/example/node/node_modules/qs/.eslintrc +0 -35
  271. package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
  272. package/example/node/node_modules/qs/.nycrc +0 -13
  273. package/example/node/node_modules/qs/CHANGELOG.md +0 -359
  274. package/example/node/node_modules/qs/LICENSE.md +0 -29
  275. package/example/node/node_modules/qs/README.md +0 -604
  276. package/example/node/node_modules/qs/dist/qs.js +0 -832
  277. package/example/node/node_modules/qs/lib/formats.js +0 -23
  278. package/example/node/node_modules/qs/lib/index.js +0 -11
  279. package/example/node/node_modules/qs/lib/parse.js +0 -257
  280. package/example/node/node_modules/qs/lib/stringify.js +0 -278
  281. package/example/node/node_modules/qs/lib/utils.js +0 -251
  282. package/example/node/node_modules/qs/package.json +0 -69
  283. package/example/node/node_modules/qs/test/parse.js +0 -772
  284. package/example/node/node_modules/qs/test/stringify.js +0 -793
  285. package/example/node/node_modules/qs/test/utils.js +0 -136
  286. package/example/node/node_modules/range-parser/HISTORY.md +0 -56
  287. package/example/node/node_modules/range-parser/LICENSE +0 -23
  288. package/example/node/node_modules/range-parser/README.md +0 -84
  289. package/example/node/node_modules/range-parser/index.js +0 -162
  290. package/example/node/node_modules/range-parser/package.json +0 -44
  291. package/example/node/node_modules/raw-body/HISTORY.md +0 -284
  292. package/example/node/node_modules/raw-body/LICENSE +0 -22
  293. package/example/node/node_modules/raw-body/README.md +0 -217
  294. package/example/node/node_modules/raw-body/index.d.ts +0 -87
  295. package/example/node/node_modules/raw-body/index.js +0 -286
  296. package/example/node/node_modules/raw-body/package.json +0 -48
  297. package/example/node/node_modules/safe-buffer/LICENSE +0 -21
  298. package/example/node/node_modules/safe-buffer/README.md +0 -584
  299. package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
  300. package/example/node/node_modules/safe-buffer/index.js +0 -65
  301. package/example/node/node_modules/safe-buffer/package.json +0 -51
  302. package/example/node/node_modules/safer-buffer/LICENSE +0 -21
  303. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
  304. package/example/node/node_modules/safer-buffer/Readme.md +0 -156
  305. package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
  306. package/example/node/node_modules/safer-buffer/package.json +0 -34
  307. package/example/node/node_modules/safer-buffer/safer.js +0 -77
  308. package/example/node/node_modules/safer-buffer/tests.js +0 -406
  309. package/example/node/node_modules/send/HISTORY.md +0 -506
  310. package/example/node/node_modules/send/LICENSE +0 -23
  311. package/example/node/node_modules/send/README.md +0 -327
  312. package/example/node/node_modules/send/index.js +0 -1133
  313. package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
  314. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
  315. package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
  316. package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
  317. package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
  318. package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
  319. package/example/node/node_modules/send/package.json +0 -61
  320. package/example/node/node_modules/serve-static/HISTORY.md +0 -459
  321. package/example/node/node_modules/serve-static/LICENSE +0 -25
  322. package/example/node/node_modules/serve-static/README.md +0 -257
  323. package/example/node/node_modules/serve-static/index.js +0 -210
  324. package/example/node/node_modules/serve-static/package.json +0 -42
  325. package/example/node/node_modules/setprototypeof/LICENSE +0 -13
  326. package/example/node/node_modules/setprototypeof/README.md +0 -31
  327. package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
  328. package/example/node/node_modules/setprototypeof/index.js +0 -17
  329. package/example/node/node_modules/setprototypeof/package.json +0 -38
  330. package/example/node/node_modules/setprototypeof/test/index.js +0 -24
  331. package/example/node/node_modules/statuses/HISTORY.md +0 -65
  332. package/example/node/node_modules/statuses/LICENSE +0 -23
  333. package/example/node/node_modules/statuses/README.md +0 -127
  334. package/example/node/node_modules/statuses/codes.json +0 -66
  335. package/example/node/node_modules/statuses/index.js +0 -113
  336. package/example/node/node_modules/statuses/package.json +0 -48
  337. package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
  338. package/example/node/node_modules/toidentifier/LICENSE +0 -21
  339. package/example/node/node_modules/toidentifier/README.md +0 -61
  340. package/example/node/node_modules/toidentifier/index.js +0 -32
  341. package/example/node/node_modules/toidentifier/package.json +0 -38
  342. package/example/node/node_modules/type-is/HISTORY.md +0 -259
  343. package/example/node/node_modules/type-is/LICENSE +0 -23
  344. package/example/node/node_modules/type-is/README.md +0 -170
  345. package/example/node/node_modules/type-is/index.js +0 -266
  346. package/example/node/node_modules/type-is/package.json +0 -45
  347. package/example/node/node_modules/unpipe/HISTORY.md +0 -4
  348. package/example/node/node_modules/unpipe/LICENSE +0 -22
  349. package/example/node/node_modules/unpipe/README.md +0 -43
  350. package/example/node/node_modules/unpipe/index.js +0 -69
  351. package/example/node/node_modules/unpipe/package.json +0 -27
  352. package/example/node/node_modules/utils-merge/LICENSE +0 -20
  353. package/example/node/node_modules/utils-merge/README.md +0 -34
  354. package/example/node/node_modules/utils-merge/index.js +0 -23
  355. package/example/node/node_modules/utils-merge/package.json +0 -40
  356. package/example/node/node_modules/vary/HISTORY.md +0 -39
  357. package/example/node/node_modules/vary/LICENSE +0 -22
  358. package/example/node/node_modules/vary/README.md +0 -101
  359. package/example/node/node_modules/vary/index.js +0 -149
  360. package/example/node/node_modules/vary/package.json +0 -43
  361. package/test/lib/crawler.test.js +0 -52
@@ -1,181 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- * @private
12
- */
13
-
14
- var createError = require('http-errors')
15
- var getBody = require('raw-body')
16
- var iconv = require('iconv-lite')
17
- var onFinished = require('on-finished')
18
- var zlib = require('zlib')
19
-
20
- /**
21
- * Module exports.
22
- */
23
-
24
- module.exports = read
25
-
26
- /**
27
- * Read a request into a buffer and parse.
28
- *
29
- * @param {object} req
30
- * @param {object} res
31
- * @param {function} next
32
- * @param {function} parse
33
- * @param {function} debug
34
- * @param {object} options
35
- * @private
36
- */
37
-
38
- function read (req, res, next, parse, debug, options) {
39
- var length
40
- var opts = options
41
- var stream
42
-
43
- // flag as parsed
44
- req._body = true
45
-
46
- // read options
47
- var encoding = opts.encoding !== null
48
- ? opts.encoding
49
- : null
50
- var verify = opts.verify
51
-
52
- try {
53
- // get the content stream
54
- stream = contentstream(req, debug, opts.inflate)
55
- length = stream.length
56
- stream.length = undefined
57
- } catch (err) {
58
- return next(err)
59
- }
60
-
61
- // set raw-body options
62
- opts.length = length
63
- opts.encoding = verify
64
- ? null
65
- : encoding
66
-
67
- // assert charset is supported
68
- if (opts.encoding === null && encoding !== null && !iconv.encodingExists(encoding)) {
69
- return next(createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
70
- charset: encoding.toLowerCase(),
71
- type: 'charset.unsupported'
72
- }))
73
- }
74
-
75
- // read body
76
- debug('read body')
77
- getBody(stream, opts, function (error, body) {
78
- if (error) {
79
- var _error
80
-
81
- if (error.type === 'encoding.unsupported') {
82
- // echo back charset
83
- _error = createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
84
- charset: encoding.toLowerCase(),
85
- type: 'charset.unsupported'
86
- })
87
- } else {
88
- // set status code on error
89
- _error = createError(400, error)
90
- }
91
-
92
- // read off entire request
93
- stream.resume()
94
- onFinished(req, function onfinished () {
95
- next(createError(400, _error))
96
- })
97
- return
98
- }
99
-
100
- // verify
101
- if (verify) {
102
- try {
103
- debug('verify body')
104
- verify(req, res, body, encoding)
105
- } catch (err) {
106
- next(createError(403, err, {
107
- body: body,
108
- type: err.type || 'entity.verify.failed'
109
- }))
110
- return
111
- }
112
- }
113
-
114
- // parse
115
- var str = body
116
- try {
117
- debug('parse body')
118
- str = typeof body !== 'string' && encoding !== null
119
- ? iconv.decode(body, encoding)
120
- : body
121
- req.body = parse(str)
122
- } catch (err) {
123
- next(createError(400, err, {
124
- body: str,
125
- type: err.type || 'entity.parse.failed'
126
- }))
127
- return
128
- }
129
-
130
- next()
131
- })
132
- }
133
-
134
- /**
135
- * Get the content stream of the request.
136
- *
137
- * @param {object} req
138
- * @param {function} debug
139
- * @param {boolean} [inflate=true]
140
- * @return {object}
141
- * @api private
142
- */
143
-
144
- function contentstream (req, debug, inflate) {
145
- var encoding = (req.headers['content-encoding'] || 'identity').toLowerCase()
146
- var length = req.headers['content-length']
147
- var stream
148
-
149
- debug('content-encoding "%s"', encoding)
150
-
151
- if (inflate === false && encoding !== 'identity') {
152
- throw createError(415, 'content encoding unsupported', {
153
- encoding: encoding,
154
- type: 'encoding.unsupported'
155
- })
156
- }
157
-
158
- switch (encoding) {
159
- case 'deflate':
160
- stream = zlib.createInflate()
161
- debug('inflate body')
162
- req.pipe(stream)
163
- break
164
- case 'gzip':
165
- stream = zlib.createGunzip()
166
- debug('gunzip body')
167
- req.pipe(stream)
168
- break
169
- case 'identity':
170
- stream = req
171
- stream.length = length
172
- break
173
- default:
174
- throw createError(415, 'unsupported content encoding "' + encoding + '"', {
175
- encoding: encoding,
176
- type: 'encoding.unsupported'
177
- })
178
- }
179
-
180
- return stream
181
- }
@@ -1,230 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014 Jonathan Ong
4
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
5
- * MIT Licensed
6
- */
7
-
8
- 'use strict'
9
-
10
- /**
11
- * Module dependencies.
12
- * @private
13
- */
14
-
15
- var bytes = require('bytes')
16
- var contentType = require('content-type')
17
- var createError = require('http-errors')
18
- var debug = require('debug')('body-parser:json')
19
- var read = require('../read')
20
- var typeis = require('type-is')
21
-
22
- /**
23
- * Module exports.
24
- */
25
-
26
- module.exports = json
27
-
28
- /**
29
- * RegExp to match the first non-space in a string.
30
- *
31
- * Allowed whitespace is defined in RFC 7159:
32
- *
33
- * ws = *(
34
- * %x20 / ; Space
35
- * %x09 / ; Horizontal tab
36
- * %x0A / ; Line feed or New line
37
- * %x0D ) ; Carriage return
38
- */
39
-
40
- var FIRST_CHAR_REGEXP = /^[\x20\x09\x0a\x0d]*(.)/ // eslint-disable-line no-control-regex
41
-
42
- /**
43
- * Create a middleware to parse JSON bodies.
44
- *
45
- * @param {object} [options]
46
- * @return {function}
47
- * @public
48
- */
49
-
50
- function json (options) {
51
- var opts = options || {}
52
-
53
- var limit = typeof opts.limit !== 'number'
54
- ? bytes.parse(opts.limit || '100kb')
55
- : opts.limit
56
- var inflate = opts.inflate !== false
57
- var reviver = opts.reviver
58
- var strict = opts.strict !== false
59
- var type = opts.type || 'application/json'
60
- var verify = opts.verify || false
61
-
62
- if (verify !== false && typeof verify !== 'function') {
63
- throw new TypeError('option verify must be function')
64
- }
65
-
66
- // create the appropriate type checking function
67
- var shouldParse = typeof type !== 'function'
68
- ? typeChecker(type)
69
- : type
70
-
71
- function parse (body) {
72
- if (body.length === 0) {
73
- // special-case empty json body, as it's a common client-side mistake
74
- // TODO: maybe make this configurable or part of "strict" option
75
- return {}
76
- }
77
-
78
- if (strict) {
79
- var first = firstchar(body)
80
-
81
- if (first !== '{' && first !== '[') {
82
- debug('strict violation')
83
- throw createStrictSyntaxError(body, first)
84
- }
85
- }
86
-
87
- try {
88
- debug('parse json')
89
- return JSON.parse(body, reviver)
90
- } catch (e) {
91
- throw normalizeJsonSyntaxError(e, {
92
- message: e.message,
93
- stack: e.stack
94
- })
95
- }
96
- }
97
-
98
- return function jsonParser (req, res, next) {
99
- if (req._body) {
100
- debug('body already parsed')
101
- next()
102
- return
103
- }
104
-
105
- req.body = req.body || {}
106
-
107
- // skip requests without bodies
108
- if (!typeis.hasBody(req)) {
109
- debug('skip empty body')
110
- next()
111
- return
112
- }
113
-
114
- debug('content-type %j', req.headers['content-type'])
115
-
116
- // determine if request should be parsed
117
- if (!shouldParse(req)) {
118
- debug('skip parsing')
119
- next()
120
- return
121
- }
122
-
123
- // assert charset per RFC 7159 sec 8.1
124
- var charset = getCharset(req) || 'utf-8'
125
- if (charset.substr(0, 4) !== 'utf-') {
126
- debug('invalid charset')
127
- next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
128
- charset: charset,
129
- type: 'charset.unsupported'
130
- }))
131
- return
132
- }
133
-
134
- // read
135
- read(req, res, next, parse, debug, {
136
- encoding: charset,
137
- inflate: inflate,
138
- limit: limit,
139
- verify: verify
140
- })
141
- }
142
- }
143
-
144
- /**
145
- * Create strict violation syntax error matching native error.
146
- *
147
- * @param {string} str
148
- * @param {string} char
149
- * @return {Error}
150
- * @private
151
- */
152
-
153
- function createStrictSyntaxError (str, char) {
154
- var index = str.indexOf(char)
155
- var partial = str.substring(0, index) + '#'
156
-
157
- try {
158
- JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation')
159
- } catch (e) {
160
- return normalizeJsonSyntaxError(e, {
161
- message: e.message.replace('#', char),
162
- stack: e.stack
163
- })
164
- }
165
- }
166
-
167
- /**
168
- * Get the first non-whitespace character in a string.
169
- *
170
- * @param {string} str
171
- * @return {function}
172
- * @private
173
- */
174
-
175
- function firstchar (str) {
176
- return FIRST_CHAR_REGEXP.exec(str)[1]
177
- }
178
-
179
- /**
180
- * Get the charset of a request.
181
- *
182
- * @param {object} req
183
- * @api private
184
- */
185
-
186
- function getCharset (req) {
187
- try {
188
- return (contentType.parse(req).parameters.charset || '').toLowerCase()
189
- } catch (e) {
190
- return undefined
191
- }
192
- }
193
-
194
- /**
195
- * Normalize a SyntaxError for JSON.parse.
196
- *
197
- * @param {SyntaxError} error
198
- * @param {object} obj
199
- * @return {SyntaxError}
200
- */
201
-
202
- function normalizeJsonSyntaxError (error, obj) {
203
- var keys = Object.getOwnPropertyNames(error)
204
-
205
- for (var i = 0; i < keys.length; i++) {
206
- var key = keys[i]
207
- if (key !== 'stack' && key !== 'message') {
208
- delete error[key]
209
- }
210
- }
211
-
212
- // replace stack before message for Node.js 0.10 and below
213
- error.stack = obj.stack.replace(error.message, obj.message)
214
- error.message = obj.message
215
-
216
- return error
217
- }
218
-
219
- /**
220
- * Get the simple type checker.
221
- *
222
- * @param {string} type
223
- * @return {function}
224
- */
225
-
226
- function typeChecker (type) {
227
- return function checkType (req) {
228
- return Boolean(typeis(req, type))
229
- }
230
- }
@@ -1,101 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- */
12
-
13
- var bytes = require('bytes')
14
- var debug = require('debug')('body-parser:raw')
15
- var read = require('../read')
16
- var typeis = require('type-is')
17
-
18
- /**
19
- * Module exports.
20
- */
21
-
22
- module.exports = raw
23
-
24
- /**
25
- * Create a middleware to parse raw bodies.
26
- *
27
- * @param {object} [options]
28
- * @return {function}
29
- * @api public
30
- */
31
-
32
- function raw (options) {
33
- var opts = options || {}
34
-
35
- var inflate = opts.inflate !== false
36
- var limit = typeof opts.limit !== 'number'
37
- ? bytes.parse(opts.limit || '100kb')
38
- : opts.limit
39
- var type = opts.type || 'application/octet-stream'
40
- var verify = opts.verify || false
41
-
42
- if (verify !== false && typeof verify !== 'function') {
43
- throw new TypeError('option verify must be function')
44
- }
45
-
46
- // create the appropriate type checking function
47
- var shouldParse = typeof type !== 'function'
48
- ? typeChecker(type)
49
- : type
50
-
51
- function parse (buf) {
52
- return buf
53
- }
54
-
55
- return function rawParser (req, res, next) {
56
- if (req._body) {
57
- debug('body already parsed')
58
- next()
59
- return
60
- }
61
-
62
- req.body = req.body || {}
63
-
64
- // skip requests without bodies
65
- if (!typeis.hasBody(req)) {
66
- debug('skip empty body')
67
- next()
68
- return
69
- }
70
-
71
- debug('content-type %j', req.headers['content-type'])
72
-
73
- // determine if request should be parsed
74
- if (!shouldParse(req)) {
75
- debug('skip parsing')
76
- next()
77
- return
78
- }
79
-
80
- // read
81
- read(req, res, next, parse, debug, {
82
- encoding: null,
83
- inflate: inflate,
84
- limit: limit,
85
- verify: verify
86
- })
87
- }
88
- }
89
-
90
- /**
91
- * Get the simple type checker.
92
- *
93
- * @param {string} type
94
- * @return {function}
95
- */
96
-
97
- function typeChecker (type) {
98
- return function checkType (req) {
99
- return Boolean(typeis(req, type))
100
- }
101
- }
@@ -1,121 +0,0 @@
1
- /*!
2
- * body-parser
3
- * Copyright(c) 2014-2015 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module dependencies.
11
- */
12
-
13
- var bytes = require('bytes')
14
- var contentType = require('content-type')
15
- var debug = require('debug')('body-parser:text')
16
- var read = require('../read')
17
- var typeis = require('type-is')
18
-
19
- /**
20
- * Module exports.
21
- */
22
-
23
- module.exports = text
24
-
25
- /**
26
- * Create a middleware to parse text bodies.
27
- *
28
- * @param {object} [options]
29
- * @return {function}
30
- * @api public
31
- */
32
-
33
- function text (options) {
34
- var opts = options || {}
35
-
36
- var defaultCharset = opts.defaultCharset || 'utf-8'
37
- var inflate = opts.inflate !== false
38
- var limit = typeof opts.limit !== 'number'
39
- ? bytes.parse(opts.limit || '100kb')
40
- : opts.limit
41
- var type = opts.type || 'text/plain'
42
- var verify = opts.verify || false
43
-
44
- if (verify !== false && typeof verify !== 'function') {
45
- throw new TypeError('option verify must be function')
46
- }
47
-
48
- // create the appropriate type checking function
49
- var shouldParse = typeof type !== 'function'
50
- ? typeChecker(type)
51
- : type
52
-
53
- function parse (buf) {
54
- return buf
55
- }
56
-
57
- return function textParser (req, res, next) {
58
- if (req._body) {
59
- debug('body already parsed')
60
- next()
61
- return
62
- }
63
-
64
- req.body = req.body || {}
65
-
66
- // skip requests without bodies
67
- if (!typeis.hasBody(req)) {
68
- debug('skip empty body')
69
- next()
70
- return
71
- }
72
-
73
- debug('content-type %j', req.headers['content-type'])
74
-
75
- // determine if request should be parsed
76
- if (!shouldParse(req)) {
77
- debug('skip parsing')
78
- next()
79
- return
80
- }
81
-
82
- // get charset
83
- var charset = getCharset(req) || defaultCharset
84
-
85
- // read
86
- read(req, res, next, parse, debug, {
87
- encoding: charset,
88
- inflate: inflate,
89
- limit: limit,
90
- verify: verify
91
- })
92
- }
93
- }
94
-
95
- /**
96
- * Get the charset of a request.
97
- *
98
- * @param {object} req
99
- * @api private
100
- */
101
-
102
- function getCharset (req) {
103
- try {
104
- return (contentType.parse(req).parameters.charset || '').toLowerCase()
105
- } catch (e) {
106
- return undefined
107
- }
108
- }
109
-
110
- /**
111
- * Get the simple type checker.
112
- *
113
- * @param {string} type
114
- * @return {function}
115
- */
116
-
117
- function typeChecker (type) {
118
- return function checkType (req) {
119
- return Boolean(typeis(req, type))
120
- }
121
- }