es6-crawler-detect 3.3.0 → 4.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (383) hide show
  1. package/.babelrc +1 -1
  2. package/.mocharc.json +5 -0
  3. package/README.md +1 -0
  4. package/dist/index.d.ts +5 -0
  5. package/dist/index.d.ts.map +1 -0
  6. package/dist/index.js +18 -0
  7. package/dist/index.js.map +1 -0
  8. package/dist/lib/crawler/crawlers.d.ts +5 -0
  9. package/dist/lib/crawler/crawlers.d.ts.map +1 -0
  10. package/dist/lib/crawler/crawlers.js +1400 -0
  11. package/dist/lib/crawler/crawlers.js.map +1 -0
  12. package/dist/lib/crawler/exclusions.d.ts +5 -0
  13. package/dist/lib/crawler/exclusions.d.ts.map +1 -0
  14. package/dist/lib/crawler/exclusions.js +59 -0
  15. package/dist/lib/crawler/exclusions.js.map +1 -0
  16. package/dist/lib/crawler/headers.d.ts +5 -0
  17. package/dist/lib/crawler/headers.d.ts.map +1 -0
  18. package/dist/lib/crawler/headers.js +21 -0
  19. package/dist/lib/crawler/headers.js.map +1 -0
  20. package/dist/lib/crawler/provider.d.ts +4 -0
  21. package/dist/lib/crawler/provider.d.ts.map +1 -0
  22. package/dist/lib/crawler/provider.js +3 -0
  23. package/dist/lib/crawler/provider.js.map +1 -0
  24. package/dist/lib/crawler.d.ts +21 -0
  25. package/dist/lib/crawler.d.ts.map +1 -0
  26. package/dist/lib/crawler.js +98 -0
  27. package/dist/lib/crawler.js.map +1 -0
  28. package/dist/types.d.ts +9 -0
  29. package/dist/types.d.ts.map +1 -0
  30. package/dist/types.js +3 -0
  31. package/dist/types.js.map +1 -0
  32. package/eslint.config.mjs +62 -0
  33. package/example/node/package.json +1 -1
  34. package/example/node/yarn.lock +222 -99
  35. package/package.json +30 -24
  36. package/src/index.ts +18 -0
  37. package/src/lib/crawler/{crawlers.js → crawlers.ts} +1397 -1403
  38. package/src/lib/crawler/{exclusions.js → exclusions.ts} +56 -62
  39. package/src/lib/crawler/{headers.js → headers.ts} +4 -10
  40. package/src/lib/crawler/provider.ts +3 -0
  41. package/src/lib/crawler.ts +128 -0
  42. package/src/types.ts +15 -0
  43. package/test/lib/crawler.spec.ts +116 -0
  44. package/tsconfig.json +28 -0
  45. package/.eslintignore +0 -6
  46. package/.eslintrc.json +0 -30
  47. package/dist/main.bundle.js +0 -1
  48. package/example/node/dist/crawler.min.js +0 -1
  49. package/example/node/node_modules/.bin/mime +0 -15
  50. package/example/node/node_modules/.bin/mime.cmd +0 -7
  51. package/example/node/node_modules/.yarn-integrity +0 -67
  52. package/example/node/node_modules/accepts/HISTORY.md +0 -236
  53. package/example/node/node_modules/accepts/LICENSE +0 -23
  54. package/example/node/node_modules/accepts/README.md +0 -142
  55. package/example/node/node_modules/accepts/index.js +0 -238
  56. package/example/node/node_modules/accepts/package.json +0 -47
  57. package/example/node/node_modules/array-flatten/LICENSE +0 -21
  58. package/example/node/node_modules/array-flatten/README.md +0 -43
  59. package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
  60. package/example/node/node_modules/array-flatten/package.json +0 -39
  61. package/example/node/node_modules/body-parser/HISTORY.md +0 -624
  62. package/example/node/node_modules/body-parser/LICENSE +0 -23
  63. package/example/node/node_modules/body-parser/README.md +0 -456
  64. package/example/node/node_modules/body-parser/index.js +0 -157
  65. package/example/node/node_modules/body-parser/lib/read.js +0 -181
  66. package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
  67. package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
  68. package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
  69. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
  70. package/example/node/node_modules/body-parser/package.json +0 -52
  71. package/example/node/node_modules/bytes/History.md +0 -92
  72. package/example/node/node_modules/bytes/LICENSE +0 -23
  73. package/example/node/node_modules/bytes/Readme.md +0 -152
  74. package/example/node/node_modules/bytes/index.js +0 -166
  75. package/example/node/node_modules/bytes/package.json +0 -42
  76. package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
  77. package/example/node/node_modules/content-disposition/LICENSE +0 -22
  78. package/example/node/node_modules/content-disposition/README.md +0 -142
  79. package/example/node/node_modules/content-disposition/index.js +0 -458
  80. package/example/node/node_modules/content-disposition/package.json +0 -44
  81. package/example/node/node_modules/content-type/HISTORY.md +0 -24
  82. package/example/node/node_modules/content-type/LICENSE +0 -22
  83. package/example/node/node_modules/content-type/README.md +0 -92
  84. package/example/node/node_modules/content-type/index.js +0 -222
  85. package/example/node/node_modules/content-type/package.json +0 -40
  86. package/example/node/node_modules/cookie/HISTORY.md +0 -128
  87. package/example/node/node_modules/cookie/LICENSE +0 -24
  88. package/example/node/node_modules/cookie/README.md +0 -257
  89. package/example/node/node_modules/cookie/index.js +0 -202
  90. package/example/node/node_modules/cookie/package.json +0 -40
  91. package/example/node/node_modules/cookie-signature/History.md +0 -38
  92. package/example/node/node_modules/cookie-signature/Readme.md +0 -42
  93. package/example/node/node_modules/cookie-signature/index.js +0 -51
  94. package/example/node/node_modules/cookie-signature/package.json +0 -18
  95. package/example/node/node_modules/debug/.coveralls.yml +0 -1
  96. package/example/node/node_modules/debug/.eslintrc +0 -11
  97. package/example/node/node_modules/debug/.travis.yml +0 -14
  98. package/example/node/node_modules/debug/CHANGELOG.md +0 -362
  99. package/example/node/node_modules/debug/LICENSE +0 -19
  100. package/example/node/node_modules/debug/Makefile +0 -50
  101. package/example/node/node_modules/debug/README.md +0 -312
  102. package/example/node/node_modules/debug/component.json +0 -19
  103. package/example/node/node_modules/debug/karma.conf.js +0 -70
  104. package/example/node/node_modules/debug/node.js +0 -1
  105. package/example/node/node_modules/debug/package.json +0 -49
  106. package/example/node/node_modules/debug/src/browser.js +0 -185
  107. package/example/node/node_modules/debug/src/debug.js +0 -202
  108. package/example/node/node_modules/debug/src/index.js +0 -10
  109. package/example/node/node_modules/debug/src/inspector-log.js +0 -15
  110. package/example/node/node_modules/debug/src/node.js +0 -248
  111. package/example/node/node_modules/depd/History.md +0 -96
  112. package/example/node/node_modules/depd/LICENSE +0 -22
  113. package/example/node/node_modules/depd/Readme.md +0 -280
  114. package/example/node/node_modules/depd/index.js +0 -522
  115. package/example/node/node_modules/depd/lib/browser/index.js +0 -77
  116. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
  117. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
  118. package/example/node/node_modules/depd/lib/compat/index.js +0 -79
  119. package/example/node/node_modules/depd/package.json +0 -41
  120. package/example/node/node_modules/destroy/LICENSE +0 -22
  121. package/example/node/node_modules/destroy/README.md +0 -60
  122. package/example/node/node_modules/destroy/index.js +0 -75
  123. package/example/node/node_modules/destroy/package.json +0 -37
  124. package/example/node/node_modules/ee-first/LICENSE +0 -22
  125. package/example/node/node_modules/ee-first/README.md +0 -80
  126. package/example/node/node_modules/ee-first/index.js +0 -95
  127. package/example/node/node_modules/ee-first/package.json +0 -29
  128. package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
  129. package/example/node/node_modules/encodeurl/LICENSE +0 -22
  130. package/example/node/node_modules/encodeurl/README.md +0 -128
  131. package/example/node/node_modules/encodeurl/index.js +0 -60
  132. package/example/node/node_modules/encodeurl/package.json +0 -40
  133. package/example/node/node_modules/escape-html/LICENSE +0 -24
  134. package/example/node/node_modules/escape-html/Readme.md +0 -43
  135. package/example/node/node_modules/escape-html/index.js +0 -78
  136. package/example/node/node_modules/escape-html/package.json +0 -24
  137. package/example/node/node_modules/etag/HISTORY.md +0 -83
  138. package/example/node/node_modules/etag/LICENSE +0 -22
  139. package/example/node/node_modules/etag/README.md +0 -159
  140. package/example/node/node_modules/etag/index.js +0 -131
  141. package/example/node/node_modules/etag/package.json +0 -47
  142. package/example/node/node_modules/express/History.md +0 -3510
  143. package/example/node/node_modules/express/LICENSE +0 -24
  144. package/example/node/node_modules/express/Readme.md +0 -158
  145. package/example/node/node_modules/express/index.js +0 -11
  146. package/example/node/node_modules/express/lib/application.js +0 -644
  147. package/example/node/node_modules/express/lib/express.js +0 -116
  148. package/example/node/node_modules/express/lib/middleware/init.js +0 -43
  149. package/example/node/node_modules/express/lib/middleware/query.js +0 -47
  150. package/example/node/node_modules/express/lib/request.js +0 -525
  151. package/example/node/node_modules/express/lib/response.js +0 -1147
  152. package/example/node/node_modules/express/lib/router/index.js +0 -668
  153. package/example/node/node_modules/express/lib/router/layer.js +0 -181
  154. package/example/node/node_modules/express/lib/router/route.js +0 -216
  155. package/example/node/node_modules/express/lib/utils.js +0 -302
  156. package/example/node/node_modules/express/lib/view.js +0 -182
  157. package/example/node/node_modules/express/package.json +0 -99
  158. package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
  159. package/example/node/node_modules/finalhandler/LICENSE +0 -22
  160. package/example/node/node_modules/finalhandler/README.md +0 -148
  161. package/example/node/node_modules/finalhandler/index.js +0 -331
  162. package/example/node/node_modules/finalhandler/package.json +0 -45
  163. package/example/node/node_modules/forwarded/HISTORY.md +0 -21
  164. package/example/node/node_modules/forwarded/LICENSE +0 -22
  165. package/example/node/node_modules/forwarded/README.md +0 -57
  166. package/example/node/node_modules/forwarded/index.js +0 -90
  167. package/example/node/node_modules/forwarded/package.json +0 -45
  168. package/example/node/node_modules/fresh/HISTORY.md +0 -70
  169. package/example/node/node_modules/fresh/LICENSE +0 -23
  170. package/example/node/node_modules/fresh/README.md +0 -119
  171. package/example/node/node_modules/fresh/index.js +0 -137
  172. package/example/node/node_modules/fresh/package.json +0 -46
  173. package/example/node/node_modules/http-errors/HISTORY.md +0 -165
  174. package/example/node/node_modules/http-errors/LICENSE +0 -23
  175. package/example/node/node_modules/http-errors/README.md +0 -169
  176. package/example/node/node_modules/http-errors/index.js +0 -299
  177. package/example/node/node_modules/http-errors/package.json +0 -49
  178. package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
  179. package/example/node/node_modules/iconv-lite/LICENSE +0 -21
  180. package/example/node/node_modules/iconv-lite/README.md +0 -156
  181. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
  182. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
  183. package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
  184. package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
  185. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
  186. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
  187. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
  188. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
  189. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
  190. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
  191. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
  192. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
  193. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
  194. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
  195. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
  196. package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
  197. package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
  198. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
  199. package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
  200. package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
  201. package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
  202. package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
  203. package/example/node/node_modules/iconv-lite/package.json +0 -46
  204. package/example/node/node_modules/inherits/LICENSE +0 -16
  205. package/example/node/node_modules/inherits/README.md +0 -42
  206. package/example/node/node_modules/inherits/inherits.js +0 -9
  207. package/example/node/node_modules/inherits/inherits_browser.js +0 -27
  208. package/example/node/node_modules/inherits/package.json +0 -29
  209. package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
  210. package/example/node/node_modules/ipaddr.js/README.md +0 -233
  211. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
  212. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
  213. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
  214. package/example/node/node_modules/ipaddr.js/package.json +0 -35
  215. package/example/node/node_modules/media-typer/HISTORY.md +0 -22
  216. package/example/node/node_modules/media-typer/LICENSE +0 -22
  217. package/example/node/node_modules/media-typer/README.md +0 -81
  218. package/example/node/node_modules/media-typer/index.js +0 -270
  219. package/example/node/node_modules/media-typer/package.json +0 -26
  220. package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
  221. package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
  222. package/example/node/node_modules/merge-descriptors/README.md +0 -48
  223. package/example/node/node_modules/merge-descriptors/index.js +0 -60
  224. package/example/node/node_modules/merge-descriptors/package.json +0 -32
  225. package/example/node/node_modules/methods/HISTORY.md +0 -29
  226. package/example/node/node_modules/methods/LICENSE +0 -24
  227. package/example/node/node_modules/methods/README.md +0 -51
  228. package/example/node/node_modules/methods/index.js +0 -69
  229. package/example/node/node_modules/methods/package.json +0 -36
  230. package/example/node/node_modules/mime/CHANGELOG.md +0 -164
  231. package/example/node/node_modules/mime/LICENSE +0 -21
  232. package/example/node/node_modules/mime/README.md +0 -90
  233. package/example/node/node_modules/mime/cli.js +0 -8
  234. package/example/node/node_modules/mime/mime.js +0 -108
  235. package/example/node/node_modules/mime/package.json +0 -44
  236. package/example/node/node_modules/mime/src/build.js +0 -53
  237. package/example/node/node_modules/mime/src/test.js +0 -60
  238. package/example/node/node_modules/mime/types.json +0 -1
  239. package/example/node/node_modules/mime-db/HISTORY.md +0 -499
  240. package/example/node/node_modules/mime-db/LICENSE +0 -22
  241. package/example/node/node_modules/mime-db/README.md +0 -100
  242. package/example/node/node_modules/mime-db/db.json +0 -8471
  243. package/example/node/node_modules/mime-db/index.js +0 -11
  244. package/example/node/node_modules/mime-db/package.json +0 -59
  245. package/example/node/node_modules/mime-types/HISTORY.md +0 -388
  246. package/example/node/node_modules/mime-types/LICENSE +0 -23
  247. package/example/node/node_modules/mime-types/README.md +0 -113
  248. package/example/node/node_modules/mime-types/index.js +0 -188
  249. package/example/node/node_modules/mime-types/package.json +0 -44
  250. package/example/node/node_modules/ms/index.js +0 -152
  251. package/example/node/node_modules/ms/license.md +0 -21
  252. package/example/node/node_modules/ms/package.json +0 -37
  253. package/example/node/node_modules/ms/readme.md +0 -51
  254. package/example/node/node_modules/negotiator/HISTORY.md +0 -103
  255. package/example/node/node_modules/negotiator/LICENSE +0 -24
  256. package/example/node/node_modules/negotiator/README.md +0 -203
  257. package/example/node/node_modules/negotiator/index.js +0 -124
  258. package/example/node/node_modules/negotiator/lib/charset.js +0 -169
  259. package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
  260. package/example/node/node_modules/negotiator/lib/language.js +0 -179
  261. package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
  262. package/example/node/node_modules/negotiator/package.json +0 -42
  263. package/example/node/node_modules/on-finished/HISTORY.md +0 -88
  264. package/example/node/node_modules/on-finished/LICENSE +0 -23
  265. package/example/node/node_modules/on-finished/README.md +0 -154
  266. package/example/node/node_modules/on-finished/index.js +0 -196
  267. package/example/node/node_modules/on-finished/package.json +0 -31
  268. package/example/node/node_modules/parseurl/HISTORY.md +0 -58
  269. package/example/node/node_modules/parseurl/LICENSE +0 -24
  270. package/example/node/node_modules/parseurl/README.md +0 -133
  271. package/example/node/node_modules/parseurl/index.js +0 -158
  272. package/example/node/node_modules/parseurl/package.json +0 -40
  273. package/example/node/node_modules/path-to-regexp/History.md +0 -36
  274. package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
  275. package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
  276. package/example/node/node_modules/path-to-regexp/index.js +0 -129
  277. package/example/node/node_modules/path-to-regexp/package.json +0 -30
  278. package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
  279. package/example/node/node_modules/proxy-addr/LICENSE +0 -22
  280. package/example/node/node_modules/proxy-addr/README.md +0 -139
  281. package/example/node/node_modules/proxy-addr/index.js +0 -327
  282. package/example/node/node_modules/proxy-addr/package.json +0 -47
  283. package/example/node/node_modules/qs/.editorconfig +0 -39
  284. package/example/node/node_modules/qs/.eslintignore +0 -2
  285. package/example/node/node_modules/qs/.eslintrc +0 -35
  286. package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
  287. package/example/node/node_modules/qs/.nycrc +0 -13
  288. package/example/node/node_modules/qs/CHANGELOG.md +0 -359
  289. package/example/node/node_modules/qs/LICENSE.md +0 -29
  290. package/example/node/node_modules/qs/README.md +0 -604
  291. package/example/node/node_modules/qs/dist/qs.js +0 -832
  292. package/example/node/node_modules/qs/lib/formats.js +0 -23
  293. package/example/node/node_modules/qs/lib/index.js +0 -11
  294. package/example/node/node_modules/qs/lib/parse.js +0 -257
  295. package/example/node/node_modules/qs/lib/stringify.js +0 -278
  296. package/example/node/node_modules/qs/lib/utils.js +0 -251
  297. package/example/node/node_modules/qs/package.json +0 -69
  298. package/example/node/node_modules/qs/test/parse.js +0 -772
  299. package/example/node/node_modules/qs/test/stringify.js +0 -793
  300. package/example/node/node_modules/qs/test/utils.js +0 -136
  301. package/example/node/node_modules/range-parser/HISTORY.md +0 -56
  302. package/example/node/node_modules/range-parser/LICENSE +0 -23
  303. package/example/node/node_modules/range-parser/README.md +0 -84
  304. package/example/node/node_modules/range-parser/index.js +0 -162
  305. package/example/node/node_modules/range-parser/package.json +0 -44
  306. package/example/node/node_modules/raw-body/HISTORY.md +0 -284
  307. package/example/node/node_modules/raw-body/LICENSE +0 -22
  308. package/example/node/node_modules/raw-body/README.md +0 -217
  309. package/example/node/node_modules/raw-body/index.d.ts +0 -87
  310. package/example/node/node_modules/raw-body/index.js +0 -286
  311. package/example/node/node_modules/raw-body/package.json +0 -48
  312. package/example/node/node_modules/safe-buffer/LICENSE +0 -21
  313. package/example/node/node_modules/safe-buffer/README.md +0 -584
  314. package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
  315. package/example/node/node_modules/safe-buffer/index.js +0 -65
  316. package/example/node/node_modules/safe-buffer/package.json +0 -51
  317. package/example/node/node_modules/safer-buffer/LICENSE +0 -21
  318. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
  319. package/example/node/node_modules/safer-buffer/Readme.md +0 -156
  320. package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
  321. package/example/node/node_modules/safer-buffer/package.json +0 -34
  322. package/example/node/node_modules/safer-buffer/safer.js +0 -77
  323. package/example/node/node_modules/safer-buffer/tests.js +0 -406
  324. package/example/node/node_modules/send/HISTORY.md +0 -506
  325. package/example/node/node_modules/send/LICENSE +0 -23
  326. package/example/node/node_modules/send/README.md +0 -327
  327. package/example/node/node_modules/send/index.js +0 -1133
  328. package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
  329. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
  330. package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
  331. package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
  332. package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
  333. package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
  334. package/example/node/node_modules/send/package.json +0 -61
  335. package/example/node/node_modules/serve-static/HISTORY.md +0 -459
  336. package/example/node/node_modules/serve-static/LICENSE +0 -25
  337. package/example/node/node_modules/serve-static/README.md +0 -257
  338. package/example/node/node_modules/serve-static/index.js +0 -210
  339. package/example/node/node_modules/serve-static/package.json +0 -42
  340. package/example/node/node_modules/setprototypeof/LICENSE +0 -13
  341. package/example/node/node_modules/setprototypeof/README.md +0 -31
  342. package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
  343. package/example/node/node_modules/setprototypeof/index.js +0 -17
  344. package/example/node/node_modules/setprototypeof/package.json +0 -38
  345. package/example/node/node_modules/setprototypeof/test/index.js +0 -24
  346. package/example/node/node_modules/statuses/HISTORY.md +0 -65
  347. package/example/node/node_modules/statuses/LICENSE +0 -23
  348. package/example/node/node_modules/statuses/README.md +0 -127
  349. package/example/node/node_modules/statuses/codes.json +0 -66
  350. package/example/node/node_modules/statuses/index.js +0 -113
  351. package/example/node/node_modules/statuses/package.json +0 -48
  352. package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
  353. package/example/node/node_modules/toidentifier/LICENSE +0 -21
  354. package/example/node/node_modules/toidentifier/README.md +0 -61
  355. package/example/node/node_modules/toidentifier/index.js +0 -32
  356. package/example/node/node_modules/toidentifier/package.json +0 -38
  357. package/example/node/node_modules/type-is/HISTORY.md +0 -259
  358. package/example/node/node_modules/type-is/LICENSE +0 -23
  359. package/example/node/node_modules/type-is/README.md +0 -170
  360. package/example/node/node_modules/type-is/index.js +0 -266
  361. package/example/node/node_modules/type-is/package.json +0 -45
  362. package/example/node/node_modules/unpipe/HISTORY.md +0 -4
  363. package/example/node/node_modules/unpipe/LICENSE +0 -22
  364. package/example/node/node_modules/unpipe/README.md +0 -43
  365. package/example/node/node_modules/unpipe/index.js +0 -69
  366. package/example/node/node_modules/unpipe/package.json +0 -27
  367. package/example/node/node_modules/utils-merge/LICENSE +0 -20
  368. package/example/node/node_modules/utils-merge/README.md +0 -34
  369. package/example/node/node_modules/utils-merge/index.js +0 -23
  370. package/example/node/node_modules/utils-merge/package.json +0 -40
  371. package/example/node/node_modules/vary/HISTORY.md +0 -39
  372. package/example/node/node_modules/vary/LICENSE +0 -22
  373. package/example/node/node_modules/vary/README.md +0 -101
  374. package/example/node/node_modules/vary/index.js +0 -149
  375. package/example/node/node_modules/vary/package.json +0 -43
  376. package/example/node/server.js +0 -38
  377. package/src/index.js +0 -16
  378. package/src/lib/crawler/provider.js +0 -11
  379. package/src/lib/crawler.js +0 -130
  380. package/test/lib/crawler.test.js +0 -94
  381. package/webpack.common.js +0 -20
  382. package/webpack.dev.js +0 -7
  383. package/webpack.prod.js +0 -14
@@ -1,139 +0,0 @@
1
- # proxy-addr
2
-
3
- [![NPM Version][npm-version-image]][npm-url]
4
- [![NPM Downloads][npm-downloads-image]][npm-url]
5
- [![Node.js Version][node-image]][node-url]
6
- [![Build Status][ci-image]][ci-url]
7
- [![Test Coverage][coveralls-image]][coveralls-url]
8
-
9
- Determine address of proxied request
10
-
11
- ## Install
12
-
13
- This is a [Node.js](https://nodejs.org/en/) module available through the
14
- [npm registry](https://www.npmjs.com/). Installation is done using the
15
- [`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
16
-
17
- ```sh
18
- $ npm install proxy-addr
19
- ```
20
-
21
- ## API
22
-
23
- ```js
24
- var proxyaddr = require('proxy-addr')
25
- ```
26
-
27
- ### proxyaddr(req, trust)
28
-
29
- Return the address of the request, using the given `trust` parameter.
30
-
31
- The `trust` argument is a function that returns `true` if you trust
32
- the address, `false` if you don't. The closest untrusted address is
33
- returned.
34
-
35
- ```js
36
- proxyaddr(req, function (addr) { return addr === '127.0.0.1' })
37
- proxyaddr(req, function (addr, i) { return i < 1 })
38
- ```
39
-
40
- The `trust` arugment may also be a single IP address string or an
41
- array of trusted addresses, as plain IP addresses, CIDR-formatted
42
- strings, or IP/netmask strings.
43
-
44
- ```js
45
- proxyaddr(req, '127.0.0.1')
46
- proxyaddr(req, ['127.0.0.0/8', '10.0.0.0/8'])
47
- proxyaddr(req, ['127.0.0.0/255.0.0.0', '192.168.0.0/255.255.0.0'])
48
- ```
49
-
50
- This module also supports IPv6. Your IPv6 addresses will be normalized
51
- automatically (i.e. `fe80::00ed:1` equals `fe80:0:0:0:0:0:ed:1`).
52
-
53
- ```js
54
- proxyaddr(req, '::1')
55
- proxyaddr(req, ['::1/128', 'fe80::/10'])
56
- ```
57
-
58
- This module will automatically work with IPv4-mapped IPv6 addresses
59
- as well to support node.js in IPv6-only mode. This means that you do
60
- not have to specify both `::ffff:a00:1` and `10.0.0.1`.
61
-
62
- As a convenience, this module also takes certain pre-defined names
63
- in addition to IP addresses, which expand into IP addresses:
64
-
65
- ```js
66
- proxyaddr(req, 'loopback')
67
- proxyaddr(req, ['loopback', 'fc00:ac:1ab5:fff::1/64'])
68
- ```
69
-
70
- * `loopback`: IPv4 and IPv6 loopback addresses (like `::1` and
71
- `127.0.0.1`).
72
- * `linklocal`: IPv4 and IPv6 link-local addresses (like
73
- `fe80::1:1:1:1` and `169.254.0.1`).
74
- * `uniquelocal`: IPv4 private addresses and IPv6 unique-local
75
- addresses (like `fc00:ac:1ab5:fff::1` and `192.168.0.1`).
76
-
77
- When `trust` is specified as a function, it will be called for each
78
- address to determine if it is a trusted address. The function is
79
- given two arguments: `addr` and `i`, where `addr` is a string of
80
- the address to check and `i` is a number that represents the distance
81
- from the socket address.
82
-
83
- ### proxyaddr.all(req, [trust])
84
-
85
- Return all the addresses of the request, optionally stopping at the
86
- first untrusted. This array is ordered from closest to furthest
87
- (i.e. `arr[0] === req.connection.remoteAddress`).
88
-
89
- ```js
90
- proxyaddr.all(req)
91
- ```
92
-
93
- The optional `trust` argument takes the same arguments as `trust`
94
- does in `proxyaddr(req, trust)`.
95
-
96
- ```js
97
- proxyaddr.all(req, 'loopback')
98
- ```
99
-
100
- ### proxyaddr.compile(val)
101
-
102
- Compiles argument `val` into a `trust` function. This function takes
103
- the same arguments as `trust` does in `proxyaddr(req, trust)` and
104
- returns a function suitable for `proxyaddr(req, trust)`.
105
-
106
- ```js
107
- var trust = proxyaddr.compile('loopback')
108
- var addr = proxyaddr(req, trust)
109
- ```
110
-
111
- This function is meant to be optimized for use against every request.
112
- It is recommend to compile a trust function up-front for the trusted
113
- configuration and pass that to `proxyaddr(req, trust)` for each request.
114
-
115
- ## Testing
116
-
117
- ```sh
118
- $ npm test
119
- ```
120
-
121
- ## Benchmarks
122
-
123
- ```sh
124
- $ npm run-script bench
125
- ```
126
-
127
- ## License
128
-
129
- [MIT](LICENSE)
130
-
131
- [ci-image]: https://badgen.net/github/checks/jshttp/proxy-addr/master?label=ci
132
- [ci-url]: https://github.com/jshttp/proxy-addr/actions?query=workflow%3Aci
133
- [coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/proxy-addr/master
134
- [coveralls-url]: https://coveralls.io/r/jshttp/proxy-addr?branch=master
135
- [node-image]: https://badgen.net/npm/node/proxy-addr
136
- [node-url]: https://nodejs.org/en/download
137
- [npm-downloads-image]: https://badgen.net/npm/dm/proxy-addr
138
- [npm-url]: https://npmjs.org/package/proxy-addr
139
- [npm-version-image]: https://badgen.net/npm/v/proxy-addr
@@ -1,327 +0,0 @@
1
- /*!
2
- * proxy-addr
3
- * Copyright(c) 2014-2016 Douglas Christopher Wilson
4
- * MIT Licensed
5
- */
6
-
7
- 'use strict'
8
-
9
- /**
10
- * Module exports.
11
- * @public
12
- */
13
-
14
- module.exports = proxyaddr
15
- module.exports.all = alladdrs
16
- module.exports.compile = compile
17
-
18
- /**
19
- * Module dependencies.
20
- * @private
21
- */
22
-
23
- var forwarded = require('forwarded')
24
- var ipaddr = require('ipaddr.js')
25
-
26
- /**
27
- * Variables.
28
- * @private
29
- */
30
-
31
- var DIGIT_REGEXP = /^[0-9]+$/
32
- var isip = ipaddr.isValid
33
- var parseip = ipaddr.parse
34
-
35
- /**
36
- * Pre-defined IP ranges.
37
- * @private
38
- */
39
-
40
- var IP_RANGES = {
41
- linklocal: ['169.254.0.0/16', 'fe80::/10'],
42
- loopback: ['127.0.0.1/8', '::1/128'],
43
- uniquelocal: ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16', 'fc00::/7']
44
- }
45
-
46
- /**
47
- * Get all addresses in the request, optionally stopping
48
- * at the first untrusted.
49
- *
50
- * @param {Object} request
51
- * @param {Function|Array|String} [trust]
52
- * @public
53
- */
54
-
55
- function alladdrs (req, trust) {
56
- // get addresses
57
- var addrs = forwarded(req)
58
-
59
- if (!trust) {
60
- // Return all addresses
61
- return addrs
62
- }
63
-
64
- if (typeof trust !== 'function') {
65
- trust = compile(trust)
66
- }
67
-
68
- for (var i = 0; i < addrs.length - 1; i++) {
69
- if (trust(addrs[i], i)) continue
70
-
71
- addrs.length = i + 1
72
- }
73
-
74
- return addrs
75
- }
76
-
77
- /**
78
- * Compile argument into trust function.
79
- *
80
- * @param {Array|String} val
81
- * @private
82
- */
83
-
84
- function compile (val) {
85
- if (!val) {
86
- throw new TypeError('argument is required')
87
- }
88
-
89
- var trust
90
-
91
- if (typeof val === 'string') {
92
- trust = [val]
93
- } else if (Array.isArray(val)) {
94
- trust = val.slice()
95
- } else {
96
- throw new TypeError('unsupported trust argument')
97
- }
98
-
99
- for (var i = 0; i < trust.length; i++) {
100
- val = trust[i]
101
-
102
- if (!Object.prototype.hasOwnProperty.call(IP_RANGES, val)) {
103
- continue
104
- }
105
-
106
- // Splice in pre-defined range
107
- val = IP_RANGES[val]
108
- trust.splice.apply(trust, [i, 1].concat(val))
109
- i += val.length - 1
110
- }
111
-
112
- return compileTrust(compileRangeSubnets(trust))
113
- }
114
-
115
- /**
116
- * Compile `arr` elements into range subnets.
117
- *
118
- * @param {Array} arr
119
- * @private
120
- */
121
-
122
- function compileRangeSubnets (arr) {
123
- var rangeSubnets = new Array(arr.length)
124
-
125
- for (var i = 0; i < arr.length; i++) {
126
- rangeSubnets[i] = parseipNotation(arr[i])
127
- }
128
-
129
- return rangeSubnets
130
- }
131
-
132
- /**
133
- * Compile range subnet array into trust function.
134
- *
135
- * @param {Array} rangeSubnets
136
- * @private
137
- */
138
-
139
- function compileTrust (rangeSubnets) {
140
- // Return optimized function based on length
141
- var len = rangeSubnets.length
142
- return len === 0
143
- ? trustNone
144
- : len === 1
145
- ? trustSingle(rangeSubnets[0])
146
- : trustMulti(rangeSubnets)
147
- }
148
-
149
- /**
150
- * Parse IP notation string into range subnet.
151
- *
152
- * @param {String} note
153
- * @private
154
- */
155
-
156
- function parseipNotation (note) {
157
- var pos = note.lastIndexOf('/')
158
- var str = pos !== -1
159
- ? note.substring(0, pos)
160
- : note
161
-
162
- if (!isip(str)) {
163
- throw new TypeError('invalid IP address: ' + str)
164
- }
165
-
166
- var ip = parseip(str)
167
-
168
- if (pos === -1 && ip.kind() === 'ipv6' && ip.isIPv4MappedAddress()) {
169
- // Store as IPv4
170
- ip = ip.toIPv4Address()
171
- }
172
-
173
- var max = ip.kind() === 'ipv6'
174
- ? 128
175
- : 32
176
-
177
- var range = pos !== -1
178
- ? note.substring(pos + 1, note.length)
179
- : null
180
-
181
- if (range === null) {
182
- range = max
183
- } else if (DIGIT_REGEXP.test(range)) {
184
- range = parseInt(range, 10)
185
- } else if (ip.kind() === 'ipv4' && isip(range)) {
186
- range = parseNetmask(range)
187
- } else {
188
- range = null
189
- }
190
-
191
- if (range <= 0 || range > max) {
192
- throw new TypeError('invalid range on address: ' + note)
193
- }
194
-
195
- return [ip, range]
196
- }
197
-
198
- /**
199
- * Parse netmask string into CIDR range.
200
- *
201
- * @param {String} netmask
202
- * @private
203
- */
204
-
205
- function parseNetmask (netmask) {
206
- var ip = parseip(netmask)
207
- var kind = ip.kind()
208
-
209
- return kind === 'ipv4'
210
- ? ip.prefixLengthFromSubnetMask()
211
- : null
212
- }
213
-
214
- /**
215
- * Determine address of proxied request.
216
- *
217
- * @param {Object} request
218
- * @param {Function|Array|String} trust
219
- * @public
220
- */
221
-
222
- function proxyaddr (req, trust) {
223
- if (!req) {
224
- throw new TypeError('req argument is required')
225
- }
226
-
227
- if (!trust) {
228
- throw new TypeError('trust argument is required')
229
- }
230
-
231
- var addrs = alladdrs(req, trust)
232
- var addr = addrs[addrs.length - 1]
233
-
234
- return addr
235
- }
236
-
237
- /**
238
- * Static trust function to trust nothing.
239
- *
240
- * @private
241
- */
242
-
243
- function trustNone () {
244
- return false
245
- }
246
-
247
- /**
248
- * Compile trust function for multiple subnets.
249
- *
250
- * @param {Array} subnets
251
- * @private
252
- */
253
-
254
- function trustMulti (subnets) {
255
- return function trust (addr) {
256
- if (!isip(addr)) return false
257
-
258
- var ip = parseip(addr)
259
- var ipconv
260
- var kind = ip.kind()
261
-
262
- for (var i = 0; i < subnets.length; i++) {
263
- var subnet = subnets[i]
264
- var subnetip = subnet[0]
265
- var subnetkind = subnetip.kind()
266
- var subnetrange = subnet[1]
267
- var trusted = ip
268
-
269
- if (kind !== subnetkind) {
270
- if (subnetkind === 'ipv4' && !ip.isIPv4MappedAddress()) {
271
- // Incompatible IP addresses
272
- continue
273
- }
274
-
275
- if (!ipconv) {
276
- // Convert IP to match subnet IP kind
277
- ipconv = subnetkind === 'ipv4'
278
- ? ip.toIPv4Address()
279
- : ip.toIPv4MappedAddress()
280
- }
281
-
282
- trusted = ipconv
283
- }
284
-
285
- if (trusted.match(subnetip, subnetrange)) {
286
- return true
287
- }
288
- }
289
-
290
- return false
291
- }
292
- }
293
-
294
- /**
295
- * Compile trust function for single subnet.
296
- *
297
- * @param {Object} subnet
298
- * @private
299
- */
300
-
301
- function trustSingle (subnet) {
302
- var subnetip = subnet[0]
303
- var subnetkind = subnetip.kind()
304
- var subnetisipv4 = subnetkind === 'ipv4'
305
- var subnetrange = subnet[1]
306
-
307
- return function trust (addr) {
308
- if (!isip(addr)) return false
309
-
310
- var ip = parseip(addr)
311
- var kind = ip.kind()
312
-
313
- if (kind !== subnetkind) {
314
- if (subnetisipv4 && !ip.isIPv4MappedAddress()) {
315
- // Incompatible IP addresses
316
- return false
317
- }
318
-
319
- // Convert IP to match subnet IP kind
320
- ip = subnetisipv4
321
- ? ip.toIPv4Address()
322
- : ip.toIPv4MappedAddress()
323
- }
324
-
325
- return ip.match(subnetip, subnetrange)
326
- }
327
- }
@@ -1,47 +0,0 @@
1
- {
2
- "name": "proxy-addr",
3
- "description": "Determine address of proxied request",
4
- "version": "2.0.7",
5
- "author": "Douglas Christopher Wilson <doug@somethingdoug.com>",
6
- "license": "MIT",
7
- "keywords": [
8
- "ip",
9
- "proxy",
10
- "x-forwarded-for"
11
- ],
12
- "repository": "jshttp/proxy-addr",
13
- "dependencies": {
14
- "forwarded": "0.2.0",
15
- "ipaddr.js": "1.9.1"
16
- },
17
- "devDependencies": {
18
- "benchmark": "2.1.4",
19
- "beautify-benchmark": "0.2.4",
20
- "deep-equal": "1.0.1",
21
- "eslint": "7.26.0",
22
- "eslint-config-standard": "14.1.1",
23
- "eslint-plugin-import": "2.23.4",
24
- "eslint-plugin-markdown": "2.2.0",
25
- "eslint-plugin-node": "11.1.0",
26
- "eslint-plugin-promise": "4.3.1",
27
- "eslint-plugin-standard": "4.1.0",
28
- "mocha": "8.4.0",
29
- "nyc": "15.1.0"
30
- },
31
- "files": [
32
- "LICENSE",
33
- "HISTORY.md",
34
- "README.md",
35
- "index.js"
36
- ],
37
- "engines": {
38
- "node": ">= 0.10"
39
- },
40
- "scripts": {
41
- "bench": "node benchmark/index.js",
42
- "lint": "eslint .",
43
- "test": "mocha --reporter spec --bail --check-leaks test/",
44
- "test-ci": "nyc --reporter=lcov --reporter=text npm test",
45
- "test-cov": "nyc --reporter=html --reporter=text npm test"
46
- }
47
- }
@@ -1,39 +0,0 @@
1
- root = true
2
-
3
- [*]
4
- indent_style = space
5
- indent_size = 4
6
- end_of_line = lf
7
- charset = utf-8
8
- trim_trailing_whitespace = true
9
- insert_final_newline = true
10
- max_line_length = 160
11
-
12
- [test/*]
13
- max_line_length = off
14
-
15
- [LICENSE.md]
16
- indent_size = off
17
-
18
- [*.md]
19
- max_line_length = off
20
-
21
- [*.json]
22
- max_line_length = off
23
-
24
- [Makefile]
25
- max_line_length = off
26
-
27
- [CHANGELOG.md]
28
- indent_style = space
29
- indent_size = 2
30
-
31
- [LICENSE]
32
- indent_size = 2
33
- max_line_length = off
34
-
35
- [coverage/**/*]
36
- indent_size = off
37
- indent_style = off
38
- indent = off
39
- max_line_length = off
@@ -1,2 +0,0 @@
1
- dist/
2
- coverage/
@@ -1,35 +0,0 @@
1
- {
2
- "root": true,
3
-
4
- "extends": "@ljharb",
5
-
6
- "rules": {
7
- "complexity": 0,
8
- "consistent-return": 1,
9
- "func-name-matching": 0,
10
- "id-length": [2, { "min": 1, "max": 25, "properties": "never" }],
11
- "indent": [2, 4],
12
- "max-lines-per-function": [2, { "max": 150 }],
13
- "max-params": [2, 14],
14
- "max-statements": [2, 52],
15
- "multiline-comment-style": 0,
16
- "no-continue": 1,
17
- "no-magic-numbers": 0,
18
- "no-restricted-syntax": [2, "BreakStatement", "DebuggerStatement", "ForInStatement", "LabeledStatement", "WithStatement"],
19
- "operator-linebreak": [2, "before"],
20
- },
21
-
22
- "overrides": [
23
- {
24
- "files": "test/**",
25
- "rules": {
26
- "function-paren-newline": 0,
27
- "max-lines-per-function": 0,
28
- "max-statements": 0,
29
- "no-buffer-constructor": 0,
30
- "no-extend-native": 0,
31
- "no-throw-literal": 0,
32
- }
33
- }
34
- ]
35
- }
@@ -1,12 +0,0 @@
1
- # These are supported funding model platforms
2
-
3
- github: [ljharb]
4
- patreon: # Replace with a single Patreon username
5
- open_collective: # Replace with a single Open Collective username
6
- ko_fi: # Replace with a single Ko-fi username
7
- tidelift: npm/qs
8
- community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
9
- liberapay: # Replace with a single Liberapay username
10
- issuehunt: # Replace with a single IssueHunt username
11
- otechie: # Replace with a single Otechie username
12
- custom: # Replace with a single custom sponsorship URL
@@ -1,13 +0,0 @@
1
- {
2
- "all": true,
3
- "check-coverage": false,
4
- "reporter": ["text-summary", "text", "html", "json"],
5
- "lines": 86,
6
- "statements": 85.93,
7
- "functions": 82.43,
8
- "branches": 76.06,
9
- "exclude": [
10
- "coverage",
11
- "dist"
12
- ]
13
- }