es6-crawler-detect 3.3.0 → 4.0.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (383) hide show
  1. package/.babelrc +1 -1
  2. package/.mocharc.json +5 -0
  3. package/README.md +1 -0
  4. package/dist/index.d.ts +5 -0
  5. package/dist/index.d.ts.map +1 -0
  6. package/dist/index.js +18 -0
  7. package/dist/index.js.map +1 -0
  8. package/dist/lib/crawler/crawlers.d.ts +5 -0
  9. package/dist/lib/crawler/crawlers.d.ts.map +1 -0
  10. package/dist/lib/crawler/crawlers.js +1400 -0
  11. package/dist/lib/crawler/crawlers.js.map +1 -0
  12. package/dist/lib/crawler/exclusions.d.ts +5 -0
  13. package/dist/lib/crawler/exclusions.d.ts.map +1 -0
  14. package/dist/lib/crawler/exclusions.js +59 -0
  15. package/dist/lib/crawler/exclusions.js.map +1 -0
  16. package/dist/lib/crawler/headers.d.ts +5 -0
  17. package/dist/lib/crawler/headers.d.ts.map +1 -0
  18. package/dist/lib/crawler/headers.js +21 -0
  19. package/dist/lib/crawler/headers.js.map +1 -0
  20. package/dist/lib/crawler/provider.d.ts +4 -0
  21. package/dist/lib/crawler/provider.d.ts.map +1 -0
  22. package/dist/lib/crawler/provider.js +3 -0
  23. package/dist/lib/crawler/provider.js.map +1 -0
  24. package/dist/lib/crawler.d.ts +21 -0
  25. package/dist/lib/crawler.d.ts.map +1 -0
  26. package/dist/lib/crawler.js +98 -0
  27. package/dist/lib/crawler.js.map +1 -0
  28. package/dist/types.d.ts +9 -0
  29. package/dist/types.d.ts.map +1 -0
  30. package/dist/types.js +3 -0
  31. package/dist/types.js.map +1 -0
  32. package/eslint.config.mjs +62 -0
  33. package/example/node/package.json +1 -1
  34. package/example/node/yarn.lock +222 -99
  35. package/package.json +30 -24
  36. package/src/index.ts +18 -0
  37. package/src/lib/crawler/{crawlers.js → crawlers.ts} +1397 -1403
  38. package/src/lib/crawler/{exclusions.js → exclusions.ts} +56 -62
  39. package/src/lib/crawler/{headers.js → headers.ts} +4 -10
  40. package/src/lib/crawler/provider.ts +3 -0
  41. package/src/lib/crawler.ts +128 -0
  42. package/src/types.ts +15 -0
  43. package/test/lib/crawler.spec.ts +116 -0
  44. package/tsconfig.json +28 -0
  45. package/.eslintignore +0 -6
  46. package/.eslintrc.json +0 -30
  47. package/dist/main.bundle.js +0 -1
  48. package/example/node/dist/crawler.min.js +0 -1
  49. package/example/node/node_modules/.bin/mime +0 -15
  50. package/example/node/node_modules/.bin/mime.cmd +0 -7
  51. package/example/node/node_modules/.yarn-integrity +0 -67
  52. package/example/node/node_modules/accepts/HISTORY.md +0 -236
  53. package/example/node/node_modules/accepts/LICENSE +0 -23
  54. package/example/node/node_modules/accepts/README.md +0 -142
  55. package/example/node/node_modules/accepts/index.js +0 -238
  56. package/example/node/node_modules/accepts/package.json +0 -47
  57. package/example/node/node_modules/array-flatten/LICENSE +0 -21
  58. package/example/node/node_modules/array-flatten/README.md +0 -43
  59. package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
  60. package/example/node/node_modules/array-flatten/package.json +0 -39
  61. package/example/node/node_modules/body-parser/HISTORY.md +0 -624
  62. package/example/node/node_modules/body-parser/LICENSE +0 -23
  63. package/example/node/node_modules/body-parser/README.md +0 -456
  64. package/example/node/node_modules/body-parser/index.js +0 -157
  65. package/example/node/node_modules/body-parser/lib/read.js +0 -181
  66. package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
  67. package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
  68. package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
  69. package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
  70. package/example/node/node_modules/body-parser/package.json +0 -52
  71. package/example/node/node_modules/bytes/History.md +0 -92
  72. package/example/node/node_modules/bytes/LICENSE +0 -23
  73. package/example/node/node_modules/bytes/Readme.md +0 -152
  74. package/example/node/node_modules/bytes/index.js +0 -166
  75. package/example/node/node_modules/bytes/package.json +0 -42
  76. package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
  77. package/example/node/node_modules/content-disposition/LICENSE +0 -22
  78. package/example/node/node_modules/content-disposition/README.md +0 -142
  79. package/example/node/node_modules/content-disposition/index.js +0 -458
  80. package/example/node/node_modules/content-disposition/package.json +0 -44
  81. package/example/node/node_modules/content-type/HISTORY.md +0 -24
  82. package/example/node/node_modules/content-type/LICENSE +0 -22
  83. package/example/node/node_modules/content-type/README.md +0 -92
  84. package/example/node/node_modules/content-type/index.js +0 -222
  85. package/example/node/node_modules/content-type/package.json +0 -40
  86. package/example/node/node_modules/cookie/HISTORY.md +0 -128
  87. package/example/node/node_modules/cookie/LICENSE +0 -24
  88. package/example/node/node_modules/cookie/README.md +0 -257
  89. package/example/node/node_modules/cookie/index.js +0 -202
  90. package/example/node/node_modules/cookie/package.json +0 -40
  91. package/example/node/node_modules/cookie-signature/History.md +0 -38
  92. package/example/node/node_modules/cookie-signature/Readme.md +0 -42
  93. package/example/node/node_modules/cookie-signature/index.js +0 -51
  94. package/example/node/node_modules/cookie-signature/package.json +0 -18
  95. package/example/node/node_modules/debug/.coveralls.yml +0 -1
  96. package/example/node/node_modules/debug/.eslintrc +0 -11
  97. package/example/node/node_modules/debug/.travis.yml +0 -14
  98. package/example/node/node_modules/debug/CHANGELOG.md +0 -362
  99. package/example/node/node_modules/debug/LICENSE +0 -19
  100. package/example/node/node_modules/debug/Makefile +0 -50
  101. package/example/node/node_modules/debug/README.md +0 -312
  102. package/example/node/node_modules/debug/component.json +0 -19
  103. package/example/node/node_modules/debug/karma.conf.js +0 -70
  104. package/example/node/node_modules/debug/node.js +0 -1
  105. package/example/node/node_modules/debug/package.json +0 -49
  106. package/example/node/node_modules/debug/src/browser.js +0 -185
  107. package/example/node/node_modules/debug/src/debug.js +0 -202
  108. package/example/node/node_modules/debug/src/index.js +0 -10
  109. package/example/node/node_modules/debug/src/inspector-log.js +0 -15
  110. package/example/node/node_modules/debug/src/node.js +0 -248
  111. package/example/node/node_modules/depd/History.md +0 -96
  112. package/example/node/node_modules/depd/LICENSE +0 -22
  113. package/example/node/node_modules/depd/Readme.md +0 -280
  114. package/example/node/node_modules/depd/index.js +0 -522
  115. package/example/node/node_modules/depd/lib/browser/index.js +0 -77
  116. package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
  117. package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
  118. package/example/node/node_modules/depd/lib/compat/index.js +0 -79
  119. package/example/node/node_modules/depd/package.json +0 -41
  120. package/example/node/node_modules/destroy/LICENSE +0 -22
  121. package/example/node/node_modules/destroy/README.md +0 -60
  122. package/example/node/node_modules/destroy/index.js +0 -75
  123. package/example/node/node_modules/destroy/package.json +0 -37
  124. package/example/node/node_modules/ee-first/LICENSE +0 -22
  125. package/example/node/node_modules/ee-first/README.md +0 -80
  126. package/example/node/node_modules/ee-first/index.js +0 -95
  127. package/example/node/node_modules/ee-first/package.json +0 -29
  128. package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
  129. package/example/node/node_modules/encodeurl/LICENSE +0 -22
  130. package/example/node/node_modules/encodeurl/README.md +0 -128
  131. package/example/node/node_modules/encodeurl/index.js +0 -60
  132. package/example/node/node_modules/encodeurl/package.json +0 -40
  133. package/example/node/node_modules/escape-html/LICENSE +0 -24
  134. package/example/node/node_modules/escape-html/Readme.md +0 -43
  135. package/example/node/node_modules/escape-html/index.js +0 -78
  136. package/example/node/node_modules/escape-html/package.json +0 -24
  137. package/example/node/node_modules/etag/HISTORY.md +0 -83
  138. package/example/node/node_modules/etag/LICENSE +0 -22
  139. package/example/node/node_modules/etag/README.md +0 -159
  140. package/example/node/node_modules/etag/index.js +0 -131
  141. package/example/node/node_modules/etag/package.json +0 -47
  142. package/example/node/node_modules/express/History.md +0 -3510
  143. package/example/node/node_modules/express/LICENSE +0 -24
  144. package/example/node/node_modules/express/Readme.md +0 -158
  145. package/example/node/node_modules/express/index.js +0 -11
  146. package/example/node/node_modules/express/lib/application.js +0 -644
  147. package/example/node/node_modules/express/lib/express.js +0 -116
  148. package/example/node/node_modules/express/lib/middleware/init.js +0 -43
  149. package/example/node/node_modules/express/lib/middleware/query.js +0 -47
  150. package/example/node/node_modules/express/lib/request.js +0 -525
  151. package/example/node/node_modules/express/lib/response.js +0 -1147
  152. package/example/node/node_modules/express/lib/router/index.js +0 -668
  153. package/example/node/node_modules/express/lib/router/layer.js +0 -181
  154. package/example/node/node_modules/express/lib/router/route.js +0 -216
  155. package/example/node/node_modules/express/lib/utils.js +0 -302
  156. package/example/node/node_modules/express/lib/view.js +0 -182
  157. package/example/node/node_modules/express/package.json +0 -99
  158. package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
  159. package/example/node/node_modules/finalhandler/LICENSE +0 -22
  160. package/example/node/node_modules/finalhandler/README.md +0 -148
  161. package/example/node/node_modules/finalhandler/index.js +0 -331
  162. package/example/node/node_modules/finalhandler/package.json +0 -45
  163. package/example/node/node_modules/forwarded/HISTORY.md +0 -21
  164. package/example/node/node_modules/forwarded/LICENSE +0 -22
  165. package/example/node/node_modules/forwarded/README.md +0 -57
  166. package/example/node/node_modules/forwarded/index.js +0 -90
  167. package/example/node/node_modules/forwarded/package.json +0 -45
  168. package/example/node/node_modules/fresh/HISTORY.md +0 -70
  169. package/example/node/node_modules/fresh/LICENSE +0 -23
  170. package/example/node/node_modules/fresh/README.md +0 -119
  171. package/example/node/node_modules/fresh/index.js +0 -137
  172. package/example/node/node_modules/fresh/package.json +0 -46
  173. package/example/node/node_modules/http-errors/HISTORY.md +0 -165
  174. package/example/node/node_modules/http-errors/LICENSE +0 -23
  175. package/example/node/node_modules/http-errors/README.md +0 -169
  176. package/example/node/node_modules/http-errors/index.js +0 -299
  177. package/example/node/node_modules/http-errors/package.json +0 -49
  178. package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
  179. package/example/node/node_modules/iconv-lite/LICENSE +0 -21
  180. package/example/node/node_modules/iconv-lite/README.md +0 -156
  181. package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
  182. package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
  183. package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
  184. package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
  185. package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
  186. package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
  187. package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
  188. package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
  189. package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
  190. package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
  191. package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
  192. package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
  193. package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
  194. package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
  195. package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
  196. package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
  197. package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
  198. package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
  199. package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
  200. package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
  201. package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
  202. package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
  203. package/example/node/node_modules/iconv-lite/package.json +0 -46
  204. package/example/node/node_modules/inherits/LICENSE +0 -16
  205. package/example/node/node_modules/inherits/README.md +0 -42
  206. package/example/node/node_modules/inherits/inherits.js +0 -9
  207. package/example/node/node_modules/inherits/inherits_browser.js +0 -27
  208. package/example/node/node_modules/inherits/package.json +0 -29
  209. package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
  210. package/example/node/node_modules/ipaddr.js/README.md +0 -233
  211. package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
  212. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
  213. package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
  214. package/example/node/node_modules/ipaddr.js/package.json +0 -35
  215. package/example/node/node_modules/media-typer/HISTORY.md +0 -22
  216. package/example/node/node_modules/media-typer/LICENSE +0 -22
  217. package/example/node/node_modules/media-typer/README.md +0 -81
  218. package/example/node/node_modules/media-typer/index.js +0 -270
  219. package/example/node/node_modules/media-typer/package.json +0 -26
  220. package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
  221. package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
  222. package/example/node/node_modules/merge-descriptors/README.md +0 -48
  223. package/example/node/node_modules/merge-descriptors/index.js +0 -60
  224. package/example/node/node_modules/merge-descriptors/package.json +0 -32
  225. package/example/node/node_modules/methods/HISTORY.md +0 -29
  226. package/example/node/node_modules/methods/LICENSE +0 -24
  227. package/example/node/node_modules/methods/README.md +0 -51
  228. package/example/node/node_modules/methods/index.js +0 -69
  229. package/example/node/node_modules/methods/package.json +0 -36
  230. package/example/node/node_modules/mime/CHANGELOG.md +0 -164
  231. package/example/node/node_modules/mime/LICENSE +0 -21
  232. package/example/node/node_modules/mime/README.md +0 -90
  233. package/example/node/node_modules/mime/cli.js +0 -8
  234. package/example/node/node_modules/mime/mime.js +0 -108
  235. package/example/node/node_modules/mime/package.json +0 -44
  236. package/example/node/node_modules/mime/src/build.js +0 -53
  237. package/example/node/node_modules/mime/src/test.js +0 -60
  238. package/example/node/node_modules/mime/types.json +0 -1
  239. package/example/node/node_modules/mime-db/HISTORY.md +0 -499
  240. package/example/node/node_modules/mime-db/LICENSE +0 -22
  241. package/example/node/node_modules/mime-db/README.md +0 -100
  242. package/example/node/node_modules/mime-db/db.json +0 -8471
  243. package/example/node/node_modules/mime-db/index.js +0 -11
  244. package/example/node/node_modules/mime-db/package.json +0 -59
  245. package/example/node/node_modules/mime-types/HISTORY.md +0 -388
  246. package/example/node/node_modules/mime-types/LICENSE +0 -23
  247. package/example/node/node_modules/mime-types/README.md +0 -113
  248. package/example/node/node_modules/mime-types/index.js +0 -188
  249. package/example/node/node_modules/mime-types/package.json +0 -44
  250. package/example/node/node_modules/ms/index.js +0 -152
  251. package/example/node/node_modules/ms/license.md +0 -21
  252. package/example/node/node_modules/ms/package.json +0 -37
  253. package/example/node/node_modules/ms/readme.md +0 -51
  254. package/example/node/node_modules/negotiator/HISTORY.md +0 -103
  255. package/example/node/node_modules/negotiator/LICENSE +0 -24
  256. package/example/node/node_modules/negotiator/README.md +0 -203
  257. package/example/node/node_modules/negotiator/index.js +0 -124
  258. package/example/node/node_modules/negotiator/lib/charset.js +0 -169
  259. package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
  260. package/example/node/node_modules/negotiator/lib/language.js +0 -179
  261. package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
  262. package/example/node/node_modules/negotiator/package.json +0 -42
  263. package/example/node/node_modules/on-finished/HISTORY.md +0 -88
  264. package/example/node/node_modules/on-finished/LICENSE +0 -23
  265. package/example/node/node_modules/on-finished/README.md +0 -154
  266. package/example/node/node_modules/on-finished/index.js +0 -196
  267. package/example/node/node_modules/on-finished/package.json +0 -31
  268. package/example/node/node_modules/parseurl/HISTORY.md +0 -58
  269. package/example/node/node_modules/parseurl/LICENSE +0 -24
  270. package/example/node/node_modules/parseurl/README.md +0 -133
  271. package/example/node/node_modules/parseurl/index.js +0 -158
  272. package/example/node/node_modules/parseurl/package.json +0 -40
  273. package/example/node/node_modules/path-to-regexp/History.md +0 -36
  274. package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
  275. package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
  276. package/example/node/node_modules/path-to-regexp/index.js +0 -129
  277. package/example/node/node_modules/path-to-regexp/package.json +0 -30
  278. package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
  279. package/example/node/node_modules/proxy-addr/LICENSE +0 -22
  280. package/example/node/node_modules/proxy-addr/README.md +0 -139
  281. package/example/node/node_modules/proxy-addr/index.js +0 -327
  282. package/example/node/node_modules/proxy-addr/package.json +0 -47
  283. package/example/node/node_modules/qs/.editorconfig +0 -39
  284. package/example/node/node_modules/qs/.eslintignore +0 -2
  285. package/example/node/node_modules/qs/.eslintrc +0 -35
  286. package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
  287. package/example/node/node_modules/qs/.nycrc +0 -13
  288. package/example/node/node_modules/qs/CHANGELOG.md +0 -359
  289. package/example/node/node_modules/qs/LICENSE.md +0 -29
  290. package/example/node/node_modules/qs/README.md +0 -604
  291. package/example/node/node_modules/qs/dist/qs.js +0 -832
  292. package/example/node/node_modules/qs/lib/formats.js +0 -23
  293. package/example/node/node_modules/qs/lib/index.js +0 -11
  294. package/example/node/node_modules/qs/lib/parse.js +0 -257
  295. package/example/node/node_modules/qs/lib/stringify.js +0 -278
  296. package/example/node/node_modules/qs/lib/utils.js +0 -251
  297. package/example/node/node_modules/qs/package.json +0 -69
  298. package/example/node/node_modules/qs/test/parse.js +0 -772
  299. package/example/node/node_modules/qs/test/stringify.js +0 -793
  300. package/example/node/node_modules/qs/test/utils.js +0 -136
  301. package/example/node/node_modules/range-parser/HISTORY.md +0 -56
  302. package/example/node/node_modules/range-parser/LICENSE +0 -23
  303. package/example/node/node_modules/range-parser/README.md +0 -84
  304. package/example/node/node_modules/range-parser/index.js +0 -162
  305. package/example/node/node_modules/range-parser/package.json +0 -44
  306. package/example/node/node_modules/raw-body/HISTORY.md +0 -284
  307. package/example/node/node_modules/raw-body/LICENSE +0 -22
  308. package/example/node/node_modules/raw-body/README.md +0 -217
  309. package/example/node/node_modules/raw-body/index.d.ts +0 -87
  310. package/example/node/node_modules/raw-body/index.js +0 -286
  311. package/example/node/node_modules/raw-body/package.json +0 -48
  312. package/example/node/node_modules/safe-buffer/LICENSE +0 -21
  313. package/example/node/node_modules/safe-buffer/README.md +0 -584
  314. package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
  315. package/example/node/node_modules/safe-buffer/index.js +0 -65
  316. package/example/node/node_modules/safe-buffer/package.json +0 -51
  317. package/example/node/node_modules/safer-buffer/LICENSE +0 -21
  318. package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
  319. package/example/node/node_modules/safer-buffer/Readme.md +0 -156
  320. package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
  321. package/example/node/node_modules/safer-buffer/package.json +0 -34
  322. package/example/node/node_modules/safer-buffer/safer.js +0 -77
  323. package/example/node/node_modules/safer-buffer/tests.js +0 -406
  324. package/example/node/node_modules/send/HISTORY.md +0 -506
  325. package/example/node/node_modules/send/LICENSE +0 -23
  326. package/example/node/node_modules/send/README.md +0 -327
  327. package/example/node/node_modules/send/index.js +0 -1133
  328. package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
  329. package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
  330. package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
  331. package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
  332. package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
  333. package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
  334. package/example/node/node_modules/send/package.json +0 -61
  335. package/example/node/node_modules/serve-static/HISTORY.md +0 -459
  336. package/example/node/node_modules/serve-static/LICENSE +0 -25
  337. package/example/node/node_modules/serve-static/README.md +0 -257
  338. package/example/node/node_modules/serve-static/index.js +0 -210
  339. package/example/node/node_modules/serve-static/package.json +0 -42
  340. package/example/node/node_modules/setprototypeof/LICENSE +0 -13
  341. package/example/node/node_modules/setprototypeof/README.md +0 -31
  342. package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
  343. package/example/node/node_modules/setprototypeof/index.js +0 -17
  344. package/example/node/node_modules/setprototypeof/package.json +0 -38
  345. package/example/node/node_modules/setprototypeof/test/index.js +0 -24
  346. package/example/node/node_modules/statuses/HISTORY.md +0 -65
  347. package/example/node/node_modules/statuses/LICENSE +0 -23
  348. package/example/node/node_modules/statuses/README.md +0 -127
  349. package/example/node/node_modules/statuses/codes.json +0 -66
  350. package/example/node/node_modules/statuses/index.js +0 -113
  351. package/example/node/node_modules/statuses/package.json +0 -48
  352. package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
  353. package/example/node/node_modules/toidentifier/LICENSE +0 -21
  354. package/example/node/node_modules/toidentifier/README.md +0 -61
  355. package/example/node/node_modules/toidentifier/index.js +0 -32
  356. package/example/node/node_modules/toidentifier/package.json +0 -38
  357. package/example/node/node_modules/type-is/HISTORY.md +0 -259
  358. package/example/node/node_modules/type-is/LICENSE +0 -23
  359. package/example/node/node_modules/type-is/README.md +0 -170
  360. package/example/node/node_modules/type-is/index.js +0 -266
  361. package/example/node/node_modules/type-is/package.json +0 -45
  362. package/example/node/node_modules/unpipe/HISTORY.md +0 -4
  363. package/example/node/node_modules/unpipe/LICENSE +0 -22
  364. package/example/node/node_modules/unpipe/README.md +0 -43
  365. package/example/node/node_modules/unpipe/index.js +0 -69
  366. package/example/node/node_modules/unpipe/package.json +0 -27
  367. package/example/node/node_modules/utils-merge/LICENSE +0 -20
  368. package/example/node/node_modules/utils-merge/README.md +0 -34
  369. package/example/node/node_modules/utils-merge/index.js +0 -23
  370. package/example/node/node_modules/utils-merge/package.json +0 -40
  371. package/example/node/node_modules/vary/HISTORY.md +0 -39
  372. package/example/node/node_modules/vary/LICENSE +0 -22
  373. package/example/node/node_modules/vary/README.md +0 -101
  374. package/example/node/node_modules/vary/index.js +0 -149
  375. package/example/node/node_modules/vary/package.json +0 -43
  376. package/example/node/server.js +0 -38
  377. package/src/index.js +0 -16
  378. package/src/lib/crawler/provider.js +0 -11
  379. package/src/lib/crawler.js +0 -130
  380. package/test/lib/crawler.test.js +0 -94
  381. package/webpack.common.js +0 -20
  382. package/webpack.dev.js +0 -7
  383. package/webpack.prod.js +0 -14
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crawlers.js","sourceRoot":"","sources":["../../../src/lib/crawler/crawlers.ts"],"names":[],"mappings":";;;AAEA,MAAa,QAAQ;IACZ,MAAM;QACX,OAAO;YACL,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,sCAAsC;YACtC,OAAO;YACP,YAAY;YACZ,eAAe;YACf,aAAa;YACb,QAAQ;YACR,WAAW;YACX,UAAU;YACV,OAAO;YACP,iBAAiB;YACjB,UAAU;YACV,OAAO;YACP,SAAS;YACT,WAAW;YACX,UAAU;YACV,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,wBAAwB;YACxB,cAAc;YACd,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,SAAS;YACT,SAAS;YACT,gBAAgB;YAChB,SAAS;YACT,SAAS;YACT,UAAU;YACV,SAAS;YACT,oBAAoB;YACpB,SAAS;YACT,cAAc;YACd,WAAW;YACX,OAAO;YACP,cAAc;YACd,kBAAkB;YAClB,cAAc;YACd,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,UAAU;YACV,SAAS;YACT,QAAQ;YACR,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kBAAkB;YAClB,uBAAuB;YACvB,OAAO;YACP,0BAA0B;YAC1B,UAAU;YACV,WAAW;YACX,QAAQ;YACR,SAAS;YACT,SAAS;YACT,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,WAAW;YACX,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,kBAAkB;YAClB,kBAAkB;YAClB,sBAAsB;YACtB,WAAW;YACX,KAAK;YACL,OAAO;YACP,YAAY;YACZ,SAAS;YACT,cAAc;YACd,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,+BAA+B;YAC/B,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,QAAQ;YACR,UAAU;YACV,wBAAwB;YACxB,SAAS;YACT,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,eAAe;YACf,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,cAAc;YACd,mBAAmB;YACnB,wBAAwB;YACxB,mBAAmB;YACnB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,aAAa;YACb,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,kBAAkB;YAClB,cAAc;YACd,SAAS;YACT,WAAW;YACX,eAAe;YACf,OAAO;YACP,WAAW;YACX,UAAU;YACV,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,WAAW;YACX,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,YAAY;YACZ,sBAAsB;YACtB,UAAU;YACV,gBAAgB;YAChB,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,kBAAkB;YAClB,QAAQ;YACR,aAAa;YACb,QAAQ;YACR,YAAY;YACZ,UAAU;YACV,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,sBAAsB;YACtB,YAAY;YACZ,SAAS;YACT,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,QAAQ;YACR,mBAAmB;YACnB,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,0BAA0B;YAC1B,YAAY;YACZ,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,WAAW;YACX,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,oBAAoB;YACpB,oBAAoB;YACpB,oBAAoB;YACpB,cAAc;YACd,aAAa;YACb,YAAY;YACZ,WAAW;YACX,cAAc;YACd,QAAQ;YACR,SAAS;YACT,OAAO;YACP,WAAW;YACX,UAAU;YACV,eAAe;YACf,aAAa;YACb,cAAc;YACd,UAAU;YACV,cAAc;YACd,SAAS;YACT,UAAU;YACV,eAAe;YACf,WAAW;YACX,eAAe;YACf,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,4BAA4B;YAC5B,iBAAiB;YACjB,6BAA6B;YAC7B,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,cAAc;YACd,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,WAAW;YACX,UAAU;YACV,aAAa;YACb,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kCAAkC;YAClC,oBAAoB;YACpB,QAAQ;YACR,kBAAkB;YAClB,YAAY;YACZ,sBAAsB;YACtB,eAAe;YACf,aAAa;YACb,yBAAyB;YACzB,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,YAAY;YACZ,SAAS;YACT,WAAW;YACX,oBAAoB;YACpB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB;YAChB,gCAAgC;YAChC,SAAS;YACT,eAAe;YACf,mBAAmB;YACnB,gBAAgB;YAChB,WAAW;YACX,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,UAAU;YACV,SAAS;YACT,MAAM;YACN,gBAAgB;YAChB,MAAM;YACN,YAAY;YACZ,YAAY;YACZ,eAAe;YACf,WAAW;YACX,sBAAsB;YACtB,WAAW;YACX,eAAe;YACf,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,sBAAsB;YACtB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,OAAO;YACP,OAAO;YACP,oDAAoD;YACpD,OAAO;YACP,MAAM;YACN,WAAW;YACX,eAAe;YACf,WAAW;YACX,wBAAwB;YACxB,aAAa;YACb,WAAW;YACX,aAAa;YACb,MAAM;YACN,WAAW;YACX,kBAAkB;YAClB,SAAS;YACT,qBAAqB;YACrB,gBAAgB;YAChB,YAAY;YACZ,oBAAoB;YACpB,yBAAyB;YACzB,6BAA6B;YAC7B,aAAa;YACb,yBAAyB;YACzB,iBAAiB;YACjB,cAAc;YACd,kBAAkB;YAClB,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,oBAAoB;YACpB,UAAU;YACV,YAAY;YACZ,QAAQ;YACR,WAAW;YACX,eAAe;YACf,gBAAgB;YAChB,QAAQ;YACR,MAAM;YACN,aAAa;YACb,MAAM;YACN,YAAY;YACZ,cAAc;YACd,SAAS;YACT,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,aAAa;YACb,mBAAmB;YACnB,SAAS;YACT,SAAS;YACT,oBAAoB;YACpB,WAAW;YACX,yBAAyB;YACzB,wBAAwB;YACxB,QAAQ;YACR,SAAS;YACT,aAAa;YACb,kBAAkB;YAClB,UAAU;YACV,MAAM;YACN,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,wBAAwB;YACxB,UAAU;YACV,QAAQ;YACR,qBAAqB;YACrB,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,oBAAoB;YACpB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,eAAe;YACf,aAAa;YACb,SAAS;YACT,aAAa;YACb,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,aAAa;YACb,UAAU;YACV,eAAe;YACf,aAAa;YACb,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,eAAe;YACf,eAAe;YACf,QAAQ;YACR,eAAe;YACf,WAAW;YACX,OAAO;YACP,UAAU;YACV,cAAc;YACd,UAAU;YACV,uBAAuB;YACvB,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,UAAU;YACV,cAAc;YACd,iBAAiB;YACjB,cAAc;YACd,UAAU;YACV,YAAY;YACZ,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,cAAc;YACd,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,eAAe;YACf,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;YACV,aAAa;YACb,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,2BAA2B;YAC3B,UAAU;YACV,iBAAiB;YACjB,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,mBAAmB;YACnB,gBAAgB;YAChB,gBAAgB;YAChB,2BAA2B;YAC3B,qBAAqB;YACrB,4BAA4B;YAC5B,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,gCAAgC;YAChC,qBAAqB;YACrB,gBAAgB;YAChB,oBAAoB;YACpB,0BAA0B;YAC1B,yBAAyB;YACzB,yBAAyB;YACzB,gBAAgB;YAChB,yBAAyB;YACzB,mBAAmB;YACnB,sBAAsB;YACtB,0BAA0B;YAC1B,iBAAiB;YACjB,eAAe;YACf,qCAAqC;YACrC,4BAA4B;YAC5B,gBAAgB;YAChB,sBAAsB;YACtB,YAAY;YACZ,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,uBAAuB;YACvB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,WAAW;YACX,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,SAAS;YACT,WAAW;YACX,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,YAAY;YACZ,SAAS;YACT,2BAA2B;YAC3B,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,eAAe;YACf,oBAAoB;YACpB,qBAAqB;YACrB,UAAU;YACV,WAAW;YACX,YAAY;YACZ,UAAU;YACV,kBAAkB;YAClB,SAAS;YACT,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,uBAAuB;YACvB,mBAAmB;YACnB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,OAAO;YACP,YAAY;YACZ,SAAS;YACT,uBAAuB;YACvB,UAAU;YACV,qBAAqB;YACrB,UAAU;YACV,iBAAiB;YACjB,WAAW;YACX,YAAY;YACZ,4BAA4B;YAC5B,gBAAgB;YAChB,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,uBAAuB;YACvB,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,uBAAuB;YACvB,YAAY;YACZ,eAAe;YACf,OAAO;YACP,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,4BAA4B;YAC5B,YAAY;YACZ,UAAU;YACV,YAAY;YACZ,eAAe;YACf,aAAa;YACb,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,cAAc;YACd,SAAS;YACT,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,6BAA6B;YAC7B,cAAc;YACd,UAAU;YACV,YAAY;YACZ,WAAW;YACX,aAAa;YACb,aAAa;YACb,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,uBAAuB;YACvB,eAAe;YACf,kBAAkB;YAClB,YAAY;YACZ,MAAM;YACN,KAAK;YACL,SAAS;YACT,WAAW;YACX,aAAa;YACb,4BAA4B;YAC5B,SAAS;YACT,MAAM;YACN,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,QAAQ;YACR,UAAU;YACV,OAAO;YACP,WAAW;YACX,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,MAAM;YACN,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,UAAU;YACV,iCAAiC;YACjC,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,OAAO;YACP,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;YACV,aAAa;YACb,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,OAAO;YACP,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,WAAW;YACX,SAAS;YACT,cAAc;YACd,aAAa;YACb,UAAU;YACV,aAAa;YACb,UAAU;YACV,cAAc;YACd,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,iBAAiB;YACjB,qBAAqB;YACrB,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,aAAa;YACb,mBAAmB;YACnB,OAAO;YACP,gBAAgB;YAChB,yBAAyB;YACzB,kBAAkB;YAClB,aAAa;YACb,aAAa;YACb,aAAa;YACb,OAAO;YACP,UAAU;YACV,WAAW;YACX,YAAY;YACZ,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,sBAAsB;YACtB,YAAY;YACZ,gBAAgB;YAChB,eAAe;YACf,aAAa;YACb,UAAU;YACV,aAAa;YACb,SAAS;YACT,iBAAiB;YACjB,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,mBAAmB;YACnB,uCAAuC;YACvC,4BAA4B;YAC5B,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,iBAAiB;YACjB,oBAAoB;YACpB,SAAS;YACT,aAAa;YACb,UAAU;YACV,QAAQ;YACR,wBAAwB;YACxB,SAAS;YACT,eAAe;YACf,UAAU;YACV,WAAW;YACX,UAAU;YACV,UAAU;YACV,yBAAyB;YACzB,eAAe;YACf,aAAa;YACb,QAAQ;YACR,UAAU;YACV,kBAAkB;YAClB,iCAAiC;YACjC,aAAa;YACb,QAAQ;YACR,aAAa;YACb,cAAc;YACd,WAAW;YACX,cAAc;YACd,eAAe;YACf,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,cAAc;YACd,aAAa;YACb,kBAAkB;YAClB,SAAS;YACT,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,SAAS;YACT,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,aAAa;YACb,wBAAwB;YACxB,YAAY;YACZ,oBAAoB;YACpB,QAAQ;YACR,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,WAAW;YACX,QAAQ;YACR,cAAc;YACd,qBAAqB;YACrB,WAAW;YACX,SAAS;YACT,SAAS;YACT,WAAW;YACX,WAAW;YACX,OAAO;YACP,iBAAiB;YACjB,kBAAkB;YAClB,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,UAAU;YACV,WAAW;YACX,oBAAoB;YACpB,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,MAAM;YACN,OAAO;YACP,QAAQ;YACR,iBAAiB;YACjB,UAAU;YACV,SAAS;YACT,KAAK;YACL,gBAAgB;YAChB,WAAW;YACX,SAAS;YACT,MAAM;YACN,kBAAkB;YAClB,mBAAmB;YACnB,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,MAAM;YACN,qBAAqB;YACrB,iBAAiB;YACjB,yBAAyB;YACzB,UAAU;YACV,kBAAkB;YAClB,aAAa;YACb,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,aAAa;YACb,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,OAAO;YACP,OAAO;YACP,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,aAAa;YACb,YAAY;YACZ,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,YAAY;YACZ,cAAc;YACd,WAAW;YACX,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,MAAM;YACN,WAAW;YACX,SAAS;YACT,wBAAwB;YACxB,cAAc;YACd,WAAW;YACX,cAAc;YACd,cAAc;YACd,YAAY;YACZ,UAAU;YACV,WAAW;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,gBAAgB;YAChB,SAAS;YACT,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,iBAAiB;YACjB,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,UAAU;YACV,mBAAmB;YACnB,SAAS;YACT,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,QAAQ;YACR,kBAAkB;YAClB,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,cAAc;YACd,SAAS;YACT,0BAA0B;YAC1B,gBAAgB;YAChB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,QAAQ;YACR,WAAW;YACX,8BAA8B;YAC9B,eAAe;YACf,aAAa;YACb,OAAO;YACP,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,aAAa;YACb,WAAW;YACX,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,uBAAuB;YACvB,OAAO;YACP,eAAe;YACf,MAAM;YACN,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,iBAAiB;YACjB,UAAU;YACV,oBAAoB;YACpB,UAAU;YACV,SAAS;YACT,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,sBAAsB;YACtB,cAAc;YACd,cAAc;YACd,cAAc;YACd,aAAa;YACb,cAAc;YACd,yBAAyB;YACzB,YAAY;YACZ,UAAU;YACV,cAAc;YACd,YAAY;YACZ,cAAc;YACd,WAAW;YACX,OAAO;YACP,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,SAAS;YACT,UAAU;YACV,aAAa;YACb,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,WAAW;YACX,QAAQ;YACR,aAAa;YACb,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,kCAAkC;YAClC,UAAU;YACV,QAAQ;YACR,eAAe;YACf,aAAa;YACb,sBAAsB;YACtB,WAAW;YACX,YAAY;YACZ,OAAO;YACP,SAAS;YACT,4BAA4B;YAC5B,UAAU;YACV,iBAAiB;YACjB,wBAAwB;YACxB,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,SAAS;YACT,aAAa;YACb,UAAU;YACV,2BAA2B;YAC3B,cAAc;YACd,eAAe;YACf,aAAa;YACb,UAAU;YACV,uBAAuB;YACvB,QAAQ;YACR,mBAAmB;YACnB,eAAe;YACf,uBAAuB;YACvB,SAAS;YACT,cAAc;YACd,WAAW;YACX,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,WAAW;YACX,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,UAAU;YACV,oBAAoB;YACpB,qBAAqB;YACrB,cAAc;YACd,SAAS;YACT,OAAO;YACP,WAAW;YACX,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,UAAU;YACV,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,aAAa;YACb,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,aAAa;YACb,iBAAiB;YACjB,aAAa;YACb,YAAY;YACZ,WAAW;YACX,WAAW;YACX,cAAc;YACd,iBAAiB;YACjB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,OAAO;YACP,WAAW;YACX,eAAe;YACf,mBAAmB;YACnB,eAAe;YACf,OAAO;YACP,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,UAAU;YACV,UAAU;YACV,gBAAgB;YAChB,YAAY;YACZ,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,WAAW;YACX,eAAe;YACf,QAAQ;YACR,OAAO;YACP,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,UAAU;YACV,WAAW;YACX,cAAc;YACd,eAAe;YACf,aAAa;YACb,YAAY;YACZ,SAAS;YACT,iBAAiB;YACjB,WAAW;YACX,aAAa;YACb,YAAY;YACZ,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,WAAW;YACX,gBAAgB;YAChB,SAAS;YACT,OAAO;YACP,oBAAoB;YACpB,qBAAqB;YACrB,YAAY;YACZ,eAAe;YACf,qBAAqB;YACrB,SAAS;YACT,SAAS;YACT,SAAS;YACT,aAAa;YACb,SAAS;YACT,cAAc;YACd,aAAa;YACb,cAAc;YACd,eAAe;YACf,mBAAmB;YACnB,aAAa;YACb,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,SAAS;YACT,aAAa;YACb,sBAAsB;YACtB,iBAAiB;YACjB,+BAA+B;YAC/B,kBAAkB;YAClB,kBAAkB;YAClB,kBAAkB;YAClB,UAAU;YACV,UAAU;YACV,SAAS;YACT,YAAY;YACZ,aAAa;YACb,cAAc;YACd,QAAQ;YACR,eAAe;YACf,aAAa;YACb,OAAO;YACP,SAAS;YACT,cAAc;YACd,eAAe;YACf,aAAa;YACb,WAAW;YACX,aAAa;YACb,aAAa;YACb,WAAW;YACX,aAAa;YACb,UAAU;YACV,UAAU;YACV,YAAY;YACZ,UAAU;YACV,UAAU;YACV,cAAc;YACd,cAAc;YACd,aAAa;YACb,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;YACT,oBAAoB;YACpB,UAAU;YACV,0BAA0B;YAC1B,WAAW;YACX,WAAW;YACX,yBAAyB;YACzB,SAAS;YACT,cAAc;YACd,qBAAqB;YACrB,aAAa;YACb,gBAAgB;YAChB,QAAQ;YACR,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,QAAQ;YACR,SAAS;YACT,cAAc;YACd,WAAW;YACX,SAAS;YACT,aAAa;YACb,SAAS;YACT,WAAW;YACX,2BAA2B;YAC3B,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,WAAW;YACX,YAAY;YACZ,OAAO;YACP,KAAK;YACL,kCAAkC;YAClC,UAAU;YACV,aAAa;YACb,SAAS;YACT,SAAS;YACT,MAAM;YACN,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,MAAM;YACN,YAAY;YACZ,eAAe;YACf,cAAc;YACd,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,cAAc;YACd,qBAAqB;YACrB,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,UAAU;YACV,UAAU;YACV,aAAa;YACb,cAAc;YACd,WAAW;YACX,UAAU;YACV,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,SAAS;YACT,aAAa;YACb,SAAS;YACT,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,UAAU;YACV,mBAAmB;YACnB,WAAW;YACX,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,QAAQ;YACR,WAAW;YACX,WAAW;YACX,aAAa;YACb,SAAS;YACT,SAAS;YACT,iBAAiB;YACjB,oBAAoB;YACpB,gBAAgB;YAChB,UAAU;YACV,YAAY;YACZ,SAAS;YACT,aAAa;YACb,WAAW;YACX,YAAY;YACZ,cAAc;YACd,aAAa;YACb,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,MAAM;YACN,OAAO;YACP,MAAM;YACN,UAAU;YACV,MAAM;YACN,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,WAAW;YACX,iBAAiB;YACjB,gBAAgB;YAChB,sBAAsB;YACtB,2BAA2B;YAC3B,YAAY;YACZ,WAAW;YACX,UAAU;YACV,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,kBAAkB;YAClB,SAAS;YACT,eAAe;YACf,QAAQ;YACR,mCAAmC;YACnC,WAAW;YACX,MAAM;YACN,sBAAsB;YACtB,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,gBAAgB;YAChB,UAAU;YACV,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,0BAA0B;YAC1B,YAAY;YACZ,kBAAkB;YAClB,yBAAyB;YACzB,sBAAsB;YACtB,qBAAqB;YACrB,OAAO;YACP,MAAM;YACN,oBAAoB;YACpB,kBAAkB;YAClB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,OAAO;YACP,gBAAgB;YAChB,cAAc;YACd,kBAAkB;YAClB,eAAe;YACf,oBAAoB;YACpB,cAAc;YACd,YAAY;YACZ,SAAS;YACT,QAAQ;YACR,MAAM;YACN,KAAK;YACL,OAAO;YACP,oBAAoB;YACpB,wBAAwB;YACxB,kBAAkB;YAClB,SAAS;YACT,OAAO;YACP,OAAO;YACP,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,yBAAyB;YACzB,QAAQ;YACR,wIAAwI;SACzI,CAAC;IACJ,CAAC;CACF;AAl3CD,4BAk3CC"}
@@ -0,0 +1,5 @@
1
+ import { Provider } from './provider';
2
+ export declare class Exclusions implements Provider {
3
+ getAll(): string[];
4
+ }
5
+ //# sourceMappingURL=exclusions.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"exclusions.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,UAAW,YAAW,QAAQ;IACzC,MAAM,IAAI,MAAM,EAAE;CAoDnB"}
@@ -0,0 +1,59 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Exclusions = void 0;
4
+ class Exclusions {
5
+ getAll() {
6
+ return [
7
+ 'Safari.[\\d\\.]*',
8
+ 'Firefox.[\\d\\.]*',
9
+ ' Chrome.[\\d\\.]*',
10
+ 'Chromium.[\\d\\.]*',
11
+ 'MSIE.[\\d\\.]',
12
+ 'Opera\\/[\\d\\.]*',
13
+ 'Mozilla.[\\d\\.]*',
14
+ 'AppleWebKit.[\\d\\.]*',
15
+ 'Trident.[\\d\\.]*',
16
+ 'Windows NT.[\\d\\.]*',
17
+ 'Android [\\d\\.]*',
18
+ 'Macintosh.',
19
+ 'Ubuntu',
20
+ 'Linux',
21
+ '[ ]Intel',
22
+ 'Mac OS X [\\d_]*',
23
+ '(like )?Gecko(.[\\d\\.]*)?',
24
+ 'KHTML,',
25
+ 'CriOS.[\\d\\.]*',
26
+ 'CPU iPhone OS ([0-9_])* like Mac OS X',
27
+ 'CPU OS ([0-9_])* like Mac OS X',
28
+ 'iPod',
29
+ 'compatible',
30
+ 'x86_..',
31
+ 'i686',
32
+ 'x64',
33
+ 'X11',
34
+ 'rv:[\\d\\.]*',
35
+ 'Version.[\\d\\.]*',
36
+ 'WOW64',
37
+ 'Win64',
38
+ 'Dalvik.[\\d\\.]*',
39
+ ' \\.NET CLR [\\d\\.]*',
40
+ 'Presto.[\\d\\.]*',
41
+ 'Media Center PC',
42
+ 'BlackBerry',
43
+ 'Build',
44
+ 'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
45
+ 'Opera',
46
+ ' \\.NET[\\d\\.]*',
47
+ 'cubot',
48
+ '; M bot',
49
+ '; CRONO',
50
+ '; B bot',
51
+ '; IDbot',
52
+ '; ID bot',
53
+ '; POWER BOT',
54
+ 'OCTOPUS-CORE',
55
+ ];
56
+ }
57
+ }
58
+ exports.Exclusions = Exclusions;
59
+ //# sourceMappingURL=exclusions.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"exclusions.js","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":";;;AAEA,MAAa,UAAU;IACrB,MAAM;QACJ,OAAO;YACL,kBAAkB;YAClB,mBAAmB;YACnB,mBAAmB;YACnB,oBAAoB;YACpB,eAAe;YACf,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,sBAAsB;YACtB,mBAAmB;YACnB,YAAY;YACZ,QAAQ;YACR,OAAO;YACP,UAAU;YACV,kBAAkB;YAClB,4BAA4B;YAC5B,QAAQ;YACR,iBAAiB;YACjB,uCAAuC;YACvC,gCAAgC;YAChC,MAAM;YACN,YAAY;YACZ,QAAQ;YACR,MAAM;YACN,KAAK;YACL,KAAK;YACL,cAAc;YACd,mBAAmB;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,kBAAkB;YAClB,iBAAiB;YACjB,YAAY;YACZ,OAAO;YACP,4DAA4D;YAC5D,OAAO;YACP,kBAAkB;YAClB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,UAAU;YACV,aAAa;YACb,cAAc;SACf,CAAC;IACJ,CAAC;CACF;AArDD,gCAqDC"}
@@ -0,0 +1,5 @@
1
+ import { Provider } from './provider';
2
+ export declare class Headers implements Provider {
3
+ getAll(): string[];
4
+ }
5
+ //# sourceMappingURL=headers.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,OAAQ,YAAW,QAAQ;IACtC,MAAM,IAAI,MAAM,EAAE;CAcnB"}
@@ -0,0 +1,21 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Headers = void 0;
4
+ class Headers {
5
+ getAll() {
6
+ return [
7
+ 'USER-AGENT',
8
+ 'X-OPERAMINI-PHONE-UA',
9
+ 'X-DEVICE-USER-AGENT',
10
+ 'X-ORIGINAL-USER-AGENT',
11
+ 'X-SKYFIRE-PHONE',
12
+ 'X-BOLT-PHONE-UA',
13
+ 'DEVICE-STOCK-UA',
14
+ 'X-UCBROWSER-DEVICE-UA',
15
+ 'FROM',
16
+ 'X-SCANNER',
17
+ ];
18
+ }
19
+ }
20
+ exports.Headers = Headers;
21
+ //# sourceMappingURL=headers.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":";;;AAEA,MAAa,OAAO;IAClB,MAAM;QACJ,OAAO;YACL,YAAY;YACZ,sBAAsB;YACtB,qBAAqB;YACrB,uBAAuB;YACvB,iBAAiB;YACjB,iBAAiB;YACjB,iBAAiB;YACjB,uBAAuB;YACvB,MAAM;YACN,WAAW;SACZ,CAAC;IACJ,CAAC;CACF;AAfD,0BAeC"}
@@ -0,0 +1,4 @@
1
+ export interface Provider {
2
+ getAll(): string[];
3
+ }
4
+ //# sourceMappingURL=provider.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,MAAM,IAAI,MAAM,EAAE,CAAC;CACpB"}
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=provider.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":""}
@@ -0,0 +1,21 @@
1
+ import { Request } from 'express-serve-static-core';
2
+ import { IncomingHttpHeaders } from 'http2';
3
+ export declare class Crawler {
4
+ private crawlers;
5
+ private headers;
6
+ private exclusions;
7
+ private request;
8
+ private compiledRegexList;
9
+ private compiledExclusions;
10
+ private httpHeaders;
11
+ private userAgent;
12
+ private matches?;
13
+ constructor(request?: Request, headers?: IncomingHttpHeaders, userAgent?: string);
14
+ compileRegex(patterns: string[], flags?: string): RegExp;
15
+ private setHttpHeaders;
16
+ private setUserAgent;
17
+ private getUaHttpHeaders;
18
+ getMatches(): string | null | object;
19
+ isCrawler(userAgent?: string): boolean;
20
+ }
21
+ //# sourceMappingURL=crawler.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAC;AAM5C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAE/B,OAAO,CAAC,OAAO,CAA2C;IAC1D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAyB;gBAGvC,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,CAAC,EAAE,mBAAmB,EAC7B,SAAS,CAAC,EAAE,MAAM;IAmBb,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAI/D,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,gBAAgB;IAIjB,UAAU,IAAI,MAAM,GAAG,IAAI,GAAG,MAAM;IAYpC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;CAsB9C"}
@@ -0,0 +1,98 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Crawler = void 0;
4
+ const crawlers_1 = require("./crawler/crawlers");
5
+ const exclusions_1 = require("./crawler/exclusions");
6
+ const headers_1 = require("./crawler/headers");
7
+ class Crawler {
8
+ crawlers;
9
+ headers;
10
+ exclusions;
11
+ request;
12
+ compiledRegexList;
13
+ compiledExclusions;
14
+ httpHeaders;
15
+ userAgent;
16
+ matches;
17
+ constructor(request, headers, userAgent) {
18
+ this.crawlers = new crawlers_1.Crawlers();
19
+ this.headers = new headers_1.Headers();
20
+ this.exclusions = new exclusions_1.Exclusions();
21
+ this.request = request ?? {};
22
+ // The regex-list must not be used with g-flag!
23
+ // See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
24
+ this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
25
+ // The exclusions should be used with g-flag in order to remove each value.
26
+ this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
27
+ this.httpHeaders = this.setHttpHeaders(headers);
28
+ this.userAgent = this.setUserAgent(userAgent);
29
+ }
30
+ compileRegex(patterns, flags) {
31
+ return new RegExp(patterns.join('|'), flags);
32
+ }
33
+ setHttpHeaders(headers) {
34
+ // Use the Request headers if httpHeaders is not defined
35
+ if (!headers || Object.keys(headers).length === 0) {
36
+ if (Object.keys(this.request).length) {
37
+ if (this.request.headers) {
38
+ return this.request.headers;
39
+ }
40
+ }
41
+ return '';
42
+ }
43
+ // Save the headers.
44
+ return headers;
45
+ }
46
+ setUserAgent(userAgent) {
47
+ if (!userAgent?.length) {
48
+ userAgent = '';
49
+ for (const header of this.getUaHttpHeaders()) {
50
+ if (typeof this.httpHeaders === 'object' &&
51
+ !Array.isArray(this.httpHeaders)) {
52
+ if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
53
+ const headerValue = this.httpHeaders[header.toLowerCase()];
54
+ if (typeof headerValue === 'string') {
55
+ const separator = userAgent.length > 0 ? ' ' : '';
56
+ userAgent += separator + headerValue;
57
+ }
58
+ else if (Array.isArray(headerValue)) {
59
+ const separator = userAgent.length > 0 ? ' ' : '';
60
+ userAgent += separator + headerValue.join(' ');
61
+ }
62
+ }
63
+ }
64
+ }
65
+ }
66
+ return userAgent;
67
+ }
68
+ getUaHttpHeaders() {
69
+ return this.headers.getAll();
70
+ }
71
+ getMatches() {
72
+ if (this.matches !== undefined) {
73
+ if (this.matches?.length) {
74
+ return this.matches[0];
75
+ }
76
+ return null;
77
+ }
78
+ return {};
79
+ }
80
+ isCrawler(userAgent) {
81
+ if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
82
+ return false;
83
+ }
84
+ let agent = userAgent ?? this.userAgent;
85
+ // test on compiled regx
86
+ agent = agent.replace(this.compiledExclusions, '');
87
+ if (agent.trim().length === 0) {
88
+ return false;
89
+ }
90
+ const matches = this.compiledRegexList.exec(agent);
91
+ if (matches) {
92
+ this.matches = matches;
93
+ }
94
+ return matches !== null && matches.length > 0;
95
+ }
96
+ }
97
+ exports.Crawler = Crawler;
98
+ //# sourceMappingURL=crawler.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":";;;AAGA,iDAA8C;AAC9C,qDAAkD;AAClD,+CAA4C;AAE5C,MAAa,OAAO;IACV,QAAQ,CAAW;IACnB,OAAO,CAAU;IACjB,UAAU,CAAa;IAEvB,OAAO,CAA2C;IAClD,iBAAiB,CAAS;IAC1B,kBAAkB,CAAS;IAC3B,WAAW,CAA0C;IACrD,SAAS,CAAS;IAClB,OAAO,CAA0B;IAEzC,YACE,OAAiB,EACjB,OAA6B,EAC7B,SAAkB;QAElB,IAAI,CAAC,QAAQ,GAAG,IAAI,mBAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAE7B,+CAA+C;QAC/C,yGAAyG;QACzG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAExE,2EAA2E;QAC3E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;QAE5E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAEM,YAAY,CAAC,QAAkB,EAAE,KAAc;QACpD,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEO,cAAc,CACpB,OAAiD;QAEjD,wDAAwD;QACxD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,oBAAoB;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,SAAkB;QACrC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YACvB,SAAS,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC7C,IACE,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ;oBACpC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAChC,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;wBAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;wBAE3D,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;4BACpC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC;wBACvC,CAAC;6BAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;4BACtC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,gBAAgB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC/B,CAAC;IAEM,UAAU;QACf,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,SAAS,CAAC,SAAkB;QACjC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACtD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAExC,wBAAwB;QACxB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,CAAC;CACF;AAxHD,0BAwHC"}
@@ -0,0 +1,9 @@
1
+ import { NextFunction, Request, Response } from 'express-serve-static-core';
2
+ import { Crawler } from './lib/crawler';
3
+ export type Middleware = (request: Request, response: Response, next: NextFunction) => Promise<Response>;
4
+ declare module 'express-serve-static-core' {
5
+ interface Request {
6
+ Crawler: Crawler;
7
+ }
8
+ }
9
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAE5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,MAAM,MAAM,UAAU,GAAG,CACvB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,YAAY,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB,OAAO,QAAQ,2BAA2B,CAAC;IACzC,UAAU,OAAO;QACf,OAAO,EAAE,OAAO,CAAC;KAClB;CACF"}
package/dist/types.js ADDED
@@ -0,0 +1,3 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
@@ -0,0 +1,62 @@
1
+ import typescriptEslint from "@typescript-eslint/eslint-plugin";
2
+ import simpleImportSort from "eslint-plugin-simple-import-sort";
3
+ import tsParser from "@typescript-eslint/parser";
4
+ import path from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import js from "@eslint/js";
7
+ import { FlatCompat } from "@eslint/eslintrc";
8
+
9
+ const __filename = fileURLToPath(import.meta.url);
10
+ const __dirname = path.dirname(__filename);
11
+ const compat = new FlatCompat({
12
+ baseDirectory: __dirname,
13
+ recommendedConfig: js.configs.recommended,
14
+ allConfig: js.configs.all
15
+ });
16
+
17
+ export default [{
18
+ ignores: [
19
+ "!**/.*",
20
+ "**/coverage/",
21
+ "**/junit/",
22
+ "**/dist/",
23
+ "**/mobiles/",
24
+ "**/node_modules/*",
25
+ ],
26
+
27
+ files: ['**/*.ts', '**/*.tsx'],
28
+
29
+ plugins: {
30
+ "@typescript-eslint": typescriptEslint,
31
+ "simple-import-sort": simpleImportSort,
32
+ },
33
+
34
+ languageOptions: {
35
+ parser: tsParser,
36
+ parserOptions: {
37
+ project: "./tsconfig.json",
38
+ },
39
+ },
40
+
41
+ rules: {
42
+ "@typescript-eslint/no-restricted-types": "error",
43
+ "@typescript-eslint/no-explicit-any": "warn",
44
+
45
+ "max-len": ["warn", {
46
+ code: 80,
47
+ tabWidth: 2,
48
+ comments: 80,
49
+ ignoreComments: true,
50
+ ignoreTrailingComments: true,
51
+ ignoreUrls: true,
52
+ ignoreStrings: true,
53
+ ignoreTemplateLiterals: true,
54
+ ignoreRegExpLiterals: true,
55
+ }],
56
+
57
+ "@typescript-eslint/no-unused-vars": ["warn"],
58
+
59
+ "simple-import-sort/imports": "error",
60
+ "simple-import-sort/exports": "error",
61
+ },
62
+ }];
@@ -6,6 +6,6 @@
6
6
  "author": "Jeffery Hussin <jefferytutorials@gmail.com> (http://github.com/jefferyhus)",
7
7
  "license": "MIT",
8
8
  "dependencies": {
9
- "express": "^4.17.2"
9
+ "express": "^4.19.2"
10
10
  }
11
11
  }