es6-crawler-detect 3.3.0 → 4.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.babelrc +1 -1
- package/.mocharc.json +5 -0
- package/README.md +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/crawler/crawlers.d.ts +5 -0
- package/dist/lib/crawler/crawlers.d.ts.map +1 -0
- package/dist/lib/crawler/crawlers.js +1400 -0
- package/dist/lib/crawler/crawlers.js.map +1 -0
- package/dist/lib/crawler/exclusions.d.ts +5 -0
- package/dist/lib/crawler/exclusions.d.ts.map +1 -0
- package/dist/lib/crawler/exclusions.js +59 -0
- package/dist/lib/crawler/exclusions.js.map +1 -0
- package/dist/lib/crawler/headers.d.ts +5 -0
- package/dist/lib/crawler/headers.d.ts.map +1 -0
- package/dist/lib/crawler/headers.js +21 -0
- package/dist/lib/crawler/headers.js.map +1 -0
- package/dist/lib/crawler/provider.d.ts +4 -0
- package/dist/lib/crawler/provider.d.ts.map +1 -0
- package/dist/lib/crawler/provider.js +3 -0
- package/dist/lib/crawler/provider.js.map +1 -0
- package/dist/lib/crawler.d.ts +21 -0
- package/dist/lib/crawler.d.ts.map +1 -0
- package/dist/lib/crawler.js +98 -0
- package/dist/lib/crawler.js.map +1 -0
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/eslint.config.mjs +62 -0
- package/example/node/package.json +1 -1
- package/example/node/yarn.lock +222 -99
- package/package.json +30 -24
- package/src/index.ts +18 -0
- package/src/lib/crawler/{crawlers.js → crawlers.ts} +1397 -1403
- package/src/lib/crawler/{exclusions.js → exclusions.ts} +56 -62
- package/src/lib/crawler/{headers.js → headers.ts} +4 -10
- package/src/lib/crawler/provider.ts +3 -0
- package/src/lib/crawler.ts +128 -0
- package/src/types.ts +15 -0
- package/test/lib/crawler.spec.ts +116 -0
- package/tsconfig.json +28 -0
- package/.eslintignore +0 -6
- package/.eslintrc.json +0 -30
- package/dist/main.bundle.js +0 -1
- package/example/node/dist/crawler.min.js +0 -1
- package/example/node/node_modules/.bin/mime +0 -15
- package/example/node/node_modules/.bin/mime.cmd +0 -7
- package/example/node/node_modules/.yarn-integrity +0 -67
- package/example/node/node_modules/accepts/HISTORY.md +0 -236
- package/example/node/node_modules/accepts/LICENSE +0 -23
- package/example/node/node_modules/accepts/README.md +0 -142
- package/example/node/node_modules/accepts/index.js +0 -238
- package/example/node/node_modules/accepts/package.json +0 -47
- package/example/node/node_modules/array-flatten/LICENSE +0 -21
- package/example/node/node_modules/array-flatten/README.md +0 -43
- package/example/node/node_modules/array-flatten/array-flatten.js +0 -64
- package/example/node/node_modules/array-flatten/package.json +0 -39
- package/example/node/node_modules/body-parser/HISTORY.md +0 -624
- package/example/node/node_modules/body-parser/LICENSE +0 -23
- package/example/node/node_modules/body-parser/README.md +0 -456
- package/example/node/node_modules/body-parser/index.js +0 -157
- package/example/node/node_modules/body-parser/lib/read.js +0 -181
- package/example/node/node_modules/body-parser/lib/types/json.js +0 -230
- package/example/node/node_modules/body-parser/lib/types/raw.js +0 -101
- package/example/node/node_modules/body-parser/lib/types/text.js +0 -121
- package/example/node/node_modules/body-parser/lib/types/urlencoded.js +0 -284
- package/example/node/node_modules/body-parser/package.json +0 -52
- package/example/node/node_modules/bytes/History.md +0 -92
- package/example/node/node_modules/bytes/LICENSE +0 -23
- package/example/node/node_modules/bytes/Readme.md +0 -152
- package/example/node/node_modules/bytes/index.js +0 -166
- package/example/node/node_modules/bytes/package.json +0 -42
- package/example/node/node_modules/content-disposition/HISTORY.md +0 -60
- package/example/node/node_modules/content-disposition/LICENSE +0 -22
- package/example/node/node_modules/content-disposition/README.md +0 -142
- package/example/node/node_modules/content-disposition/index.js +0 -458
- package/example/node/node_modules/content-disposition/package.json +0 -44
- package/example/node/node_modules/content-type/HISTORY.md +0 -24
- package/example/node/node_modules/content-type/LICENSE +0 -22
- package/example/node/node_modules/content-type/README.md +0 -92
- package/example/node/node_modules/content-type/index.js +0 -222
- package/example/node/node_modules/content-type/package.json +0 -40
- package/example/node/node_modules/cookie/HISTORY.md +0 -128
- package/example/node/node_modules/cookie/LICENSE +0 -24
- package/example/node/node_modules/cookie/README.md +0 -257
- package/example/node/node_modules/cookie/index.js +0 -202
- package/example/node/node_modules/cookie/package.json +0 -40
- package/example/node/node_modules/cookie-signature/History.md +0 -38
- package/example/node/node_modules/cookie-signature/Readme.md +0 -42
- package/example/node/node_modules/cookie-signature/index.js +0 -51
- package/example/node/node_modules/cookie-signature/package.json +0 -18
- package/example/node/node_modules/debug/.coveralls.yml +0 -1
- package/example/node/node_modules/debug/.eslintrc +0 -11
- package/example/node/node_modules/debug/.travis.yml +0 -14
- package/example/node/node_modules/debug/CHANGELOG.md +0 -362
- package/example/node/node_modules/debug/LICENSE +0 -19
- package/example/node/node_modules/debug/Makefile +0 -50
- package/example/node/node_modules/debug/README.md +0 -312
- package/example/node/node_modules/debug/component.json +0 -19
- package/example/node/node_modules/debug/karma.conf.js +0 -70
- package/example/node/node_modules/debug/node.js +0 -1
- package/example/node/node_modules/debug/package.json +0 -49
- package/example/node/node_modules/debug/src/browser.js +0 -185
- package/example/node/node_modules/debug/src/debug.js +0 -202
- package/example/node/node_modules/debug/src/index.js +0 -10
- package/example/node/node_modules/debug/src/inspector-log.js +0 -15
- package/example/node/node_modules/debug/src/node.js +0 -248
- package/example/node/node_modules/depd/History.md +0 -96
- package/example/node/node_modules/depd/LICENSE +0 -22
- package/example/node/node_modules/depd/Readme.md +0 -280
- package/example/node/node_modules/depd/index.js +0 -522
- package/example/node/node_modules/depd/lib/browser/index.js +0 -77
- package/example/node/node_modules/depd/lib/compat/callsite-tostring.js +0 -103
- package/example/node/node_modules/depd/lib/compat/event-listener-count.js +0 -22
- package/example/node/node_modules/depd/lib/compat/index.js +0 -79
- package/example/node/node_modules/depd/package.json +0 -41
- package/example/node/node_modules/destroy/LICENSE +0 -22
- package/example/node/node_modules/destroy/README.md +0 -60
- package/example/node/node_modules/destroy/index.js +0 -75
- package/example/node/node_modules/destroy/package.json +0 -37
- package/example/node/node_modules/ee-first/LICENSE +0 -22
- package/example/node/node_modules/ee-first/README.md +0 -80
- package/example/node/node_modules/ee-first/index.js +0 -95
- package/example/node/node_modules/ee-first/package.json +0 -29
- package/example/node/node_modules/encodeurl/HISTORY.md +0 -14
- package/example/node/node_modules/encodeurl/LICENSE +0 -22
- package/example/node/node_modules/encodeurl/README.md +0 -128
- package/example/node/node_modules/encodeurl/index.js +0 -60
- package/example/node/node_modules/encodeurl/package.json +0 -40
- package/example/node/node_modules/escape-html/LICENSE +0 -24
- package/example/node/node_modules/escape-html/Readme.md +0 -43
- package/example/node/node_modules/escape-html/index.js +0 -78
- package/example/node/node_modules/escape-html/package.json +0 -24
- package/example/node/node_modules/etag/HISTORY.md +0 -83
- package/example/node/node_modules/etag/LICENSE +0 -22
- package/example/node/node_modules/etag/README.md +0 -159
- package/example/node/node_modules/etag/index.js +0 -131
- package/example/node/node_modules/etag/package.json +0 -47
- package/example/node/node_modules/express/History.md +0 -3510
- package/example/node/node_modules/express/LICENSE +0 -24
- package/example/node/node_modules/express/Readme.md +0 -158
- package/example/node/node_modules/express/index.js +0 -11
- package/example/node/node_modules/express/lib/application.js +0 -644
- package/example/node/node_modules/express/lib/express.js +0 -116
- package/example/node/node_modules/express/lib/middleware/init.js +0 -43
- package/example/node/node_modules/express/lib/middleware/query.js +0 -47
- package/example/node/node_modules/express/lib/request.js +0 -525
- package/example/node/node_modules/express/lib/response.js +0 -1147
- package/example/node/node_modules/express/lib/router/index.js +0 -668
- package/example/node/node_modules/express/lib/router/layer.js +0 -181
- package/example/node/node_modules/express/lib/router/route.js +0 -216
- package/example/node/node_modules/express/lib/utils.js +0 -302
- package/example/node/node_modules/express/lib/view.js +0 -182
- package/example/node/node_modules/express/package.json +0 -99
- package/example/node/node_modules/finalhandler/HISTORY.md +0 -187
- package/example/node/node_modules/finalhandler/LICENSE +0 -22
- package/example/node/node_modules/finalhandler/README.md +0 -148
- package/example/node/node_modules/finalhandler/index.js +0 -331
- package/example/node/node_modules/finalhandler/package.json +0 -45
- package/example/node/node_modules/forwarded/HISTORY.md +0 -21
- package/example/node/node_modules/forwarded/LICENSE +0 -22
- package/example/node/node_modules/forwarded/README.md +0 -57
- package/example/node/node_modules/forwarded/index.js +0 -90
- package/example/node/node_modules/forwarded/package.json +0 -45
- package/example/node/node_modules/fresh/HISTORY.md +0 -70
- package/example/node/node_modules/fresh/LICENSE +0 -23
- package/example/node/node_modules/fresh/README.md +0 -119
- package/example/node/node_modules/fresh/index.js +0 -137
- package/example/node/node_modules/fresh/package.json +0 -46
- package/example/node/node_modules/http-errors/HISTORY.md +0 -165
- package/example/node/node_modules/http-errors/LICENSE +0 -23
- package/example/node/node_modules/http-errors/README.md +0 -169
- package/example/node/node_modules/http-errors/index.js +0 -299
- package/example/node/node_modules/http-errors/package.json +0 -49
- package/example/node/node_modules/iconv-lite/Changelog.md +0 -162
- package/example/node/node_modules/iconv-lite/LICENSE +0 -21
- package/example/node/node_modules/iconv-lite/README.md +0 -156
- package/example/node/node_modules/iconv-lite/encodings/dbcs-codec.js +0 -555
- package/example/node/node_modules/iconv-lite/encodings/dbcs-data.js +0 -176
- package/example/node/node_modules/iconv-lite/encodings/index.js +0 -22
- package/example/node/node_modules/iconv-lite/encodings/internal.js +0 -188
- package/example/node/node_modules/iconv-lite/encodings/sbcs-codec.js +0 -72
- package/example/node/node_modules/iconv-lite/encodings/sbcs-data-generated.js +0 -451
- package/example/node/node_modules/iconv-lite/encodings/sbcs-data.js +0 -174
- package/example/node/node_modules/iconv-lite/encodings/tables/big5-added.json +0 -122
- package/example/node/node_modules/iconv-lite/encodings/tables/cp936.json +0 -264
- package/example/node/node_modules/iconv-lite/encodings/tables/cp949.json +0 -273
- package/example/node/node_modules/iconv-lite/encodings/tables/cp950.json +0 -177
- package/example/node/node_modules/iconv-lite/encodings/tables/eucjp.json +0 -182
- package/example/node/node_modules/iconv-lite/encodings/tables/gb18030-ranges.json +0 -1
- package/example/node/node_modules/iconv-lite/encodings/tables/gbk-added.json +0 -55
- package/example/node/node_modules/iconv-lite/encodings/tables/shiftjis.json +0 -125
- package/example/node/node_modules/iconv-lite/encodings/utf16.js +0 -177
- package/example/node/node_modules/iconv-lite/encodings/utf7.js +0 -290
- package/example/node/node_modules/iconv-lite/lib/bom-handling.js +0 -52
- package/example/node/node_modules/iconv-lite/lib/extend-node.js +0 -217
- package/example/node/node_modules/iconv-lite/lib/index.d.ts +0 -24
- package/example/node/node_modules/iconv-lite/lib/index.js +0 -153
- package/example/node/node_modules/iconv-lite/lib/streams.js +0 -121
- package/example/node/node_modules/iconv-lite/package.json +0 -46
- package/example/node/node_modules/inherits/LICENSE +0 -16
- package/example/node/node_modules/inherits/README.md +0 -42
- package/example/node/node_modules/inherits/inherits.js +0 -9
- package/example/node/node_modules/inherits/inherits_browser.js +0 -27
- package/example/node/node_modules/inherits/package.json +0 -29
- package/example/node/node_modules/ipaddr.js/LICENSE +0 -19
- package/example/node/node_modules/ipaddr.js/README.md +0 -233
- package/example/node/node_modules/ipaddr.js/ipaddr.min.js +0 -1
- package/example/node/node_modules/ipaddr.js/lib/ipaddr.js +0 -673
- package/example/node/node_modules/ipaddr.js/lib/ipaddr.js.d.ts +0 -68
- package/example/node/node_modules/ipaddr.js/package.json +0 -35
- package/example/node/node_modules/media-typer/HISTORY.md +0 -22
- package/example/node/node_modules/media-typer/LICENSE +0 -22
- package/example/node/node_modules/media-typer/README.md +0 -81
- package/example/node/node_modules/media-typer/index.js +0 -270
- package/example/node/node_modules/media-typer/package.json +0 -26
- package/example/node/node_modules/merge-descriptors/HISTORY.md +0 -21
- package/example/node/node_modules/merge-descriptors/LICENSE +0 -23
- package/example/node/node_modules/merge-descriptors/README.md +0 -48
- package/example/node/node_modules/merge-descriptors/index.js +0 -60
- package/example/node/node_modules/merge-descriptors/package.json +0 -32
- package/example/node/node_modules/methods/HISTORY.md +0 -29
- package/example/node/node_modules/methods/LICENSE +0 -24
- package/example/node/node_modules/methods/README.md +0 -51
- package/example/node/node_modules/methods/index.js +0 -69
- package/example/node/node_modules/methods/package.json +0 -36
- package/example/node/node_modules/mime/CHANGELOG.md +0 -164
- package/example/node/node_modules/mime/LICENSE +0 -21
- package/example/node/node_modules/mime/README.md +0 -90
- package/example/node/node_modules/mime/cli.js +0 -8
- package/example/node/node_modules/mime/mime.js +0 -108
- package/example/node/node_modules/mime/package.json +0 -44
- package/example/node/node_modules/mime/src/build.js +0 -53
- package/example/node/node_modules/mime/src/test.js +0 -60
- package/example/node/node_modules/mime/types.json +0 -1
- package/example/node/node_modules/mime-db/HISTORY.md +0 -499
- package/example/node/node_modules/mime-db/LICENSE +0 -22
- package/example/node/node_modules/mime-db/README.md +0 -100
- package/example/node/node_modules/mime-db/db.json +0 -8471
- package/example/node/node_modules/mime-db/index.js +0 -11
- package/example/node/node_modules/mime-db/package.json +0 -59
- package/example/node/node_modules/mime-types/HISTORY.md +0 -388
- package/example/node/node_modules/mime-types/LICENSE +0 -23
- package/example/node/node_modules/mime-types/README.md +0 -113
- package/example/node/node_modules/mime-types/index.js +0 -188
- package/example/node/node_modules/mime-types/package.json +0 -44
- package/example/node/node_modules/ms/index.js +0 -152
- package/example/node/node_modules/ms/license.md +0 -21
- package/example/node/node_modules/ms/package.json +0 -37
- package/example/node/node_modules/ms/readme.md +0 -51
- package/example/node/node_modules/negotiator/HISTORY.md +0 -103
- package/example/node/node_modules/negotiator/LICENSE +0 -24
- package/example/node/node_modules/negotiator/README.md +0 -203
- package/example/node/node_modules/negotiator/index.js +0 -124
- package/example/node/node_modules/negotiator/lib/charset.js +0 -169
- package/example/node/node_modules/negotiator/lib/encoding.js +0 -184
- package/example/node/node_modules/negotiator/lib/language.js +0 -179
- package/example/node/node_modules/negotiator/lib/mediaType.js +0 -294
- package/example/node/node_modules/negotiator/package.json +0 -42
- package/example/node/node_modules/on-finished/HISTORY.md +0 -88
- package/example/node/node_modules/on-finished/LICENSE +0 -23
- package/example/node/node_modules/on-finished/README.md +0 -154
- package/example/node/node_modules/on-finished/index.js +0 -196
- package/example/node/node_modules/on-finished/package.json +0 -31
- package/example/node/node_modules/parseurl/HISTORY.md +0 -58
- package/example/node/node_modules/parseurl/LICENSE +0 -24
- package/example/node/node_modules/parseurl/README.md +0 -133
- package/example/node/node_modules/parseurl/index.js +0 -158
- package/example/node/node_modules/parseurl/package.json +0 -40
- package/example/node/node_modules/path-to-regexp/History.md +0 -36
- package/example/node/node_modules/path-to-regexp/LICENSE +0 -21
- package/example/node/node_modules/path-to-regexp/Readme.md +0 -35
- package/example/node/node_modules/path-to-regexp/index.js +0 -129
- package/example/node/node_modules/path-to-regexp/package.json +0 -30
- package/example/node/node_modules/proxy-addr/HISTORY.md +0 -161
- package/example/node/node_modules/proxy-addr/LICENSE +0 -22
- package/example/node/node_modules/proxy-addr/README.md +0 -139
- package/example/node/node_modules/proxy-addr/index.js +0 -327
- package/example/node/node_modules/proxy-addr/package.json +0 -47
- package/example/node/node_modules/qs/.editorconfig +0 -39
- package/example/node/node_modules/qs/.eslintignore +0 -2
- package/example/node/node_modules/qs/.eslintrc +0 -35
- package/example/node/node_modules/qs/.github/FUNDING.yml +0 -12
- package/example/node/node_modules/qs/.nycrc +0 -13
- package/example/node/node_modules/qs/CHANGELOG.md +0 -359
- package/example/node/node_modules/qs/LICENSE.md +0 -29
- package/example/node/node_modules/qs/README.md +0 -604
- package/example/node/node_modules/qs/dist/qs.js +0 -832
- package/example/node/node_modules/qs/lib/formats.js +0 -23
- package/example/node/node_modules/qs/lib/index.js +0 -11
- package/example/node/node_modules/qs/lib/parse.js +0 -257
- package/example/node/node_modules/qs/lib/stringify.js +0 -278
- package/example/node/node_modules/qs/lib/utils.js +0 -251
- package/example/node/node_modules/qs/package.json +0 -69
- package/example/node/node_modules/qs/test/parse.js +0 -772
- package/example/node/node_modules/qs/test/stringify.js +0 -793
- package/example/node/node_modules/qs/test/utils.js +0 -136
- package/example/node/node_modules/range-parser/HISTORY.md +0 -56
- package/example/node/node_modules/range-parser/LICENSE +0 -23
- package/example/node/node_modules/range-parser/README.md +0 -84
- package/example/node/node_modules/range-parser/index.js +0 -162
- package/example/node/node_modules/range-parser/package.json +0 -44
- package/example/node/node_modules/raw-body/HISTORY.md +0 -284
- package/example/node/node_modules/raw-body/LICENSE +0 -22
- package/example/node/node_modules/raw-body/README.md +0 -217
- package/example/node/node_modules/raw-body/index.d.ts +0 -87
- package/example/node/node_modules/raw-body/index.js +0 -286
- package/example/node/node_modules/raw-body/package.json +0 -48
- package/example/node/node_modules/safe-buffer/LICENSE +0 -21
- package/example/node/node_modules/safe-buffer/README.md +0 -584
- package/example/node/node_modules/safe-buffer/index.d.ts +0 -187
- package/example/node/node_modules/safe-buffer/index.js +0 -65
- package/example/node/node_modules/safe-buffer/package.json +0 -51
- package/example/node/node_modules/safer-buffer/LICENSE +0 -21
- package/example/node/node_modules/safer-buffer/Porting-Buffer.md +0 -268
- package/example/node/node_modules/safer-buffer/Readme.md +0 -156
- package/example/node/node_modules/safer-buffer/dangerous.js +0 -58
- package/example/node/node_modules/safer-buffer/package.json +0 -34
- package/example/node/node_modules/safer-buffer/safer.js +0 -77
- package/example/node/node_modules/safer-buffer/tests.js +0 -406
- package/example/node/node_modules/send/HISTORY.md +0 -506
- package/example/node/node_modules/send/LICENSE +0 -23
- package/example/node/node_modules/send/README.md +0 -327
- package/example/node/node_modules/send/index.js +0 -1133
- package/example/node/node_modules/send/node_modules/.bin/mime +0 -15
- package/example/node/node_modules/send/node_modules/.bin/mime.cmd +0 -7
- package/example/node/node_modules/send/node_modules/ms/index.js +0 -162
- package/example/node/node_modules/send/node_modules/ms/license.md +0 -21
- package/example/node/node_modules/send/node_modules/ms/package.json +0 -38
- package/example/node/node_modules/send/node_modules/ms/readme.md +0 -59
- package/example/node/node_modules/send/package.json +0 -61
- package/example/node/node_modules/serve-static/HISTORY.md +0 -459
- package/example/node/node_modules/serve-static/LICENSE +0 -25
- package/example/node/node_modules/serve-static/README.md +0 -257
- package/example/node/node_modules/serve-static/index.js +0 -210
- package/example/node/node_modules/serve-static/package.json +0 -42
- package/example/node/node_modules/setprototypeof/LICENSE +0 -13
- package/example/node/node_modules/setprototypeof/README.md +0 -31
- package/example/node/node_modules/setprototypeof/index.d.ts +0 -2
- package/example/node/node_modules/setprototypeof/index.js +0 -17
- package/example/node/node_modules/setprototypeof/package.json +0 -38
- package/example/node/node_modules/setprototypeof/test/index.js +0 -24
- package/example/node/node_modules/statuses/HISTORY.md +0 -65
- package/example/node/node_modules/statuses/LICENSE +0 -23
- package/example/node/node_modules/statuses/README.md +0 -127
- package/example/node/node_modules/statuses/codes.json +0 -66
- package/example/node/node_modules/statuses/index.js +0 -113
- package/example/node/node_modules/statuses/package.json +0 -48
- package/example/node/node_modules/toidentifier/HISTORY.md +0 -9
- package/example/node/node_modules/toidentifier/LICENSE +0 -21
- package/example/node/node_modules/toidentifier/README.md +0 -61
- package/example/node/node_modules/toidentifier/index.js +0 -32
- package/example/node/node_modules/toidentifier/package.json +0 -38
- package/example/node/node_modules/type-is/HISTORY.md +0 -259
- package/example/node/node_modules/type-is/LICENSE +0 -23
- package/example/node/node_modules/type-is/README.md +0 -170
- package/example/node/node_modules/type-is/index.js +0 -266
- package/example/node/node_modules/type-is/package.json +0 -45
- package/example/node/node_modules/unpipe/HISTORY.md +0 -4
- package/example/node/node_modules/unpipe/LICENSE +0 -22
- package/example/node/node_modules/unpipe/README.md +0 -43
- package/example/node/node_modules/unpipe/index.js +0 -69
- package/example/node/node_modules/unpipe/package.json +0 -27
- package/example/node/node_modules/utils-merge/LICENSE +0 -20
- package/example/node/node_modules/utils-merge/README.md +0 -34
- package/example/node/node_modules/utils-merge/index.js +0 -23
- package/example/node/node_modules/utils-merge/package.json +0 -40
- package/example/node/node_modules/vary/HISTORY.md +0 -39
- package/example/node/node_modules/vary/LICENSE +0 -22
- package/example/node/node_modules/vary/README.md +0 -101
- package/example/node/node_modules/vary/index.js +0 -149
- package/example/node/node_modules/vary/package.json +0 -43
- package/example/node/server.js +0 -38
- package/src/index.js +0 -16
- package/src/lib/crawler/provider.js +0 -11
- package/src/lib/crawler.js +0 -130
- package/test/lib/crawler.test.js +0 -94
- package/webpack.common.js +0 -20
- package/webpack.dev.js +0 -7
- package/webpack.prod.js +0 -14
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawlers.js","sourceRoot":"","sources":["../../../src/lib/crawler/crawlers.ts"],"names":[],"mappings":";;;AAEA,MAAa,QAAQ;IACZ,MAAM;QACX,OAAO;YACL,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,sCAAsC;YACtC,OAAO;YACP,YAAY;YACZ,eAAe;YACf,aAAa;YACb,QAAQ;YACR,WAAW;YACX,UAAU;YACV,OAAO;YACP,iBAAiB;YACjB,UAAU;YACV,OAAO;YACP,SAAS;YACT,WAAW;YACX,UAAU;YACV,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,wBAAwB;YACxB,cAAc;YACd,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,SAAS;YACT,SAAS;YACT,gBAAgB;YAChB,SAAS;YACT,SAAS;YACT,UAAU;YACV,SAAS;YACT,oBAAoB;YACpB,SAAS;YACT,cAAc;YACd,WAAW;YACX,OAAO;YACP,cAAc;YACd,kBAAkB;YAClB,cAAc;YACd,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,UAAU;YACV,SAAS;YACT,QAAQ;YACR,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kBAAkB;YAClB,uBAAuB;YACvB,OAAO;YACP,0BAA0B;YAC1B,UAAU;YACV,WAAW;YACX,QAAQ;YACR,SAAS;YACT,SAAS;YACT,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,WAAW;YACX,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,kBAAkB;YAClB,kBAAkB;YAClB,sBAAsB;YACtB,WAAW;YACX,KAAK;YACL,OAAO;YACP,YAAY;YACZ,SAAS;YACT,cAAc;YACd,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,+BAA+B;YAC/B,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,QAAQ;YACR,UAAU;YACV,wBAAwB;YACxB,SAAS;YACT,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,eAAe;YACf,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,cAAc;YACd,mBAAmB;YACnB,wBAAwB;YACxB,mBAAmB;YACnB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,aAAa;YACb,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,kBAAkB;YAClB,cAAc;YACd,SAAS;YACT,WAAW;YACX,eAAe;YACf,OAAO;YACP,WAAW;YACX,UAAU;YACV,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,WAAW;YACX,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,YAAY;YACZ,sBAAsB;YACtB,UAAU;YACV,gBAAgB;YAChB,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,kBAAkB;YAClB,QAAQ;YACR,aAAa;YACb,QAAQ;YACR,YAAY;YACZ,UAAU;YACV,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,sBAAsB;YACtB,YAAY;YACZ,SAAS;YACT,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,QAAQ;YACR,mBAAmB;YACnB,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,0BAA0B;YAC1B,YAAY;YACZ,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,WAAW;YACX,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,oBAAoB;YACpB,oBAAoB;YACpB,oBAAoB;YACpB,cAAc;YACd,aAAa;YACb,YAAY;YACZ,WAAW;YACX,cAAc;YACd,QAAQ;YACR,SAAS;YACT,OAAO;YACP,WAAW;YACX,UAAU;YACV,eAAe;YACf,aAAa;YACb,cAAc;YACd,UAAU;YACV,cAAc;YACd,SAAS;YACT,UAAU;YACV,eAAe;YACf,WAAW;YACX,eAAe;YACf,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,4BAA4B;YAC5B,iBAAiB;YACjB,6BAA6B;YAC7B,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,cAAc;YACd,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,WAAW;YACX,UAAU;YACV,aAAa;YACb,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kCAAkC;YAClC,oBAAoB;YACpB,QAAQ;YACR,kBAAkB;YAClB,YAAY;YACZ,sBAAsB;YACtB,eAAe;YACf,aAAa;YACb,yBAAyB;YACzB,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,YAAY;YACZ,SAAS;YACT,WAAW;YACX,oBAAoB;YACpB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB;YAChB,gCAAgC;YAChC,SAAS;YACT,eAAe;YACf,mBAAmB;YACnB,gBAAgB;YAChB,WAAW;YACX,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,UAAU;YACV,SAAS;YACT,MAAM;YACN,gBAAgB;YAChB,MAAM;YACN,YAAY;YACZ,YAAY;YACZ,eAAe;YACf,WAAW;YACX,sBAAsB;YACtB,WAAW;YACX,eAAe;YACf,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,sBAAsB;YACtB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,OAAO;YACP,OAAO;YACP,oDAAoD;YACpD,OAAO;YACP,MAAM;YACN,WAAW;YACX,eAAe;YACf,WAAW;YACX,wBAAwB;YACxB,aAAa;YACb,WAAW;YACX,aAAa;YACb,MAAM;YACN,WAAW;YACX,kBAAkB;YAClB,SAAS;YACT,qBAAqB;YACrB,gBAAgB;YAChB,YAAY;YACZ,oBAAoB;YACpB,yBAAyB;YACzB,6BAA6B;YAC7B,aAAa;YACb,yBAAyB;YACzB,iBAAiB;YACjB,cAAc;YACd,kBAAkB;YAClB,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,oBAAoB;YACpB,UAAU;YACV,YAAY;YACZ,QAAQ;YACR,WAAW;YACX,eAAe;YACf,gBAAgB;YAChB,QAAQ;YACR,MAAM;YACN,aAAa;YACb,MAAM;YACN,YAAY;YACZ,cAAc;YACd,SAAS;YACT,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,aAAa;YACb,mBAAmB;YACnB,SAAS;YACT,SAAS;YACT,oBAAoB;YACpB,WAAW;YACX,yBAAyB;YACzB,wBAAwB;YACxB,QAAQ;YACR,SAAS;YACT,aAAa;YACb,kBAAkB;YAClB,UAAU;YACV,MAAM;YACN,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,wBAAwB;YACxB,UAAU;YACV,QAAQ;YACR,qBAAqB;YACrB,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,oBAAoB;YACpB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,eAAe;YACf,aAAa;YACb,SAAS;YACT,aAAa;YACb,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,aAAa;YACb,UAAU;YACV,eAAe;YACf,aAAa;YACb,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,eAAe;YACf,eAAe;YACf,QAAQ;YACR,eAAe;YACf,WAAW;YACX,OAAO;YACP,UAAU;YACV,cAAc;YACd,UAAU;YACV,uBAAuB;YACvB,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,UAAU;YACV,cAAc;YACd,iBAAiB;YACjB,cAAc;YACd,UAAU;YACV,YAAY;YACZ,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,cAAc;YACd,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,eAAe;YACf,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;YACV,aAAa;YACb,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,2BAA2B;YAC3B,UAAU;YACV,iBAAiB;YACjB,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,mBAAmB;YACnB,gBAAgB;YAChB,gBAAgB;YAChB,2BAA2B;YAC3B,qBAAqB;YACrB,4BAA4B;YAC5B,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,gCAAgC;YAChC,qBAAqB;YACrB,gBAAgB;YAChB,oBAAoB;YACpB,0BAA0B;YAC1B,yBAAyB;YACzB,yBAAyB;YACzB,gBAAgB;YAChB,yBAAyB;YACzB,mBAAmB;YACnB,sBAAsB;YACtB,0BAA0B;YAC1B,iBAAiB;YACjB,eAAe;YACf,qCAAqC;YACrC,4BAA4B;YAC5B,gBAAgB;YAChB,sBAAsB;YACtB,YAAY;YACZ,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,uBAAuB;YACvB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,WAAW;YACX,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,SAAS;YACT,WAAW;YACX,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,YAAY;YACZ,SAAS;YACT,2BAA2B;YAC3B,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,eAAe;YACf,oBAAoB;YACpB,qBAAqB;YACrB,UAAU;YACV,WAAW;YACX,YAAY;YACZ,UAAU;YACV,kBAAkB;YAClB,SAAS;YACT,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,uBAAuB;YACvB,mBAAmB;YACnB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,OAAO;YACP,YAAY;YACZ,SAAS;YACT,uBAAuB;YACvB,UAAU;YACV,qBAAqB;YACrB,UAAU;YACV,iBAAiB;YACjB,WAAW;YACX,YAAY;YACZ,4BAA4B;YAC5B,gBAAgB;YAChB,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,uBAAuB;YACvB,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,uBAAuB;YACvB,YAAY;YACZ,eAAe;YACf,OAAO;YACP,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,4BAA4B;YAC5B,YAAY;YACZ,UAAU;YACV,YAAY;YACZ,eAAe;YACf,aAAa;YACb,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,cAAc;YACd,SAAS;YACT,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,6BAA6B;YAC7B,cAAc;YACd,UAAU;YACV,YAAY;YACZ,WAAW;YACX,aAAa;YACb,aAAa;YACb,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,uBAAuB;YACvB,eAAe;YACf,kBAAkB;YAClB,YAAY;YACZ,MAAM;YACN,KAAK;YACL,SAAS;YACT,WAAW;YACX,aAAa;YACb,4BAA4B;YAC5B,SAAS;YACT,MAAM;YACN,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,QAAQ;YACR,UAAU;YACV,OAAO;YACP,WAAW;YACX,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,MAAM;YACN,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,UAAU;YACV,iCAAiC;YACjC,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,OAAO;YACP,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;YACV,aAAa;YACb,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,OAAO;YACP,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,WAAW;YACX,SAAS;YACT,cAAc;YACd,aAAa;YACb,UAAU;YACV,aAAa;YACb,UAAU;YACV,cAAc;YACd,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,iBAAiB;YACjB,qBAAqB;YACrB,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,aAAa;YACb,mBAAmB;YACnB,OAAO;YACP,gBAAgB;YAChB,yBAAyB;YACzB,kBAAkB;YAClB,aAAa;YACb,aAAa;YACb,aAAa;YACb,OAAO;YACP,UAAU;YACV,WAAW;YACX,YAAY;YACZ,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,sBAAsB;YACtB,YAAY;YACZ,gBAAgB;YAChB,eAAe;YACf,aAAa;YACb,UAAU;YACV,aAAa;YACb,SAAS;YACT,iBAAiB;YACjB,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,mBAAmB;YACnB,uCAAuC;YACvC,4BAA4B;YAC5B,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,iBAAiB;YACjB,oBAAoB;YACpB,SAAS;YACT,aAAa;YACb,UAAU;YACV,QAAQ;YACR,wBAAwB;YACxB,SAAS;YACT,eAAe;YACf,UAAU;YACV,WAAW;YACX,UAAU;YACV,UAAU;YACV,yBAAyB;YACzB,eAAe;YACf,aAAa;YACb,QAAQ;YACR,UAAU;YACV,kBAAkB;YAClB,iCAAiC;YACjC,aAAa;YACb,QAAQ;YACR,aAAa;YACb,cAAc;YACd,WAAW;YACX,cAAc;YACd,eAAe;YACf,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,cAAc;YACd,aAAa;YACb,kBAAkB;YAClB,SAAS;YACT,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,SAAS;YACT,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,aAAa;YACb,wBAAwB;YACxB,YAAY;YACZ,oBAAoB;YACpB,QAAQ;YACR,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,WAAW;YACX,QAAQ;YACR,cAAc;YACd,qBAAqB;YACrB,WAAW;YACX,SAAS;YACT,SAAS;YACT,WAAW;YACX,WAAW;YACX,OAAO;YACP,iBAAiB;YACjB,kBAAkB;YAClB,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,UAAU;YACV,WAAW;YACX,oBAAoB;YACpB,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,MAAM;YACN,OAAO;YACP,QAAQ;YACR,iBAAiB;YACjB,UAAU;YACV,SAAS;YACT,KAAK;YACL,gBAAgB;YAChB,WAAW;YACX,SAAS;YACT,MAAM;YACN,kBAAkB;YAClB,mBAAmB;YACnB,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,MAAM;YACN,qBAAqB;YACrB,iBAAiB;YACjB,yBAAyB;YACzB,UAAU;YACV,kBAAkB;YAClB,aAAa;YACb,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,aAAa;YACb,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,OAAO;YACP,OAAO;YACP,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,aAAa;YACb,YAAY;YACZ,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,YAAY;YACZ,cAAc;YACd,WAAW;YACX,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,MAAM;YACN,WAAW;YACX,SAAS;YACT,wBAAwB;YACxB,cAAc;YACd,WAAW;YACX,cAAc;YACd,cAAc;YACd,YAAY;YACZ,UAAU;YACV,WAAW;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,gBAAgB;YAChB,SAAS;YACT,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,iBAAiB;YACjB,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,UAAU;YACV,mBAAmB;YACnB,SAAS;YACT,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,QAAQ;YACR,kBAAkB;YAClB,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,cAAc;YACd,SAAS;YACT,0BAA0B;YAC1B,gBAAgB;YAChB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,QAAQ;YACR,WAAW;YACX,8BAA8B;YAC9B,eAAe;YACf,aAAa;YACb,OAAO;YACP,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,aAAa;YACb,WAAW;YACX,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,uBAAuB;YACvB,OAAO;YACP,eAAe;YACf,MAAM;YACN,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,iBAAiB;YACjB,UAAU;YACV,oBAAoB;YACpB,UAAU;YACV,SAAS;YACT,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,sBAAsB;YACtB,cAAc;YACd,cAAc;YACd,cAAc;YACd,aAAa;YACb,cAAc;YACd,yBAAyB;YACzB,YAAY;YACZ,UAAU;YACV,cAAc;YACd,YAAY;YACZ,cAAc;YACd,WAAW;YACX,OAAO;YACP,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,SAAS;YACT,UAAU;YACV,aAAa;YACb,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,WAAW;YACX,QAAQ;YACR,aAAa;YACb,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,kCAAkC;YAClC,UAAU;YACV,QAAQ;YACR,eAAe;YACf,aAAa;YACb,sBAAsB;YACtB,WAAW;YACX,YAAY;YACZ,OAAO;YACP,SAAS;YACT,4BAA4B;YAC5B,UAAU;YACV,iBAAiB;YACjB,wBAAwB;YACxB,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,SAAS;YACT,aAAa;YACb,UAAU;YACV,2BAA2B;YAC3B,cAAc;YACd,eAAe;YACf,aAAa;YACb,UAAU;YACV,uBAAuB;YACvB,QAAQ;YACR,mBAAmB;YACnB,eAAe;YACf,uBAAuB;YACvB,SAAS;YACT,cAAc;YACd,WAAW;YACX,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,WAAW;YACX,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,UAAU;YACV,oBAAoB;YACpB,qBAAqB;YACrB,cAAc;YACd,SAAS;YACT,OAAO;YACP,WAAW;YACX,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,UAAU;YACV,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,aAAa;YACb,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,aAAa;YACb,iBAAiB;YACjB,aAAa;YACb,YAAY;YACZ,WAAW;YACX,WAAW;YACX,cAAc;YACd,iBAAiB;YACjB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,OAAO;YACP,WAAW;YACX,eAAe;YACf,mBAAmB;YACnB,eAAe;YACf,OAAO;YACP,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,UAAU;YACV,UAAU;YACV,gBAAgB;YAChB,YAAY;YACZ,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,WAAW;YACX,eAAe;YACf,QAAQ;YACR,OAAO;YACP,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,UAAU;YACV,WAAW;YACX,cAAc;YACd,eAAe;YACf,aAAa;YACb,YAAY;YACZ,SAAS;YACT,iBAAiB;YACjB,WAAW;YACX,aAAa;YACb,YAAY;YACZ,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,WAAW;YACX,gBAAgB;YAChB,SAAS;YACT,OAAO;YACP,oBAAoB;YACpB,qBAAqB;YACrB,YAAY;YACZ,eAAe;YACf,qBAAqB;YACrB,SAAS;YACT,SAAS;YACT,SAAS;YACT,aAAa;YACb,SAAS;YACT,cAAc;YACd,aAAa;YACb,cAAc;YACd,eAAe;YACf,mBAAmB;YACnB,aAAa;YACb,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,SAAS;YACT,aAAa;YACb,sBAAsB;YACtB,iBAAiB;YACjB,+BAA+B;YAC/B,kBAAkB;YAClB,kBAAkB;YAClB,kBAAkB;YAClB,UAAU;YACV,UAAU;YACV,SAAS;YACT,YAAY;YACZ,aAAa;YACb,cAAc;YACd,QAAQ;YACR,eAAe;YACf,aAAa;YACb,OAAO;YACP,SAAS;YACT,cAAc;YACd,eAAe;YACf,aAAa;YACb,WAAW;YACX,aAAa;YACb,aAAa;YACb,WAAW;YACX,aAAa;YACb,UAAU;YACV,UAAU;YACV,YAAY;YACZ,UAAU;YACV,UAAU;YACV,cAAc;YACd,cAAc;YACd,aAAa;YACb,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;YACT,oBAAoB;YACpB,UAAU;YACV,0BAA0B;YAC1B,WAAW;YACX,WAAW;YACX,yBAAyB;YACzB,SAAS;YACT,cAAc;YACd,qBAAqB;YACrB,aAAa;YACb,gBAAgB;YAChB,QAAQ;YACR,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,QAAQ;YACR,SAAS;YACT,cAAc;YACd,WAAW;YACX,SAAS;YACT,aAAa;YACb,SAAS;YACT,WAAW;YACX,2BAA2B;YAC3B,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,WAAW;YACX,YAAY;YACZ,OAAO;YACP,KAAK;YACL,kCAAkC;YAClC,UAAU;YACV,aAAa;YACb,SAAS;YACT,SAAS;YACT,MAAM;YACN,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,MAAM;YACN,YAAY;YACZ,eAAe;YACf,cAAc;YACd,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,cAAc;YACd,qBAAqB;YACrB,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,UAAU;YACV,UAAU;YACV,aAAa;YACb,cAAc;YACd,WAAW;YACX,UAAU;YACV,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,SAAS;YACT,aAAa;YACb,SAAS;YACT,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,UAAU;YACV,mBAAmB;YACnB,WAAW;YACX,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,QAAQ;YACR,WAAW;YACX,WAAW;YACX,aAAa;YACb,SAAS;YACT,SAAS;YACT,iBAAiB;YACjB,oBAAoB;YACpB,gBAAgB;YAChB,UAAU;YACV,YAAY;YACZ,SAAS;YACT,aAAa;YACb,WAAW;YACX,YAAY;YACZ,cAAc;YACd,aAAa;YACb,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,MAAM;YACN,OAAO;YACP,MAAM;YACN,UAAU;YACV,MAAM;YACN,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,WAAW;YACX,iBAAiB;YACjB,gBAAgB;YAChB,sBAAsB;YACtB,2BAA2B;YAC3B,YAAY;YACZ,WAAW;YACX,UAAU;YACV,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,kBAAkB;YAClB,SAAS;YACT,eAAe;YACf,QAAQ;YACR,mCAAmC;YACnC,WAAW;YACX,MAAM;YACN,sBAAsB;YACtB,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,gBAAgB;YAChB,UAAU;YACV,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,0BAA0B;YAC1B,YAAY;YACZ,kBAAkB;YAClB,yBAAyB;YACzB,sBAAsB;YACtB,qBAAqB;YACrB,OAAO;YACP,MAAM;YACN,oBAAoB;YACpB,kBAAkB;YAClB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,OAAO;YACP,gBAAgB;YAChB,cAAc;YACd,kBAAkB;YAClB,eAAe;YACf,oBAAoB;YACpB,cAAc;YACd,YAAY;YACZ,SAAS;YACT,QAAQ;YACR,MAAM;YACN,KAAK;YACL,OAAO;YACP,oBAAoB;YACpB,wBAAwB;YACxB,kBAAkB;YAClB,SAAS;YACT,OAAO;YACP,OAAO;YACP,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,yBAAyB;YACzB,QAAQ;YACR,wIAAwI;SACzI,CAAC;IACJ,CAAC;CACF;AAl3CD,4BAk3CC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"exclusions.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,UAAW,YAAW,QAAQ;IACzC,MAAM,IAAI,MAAM,EAAE;CAoDnB"}
|
@@ -0,0 +1,59 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Exclusions = void 0;
|
4
|
+
class Exclusions {
|
5
|
+
getAll() {
|
6
|
+
return [
|
7
|
+
'Safari.[\\d\\.]*',
|
8
|
+
'Firefox.[\\d\\.]*',
|
9
|
+
' Chrome.[\\d\\.]*',
|
10
|
+
'Chromium.[\\d\\.]*',
|
11
|
+
'MSIE.[\\d\\.]',
|
12
|
+
'Opera\\/[\\d\\.]*',
|
13
|
+
'Mozilla.[\\d\\.]*',
|
14
|
+
'AppleWebKit.[\\d\\.]*',
|
15
|
+
'Trident.[\\d\\.]*',
|
16
|
+
'Windows NT.[\\d\\.]*',
|
17
|
+
'Android [\\d\\.]*',
|
18
|
+
'Macintosh.',
|
19
|
+
'Ubuntu',
|
20
|
+
'Linux',
|
21
|
+
'[ ]Intel',
|
22
|
+
'Mac OS X [\\d_]*',
|
23
|
+
'(like )?Gecko(.[\\d\\.]*)?',
|
24
|
+
'KHTML,',
|
25
|
+
'CriOS.[\\d\\.]*',
|
26
|
+
'CPU iPhone OS ([0-9_])* like Mac OS X',
|
27
|
+
'CPU OS ([0-9_])* like Mac OS X',
|
28
|
+
'iPod',
|
29
|
+
'compatible',
|
30
|
+
'x86_..',
|
31
|
+
'i686',
|
32
|
+
'x64',
|
33
|
+
'X11',
|
34
|
+
'rv:[\\d\\.]*',
|
35
|
+
'Version.[\\d\\.]*',
|
36
|
+
'WOW64',
|
37
|
+
'Win64',
|
38
|
+
'Dalvik.[\\d\\.]*',
|
39
|
+
' \\.NET CLR [\\d\\.]*',
|
40
|
+
'Presto.[\\d\\.]*',
|
41
|
+
'Media Center PC',
|
42
|
+
'BlackBerry',
|
43
|
+
'Build',
|
44
|
+
'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
|
45
|
+
'Opera',
|
46
|
+
' \\.NET[\\d\\.]*',
|
47
|
+
'cubot',
|
48
|
+
'; M bot',
|
49
|
+
'; CRONO',
|
50
|
+
'; B bot',
|
51
|
+
'; IDbot',
|
52
|
+
'; ID bot',
|
53
|
+
'; POWER BOT',
|
54
|
+
'OCTOPUS-CORE',
|
55
|
+
];
|
56
|
+
}
|
57
|
+
}
|
58
|
+
exports.Exclusions = Exclusions;
|
59
|
+
//# sourceMappingURL=exclusions.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"exclusions.js","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":";;;AAEA,MAAa,UAAU;IACrB,MAAM;QACJ,OAAO;YACL,kBAAkB;YAClB,mBAAmB;YACnB,mBAAmB;YACnB,oBAAoB;YACpB,eAAe;YACf,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,sBAAsB;YACtB,mBAAmB;YACnB,YAAY;YACZ,QAAQ;YACR,OAAO;YACP,UAAU;YACV,kBAAkB;YAClB,4BAA4B;YAC5B,QAAQ;YACR,iBAAiB;YACjB,uCAAuC;YACvC,gCAAgC;YAChC,MAAM;YACN,YAAY;YACZ,QAAQ;YACR,MAAM;YACN,KAAK;YACL,KAAK;YACL,cAAc;YACd,mBAAmB;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,kBAAkB;YAClB,iBAAiB;YACjB,YAAY;YACZ,OAAO;YACP,4DAA4D;YAC5D,OAAO;YACP,kBAAkB;YAClB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,UAAU;YACV,aAAa;YACb,cAAc;SACf,CAAC;IACJ,CAAC;CACF;AArDD,gCAqDC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,OAAQ,YAAW,QAAQ;IACtC,MAAM,IAAI,MAAM,EAAE;CAcnB"}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Headers = void 0;
|
4
|
+
class Headers {
|
5
|
+
getAll() {
|
6
|
+
return [
|
7
|
+
'USER-AGENT',
|
8
|
+
'X-OPERAMINI-PHONE-UA',
|
9
|
+
'X-DEVICE-USER-AGENT',
|
10
|
+
'X-ORIGINAL-USER-AGENT',
|
11
|
+
'X-SKYFIRE-PHONE',
|
12
|
+
'X-BOLT-PHONE-UA',
|
13
|
+
'DEVICE-STOCK-UA',
|
14
|
+
'X-UCBROWSER-DEVICE-UA',
|
15
|
+
'FROM',
|
16
|
+
'X-SCANNER',
|
17
|
+
];
|
18
|
+
}
|
19
|
+
}
|
20
|
+
exports.Headers = Headers;
|
21
|
+
//# sourceMappingURL=headers.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":";;;AAEA,MAAa,OAAO;IAClB,MAAM;QACJ,OAAO;YACL,YAAY;YACZ,sBAAsB;YACtB,qBAAqB;YACrB,uBAAuB;YACvB,iBAAiB;YACjB,iBAAiB;YACjB,iBAAiB;YACjB,uBAAuB;YACvB,MAAM;YACN,WAAW;SACZ,CAAC;IACJ,CAAC;CACF;AAfD,0BAeC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,MAAM,IAAI,MAAM,EAAE,CAAC;CACpB"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":""}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
import { Request } from 'express-serve-static-core';
|
2
|
+
import { IncomingHttpHeaders } from 'http2';
|
3
|
+
export declare class Crawler {
|
4
|
+
private crawlers;
|
5
|
+
private headers;
|
6
|
+
private exclusions;
|
7
|
+
private request;
|
8
|
+
private compiledRegexList;
|
9
|
+
private compiledExclusions;
|
10
|
+
private httpHeaders;
|
11
|
+
private userAgent;
|
12
|
+
private matches?;
|
13
|
+
constructor(request?: Request, headers?: IncomingHttpHeaders, userAgent?: string);
|
14
|
+
compileRegex(patterns: string[], flags?: string): RegExp;
|
15
|
+
private setHttpHeaders;
|
16
|
+
private setUserAgent;
|
17
|
+
private getUaHttpHeaders;
|
18
|
+
getMatches(): string | null | object;
|
19
|
+
isCrawler(userAgent?: string): boolean;
|
20
|
+
}
|
21
|
+
//# sourceMappingURL=crawler.d.ts.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAC;AAM5C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAE/B,OAAO,CAAC,OAAO,CAA2C;IAC1D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAyB;gBAGvC,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,CAAC,EAAE,mBAAmB,EAC7B,SAAS,CAAC,EAAE,MAAM;IAmBb,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAI/D,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,gBAAgB;IAIjB,UAAU,IAAI,MAAM,GAAG,IAAI,GAAG,MAAM;IAYpC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;CAsB9C"}
|
@@ -0,0 +1,98 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Crawler = void 0;
|
4
|
+
const crawlers_1 = require("./crawler/crawlers");
|
5
|
+
const exclusions_1 = require("./crawler/exclusions");
|
6
|
+
const headers_1 = require("./crawler/headers");
|
7
|
+
class Crawler {
|
8
|
+
crawlers;
|
9
|
+
headers;
|
10
|
+
exclusions;
|
11
|
+
request;
|
12
|
+
compiledRegexList;
|
13
|
+
compiledExclusions;
|
14
|
+
httpHeaders;
|
15
|
+
userAgent;
|
16
|
+
matches;
|
17
|
+
constructor(request, headers, userAgent) {
|
18
|
+
this.crawlers = new crawlers_1.Crawlers();
|
19
|
+
this.headers = new headers_1.Headers();
|
20
|
+
this.exclusions = new exclusions_1.Exclusions();
|
21
|
+
this.request = request ?? {};
|
22
|
+
// The regex-list must not be used with g-flag!
|
23
|
+
// See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
|
24
|
+
this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
|
25
|
+
// The exclusions should be used with g-flag in order to remove each value.
|
26
|
+
this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
|
27
|
+
this.httpHeaders = this.setHttpHeaders(headers);
|
28
|
+
this.userAgent = this.setUserAgent(userAgent);
|
29
|
+
}
|
30
|
+
compileRegex(patterns, flags) {
|
31
|
+
return new RegExp(patterns.join('|'), flags);
|
32
|
+
}
|
33
|
+
setHttpHeaders(headers) {
|
34
|
+
// Use the Request headers if httpHeaders is not defined
|
35
|
+
if (!headers || Object.keys(headers).length === 0) {
|
36
|
+
if (Object.keys(this.request).length) {
|
37
|
+
if (this.request.headers) {
|
38
|
+
return this.request.headers;
|
39
|
+
}
|
40
|
+
}
|
41
|
+
return '';
|
42
|
+
}
|
43
|
+
// Save the headers.
|
44
|
+
return headers;
|
45
|
+
}
|
46
|
+
setUserAgent(userAgent) {
|
47
|
+
if (!userAgent?.length) {
|
48
|
+
userAgent = '';
|
49
|
+
for (const header of this.getUaHttpHeaders()) {
|
50
|
+
if (typeof this.httpHeaders === 'object' &&
|
51
|
+
!Array.isArray(this.httpHeaders)) {
|
52
|
+
if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
|
53
|
+
const headerValue = this.httpHeaders[header.toLowerCase()];
|
54
|
+
if (typeof headerValue === 'string') {
|
55
|
+
const separator = userAgent.length > 0 ? ' ' : '';
|
56
|
+
userAgent += separator + headerValue;
|
57
|
+
}
|
58
|
+
else if (Array.isArray(headerValue)) {
|
59
|
+
const separator = userAgent.length > 0 ? ' ' : '';
|
60
|
+
userAgent += separator + headerValue.join(' ');
|
61
|
+
}
|
62
|
+
}
|
63
|
+
}
|
64
|
+
}
|
65
|
+
}
|
66
|
+
return userAgent;
|
67
|
+
}
|
68
|
+
getUaHttpHeaders() {
|
69
|
+
return this.headers.getAll();
|
70
|
+
}
|
71
|
+
getMatches() {
|
72
|
+
if (this.matches !== undefined) {
|
73
|
+
if (this.matches?.length) {
|
74
|
+
return this.matches[0];
|
75
|
+
}
|
76
|
+
return null;
|
77
|
+
}
|
78
|
+
return {};
|
79
|
+
}
|
80
|
+
isCrawler(userAgent) {
|
81
|
+
if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
|
82
|
+
return false;
|
83
|
+
}
|
84
|
+
let agent = userAgent ?? this.userAgent;
|
85
|
+
// test on compiled regx
|
86
|
+
agent = agent.replace(this.compiledExclusions, '');
|
87
|
+
if (agent.trim().length === 0) {
|
88
|
+
return false;
|
89
|
+
}
|
90
|
+
const matches = this.compiledRegexList.exec(agent);
|
91
|
+
if (matches) {
|
92
|
+
this.matches = matches;
|
93
|
+
}
|
94
|
+
return matches !== null && matches.length > 0;
|
95
|
+
}
|
96
|
+
}
|
97
|
+
exports.Crawler = Crawler;
|
98
|
+
//# sourceMappingURL=crawler.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":";;;AAGA,iDAA8C;AAC9C,qDAAkD;AAClD,+CAA4C;AAE5C,MAAa,OAAO;IACV,QAAQ,CAAW;IACnB,OAAO,CAAU;IACjB,UAAU,CAAa;IAEvB,OAAO,CAA2C;IAClD,iBAAiB,CAAS;IAC1B,kBAAkB,CAAS;IAC3B,WAAW,CAA0C;IACrD,SAAS,CAAS;IAClB,OAAO,CAA0B;IAEzC,YACE,OAAiB,EACjB,OAA6B,EAC7B,SAAkB;QAElB,IAAI,CAAC,QAAQ,GAAG,IAAI,mBAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAE7B,+CAA+C;QAC/C,yGAAyG;QACzG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAExE,2EAA2E;QAC3E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;QAE5E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAEM,YAAY,CAAC,QAAkB,EAAE,KAAc;QACpD,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEO,cAAc,CACpB,OAAiD;QAEjD,wDAAwD;QACxD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,oBAAoB;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,SAAkB;QACrC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YACvB,SAAS,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC7C,IACE,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ;oBACpC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAChC,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;wBAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;wBAE3D,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;4BACpC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC;wBACvC,CAAC;6BAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;4BACtC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,gBAAgB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC/B,CAAC;IAEM,UAAU;QACf,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,SAAS,CAAC,SAAkB;QACjC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACtD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAExC,wBAAwB;QACxB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,CAAC;CACF;AAxHD,0BAwHC"}
|
package/dist/types.d.ts
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
import { NextFunction, Request, Response } from 'express-serve-static-core';
|
2
|
+
import { Crawler } from './lib/crawler';
|
3
|
+
export type Middleware = (request: Request, response: Response, next: NextFunction) => Promise<Response>;
|
4
|
+
declare module 'express-serve-static-core' {
|
5
|
+
interface Request {
|
6
|
+
Crawler: Crawler;
|
7
|
+
}
|
8
|
+
}
|
9
|
+
//# sourceMappingURL=types.d.ts.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAE5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,MAAM,MAAM,UAAU,GAAG,CACvB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,YAAY,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB,OAAO,QAAQ,2BAA2B,CAAC;IACzC,UAAU,OAAO;QACf,OAAO,EAAE,OAAO,CAAC;KAClB;CACF"}
|
package/dist/types.js
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
@@ -0,0 +1,62 @@
|
|
1
|
+
import typescriptEslint from "@typescript-eslint/eslint-plugin";
|
2
|
+
import simpleImportSort from "eslint-plugin-simple-import-sort";
|
3
|
+
import tsParser from "@typescript-eslint/parser";
|
4
|
+
import path from "node:path";
|
5
|
+
import { fileURLToPath } from "node:url";
|
6
|
+
import js from "@eslint/js";
|
7
|
+
import { FlatCompat } from "@eslint/eslintrc";
|
8
|
+
|
9
|
+
const __filename = fileURLToPath(import.meta.url);
|
10
|
+
const __dirname = path.dirname(__filename);
|
11
|
+
const compat = new FlatCompat({
|
12
|
+
baseDirectory: __dirname,
|
13
|
+
recommendedConfig: js.configs.recommended,
|
14
|
+
allConfig: js.configs.all
|
15
|
+
});
|
16
|
+
|
17
|
+
export default [{
|
18
|
+
ignores: [
|
19
|
+
"!**/.*",
|
20
|
+
"**/coverage/",
|
21
|
+
"**/junit/",
|
22
|
+
"**/dist/",
|
23
|
+
"**/mobiles/",
|
24
|
+
"**/node_modules/*",
|
25
|
+
],
|
26
|
+
|
27
|
+
files: ['**/*.ts', '**/*.tsx'],
|
28
|
+
|
29
|
+
plugins: {
|
30
|
+
"@typescript-eslint": typescriptEslint,
|
31
|
+
"simple-import-sort": simpleImportSort,
|
32
|
+
},
|
33
|
+
|
34
|
+
languageOptions: {
|
35
|
+
parser: tsParser,
|
36
|
+
parserOptions: {
|
37
|
+
project: "./tsconfig.json",
|
38
|
+
},
|
39
|
+
},
|
40
|
+
|
41
|
+
rules: {
|
42
|
+
"@typescript-eslint/no-restricted-types": "error",
|
43
|
+
"@typescript-eslint/no-explicit-any": "warn",
|
44
|
+
|
45
|
+
"max-len": ["warn", {
|
46
|
+
code: 80,
|
47
|
+
tabWidth: 2,
|
48
|
+
comments: 80,
|
49
|
+
ignoreComments: true,
|
50
|
+
ignoreTrailingComments: true,
|
51
|
+
ignoreUrls: true,
|
52
|
+
ignoreStrings: true,
|
53
|
+
ignoreTemplateLiterals: true,
|
54
|
+
ignoreRegExpLiterals: true,
|
55
|
+
}],
|
56
|
+
|
57
|
+
"@typescript-eslint/no-unused-vars": ["warn"],
|
58
|
+
|
59
|
+
"simple-import-sort/imports": "error",
|
60
|
+
"simple-import-sort/exports": "error",
|
61
|
+
},
|
62
|
+
}];
|