es6-crawler-detect 4.0.0-beta.1 → 4.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/crawler/crawlers.d.ts +5 -0
- package/dist/lib/crawler/crawlers.d.ts.map +1 -0
- package/dist/lib/crawler/crawlers.js +1400 -0
- package/dist/lib/crawler/crawlers.js.map +1 -0
- package/dist/lib/crawler/exclusions.d.ts +5 -0
- package/dist/lib/crawler/exclusions.d.ts.map +1 -0
- package/dist/lib/crawler/exclusions.js +59 -0
- package/dist/lib/crawler/exclusions.js.map +1 -0
- package/dist/lib/crawler/headers.d.ts +5 -0
- package/dist/lib/crawler/headers.d.ts.map +1 -0
- package/dist/lib/crawler/headers.js +21 -0
- package/dist/lib/crawler/headers.js.map +1 -0
- package/dist/lib/crawler/provider.d.ts +4 -0
- package/dist/lib/crawler/provider.d.ts.map +1 -0
- package/dist/lib/crawler/provider.js +3 -0
- package/dist/lib/crawler/provider.js.map +1 -0
- package/dist/lib/crawler.d.ts +21 -0
- package/dist/lib/crawler.d.ts.map +1 -0
- package/dist/lib/crawler.js +98 -0
- package/dist/lib/crawler.js.map +1 -0
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +3 -0
- package/dist/types.js.map +1 -0
- package/package.json +3 -2
- package/tsconfig.json +13 -1
- package/dist/main.bundle.js +0 -1
- package/example/node/dist/crawler.min.js +0 -1
- package/example/node/server.js +0 -38
- package/src/index.js +0 -22
- package/src/lib/crawler/crawlers.js +0 -1399
- package/src/lib/crawler/exclusions.js +0 -58
- package/src/lib/crawler/headers.js +0 -20
- package/src/lib/crawler/provider.js +0 -2
- package/src/lib/crawler.js +0 -98
- package/src/types.js +0 -2
- package/test/lib/crawler.spec.js +0 -149
- package/webpack.common.js +0 -20
- package/webpack.dev.js +0 -7
- package/webpack.prod.js +0 -14
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawlers.js","sourceRoot":"","sources":["../../../src/lib/crawler/crawlers.ts"],"names":[],"mappings":";;;AAEA,MAAa,QAAQ;IACZ,MAAM;QACX,OAAO;YACL,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,sCAAsC;YACtC,OAAO;YACP,YAAY;YACZ,eAAe;YACf,aAAa;YACb,QAAQ;YACR,WAAW;YACX,UAAU;YACV,OAAO;YACP,iBAAiB;YACjB,UAAU;YACV,OAAO;YACP,SAAS;YACT,WAAW;YACX,UAAU;YACV,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,wBAAwB;YACxB,cAAc;YACd,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,SAAS;YACT,SAAS;YACT,gBAAgB;YAChB,SAAS;YACT,SAAS;YACT,UAAU;YACV,SAAS;YACT,oBAAoB;YACpB,SAAS;YACT,cAAc;YACd,WAAW;YACX,OAAO;YACP,cAAc;YACd,kBAAkB;YAClB,cAAc;YACd,OAAO;YACP,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,UAAU;YACV,UAAU;YACV,UAAU;YACV,SAAS;YACT,QAAQ;YACR,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kBAAkB;YAClB,uBAAuB;YACvB,OAAO;YACP,0BAA0B;YAC1B,UAAU;YACV,WAAW;YACX,QAAQ;YACR,SAAS;YACT,SAAS;YACT,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,WAAW;YACX,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,kBAAkB;YAClB,kBAAkB;YAClB,sBAAsB;YACtB,WAAW;YACX,KAAK;YACL,OAAO;YACP,YAAY;YACZ,SAAS;YACT,cAAc;YACd,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,+BAA+B;YAC/B,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,QAAQ;YACR,UAAU;YACV,wBAAwB;YACxB,SAAS;YACT,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,eAAe;YACf,WAAW;YACX,kBAAkB;YAClB,eAAe;YACf,cAAc;YACd,mBAAmB;YACnB,wBAAwB;YACxB,mBAAmB;YACnB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,aAAa;YACb,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,kBAAkB;YAClB,cAAc;YACd,SAAS;YACT,WAAW;YACX,eAAe;YACf,OAAO;YACP,WAAW;YACX,UAAU;YACV,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,WAAW;YACX,UAAU;YACV,mBAAmB;YACnB,UAAU;YACV,YAAY;YACZ,sBAAsB;YACtB,UAAU;YACV,gBAAgB;YAChB,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,SAAS;YACT,kBAAkB;YAClB,QAAQ;YACR,aAAa;YACb,QAAQ;YACR,YAAY;YACZ,UAAU;YACV,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,SAAS;YACT,kBAAkB;YAClB,sBAAsB;YACtB,YAAY;YACZ,SAAS;YACT,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,QAAQ;YACR,mBAAmB;YACnB,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,0BAA0B;YAC1B,YAAY;YACZ,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,WAAW;YACX,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,oBAAoB;YACpB,oBAAoB;YACpB,oBAAoB;YACpB,cAAc;YACd,aAAa;YACb,YAAY;YACZ,WAAW;YACX,cAAc;YACd,QAAQ;YACR,SAAS;YACT,OAAO;YACP,WAAW;YACX,UAAU;YACV,eAAe;YACf,aAAa;YACb,cAAc;YACd,UAAU;YACV,cAAc;YACd,SAAS;YACT,UAAU;YACV,eAAe;YACf,WAAW;YACX,eAAe;YACf,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,4BAA4B;YAC5B,iBAAiB;YACjB,6BAA6B;YAC7B,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,cAAc;YACd,WAAW;YACX,WAAW;YACX,cAAc;YACd,cAAc;YACd,WAAW;YACX,UAAU;YACV,aAAa;YACb,QAAQ;YACR,UAAU;YACV,gBAAgB;YAChB,kCAAkC;YAClC,oBAAoB;YACpB,QAAQ;YACR,kBAAkB;YAClB,YAAY;YACZ,sBAAsB;YACtB,eAAe;YACf,aAAa;YACb,yBAAyB;YACzB,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,YAAY;YACZ,SAAS;YACT,WAAW;YACX,oBAAoB;YACpB,YAAY;YACZ,kBAAkB;YAClB,gBAAgB;YAChB,gCAAgC;YAChC,SAAS;YACT,eAAe;YACf,mBAAmB;YACnB,gBAAgB;YAChB,WAAW;YACX,WAAW;YACX,qBAAqB;YACrB,aAAa;YACb,SAAS;YACT,UAAU;YACV,QAAQ;YACR,WAAW;YACX,QAAQ;YACR,UAAU;YACV,SAAS;YACT,MAAM;YACN,gBAAgB;YAChB,MAAM;YACN,YAAY;YACZ,YAAY;YACZ,eAAe;YACf,WAAW;YACX,sBAAsB;YACtB,WAAW;YACX,eAAe;YACf,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,sBAAsB;YACtB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,OAAO;YACP,OAAO;YACP,oDAAoD;YACpD,OAAO;YACP,MAAM;YACN,WAAW;YACX,eAAe;YACf,WAAW;YACX,wBAAwB;YACxB,aAAa;YACb,WAAW;YACX,aAAa;YACb,MAAM;YACN,WAAW;YACX,kBAAkB;YAClB,SAAS;YACT,qBAAqB;YACrB,gBAAgB;YAChB,YAAY;YACZ,oBAAoB;YACpB,yBAAyB;YACzB,6BAA6B;YAC7B,aAAa;YACb,yBAAyB;YACzB,iBAAiB;YACjB,cAAc;YACd,kBAAkB;YAClB,MAAM;YACN,SAAS;YACT,4CAA4C;YAC5C,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,oBAAoB;YACpB,UAAU;YACV,YAAY;YACZ,QAAQ;YACR,WAAW;YACX,eAAe;YACf,gBAAgB;YAChB,QAAQ;YACR,MAAM;YACN,aAAa;YACb,MAAM;YACN,YAAY;YACZ,cAAc;YACd,SAAS;YACT,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,aAAa;YACb,mBAAmB;YACnB,SAAS;YACT,SAAS;YACT,oBAAoB;YACpB,WAAW;YACX,yBAAyB;YACzB,wBAAwB;YACxB,QAAQ;YACR,SAAS;YACT,aAAa;YACb,kBAAkB;YAClB,UAAU;YACV,MAAM;YACN,SAAS;YACT,aAAa;YACb,qBAAqB;YACrB,wBAAwB;YACxB,UAAU;YACV,QAAQ;YACR,qBAAqB;YACrB,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,oBAAoB;YACpB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,eAAe;YACf,aAAa;YACb,SAAS;YACT,aAAa;YACb,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,gBAAgB;YAChB,aAAa;YACb,UAAU;YACV,eAAe;YACf,aAAa;YACb,gBAAgB;YAChB,YAAY;YACZ,WAAW;YACX,eAAe;YACf,eAAe;YACf,QAAQ;YACR,eAAe;YACf,WAAW;YACX,OAAO;YACP,UAAU;YACV,cAAc;YACd,UAAU;YACV,uBAAuB;YACvB,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,UAAU;YACV,UAAU;YACV,cAAc;YACd,iBAAiB;YACjB,cAAc;YACd,UAAU;YACV,YAAY;YACZ,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,cAAc;YACd,eAAe;YACf,YAAY;YACZ,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,eAAe;YACf,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,qBAAqB;YACrB,UAAU;YACV,aAAa;YACb,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,2BAA2B;YAC3B,UAAU;YACV,iBAAiB;YACjB,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,iBAAiB;YACjB,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,mBAAmB;YACnB,gBAAgB;YAChB,gBAAgB;YAChB,2BAA2B;YAC3B,qBAAqB;YACrB,4BAA4B;YAC5B,mBAAmB;YACnB,uBAAuB;YACvB,oBAAoB;YACpB,gCAAgC;YAChC,qBAAqB;YACrB,gBAAgB;YAChB,oBAAoB;YACpB,0BAA0B;YAC1B,yBAAyB;YACzB,yBAAyB;YACzB,gBAAgB;YAChB,yBAAyB;YACzB,mBAAmB;YACnB,sBAAsB;YACtB,0BAA0B;YAC1B,iBAAiB;YACjB,eAAe;YACf,qCAAqC;YACrC,4BAA4B;YAC5B,gBAAgB;YAChB,sBAAsB;YACtB,YAAY;YACZ,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,aAAa;YACb,uBAAuB;YACvB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,WAAW;YACX,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,SAAS;YACT,WAAW;YACX,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,YAAY;YACZ,SAAS;YACT,2BAA2B;YAC3B,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,oBAAoB;YACpB,WAAW;YACX,QAAQ;YACR,OAAO;YACP,UAAU;YACV,gBAAgB;YAChB,eAAe;YACf,oBAAoB;YACpB,qBAAqB;YACrB,UAAU;YACV,WAAW;YACX,YAAY;YACZ,UAAU;YACV,kBAAkB;YAClB,SAAS;YACT,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,uBAAuB;YACvB,mBAAmB;YACnB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,OAAO;YACP,YAAY;YACZ,SAAS;YACT,uBAAuB;YACvB,UAAU;YACV,qBAAqB;YACrB,UAAU;YACV,iBAAiB;YACjB,WAAW;YACX,YAAY;YACZ,4BAA4B;YAC5B,gBAAgB;YAChB,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,uBAAuB;YACvB,UAAU;YACV,eAAe;YACf,gBAAgB;YAChB,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,uBAAuB;YACvB,YAAY;YACZ,eAAe;YACf,OAAO;YACP,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,4BAA4B;YAC5B,YAAY;YACZ,UAAU;YACV,YAAY;YACZ,eAAe;YACf,aAAa;YACb,aAAa;YACb,cAAc;YACd,gBAAgB;YAChB,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,qBAAqB;YACrB,eAAe;YACf,cAAc;YACd,SAAS;YACT,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,6BAA6B;YAC7B,cAAc;YACd,UAAU;YACV,YAAY;YACZ,WAAW;YACX,aAAa;YACb,aAAa;YACb,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,uBAAuB;YACvB,eAAe;YACf,kBAAkB;YAClB,YAAY;YACZ,MAAM;YACN,KAAK;YACL,SAAS;YACT,WAAW;YACX,aAAa;YACb,4BAA4B;YAC5B,SAAS;YACT,MAAM;YACN,QAAQ;YACR,cAAc;YACd,SAAS;YACT,WAAW;YACX,iBAAiB;YACjB,QAAQ;YACR,UAAU;YACV,OAAO;YACP,WAAW;YACX,UAAU;YACV,gBAAgB;YAChB,cAAc;YACd,UAAU;YACV,WAAW;YACX,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,mBAAmB;YACnB,MAAM;YACN,WAAW;YACX,gBAAgB;YAChB,oBAAoB;YACpB,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,oBAAoB;YACpB,UAAU;YACV,iCAAiC;YACjC,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,OAAO;YACP,YAAY;YACZ,SAAS;YACT,YAAY;YACZ,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,cAAc;YACd,UAAU;YACV,UAAU;YACV,aAAa;YACb,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,OAAO;YACP,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,WAAW;YACX,SAAS;YACT,cAAc;YACd,aAAa;YACb,UAAU;YACV,aAAa;YACb,UAAU;YACV,cAAc;YACd,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,iBAAiB;YACjB,qBAAqB;YACrB,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,mBAAmB;YACnB,aAAa;YACb,mBAAmB;YACnB,OAAO;YACP,gBAAgB;YAChB,yBAAyB;YACzB,kBAAkB;YAClB,aAAa;YACb,aAAa;YACb,aAAa;YACb,OAAO;YACP,UAAU;YACV,WAAW;YACX,YAAY;YACZ,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,WAAW;YACX,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,aAAa;YACb,sBAAsB;YACtB,YAAY;YACZ,gBAAgB;YAChB,eAAe;YACf,aAAa;YACb,UAAU;YACV,aAAa;YACb,SAAS;YACT,iBAAiB;YACjB,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,mBAAmB;YACnB,uCAAuC;YACvC,4BAA4B;YAC5B,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,iBAAiB;YACjB,YAAY;YACZ,iBAAiB;YACjB,oBAAoB;YACpB,SAAS;YACT,aAAa;YACb,UAAU;YACV,QAAQ;YACR,wBAAwB;YACxB,SAAS;YACT,eAAe;YACf,UAAU;YACV,WAAW;YACX,UAAU;YACV,UAAU;YACV,yBAAyB;YACzB,eAAe;YACf,aAAa;YACb,QAAQ;YACR,UAAU;YACV,kBAAkB;YAClB,iCAAiC;YACjC,aAAa;YACb,QAAQ;YACR,aAAa;YACb,cAAc;YACd,WAAW;YACX,cAAc;YACd,eAAe;YACf,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,cAAc;YACd,aAAa;YACb,kBAAkB;YAClB,SAAS;YACT,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,aAAa;YACb,SAAS;YACT,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,aAAa;YACb,wBAAwB;YACxB,YAAY;YACZ,oBAAoB;YACpB,QAAQ;YACR,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,WAAW;YACX,QAAQ;YACR,cAAc;YACd,qBAAqB;YACrB,WAAW;YACX,SAAS;YACT,SAAS;YACT,WAAW;YACX,WAAW;YACX,OAAO;YACP,iBAAiB;YACjB,kBAAkB;YAClB,uBAAuB;YACvB,YAAY;YACZ,iBAAiB;YACjB,aAAa;YACb,WAAW;YACX,UAAU;YACV,WAAW;YACX,oBAAoB;YACpB,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,aAAa;YACb,gBAAgB;YAChB,MAAM;YACN,OAAO;YACP,QAAQ;YACR,iBAAiB;YACjB,UAAU;YACV,SAAS;YACT,KAAK;YACL,gBAAgB;YAChB,WAAW;YACX,SAAS;YACT,MAAM;YACN,kBAAkB;YAClB,mBAAmB;YACnB,UAAU;YACV,QAAQ;YACR,QAAQ;YACR,MAAM;YACN,qBAAqB;YACrB,iBAAiB;YACjB,yBAAyB;YACzB,UAAU;YACV,kBAAkB;YAClB,aAAa;YACb,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,aAAa;YACb,iBAAiB;YACjB,iBAAiB;YACjB,aAAa;YACb,OAAO;YACP,OAAO;YACP,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,aAAa;YACb,YAAY;YACZ,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,YAAY;YACZ,cAAc;YACd,WAAW;YACX,eAAe;YACf,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,OAAO;YACP,YAAY;YACZ,WAAW;YACX,YAAY;YACZ,kBAAkB;YAClB,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,MAAM;YACN,WAAW;YACX,SAAS;YACT,wBAAwB;YACxB,cAAc;YACd,WAAW;YACX,cAAc;YACd,cAAc;YACd,YAAY;YACZ,UAAU;YACV,WAAW;YACX,eAAe;YACf,WAAW;YACX,aAAa;YACb,gBAAgB;YAChB,SAAS;YACT,YAAY;YACZ,UAAU;YACV,iBAAiB;YACjB,iBAAiB;YACjB,QAAQ;YACR,SAAS;YACT,YAAY;YACZ,UAAU;YACV,mBAAmB;YACnB,SAAS;YACT,eAAe;YACf,kBAAkB;YAClB,cAAc;YACd,QAAQ;YACR,kBAAkB;YAClB,2BAA2B;YAC3B,aAAa;YACb,QAAQ;YACR,SAAS;YACT,cAAc;YACd,SAAS;YACT,0BAA0B;YAC1B,gBAAgB;YAChB,mBAAmB;YACnB,UAAU;YACV,UAAU;YACV,eAAe;YACf,QAAQ;YACR,WAAW;YACX,8BAA8B;YAC9B,eAAe;YACf,aAAa;YACb,OAAO;YACP,aAAa;YACb,gBAAgB;YAChB,mBAAmB;YACnB,aAAa;YACb,WAAW;YACX,cAAc;YACd,UAAU;YACV,sBAAsB;YACtB,uBAAuB;YACvB,OAAO;YACP,eAAe;YACf,MAAM;YACN,iBAAiB;YACjB,cAAc;YACd,iBAAiB;YACjB,eAAe;YACf,gBAAgB;YAChB,YAAY;YACZ,YAAY;YACZ,QAAQ;YACR,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,iBAAiB;YACjB,UAAU;YACV,oBAAoB;YACpB,UAAU;YACV,SAAS;YACT,yBAAyB;YACzB,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,sBAAsB;YACtB,cAAc;YACd,cAAc;YACd,cAAc;YACd,aAAa;YACb,cAAc;YACd,yBAAyB;YACzB,YAAY;YACZ,UAAU;YACV,cAAc;YACd,YAAY;YACZ,cAAc;YACd,WAAW;YACX,OAAO;YACP,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,kBAAkB;YAClB,WAAW;YACX,SAAS;YACT,UAAU;YACV,aAAa;YACb,WAAW;YACX,YAAY;YACZ,gBAAgB;YAChB,WAAW;YACX,QAAQ;YACR,aAAa;YACb,oBAAoB;YACpB,kBAAkB;YAClB,WAAW;YACX,kCAAkC;YAClC,UAAU;YACV,QAAQ;YACR,eAAe;YACf,aAAa;YACb,sBAAsB;YACtB,WAAW;YACX,YAAY;YACZ,OAAO;YACP,SAAS;YACT,4BAA4B;YAC5B,UAAU;YACV,iBAAiB;YACjB,wBAAwB;YACxB,QAAQ;YACR,WAAW;YACX,mBAAmB;YACnB,SAAS;YACT,aAAa;YACb,UAAU;YACV,2BAA2B;YAC3B,cAAc;YACd,eAAe;YACf,aAAa;YACb,UAAU;YACV,uBAAuB;YACvB,QAAQ;YACR,mBAAmB;YACnB,eAAe;YACf,uBAAuB;YACvB,SAAS;YACT,cAAc;YACd,WAAW;YACX,aAAa;YACb,YAAY;YACZ,kBAAkB;YAClB,UAAU;YACV,WAAW;YACX,WAAW;YACX,UAAU;YACV,UAAU;YACV,UAAU;YACV,QAAQ;YACR,aAAa;YACb,aAAa;YACb,WAAW;YACX,gBAAgB;YAChB,UAAU;YACV,WAAW;YACX,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,QAAQ;YACR,WAAW;YACX,UAAU;YACV,oBAAoB;YACpB,qBAAqB;YACrB,cAAc;YACd,SAAS;YACT,OAAO;YACP,WAAW;YACX,YAAY;YACZ,QAAQ;YACR,SAAS;YACT,aAAa;YACb,cAAc;YACd,UAAU;YACV,SAAS;YACT,UAAU;YACV,aAAa;YACb,YAAY;YACZ,cAAc;YACd,cAAc;YACd,aAAa;YACb,aAAa;YACb,uBAAuB;YACvB,kBAAkB;YAClB,aAAa;YACb,iBAAiB;YACjB,aAAa;YACb,YAAY;YACZ,WAAW;YACX,WAAW;YACX,cAAc;YACd,iBAAiB;YACjB,UAAU;YACV,mBAAmB;YACnB,cAAc;YACd,OAAO;YACP,WAAW;YACX,eAAe;YACf,mBAAmB;YACnB,eAAe;YACf,OAAO;YACP,QAAQ;YACR,YAAY;YACZ,YAAY;YACZ,UAAU;YACV,aAAa;YACb,QAAQ;YACR,iBAAiB;YACjB,WAAW;YACX,UAAU;YACV,UAAU;YACV,gBAAgB;YAChB,YAAY;YACZ,aAAa;YACb,SAAS;YACT,SAAS;YACT,YAAY;YACZ,WAAW;YACX,eAAe;YACf,QAAQ;YACR,OAAO;YACP,SAAS;YACT,WAAW;YACX,WAAW;YACX,UAAU;YACV,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,UAAU;YACV,WAAW;YACX,cAAc;YACd,eAAe;YACf,aAAa;YACb,YAAY;YACZ,SAAS;YACT,iBAAiB;YACjB,WAAW;YACX,aAAa;YACb,YAAY;YACZ,aAAa;YACb,QAAQ;YACR,QAAQ;YACR,SAAS;YACT,WAAW;YACX,gBAAgB;YAChB,SAAS;YACT,OAAO;YACP,oBAAoB;YACpB,qBAAqB;YACrB,YAAY;YACZ,eAAe;YACf,qBAAqB;YACrB,SAAS;YACT,SAAS;YACT,SAAS;YACT,aAAa;YACb,SAAS;YACT,cAAc;YACd,aAAa;YACb,cAAc;YACd,eAAe;YACf,mBAAmB;YACnB,aAAa;YACb,UAAU;YACV,UAAU;YACV,aAAa;YACb,aAAa;YACb,YAAY;YACZ,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,SAAS;YACT,aAAa;YACb,sBAAsB;YACtB,iBAAiB;YACjB,+BAA+B;YAC/B,kBAAkB;YAClB,kBAAkB;YAClB,kBAAkB;YAClB,UAAU;YACV,UAAU;YACV,SAAS;YACT,YAAY;YACZ,aAAa;YACb,cAAc;YACd,QAAQ;YACR,eAAe;YACf,aAAa;YACb,OAAO;YACP,SAAS;YACT,cAAc;YACd,eAAe;YACf,aAAa;YACb,WAAW;YACX,aAAa;YACb,aAAa;YACb,WAAW;YACX,aAAa;YACb,UAAU;YACV,UAAU;YACV,YAAY;YACZ,UAAU;YACV,UAAU;YACV,cAAc;YACd,cAAc;YACd,aAAa;YACb,SAAS;YACT,OAAO;YACP,QAAQ;YACR,SAAS;YACT,oBAAoB;YACpB,UAAU;YACV,0BAA0B;YAC1B,WAAW;YACX,WAAW;YACX,yBAAyB;YACzB,SAAS;YACT,cAAc;YACd,qBAAqB;YACrB,aAAa;YACb,gBAAgB;YAChB,QAAQ;YACR,eAAe;YACf,SAAS;YACT,oBAAoB;YACpB,QAAQ;YACR,SAAS;YACT,cAAc;YACd,WAAW;YACX,SAAS;YACT,aAAa;YACb,SAAS;YACT,WAAW;YACX,2BAA2B;YAC3B,cAAc;YACd,gBAAgB;YAChB,aAAa;YACb,QAAQ;YACR,WAAW;YACX,YAAY;YACZ,OAAO;YACP,KAAK;YACL,kCAAkC;YAClC,UAAU;YACV,aAAa;YACb,SAAS;YACT,SAAS;YACT,MAAM;YACN,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,MAAM;YACN,YAAY;YACZ,eAAe;YACf,cAAc;YACd,aAAa;YACb,SAAS;YACT,qBAAqB;YACrB,cAAc;YACd,qBAAqB;YACrB,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,UAAU;YACV,UAAU;YACV,aAAa;YACb,cAAc;YACd,WAAW;YACX,UAAU;YACV,SAAS;YACT,YAAY;YACZ,YAAY;YACZ,YAAY;YACZ,mBAAmB;YACnB,aAAa;YACb,SAAS;YACT,aAAa;YACb,SAAS;YACT,YAAY;YACZ,cAAc;YACd,YAAY;YACZ,YAAY;YACZ,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,aAAa;YACb,UAAU;YACV,SAAS;YACT,UAAU;YACV,UAAU;YACV,mBAAmB;YACnB,WAAW;YACX,UAAU;YACV,YAAY;YACZ,YAAY;YACZ,iBAAiB;YACjB,SAAS;YACT,QAAQ;YACR,WAAW;YACX,WAAW;YACX,aAAa;YACb,SAAS;YACT,SAAS;YACT,iBAAiB;YACjB,oBAAoB;YACpB,gBAAgB;YAChB,UAAU;YACV,YAAY;YACZ,SAAS;YACT,aAAa;YACb,WAAW;YACX,YAAY;YACZ,cAAc;YACd,aAAa;YACb,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,MAAM;YACN,OAAO;YACP,MAAM;YACN,UAAU;YACV,MAAM;YACN,SAAS;YACT,UAAU;YACV,WAAW;YACX,SAAS;YACT,cAAc;YACd,QAAQ;YACR,WAAW;YACX,iBAAiB;YACjB,gBAAgB;YAChB,sBAAsB;YACtB,2BAA2B;YAC3B,YAAY;YACZ,WAAW;YACX,UAAU;YACV,QAAQ;YACR,MAAM;YACN,oBAAoB;YACpB,eAAe;YACf,cAAc;YACd,kBAAkB;YAClB,SAAS;YACT,eAAe;YACf,QAAQ;YACR,mCAAmC;YACnC,WAAW;YACX,MAAM;YACN,sBAAsB;YACtB,QAAQ;YACR,SAAS;YACT,QAAQ;YACR,iBAAiB;YACjB,eAAe;YACf,eAAe;YACf,gBAAgB;YAChB,UAAU;YACV,oBAAoB;YACpB,YAAY;YACZ,YAAY;YACZ,0BAA0B;YAC1B,YAAY;YACZ,kBAAkB;YAClB,yBAAyB;YACzB,sBAAsB;YACtB,qBAAqB;YACrB,OAAO;YACP,MAAM;YACN,oBAAoB;YACpB,kBAAkB;YAClB,gBAAgB;YAChB,YAAY;YACZ,QAAQ;YACR,kBAAkB;YAClB,OAAO;YACP,MAAM;YACN,OAAO;YACP,gBAAgB;YAChB,cAAc;YACd,kBAAkB;YAClB,eAAe;YACf,oBAAoB;YACpB,cAAc;YACd,YAAY;YACZ,SAAS;YACT,QAAQ;YACR,MAAM;YACN,KAAK;YACL,OAAO;YACP,oBAAoB;YACpB,wBAAwB;YACxB,kBAAkB;YAClB,SAAS;YACT,OAAO;YACP,OAAO;YACP,YAAY;YACZ,QAAQ;YACR,aAAa;YACb,YAAY;YACZ,yBAAyB;YACzB,QAAQ;YACR,wIAAwI;SACzI,CAAC;IACJ,CAAC;CACF;AAl3CD,4BAk3CC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"exclusions.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,UAAW,YAAW,QAAQ;IACzC,MAAM,IAAI,MAAM,EAAE;CAoDnB"}
|
@@ -0,0 +1,59 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Exclusions = void 0;
|
4
|
+
class Exclusions {
|
5
|
+
getAll() {
|
6
|
+
return [
|
7
|
+
'Safari.[\\d\\.]*',
|
8
|
+
'Firefox.[\\d\\.]*',
|
9
|
+
' Chrome.[\\d\\.]*',
|
10
|
+
'Chromium.[\\d\\.]*',
|
11
|
+
'MSIE.[\\d\\.]',
|
12
|
+
'Opera\\/[\\d\\.]*',
|
13
|
+
'Mozilla.[\\d\\.]*',
|
14
|
+
'AppleWebKit.[\\d\\.]*',
|
15
|
+
'Trident.[\\d\\.]*',
|
16
|
+
'Windows NT.[\\d\\.]*',
|
17
|
+
'Android [\\d\\.]*',
|
18
|
+
'Macintosh.',
|
19
|
+
'Ubuntu',
|
20
|
+
'Linux',
|
21
|
+
'[ ]Intel',
|
22
|
+
'Mac OS X [\\d_]*',
|
23
|
+
'(like )?Gecko(.[\\d\\.]*)?',
|
24
|
+
'KHTML,',
|
25
|
+
'CriOS.[\\d\\.]*',
|
26
|
+
'CPU iPhone OS ([0-9_])* like Mac OS X',
|
27
|
+
'CPU OS ([0-9_])* like Mac OS X',
|
28
|
+
'iPod',
|
29
|
+
'compatible',
|
30
|
+
'x86_..',
|
31
|
+
'i686',
|
32
|
+
'x64',
|
33
|
+
'X11',
|
34
|
+
'rv:[\\d\\.]*',
|
35
|
+
'Version.[\\d\\.]*',
|
36
|
+
'WOW64',
|
37
|
+
'Win64',
|
38
|
+
'Dalvik.[\\d\\.]*',
|
39
|
+
' \\.NET CLR [\\d\\.]*',
|
40
|
+
'Presto.[\\d\\.]*',
|
41
|
+
'Media Center PC',
|
42
|
+
'BlackBerry',
|
43
|
+
'Build',
|
44
|
+
'Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.',
|
45
|
+
'Opera',
|
46
|
+
' \\.NET[\\d\\.]*',
|
47
|
+
'cubot',
|
48
|
+
'; M bot',
|
49
|
+
'; CRONO',
|
50
|
+
'; B bot',
|
51
|
+
'; IDbot',
|
52
|
+
'; ID bot',
|
53
|
+
'; POWER BOT',
|
54
|
+
'OCTOPUS-CORE',
|
55
|
+
];
|
56
|
+
}
|
57
|
+
}
|
58
|
+
exports.Exclusions = Exclusions;
|
59
|
+
//# sourceMappingURL=exclusions.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"exclusions.js","sourceRoot":"","sources":["../../../src/lib/crawler/exclusions.ts"],"names":[],"mappings":";;;AAEA,MAAa,UAAU;IACrB,MAAM;QACJ,OAAO;YACL,kBAAkB;YAClB,mBAAmB;YACnB,mBAAmB;YACnB,oBAAoB;YACpB,eAAe;YACf,mBAAmB;YACnB,mBAAmB;YACnB,uBAAuB;YACvB,mBAAmB;YACnB,sBAAsB;YACtB,mBAAmB;YACnB,YAAY;YACZ,QAAQ;YACR,OAAO;YACP,UAAU;YACV,kBAAkB;YAClB,4BAA4B;YAC5B,QAAQ;YACR,iBAAiB;YACjB,uCAAuC;YACvC,gCAAgC;YAChC,MAAM;YACN,YAAY;YACZ,QAAQ;YACR,MAAM;YACN,KAAK;YACL,KAAK;YACL,cAAc;YACd,mBAAmB;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,uBAAuB;YACvB,kBAAkB;YAClB,iBAAiB;YACjB,YAAY;YACZ,OAAO;YACP,4DAA4D;YAC5D,OAAO;YACP,kBAAkB;YAClB,OAAO;YACP,SAAS;YACT,SAAS;YACT,SAAS;YACT,SAAS;YACT,UAAU;YACV,aAAa;YACb,cAAc;SACf,CAAC;IACJ,CAAC;CACF;AArDD,gCAqDC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,OAAQ,YAAW,QAAQ;IACtC,MAAM,IAAI,MAAM,EAAE;CAcnB"}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Headers = void 0;
|
4
|
+
class Headers {
|
5
|
+
getAll() {
|
6
|
+
return [
|
7
|
+
'USER-AGENT',
|
8
|
+
'X-OPERAMINI-PHONE-UA',
|
9
|
+
'X-DEVICE-USER-AGENT',
|
10
|
+
'X-ORIGINAL-USER-AGENT',
|
11
|
+
'X-SKYFIRE-PHONE',
|
12
|
+
'X-BOLT-PHONE-UA',
|
13
|
+
'DEVICE-STOCK-UA',
|
14
|
+
'X-UCBROWSER-DEVICE-UA',
|
15
|
+
'FROM',
|
16
|
+
'X-SCANNER',
|
17
|
+
];
|
18
|
+
}
|
19
|
+
}
|
20
|
+
exports.Headers = Headers;
|
21
|
+
//# sourceMappingURL=headers.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../../src/lib/crawler/headers.ts"],"names":[],"mappings":";;;AAEA,MAAa,OAAO;IAClB,MAAM;QACJ,OAAO;YACL,YAAY;YACZ,sBAAsB;YACtB,qBAAqB;YACrB,uBAAuB;YACvB,iBAAiB;YACjB,iBAAiB;YACjB,iBAAiB;YACjB,uBAAuB;YACvB,MAAM;YACN,WAAW;SACZ,CAAC;IACJ,CAAC;CACF;AAfD,0BAeC"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,MAAM,IAAI,MAAM,EAAE,CAAC;CACpB"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/lib/crawler/provider.ts"],"names":[],"mappings":""}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
import { Request } from 'express-serve-static-core';
|
2
|
+
import { IncomingHttpHeaders } from 'http2';
|
3
|
+
export declare class Crawler {
|
4
|
+
private crawlers;
|
5
|
+
private headers;
|
6
|
+
private exclusions;
|
7
|
+
private request;
|
8
|
+
private compiledRegexList;
|
9
|
+
private compiledExclusions;
|
10
|
+
private httpHeaders;
|
11
|
+
private userAgent;
|
12
|
+
private matches?;
|
13
|
+
constructor(request?: Request, headers?: IncomingHttpHeaders, userAgent?: string);
|
14
|
+
compileRegex(patterns: string[], flags?: string): RegExp;
|
15
|
+
private setHttpHeaders;
|
16
|
+
private setUserAgent;
|
17
|
+
private getUaHttpHeaders;
|
18
|
+
getMatches(): string | null | object;
|
19
|
+
isCrawler(userAgent?: string): boolean;
|
20
|
+
}
|
21
|
+
//# sourceMappingURL=crawler.d.ts.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,2BAA2B,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,OAAO,CAAC;AAM5C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAW;IAC3B,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAE/B,OAAO,CAAC,OAAO,CAA2C;IAC1D,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAyB;gBAGvC,OAAO,CAAC,EAAE,OAAO,EACjB,OAAO,CAAC,EAAE,mBAAmB,EAC7B,SAAS,CAAC,EAAE,MAAM;IAmBb,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM;IAI/D,OAAO,CAAC,cAAc;IAkBtB,OAAO,CAAC,YAAY;IA0BpB,OAAO,CAAC,gBAAgB;IAIjB,UAAU,IAAI,MAAM,GAAG,IAAI,GAAG,MAAM;IAYpC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO;CAsB9C"}
|
@@ -0,0 +1,98 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.Crawler = void 0;
|
4
|
+
const crawlers_1 = require("./crawler/crawlers");
|
5
|
+
const exclusions_1 = require("./crawler/exclusions");
|
6
|
+
const headers_1 = require("./crawler/headers");
|
7
|
+
class Crawler {
|
8
|
+
crawlers;
|
9
|
+
headers;
|
10
|
+
exclusions;
|
11
|
+
request;
|
12
|
+
compiledRegexList;
|
13
|
+
compiledExclusions;
|
14
|
+
httpHeaders;
|
15
|
+
userAgent;
|
16
|
+
matches;
|
17
|
+
constructor(request, headers, userAgent) {
|
18
|
+
this.crawlers = new crawlers_1.Crawlers();
|
19
|
+
this.headers = new headers_1.Headers();
|
20
|
+
this.exclusions = new exclusions_1.Exclusions();
|
21
|
+
this.request = request ?? {};
|
22
|
+
// The regex-list must not be used with g-flag!
|
23
|
+
// See: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
|
24
|
+
this.compiledRegexList = this.compileRegex(this.crawlers.getAll(), 'i');
|
25
|
+
// The exclusions should be used with g-flag in order to remove each value.
|
26
|
+
this.compiledExclusions = this.compileRegex(this.exclusions.getAll(), 'gi');
|
27
|
+
this.httpHeaders = this.setHttpHeaders(headers);
|
28
|
+
this.userAgent = this.setUserAgent(userAgent);
|
29
|
+
}
|
30
|
+
compileRegex(patterns, flags) {
|
31
|
+
return new RegExp(patterns.join('|'), flags);
|
32
|
+
}
|
33
|
+
setHttpHeaders(headers) {
|
34
|
+
// Use the Request headers if httpHeaders is not defined
|
35
|
+
if (!headers || Object.keys(headers).length === 0) {
|
36
|
+
if (Object.keys(this.request).length) {
|
37
|
+
if (this.request.headers) {
|
38
|
+
return this.request.headers;
|
39
|
+
}
|
40
|
+
}
|
41
|
+
return '';
|
42
|
+
}
|
43
|
+
// Save the headers.
|
44
|
+
return headers;
|
45
|
+
}
|
46
|
+
setUserAgent(userAgent) {
|
47
|
+
if (!userAgent?.length) {
|
48
|
+
userAgent = '';
|
49
|
+
for (const header of this.getUaHttpHeaders()) {
|
50
|
+
if (typeof this.httpHeaders === 'object' &&
|
51
|
+
!Array.isArray(this.httpHeaders)) {
|
52
|
+
if (Object.hasOwn(this.httpHeaders, header.toLowerCase())) {
|
53
|
+
const headerValue = this.httpHeaders[header.toLowerCase()];
|
54
|
+
if (typeof headerValue === 'string') {
|
55
|
+
const separator = userAgent.length > 0 ? ' ' : '';
|
56
|
+
userAgent += separator + headerValue;
|
57
|
+
}
|
58
|
+
else if (Array.isArray(headerValue)) {
|
59
|
+
const separator = userAgent.length > 0 ? ' ' : '';
|
60
|
+
userAgent += separator + headerValue.join(' ');
|
61
|
+
}
|
62
|
+
}
|
63
|
+
}
|
64
|
+
}
|
65
|
+
}
|
66
|
+
return userAgent;
|
67
|
+
}
|
68
|
+
getUaHttpHeaders() {
|
69
|
+
return this.headers.getAll();
|
70
|
+
}
|
71
|
+
getMatches() {
|
72
|
+
if (this.matches !== undefined) {
|
73
|
+
if (this.matches?.length) {
|
74
|
+
return this.matches[0];
|
75
|
+
}
|
76
|
+
return null;
|
77
|
+
}
|
78
|
+
return {};
|
79
|
+
}
|
80
|
+
isCrawler(userAgent) {
|
81
|
+
if (Buffer.byteLength(userAgent ?? '', 'utf8') > 4096) {
|
82
|
+
return false;
|
83
|
+
}
|
84
|
+
let agent = userAgent ?? this.userAgent;
|
85
|
+
// test on compiled regx
|
86
|
+
agent = agent.replace(this.compiledExclusions, '');
|
87
|
+
if (agent.trim().length === 0) {
|
88
|
+
return false;
|
89
|
+
}
|
90
|
+
const matches = this.compiledRegexList.exec(agent);
|
91
|
+
if (matches) {
|
92
|
+
this.matches = matches;
|
93
|
+
}
|
94
|
+
return matches !== null && matches.length > 0;
|
95
|
+
}
|
96
|
+
}
|
97
|
+
exports.Crawler = Crawler;
|
98
|
+
//# sourceMappingURL=crawler.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"crawler.js","sourceRoot":"","sources":["../../src/lib/crawler.ts"],"names":[],"mappings":";;;AAGA,iDAA8C;AAC9C,qDAAkD;AAClD,+CAA4C;AAE5C,MAAa,OAAO;IACV,QAAQ,CAAW;IACnB,OAAO,CAAU;IACjB,UAAU,CAAa;IAEvB,OAAO,CAA2C;IAClD,iBAAiB,CAAS;IAC1B,kBAAkB,CAAS;IAC3B,WAAW,CAA0C;IACrD,SAAS,CAAS;IAClB,OAAO,CAA0B;IAEzC,YACE,OAAiB,EACjB,OAA6B,EAC7B,SAAkB;QAElB,IAAI,CAAC,QAAQ,GAAG,IAAI,mBAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,iBAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAU,EAAE,CAAC;QAEnC,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,EAAE,CAAC;QAE7B,+CAA+C;QAC/C,yGAAyG;QACzG,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QAExE,2EAA2E;QAC3E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;QAE5E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAEM,YAAY,CAAC,QAAkB,EAAE,KAAc;QACpD,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/C,CAAC;IAEO,cAAc,CACpB,OAAiD;QAEjD,wDAAwD;QACxD,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;gBACrC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC9B,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,oBAAoB;QACpB,OAAO,OAAO,CAAC;IACjB,CAAC;IAEO,YAAY,CAAC,SAAkB;QACrC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YACvB,SAAS,GAAG,EAAE,CAAC;YACf,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;gBAC7C,IACE,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ;oBACpC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAChC,CAAC;oBACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;wBAC1D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;wBAE3D,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;4BACpC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC;wBACvC,CAAC;6BAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;4BACtC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;4BAClD,SAAS,IAAI,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,gBAAgB;QACtB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IAC/B,CAAC;IAEM,UAAU;QACf,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBACzB,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,CAAC;YAED,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAEM,SAAS,CAAC,SAAkB;QACjC,IAAI,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC;YACtD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,KAAK,GAAG,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAExC,wBAAwB;QACxB,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnD,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACzB,CAAC;QAED,OAAO,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAChD,CAAC;CACF;AAxHD,0BAwHC"}
|
package/dist/types.d.ts
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
import { NextFunction, Request, Response } from 'express-serve-static-core';
|
2
|
+
import { Crawler } from './lib/crawler';
|
3
|
+
export type Middleware = (request: Request, response: Response, next: NextFunction) => Promise<Response>;
|
4
|
+
declare module 'express-serve-static-core' {
|
5
|
+
interface Request {
|
6
|
+
Crawler: Crawler;
|
7
|
+
}
|
8
|
+
}
|
9
|
+
//# sourceMappingURL=types.d.ts.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAE5E,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAExC,MAAM,MAAM,UAAU,GAAG,CACvB,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,IAAI,EAAE,YAAY,KACf,OAAO,CAAC,QAAQ,CAAC,CAAC;AAEvB,OAAO,QAAQ,2BAA2B,CAAC;IACzC,UAAU,OAAO;QACf,OAAO,EAAE,OAAO,CAAC;KAClB;CACF"}
|
package/dist/types.js
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "es6-crawler-detect",
|
3
|
-
"version": "4.0.0-beta.
|
3
|
+
"version": "4.0.0-beta.2",
|
4
4
|
"description": "This is an ES6 adaptation of the original PHP library CrawlerDetect, this library will help you detect bots/crawlers/spiders vie the useragent.",
|
5
5
|
"main": "src/index.js",
|
6
6
|
"directories": {
|
@@ -60,5 +60,6 @@
|
|
60
60
|
"webpack": "^5.94.0",
|
61
61
|
"webpack-cli": "^5.1.4",
|
62
62
|
"webpack-merge": "^6.0.1"
|
63
|
-
}
|
63
|
+
},
|
64
|
+
"types": "dist/index.d.ts"
|
64
65
|
}
|
package/tsconfig.json
CHANGED
@@ -8,9 +8,21 @@
|
|
8
8
|
"module": "node16",
|
9
9
|
"target": "es2022",
|
10
10
|
|
11
|
+
"declaration": true,
|
12
|
+
"declarationMap": true,
|
13
|
+
"sourceMap": true,
|
14
|
+
"emitDeclarationOnly": false,
|
15
|
+
"outDir": "dist",
|
16
|
+
|
11
17
|
"strict": true,
|
12
18
|
"esModuleInterop": true,
|
13
19
|
"skipLibCheck": true,
|
14
20
|
"moduleResolution": "node16"
|
15
|
-
}
|
21
|
+
},
|
22
|
+
"ts-node": {
|
23
|
+
"esm": true,
|
24
|
+
},
|
25
|
+
|
26
|
+
"include": ["src/**/*"],
|
27
|
+
"exclude": ["node_modules", "dist"]
|
16
28
|
}
|
package/dist/main.bundle.js
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
var Crawler;(()=>{var e={556:(e,t,o)=>{const r=o(636);e.exports={Crawler:r,middleware:e=>(t,o,i)=>{"function"==typeof e&&e.call(t,o),t.Crawler=new r(t),i()}}},636:(e,t,o)=>{"use strict";const r=o(10),i=o(902),a=o(317);e.exports=class{constructor(e,t,o){this._init(),this.request="object"==typeof e?e:{},this.compiledRegexList=this.compileRegex(this.crawlers.getAll(),"i"),this.compiledExclusions=this.compileRegex(this.exclusions.getAll(),"gi"),this.setHttpHeaders(t),this.userAgent=this.setUserAgent(o)}_init(){this.crawlers=new r,this.headers=new a,this.exclusions=new i}compileRegex(e,t){return new RegExp(e.join("|"),t)}setHttpHeaders(e){void 0!==e&&0!==Object.keys(e).length||(e=Object.keys(this.request).length?this.request.headers:{}),this.httpHeaders=e}setUserAgent(e){if(null==e||!e.length){e="";for(const t of this.getUaHttpHeaders())if(Object.keys(this.httpHeaders).indexOf(t.toLowerCase())>=0){e+=(e.length>0?" ":"")+this.httpHeaders[t.toLowerCase()]}}return e}getUaHttpHeaders(){return this.headers.getAll()}isCrawler(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:void 0;if(Buffer.byteLength(e||"","utf8")>4096)return!1;var t=null==e?this.userAgent:e;if(0===(t=t.replace(this.compiledExclusions,"")).trim().length)return!1;var o=this.compiledRegexList.exec(t);return o&&(this.matches=o),null!==o&&!!o.length}getMatches(){return void 0!==this.matches?this.matches.length?this.matches[0]:null:{}}}},10:(e,t,o)=>{"use strict";const r=o(174);e.exports=class extends r{constructor(){super(),this.data=[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\/[0-9]","^Expanse","^FDM ","^git\\/","^Goose\\/","^Grabber","^Gradle\\/","^HTTPClient\\/","^HTTPing","^Java\\/","^Jeode\\/","^Jetty\\/","^Mail\\/","^Mget","^Microsoft URL Control","^Mikrotik\\/","^Netlab360","^NG\\/[0-9\\.]","^NING\\/","^npm\\/","^Nuclei","^PHP-AYMAPI\\/","^PHP\\/","^pip\\/","^pnpm\\/","^RMA\\/","^Ruby|Ruby\\/[0-9]","^Swurl ","^TLS tester ","^twine\\/","^ureq","^VSE\\/[0-9]","^WordPress\\.com","^XRL\\/[0-9]","^ZmEu","008\\/","13TABS","192\\.comAgent","2GDPR\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\/","acoon","acrylicapps\\.com\\/pulp","Acunetix","AdAuth\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\/","Airmail","akka-http\\/","akula\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\/","Apache Ant\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\/","AppBeat\\/","AppEngine-Google","AppleSyndication","Aprc\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\/","Browsershots","BUbiNG","Buck\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\/","BuzzSumo","CAAM\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\/","curb","Curious George","curl","cuwhois\\/","cybo\\.com","DAP\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\/\\+\\/web\\/snippet\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\/","Disqus\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\/\\/drupal\\.org\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\/","Feedspot","FeedViewer\\/","Feedwind\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\/[0-9]","Fever\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\/","Florienzh\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool","free thumbnails","Freeuploader","FreshRSS","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\/","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\/\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\/","HTTP-Tiny","HTTP::Lite","http:\\/\\/www.neomo.de\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\/","ImageVisu\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\/[0-9]","IPWorks HTTP\\/S Component","iqdb\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\/","Kerrigan\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\/","KickFire","KimonoLabs\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","monitis","Monitority\\/","Monit\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\/","muhstik-scan","MVAClient","MxToolbox\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\/","Nymesis","NYU","Observatory\\/","Ocelli\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\/","Photon\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\/","POE-Component-Client-HTTP","Polymail\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\/","redback\\/","ReederForMac","Reeder\\/","ReGet","RepoMonkey","request\\.js","reqwest\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\/","Syndirella\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","TelegramBot","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\/","VSAgent\\/","VSB-TUO\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\/","WebThumbnail","Webthumb\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\/","WordupinfoSearch","Word\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"]}}},902:(e,t,o)=>{"use strict";const r=o(174);e.exports=class extends r{constructor(){super(),this.data=["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE"]}}},317:(e,t,o)=>{"use strict";const r=o(174);e.exports=class extends r{constructor(){super(),this.data=["USER-AGENT","X-OPERAMINI-PHONE-UA","X-DEVICE-USER-AGENT","X-ORIGINAL-USER-AGENT","X-SKYFIRE-PHONE","X-BOLT-PHONE-UA","DEVICE-STOCK-UA","X-UCBROWSER-DEVICE-UA","FROM","X-SCANNER"]}}},174:e=>{"use strict";e.exports=class{constructor(){}getAll(){return this.data}}}},t={};var o=function o(r){var i=t[r];if(void 0!==i)return i.exports;var a=t[r]={exports:{}};return e[r](a,a.exports,o),a.exports}(556);Crawler=o})();
|