es6-crawler-detect 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/example/node/server.js +38 -32
- package/package.json +1 -1
- package/src/lib/crawler/crawlers.js +1403 -1402
- package/src/lib/crawler/exclusions.js +62 -62
- package/src/lib/crawler.js +130 -135
- package/test/lib/crawler.test.js +94 -52
- package/test/lib/database/crawlers.txt +3651 -0
- package/test/lib/database/devices.txt +165636 -0
package/example/node/server.js
CHANGED
@@ -1,32 +1,38 @@
|
|
1
|
-
'use strict';
|
2
|
-
|
3
|
-
const path = require('path');
|
4
|
-
const express = require('express');
|
5
|
-
const { middleware } = require('../../src/index');
|
6
|
-
|
7
|
-
const app = express();
|
8
|
-
const port = 3000;
|
9
|
-
|
10
|
-
app.use(
|
11
|
-
middleware(() => {
|
12
|
-
console.log('Testing the callback\n');
|
13
|
-
})
|
14
|
-
);
|
15
|
-
|
16
|
-
app.use('/dist', express.static(path.join(__dirname + '/dist')));
|
17
|
-
|
18
|
-
app.get('/', function (req, res) {
|
19
|
-
res.sendFile(path.join(__dirname + '/index.html'));
|
20
|
-
});
|
21
|
-
|
22
|
-
app.get('/crawler', function async(request, response) {
|
23
|
-
// or check a user agent string
|
24
|
-
request.Crawler.isCrawler(
|
25
|
-
'
|
26
|
-
);
|
27
|
-
|
28
|
-
// Output the name of the bot that matched (if any)
|
29
|
-
response.send(request.Crawler.getMatches());
|
30
|
-
});
|
31
|
-
|
32
|
-
app.
|
1
|
+
'use strict';
|
2
|
+
|
3
|
+
const path = require('path');
|
4
|
+
const express = require('express');
|
5
|
+
const { middleware, Crawler } = require('../../src/index');
|
6
|
+
|
7
|
+
const app = express();
|
8
|
+
const port = 3000;
|
9
|
+
|
10
|
+
app.use(
|
11
|
+
middleware(() => {
|
12
|
+
console.log('Testing the callback\n');
|
13
|
+
})
|
14
|
+
);
|
15
|
+
|
16
|
+
app.use('/dist', express.static(path.join(__dirname + '/dist')));
|
17
|
+
|
18
|
+
app.get('/', function (req, res) {
|
19
|
+
res.sendFile(path.join(__dirname + '/index.html'));
|
20
|
+
});
|
21
|
+
|
22
|
+
app.get('/crawler', function async(request, response) {
|
23
|
+
// or check a user agent string
|
24
|
+
request.Crawler.isCrawler(
|
25
|
+
'TinEye-bot/0.51 (see http://www.tineye.com/crawler.html)'
|
26
|
+
);
|
27
|
+
|
28
|
+
// Output the name of the bot that matched (if any)
|
29
|
+
response.send(request.Crawler.getMatches());
|
30
|
+
});
|
31
|
+
|
32
|
+
app.get('/curl', function async(request, response) {
|
33
|
+
var CrawlerDetector = new Crawler(request);
|
34
|
+
CrawlerDetector.isCrawler(); // true
|
35
|
+
response.send(CrawlerDetector.getMatches());
|
36
|
+
});
|
37
|
+
|
38
|
+
app.listen(port, () => console.log(`Example app listening on port ${port}!`));
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "es6-crawler-detect",
|
3
|
-
"version": "3.
|
3
|
+
"version": "3.3.0",
|
4
4
|
"description": "This is an ES6 adaptation of the original PHP library CrawlerDetect, this library will help you detect bots/crawlers/spiders vie the useragent.",
|
5
5
|
"main": "src/index.js",
|
6
6
|
"directories": {
|