logstash-input-blueliv 1.0.1 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +37 -10
- data/lib/logstash/inputs/blueliv.rb +88 -21
- data/logstash-input-blueliv.gemspec +1 -1
- metadata +28 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 239a66eaf18c2e97d86c305e8150ea1c3600656e
|
4
|
+
data.tar.gz: d463808c18f1980f338dc13c4308c2ff4942e79a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 287345891c6a327df5d89630b95f05d425d4e25c3f54538f1355c017e587915a372cb446aef990fcd758bf8b36c8ee1880153d4d786b9624f4023cd16148fdfc
|
7
|
+
data.tar.gz: 8478591f6323e46467bf516ac5dae592f013499b2cd112ca113e19680ea9227916263c8f88d9c8eb92faab4ebc2dbec1e33a800f057edff02ee593720124d2e1
|
data/README.md
CHANGED
@@ -6,12 +6,19 @@ This is an input plugin for [Logstash](https://github.com/elasticsearch/logstash
|
|
6
6
|
|
7
7
|
* API key (get yours <a href="https://map.blueliv.com" target="_blank">here</a>)
|
8
8
|
* Logstash >= 1.5.0
|
9
|
-
*
|
9
|
+
* ElasticSearch >= 2.0.0 (Tested on 2.4.0)
|
10
10
|
|
11
11
|
## Installing
|
12
12
|
|
13
13
|
```
|
14
|
-
|
14
|
+
# In logstash version < 2.3
|
15
|
+
|
16
|
+
$LS_HOME/bin/plugin install --version 1.1.0 logstash-input-blueliv
|
17
|
+
|
18
|
+
# In logstash version >2.3
|
19
|
+
$LS_HOME/bin/logstash-plugin install logstash-input-blueliv
|
20
|
+
|
21
|
+
|
15
22
|
```
|
16
23
|
|
17
24
|
### Configuration
|
@@ -23,21 +30,33 @@ This plugin has the following configuration parameters:
|
|
23
30
|
+ ``http_timeout``(default: ``500`` seconds): HTTP timeout for each API call.
|
24
31
|
+ ``feeds``: It is a [hash](http://ruby-doc.org/core-1.9.3/Hash.html) that specifies the parameters to access each one of our feeds. Each feed may be configured with the following properties:
|
25
32
|
+ ``active`` (default: ``false``): if the feed is active or not.
|
26
|
-
+ ``feed_type`` (default: ``test``): the type of the feed that you want. For **Crime Servers** apart from ``test`` (for _debug_ purposes) you
|
33
|
+
+ ``feed_type`` (default: ``test``): the type of the feed that you want. For **Crime Servers** apart from ``test`` (for _debug_ purposes) you have ``recent`` (1 hour updates) and ``last`` (15 minutes updates). As of **Bot IPs** you may choose between ``non_pos`` (all BotIPs **but** the ones from Point-of-Sale), ``pos`` (only from POS) or ``full`` (all of them) feed.
|
27
34
|
+ ``interval`` (default: ``600`` seconds for BotIPs and ``900`` seconds for Crime Servers). The intervall of polling data from our API.
|
28
35
|
|
29
36
|
The default configuration for ``feeds`` field is the following:
|
30
37
|
```javascript
|
31
|
-
|
38
|
+
{
|
39
|
+
"attacks" => {
|
32
40
|
"active" => false,
|
33
|
-
"feed_type" => "
|
34
|
-
"interval" =>
|
41
|
+
"feed_type" => "recent",
|
42
|
+
"interval" => 600
|
35
43
|
},
|
36
44
|
"botips" => {
|
37
45
|
"active" => false,
|
38
46
|
"feed_type" => "test",
|
39
47
|
"interval" => 600
|
48
|
+
},
|
49
|
+
"crimeservers" => {
|
50
|
+
"active" => true,
|
51
|
+
"feed_type" => "test",
|
52
|
+
"interval" => 900
|
53
|
+
},
|
54
|
+
"malwares" => {
|
55
|
+
"active" => false,
|
56
|
+
"feed_type" => "recent",
|
57
|
+
"interval" => 3600
|
40
58
|
}
|
59
|
+
}
|
41
60
|
```
|
42
61
|
|
43
62
|
|
@@ -48,18 +67,26 @@ input {
|
|
48
67
|
blueliv {
|
49
68
|
api_key => "<YOUR API KEY>"
|
50
69
|
feeds => {
|
70
|
+
"attacks" => {
|
71
|
+
"active" => "true"
|
72
|
+
"feed_type" => "recent"
|
73
|
+
}
|
51
74
|
"botips" => {
|
52
|
-
"active" => true
|
75
|
+
"active" => "true"
|
53
76
|
"feed_type" => "non_pos"
|
54
77
|
}
|
55
78
|
"crimeservers" => {
|
56
|
-
"active" => true
|
57
|
-
"feed_type" => "
|
79
|
+
"active" => "true"
|
80
|
+
"feed_type" => "recent"
|
81
|
+
}
|
82
|
+
"malwares" => {
|
83
|
+
"active" => "true"
|
84
|
+
"feed_type" => "recent"
|
58
85
|
}
|
59
86
|
}
|
60
|
-
}
|
61
87
|
}
|
62
88
|
```
|
89
|
+
|
63
90
|
Be aware that if you do not specify a given field, the default value will be configured. In this case, we did not touch the ``interval`` field for the feeds, so the defaults will apply.
|
64
91
|
|
65
92
|
## Need Help?
|
@@ -8,7 +8,7 @@ require "rest-client"
|
|
8
8
|
require "securerandom"
|
9
9
|
|
10
10
|
|
11
|
-
USER_AGENT = "Logstash
|
11
|
+
USER_AGENT = "Logstash 1.1.0"
|
12
12
|
API_CLIENT = "6ee37a93-d064-464b-b4c1-c37e9656273f"
|
13
13
|
|
14
14
|
RESOURCES = {
|
@@ -32,35 +32,69 @@ RESOURCES = {
|
|
32
32
|
:endpoint => "/v1/ip",
|
33
33
|
:feeds => {
|
34
34
|
:non_pos => {
|
35
|
-
600 => "/
|
36
|
-
3600 => "/
|
35
|
+
600 => "/last",
|
36
|
+
3600 => "/recent"
|
37
37
|
},
|
38
38
|
:pos => {
|
39
|
-
600 => "/pos/
|
40
|
-
3600 => "/pos/
|
39
|
+
600 => "/pos/last",
|
40
|
+
3600 => "/pos/recent"
|
41
41
|
},
|
42
42
|
:full => {
|
43
|
-
600 => "/full/
|
44
|
-
3600 => "/full/
|
43
|
+
600 => "/full/last",
|
44
|
+
3600 => "/full/recent"
|
45
45
|
},
|
46
46
|
:test => {
|
47
47
|
600 => "/test"
|
48
48
|
}
|
49
49
|
}
|
50
|
+
},
|
51
|
+
:attacks => {
|
52
|
+
:items => "attacks",
|
53
|
+
:endpoint => "/v1/attack",
|
54
|
+
:feeds => {
|
55
|
+
:last => {
|
56
|
+
1800 => "/last"
|
57
|
+
},
|
58
|
+
:recent => {
|
59
|
+
10800 => "/recent"
|
60
|
+
}
|
61
|
+
}
|
62
|
+
},
|
63
|
+
:malwares => {
|
64
|
+
:items => "malwares",
|
65
|
+
:endpoint => "/v1/malware",
|
66
|
+
:feeds => {
|
67
|
+
:last => {
|
68
|
+
600 => "/last"
|
69
|
+
},
|
70
|
+
:recent => {
|
71
|
+
3600 => "/recent"
|
72
|
+
}
|
73
|
+
}
|
50
74
|
}
|
51
75
|
}
|
52
76
|
|
53
77
|
DEFAULT_CONFIG = {
|
78
|
+
"attacks" => {
|
79
|
+
"active" => false,
|
80
|
+
"feed_type" => "recent",
|
81
|
+
"interval" => 600
|
82
|
+
},
|
83
|
+
"botips" => {
|
84
|
+
"active" => false,
|
85
|
+
"feed_type" => "test",
|
86
|
+
"interval" => 600
|
87
|
+
},
|
54
88
|
"crimeservers" => {
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
89
|
+
"active" => true,
|
90
|
+
"feed_type" => "test",
|
91
|
+
"interval" => 900
|
92
|
+
},
|
93
|
+
"malwares" => {
|
94
|
+
"active" => false,
|
95
|
+
"feed_type" => "recent",
|
96
|
+
"interval" => 3600
|
97
|
+
}
|
64
98
|
}
|
65
99
|
|
66
100
|
INITIALIZE_FILE = "blueliv.ini"
|
@@ -93,8 +127,8 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
|
|
93
127
|
def run(queue)
|
94
128
|
threads = []
|
95
129
|
@feeds.each do |name, conf|
|
96
|
-
if
|
97
|
-
url, interval = get_url(name,
|
130
|
+
if conf["active"] == 'true'
|
131
|
+
url, interval = get_url(name, conf["feed_type"], conf["interval"])
|
98
132
|
threads << Thread.new{get_feed_each(queue, name, url, interval)}
|
99
133
|
end
|
100
134
|
end
|
@@ -165,10 +199,11 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
|
|
165
199
|
response = client.get("#{url}?key=#{API_CLIENT}", :Authorization => @auth, :timeout => @timeout,
|
166
200
|
:user_agent => USER_AGENT, :headers => {"X-API-CLIENT" => API_CLIENT})
|
167
201
|
response_json = JSON.parse(response.body)
|
168
|
-
|
202
|
+
keyItems = RESOURCES[name.to_sym][:items]
|
203
|
+
items = response_json[keyItems]
|
169
204
|
items.each do |it|
|
170
|
-
|
171
|
-
|
205
|
+
collection = keyItems.downcase
|
206
|
+
it = send("map_"+collection, it)
|
172
207
|
it["@collection"] = collection
|
173
208
|
it["document_id"] = if it.has_key?("_id") then it["_id"] else SecureRandom.base64(32) end
|
174
209
|
it.delete("_id") if it.has_key?("_id")
|
@@ -198,6 +233,7 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
|
|
198
233
|
end
|
199
234
|
rescue Exception => e
|
200
235
|
@logger.info("Will retry in #{FAILURE_SLEEP} seconds")
|
236
|
+
@logger.info(e.message+"\n"+e.backtrace.join("\n"))
|
201
237
|
sleep(FAILURE_SLEEP)
|
202
238
|
end
|
203
239
|
end
|
@@ -207,4 +243,35 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
|
|
207
243
|
RestClient
|
208
244
|
end
|
209
245
|
|
246
|
+
private
|
247
|
+
def map_global_location(it)
|
248
|
+
it["location"] = [it["longitude"].to_f, it["latitude"].to_f]
|
249
|
+
return it
|
250
|
+
end
|
251
|
+
|
252
|
+
def map_ips(it)
|
253
|
+
map_global_location(it)
|
254
|
+
end
|
255
|
+
|
256
|
+
def map_crimeservers(it)
|
257
|
+
map_global_location(it)
|
258
|
+
end
|
259
|
+
|
260
|
+
def map_attacks(it)
|
261
|
+
source = it["source"];
|
262
|
+
#map_global_location(source);
|
263
|
+
it["location"] = [source["longitude"].to_f, source["latitude"].to_f]
|
264
|
+
firstEvent = DateTime.parse(it["firstEvent"])
|
265
|
+
lastEvent = DateTime.parse(it["lastEvent"])
|
266
|
+
duration = ((lastEvent - firstEvent)* 24 * 60 * 60).to_i
|
267
|
+
if duration < 0
|
268
|
+
duration = -duration
|
269
|
+
end
|
270
|
+
it[:duration] = duration
|
271
|
+
return it
|
272
|
+
end
|
273
|
+
def map_malwares(it)
|
274
|
+
it["_id"] = it["sha256"]
|
275
|
+
return it
|
276
|
+
end
|
210
277
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = "logstash-input-blueliv"
|
3
|
-
s.version = "1.0
|
3
|
+
s.version = "1.1.0"
|
4
4
|
s.licenses = ["Apache License (2.0)"]
|
5
5
|
s.summary = "This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds."
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,78 +1,78 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-blueliv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Blueliv
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - '>='
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 1.4.0
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
|
-
- -
|
16
|
+
- - ">="
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: 1.4.0
|
19
|
+
name: logstash-core
|
25
20
|
prerelease: false
|
26
21
|
type: :runtime
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: logstash-codec-plain
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ">="
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
26
|
+
version: 1.4.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
|
-
- -
|
30
|
+
- - ">="
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '0'
|
33
|
+
name: logstash-codec-plain
|
39
34
|
prerelease: false
|
40
35
|
type: :runtime
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: rest-client
|
43
36
|
version_requirements: !ruby/object:Gem::Requirement
|
44
37
|
requirements:
|
45
|
-
- -
|
38
|
+
- - ">="
|
46
39
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
48
42
|
requirement: !ruby/object:Gem::Requirement
|
49
43
|
requirements:
|
50
|
-
- - ~>
|
44
|
+
- - "~>"
|
51
45
|
- !ruby/object:Gem::Version
|
52
46
|
version: 1.8.0
|
47
|
+
name: rest-client
|
53
48
|
prerelease: false
|
54
49
|
type: :runtime
|
55
|
-
- !ruby/object:Gem::Dependency
|
56
|
-
name: logstash-devutils
|
57
50
|
version_requirements: !ruby/object:Gem::Requirement
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - "~>"
|
60
53
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
54
|
+
version: 1.8.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
62
56
|
requirement: !ruby/object:Gem::Requirement
|
63
57
|
requirements:
|
64
|
-
- -
|
58
|
+
- - ">="
|
65
59
|
- !ruby/object:Gem::Version
|
66
60
|
version: '0'
|
61
|
+
name: logstash-devutils
|
67
62
|
prerelease: false
|
68
63
|
type: :development
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
69
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program
|
70
70
|
email: community@blueliv.com
|
71
71
|
executables: []
|
72
72
|
extensions: []
|
73
73
|
extra_rdoc_files: []
|
74
74
|
files:
|
75
|
-
- .gitignore
|
75
|
+
- ".gitignore"
|
76
76
|
- CHANGELOG.md
|
77
77
|
- Gemfile
|
78
78
|
- LICENSE
|
@@ -93,17 +93,17 @@ require_paths:
|
|
93
93
|
- lib
|
94
94
|
required_ruby_version: !ruby/object:Gem::Requirement
|
95
95
|
requirements:
|
96
|
-
- -
|
96
|
+
- - ">="
|
97
97
|
- !ruby/object:Gem::Version
|
98
98
|
version: '0'
|
99
99
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
requirements: []
|
105
105
|
rubyforge_project:
|
106
|
-
rubygems_version: 2.4.
|
106
|
+
rubygems_version: 2.4.8
|
107
107
|
signing_key:
|
108
108
|
specification_version: 4
|
109
109
|
summary: This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds.
|