logstash-filter-ciseipdb 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +2 -0
- data/LICENSE +14 -0
- data/NOTICE.TXT +12 -0
- data/README.md +25 -0
- data/lib/logstash/filters/ciseipdb.rb +167 -0
- data/logstash-filter-ciseipdb.gemspec +29 -0
- data/spec/filters/ciseipdb_spec.rb +24 -0
- metadata +110 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6d629463df4731b72259a3fd44fbfbdfbc3abad8
|
4
|
+
data.tar.gz: be938079e6e753bdcd0d15511dd7841a41b8bae4
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0edb982adc2ab31aceb74f2a1e2b394dcc0546b3e23314238031027b5786fa1c31907a591365c7ee7cb8ea39936c00d1b739c80839b1684e37d8ec4294ab2b5f
|
7
|
+
data.tar.gz: 9487277cde7ac75c49c769842695f0924f7b168b3ad5d6f8e4e0b5d3cc8382b909c82ba73fee6e1f91c6839b046b5ffc136e17688b5b2de09e5cfd83bf158270
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
|
2
|
+
Copyright (c) 2016 Sohonet <http://www.sohonet.com>
|
3
|
+
|
4
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
you may not use this file except in compliance with the License.
|
6
|
+
You may obtain a copy of the License at
|
7
|
+
|
8
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
|
10
|
+
Unless required by applicable law or agreed to in writing, software
|
11
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
See the License for the specific language governing permissions and
|
14
|
+
limitations under the License.
|
data/NOTICE.TXT
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
logstash-filter-ciseipdb logstash plugin
|
2
|
+
|
3
|
+
Copyright, 2016 Sohonet
|
4
|
+
|
5
|
+
This software contains code derived from logstash-filter-elasticsearch
|
6
|
+
Copyright 2012-2015 Elasticsearch
|
7
|
+
|
8
|
+
Elasticsearch
|
9
|
+
Copyright 2012-2015 Elasticsearch
|
10
|
+
|
11
|
+
This product includes software developed by The Apache Software
|
12
|
+
Foundation (http://www.apache.org/).
|
data/README.md
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# Ciseipdb Logstash Plugin
|
2
|
+
|
3
|
+
[](https://travis-ci.org/sohonetlabs/logstash-filter-ciseipdb)
|
4
|
+
|
5
|
+
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
6
|
+
|
7
|
+
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
8
|
+
|
9
|
+
## Documentation
|
10
|
+
|
11
|
+
This plugin allows you to search for matching IPs in Elasticsearch IP database indexes and add that information into events.
|
12
|
+
|
13
|
+
This is intended to work with [generate-ipdatabase](https://github.com/sohonetlabs/generate-ipdatabase) which will create the Elasticsearch IP database entries.
|
14
|
+
|
15
|
+
Matching IPs are cached in redis.
|
16
|
+
|
17
|
+
Example:
|
18
|
+
|
19
|
+
ciseipdb {
|
20
|
+
hosts => [ "elasticsearch" ]
|
21
|
+
indexes => [ "ipdatabase" ]
|
22
|
+
ipaddress => "%{ip_dst}"
|
23
|
+
target => "dst_info"
|
24
|
+
redis_host => "localhost'
|
25
|
+
}
|
@@ -0,0 +1,167 @@
|
|
1
|
+
require "logstash/filters/base"
|
2
|
+
require "logstash/namespace"
|
3
|
+
require "base64"
|
4
|
+
|
5
|
+
# Search elasticsearch for matching IPs in Elasticsearch IP database indexes i
|
6
|
+
# and add that information into events.
|
7
|
+
#
|
8
|
+
# Caches matching IPs in redis.
|
9
|
+
#
|
10
|
+
# Example:
|
11
|
+
#
|
12
|
+
# ciseipdb {
|
13
|
+
# hosts => [ "elasticsearch" ]
|
14
|
+
# indexes => [ "ipdatabase" ]
|
15
|
+
# ipaddress => "%{ip_dst}"
|
16
|
+
# target => "dst_info"
|
17
|
+
# }
|
18
|
+
|
19
|
+
class LogStash::Filters::Ciseipdb < LogStash::Filters::Base
|
20
|
+
config_name "ciseipdb"
|
21
|
+
|
22
|
+
# List of elasticsearch hosts to use for querying.
|
23
|
+
config :hosts, :validate => :array, :required => true
|
24
|
+
|
25
|
+
# List of indexes to perform the search query against.
|
26
|
+
config :indexes, :validate => :array, :default => [""]
|
27
|
+
|
28
|
+
# IP Addresses
|
29
|
+
config :ipaddress, :validate => :string, :required => true
|
30
|
+
|
31
|
+
# Target field for added fields
|
32
|
+
config :target, :validate => :string, :required => true
|
33
|
+
|
34
|
+
# Basic Auth - username
|
35
|
+
config :user, :validate => :string
|
36
|
+
|
37
|
+
# Basic Auth - password
|
38
|
+
config :password, :validate => :password
|
39
|
+
|
40
|
+
# SSL
|
41
|
+
config :ssl, :validate => :boolean, :default => false
|
42
|
+
|
43
|
+
# SSL Certificate Authority file
|
44
|
+
config :ca_file, :validate => :path
|
45
|
+
|
46
|
+
# Redis host
|
47
|
+
config :redis_host, :validate => :string, :default => "localhost"
|
48
|
+
|
49
|
+
# Redis key TTL
|
50
|
+
config :redis_ttl, :validate => :number, :default => 3600
|
51
|
+
|
52
|
+
|
53
|
+
public
|
54
|
+
def register
|
55
|
+
require "elasticsearch"
|
56
|
+
require "redis"
|
57
|
+
|
58
|
+
transport_options = {}
|
59
|
+
|
60
|
+
if @user && @password
|
61
|
+
token = Base64.strict_encode64("#{@user}:#{@password.value}")
|
62
|
+
transport_options[:headers] = { Authorization: "Basic #{token}" }
|
63
|
+
end
|
64
|
+
|
65
|
+
hosts = if @ssl then
|
66
|
+
@hosts.map {|h| { host: h, scheme: 'https' } }
|
67
|
+
else
|
68
|
+
@hosts
|
69
|
+
end
|
70
|
+
|
71
|
+
if @ssl && @ca_file
|
72
|
+
transport_options[:ssl] = { ca_file: @ca_file }
|
73
|
+
end
|
74
|
+
|
75
|
+
@logger.info("New CISE IPDB filter", :hosts => hosts)
|
76
|
+
@client = Elasticsearch::Client.new hosts: hosts, transport_options: transport_options
|
77
|
+
|
78
|
+
@redis = Redis.new(:host => redis_host)
|
79
|
+
end # def register
|
80
|
+
|
81
|
+
public
|
82
|
+
def filter(event)
|
83
|
+
|
84
|
+
ipaddress = event.sprintf(@ipaddress)
|
85
|
+
|
86
|
+
# Check ip address in redis
|
87
|
+
data = check_redis(ipaddress)
|
88
|
+
|
89
|
+
# IP not in redis, lookup elasticsearch, add to redis
|
90
|
+
if data.nil?
|
91
|
+
data = search(ipaddress)
|
92
|
+
update_redis(ipaddress, data)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Update event
|
96
|
+
data.each_pair do |k,v|
|
97
|
+
targetname = "#{@target}_#{k}"
|
98
|
+
event[targetname] = v
|
99
|
+
end
|
100
|
+
filter_matched(event)
|
101
|
+
|
102
|
+
end # def filter
|
103
|
+
|
104
|
+
def search(ip)
|
105
|
+
output = Hash.new
|
106
|
+
|
107
|
+
begin
|
108
|
+
query = {
|
109
|
+
query: {
|
110
|
+
filtered: {
|
111
|
+
filter: {
|
112
|
+
and: [
|
113
|
+
{ term: { IPADDRESS: ip } },
|
114
|
+
{ range: { "@timestamp" => { gte: "now-1d/d", lt: "now" } } }
|
115
|
+
]
|
116
|
+
}
|
117
|
+
}
|
118
|
+
}
|
119
|
+
}
|
120
|
+
results = @client.search index: @indexes, body: query
|
121
|
+
|
122
|
+
if results['hits']['total'] >= 1
|
123
|
+
output['databases'] = Array.new
|
124
|
+
output['reputation_score'] = 0
|
125
|
+
results['hits']['hits'].each do |hit|
|
126
|
+
output['databases'] << hit['_source']['database']['shortname']
|
127
|
+
output['reputation_score'] += hit['_source']['database']['reputation_score'].to_i
|
128
|
+
|
129
|
+
# Extra data from nipap
|
130
|
+
if hit['_source']['database']['shortname'] == 'nipap'
|
131
|
+
output['service_slug'] = hit['_source']['service_slug']
|
132
|
+
output['description'] = hit['_source']['description']
|
133
|
+
output['router'] = hit['_source']['router']
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
rescue => e
|
139
|
+
@logger.debug("No hits for ipaddresses", :query => query, :error => e)
|
140
|
+
end #begin..rescue
|
141
|
+
|
142
|
+
output
|
143
|
+
end # def search
|
144
|
+
|
145
|
+
def check_redis(ip)
|
146
|
+
begin
|
147
|
+
output = @redis.get(ip)
|
148
|
+
if output.nil?
|
149
|
+
output
|
150
|
+
else
|
151
|
+
eval(output)
|
152
|
+
end
|
153
|
+
rescue => e
|
154
|
+
@logger.warn("Problem getting key from redis", :ip => ip, :error => e)
|
155
|
+
end
|
156
|
+
end # def check_redis
|
157
|
+
|
158
|
+
def update_redis(ip, data)
|
159
|
+
begin
|
160
|
+
@redis.set(ip, data)
|
161
|
+
@redis.expire(ip, @redis_ttl)
|
162
|
+
rescue => e
|
163
|
+
@logger.warn("Problem updating redis", :ip => ip, :data => data , :error => e)
|
164
|
+
end
|
165
|
+
end # def update_redis
|
166
|
+
|
167
|
+
end # class LogStash::Filters::Elasticsearch
|
@@ -0,0 +1,29 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
|
3
|
+
s.name = 'logstash-filter-ciseipdb'
|
4
|
+
s.version = '0.10.0'
|
5
|
+
s.licenses = ['Apache-2.0']
|
6
|
+
s.summary = "Lookup and inject IP database information into events"
|
7
|
+
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
8
|
+
s.authors = ["Elastic", "Sohonet"]
|
9
|
+
s.email = 'support@sohonet.com'
|
10
|
+
s.homepage = "https://github.com/sohonetlabs/logstash-filter-ciseipdb"
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
|
24
|
+
s.add_runtime_dependency "elasticsearch", "~> 1.0"
|
25
|
+
s.add_runtime_dependency "redis", ">= 0"
|
26
|
+
|
27
|
+
s.add_development_dependency 'logstash-devutils'
|
28
|
+
end
|
29
|
+
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
require "logstash/devutils/rspec/spec_helper"
|
4
|
+
require "logstash/plugin"
|
5
|
+
require "logstash/filters/ciseipdb"
|
6
|
+
|
7
|
+
describe LogStash::Filters::Ciseipdb do
|
8
|
+
|
9
|
+
let (:cise_config) {{
|
10
|
+
'hosts' => [ 'elasticsearch' ],
|
11
|
+
'ipaddress' => '127.0.0.1',
|
12
|
+
'target' => 'destination',
|
13
|
+
}}
|
14
|
+
|
15
|
+
context "registration" do
|
16
|
+
|
17
|
+
let(:plugin) { LogStash::Plugin.lookup("filter", "ciseipdb").new(cise_config) }
|
18
|
+
|
19
|
+
it "should not raise an exception" do
|
20
|
+
expect {plugin.register}.to_not raise_error
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-filter-ciseipdb
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.10.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Elastic
|
8
|
+
- Sohonet
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2016-08-10 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.0'
|
20
|
+
name: logstash-core-plugin-api
|
21
|
+
prerelease: false
|
22
|
+
type: :runtime
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.0'
|
34
|
+
name: elasticsearch
|
35
|
+
prerelease: false
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '1.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
name: redis
|
49
|
+
prerelease: false
|
50
|
+
type: :runtime
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
name: logstash-devutils
|
63
|
+
prerelease: false
|
64
|
+
type: :development
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
71
|
+
email: support@sohonet.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- Gemfile
|
77
|
+
- LICENSE
|
78
|
+
- NOTICE.TXT
|
79
|
+
- README.md
|
80
|
+
- lib/logstash/filters/ciseipdb.rb
|
81
|
+
- logstash-filter-ciseipdb.gemspec
|
82
|
+
- spec/filters/ciseipdb_spec.rb
|
83
|
+
homepage: https://github.com/sohonetlabs/logstash-filter-ciseipdb
|
84
|
+
licenses:
|
85
|
+
- Apache-2.0
|
86
|
+
metadata:
|
87
|
+
logstash_plugin: 'true'
|
88
|
+
logstash_group: filter
|
89
|
+
post_install_message:
|
90
|
+
rdoc_options: []
|
91
|
+
require_paths:
|
92
|
+
- lib
|
93
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - ">="
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
requirements: []
|
104
|
+
rubyforge_project:
|
105
|
+
rubygems_version: 2.4.8
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Lookup and inject IP database information into events
|
109
|
+
test_files:
|
110
|
+
- spec/filters/ciseipdb_spec.rb
|