logstash-filter-ciseipdb 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 6d629463df4731b72259a3fd44fbfbdfbc3abad8
4
+ data.tar.gz: be938079e6e753bdcd0d15511dd7841a41b8bae4
5
+ SHA512:
6
+ metadata.gz: 0edb982adc2ab31aceb74f2a1e2b394dcc0546b3e23314238031027b5786fa1c31907a591365c7ee7cb8ea39936c00d1b739c80839b1684e37d8ec4294ab2b5f
7
+ data.tar.gz: 9487277cde7ac75c49c769842695f0924f7b168b3ad5d6f8e4e0b5d3cc8382b909c82ba73fee6e1f91c6839b046b5ffc136e17688b5b2de09e5cfd83bf158270
data/Gemfile ADDED
@@ -0,0 +1,2 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,14 @@
1
+ Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
2
+ Copyright (c) 2016 Sohonet <http://www.sohonet.com>
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
@@ -0,0 +1,12 @@
1
+ logstash-filter-ciseipdb logstash plugin
2
+
3
+ Copyright, 2016 Sohonet
4
+
5
+ This software contains code derived from logstash-filter-elasticsearch
6
+ Copyright 2012-2015 Elasticsearch
7
+
8
+ Elasticsearch
9
+ Copyright 2012-2015 Elasticsearch
10
+
11
+ This product includes software developed by The Apache Software
12
+ Foundation (http://www.apache.org/).
@@ -0,0 +1,25 @@
1
+ # Ciseipdb Logstash Plugin
2
+
3
+ [![Build Status](https://travis-ci.org/sohonetlabs/logstash-filter-ciseipdb.svg?branch=master)](https://travis-ci.org/sohonetlabs/logstash-filter-ciseipdb)
4
+
5
+ This is a plugin for [Logstash](https://github.com/elastic/logstash).
6
+
7
+ It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
8
+
9
+ ## Documentation
10
+
11
+ This plugin allows you to search for matching IPs in Elasticsearch IP database indexes and add that information into events.
12
+
13
+ This is intended to work with [generate-ipdatabase](https://github.com/sohonetlabs/generate-ipdatabase) which will create the Elasticsearch IP database entries.
14
+
15
+ Matching IPs are cached in redis.
16
+
17
+ Example:
18
+
19
+ ciseipdb {
20
+ hosts => [ "elasticsearch" ]
21
+ indexes => [ "ipdatabase" ]
22
+ ipaddress => "%{ip_dst}"
23
+ target => "dst_info"
24
+ redis_host => "localhost'
25
+ }
@@ -0,0 +1,167 @@
1
+ require "logstash/filters/base"
2
+ require "logstash/namespace"
3
+ require "base64"
4
+
5
+ # Search elasticsearch for matching IPs in Elasticsearch IP database indexes i
6
+ # and add that information into events.
7
+ #
8
+ # Caches matching IPs in redis.
9
+ #
10
+ # Example:
11
+ #
12
+ # ciseipdb {
13
+ # hosts => [ "elasticsearch" ]
14
+ # indexes => [ "ipdatabase" ]
15
+ # ipaddress => "%{ip_dst}"
16
+ # target => "dst_info"
17
+ # }
18
+
19
+ class LogStash::Filters::Ciseipdb < LogStash::Filters::Base
20
+ config_name "ciseipdb"
21
+
22
+ # List of elasticsearch hosts to use for querying.
23
+ config :hosts, :validate => :array, :required => true
24
+
25
+ # List of indexes to perform the search query against.
26
+ config :indexes, :validate => :array, :default => [""]
27
+
28
+ # IP Addresses
29
+ config :ipaddress, :validate => :string, :required => true
30
+
31
+ # Target field for added fields
32
+ config :target, :validate => :string, :required => true
33
+
34
+ # Basic Auth - username
35
+ config :user, :validate => :string
36
+
37
+ # Basic Auth - password
38
+ config :password, :validate => :password
39
+
40
+ # SSL
41
+ config :ssl, :validate => :boolean, :default => false
42
+
43
+ # SSL Certificate Authority file
44
+ config :ca_file, :validate => :path
45
+
46
+ # Redis host
47
+ config :redis_host, :validate => :string, :default => "localhost"
48
+
49
+ # Redis key TTL
50
+ config :redis_ttl, :validate => :number, :default => 3600
51
+
52
+
53
+ public
54
+ def register
55
+ require "elasticsearch"
56
+ require "redis"
57
+
58
+ transport_options = {}
59
+
60
+ if @user && @password
61
+ token = Base64.strict_encode64("#{@user}:#{@password.value}")
62
+ transport_options[:headers] = { Authorization: "Basic #{token}" }
63
+ end
64
+
65
+ hosts = if @ssl then
66
+ @hosts.map {|h| { host: h, scheme: 'https' } }
67
+ else
68
+ @hosts
69
+ end
70
+
71
+ if @ssl && @ca_file
72
+ transport_options[:ssl] = { ca_file: @ca_file }
73
+ end
74
+
75
+ @logger.info("New CISE IPDB filter", :hosts => hosts)
76
+ @client = Elasticsearch::Client.new hosts: hosts, transport_options: transport_options
77
+
78
+ @redis = Redis.new(:host => redis_host)
79
+ end # def register
80
+
81
+ public
82
+ def filter(event)
83
+
84
+ ipaddress = event.sprintf(@ipaddress)
85
+
86
+ # Check ip address in redis
87
+ data = check_redis(ipaddress)
88
+
89
+ # IP not in redis, lookup elasticsearch, add to redis
90
+ if data.nil?
91
+ data = search(ipaddress)
92
+ update_redis(ipaddress, data)
93
+ end
94
+
95
+ # Update event
96
+ data.each_pair do |k,v|
97
+ targetname = "#{@target}_#{k}"
98
+ event[targetname] = v
99
+ end
100
+ filter_matched(event)
101
+
102
+ end # def filter
103
+
104
+ def search(ip)
105
+ output = Hash.new
106
+
107
+ begin
108
+ query = {
109
+ query: {
110
+ filtered: {
111
+ filter: {
112
+ and: [
113
+ { term: { IPADDRESS: ip } },
114
+ { range: { "@timestamp" => { gte: "now-1d/d", lt: "now" } } }
115
+ ]
116
+ }
117
+ }
118
+ }
119
+ }
120
+ results = @client.search index: @indexes, body: query
121
+
122
+ if results['hits']['total'] >= 1
123
+ output['databases'] = Array.new
124
+ output['reputation_score'] = 0
125
+ results['hits']['hits'].each do |hit|
126
+ output['databases'] << hit['_source']['database']['shortname']
127
+ output['reputation_score'] += hit['_source']['database']['reputation_score'].to_i
128
+
129
+ # Extra data from nipap
130
+ if hit['_source']['database']['shortname'] == 'nipap'
131
+ output['service_slug'] = hit['_source']['service_slug']
132
+ output['description'] = hit['_source']['description']
133
+ output['router'] = hit['_source']['router']
134
+ end
135
+ end
136
+ end
137
+
138
+ rescue => e
139
+ @logger.debug("No hits for ipaddresses", :query => query, :error => e)
140
+ end #begin..rescue
141
+
142
+ output
143
+ end # def search
144
+
145
+ def check_redis(ip)
146
+ begin
147
+ output = @redis.get(ip)
148
+ if output.nil?
149
+ output
150
+ else
151
+ eval(output)
152
+ end
153
+ rescue => e
154
+ @logger.warn("Problem getting key from redis", :ip => ip, :error => e)
155
+ end
156
+ end # def check_redis
157
+
158
+ def update_redis(ip, data)
159
+ begin
160
+ @redis.set(ip, data)
161
+ @redis.expire(ip, @redis_ttl)
162
+ rescue => e
163
+ @logger.warn("Problem updating redis", :ip => ip, :data => data , :error => e)
164
+ end
165
+ end # def update_redis
166
+
167
+ end # class LogStash::Filters::Elasticsearch
@@ -0,0 +1,29 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-filter-ciseipdb'
4
+ s.version = '0.10.0'
5
+ s.licenses = ['Apache-2.0']
6
+ s.summary = "Lookup and inject IP database information into events"
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
+ s.authors = ["Elastic", "Sohonet"]
9
+ s.email = 'support@sohonet.com'
10
+ s.homepage = "https://github.com/sohonetlabs/logstash-filter-ciseipdb"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
24
+ s.add_runtime_dependency "elasticsearch", "~> 1.0"
25
+ s.add_runtime_dependency "redis", ">= 0"
26
+
27
+ s.add_development_dependency 'logstash-devutils'
28
+ end
29
+
@@ -0,0 +1,24 @@
1
+ # encoding: utf-8
2
+
3
+ require "logstash/devutils/rspec/spec_helper"
4
+ require "logstash/plugin"
5
+ require "logstash/filters/ciseipdb"
6
+
7
+ describe LogStash::Filters::Ciseipdb do
8
+
9
+ let (:cise_config) {{
10
+ 'hosts' => [ 'elasticsearch' ],
11
+ 'ipaddress' => '127.0.0.1',
12
+ 'target' => 'destination',
13
+ }}
14
+
15
+ context "registration" do
16
+
17
+ let(:plugin) { LogStash::Plugin.lookup("filter", "ciseipdb").new(cise_config) }
18
+
19
+ it "should not raise an exception" do
20
+ expect {plugin.register}.to_not raise_error
21
+ end
22
+ end
23
+
24
+ end
metadata ADDED
@@ -0,0 +1,110 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-filter-ciseipdb
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.10.0
5
+ platform: ruby
6
+ authors:
7
+ - Elastic
8
+ - Sohonet
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2016-08-10 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ name: logstash-core-plugin-api
21
+ prerelease: false
22
+ type: :runtime
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - "~>"
26
+ - !ruby/object:Gem::Version
27
+ version: '1.0'
28
+ - !ruby/object:Gem::Dependency
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.0'
34
+ name: elasticsearch
35
+ prerelease: false
36
+ type: :runtime
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - "~>"
40
+ - !ruby/object:Gem::Version
41
+ version: '1.0'
42
+ - !ruby/object:Gem::Dependency
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ name: redis
49
+ prerelease: false
50
+ type: :runtime
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ name: logstash-devutils
63
+ prerelease: false
64
+ type: :development
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
71
+ email: support@sohonet.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - Gemfile
77
+ - LICENSE
78
+ - NOTICE.TXT
79
+ - README.md
80
+ - lib/logstash/filters/ciseipdb.rb
81
+ - logstash-filter-ciseipdb.gemspec
82
+ - spec/filters/ciseipdb_spec.rb
83
+ homepage: https://github.com/sohonetlabs/logstash-filter-ciseipdb
84
+ licenses:
85
+ - Apache-2.0
86
+ metadata:
87
+ logstash_plugin: 'true'
88
+ logstash_group: filter
89
+ post_install_message:
90
+ rdoc_options: []
91
+ require_paths:
92
+ - lib
93
+ required_ruby_version: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ">="
96
+ - !ruby/object:Gem::Version
97
+ version: '0'
98
+ required_rubygems_version: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ requirements: []
104
+ rubyforge_project:
105
+ rubygems_version: 2.4.8
106
+ signing_key:
107
+ specification_version: 4
108
+ summary: Lookup and inject IP database information into events
109
+ test_files:
110
+ - spec/filters/ciseipdb_spec.rb