logstash-filter-ciseipdb 0.10.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6d629463df4731b72259a3fd44fbfbdfbc3abad8
+  data.tar.gz: be938079e6e753bdcd0d15511dd7841a41b8bae4
+SHA512:
+  metadata.gz: 0edb982adc2ab31aceb74f2a1e2b394dcc0546b3e23314238031027b5786fa1c31907a591365c7ee7cb8ea39936c00d1b739c80839b1684e37d8ec4294ab2b5f
+  data.tar.gz: 9487277cde7ac75c49c769842695f0924f7b168b3ad5d6f8e4e0b5d3cc8382b909c82ba73fee6e1f91c6839b046b5ffc136e17688b5b2de09e5cfd83bf158270

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2016 Sohonet <http://www.sohonet.com>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,12 @@
+logstash-filter-ciseipdb logstash plugin
+
+Copyright, 2016 Sohonet
+
+This software contains code derived from logstash-filter-elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,25 @@
+# Ciseipdb Logstash Plugin
+
+[![Build Status](https://travis-ci.org/sohonetlabs/logstash-filter-ciseipdb.svg?branch=master)](https://travis-ci.org/sohonetlabs/logstash-filter-ciseipdb)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+This plugin allows you to search for matching IPs in Elasticsearch IP database indexes and add that information into events.
+
+This is intended to work with [generate-ipdatabase](https://github.com/sohonetlabs/generate-ipdatabase) which will create the Elasticsearch IP database entries.
+
+Matching IPs are cached in redis.
+
+Example:
+
+    ciseipdb {
+      hosts   => [ "elasticsearch" ]
+      indexes => [ "ipdatabase" ]
+      ipaddress => "%{ip_dst}"
+      target  => "dst_info"
+      redis_host => "localhost'
+    }

data/lib/logstash/filters/ciseipdb.rb

@@ -0,0 +1,167 @@
+require "logstash/filters/base"
+require "logstash/namespace"
+require "base64"
+
+# Search elasticsearch for matching IPs in Elasticsearch IP database indexes i
+# and add that information into events.
+#
+# Caches matching IPs in redis.
+#
+# Example:
+#
+# ciseipdb {
+#    hosts   => [ "elasticsearch" ]
+#    indexes => [ "ipdatabase" ]
+#    ipaddress => "%{ip_dst}"
+#    target  => "dst_info"
+# }
+
+class LogStash::Filters::Ciseipdb < LogStash::Filters::Base
+  config_name "ciseipdb"
+
+  # List of elasticsearch hosts to use for querying.
+  config :hosts, :validate => :array, :required => true
+
+  # List of indexes to perform the search query against.
+  config :indexes, :validate => :array, :default => [""]
+
+  # IP Addresses
+  config :ipaddress, :validate => :string, :required => true
+
+  # Target field for added fields
+  config :target, :validate => :string, :required => true
+
+  # Basic Auth - username
+  config :user, :validate => :string
+
+  # Basic Auth - password
+  config :password, :validate => :password
+
+  # SSL
+  config :ssl, :validate => :boolean, :default => false
+
+  # SSL Certificate Authority file
+  config :ca_file, :validate => :path
+
+  # Redis host
+  config :redis_host, :validate => :string, :default => "localhost"
+
+  # Redis key TTL
+  config :redis_ttl, :validate => :number, :default => 3600
+
+
+  public
+  def register
+    require "elasticsearch"
+    require "redis"
+
+    transport_options = {}
+
+    if @user && @password
+      token = Base64.strict_encode64("#{@user}:#{@password.value}")
+      transport_options[:headers] = { Authorization: "Basic #{token}" }
+    end
+
+    hosts = if @ssl then
+      @hosts.map {|h| { host: h, scheme: 'https' } }
+    else
+      @hosts
+    end
+
+    if @ssl && @ca_file
+      transport_options[:ssl] = { ca_file: @ca_file }
+    end
+
+    @logger.info("New CISE IPDB filter", :hosts => hosts)
+    @client = Elasticsearch::Client.new hosts: hosts, transport_options: transport_options
+
+    @redis = Redis.new(:host => redis_host)
+  end # def register
+
+  public
+  def filter(event)
+
+    ipaddress = event.sprintf(@ipaddress)
+
+    # Check ip address in redis
+    data = check_redis(ipaddress)
+
+    # IP not in redis, lookup elasticsearch, add to redis
+    if data.nil?
+      data = search(ipaddress)
+      update_redis(ipaddress, data)
+    end
+
+    # Update event
+    data.each_pair do |k,v|
+      targetname = "#{@target}_#{k}"
+      event[targetname] = v
+    end
+    filter_matched(event)
+
+  end # def filter
+
+  def search(ip)
+    output = Hash.new
+
+    begin
+      query = {
+        query: {
+          filtered: {
+            filter: {
+              and: [
+                { term: { IPADDRESS: ip } },
+                { range: { "@timestamp" => { gte: "now-1d/d", lt: "now" } } }
+              ]
+            }
+          }
+        }
+      }
+      results = @client.search index: @indexes, body: query
+
+      if results['hits']['total'] >= 1
+        output['databases'] = Array.new
+        output['reputation_score'] = 0
+        results['hits']['hits'].each do |hit|
+          output['databases'] << hit['_source']['database']['shortname']
+          output['reputation_score'] += hit['_source']['database']['reputation_score'].to_i
+
+          # Extra data from nipap
+          if hit['_source']['database']['shortname'] == 'nipap'
+            output['service_slug'] = hit['_source']['service_slug']
+            output['description'] = hit['_source']['description']
+            output['router'] = hit['_source']['router']
+          end
+        end
+      end
+
+    rescue => e
+      @logger.debug("No hits for ipaddresses", :query => query, :error => e)
+    end #begin..rescue
+
+    output
+  end # def search
+
+  def check_redis(ip)
+    begin
+      output = @redis.get(ip)
+      if output.nil?
+        output
+      else
+        eval(output)
+      end
+    rescue => e
+      @logger.warn("Problem getting key from redis", :ip => ip, :error => e)
+    end
+  end # def check_redis
+
+  def update_redis(ip, data)
+    begin
+      @redis.set(ip, data)
+      @redis.expire(ip, @redis_ttl)
+    rescue => e
+      @logger.warn("Problem updating redis", :ip => ip, :data => data , :error => e)
+    end
+  end # def update_redis
+
+end # class LogStash::Filters::Elasticsearch

data/logstash-filter-ciseipdb.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-ciseipdb'
+  s.version         = '0.10.0'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "Lookup and inject IP database information into events"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Elastic", "Sohonet"]
+  s.email           = 'support@sohonet.com'
+  s.homepage        = "https://github.com/sohonetlabs/logstash-filter-ciseipdb"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "elasticsearch", "~> 1.0"
+  s.add_runtime_dependency "redis", ">= 0"
+
+  s.add_development_dependency 'logstash-devutils'
+end
+

data/spec/filters/ciseipdb_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin"
+require "logstash/filters/ciseipdb"
+
+describe LogStash::Filters::Ciseipdb do
+
+  let (:cise_config) {{
+    'hosts'      => [ 'elasticsearch' ],
+    'ipaddress'  => '127.0.0.1',
+    'target'     => 'destination',
+  }}
+
+  context "registration" do
+
+    let(:plugin) { LogStash::Plugin.lookup("filter", "ciseipdb").new(cise_config) }
+
+    it "should not raise an exception" do
+      expect {plugin.register}.to_not raise_error
+    end
+  end
+
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-ciseipdb
+version: !ruby/object:Gem::Version
+  version: 0.10.0
+platform: ruby
+authors:
+- Elastic
+- Sohonet
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-08-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: redis
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: support@sohonet.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/filters/ciseipdb.rb
+- logstash-filter-ciseipdb.gemspec
+- spec/filters/ciseipdb_spec.rb
+homepage: https://github.com/sohonetlabs/logstash-filter-ciseipdb
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Lookup and inject IP database information into events
+test_files:
+- spec/filters/ciseipdb_spec.rb