logstash-filter-private_geoip 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bb1d1e7b4b8854ff8ed70dad9b8f2cd995deff51
+  data.tar.gz: 4fd7f2f248d4cd01ef4dd31f8c32c41281121f21
+SHA512:
+  metadata.gz: 1989fd62d6cd32f95890430baced04915fefaf89686a0715b5e71a97ebb40e5530e755623a1bc0fa856e63c508f55c0d0d8380523fe8e5c4860f19b5cec556b5
+  data.tar.gz: 114a016e0540a3f77147054f6ee4a9ee29f04efddea782487942996afb32b1364269c229a286d5070b880d5ad78fade46a9f4f3eccce332e9ae9fd70b94fd5df

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2015 KULeuven/LIBIS <http://www.libis.be>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,49 @@
+# logstash-filter-private_geoip
+
+Extends or overwrites the GeoIP event data. If you have a large network with _mixed_ calls from **public** and **private** 
+IP address the **private** IP addresses are NOT resolved by GeoIP.
+   
+If you supply the different local networks they will get mapped.
+   
+## Filter definition   
+```json
+    filter {
+        private_geoip {
+            source => "client_ip"
+            database => "/path/to/network_file.json"
+            target => 'geoip'
+            merge => false
+        }
+    }
+```  
+
+* source: field that contains the ip address that needs to be mapped. This can be http_clientip, http_x_forwarded_for or ...
+* database: location of the the JSON mapping file.
+* target: where should the mapping result be stored. default = private_geoip but you could change it to geoip
+* merge: should the target be overwritten with the mapping result. default = true
+  
+## Mapping definition
+```JSON
+    [
+        {"cidr":"10.33.104.0/22",
+         "data":{"organization_name":"XYZ Faculty",
+                 "city_name":"Leuven",
+                 "country_name":"Belgium",
+                 ...
+                }
+        },
+        ...
+    ]
+``` 
+ 
+ * cidr: contains the network 
+ * data: can contain any key/value pair. If you want to extend/overwrite GeoIP take those fields ...
+   
+   
+# Install
+   
+```sh
+   bin/plugin install /your/local/plugin/logstash-filter-private_geoip.gem
+```   
+
+Restart LogStash.

data/lib/logstash/filters/private_geoip.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+# The Private GeoIP filter add information about local private IP addresses
+# This is a CIDR filter on steroids
+
+class LogStash::Filters::PrivateGeoIP < LogStash::Filters::Base
+  config_name "private_geoip"
+
+  #The field that that has the IP address to compare
+  config :source, validate: :string, required: true
+
+  #The path to the JSON file. "data" can contain any key/value pair
+  #[
+  # {
+  #   "cidr":"10.32.1.0/24",
+  #   "data": {
+  #     "organization_name": "MyOrganization or Description",
+  #     "city_name": "Leuven",
+  #     "country_name": "Belgium",
+  #     "country_code2": "BE",
+  #     "country_code3": "BEL",
+  #     "continent_code": "Europe/Brussels"
+  #   }
+  # }
+  #]
+  config :database, validate: :string, required: true
+
+  #contains the target object. it is by default private_geoip but you could change it to geoip
+  config :target, validate: :string, default: "private_geoip"
+
+  #if a geoip object exists on the event should it be merged and thus overwritten
+  config :merge, validate: :boolean, default: true
+
+  public
+  def register
+    require 'ip'
+    require 'json'
+
+    @cidrs = {}
+    JSON.parse(File.read(@database)).each do |entry|
+      @cidrs.store(IP::CIDR.new(entry['cidr']), entry['data'])
+    end
+
+  rescue Exception => e
+    raise "Error registering filter: #{e.message}"
+  end
+
+  public
+  def filter(event)
+    ip = IP::Address::Util.string_to_ip(event[@source])
+
+    matched_cidr_data = {}
+
+    @cidrs.each do |cidr, data|
+      if cidr.includes?(ip)
+        matched_cidr_data = data
+        break
+      end
+    end
+
+    return if matched_cidr_data.nil? || matched_cidr_data.empty?
+
+    geo_data = event[@target].nil? ? {} : event[@target]
+
+    if @merge
+      geo_data.merge!(matched_cidr_data)
+      geo_data['ip'] = event[@source]
+    else
+      add_keys = matched_cidr_data.keys - geo_data.keys
+      geo_data.merge!(matched_cidr_data.select{|k,v| add_keys.include?(k)})
+    end
+
+    event[@target] = geo_data
+
+    filter_matched(event)
+  end
+end

data/logstash-filter-private_geoip.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-private_geoip'
+  s.version         = '1.0.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "$summary"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["Mehmet Celik"]
+  s.email           = 'mehmet.celik@kuleuven.libis.be'
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  #s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency 'ip', "0.3.1"
+
+
+  s.add_development_dependency 'logstash-devutils', '~> 0'
+end

data/spec/filters/private_geoip_spec.rb

@@ -0,0 +1,140 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/private_geoip"
+require 'pp'
+
+DATABASE = "spec/resources/cidr.json"
+
+describe LogStash::Filters::PrivateGeoIP do
+  describe "does find match" do
+    config <<-CONFIG
+      filter {
+        private_geoip {
+          source => "ip"
+          database => "#{DATABASE}"
+        }
+      }
+    CONFIG
+
+    sample("ip" => "10.33.108.120") do
+      insist {subject}.include?("private_geoip")
+      insist {subject["private_geoip"]["organization_name"]}.eql?("centrale bibliotheek")
+    end
+  end
+
+  describe "does not find match" do
+    config <<-CONFIG
+      filter {
+        private_geoip {
+          source => "ip"
+          database => "#{DATABASE}"
+        }
+      }
+    CONFIG
+
+    sample("ip" => "10.33.101.120") do
+      reject {subject}.include?("private_geoip")
+
+    end
+  end
+
+  describe "target = geoip instead of private_geoip" do
+    config <<-CONFIG
+      filter {
+        private_geoip {
+          source => "ip"
+          database => "#{DATABASE}"
+          target => "geoip"
+        }
+      }
+    CONFIG
+
+
+    sample("ip" => "10.33.108.120") do
+      insist {subject}.include?("geoip")
+      reject {subject}.include?("private_geoip")
+    end
+  end
+
+  describe "do merge with overwrite" do
+    config <<-CONFIG
+      filter {
+        private_geoip {
+          source => "client_ip"
+          database => "#{DATABASE}"
+          target => "geoip"
+        }
+      }
+    CONFIG
+
+    sample_data = {
+        "client_ip" => "10.33.108.120",
+        "geoip" => {
+          "ip" => "134.58.179.35",
+          "country_code2" => "BE",
+          "country_code3" => "BEL",
+          "country_name" => "Belgium",
+          "continent_code" => "EU",
+          "region_name" => "12",
+          "city_name" => "Leuven",
+          "postal_code" => "3000",
+          "latitude" => 50.88329999999999,
+          "longitude" => 4.699999999999989,
+          "timezone" => "Europe/Brussels",
+          "real_region_name" => "Vlaams-Brabant",
+          "location" => [
+              4.699999999999989,
+              50.88329999999999
+          ]
+        }
+    }
+
+
+    sample(sample_data) do
+      #insist {subject}.include?("geoip")
+      insist {subject["geoip"]["city_name"]} == "KULASSOC"
+    end
+  end
+
+  describe "do merge NO overwrite" do
+    config <<-CONFIG
+      filter {
+        private_geoip {
+          source => "client_ip"
+          database => "#{DATABASE}"
+          target => "geoip"
+          merge => false
+        }
+      }
+    CONFIG
+
+    sample_data = {
+        "client_ip" => "10.33.108.120",
+        "geoip" => {
+            "ip" => "134.58.179.35",
+            "country_code2" => "BE",
+            "country_code3" => "BEL",
+            "country_name" => "Belgium",
+            "continent_code" => "EU",
+            "region_name" => "12",
+            "city_name" => "Leuven",
+            "postal_code" => "3000",
+            "latitude" => 50.88329999999999,
+            "longitude" => 4.699999999999989,
+            "timezone" => "Europe/Brussels",
+            "real_region_name" => "Vlaams-Brabant",
+            "location" => [
+                4.699999999999989,
+                50.88329999999999
+            ]
+        }
+    }
+
+    sample(sample_data) do
+      insist {subject}.include?("geoip")
+      reject {subject}.include?("private_geoip")
+      insist {subject["geoip"]["city_name"]} == "Leuven"
+    end
+  end
+
+
+end

data/spec/resources/cidr.json

@@ -0,0 +1,5 @@
+[
+  {"cidr":"10.33.104.0/22","data":{"organization_name":"Faculteit Landbouw","city_name":"KULASSOC","country_name":"Belgium","country_code2":"BE","country_code3":"BEL","continent_code":"Europe/Brussels"}},
+  {"cidr":"10.33.108.0/24","data":{"organization_name":"Centrale Bibliotheek","city_name":"KULASSOC","country_name":"Belgium","country_code2":"BE","country_code3":"BEL","continent_code":"Europe/Brussels"}},
+  {"cidr":"10.33.109.0/24","data":{"organization_name":"Burgerlijke Bouwkunde","city_name":"KULASSOC","country_name":"Belgium","country_code2":"BE","country_code3":"BEL","continent_code":"Europe/Brussels"}}
+]

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-private_geoip
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Mehmet Celik
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-10-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+  name: ip
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: mehmet.celik@kuleuven.libis.be
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/private_geoip.rb
+- logstash-filter-private_geoip.gemspec
+- spec/filters/private_geoip_spec.rb
+- spec/resources/cidr.json
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: $summary
+test_files:
+- spec/filters/private_geoip_spec.rb
+- spec/resources/cidr.json