logstash-filter-ip2location 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb5215e6f1580d59313b0525104e612219c879fc
-  data.tar.gz: f5a116257db7d3d6e48c7daeb6f445276ee665f4
+  metadata.gz: 182689a84724ac812b556c4f0072b4673b7cb82b
+  data.tar.gz: 4ac5a8d3b65c417c2284fa1c72e27dd0b9404d3b
 SHA512:
-  metadata.gz: a1ec379054e4e326e81a2054f52c7b49d7af830f328641d1a2579605caa10059ae5e6ee3d5eebb2a62b1cc620cb64d37c5aae42462c4ed738ea83e5ed456f7f6
-  data.tar.gz: 306966e50bbc68397d5ebfae0b8c422694dad97a3e5fa821717422e2fbd44d4b1fcbcedfb3ec6b41dd44ee7123ef9137292ebd1caaec119a04fa517b1236438e
+  metadata.gz: 4c19320e9fdb7447211a3030b622f14ca370d67daf9b41f0d6491b076163f8b12bdbd16c7186fab9068e2fc49abbc678403869f99566ff216223a8aadf09bf05
+  data.tar.gz: 03156e6e53e4e5db86ba9d5d838cd37552229264817d71b32df349ff04b707a5c49a0f5c22aa300b57a774f0cece1107fe0838cf0c9aae3540aaae7aa4b08fbd

data/Gemfile

@@ -1,11 +1,11 @@
-source 'https://rubygems.org'
-
-gemspec
-
-logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
-use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
-
-if Dir.exist?(logstash_path) && use_logstash_source
-  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
-  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
-end
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -1,13 +1,13 @@
-Copyright (c) 2017 IP2Location.com
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
+Copyright (c) 2018 IP2Location.com
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
 limitations under the License.

data/README.md

@@ -1,90 +1,89 @@
-# IP2Location Filter Plugin
-This is IP2Location filter plugin for Logstash that enables Logstash's users to add geolocation information such as country, region, city, latitude, longitude, ZIP code, time zone, Internet Service Provider (ISP), domain name, connection speed, IDD code, area code, weather station code, weather station name, mobile country code (MCC), mobile network code (MNC), mobile brand, elevation, and usage type by IP address. The library reads the geo location information from **IP2Location BIN data** file.
-
-Supported IPv4 and IPv6 address.
-
-For the methods to use IP2Location filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://www.ip2location.com/tutorials/how-to-use-ip2location-filter-plugin-with-elastic-stack).
-
-
-## Dependencies (IP2LOCATION BIN DATA FILE)
-This plugin requires IP2Location BIN data file to function. You may download the BIN data file at
-* IP2Location LITE BIN Data (Free): https://lite.ip2location.com
-* IP2Location Commercial BIN Data (Commercial): https://www.ip2location.com
-
-
-## Installation
-Install this plugin by the following code:
-```
-bin/logstash-plugin install logstash-filter-ip2location
-```
-
-
-## Config File Example
-```
-input {
-  beats {
-    port => "5043"
-  }
-}
-
-filter {
-  grok {
-    match => { "message" => "%{COMBINEDAPACHELOG}"}
-  }
-  ip2location {
-    source => "clientip"
-  }
-}
-
-
-output {
-  elasticsearch {
-    hosts => [ "localhost:9200" ]
-  }
-}
-```
-
-
-## IP2Location Filter Configuration
-|Setting|Input type|Required|
-|---|---|---|
-|source|string|Yes|
-|database|a valid filesystem path|No|
-
-* **source** field is a required setting that containing the IP address or hostname to get the ip information.
-* **database** field is an optional setting that containing the path to the IP2Location BIN database file.
-
-
-## Sample Output
-|Field|Description|
-|---|---|
-|ip2location.area_code|the varying length number assigned to geographic areas for call between cities|
-|ip2location.city|the city name|
-|ip2location.country_long|the country name based on ISO 3166|
-|ip2location.country_short|the two-character country code based on ISO 3166|
-|ip2location.domain|the Internet domain name associated to IP address range|
-|ip2location.elevation|the elevation|
-|ip2location.idd_code|the IDD prefix to call the city from another country|
-|ip2location.ip_address|the IP address|
-|ip2location.isp|the Internet Service Provider (ISP) name|
-|ip2location.latitude|the city latitude|
-|ip2location.longitude|the city longitude|
-|ip2location.location|the location of city|
-|ip2location.mcc|the mobile country code|
-|ip2location.mnc|mobile network code|
-|ip2location.mobile_brand|the mobile brand|
-|ip2location.net_speed|the Internet Connection Speed (DIAL) DIAL-UP,(DSL) DSL/CABLE or(COMP) COMPANY|
-|ip2location.region|the region or state name|
-|ip2location.time_zone|the Time zone in UTC (Coordinated Universal Time)|
-|ip2location.usage_type|the usage type|
-|ip2location.weather_station_code|the special code to identify the nearest weather observation station|
-|ip2location.weather_station_name|the name of the nearest weather observation station|
-|ip2location.zip_code|the ZIP code|
-
-![Example of data](https://www.ip2location.com/images/tutorial/logstash-filter-ip2location-screenshot2.png?)
-
-
-## Support
-Email: support@ip2location.com
-
-URL: [https://www.ip2location.com](https://www.ip2location.com)
+# IP2Location Filter Plugin
+This is IP2Location filter plugin for Logstash that enables Logstash's users to add geolocation information such as country, region, city, latitude, longitude, ZIP code, time zone, Internet Service Provider (ISP), domain name, connection speed, IDD code, area code, weather station code, weather station name, mobile country code (MCC), mobile network code (MNC), mobile brand, elevation, and usage type by IP address. The library reads the geo location information from **IP2Location BIN data** file.
+
+Supported IPv4 and IPv6 address.
+
+For the methods to use IP2Location filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://www.ip2location.com/tutorials/how-to-use-ip2location-filter-plugin-with-elastic-stack).
+
+
+## Dependencies (IP2LOCATION BIN DATA FILE)
+This plugin requires IP2Location BIN data file to function. You may download the BIN data file at
+* IP2Location LITE BIN Data (Free): https://lite.ip2location.com
+* IP2Location Commercial BIN Data (Commercial): https://www.ip2location.com
+
+
+## Installation
+Install this plugin by the following code:
+```
+bin/logstash-plugin install logstash-filter-ip2location
+```
+
+
+## Config File Example
+```
+input {
+  beats {
+    port => "5043"
+  }
+}
+
+filter {
+  grok {
+    match => { "message" => "%{COMBINEDAPACHELOG}"}
+  }
+  ip2location {
+    source => "clientip"
+  }
+}
+
+
+output {
+  elasticsearch {
+    hosts => [ "localhost:9200" ]
+  }
+}
+```
+
+
+## IP2Location Filter Configuration
+|Setting|Input type|Required|
+|---|---|---|
+|source|string|Yes|
+|database|a valid filesystem path|No|
+
+* **source** field is a required setting that containing the IP address or hostname to get the ip information.
+* **database** field is an optional setting that containing the path to the IP2Location BIN database file.
+
+
+## Sample Output
+|Field|Description|
+|---|---|
+|ip2location.area_code|the varying length number assigned to geographic areas for call between cities|
+|ip2location.city|the city name|
+|ip2location.country_long|the country name based on ISO 3166|
+|ip2location.country_short|the two-character country code based on ISO 3166|
+|ip2location.domain|the Internet domain name associated to IP address range|
+|ip2location.elevation|the elevation|
+|ip2location.idd_code|the IDD prefix to call the city from another country|
+|ip2location.ip_address|the IP address|
+|ip2location.isp|the Internet Service Provider (ISP) name|
+|ip2location.latitude|the city latitude|
+|ip2location.longitude|the city longitude|
+|ip2location.mcc|the mobile country code|
+|ip2location.mnc|mobile network code|
+|ip2location.mobile_brand|the mobile brand|
+|ip2location.net_speed|the Internet Connection Speed (DIAL) DIAL-UP,(DSL) DSL/CABLE or(COMP) COMPANY|
+|ip2location.region|the region or state name|
+|ip2location.time_zone|the Time zone in UTC (Coordinated Universal Time)|
+|ip2location.usage_type|the usage type|
+|ip2location.weather_station_code|the special code to identify the nearest weather observation station|
+|ip2location.weather_station_name|the name of the nearest weather observation station|
+|ip2location.zip_code|the ZIP code|
+
+![Example of data](https://www.ip2location.com/images/tutorial/logstash-filter-ip2location-screenshot2.png?)
+
+
+## Support
+Email: support@ip2location.com
+
+URL: [https://www.ip2location.com](https://www.ip2location.com)

data/lib/logstash-filter-ip2location_jars.rb

@@ -1,3 +1,3 @@
-require 'jar_dependencies'
-require_jar('com.ip2location.ip2location', 'ip2location', '8.0.3')
-require_jar('org.logstash.filters', 'logstash-filter-ip2location', '1.0.3')
+require 'jar_dependencies'
+require_jar('com.ip2location.ip2location', 'ip2location', '8.0.3')
+require_jar('org.logstash.filters', 'logstash-filter-ip2location', '1.1.0')

data/lib/logstash/filters/ip2location.rb

@@ -1,50 +1,160 @@
-# encoding: utf-8
-require "logstash/filters/base"
-require "logstash/namespace"
-
-require "logstash-filter-ip2location_jars"
-
-class LogStash::Filters::IP2Location < LogStash::Filters::Base
-  config_name "ip2location"
-
-  # The path to the IP2Location.BIN database file which Logstash should use.
-  # If not specified, this will default to the IP2LOCATION-LITE-DB3.IPV6.BIN database that embedded in the plugin.
-  config :database, :validate => :path
-
-  # The field containing the IP address.
-  # If this field is an array, only the first value will be used.
-  config :source, :validate => :string, :required => true
-
-  # The field used to define iplocation as target.
-  config :target, :validate => :string, :default => 'ip2location'
-
-  public
-  def register
-    if @database.nil?
-      @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2LOCATION-LITE-DB3.IPV6.BIN")).first
-
-      if @database.nil? || !File.exists?(@database)
-        raise "You must specify 'database => ...' in your ip2location filter (I looked for '#{@database}')"
-      end
-    end
-
-    @logger.info("Using ip2location database", :path => @database)
-    
-    @ip2locationfilter = org.logstash.filters.IP2LocationFilter.new(@source, @target, @database)
-  end
-
-  public
-  def filter(event)
-    return unless filter?(event)
-    if @ip2locationfilter.handleEvent(event)
-      filter_matched(event)
-    else
-      tag_iplookup_unsuccessful(event)
-    end
-  end
-
-  def tag_iplookup_unsuccessful(event)
-    @logger.debug? && @logger.debug("IP #{event.get(@source)} was not found in the database", :event => event)
-  end
-
-end # class LogStash::Filters::IP2Location
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+
+require "logstash-filter-ip2location_jars"
+require "json"
+
+require 'thread'
+
+
+class LogStash::Filters::IP2Location < LogStash::Filters::Base
+  config_name "ip2location"
+
+  # The path to the IP2Location.BIN database file which Logstash should use.
+  # If not specified, this will default to the IP2LOCATION-LITE-DB3.IPV6.BIN database that embedded in the plugin.
+  config :database, :validate => :path
+
+  # The field containing the IP address.
+  # If this field is an array, only the first value will be used.
+  config :source, :validate => :string, :required => true
+
+  # The field used to define iplocation as target.
+  config :target, :validate => :string, :default => 'ip2location'
+
+  # The field used to define the size of the cache. It is not required and the default value is 10 000 
+  config :cache_size, :validate => :number, :required => false, :default => 10_000
+
+  public
+  def register
+    if @database.nil?
+      @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2LOCATION-LITE-DB3.IPV6.BIN")).first
+
+      if @database.nil? || !File.exists?(@database)
+        raise "You must specify 'database => ...' in your ip2location filter (I looked for '#{@database}')"
+      end
+    end
+
+    @logger.info("Using ip2location database", :path => @database)
+
+    @ip2locationfilter = org.logstash.filters.IP2LocationFilter.new(@source, @target, @database)
+  end
+
+  public
+  def filter(event)
+    json = JSON.parse(event.to_json)
+    ip = json["clientip"]
+
+    return unless filter?(event)
+    if value = Cache.find(event, ip, @ip2locationfilter, @cache_size).get('ip2location')
+      event.set('ip2location', value)
+      filter_matched(event)
+    else
+      tag_iplookup_unsuccessful(event)
+    end
+  end
+
+  def tag_iplookup_unsuccessful(event)
+    @logger.debug? && @logger.debug("IP #{event.get(@source)} was not found in the database", :event => event)
+  end
+
+end # class LogStash::Filters::IP2Location
+
+class OrderedHash
+  ONE = 1
+
+  attr_reader :times_queried # ip -> times queried
+  attr_reader :hash
+
+  def initialize
+    @times_queried = Hash.new(0) # ip -> times queried
+    @hash = {} # number of hits -> array of ips
+  end
+
+  def add(key)
+    hash[ONE] ||= []
+    hash[ONE] << key
+    times_queried[key] = ONE
+  end
+
+  def reorder(key)
+    number_of_queries = times_queried[key]
+
+    hash[number_of_queries].delete(key)
+    hash.delete(number_of_queries) if hash[number_of_queries].empty?
+
+    hash[number_of_queries + 1] ||= []
+    hash[number_of_queries + 1] << key
+  end
+
+  def increment(key)
+    add(key) unless times_queried.has_key?(key)
+    reorder(key)
+    times_queried[key] += 1
+  end
+
+  def delete_least_used
+    first_pile_with_someting.shift.tap { |key| times_queried.delete(key) }
+  end
+
+  def first_pile_with_someting
+    hash[hash.keys.min]
+  end
+end
+
+class Cache
+  ONE_DAY_IN_SECONDS = 86_400
+
+  @cache         = {}            # ip -> event
+  @timestamps    = {}            # ip -> time of caching
+  @times_queried = OrderedHash.new # ip -> times queried
+  @mutex         = Mutex.new
+
+  class << self
+    attr_reader :cache
+    attr_reader :timestamps
+    attr_reader :times_queried
+
+
+    def find(event, ip, filter, cache_size)
+      synchronize do
+        if cache.has_key?(ip)
+          refresh_event(ip) if too_old?(ip)
+        else
+          if cache_full?(cache_size)
+            make_room 
+          end
+          cache_event(event, ip, filter)
+        end
+        times_queried.increment(ip)
+        cache[ip]
+      end
+    end
+
+    def too_old?(ip)
+      timestamps[ip] < Time.now - ONE_DAY_IN_SECONDS
+    end
+
+    def make_room
+      key = times_queried.delete_least_used
+      cache.delete(key)
+      timestamps.delete(key)
+    end
+
+    def cache_full?(cache_size)
+      cache.size >= cache_size
+    end
+
+    def cache_event(event, ip, filter)
+      filter.handleEvent(event)
+      cache[ip] = event
+      timestamps[ip] = Time.now
+    end
+
+    def synchronize(&block)
+      @mutex.synchronize(&block)
+    end
+
+    alias_method :refresh_event, :cache_event
+  end
+end

data/logstash-filter-ip2location.gemspec

@@ -1,25 +1,25 @@
-Gem::Specification.new do |s|
-
-  s.name            = 'logstash-filter-ip2location'
-  s.version         = '1.0.3'
-  s.licenses        = ['Apache License (2.0)']
-  s.summary         = "Logstash filter IP2Location"
-  s.description     = "IP2Location filter plugin for Logstash enables Logstash's users to add geolocation information such as country, state, city, latitude, longitude, ZIP code, time zone, ISP, domain name, connection speed, IDD code, area code, weather station code, weather station name, MNC, MCC, mobile brand, elevation and usage type by IP address."
-  s.authors         = ["IP2Location"]
-  s.email           = 'support@ip2location.com'
-  s.homepage        = "https://www.ip2location.com"
-  s.require_paths   = ["lib", "vendor/jar-dependencies"]
-
-  # Files
-  s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
-
-  # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
-
-  # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_development_dependency "logstash-devutils"
-end
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-ip2location'
+  s.version         = '1.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Logstash filter IP2Location"
+  s.description     = "IP2Location filter plugin for Logstash enables Logstash's users to add geolocation information such as country, state, city, latitude, longitude, ZIP code, time zone, ISP, domain name, connection speed, IDD code, area code, weather station code, weather station name, MNC, MCC, mobile brand, elevation and usage type by IP address."
+  s.authors         = ["IP2Location"]
+  s.email           = 'support@ip2location.com'
+  s.homepage        = "https://www.ip2location.com"
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/filters/ip2location_spec.rb

@@ -1,32 +1,32 @@
-# encoding: utf-8
-require_relative '../spec_helper'
-require "logstash/filters/ip2location"
-
-IP2LOCATIONDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2LOCATION-LITE-DB3.IPV6.BIN").first
-
-describe LogStash::Filters::IP2Location do
-
-  describe "normal test" do
-    config <<-CONFIG
-      filter {
-        ip2location {
-          source => "ip"
-          #database => "#{IP2LOCATIONDB}"
-        }
-      }
-    CONFIG
-
-    sample("ip" => "8.8.8.8") do
-      expect(subject.get("ip2location")).not_to be_empty
-      expect(subject.get("ip2location")["country_short"]).to eq("US")
-      end
-    end
-
-    sample("ip" => "2a01:04f8:0d16:26c2::") do
-      expect(subject.get("ip2location")).not_to be_empty
-      expect(subject.get("ip2location")["country_short"]).to eq("DE")
-      end
-    end
-  end
-
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/ip2location"
+
+IP2LOCATIONDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2LOCATION-LITE-DB3.IPV6.BIN").first
+
+describe LogStash::Filters::IP2Location do
+
+  describe "normal test" do
+    config <<-CONFIG
+      filter {
+        ip2location {
+          source => "ip"
+          #database => "#{IP2LOCATIONDB}"
+        }
+      }
+    CONFIG
+
+    sample("ip" => "8.8.8.8") do
+      expect(subject.get("ip2location")).not_to be_empty
+      expect(subject.get("ip2location")["country_short"]).to eq("US")
+      end
+    end
+
+    sample("ip" => "2a01:04f8:0d16:26c2::") do
+      expect(subject.get("ip2location")).not_to be_empty
+      expect(subject.get("ip2location")["country_short"]).to eq("DE")
+      end
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,2 +1,2 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-ip2location
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - IP2Location
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -58,7 +58,7 @@ files:
 - spec/spec_helper.rb
 - vendor/IP2LOCATION-LITE-DB3.IPV6.BIN
 - vendor/jar-dependencies/com/ip2location/ip2location/ip2location/8.0.3/ip2location-8.0.3.jar
-- vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2location/1.0.3/logstash-filter-ip2location-1.0.3.jar
+- vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2location/1.1.0/logstash-filter-ip2location-1.1.0.jar
 homepage: https://www.ip2location.com
 licenses:
 - Apache License (2.0)