logstash-filter-ip2proxy 2.3.2 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f13634facd68b8cccdbd6cc6117ae1ca19203010c7a26209c43d6c78ef2427b
-  data.tar.gz: f02c8d0f4626fc8cd549393ddfc7e416f60dd144a5bad19a0b2bc8507604dbab
+  metadata.gz: fa9eb567d56fcb6e98d32a54a8145b0888f6e2b50d1755aa91304a1edee714ab
+  data.tar.gz: d336a63fa77ee0b5dbb5b05d708a97b9ab666e8765fc68f8ebcf94be63a85d36
 SHA512:
-  metadata.gz: 8781be17fa99fdcf69042727ab4d518eac2e193e47bad8e5de72de41247959fa73e25f48ab22cff8cea43e27f3aec260b8aaf929e115c720a39fb00a940f4d0f
-  data.tar.gz: 15b88d65bc35baedbe899129d0247896a7244aac49fc5e3229b0cc713296c8f2ff2e6fcdd138bba2fc407e652896b54e096b2c34a9d8b7a5805f693dd3c36066
+  metadata.gz: d92c46b2b301680c47b9b04083c23068064273a23ef67821e3c856bec1e03705c6df79f030f54031d18b55b4a5397121c7cd765dff06d0907d6b26daf5ab13ac
+  data.tar.gz: 1fc1c255a8dcb9abe9afcec1f8cde4f845441673da5abc32e84eb32ab86607d3c1252d371a7faaf1e5ab564660de19aac2ddbe4a605ee1e11787aa3191521e43

data/Gemfile

@@ -1,11 +1,11 @@
-source 'https://rubygems.org'
-
-gemspec
-
-logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
-use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
-
-if Dir.exist?(logstash_path) && use_logstash_source
-  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
-  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
-end
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -1,13 +1,13 @@
-Copyright (c) 2022 IP2Location.com
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
+Copyright (c) 2024 IP2Location.com
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
 limitations under the License.

data/README.md

@@ -1,85 +1,118 @@
-# IP2Proxy Filter Plugin
-This is IP2Proxy filter plugin for Logstash that enables Logstash's users to reverse search of IP address to detect VPN servers, open proxies, web proxies, Tor exit nodes, search engine robots, data center ranges and residential proxies using IP2Proxy BIN database. Other information available includes proxy type, country, state, city, ISP, domain name, usage type, AS number, AS name, threats, last seen date and provider names. The library took the proxy IP address from **IP2Proxy BIN Data** file.
-
-For the methods to use IP2Proxy filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://blog.ip2location.com/knowledge-base/how-to-use-ip2proxy-filter-plugin-with-elastic-stack).
-
-*Note: This plugin works in Logstash 7 and Logstash 8.*
-
-
-## Dependencies (IP2PROXY BIN DATA FILE)
-This plugin requires IP2Proxy BIN data file to function. You may download the BIN data file at
-* IP2Proxy LITE BIN Data (Free): https://lite.ip2location.com
-* IP2Proxy Commercial BIN Data (Commercial): https://www.ip2location.com
-
-
-## Installation
-Install this plugin by the following code:
-```
-bin/logstash-plugin install logstash-filter-ip2proxy
-```
-
-
-## Config File Example
-```
-input {
-  beats {
-    port => "5043"
-  }
-}
-
-filter {
-  grok {
-    match => { "message" => "%{COMBINEDAPACHELOG}"}
-  }
-  ip2proxy {
-    source => "clientip"
-  }
-}
-
-output {
-  elasticsearch {
-    hosts => [ "localhost:9200" ]
-  }
-}
-```
-
-
-## IP2Proxy Filter Configuration
-|Setting|Input type|Required|
-|---|---|---|
-|source|string|Yes|
-|database|a valid filesystem path|No|
-|use_memory_mapped|boolean|No|
-|use_cache|boolean|No|
-|hide_unsupported_fields|boolean|No|
-
-* **source** field is a required setting that containing the IP address or hostname to get the ip information.
-* **database** field is an optional setting that containing the path to the IP2Proxy BIN database file.
-* **use_memory_mapped** field is an optional setting that used to allow user to enable the use of memory mapped file. Default value is false.
-* **use_cache** field is an optional setting that used to allow user to enable the use of cache. Default value is true.
-* **hide_unsupported_fields** field is an optional setting that used to allow user to hide unsupported fields. Default value is false.
-
-
-## Sample Output
-|Field|Description|
-|---|---|
-|ip2proxy.as|the autonomous system (AS) name of proxy's IP address or domain name|
-|ip2proxy.asn|the autonomous system number (ASN) of proxy's IP address or domain name|
-|ip2proxy.city|the city name of the proxy|
-|ip2proxy.country_long|the ISO3166-1 country name of the proxy|
-|ip2proxy.country_short|the ISO3166-1 country code (two-characters) of the proxy|
-|ip2proxy.domain|the domain name of proxy's IP address or domain name|
-|ip2proxy.is_proxy|Check whether if an IP address was a proxy. Returned value:<ul><li>-1 : errors</li><li>0 : not a proxy</li><li>1 : a proxy</li><li>2 : a data center IP address</li></ul>|
-|ip2proxy.isp|the ISP name of the proxy|
-|ip2proxy.last_seen|the last seen days ago value of proxy's IP address or domain name|
-|ip2proxy.provider|the VPN service provider name if available|
-|ip2proxy.proxy_type|the proxy type. Please visit  <a href="https://www.ip2location.com/database/px11-ip-proxytype-country-region-city-isp-domain-usagetype-asn-lastseen-threat-residential-provider" target="_blank">IP2Location</a> for the list of proxy types supported|
-|ip2proxy.region|the ISO3166-2 region name of the proxy. Please visit <a href="https://www.ip2location.com/free/iso3166-2" target="_blank">ISO3166-2 Subdivision Code</a> for the information of ISO3166-2 supported|
-|ip2proxy.thread|the threat type of the proxy|
-|ip2proxy.usage_type|the usage type classification of the proxy. Please visit <a href="https://www.ip2location.com/database/px11-ip-proxytype-country-region-city-isp-domain-usagetype-asn-lastseen-threat-residential-provider" target="_blank">IP2Location</a> for the list of usage types supported|
-
-
-## Support
-Email: support@ip2location.com
-
-URL: [https://www.ip2location.com](https://www.ip2location.com)
+# IP2Proxy Filter Plugin
+This is IP2Proxy filter plugin for Logstash that enables Logstash's users to reverse search of IP address to detect VPN servers, open proxies, web proxies, Tor exit nodes, search engine robots, data center ranges, residential proxies, consumer privacy networks, and enterprise private networks using IP2Proxy BIN database. Other information available includes proxy type, country, state, city, ISP, domain name, usage type, AS number, AS name, threats, last seen date and provider names. The library took the proxy IP address from **IP2Proxy BIN Data** file and **IP2Location.io** data.
+
+For the methods to use IP2Proxy filter plugin with Elastic Stack (Elasticsearch, Filebeat, Logstash, and Kibana), please take a look on this [tutorial](https://blog.ip2location.com/knowledge-base/how-to-use-ip2proxy-filter-plugin-with-elastic-stack).
+
+*Note: This plugin works in Logstash 7 and Logstash 8.*
+
+
+## Dependencies (IP2PROXY BIN DATA FILE)
+This plugin requires IP2Proxy BIN data file to function. You may download the BIN data file at
+* IP2Proxy LITE BIN Data (Free): https://lite.ip2location.com
+* IP2Proxy Commercial BIN Data (Commercial): https://www.ip2location.com
+
+## Dependencies (IP2LOCATION.IO DATA)
+This plugin requires API key to function. You may sign up for a free API key at https://www.ip2location.io/pricing.
+
+
+## Installation
+Install this plugin by the following code:
+```
+bin/logstash-plugin install logstash-filter-ip2proxy
+```
+
+
+## Config File Example
+```
+input {
+  beats {
+    port => "5043"
+  }
+}
+
+filter {
+  grok {
+    match => { "message" => "%{COMBINEDAPACHELOG}"}
+  }
+  ip2proxy {
+    source => "[source][address]"
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => [ "localhost:9200" ]
+  }
+}
+```
+
+## Config File Example using IP2Location.io
+```
+input {
+  beats {
+    port => "5043"
+  }
+}
+
+filter {
+  grok {
+    match => { "message" => "%{COMBINEDAPACHELOG}"}
+  }
+  ip2proxy {
+    source => "[source][address]"
+    lookup_type => "ws"
+    api_key => "YOUR_API_KEY"
+  }
+}
+
+output {
+  elasticsearch {
+    hosts => [ "localhost:9200" ]
+  }
+}
+```
+
+
+## IP2Proxy Filter Configuration
+|Setting|Input type|Required|
+|---|---|---|
+|source|string|Yes|
+|database|a valid filesystem path|No|
+|use_memory_mapped|boolean|No|
+|use_cache|boolean|No|
+|lookup_type|string|No|
+|api_key|string|No|
+|hide_unsupported_fields|boolean|No|
+
+* **source** field is a required setting that containing the IP address or hostname to get the ip information.
+* **database** field is an optional setting that containing the path to the IP2Proxy BIN database file.
+* **use_memory_mapped** field is an optional setting that used to allow user to enable the use of memory mapped file. Default value is false.
+* **use_cache** field is an optional setting that used to allow user to enable the use of cache. Default value is true.
+* **lookup_type** field is an optional setting that used to allow user to decide the lookup method either using IP2Proxy BIN database file(db) or IP2Location.io data(ws). Default value is db.
+* **api_key** field is an optional setting that used to allow user to set the API Key of the IP2Location.io lookup.
+* **hide_unsupported_fields** field is an optional setting that used to allow user to hide unsupported fields. Default value is false.
+
+
+## Sample Output
+|Field|Description|
+|---|---|
+|ip2proxy.as|the autonomous system (AS) name of proxy's IP address or domain name|
+|ip2proxy.asn|the autonomous system number (ASN) of proxy's IP address or domain name|
+|ip2proxy.city|the city name of the proxy|
+|ip2proxy.country_long|the ISO3166-1 country name of the proxy|
+|ip2proxy.country_short|the ISO3166-1 country code (two-characters) of the proxy|
+|ip2proxy.domain|the domain name of proxy's IP address or domain name|
+|ip2proxy.is_proxy|Check whether if an IP address was a proxy. Returned value:<ul><li>-1 : errors</li><li>0 : not a proxy</li><li>1 : a proxy</li><li>2 : a data center IP address</li></ul>|
+|ip2proxy.isp|the ISP name of the proxy|
+|ip2proxy.last_seen|the last seen days ago value of proxy's IP address or domain name|
+|ip2proxy.provider|the VPN service provider name if available|
+|ip2proxy.proxy_type|the proxy type. Please visit  <a href="https://www.ip2location.com/database/px11-ip-proxytype-country-region-city-isp-domain-usagetype-asn-lastseen-threat-residential-provider" target="_blank">IP2Location</a> for the list of proxy types supported|
+|ip2proxy.region|the ISO3166-2 region name of the proxy. Please visit <a href="https://www.ip2location.com/free/iso3166-2" target="_blank">ISO3166-2 Subdivision Code</a> for the information of ISO3166-2 supported|
+|ip2proxy.thread|the threat type of the proxy|
+|ip2proxy.usage_type|the usage type classification of the proxy. Please visit <a href="https://www.ip2location.com/database/px11-ip-proxytype-country-region-city-isp-domain-usagetype-asn-lastseen-threat-residential-provider" target="_blank">IP2Location</a> for the list of usage types supported|
+
+
+## Support
+Email: support@ip2location.com
+
+URL: [https://www.ip2location.com](https://www.ip2location.com)

data/lib/logstash/filters/ip2proxy.rb

@@ -27,22 +27,34 @@ class LogStash::Filters::IP2Proxy < LogStash::Filters::Base
   # The field used to allow user to hide unsupported fields.
   config :hide_unsupported_fields, :validate => :boolean, :default => false
 
+  # The field used to define lookup type.
+  config :lookup_type, :validate => :string, :default => 'db'
+
+  # The field used to define the apikey of IP2location.io.
+  config :api_key, :validate => :string, :default => ''
+
   # The field used to define the size of the cache. It is not required and the default value is 10 000 
   config :cache_size, :validate => :number, :required => false, :default => 10_000
 
   public
   def register
-    if @database.nil?
-      @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2PROXY-LITE-PX1.BIN")).first
-
-      if @database.nil? || !File.exists?(@database)
-        raise "You must specify 'database => ...' in your ip2proxy filter (I looked for '#{@database}')"
+    if @lookup_type == "ws"
+      @logger.info("Using IP2Location.io API")
+      if @api_key == ""
+        raise "An IP2Location.io API key is required. You may sign up for a free API key at https://www.ip2location.io/pricing."
+      end
+    else
+      if @database.nil?
+        @database = ::Dir.glob(::File.join(::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)),"IP2PROXY-LITE-PX1.BIN")).first
+  
+        if @database.nil? || !File.exists?(@database)
+          raise "You must specify 'database => ...' in your ip2proxy filter (I looked for '#{@database}')"
+        end
       end
+      @logger.info("Using ip2proxy database", :path => @database)
     end
 
-    @logger.info("Using ip2proxy database", :path => @database)
-
-    @ip2proxyfilter = org.logstash.filters.IP2ProxyFilter.new(@source, @target, @database, @use_memory_mapped, @hide_unsupported_fields)
+    @ip2proxyfilter = org.logstash.filters.IP2ProxyFilter.new(@source, @target, @database, @use_memory_mapped, @hide_unsupported_fields, @lookup_type, @api_key)
   end
 
   public
@@ -50,18 +62,26 @@ class LogStash::Filters::IP2Proxy < LogStash::Filters::Base
     ip = event.get(@source)
 
     return unless filter?(event)
-    if @use_cache
-      if value = IP2ProxyCache.find(event, ip, @ip2proxyfilter, @cache_size).get('ip2proxy')
-        event.set('ip2proxy', value)
+    if @lookup_type == "ws"
+      if @ip2proxyfilter.handleEvent(event)
         filter_matched(event)
       else
         tag_iplookup_unsuccessful(event)
       end
     else
-      if @ip2proxyfilter.handleEvent(event)
-        filter_matched(event)
+      if @use_cache
+        if value = IP2ProxyCache.find(event, ip, @ip2proxyfilter, @cache_size).get('ip2proxy')
+          event.set('ip2proxy', value)
+          filter_matched(event)
+        else
+          tag_iplookup_unsuccessful(event)
+        end
       else
-        tag_iplookup_unsuccessful(event)
+        if @ip2proxyfilter.handleEvent(event)
+          filter_matched(event)
+        else
+          tag_iplookup_unsuccessful(event)
+        end
       end
     end
   end

data/lib/logstash-filter-ip2proxy_jars.rb

@@ -1,3 +1,4 @@
 require 'jar_dependencies'
 require_jar('com.ip2proxy.ip2proxy', 'ip2proxy', '3.4.0')
-require_jar('org.logstash.filters', 'logstash-filter-ip2proxy', '2.3.2')
+require_jar('com.google.gson', 'gson', '2.11.0')
+require_jar('org.logstash.filters', 'logstash-filter-ip2proxy', '2.4.0')

data/logstash-filter-ip2proxy.gemspec

@@ -1,25 +1,25 @@
-Gem::Specification.new do |s|
-
-  s.name            = 'logstash-filter-ip2proxy'
-  s.version         = '2.3.2'
-  s.licenses        = ['Apache-2.0']
-  s.summary         = "Logstash filter IP2Proxy"
-  s.description     = "IP2Proxy filter plugin for Logstash enables Logstash's users to reverse search of IP address to detect VPN servers, open proxies, web proxies, Tor exit nodes, search engine robots, data center ranges and residential proxies using IP2Proxy BIN database."
-  s.authors         = ["IP2Location"]
-  s.email           = 'support@ip2location.com'
-  s.homepage        = "https://www.ip2location.com"
-  s.require_paths   = ["lib", "vendor/jar-dependencies"]
-
-  # Files
-  s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
-
-  # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
-
-  # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_development_dependency "logstash-devutils"
-end
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-filter-ip2proxy'
+  s.version         = '2.4.0'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "Logstash filter IP2Proxy"
+  s.description     = "IP2Proxy filter plugin for Logstash enables Logstash's users to reverse search of IP address to detect VPN servers, open proxies, web proxies, Tor exit nodes, search engine robots, data center ranges and residential proxies using IP2Proxy BIN database."
+  s.authors         = ["IP2Location"]
+  s.email           = 'support@ip2location.com'
+  s.homepage        = "https://www.ip2location.com"
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
+
+  # Files
+  s.files = Dir["lib/**/*",'spec/**/*',"vendor/**/*","vendor/jar-dependencies/**/*.jar","*.gemspec","*.md","Gemfile","LICENSE"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/filters/ip2proxy_spec.rb

@@ -1,27 +1,27 @@
-# encoding: utf-8
-require_relative '../spec_helper'
-require "logstash/filters/ip2proxy"
-
-IP2PROXYDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2PROXY-LITE-PX1.BIN").first
-
-describe LogStash::Filters::IP2Proxy do
-
-  describe "normal test" do
-    config <<-CONFIG
-      filter {
-        ip2proxy {
-          source => "ip"
-          #database => "#{IP2PROXYDB}"
-        }
-      }
-    CONFIG
-
-    sample("ip" => "8.8.8.8") do
-      expect(subject.get("ip2proxy")).not_to be_empty
-      expect(subject.get("ip2proxy")["country_short"]).to eq("US")
-      end
-    end
-
-  end
-
+# encoding: utf-8
+require_relative '../spec_helper'
+require "logstash/filters/ip2proxy"
+
+IP2PROXYDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/IP2PROXY-LITE-PX1.BIN").first
+
+describe LogStash::Filters::IP2Proxy do
+
+  describe "normal test" do
+    config <<-CONFIG
+      filter {
+        ip2proxy {
+          source => "ip"
+          #database => "#{IP2PROXYDB}"
+        }
+      }
+    CONFIG
+
+    sample("ip" => "8.8.8.8") do
+      expect(subject.get("ip2proxy")).not_to be_empty
+      expect(subject.get("ip2proxy")["country_short"]).to eq("US")
+      end
+    end
+
+  end
+
 end

data/spec/spec_helper.rb

@@ -1,2 +1,2 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-ip2proxy
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.4.0
 platform: ruby
 authors:
 - IP2Location
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-14 00:00:00.000000000 Z
+date: 2024-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -55,9 +55,9 @@ files:
 - logstash-filter-ip2proxy.gemspec
 - spec/filters/ip2proxy_spec.rb
 - spec/spec_helper.rb
-- vendor/IP2PROXY-LITE-PX1.BIN
+- vendor/jar-dependencies/com/google/gson/gson/2.11.0/gson-2.11.0.jar
 - vendor/jar-dependencies/com/ip2proxy/ip2proxy/ip2proxy/3.4.0/ip2proxy-3.4.0.jar
-- vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2proxy/2.3.2/logstash-filter-ip2proxy-2.3.2.jar
+- vendor/jar-dependencies/org/logstash/filters/logstash-filter-ip2proxy/2.4.0/logstash-filter-ip2proxy-2.4.0.jar
 homepage: https://www.ip2location.com
 licenses:
 - Apache-2.0
@@ -80,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.18
 signing_key:
 specification_version: 4
 summary: Logstash filter IP2Proxy

data/vendor/IP2PROXY-LITE-PX1.BIN

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:852f55814fb14e8de5b47d416b95f1ce58757256484fda7d2d4462900db4229a
-size 229021771