logstash-filter-geoip 7.2.12-java → 7.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8016c2fb0b715fc6452e41ab3d36598c5fab654317362b52dc7974aba5d1dbe0
-  data.tar.gz: 9e97baedf9827069590d33a18999118401f42565e90304b127e4ab0eba111740
+  metadata.gz: f7db08a266d05be61267f7921ede9b9e9c7177804574c96fcc94a25615b6449d
+  data.tar.gz: bdd2133e4acbea8e12dfd6dc57ce45a9b5e628cf7672a0c299768041855ec001
 SHA512:
-  metadata.gz: 8985a3708143ff541037379491ec98c592766a0fd6d06dd43247cbea8b2675c754475fd37300eebd247c2cd2562174acdb3f3fd19057a716550969988b560b9b
-  data.tar.gz: 5dd029823bab8ff4c816b379e9f1fe258c62a81058fda64e18f0a93d12e475750b5b62ea6bc1b5edfbe385eb93b9cfcd135389f5aa3f73518203362cdf6178ba
+  metadata.gz: d659cea20b030c2dfbfe30d7b1ce283b308250a710e90287595c836a0cb6ef678b25d473cca6da7b7d82d87e31dc5557ad37802e85e06c437a4f5b104d5c5b48
+  data.tar.gz: 5157efccb1f54d898d53ad6db5ae9ba1659473858de168d353789ca6f1fd7b78ffe8777e76130e9adda0d33cd23ebb2215c5d4812ec127b254ec925dfc6937ba

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 7.3.0
+  - Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases ([#223](https://github.com/logstash-plugins/logstash-filter-geoip/pull/223))
+  - Updated MaxMind dependencies.
+  - Added tests for the Java classes.
+
+## 7.2.13
+  - [DOC] Add documentation for database auto-update configuration [#210](https://github.com/logstash-plugins/logstash-filter-geoip/pull/210)
+
 ## 7.2.12
   - [DOC] Add `http_proxy` environment variable for GeoIP service endpoint. The feature is included in 8.1.0, and was back-ported to 7.17.2 [#207](https://github.com/logstash-plugins/logstash-filter-geoip/pull/207) 
 

data/docs/index.asciidoc

@@ -47,8 +47,8 @@ within 30 days of a database update.
 
 The GeoIP filter plugin can manage the database for users running the Logstash default
 distribution, or you can manage
-database updates on your own. The behavior is controlled by the `database` setting.
-When you use the default `database` setting, the auto-update feature ensures that the plugin is
+database updates on your own. The behavior is controlled by the `database` setting and by the auto-update feature.
+When you use the default `database` setting and the auto-update feature is enabled, Logstash ensures that the plugin is
 using the latest version of the database.
 Otherwise, you are responsible for maintaining compliance.
 
@@ -59,19 +59,26 @@ database by default.
 ==== Database Auto-update
 
 This plugin bundles Creative Commons (CC) license databases.
-Logstash checks for database updates every day. It downloads the latest and can replace the old database
+If the auto-update feature is enabled in `logstash.yml`(as it is by default), Logstash checks for database updates every day. It downloads the latest and can replace the old database
 while the plugin is running.
-After Logstash downloads EULA license databases, it will not fallback to CC license databases.
 
-NOTE: If the database has never been updated successfully, as in air-gapped environments, Logstash can use CC license databases indefinitely.
+NOTE: If the auto-update feature is disabled or the database has never been updated successfully, as in air-gapped environments, Logstash can use CC license databases indefinitely.
 
 After Logstash has switched to a EULA licensed database, the geoip filter will
 stop enriching events in order to maintain compliance if Logstash fails to
 check for database updates for 30 days.
 Events will be tagged with `_geoip_expired_database` tag to facilitate the handling of this situation.
 
+NOTE: If the auto-update feature is enabled, Logstash upgrades from the CC database license to the EULA version on the first download.
+
 TIP: When possible, allow Logstash to access the internet to download databases so that they are always up-to-date.
 
+**Disable the auto-update feature**
+
+If you work in air-gapped environment and want to disable the database auto-update feature, set the `xpack.geoip.downloader.enabled` value to `false` in `logstash.yml`.
+
+When the auto-update feature is disabled, Logstash uses the Creative Commons (CC) license databases indefinitely, and any previously downloaded version of the EULA databases will be deleted.
+
 [id="plugins-{type}s-{plugin}-manage_update"]
 ==== Manage your own database updates
 
@@ -183,37 +190,44 @@ When ECS compatibility is enabled, the fields are structured to fit into an ECS
 |===========================
 | Database Field Name | ECS Field | Example
 
-| `ip`               | `[ip]`                     | `12.34.56.78`
-
-| `city_name`        | `[geo][city_name]`         | `Seattle`
-| `country_name`     | `[geo][country_name]`      | `United States`
-| `continent_code`   | `[geo][continent_code]`    | `NA`
-| `continent_name`   | `[geo][continent_name]`    | `North America`
-| `country_code2`    | `[geo][country_iso_code]`  | `US`
-| `country_code3`    | _N/A_                      | `US`
-
-                                                    _maintained for legacy
-                                                    support, but populated
-                                                    with 2-character country
-                                                    code_
-
-| `postal_code`      | `[geo][postal_code]`       | `98106`
-| `region_name`      | `[geo][region_name]`       | `Washington`
-| `region_code`      | `[geo][region_code]`       | `WA`
-| `region_iso_code`* | `[geo][region_iso_code]`   | `US-WA`
-| `timezone`         | `[geo][timezone]`          | `America/Los_Angeles`
-| `location`*        | `[geo][location]`          | `{"lat": 47.6062, "lon": -122.3321}"`
-| `latitude`         | `[geo][location][lat]`     | `47.6062`
-| `longitude`        | `[geo][location][lon]`     | `-122.3321`
-
-| `domain`           | `[domain]`                 | `example.com`
-
-| `asn`              | `[as][number]`             | `98765`
-| `as_org`           | `[as][organization][name]` | `Elastic, NV`
-
-| `isp`              | `[mmdb][isp]`              | `InterLink Supra LLC`
-| `dma_code`         | `[mmdb][dma_code]`         | `819`
-| `organization`     | `[mmdb][organization]`     | `Elastic, NV`
+| `ip`                | `[ip]`                           | `12.34.56.78`
+| `anonymous`         | `[ip_traits][anonymous]`         | `false`
+| `anonymous_vpn`     | `[ip_traits][anonymous_vpn]`     | `false`
+| `hosting_provider`  | `[ip_traits][hosting_provider]`  | `true`
+| `network`           | `[ip_traits][network]`           | `12.34.56.78/20`
+| `public_proxy`      | `[ip_traits][public_proxy]`      | `true`
+| `residential_proxy` | `[ip_traits][residential_proxy]` | `false`
+| `tor_exit_node`     | `[ip_traits][tor_exit_node]`     | `true`
+
+| `city_name`         | `[geo][city_name]`               | `Seattle`
+| `country_name`      | `[geo][country_name]`            | `United States`
+| `continent_code`    | `[geo][continent_code]`          | `NA`
+| `continent_name`    | `[geo][continent_name]`          | `North America`
+| `country_code2`     | `[geo][country_iso_code]`        | `US`
+| `country_code3`     | _N/A_                            | `US`
+
+                                                           _maintained for legacy
+                                                           support, but populated
+                                                           with 2-character country
+                                                           code_
+
+| `postal_code`       | `[geo][postal_code]`             | `98106`
+| `region_name`       | `[geo][region_name]`             | `Washington`
+| `region_code`       | `[geo][region_code]`             | `WA`
+| `region_iso_code`*  | `[geo][region_iso_code]`         | `US-WA`
+| `timezone`          | `[geo][timezone]`                | `America/Los_Angeles`
+| `location`*         | `[geo][location]`                | `{"lat": 47.6062, "lon": -122.3321}"`
+| `latitude`          | `[geo][location][lat]`           | `47.6062`
+| `longitude`         | `[geo][location][lon]`           | `-122.3321`
+
+| `domain`            | `[domain]`                       | `example.com`
+
+| `asn`               | `[as][number]`                   | `98765`
+| `as_org`            | `[as][organization][name]`       | `Elastic, NV`
+
+| `isp`               | `[mmdb][isp]`                    | `InterLink Supra LLC`
+| `dma_code`          | `[mmdb][dma_code]`               | `819`
+| `organization`      | `[mmdb][organization]`           | `Elastic, NV`
 |===========================
 
 NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components.
@@ -294,7 +308,7 @@ number of cache misses and waste memory.
 The path to MaxMind's database file that Logstash should use.
 The default database is `GeoLite2-City`.
 This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`)
-and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`).
+and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`).
 
 Database auto-update applies to the default distribution.
 When `database` points to user's database path, auto-update is disabled.

data/lib/logstash/filters/geoip.rb

@@ -171,7 +171,17 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   end
 
   def close
-    @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    begin
+      @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    rescue => e
+      @logger.error("Error unsubscribing geoip database path", :path => @database, :exception => e)
+    end
+
+    begin
+      @geoipfilter.close if @geoipfilter
+    rescue => e
+      @logger.error("Error closing GeoIPFilter",  :exception => e)
+    end
   end
 
   def select_database_path

data/lib/logstash-filter-geoip_jars.rb

@@ -1,6 +1,6 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.maxmind.geoip2', 'geoip2', '2.9.0')
-require_jar('com.maxmind.db', 'maxmind-db', '1.2.2')
-require_jar('org.logstash.filters', 'logstash-filter-geoip', '6.0.0')
+require_jar('com.maxmind.geoip2', 'geoip2', '2.17.0')
+require_jar('com.maxmind.db', 'maxmind-db', '2.1.0')
+require_jar('org.logstash.filters', 'logstash-filter-geoip', '7.3.0')

data/logstash-filter-geoip.gemspec

@@ -1,7 +1,9 @@
+VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(VERSION)
+
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.2.12'
+  s.version         = VERSION
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/geoip_offline_spec.rb

@@ -112,7 +112,7 @@ describe LogStash::Filters::GeoIP do
         CONFIG
 
     context "should return the correct sourcefield in the logging message" do
-      sample("ip" => "8.8.8.8") do
+      sample({"ip" => "8.8.8.8"}) do
         expect { subject }.to raise_error(java.lang.IllegalArgumentException, "The database provided is invalid or corrupted.")
       end
     end

data/spec/filters/geoip_online_spec.rb

@@ -1,65 +1,97 @@
 # encoding: utf-8
+require 'pathname'
 require "logstash/devutils/rspec/spec_helper"
 require "insist"
 require "logstash/filters/geoip"
 require_relative 'test_helper'
 
 describe LogStash::Filters::GeoIP do
+  context "when no database_path is given" do
 
-  before(:each) do
-    ::File.delete(METADATA_PATH) if ::File.exist?(METADATA_PATH)
-  end
+    let(:last_db_path_recorder) do
+      Module.new do
+        attr_reader :last_db_path
+        def setup_filter(db_path)
+          @last_db_path = db_path
+          super
+        end
+      end
+    end
 
-  describe "config without database path in LS >= 7.14", :aggregate_failures do
-    before(:each) do
-      dir_path = Stud::Temporary.directory
-      File.open(dir_path + '/uuid', 'w') { |f| f.write(SecureRandom.uuid) }
-      allow(LogStash::SETTINGS).to receive(:get).and_call_original
-      allow(LogStash::SETTINGS).to receive(:get).with("path.data").and_return(dir_path)
+    let(:plugin_config) { Hash["source" => "[source][ip]", "target" => "[target]"] }
+    let(:plugin) { described_class.new(plugin_config).extend(last_db_path_recorder) }
+    let(:event) { LogStash::Event.new("source" => { "ip" => "173.9.34.107" }) }
+
+    shared_examples "event enrichment" do
+      it 'enriches events' do
+        plugin.register
+        plugin.filter(event)
+
+        expect(event.get("target")).to include('ip')
+      end
     end
 
-    let(:plugin) { LogStash::Filters::GeoIP.new("source" => "[target][ip]") }
+    database_management_available = (MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)) && !LogStash::OSS
+    if database_management_available
+      context "when geoip database management is available" do
+
+        let(:mock_manager) do
+          double('LogStash::Filters::Geoip::DatabaseManager').tap do |m|
+            allow(m).to receive(:subscribe_database_path) do |db_type, explicit_path, plugin_instance|
+              explicit_path || mock_managed[db_type]
+            end
+            allow(m).to receive(:unsubscribe_database_path).with(any_args)
+          end
+        end
+
+        # The extension to this plugin that lives in Logstash core will _always_ provide a valid
+        # database path, and how it does so is not the concern of this plugin. We emulate this
+        # behaviour here by copying the vendored CC-licensed db's into a temporary path
+        let(:mock_managed) do
+          managed_path = Pathname.new(temp_data_path).join("managed", Time.now.to_i.to_s).tap(&:mkpath)
+
+          managed_city_db_path = Pathname.new(DEFAULT_CITY_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_CITY_DB_PATH, managed_city_db_path)
 
-    context "restart the plugin" do
-      let(:event) { LogStash::Event.new("target" => { "ip" => "173.9.34.107" }) }
-      let(:event2) { LogStash::Event.new("target" => { "ip" => "55.159.212.43" }) }
+          managed_asn_db_path = Pathname.new(DEFAULT_ASN_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_ASN_DB_PATH, managed_asn_db_path)
 
-      it "should use the same database" do
-        unless plugin.load_database_manager?
-          logstash_path = ENV['LOGSTASH_PATH'] || '/usr/share/logstash' # docker logstash home
-          stub_const('LogStash::Environment::LOGSTASH_HOME', logstash_path)
+          {
+            'City' => managed_city_db_path,
+            'ASN' => managed_asn_db_path,
+          }
         end
 
-        plugin.register
-        plugin.filter(event)
-        plugin.close
-        first_dirname = get_metadata_city_database_name
-        plugin.register
-        plugin.filter(event2)
-        plugin.close
-        second_dirname = get_metadata_city_database_name
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:load_database_manager?).and_return(true)
+          stub_const("LogStash::Filters::Geoip::DatabaseManager", double("DatabaseManager.Class", :instance => mock_manager))
+        end
 
-        expect(first_dirname).not_to be_nil
-        expect(first_dirname).to eq(second_dirname)
-        expect(File).to exist(get_file_path(first_dirname))
+        let(:temp_data_path) { Stud::Temporary.directory }
+        after(:each) do
+          FileUtils.rm_rf(temp_data_path) if File.exist?(temp_data_path)
+        end
+
+        it "uses a managed database" do
+          plugin.register
+          plugin.filter(event)
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to start_with(temp_data_path)
+        end
+
+        include_examples "event enrichment"
       end
-    end
-  end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
-
-  describe "config without database path in LS < 7.14" do
-    context "should run in offline mode" do
-      config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-        }
-      }
-      CONFIG
-
-      sample("ip" => "173.9.34.107") do
-        insist { subject.get("geoip") }.include?("ip")
-        expect(::File.exist?(METADATA_PATH)).to be_falsey
+    else
+      context "when geoip database management is not available" do
+
+        include_examples "event enrichment"
+
+        it "uses a plugin-vendored database" do
+          plugin.register
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to include("/vendor/")
+        end
       end
     end
-  end if MAJOR < 7 || (MAJOR == 7 && MINOR < 14)
+  end
 end

data/spec/filters/geoip_spec.rb

@@ -19,13 +19,13 @@ describe LogStash::Filters::GeoIP do
       end
     end
 
-    describe ">= 7.14" do
+    shared_examples "with database manager" do
       it "load_database_manager? should be true" do
         expect(plugin.load_database_manager?).to be_truthy
       end
-    end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+    end
 
-    describe "<= 7.13" do
+    shared_examples "without database manager" do
       it "load_database_manager? should be false" do
         expect(plugin.load_database_manager?).to be_falsey
       end
@@ -37,6 +37,24 @@ describe LogStash::Filters::GeoIP do
           expect(plugin.select_database_path).to eql(DEFAULT_CITY_DB_PATH)
         end
       end
-    end if MAJOR < 7 || (MAJOR == 7 && MINOR <= 13)
+    end
+
+    if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+      context "Logstash >= 7.14" do
+        if LogStash::OSS
+          context "OSS-only" do
+            include_examples "without database manager"
+          end
+        else
+          context "default distro" do
+            include_examples "with database manager"
+          end
+        end
+      end
+    else
+      describe "Logstash < 7.14" do
+        include_examples "without database manager"
+      end
+    end
   end
 end

data/spec/filters/test_helper.rb

@@ -2,28 +2,46 @@ require "logstash-core/logstash-core"
 require "digest"
 require "csv"
 
-def get_vendor_path(filename)
-  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
-end
-
-def get_data_dir
-  ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
-end
-
-def get_file_path(filename)
-  ::File.join(get_data_dir, filename)
-end
+# Since we use Logstash's x-pack WITHOUT the LogStash::Runner,
+# we must find it relative to logstash-core and add it to the load path.
+require 'pathname'
+logstash_core_path = Gem.loaded_specs['logstash-core']&.full_gem_path or fail("logstash-core lib not found")
+logstash_xpack_load_path = Pathname.new(logstash_core_path).join("../x-pack/lib").cleanpath.to_s
+if ENV['OSS'] == "true" || !File.exists?(logstash_xpack_load_path)
+  $stderr.puts("X-PACK is not available")
+  LogStash::OSS = true
+else
+  if !$LOAD_PATH.include?(logstash_xpack_load_path)
+    $stderr.puts("ADDING LOGSTASH X-PACK to load path: #{logstash_xpack_load_path}")
+    $LOAD_PATH.unshift(logstash_xpack_load_path)
+  end
+  LogStash::OSS = false
 
-def get_metadata_city_database_name
-  if ::File.exist?(METADATA_PATH)
-    city = ::CSV.read(METADATA_PATH, headers: false).select { |row| row[0].eql?("City") }.last
-    city[3]
+  # when running in a Logstash process that has a geoip extension available, it will
+  # be loaded before this plugin is instantiated. In tests, we need to find and load the
+  # appropriate extension ourselves.
+  extension = nil
+  extension ||= begin; require 'geoip_database_management/extension'; LogStash::const_get("GeoipDatabaseManagement::Extension"); rescue Exception; nil; end
+  extension ||= begin; require 'filters/geoip/extension'; LogStash::const_get("Filters::Geoip::Extension"); rescue Exception; nil; end
+  if extension
+    $stderr.puts("loading logstash extension for geoip: #{extension}")
+    extension.new.tap do |instance|
+      # the extensions require logstash/runner even though they don't need to,
+      # resulting in _all_ extensions being loaded into the registry, including
+      # those whose dependencies are not met by this plugin's dependency graph.
+      def instance.require(path)
+        super unless path == "logstash/runner"
+      end
+    end.additionals_settings(LogStash::SETTINGS)
   else
-    nil
+    $stderr.puts("no logstash extension for geoip is available")
   end
 end
 
-METADATA_PATH = get_file_path("metadata.csv")
+def get_vendor_path(filename)
+  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
+end
+
 DEFAULT_CITY_DB_PATH = get_vendor_path("GeoLite2-City.mmdb")
 DEFAULT_ASN_DB_PATH = get_vendor_path("GeoLite2-ASN.mmdb")
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 7.2.12
+  version: 7.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-18 00:00:00.000000000 Z
+date: 2024-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -127,9 +127,9 @@ files:
 - spec/filters/test_helper.rb
 - vendor/GeoLite2-ASN.mmdb
 - vendor/GeoLite2-City.mmdb
-- vendor/jar-dependencies/com/maxmind/db/maxmind-db/1.2.2/maxmind-db-1.2.2.jar
-- vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.9.0/geoip2-2.9.0.jar
-- vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/6.0.0/logstash-filter-geoip-6.0.0.jar
+- vendor/jar-dependencies/com/maxmind/db/maxmind-db/2.1.0/maxmind-db-2.1.0.jar
+- vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.17.0/geoip2-2.17.0.jar
+- vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/7.3.0/logstash-filter-geoip-7.3.0.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Adds geographical information about an IP address