logstash-filter-geoip 7.2.13-java → 7.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caca9f4a1ba058cc744e60279252429de5e33f041ecac3a2872b1973714961e2
-  data.tar.gz: 248298b87dc000636fe7775910cd1cf28eb18842200426b831c46a8c835d7db2
+  metadata.gz: 381f84a6a7bd07469d152e2c97c1841b3d77c229f23c7f8e9a88976a546a5117
+  data.tar.gz: 42c1922d9d8ee49a7e527bc1159412a531d92742c783d91d9d1662f083664be5
 SHA512:
-  metadata.gz: f2a9c1475b3832ef62a61998427fd0f8cac9aa18c28cba340cb0f9086ca298882dc99e1fcf9c0c6f08dc9dcf5eeb6543f4145d6536c74b85235a46c1bc4a1ff4
-  data.tar.gz: 10612c6801e46e32763c913ac55b9598959341b44f201a89cf1738ec2db84efdc3df3cf8edf1569d46dd5a5e06963023c1dc49f6e6127bb41aa2d25e655e8632
+  metadata.gz: 7ad313fccdc5174dd735f768adb10576c89acadea3c4dfc3e640ac8a4fd5d9ace7fa9d89a34cef91610fa12a7e84ebba03183c5db3613bb4112b8ad1b4735eb8
+  data.tar.gz: 627cfd2e7cabaed7cd3a0fee677285db3c1c93202f88656ae61b51edba01228d8451a0a20aa38d7eaa7963bf9a962f6646e6a72dd406efecde07f71eb351e33e

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 7.3.1
+  - Avoid to crash pipelines when lookup a database with customised fields. [#225](https://github.com/logstash-plugins/logstash-filter-geoip/pull/225)
+
+## 7.3.0
+  - Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases ([#223](https://github.com/logstash-plugins/logstash-filter-geoip/pull/223))
+  - Updated MaxMind dependencies.
+  - Added tests for the Java classes.
+
 ## 7.2.13
   - [DOC] Add documentation for database auto-update configuration [#210](https://github.com/logstash-plugins/logstash-filter-geoip/pull/210)
 

data/docs/index.asciidoc

@@ -190,37 +190,44 @@ When ECS compatibility is enabled, the fields are structured to fit into an ECS
 |===========================
 | Database Field Name | ECS Field | Example
 
-| `ip`               | `[ip]`                     | `12.34.56.78`
-
-| `city_name`        | `[geo][city_name]`         | `Seattle`
-| `country_name`     | `[geo][country_name]`      | `United States`
-| `continent_code`   | `[geo][continent_code]`    | `NA`
-| `continent_name`   | `[geo][continent_name]`    | `North America`
-| `country_code2`    | `[geo][country_iso_code]`  | `US`
-| `country_code3`    | _N/A_                      | `US`
-
-                                                    _maintained for legacy
-                                                    support, but populated
-                                                    with 2-character country
-                                                    code_
-
-| `postal_code`      | `[geo][postal_code]`       | `98106`
-| `region_name`      | `[geo][region_name]`       | `Washington`
-| `region_code`      | `[geo][region_code]`       | `WA`
-| `region_iso_code`* | `[geo][region_iso_code]`   | `US-WA`
-| `timezone`         | `[geo][timezone]`          | `America/Los_Angeles`
-| `location`*        | `[geo][location]`          | `{"lat": 47.6062, "lon": -122.3321}"`
-| `latitude`         | `[geo][location][lat]`     | `47.6062`
-| `longitude`        | `[geo][location][lon]`     | `-122.3321`
-
-| `domain`           | `[domain]`                 | `example.com`
-
-| `asn`              | `[as][number]`             | `98765`
-| `as_org`           | `[as][organization][name]` | `Elastic, NV`
-
-| `isp`              | `[mmdb][isp]`              | `InterLink Supra LLC`
-| `dma_code`         | `[mmdb][dma_code]`         | `819`
-| `organization`     | `[mmdb][organization]`     | `Elastic, NV`
+| `ip`                | `[ip]`                           | `12.34.56.78`
+| `anonymous`         | `[ip_traits][anonymous]`         | `false`
+| `anonymous_vpn`     | `[ip_traits][anonymous_vpn]`     | `false`
+| `hosting_provider`  | `[ip_traits][hosting_provider]`  | `true`
+| `network`           | `[ip_traits][network]`           | `12.34.56.78/20`
+| `public_proxy`      | `[ip_traits][public_proxy]`      | `true`
+| `residential_proxy` | `[ip_traits][residential_proxy]` | `false`
+| `tor_exit_node`     | `[ip_traits][tor_exit_node]`     | `true`
+
+| `city_name`         | `[geo][city_name]`               | `Seattle`
+| `country_name`      | `[geo][country_name]`            | `United States`
+| `continent_code`    | `[geo][continent_code]`          | `NA`
+| `continent_name`    | `[geo][continent_name]`          | `North America`
+| `country_code2`     | `[geo][country_iso_code]`        | `US`
+| `country_code3`     | _N/A_                            | `US`
+
+                                                           _maintained for legacy
+                                                           support, but populated
+                                                           with 2-character country
+                                                           code_
+
+| `postal_code`       | `[geo][postal_code]`             | `98106`
+| `region_name`       | `[geo][region_name]`             | `Washington`
+| `region_code`       | `[geo][region_code]`             | `WA`
+| `region_iso_code`*  | `[geo][region_iso_code]`         | `US-WA`
+| `timezone`          | `[geo][timezone]`                | `America/Los_Angeles`
+| `location`*         | `[geo][location]`                | `{"lat": 47.6062, "lon": -122.3321}"`
+| `latitude`          | `[geo][location][lat]`           | `47.6062`
+| `longitude`         | `[geo][location][lon]`           | `-122.3321`
+
+| `domain`            | `[domain]`                       | `example.com`
+
+| `asn`               | `[as][number]`                   | `98765`
+| `as_org`            | `[as][organization][name]`       | `Elastic, NV`
+
+| `isp`               | `[mmdb][isp]`                    | `InterLink Supra LLC`
+| `dma_code`          | `[mmdb][dma_code]`               | `819`
+| `organization`      | `[mmdb][organization]`           | `Elastic, NV`
 |===========================
 
 NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components.
@@ -301,7 +308,7 @@ number of cache misses and waste memory.
 The path to MaxMind's database file that Logstash should use.
 The default database is `GeoLite2-City`.
 This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`)
-and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`).
+and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`).
 
 Database auto-update applies to the default distribution.
 When `database` points to user's database path, auto-update is disabled.

data/lib/logstash/filters/geoip.rb

@@ -171,7 +171,17 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   end
 
   def close
-    @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    begin
+      @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    rescue => e
+      @logger.error("Error unsubscribing geoip database path", :path => @database, :exception => e)
+    end
+
+    begin
+      @geoipfilter.close if @geoipfilter
+    rescue => e
+      @logger.error("Error closing GeoIPFilter",  :exception => e)
+    end
   end
 
   def select_database_path

data/lib/logstash-filter-geoip_jars.rb

@@ -1,6 +1,6 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.maxmind.geoip2', 'geoip2', '2.9.0')
-require_jar('com.maxmind.db', 'maxmind-db', '1.2.2')
-require_jar('org.logstash.filters', 'logstash-filter-geoip', '6.0.0')
+require_jar('com.maxmind.geoip2', 'geoip2', '2.17.0')
+require_jar('com.maxmind.db', 'maxmind-db', '2.1.0')
+require_jar('org.logstash.filters', 'logstash-filter-geoip', '7.3.1')

data/logstash-filter-geoip.gemspec

@@ -1,7 +1,9 @@
+VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(VERSION)
+
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.2.13'
+  s.version         = VERSION
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/geoip_ecs_spec.rb

@@ -22,7 +22,7 @@ describe LogStash::Filters::GeoIP do
         let(:common_options) { {"source" => "message", "database" => CITYDB, "target" => target} }
 
         before(:each) do
-          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+          allow(plugin).to receive(:ecs_compatibility).and_return(ecs_compatibility)
           plugin.register
         end
 
@@ -170,7 +170,7 @@ describe LogStash::Filters::GeoIP do
 
       context "ECS disabled" do
         before do
-          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(:disabled)
+          allow(plugin).to receive(:ecs_compatibility).and_return(:disabled)
           plugin.register
           plugin.filter(event)
         end
@@ -193,7 +193,7 @@ describe LogStash::Filters::GeoIP do
 
       context "ECS mode" do
         before do
-          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(:v1)
+          allow(plugin).to receive(:ecs_compatibility).and_return(:v1)
         end
 
         context "`target` is unset" do

data/spec/filters/geoip_offline_spec.rb

@@ -112,7 +112,7 @@ describe LogStash::Filters::GeoIP do
         CONFIG
 
     context "should return the correct sourcefield in the logging message" do
-      sample("ip" => "8.8.8.8") do
+      sample({"ip" => "8.8.8.8"}) do
         expect { subject }.to raise_error(java.lang.IllegalArgumentException, "The database provided is invalid or corrupted.")
       end
     end

data/spec/filters/geoip_online_spec.rb

@@ -1,67 +1,97 @@
 # encoding: utf-8
+require 'pathname'
 require "logstash/devutils/rspec/spec_helper"
 require "insist"
 require "logstash/filters/geoip"
 require_relative 'test_helper'
 
 describe LogStash::Filters::GeoIP do
+  context "when no database_path is given" do
 
-  before(:each) do
-    ::File.delete(METADATA_PATH) if ::File.exist?(METADATA_PATH)
-  end
+    let(:last_db_path_recorder) do
+      Module.new do
+        attr_reader :last_db_path
+        def setup_filter(db_path)
+          @last_db_path = db_path
+          super
+        end
+      end
+    end
 
-  describe "config without database path in LS >= 7.14", :aggregate_failures do
-    before(:each) do
-      dir_path = Stud::Temporary.directory
-      File.open(dir_path + '/uuid', 'w') { |f| f.write(SecureRandom.uuid) }
-      allow(LogStash::SETTINGS).to receive(:get).and_call_original
-      allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(true)
-      allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(nil)
-      allow(LogStash::SETTINGS).to receive(:get).with("path.data").and_return(dir_path)
+    let(:plugin_config) { Hash["source" => "[source][ip]", "target" => "[target]"] }
+    let(:plugin) { described_class.new(plugin_config).extend(last_db_path_recorder) }
+    let(:event) { LogStash::Event.new("source" => { "ip" => "173.9.34.107" }) }
+
+    shared_examples "event enrichment" do
+      it 'enriches events' do
+        plugin.register
+        plugin.filter(event)
+
+        expect(event.get("target")).to include('ip')
+      end
     end
 
-    let(:plugin) { LogStash::Filters::GeoIP.new("source" => "[target][ip]") }
+    database_management_available = (MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)) && !LogStash::OSS
+    if database_management_available
+      context "when geoip database management is available" do
+
+        let(:mock_manager) do
+          double('LogStash::Filters::Geoip::DatabaseManager').tap do |m|
+            allow(m).to receive(:subscribe_database_path) do |db_type, explicit_path, plugin_instance|
+              explicit_path || mock_managed[db_type]
+            end
+            allow(m).to receive(:unsubscribe_database_path).with(any_args)
+          end
+        end
+
+        # The extension to this plugin that lives in Logstash core will _always_ provide a valid
+        # database path, and how it does so is not the concern of this plugin. We emulate this
+        # behaviour here by copying the vendored CC-licensed db's into a temporary path
+        let(:mock_managed) do
+          managed_path = Pathname.new(temp_data_path).join("managed", Time.now.to_i.to_s).tap(&:mkpath)
+
+          managed_city_db_path = Pathname.new(DEFAULT_CITY_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_CITY_DB_PATH, managed_city_db_path)
 
-    context "restart the plugin" do
-      let(:event) { LogStash::Event.new("target" => { "ip" => "173.9.34.107" }) }
-      let(:event2) { LogStash::Event.new("target" => { "ip" => "55.159.212.43" }) }
+          managed_asn_db_path = Pathname.new(DEFAULT_ASN_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_ASN_DB_PATH, managed_asn_db_path)
 
-      it "should use the same database" do
-        unless plugin.load_database_manager?
-          logstash_path = ENV['LOGSTASH_PATH'] || '/usr/share/logstash' # docker logstash home
-          stub_const('LogStash::Environment::LOGSTASH_HOME', logstash_path)
+          {
+            'City' => managed_city_db_path,
+            'ASN' => managed_asn_db_path,
+          }
         end
 
-        plugin.register
-        plugin.filter(event)
-        plugin.close
-        first_dirname = get_metadata_city_database_name
-        plugin.register
-        plugin.filter(event2)
-        plugin.close
-        second_dirname = get_metadata_city_database_name
+        before(:each) do
+          allow(plugin).to receive(:load_database_manager?).and_return(true)
+          stub_const("LogStash::Filters::Geoip::DatabaseManager", double("DatabaseManager.Class", :instance => mock_manager))
+        end
 
-        expect(first_dirname).not_to be_nil
-        expect(first_dirname).to eq(second_dirname)
-        expect(File).to exist(get_file_path(first_dirname))
+        let(:temp_data_path) { Stud::Temporary.directory }
+        after(:each) do
+          FileUtils.rm_rf(temp_data_path) if File.exist?(temp_data_path)
+        end
+
+        it "uses a managed database" do
+          plugin.register
+          plugin.filter(event)
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to start_with(temp_data_path)
+        end
+
+        include_examples "event enrichment"
       end
-    end
-  end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
-
-  describe "config without database path in LS < 7.14" do
-    context "should run in offline mode" do
-      config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-        }
-      }
-      CONFIG
-
-      sample("ip" => "173.9.34.107") do
-        insist { subject.get("geoip") }.include?("ip")
-        expect(::File.exist?(METADATA_PATH)).to be_falsey
+    else
+      context "when geoip database management is not available" do
+
+        include_examples "event enrichment"
+
+        it "uses a plugin-vendored database" do
+          plugin.register
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to include("/vendor/")
+        end
       end
     end
-  end if MAJOR < 7 || (MAJOR == 7 && MINOR < 14)
+  end
 end

data/spec/filters/geoip_spec.rb

@@ -19,13 +19,13 @@ describe LogStash::Filters::GeoIP do
       end
     end
 
-    describe ">= 7.14" do
+    shared_examples "with database manager" do
       it "load_database_manager? should be true" do
         expect(plugin.load_database_manager?).to be_truthy
       end
-    end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+    end
 
-    describe "<= 7.13" do
+    shared_examples "without database manager" do
       it "load_database_manager? should be false" do
         expect(plugin.load_database_manager?).to be_falsey
       end
@@ -37,6 +37,24 @@ describe LogStash::Filters::GeoIP do
           expect(plugin.select_database_path).to eql(DEFAULT_CITY_DB_PATH)
         end
       end
-    end if MAJOR < 7 || (MAJOR == 7 && MINOR <= 13)
+    end
+
+    if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+      context "Logstash >= 7.14" do
+        if LogStash::OSS
+          context "OSS-only" do
+            include_examples "without database manager"
+          end
+        else
+          context "default distro" do
+            include_examples "with database manager"
+          end
+        end
+      end
+    else
+      describe "Logstash < 7.14" do
+        include_examples "without database manager"
+      end
+    end
   end
 end

data/spec/filters/test_helper.rb

@@ -2,28 +2,46 @@ require "logstash-core/logstash-core"
 require "digest"
 require "csv"
 
-def get_vendor_path(filename)
-  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
-end
-
-def get_data_dir
-  ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
-end
-
-def get_file_path(filename)
-  ::File.join(get_data_dir, filename)
-end
+# Since we use Logstash's x-pack WITHOUT the LogStash::Runner,
+# we must find it relative to logstash-core and add it to the load path.
+require 'pathname'
+logstash_core_path = Gem.loaded_specs['logstash-core']&.full_gem_path or fail("logstash-core lib not found")
+logstash_xpack_load_path = Pathname.new(logstash_core_path).join("../x-pack/lib").cleanpath.to_s
+if ENV['OSS'] == "true" || !File.exists?(logstash_xpack_load_path)
+  $stderr.puts("X-PACK is not available")
+  LogStash::OSS = true
+else
+  if !$LOAD_PATH.include?(logstash_xpack_load_path)
+    $stderr.puts("ADDING LOGSTASH X-PACK to load path: #{logstash_xpack_load_path}")
+    $LOAD_PATH.unshift(logstash_xpack_load_path)
+  end
+  LogStash::OSS = false
 
-def get_metadata_city_database_name
-  if ::File.exist?(METADATA_PATH)
-    city = ::CSV.read(METADATA_PATH, headers: false).select { |row| row[0].eql?("City") }.last
-    city[3]
+  # when running in a Logstash process that has a geoip extension available, it will
+  # be loaded before this plugin is instantiated. In tests, we need to find and load the
+  # appropriate extension ourselves.
+  extension = nil
+  extension ||= begin; require 'geoip_database_management/extension'; LogStash::const_get("GeoipDatabaseManagement::Extension"); rescue Exception; nil; end
+  extension ||= begin; require 'filters/geoip/extension'; LogStash::const_get("Filters::Geoip::Extension"); rescue Exception; nil; end
+  if extension
+    $stderr.puts("loading logstash extension for geoip: #{extension}")
+    extension.new.tap do |instance|
+      # the extensions require logstash/runner even though they don't need to,
+      # resulting in _all_ extensions being loaded into the registry, including
+      # those whose dependencies are not met by this plugin's dependency graph.
+      def instance.require(path)
+        super unless path == "logstash/runner"
+      end
+    end.additionals_settings(LogStash::SETTINGS)
   else
-    nil
+    $stderr.puts("no logstash extension for geoip is available")
   end
 end
 
-METADATA_PATH = get_file_path("metadata.csv")
+def get_vendor_path(filename)
+  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
+end
+
 DEFAULT_CITY_DB_PATH = get_vendor_path("GeoLite2-City.mmdb")
 DEFAULT_ASN_DB_PATH = get_vendor_path("GeoLite2-ASN.mmdb")
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 7.2.13
+  version: 7.3.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-07 00:00:00.000000000 Z
+date: 2024-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.2'
   name: logstash-mixin-ecs_compatibility_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: insist
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: benchmark-ips
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -93,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 7.14.0
   name: logstash-core
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -127,9 +127,9 @@ files:
 - spec/filters/test_helper.rb
 - vendor/GeoLite2-ASN.mmdb
 - vendor/GeoLite2-City.mmdb
-- vendor/jar-dependencies/com/maxmind/db/maxmind-db/1.2.2/maxmind-db-1.2.2.jar
-- vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.9.0/geoip2-2.9.0.jar
-- vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/6.0.0/logstash-filter-geoip-6.0.0.jar
+- vendor/jar-dependencies/com/maxmind/db/maxmind-db/2.1.0/maxmind-db-2.1.0.jar
+- vendor/jar-dependencies/com/maxmind/geoip2/geoip2/2.17.0/geoip2-2.17.0.jar
+- vendor/jar-dependencies/org/logstash/filters/logstash-filter-geoip/7.3.1/logstash-filter-geoip-7.3.1.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Adds geographical information about an IP address