logstash-filter-geoip 7.2.6-java → 7.2.10-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 433ba5fee36660147ee97dfe1f43b301184840886bc7bbc5d831b02337ba4606
-  data.tar.gz: 897f4da52600922cfb828444b349123dedc03a79b0062fb494db56561894f8aa
+  metadata.gz: 0e7b28476c126af68bc29d44c7d64ec15aa5e09eed39c45a53d1c52d62f6c2c4
+  data.tar.gz: 6679da72cdaf947fccaf8140a82019668bf6c099f708638fdcf15e0dde8701a6
 SHA512:
-  metadata.gz: 7575caa6518fb518269defd3186b02f04996c723692dd0f0e3e12b3d05b03be51b97f65c74a3b1c110a43d22ce1880601c0729764bb162099cae8c21b12d85a5
-  data.tar.gz: cc3761f22ac75732293f0812851a2b5eab426f510def361cbe8f251dc2685f0ab10e2b0e98bd4d1ee40ff72f43a6e19ccbcac7c5399c0bf12d245db6dcc50484
+  metadata.gz: fc1051fee22e58527e8e091108f9e24d5f314ea92357817bf700765863c6bb320bca5f59835f1a873bb40f4f26df9c1847c18babdc1ebccb94d9f0e60d916f03
+  data.tar.gz: 694bbeeec97b179ccc6d52d5c94a56dfbe693c4bef95a44cbaadaa9bda2bc84b8e6c0eb5b6e73a8b94fb8930a649e59fa4a0f3206b4616baedabcf90015e0372

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 7.2.10
+  - [DOC] Air-gapped environment requires both ASN and City databases [#204](https://github.com/logstash-plugins/logstash-filter-geoip/pull/204)
+
+## 7.2.9
+  - Fix: red CI in Logstash 8.0 [#201](https://github.com/logstash-plugins/logstash-filter-geoip/pull/201)
+  - Update Log4j dependency to 2.17.1
+
+## 7.2.8
+  - Update Log4j dependency to 2.17.0
+
+## 7.2.7
+  - Ensure java 8 compatibility [#197](https://github.com/logstash-plugins/logstash-filter-geoip/pull/197)
+
 ## 7.2.6
   - Update Log4J dependencies [#196](https://github.com/logstash-plugins/logstash-filter-geoip/pull/196)
 

data/docs/index.asciidoc

@@ -86,10 +86,10 @@ a secure proxy. You can then specify the proxy endpoint URL in the
 If you work in air-gapped environment and can't update your databases from the Elastic endpoint,
 You can then download databases from MaxMind and bootstrap the service.
 
-. Download your `.mmdb` database files from the
+. Download both `GeoLite2-ASN.mmdb` and `GeoLite2-City.mmdb` database files from the
 http://dev.maxmind.com/geoip/geoip2/geolite2[MaxMind site].
 
-. Copy your database files to a single directory.
+. Copy both database files to a single directory.
 
 . https://www.elastic.co/downloads/elasticsearch[Download {es}].
 

data/logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.2.6'
+  s.version         = '7.2.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/geoip_ecs_spec.rb

@@ -48,7 +48,6 @@ describe LogStash::Filters::GeoIP do
           end
         end
 
-
         context "with ASN database" do
           let(:options) { common_options.merge({"database" => ASNDB}) }
 
@@ -59,27 +58,96 @@ describe LogStash::Filters::GeoIP do
             expect( event.get ecs_select[disabled: "[#{target}][asn]", v1: "[#{target}][as][number]"] ).to eq 15169
             expect( event.get ecs_select[disabled: "[#{target}][as_org]", v1: "[#{target}][as][organization][name]"] ).to eq "Google LLC"
           end
+
+          context "with customize fields" do
+            let(:fields) { ["AUTONOMOUS_SYSTEM_NUMBER"] }
+            let(:options) { common_options.merge({"database" => ASNDB, "fields" => fields}) }
+
+            it "should give asn field" do
+              plugin.filter(event)
+
+              expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][as_org]", v1: "[#{target}][as][organization][name]"] ).to be_nil
+
+              expect( event.get ecs_select[disabled: "[#{target}][asn]", v1: "[#{target}][as][number]"] ).to eq 15169
+            end
+          end
         end
 
         context "with customize fields" do
-          let(:fields) { ["continent_name", "timezone"] }
-          let(:options) { common_options.merge({"fields" => fields}) }
+          context "continent_name and timezone" do
+            let(:fields) { ["continent_name", "timezone"] }
+            let(:options) { common_options.merge({"fields" => fields}) }
 
-          it "should return fields" do
-            plugin.filter(event)
+            it "should return fields in UTF8" do
+              plugin.filter(event)
 
-            expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to be_nil
-            expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to be_nil
-            expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to be_nil
-            expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to be_nil
-            expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to be_nil
-            expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to be_nil
+
+              continent_name = event.get ecs_select[disabled: "[#{target}][continent_name]", v1: "[#{target}][geo][continent_name]"]
+              timezone = event.get ecs_select[disabled: "[#{target}][timezone]", v1: "[#{target}][geo][timezone]"]
+              expect( continent_name ).to eq "North America"
+              expect( timezone ).to eq "America/Chicago"
+              expect( continent_name.encoding ).to eq Encoding::UTF_8
+              expect( timezone.encoding ).to eq Encoding::UTF_8
+            end
+          end
 
-            expect( event.get ecs_select[disabled: "[#{target}][continent_name]", v1: "[#{target}][geo][continent_name]"] ).to eq "North America"
-            expect( event.get ecs_select[disabled: "[#{target}][timezone]", v1: "[#{target}][geo][timezone]"] ).to eq "America/Chicago"
+          context "location" do
+            shared_examples "provide location, lat and lon" do
+              it "should return location, lat and lon" do
+                plugin.filter(event)
+
+                expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][continent_name]", v1: "[#{target}][geo][continent_name]"] ).to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][timezone]", v1: "[#{target}][geo][timezone]"] ).to be_nil
+
+                expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).not_to be_nil
+                expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).not_to be_nil
+              end
+            end
+
+            context "location and longitude" do
+              let(:fields) { ["location", "longitude"] }
+              let(:options) { common_options.merge({"fields" => fields}) }
+              it_behaves_like "provide location, lat and lon"
+            end
+
+            context "location and latitude" do
+              let(:fields) { ["location", "latitude"] }
+              let(:options) { common_options.merge({"fields" => fields}) }
+              it_behaves_like "provide location, lat and lon"
+            end
           end
-        end
 
+          context "continent_code and IP is IPv6 format" do
+            let(:ip) { "2607:f0d0:1002:51::4" }
+            let(:fields) { ["continent_code", "ip"] }
+            let(:options) { common_options.merge({"fields" => fields}) }
+
+            it "should return fields" do
+              plugin.filter(event)
+
+              expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][continent_name]", v1: "[#{target}][geo][continent_name]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to be_nil
+              expect( event.get ecs_select[disabled: "[#{target}][timezone]", v1: "[#{target}][geo][timezone]"] ).to be_nil
+
+              expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to eq("2607:f0d0:1002:51:0:0:0:4")
+              expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to eq("NA")
+            end
+          end
+        end
       end
     end
 

data/spec/filters/geoip_offline_spec.rb

@@ -6,78 +6,33 @@ require "logstash/filters/geoip"
 CITYDB = ::Dir.glob(::File.expand_path(::File.join("..", "..", "..", "vendor", "GeoLite2-City.mmdb"), __FILE__)).first
 ASNDB = ::Dir.glob(::File.expand_path(::File.join("..", "..", "..", "vendor", "GeoLite2-ASN.mmdb"), __FILE__)).first
 
-describe LogStash::Filters::GeoIP do
-  describe "defaults" do
-    config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-          database => "#{CITYDB}"
-        }
-      }
-    CONFIG
-
-    sample("ip" => "8.8.8.8") do
-      insist { subject }.include?("geoip")
-
-      expected_fields = %w(ip country_code2 country_code3 country_name
-                           continent_code latitude longitude location)
-      expected_fields.each do |f|
-        insist { subject.get("geoip") }.include?(f)
-      end
-    end
 
-    sample("ip" => "127.0.0.1") do
-      # assume geoip fails on localhost lookups
-      expect(subject.get("geoip")).to eq({})
+describe LogStash::Filters::GeoIP do
+  shared_examples "invalid empty IP" do
+    it "should not give target field" do
+      expect(event.get(target)).to be_nil
+      expect(event.get("tags")).to include("_geoip_lookup_failure")
     end
   end
 
-  describe "normal operations" do
-    config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-          database => "#{CITYDB}"
-          target => src_ip
-          add_tag => "done"
-        }
-      }
-    CONFIG
-
-    context "when specifying the target" do
-
-      sample("ip" => "8.8.8.8") do
-        expect(subject).to include("src_ip")
-
-        expected_fields = %w(ip country_code2 country_code3 country_name
-                             continent_code latitude longitude location)
-        expected_fields.each do |f|
-          expect(subject.get("src_ip")).to include(f)
-        end
-      end
-
-      sample("ip" => "127.0.0.1") do
-        # assume geoip fails on localhost lookups
-        expect(subject.get("src_ip")).to eq({})
-      end
-    end
-
-    context "when specifying add_tag" do
-      sample("ip" => "8.8.8.8") do
-        expect(subject.get("tags")).to include("done")
-      end
+  shared_examples "invalid string IP" do
+    it "should give empty hash in target field" do
+      expect(event.get(target)).to eq({})
+      expect(event.get("tags")).to include("_geoip_lookup_failure")
     end
   end
 
-  describe "source is derived from target" do
-    subject(:event) { LogStash::Event.new("target" => { "ip" => "173.9.34.107" } ) }
+  let(:target) { "server" }
+
+  describe "invalid IP" do
+    let(:ip) { "173.9.34.107" }
+    let(:event) { LogStash::Event.new("client" => { "ip" => ip } ) }
     let(:plugin) {
       LogStash::Filters::GeoIP.new(
-        "source" => "[target][ip]",
-        "target" => "target",
-        "fields" => [ "city_name", "region_name" ],
-        "add_tag" => "done", "database" => CITYDB
+        "source" => "[client][ip]",
+        "target" => target,
+        "fields" => %w[country_name continent_code],
+        "database" => CITYDB
       )
     }
 
@@ -86,200 +41,71 @@ describe LogStash::Filters::GeoIP do
       plugin.filter(event)
     end
 
-    context "when source field 'ip' is a subfield of 'target'" do
-
-      it "should preserve value in [target][ip]" do
-        expect(event.get("[target][ip]")).to eq("173.9.34.107")
+    context "when ip is 127.0.0.1" do
+      let(:ip) { "127.0.0.1" }
+      it "should give empty hash" do
+        expect(event.get(target)).to eq({})
       end
-
-      it "should set other subfields of 'target' properly" do
-        expect(event.get("target").to_hash.keys.sort).to eq(["city_name", "ip", "region_name"])
-        expect(event.get("[target][city_name]")).to eq("Malden")
-        expect(event.get("[target][region_name]")).to eq("Massachusetts")
-      end
-
     end
 
-  end
-
-  describe "correct encodings with default db" do
-    config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-          database => "#{CITYDB}"
-        }
-      }
-    CONFIG
-    expected_fields = %w(ip country_code2 country_code3 country_name
-                           continent_code region_name city_name postal_code
-                           dma_code timezone)
-
-    sample("ip" => "1.1.1.1") do
-      checked = 0
-      expected_fields.each do |f|
-        next unless subject.get("geoip")[f]
-        checked += 1
-        insist { subject.get("geoip")[f].encoding } == Encoding::UTF_8
-      end
-      insist { checked } > 0
+    context "when ip is empty string" do
+      let(:ip) { "" }
+      it_behaves_like "invalid empty IP"
     end
 
-    sample("ip" => "189.2.0.0") do
-      checked = 0
-      expected_fields.each do |f|
-        next unless subject.get("geoip")[f]
-        checked += 1
-        insist { subject.get("geoip")[f].encoding } == Encoding::UTF_8
-      end
-      insist { checked } > 0
+    context "when ip is space" do
+      let(:ip) { "      " }
+      it_behaves_like "invalid empty IP"
     end
 
-  end
-
-  describe "location field" do
-    shared_examples_for "an event with a [geoip][location] field" do
-      subject(:event) { LogStash::Event.new("message" => "8.8.8.8") }
-      let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message", "fields" => fields, "database" => CITYDB) }
-
-      before do
-        plugin.register
-        plugin.filter(event)
-      end
-
-      it "should have a location field" do
-        expect(event.get("[geoip][location]")).not_to(be_nil)
-      end
+    context "when ip is dash" do
+      let(:ip) { "-" }
+      it_behaves_like "invalid string IP"
     end
 
-    context "when latitude field is excluded" do
-      let(:fields) { ["country_name", "location", "longitude"] }
-      it_behaves_like "an event with a [geoip][location] field"
+    context "when ip is N/A" do
+      let(:ip) { "N/A" }
+      it_behaves_like "invalid string IP"
     end
 
-    context "when longitude field is excluded" do
-      let(:fields) { ["country_name", "location", "latitude"] }
-      it_behaves_like "an event with a [geoip][location] field"
+    context "when ip is two ip comma separated" do
+      let(:ip) { "123.45.67.89,61.160.232.222" }
+      it_behaves_like "invalid string IP"
     end
 
-    context "when both latitude and longitude field are excluded" do
-      let(:fields) { ["country_name", "location"] }
-      it_behaves_like "an event with a [geoip][location] field"
+    context "when ip is not found in the DB" do
+      let(:ip) { "0.0.0.0" }
+      it_behaves_like "invalid string IP"
     end
-  end
-
-  describe "an invalid IP" do
-    config <<-CONFIG
-          filter {
-            geoip {
-              source => "ip"
-              database => "#{CITYDB}"
-            }
-          }
-        CONFIG
-    describe "should not raise an error" do
-      sample("ip" => "-") do
-        expect{ subject }.to_not raise_error
-      end
 
-      sample("ip" => "~") do
-        expect{ subject }.to_not raise_error
-      end
-
-      sample("ip" => "") do
-        expect{ subject }.to_not raise_error
-      end
-
-      sample("ip" => "      ") do
-        expect{ subject }.to_not raise_error
-      end
+    context "when ip is IPv6 format for localhost" do
+      let(:ip) { "::1" }
+      it_behaves_like "invalid string IP"
     end
+  end
 
-    describe "empty database path" do
-      let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message") }
-      let(:event) { LogStash::Event.new("message" => "8.8.8.8") }
-
-      context "when database manager give nil database path" do
-        it "should tag expired database" do
-          expect(plugin).to receive(:select_database_path).and_return(nil)
-
-          plugin.register
-          plugin.filter(event)
-
-          expect(event.get("tags")).to include("_geoip_expired_database")
-        end
-      end
-    end
+  describe "database path is empty" do
+    let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message", "target" => target) }
+    let(:event) { LogStash::Event.new("message" => "8.8.8.8") }
 
-    describe "filter method outcomes" do
-      let(:plugin) { LogStash::Filters::GeoIP.new("source" => "message", "add_tag" => "done", "database" => CITYDB) }
-      let(:event) { LogStash::Event.new("message" => ipstring) }
+    context "when database manager give nil database path" do
+      it "should tag expired database" do
+        expect(plugin).to receive(:select_database_path).and_return(nil)
 
-      before do
         plugin.register
         plugin.filter(event)
-      end
-
-      context "when the bad IP is N/A" do
-        # regression test for issue https://github.com/logstash-plugins/logstash-filter-geoip/issues/50
-        let(:ipstring) { "N/A" }
-
-        it "should set the target field to an empty hash" do
-          expect(event.get("geoip")).to eq({})
-        end
-
-        it "should add failure tags" do
-          expect(event.get("tags")).to include("_geoip_lookup_failure")
-        end
-      end
-
-      context "when the bad IP is two ip comma separated" do
-        # regression test for issue https://github.com/logstash-plugins/logstash-filter-geoip/issues/51
-        let(:ipstring) { "123.45.67.89,61.160.232.222" }
 
-        it "should set the target field to an empty hash" do
-          expect(event.get("geoip")).to eq({})
-        end
+        expect(event.get("tags")).to include("_geoip_expired_database")
       end
-
-      context "when a IP is not found in the DB" do
-        let(:ipstring) { "0.0.0.0" }
-
-        it "should set the target field to an empty hash" do
-          expect(event.get("geoip")).to eq({})
-          expect(event.get("tags")).to include("_geoip_lookup_failure")
-        end
-      end
-
-      context "when IP is IPv6 format for localhost" do
-        let(:ipstring) { "::1" }
-
-        it "should set the target field to an empty hash" do
-          expect(event.get("geoip")).to eq({})
-        end
-      end
-
-      context "when IP is valid IPv6 format" do
-        let(:ipstring) { "2607:f0d0:1002:51::4" }
-
-        it "should set the target fields properly" do
-          expect(event.get("geoip")).not_to be_empty
-          expect(event.get("geoip")["ip"]).to eq("2607:f0d0:1002:51:0:0:0:4")
-          expect(event.get("geoip").to_hash.keys.sort).to eq(
-              ["continent_code", "country_code2", "country_code3", "country_name", "ip", "latitude", "location", "longitude", "timezone"]
-                                                          )
-        end
-      end
-
     end
-
   end
 
-  describe "an invalid database" do
+  describe "database path is an invalid database file" do
     config <<-CONFIG
           filter {
             geoip {
               source => "ip"
+              target => "geo"
               database => "./Gemfile"
             }
           }
@@ -292,40 +118,4 @@ describe LogStash::Filters::GeoIP do
     end
   end
 
-  describe "GeoIP2-ASN database" do
-    config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-          database => "#{ASNDB}"
-          default_database_type => "ASN"
-        }
-      }
-    CONFIG
-
-    sample("ip" => "8.8.8.8") do
-      expect(subject.get("geoip")).not_to be_empty
-      expect(subject.get("geoip")["asn"]).to eq(15169)
-      expect(subject.get("geoip")["as_org"]).to eq("Google LLC")
-    end
-  end
-
-  describe "GeoIP2-ASN database with fields" do
-    config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-          database => "#{ASNDB}"
-          default_database_type => "ASN"
-          fields => [ "AUTONOMOUS_SYSTEM_NUMBER" ]
-        }
-      }
-    CONFIG
-
-    sample("ip" => "8.8.8.8") do
-      expect(subject.get("geoip")).not_to be_empty
-      expect(subject.get("geoip")["asn"]).to eq(15169)
-    end
-  end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-geoip
 version: !ruby/object:Gem::Version
-  version: 7.2.6
+  version: 7.2.10
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-10 00:00:00.000000000 Z
+date: 2022-01-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement