logstash-filter-real_ip 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03a6d392653c84b6e9979c4242b33addc2bef2b9edbc06a02ab12bf4b0e817eb
-  data.tar.gz: b86af8b9cb8beeb0018548058bda54666cc96a8c9a79dfa808e537ea24910ef9
+  metadata.gz: df7a4bbe848333419b6bd9cdddcb6396caa6469e1e80db081282c0ce3d6afe1a
+  data.tar.gz: d56cd9f9f44694175f181777d3b25244236c80c97d129a8bd4eb90c42c8ad2c4
 SHA512:
-  metadata.gz: 80d539a35dc6a9ae7957e2051e75a3440aed0a885ea52bc565d757d5166a6623f17e54a933d4c1440922d89a44d1b0d19ab98b4e9613d298a8a71ba979e0ba9c
-  data.tar.gz: c8a5626f432719c803a2abd375f31ae4d23390702e124cd9ed90233dd9bbd32c9f2ab77569d5ba7155c18e16999123783c120360d6fd88686a33eed975677d73
+  metadata.gz: d2f799ae54fffe451882944901b007d426f6aaa2a9909ed339bf50f5112f38a186d49e48d667d4513eb0722035aea2f3a05b9da6ed0e2aa56444c5838e653d68
+  data.tar.gz: 7df5f2176930a09f686bc63a2f5841a9b9985acef23d33ce0f461b0d23f002c866a2b141545323af6636b2369d7c85e442dce1c0c74ef26b549df541775f9e20

data/CHANGELOG.md

@@ -1,4 +1,8 @@
 
+## 0.2.0
+ - Feature: Write all x-forwarded-for IPs as array in new field
+   and detect invalid IPs, tagging the event if necessary
+
 ## 0.1.1
  - fix: Fix edge case where x-forwarded-for is empty or only contains trusted IPs
 

data/lib/logstash/filters/real_ip.rb

@@ -98,6 +98,16 @@ class LogStash::Filters::RealIp < LogStash::Filters::Base
   # address to.
   config :target_field, :validate => :string, :default => "real_ip"
 
+  # writes all valid IPs found in the x_forwarded_for header to this field
+  config :x_forwarded_for_target, :validate => :string, :default => ""
+
+  # In case any IPs processed are invalid IP addresses,
+  # write all of them as one string to the field specified.
+  config :target_on_invalid_ip, :validate => :string, :default => ""
+
+  # In case any IPs processed are invalid IP addresses, these tags will be set.
+  config :tags_in_invalid_ip, :valudate => :array, :default => ["_real_ip_invalid_ip"]
+
   # In case of error during evaluation, these tags will be set.
   config :tags_on_failure, :validate => :array, :default => ["_real_ip_lookup_failure"]
 
@@ -122,6 +132,7 @@ class LogStash::Filters::RealIp < LogStash::Filters::Base
     end
 
     @trusted_networks.map! {|e| IPAddr.new(e)}
+    @need_all = x_forwarded_for_target.length > 0
   end # def register
 
   private
@@ -151,7 +162,7 @@ class LogStash::Filters::RealIp < LogStash::Filters::Base
 
     # check for presence of x_forwarded_for_field
     if fwdfor == nil
-      @logger.info("x_forwarded_for_field missing from event", :event => event)
+      @logger.debug? and @logger.debug("x_forwarded_for_field missing from event", :event => event)
       event.set(@target_field, remote_addr)
       filter_matched(event)
       return
@@ -206,29 +217,43 @@ class LogStash::Filters::RealIp < LogStash::Filters::Base
       return
     end
 
+    found = false
+    fatal = false
+    target = []
     # check each IP in x_forwarded_for_field from last to first
     (fwdfor.length - 1).downto(0) do |i|
       begin
         ip = IPAddr.new(fwdfor[i])
       rescue ArgumentError => e
         @logger.warn("Invalid IP address", :address => fwdfor[i], :event => event)
-        @tags_on_failure.each {|tag| event.tag(tag)}
-        return
+        if not found
+          @tags_on_failure.each {|tag| event.tag(tag)}
+          fatal = true
+        end
+        @tags_in_invalid_ip.each {|tag| event.tag(tag)}
+        next
       end
 
+      target.unshift(ip.to_s()) if @need_all
+
       # return on the first non-match against our trusted networks
-      if match(ip) == false
+      if found == false and fatal == false and match(ip) == false
         event.set(@target_field, fwdfor[i])
         filter_matched(event)
-        return
+        return if not @need_all
+        found = true
       end
     end
 
-    # in case remote_addr and all x_forwarded_for IPs are trusted, use the
-    # left-most IP from x_forwarded_for
-    event.set(@target_field, fwdfor[0])
-    filter_matched(event)
-    return
+    event.set(@x_forwarded_for_target, target) if @need_all
+
+    if found == false and fatal == false
+      # in case remote_addr and all x_forwarded_for IPs are trusted, use the
+      # left-most IP from x_forwarded_for
+      event.set(@target_field, fwdfor[0])
+      filter_matched(event)
+      return
+    end
 
   end # def filter
 end # class LogStash::Filters::RealIp

data/logstash-filter-real_ip.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-real_ip'
-  s.version       = '0.1.1'
+  s.version       = '0.2.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Evaluates real client IP from X-Forwarded-For header'
   s.description   = 'See https://github.com/fholzer/logstash-filter-real_ip/blob/master/README.md for details.'

data/spec/filters/real_ip_spec.rb

@@ -10,57 +10,81 @@ describe LogStash::Filters::RealIp do
           remote_address_field => "remote_addr"
           x_forwarded_for_field => "xfwdfor"
           trusted_networks => ["10.0.0.0/8", "192.168.0.0/16"]
+          add_field => { "success" => "success" }
         }
       }
     CONFIG
     end
 
     sample("dummy") do
+      expect(subject).not_to include("success")
       expect(subject).to include("tags")
       expect(subject.get('tags')).to include("_real_ip_lookup_failure")
     end
 
     sample("remote_addr" => "1.2.3.4") do
-      if subject.get('tags') then expect(subject.get('tags')).not_to include("_real_ip_lookup_failure") end
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('10.2.3.4')
     end
 
     sample("remote_addr" => "1.2.3.4", "xfwdfor" => "") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "1.2.3.4") do
-      expect(subject.get("tags")).to eq(nil)
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["1.2.3.4", "192.168.3.4"]) do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["192.168.5.6", "192.168.3.4"]) do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('192.168.5.6')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => []) do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('10.2.3.4')
     end
+
+    sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["10.5", "1.2.3.4", "192.168.3.4"]) do
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
+      expect(subject).to include("real_ip")
+      expect(subject.get('real_ip')).to eq('1.2.3.4')
+    end
+
+    sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["10.5", "192.168.3.4"]) do
+      expect(subject).not_to include("success")
+      expect(subject).to include("tags")
+      expect(subject).not_to include("real_ip")
+      expect(subject.get('tags')).to include('_real_ip_invalid_ip')
+      expect(subject.get('tags')).to include('_real_ip_lookup_failure')
+    end
   end
 
   describe "Evaluates real IP correctly with flat string xfwfor" do
@@ -71,24 +95,28 @@ describe LogStash::Filters::RealIp do
           x_forwarded_for_field => "xfwdfor"
           x_forwarded_for_is_string => true
           trusted_networks => ["10.0.0.0/8", "192.168.0.0/16"]
+          add_field => { "success" => "success" }
         }
       }
     CONFIG
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "1.2.3.4") do
-      expect(subject.get("tags")).to eq(nil)
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "1.2.3.4,+192.168.3.4") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "1.2.3.4, 192.168.3.4,+ 192.168.4.5") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
@@ -103,30 +131,35 @@ describe LogStash::Filters::RealIp do
           x_forwarded_for_field => "xfwdfor"
           target_field => "evaluated_ip"
           trusted_networks => ["10.0.0.0/8", "192.168.0.0/16"]
+          add_field => { "success" => "success" }
         }
       }
     CONFIG
     end
 
     sample("remote_addr" => "1.2.3.4") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("evaluated_ip")
       expect(subject.get('evaluated_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "") do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("evaluated_ip")
       expect(subject.get('evaluated_ip')).to eq('10.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => "1.2.3.4") do
-      expect(subject.get("tags")).to eq(nil)
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
       expect(subject).to include("evaluated_ip")
       expect(subject.get('evaluated_ip')).to eq('1.2.3.4')
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["1.2.3.4", "192.168.3.4"]) do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("evaluated_ip")
       expect(subject.get('evaluated_ip')).to eq('1.2.3.4')
@@ -140,15 +173,60 @@ describe LogStash::Filters::RealIp do
           remote_address_field => "remote_addr"
           x_forwarded_for_field => "xfwdfor"
           trusted_networks => ["10.0.0.0/8", "2606:2800:220:1:248:1893:25c8:1946/120"]
+          add_field => { "success" => "success" }
         }
       }
     CONFIG
     end
 
     sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["1.2.3.4", "2606:2800:220:1:248:1893:25c8:1946"]) do
+      expect(subject).to include("success")
+      expect(subject).not_to include("tags")
+      expect(subject).to include("real_ip")
+      expect(subject.get('real_ip')).to eq('1.2.3.4')
+    end
+  end
+
+  describe "writes IPs to target array" do
+    let(:config) do <<-CONFIG
+      filter {
+        real_ip {
+          remote_address_field => "remote_addr"
+          x_forwarded_for_field => "xfwdfor"
+          x_forwarded_for_target => "xtarget"
+          trusted_networks => ["10.0.0.0/8", "192.168.0.0/16"]
+          add_field => { "success" => "success" }
+        }
+      }
+    CONFIG
+    end
+
+    sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["5.6.7.8", "1.2.3.4", "192.168.3.4"]) do
+      expect(subject).to include("success")
       expect(subject).not_to include("tags")
       expect(subject).to include("real_ip")
       expect(subject.get('real_ip')).to eq('1.2.3.4')
+      expect(subject.get('xtarget')).to eq(["5.6.7.8", "1.2.3.4", "192.168.3.4"])
+    end
+
+    sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["10.5", "1.2.3.4", "192.168.3.4"]) do
+      expect(subject).to include("success")
+      expect(subject).to include("tags")
+      expect(subject).to include("real_ip")
+      expect(subject.get('real_ip')).to eq('1.2.3.4')
+      expect(subject.get('xtarget')).to eq(["1.2.3.4", "192.168.3.4"])
+      expect(subject.get('tags')).to include('_real_ip_invalid_ip')
+      expect(subject.get('tags')).not_to include('_real_ip_lookup_failure')
+    end
+
+    sample("remote_addr" => "10.2.3.4", "xfwdfor" => ["10.5", "192.168.3.4"]) do
+      expect(subject).not_to include("success")
+      expect(subject).to include("tags")
+      expect(subject).not_to include("real_ip")
+      expect(subject.get('xtarget')).to eq(["192.168.3.4"])
+      expect(subject.get('tags')).to include('_real_ip_invalid_ip')
+      expect(subject.get('tags')).to include('_real_ip_lookup_failure')
     end
   end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-real_ip
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Ferdinand Holzer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-04 00:00:00.000000000 Z
+date: 2018-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement