logstash-lib 1.3.2

data/spec/examples/syslog.rb

@@ -0,0 +1,48 @@
+require "test_utils"
+
+describe "parse syslog", :if => RUBY_ENGINE == "jruby" do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grok {
+          type => "syslog"
+          singles => true
+          pattern => [ "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" ]
+          add_field => [ "received_at", "%{@timestamp}" ]
+          add_field => [ "received_from", "%{source_host}" ]
+      }
+      syslog_pri {
+          type => "syslog"
+      }
+      date {
+          type => "syslog"
+          match => ["syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+      }
+      mutate {
+          type => "syslog"
+          exclude_tags => "_grokparsefailure"
+          replace => [ "source_host", "%{syslog_hostname}" ]
+          replace => [ "message", "%{syslog_message}" ]
+      }
+      mutate {
+          type => "syslog"
+          remove => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ]
+      }
+    }
+  CONFIG
+
+  sample("message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]", "type" => "syslog") do
+    insist { subject["type"] } == "syslog"
+    insist { subject["tags"] }.nil?
+    insist { subject["syslog_pri"] } == "164"
+  end
+
+  # Single digit day
+  sample("message" => "<164>Oct  6 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]", "type" => "syslog") do
+    insist { subject["type"] } == "syslog"
+    insist { subject["tags"] }.nil?
+    insist { subject["syslog_pri"] } == "164"
+    #insist { subject.timestamp } == "2012-10-26T15:19:25.000Z"
+  end
+end

data/spec/filters/alter.rb

@@ -0,0 +1,96 @@
+require "test_utils"
+require "logstash/filters/alter"
+
+describe LogStash::Filters::Alter do
+  extend LogStash::RSpec
+
+  describe "condrewrite with static values" do
+    config <<-CONFIG
+    filter {
+      alter {
+        condrewrite => ["rewrite-me", "hello", "goodbye"]
+      }
+    }
+    CONFIG
+
+    sample("rewrite-me"  => "hello") do
+      insist { subject["rewrite-me"] } == "goodbye"
+    end
+
+    sample("rewrite-me"  => "greetings") do
+      insist { subject["rewrite-me"] } == "greetings"
+    end
+  end
+
+  describe "condrewrite with dynamic values" do
+    config <<-CONFIG
+    filter {
+      alter {
+        condrewrite => ["rewrite-me", "%{test}", "%{rewrite-value}"]
+      }
+    }
+    CONFIG
+
+    sample("rewrite-me"  => "hello", "test" => "hello",
+           "rewrite-value" => "goodbye") do
+      insist { subject["rewrite-me"] } == "goodbye"
+    end
+
+    sample("rewrite-me"  => "hello") do
+      insist { subject["rewrite-me"] } == "hello"
+    end
+
+    sample("rewrite-me"  => "%{test}") do
+      insist { subject["rewrite-me"] } == "%{rewrite-value}"
+    end
+
+    sample("rewrite-me"  => "hello", "test" => "hello") do
+      insist { subject["rewrite-me"] } == "%{rewrite-value}"
+    end
+
+    sample("rewrite-me"  => "greetings", "test" => "hello") do
+      insist { subject["rewrite-me"] } == "greetings"
+    end
+  end
+
+  describe "condrewriteother" do
+    config <<-CONFIG
+    filter {
+      alter {
+        condrewriteother => ["test-me", "hello", "rewrite-me","goodbye"]
+      }
+    }
+    CONFIG
+
+    sample("test-me"  => "hello") do
+      insist { subject["rewrite-me"] } == "goodbye"
+    end
+
+    sample("test-me"  => "hello", "rewrite-me"  => "hello2") do
+      insist { subject["rewrite-me"] } == "goodbye"
+    end
+
+    sample("test-me"  => "greetings") do
+      insist { subject["rewrite-me"] }.nil?
+    end
+
+    sample("test-me"  => "greetings",
+      "rewrite-me"  => "hello2") do
+      insist { subject["rewrite-me"] } == "hello2"
+    end
+  end
+
+  describe "coalesce" do
+    config <<-CONFIG
+    filter {
+      alter {
+        coalesce => ["coalesce-me", "%{non-existing-field}", "mydefault"]
+      }
+    }
+    CONFIG
+
+    sample("coalesce-me"  => "Hello") do
+      insist { subject["coalesce-me"] } == "mydefault" 
+    end
+  end
+end

data/spec/filters/anonymize.rb

@@ -0,0 +1,189 @@
+require "test_utils"
+require "logstash/filters/anonymize"
+
+describe LogStash::Filters::Anonymize do
+  extend LogStash::RSpec
+
+  describe "anonymize ipaddress with IPV4_NETWORK algorithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          algorithm => "IPV4_NETWORK"
+          key => 24
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "233.255.13.44") do
+      insist { subject["clientip"] } == "233.255.13.0"
+    end
+  end
+
+  describe "anonymize string with MURMUR3 algorithm" do
+    config <<-CONFIG
+      filter { 
+        anonymize { 
+          fields => ["clientip"]
+          algorithm => "MURMUR3"
+          key => ""
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.52.122.33") do
+      insist { subject["clientip"] } == 1541804874
+    end
+  end
+ 
+   describe "anonymize string with SHA1 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'SHA1'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject["clientip"] } == "fdc60acc4773dc5ac569ffb78fcb93c9630797f4"
+    end
+  end
+
+  # HMAC-SHA224 isn't implemented in JRuby OpenSSL
+  #describe "anonymize string with SHA224 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    #config <<-CONFIG
+      #filter {
+        #anonymize {
+          #fields => ["clientip"]
+          #key => "longencryptionkey"
+          #algorithm => 'SHA224'
+        #}
+      #}
+    #CONFIG
+
+    #sample("clientip" => "123.123.123.123") do
+      #insist { subject["clientip"] } == "5744bbcc4f64acb6a805b7fee3013a8958cc8782d3fb0fb318cec915"
+    #end
+  #end
+
+  describe "anonymize string with SHA256 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'SHA256'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject["clientip"] } == "345bec3eff242d53b568916c2610b3e393d885d6b96d643f38494fd74bf4a9ca"
+    end
+  end
+
+  describe "anonymize string with SHA384 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'SHA384'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject["clientip"] } == "22d4c0e8c4fbcdc4887d2038fca7650f0e2e0e2457ff41c06eb2a980dded6749561c814fe182aff93e2538d18593947a"
+    end
+  end
+
+  describe "anonymize string with SHA512 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'SHA512'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject["clientip"] } == "11c19b326936c08d6c50a3c847d883e5a1362e6a64dd55201a25f2c1ac1b673f7d8bf15b8f112a4978276d573275e3b14166e17246f670c2a539401c5bfdace8"
+    end
+  end
+
+  # HMAC-MD4 isn't implemented in JRuby OpenSSL
+  #describe "anonymize string with MD4 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    #config <<-CONFIG
+      #filter {
+        #anonymize {
+          #fields => ["clientip"]
+          #key => "longencryptionkey"
+          #algorithm => 'MD4'
+        #}
+      #}
+    #CONFIG
+#
+    #sample("clientip" => "123.123.123.123") do
+      #insist { subject["clientip"] } == "0845cb571ab3646e51a07bcabf05e33d"
+    #end
+  #end
+
+  describe "anonymize string with MD5 alogrithm" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'MD5'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject["clientip"] } == "9336c879e305c9604a3843fc3e75948f"
+    end
+  end
+
+  describe "Test field with multiple values" do
+    # The logstash config goes here.
+    # At this time, only filters are supported.
+    config <<-CONFIG
+      filter {
+        anonymize {
+          fields => ["clientip"]
+          key => "longencryptionkey"
+          algorithm => 'MD5'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => [ "123.123.123.123", "223.223.223.223" ]) do
+      insist { subject["clientip"]} == [ "9336c879e305c9604a3843fc3e75948f", "7a6c66b8d3f42a7d650e3354af508df3" ]
+    end
+  end
+
+  
+
+end

data/spec/filters/checksum.rb

@@ -0,0 +1,41 @@
+require "test_utils"
+require "logstash/filters/checksum"
+require 'openssl'
+
+describe LogStash::Filters::Checksum do
+  extend LogStash::RSpec
+  
+  LogStash::Filters::Checksum::ALGORITHMS.each do |alg|
+    describe "#{alg} checksum with single field" do
+      config <<-CONFIG
+        filter {
+          checksum {
+            algorithm => "#{alg}"
+            keys => ["test"]
+          }
+        }
+        CONFIG
+
+      sample "test" => "foo bar" do
+        insist { !subject["logstash_checksum"].nil? }
+        insist { subject["logstash_checksum"] } == OpenSSL::Digest.hexdigest(alg, "|test|foo bar|")
+      end
+    end
+    
+    describe "#{alg} checksum with multiple keys" do
+      config <<-CONFIG
+        filter {
+          checksum {
+            algorithm => "#{alg}"
+            keys => ["test1", "test2"]
+          }
+        }
+        CONFIG
+
+      sample "test1" => "foo", "test2" => "bar" do
+        insist { !subject["logstash_checksum"].nil? }
+        insist { subject["logstash_checksum"] } == OpenSSL::Digest.hexdigest(alg, "|test1|foo|test2|bar|")
+      end
+    end
+  end
+end

data/spec/filters/clone.rb

@@ -0,0 +1,67 @@
+require "test_utils"
+require "logstash/filters/clone"
+
+describe LogStash::Filters::Clone do
+  extend LogStash::RSpec
+
+  describe "all defaults" do
+    type "original"
+    config <<-CONFIG
+      filter {
+        clone {
+          type => "original"
+          clones => ["clone", "clone", "clone"]
+        }
+      }
+    CONFIG
+
+    sample("message" => "hello world", "type" => "original") do
+      insist { subject }.is_a? Array
+      insist { subject.length } == 4
+      subject.each_with_index do |s,i|
+        if i == 0 # last one should be 'original'
+          insist { s["type"] } == "original"
+        else
+          insist { s["type"]} == "clone"
+        end
+        insist { s["message"] } == "hello world"
+      end
+    end
+  end
+
+  describe "Complex use" do
+    config <<-CONFIG
+      filter {
+        clone {
+          type => "nginx-access"
+          tags => ['TESTLOG']
+          clones => ["nginx-access-clone1", "nginx-access-clone2"]
+          add_tag => ['RABBIT','NO_ES']
+          remove_tag => ["TESTLOG"]
+        }
+      }
+    CONFIG
+
+    sample("type" => "nginx-access", "tags" => ["TESTLOG"], "message" => "hello world") do
+      insist { subject }.is_a? Array
+      insist { subject.length } == 3
+
+      insist { subject[0]["type"] } == "nginx-access"
+      #Initial event remains unchanged
+      insist { subject[0]["tags"] }.include? "TESTLOG"
+      reject { subject[0]["tags"] }.include? "RABBIT"
+      reject { subject[0]["tags"] }.include? "NO_ES"
+      #All clones go through filter_matched
+      insist { subject[1]["type"] } == "nginx-access-clone1"
+      reject { subject[1]["tags"] }.include? "TESTLOG"
+      insist { subject[1]["tags"] }.include? "RABBIT"
+      insist { subject[1]["tags"] }.include? "NO_ES"
+
+      insist { subject[2]["type"] } == "nginx-access-clone2"
+      reject { subject[2]["tags"] }.include? "TESTLOG"
+      insist { subject[2]["tags"] }.include? "RABBIT"
+      insist { subject[2]["tags"] }.include? "NO_ES"
+
+    end
+  end
+end

data/spec/filters/collate.rb

@@ -0,0 +1,122 @@
+require "test_utils"
+require "logstash/filters/collate"
+
+describe LogStash::Filters::Collate do
+  extend LogStash::RSpec
+
+  describe "collate when count is full" do
+    config <<-CONFIG
+      filter {
+        collate {
+          count => 2
+        }
+      }
+    CONFIG
+
+    events = [
+      {
+        "@timestamp" => Time.iso8601("2013-01-02T00:00:00.000Z"),
+        "message" => "later message"
+      },
+      {
+        "@timestamp" => Time.iso8601("2013-01-01T00:00:00.000Z"),
+        "message" => "earlier message"
+      }
+    ]
+
+    sample(events) do
+      insist { subject }.is_a? Array
+      insist { subject.length } == 2
+      subject.each_with_index do |s,i|
+        if i == 0 # first one should be the earlier message
+          insist { s["message"] } == "earlier message"
+        end
+        if i == 1 # second one should be the later message
+          insist { s["message"]} == "later message"
+        end
+      end
+    end
+  end
+
+  describe "collate by desc" do
+    config <<-CONFIG
+      filter {
+        collate {
+          count => 3
+          order => "descending"
+        }
+      }
+    CONFIG
+
+    events = [
+      {
+        "@timestamp" => Time.iso8601("2013-01-03T00:00:00.000Z"),
+        "message" => "third message"
+      },
+      {
+        "@timestamp" => Time.iso8601("2013-01-01T00:00:00.000Z"),
+        "message" => "first message"
+      },
+      {
+        "@timestamp" => Time.iso8601("2013-01-02T00:00:00.000Z"),
+        "message" => "second message"
+      }
+    ]
+
+    sample(events) do
+      insist { subject }.is_a? Array
+      insist { subject.length } == 3
+      subject.each_with_index do |s,i|
+        if i == 0 # first one should be the third message
+          insist { s["message"] } == "third message"
+        end
+        if i == 1 # second one should be the second message
+          insist { s["message"]} == "second message"
+        end
+        if i == 2 # third one should be the third message
+          insist { s["message"]} == "first message"
+        end
+      end
+    end
+  end
+
+  # (Ignored) Currently this case can't pass because of the case depends on the flush function of the filter in the test, 
+  # there was a TODO marked in the code (test_utils.rb, # TODO(sissel): pipeline flush needs to be implemented.), 
+  # and the case wants to test the scenario which collate was triggered by a scheduler, so in this case, it needs to sleep few seconds 
+  # waiting the scheduler triggered, and after the events were flushed, then the result can be checked.
+
+  # describe "collate when interval reached" do
+  #   config <<-CONFIG
+  #     filter {
+  #       collate {
+  #         interval => "1s"
+  #       }
+  #     }
+  #   CONFIG
+
+  #   events = [
+  #     {
+  #       "@timestamp" => Time.iso8601("2013-01-02T00:00:00.000Z"),
+  #       "message" => "later message"
+  #     },
+  #     {
+  #       "@timestamp" => Time.iso8601("2013-01-01T00:00:00.000Z"),
+  #       "message" => "earlier message"
+  #     }
+  #   ]
+
+  #   sample(events) do
+  #     sleep(2)
+  #     insist { subject }.is_a? Array
+  #     insist { subject.length } == 2
+  #     subject.each_with_index do |s,i|
+  #       if i == 0 # first one should be the earlier message
+  #         insist { s["message"] } == "earlier message"
+  #       end
+  #       if i == 1 # second one should be the later message
+  #         insist { s["message"]} == "later message"
+  #       end
+  #     end
+  #   end
+  # end
+end