logstash-lib 1.3.2

data/spec/filters/metrics.rb

@@ -0,0 +1,212 @@
+require "logstash/filters/metrics"
+
+describe LogStash::Filters::Metrics do
+
+  context "with basic meter config" do
+    context "when no events were received" do
+      it "should not flush" do
+        config = {"meter" => ["http.%{response}"]}
+        filter = LogStash::Filters::Metrics.new config
+        filter.register
+
+        events = filter.flush
+        insist { events }.nil?
+      end
+    end
+
+    context "when events are received" do
+      context "on the first flush" do
+        subject {
+          config = {"meter" => ["http.%{response}"]}
+          filter = LogStash::Filters::Metrics.new config
+          filter.register
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 404})
+          filter.flush
+        }
+
+        it "should flush counts" do
+          insist { subject.length } == 1
+          insist { subject.first["http.200.count"] } == 2
+          insist { subject.first["http.404.count"] } == 1
+        end
+
+        it "should include rates and percentiles" do
+          metrics = ["http.200.rate_1m", "http.200.rate_5m", "http.200.rate_15m",
+                     "http.404.rate_1m", "http.404.rate_5m", "http.404.rate_15m"]
+          metrics.each do |metric|
+            insist { subject.first }.include? metric
+          end
+        end
+      end
+
+      context "on the second flush" do
+        it "should not reset counts" do
+          config = {"meter" => ["http.%{response}"]}
+          filter = LogStash::Filters::Metrics.new config
+          filter.register
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 404})
+
+          events = filter.flush
+          events = filter.flush
+          insist { events.length } == 1
+          insist { events.first["http.200.count"] } == 2
+          insist { events.first["http.404.count"] } == 1
+        end
+      end
+    end
+
+    context "when custom rates and percentiles are selected" do
+      context "on the first flush" do
+        subject {
+          config = {
+            "meter" => ["http.%{response}"],
+            "rates" => [1]
+          }
+          filter = LogStash::Filters::Metrics.new config
+          filter.register
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 200})
+          filter.filter LogStash::Event.new({"response" => 404})
+          filter.flush
+        }
+
+        it "should include only the requested rates" do
+          rate_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.200.rate") }
+          insist { rate_fields.length } == 1
+          insist { rate_fields }.include? "http.200.rate_1m"
+        end
+      end
+    end
+  end
+
+  context "with multiple instances" do
+    it "counts should be independent" do
+      config_tag1 = {"meter" => ["http.%{response}"], "tags" => ["tag1"]}
+      config_tag2 = {"meter" => ["http.%{response}"], "tags" => ["tag2"]}
+      filter_tag1 = LogStash::Filters::Metrics.new config_tag1
+      filter_tag2 = LogStash::Filters::Metrics.new config_tag2
+      event_tag1 = LogStash::Event.new({"response" => 200, "tags" => [ "tag1" ]})
+      event_tag2 = LogStash::Event.new({"response" => 200, "tags" => [ "tag2" ]})
+      event2_tag2 = LogStash::Event.new({"response" => 200, "tags" => [ "tag2" ]})
+      filter_tag1.register
+      filter_tag2.register
+
+      [event_tag1, event_tag2, event2_tag2].each do |event|
+        filter_tag1.filter event
+        filter_tag2.filter event
+      end
+
+      events_tag1 = filter_tag1.flush
+      events_tag2 = filter_tag2.flush
+
+      insist { events_tag1.first["http.200.count"] } == 1
+      insist { events_tag2.first["http.200.count"] } == 2
+    end
+  end
+
+  context "with timer config" do
+    context "on the first flush" do
+      subject {
+        config = {"timer" => ["http.request_time", "request_time"]}
+        filter = LogStash::Filters::Metrics.new config
+        filter.register
+        filter.filter LogStash::Event.new({"request_time" => 1})
+        filter.filter LogStash::Event.new({"request_time" => 2})
+        filter.filter LogStash::Event.new({"request_time" => 3})
+        filter.flush
+      }
+
+      it "should flush counts" do
+        insist { subject.length } == 1
+        insist { subject.first["http.request_time.count"] } == 3
+      end
+
+      it "should include rates and percentiles" do
+        metrics = ["rate_1m", "rate_5m", "rate_15m", "p1", "p5", "p10", "p90", "p95", "p99"]
+        metrics.each do |metric|
+          insist { subject.first }.include? "http.request_time.#{metric}"
+        end
+      end
+    end
+  end
+
+  context "when custom rates and percentiles are selected" do
+    context "on the first flush" do
+      subject {
+        config = {
+          "timer" => ["http.request_time", "request_time"],
+          "rates" => [1],
+          "percentiles" => [1, 2]
+        }
+        filter = LogStash::Filters::Metrics.new config
+        filter.register
+        filter.filter LogStash::Event.new({"request_time" => 1})
+        filter.flush
+      }
+
+      it "should flush counts" do
+        insist { subject.length } == 1
+        insist { subject.first["http.request_time.count"] } == 1
+      end
+
+      it "should include only the requested rates" do
+        rate_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.request_time.rate") }
+        insist { rate_fields.length } == 1
+        insist { rate_fields }.include? "http.request_time.rate_1m"
+      end
+
+      it "should include only the requested percentiles" do
+        percentile_fields = subject.first.to_hash.keys.select {|field| field.start_with?("http.request_time.p") }
+        insist { percentile_fields.length } == 2
+        insist { percentile_fields }.include? "http.request_time.p1"
+        insist { percentile_fields }.include? "http.request_time.p2"
+      end
+    end
+  end
+
+
+  context "when a custom flush_interval is set" do
+    it "should flush only when required" do
+      config = {"meter" => ["http.%{response}"], "flush_interval" => 15}
+      filter = LogStash::Filters::Metrics.new config
+      filter.register
+      filter.filter LogStash::Event.new({"response" => 200})
+
+      insist { filter.flush }.nil?        # 5s
+      insist { filter.flush }.nil?        # 10s
+      insist { filter.flush.length } == 1 # 15s
+      insist { filter.flush }.nil?        # 20s
+      insist { filter.flush }.nil?        # 25s
+      insist { filter.flush.length } == 1 # 30s
+    end
+  end
+
+  context "when a custom clear_interval is set" do
+    it "should clear the metrics after interval has passed" do
+      config = {"meter" => ["http.%{response}"], "clear_interval" => 15}
+      filter = LogStash::Filters::Metrics.new config
+      filter.register
+      filter.filter LogStash::Event.new({"response" => 200})
+
+      insist { filter.flush.first["http.200.count"] } == 1 # 5s
+      insist { filter.flush.first["http.200.count"] } == 1 # 10s
+      insist { filter.flush.first["http.200.count"] } == 1 # 15s
+      insist { filter.flush }.nil?                         # 20s
+    end
+  end
+
+  context "when invalid rates are set" do
+    subject {
+      config = {"meter" => ["http.%{response}"], "rates" => [90]}
+      filter = LogStash::Filters::Metrics.new config
+    }
+
+    it "should raise an error" do
+      insist {subject.register }.raises(LogStash::ConfigurationError)
+    end
+  end
+end

data/spec/filters/multiline.rb

@@ -0,0 +1,119 @@
+require "test_utils"
+require "logstash/filters/multiline"
+
+puts "MULTILINE FILTER TEST DISABLED"
+describe LogStash::Filters::Multiline, :if => false do
+
+  extend LogStash::RSpec
+
+  describe "simple multiline" do
+    config <<-CONFIG
+    filter {
+      multiline {
+        pattern => "^\\s"
+        what => previous
+      }
+    }
+    CONFIG
+
+    sample [ "hello world", "   second line", "another first line" ] do
+      p subject.to_hash
+      insist { subject.length } == 2
+      insist { subject[0]["message"] } == "hello world\n   second line"
+      insist { subject[1]["message"] } == "another first line"
+    end
+  end
+
+  describe "multiline using grok patterns" do
+    config <<-CONFIG
+    filter {
+      multiline {
+        pattern => "^%{NUMBER} %{TIME}"
+        negate => true
+        what => previous
+      }
+    }
+    CONFIG
+
+    sample [ "120913 12:04:33 first line", "second line", "third line" ] do
+      insist { subject.length } == 1
+      insist { subject[0]["message"] } ==  "120913 12:04:33 first line\nsecond line\nthird line"
+    end
+  end
+
+  describe "multiline safety among multiple concurrent streams" do
+    config <<-CONFIG
+      filter {
+        multiline {
+          pattern => "^\\s"
+          what => previous
+        }
+      }
+    CONFIG
+
+    multiline_event = [
+      "hello world",
+    ]
+
+    count = 20
+    stream_count = 2
+    id = 0
+    eventstream = count.times.collect do |i|
+      stream = "stream#{i % stream_count}"
+      (
+        [ "hello world #{stream}" ] \
+        + rand(5).times.collect { |n| id += 1; "   extra line #{n} in #{stream} event #{id}" }
+      ) .collect do |line|
+        LogStash::Event.new("message" => line,
+                            "host" => stream, "type" => stream,
+                            "event" => i)
+      end
+    end
+
+    alllines = eventstream.flatten
+
+    # Take whole events and mix them with other events (maintain order)
+    # This simulates a mixing of multiple streams being received 
+    # and processed. It requires that the multiline filter correctly partition
+    # by stream_identity
+    concurrent_stream = eventstream.flatten.count.times.collect do 
+      index = rand(eventstream.count)
+      event = eventstream[index].shift
+      eventstream.delete_at(index) if eventstream[index].empty?
+      event
+    end
+
+    sample concurrent_stream do 
+      insist { subject.count } == count
+      subject.each_with_index do |event, i|
+        #puts "#{i}/#{event["event"]}: #{event.to_json}"
+        #insist { event.type } == stream
+        #insist { event.source } == stream
+        insist { event["message"].split("\n").first } =~ /hello world /
+      end
+    end
+  end
+
+  describe "multiline add/remove tags and fields only when matched" do
+    config <<-CONFIG
+      filter {
+        mutate {
+          add_tag => "dummy"
+        }
+        multiline {
+          add_tag => [ "nope" ]
+          remove_tag => "dummy"
+          add_field => [ "dummy2", "value" ]
+          pattern => "an unlikely match"
+          what => previous
+        }
+      }
+    CONFIG
+
+    sample [ "120913 12:04:33 first line", "120913 12:04:33 second line" ] do
+      subject.each do |s|
+        insist { s.tags.find_index("nope").nil? && s.tags.find_index("dummy") != nil && !s.fields.has_key?("dummy2") } == true
+      end
+    end
+  end 
+end

data/spec/filters/mutate.rb

@@ -0,0 +1,180 @@
+require "test_utils"
+require "logstash/filters/mutate"
+
+describe LogStash::Filters::Mutate do
+  extend LogStash::RSpec
+
+  describe "basics" do
+    config <<-CONFIG
+      filter {
+        mutate {
+          lowercase => "lowerme"
+          uppercase => "upperme"
+          convert => [ "intme", "integer", "floatme", "float" ]
+          rename => [ "rename1", "rename2" ]
+          replace => [ "replaceme", "hello world" ]
+          replace => [ "newfield", "newnew" ]
+          update => [ "nosuchfield", "weee" ]
+          update => [ "updateme", "updated" ]
+          remove => [ "removeme" ]
+        }
+      }
+    CONFIG
+
+    event = {
+      "lowerme" => [ "ExAmPlE" ],
+      "upperme" => [ "ExAmPlE" ],
+      "intme" => [ "1234", "7890.4", "7.9" ],
+      "floatme" => [ "1234.455" ],
+      "rename1" => [ "hello world" ],
+      "updateme" => [ "who cares" ],
+      "replaceme" => [ "who cares" ],
+      "removeme" => [ "something" ]
+    }
+
+    sample event do
+      insist { subject["lowerme"] } == ['example']
+      insist { subject["upperme"] } == ['EXAMPLE']
+      insist { subject["intme"] }   == [1234, 7890, 7]
+      insist { subject["floatme"] } == [1234.455]
+      reject { subject }.include?("rename1")
+      insist { subject["rename2"] } == [ "hello world" ]
+      reject { subject }.include?("removeme")
+
+      insist { subject }.include?("newfield")
+      insist { subject["newfield"] } == "newnew"
+      reject { subject }.include?("nosuchfield")
+      insist { subject["updateme"] } == "updated"
+    end
+  end
+
+  describe "remove multiple fields" do
+    config '
+      filter {
+        mutate {
+          remove => [ "remove-me", "remove-me2", "diedie", "[one][two]" ]
+        }
+      }'
+
+    sample(
+      "remove-me"  => "Goodbye!",
+      "remove-me2" => 1234,
+      "diedie"     => [1, 2, 3, 4],
+      "survivor"   => "Hello.",
+      "one" => { "two" => "wee" }
+    ) do
+      insist { subject["survivor"] } == "Hello."
+      reject { subject }.include?("remove-me")
+      reject { subject }.include?("remove-me2")
+      reject { subject }.include?("diedie")
+      reject { subject["one"] }.include?("two")
+    end
+  end
+
+  describe "convert one field to string" do
+    config '
+      filter {
+        mutate {
+          convert => [ "unicorns", "string" ]
+        }
+      }'
+
+    sample("unicorns" => 1234) do
+      insist { subject["unicorns"] } == "1234"
+    end
+  end
+
+  describe "gsub on a String" do
+    config '
+      filter {
+        mutate {
+          gsub => [ "unicorns", "but extinct", "and common" ]
+        }
+      }'
+
+    sample("unicorns" => "Magnificient, but extinct, animals") do
+      insist { subject["unicorns"] } == "Magnificient, and common, animals"
+    end
+  end
+
+  describe "gsub on an Array of Strings" do
+    config '
+      filter {
+        mutate {
+          gsub => [ "unicorns", "extinct", "common" ]
+        }
+      }'
+
+    sample("unicorns" => [ 
+      "Magnificient extinct animals", "Other extinct ideas" ]
+    ) do
+      insist { subject["unicorns"] } == [
+        "Magnificient common animals",
+        "Other common ideas"
+      ]
+    end
+  end
+
+  describe "gsub on multiple fields" do
+    config '
+      filter {
+        mutate {
+          gsub => [ "colors", "red", "blue",
+                    "shapes", "square", "circle" ]
+        }
+      }'
+
+    sample("colors" => "One red car", "shapes" => "Four red squares") do
+      insist { subject["colors"] } == "One blue car"
+      insist { subject["shapes"] } == "Four red circles"
+    end
+  end
+
+  describe "regression - mutate should lowercase a field created by grok" do
+    config <<-CONFIG
+      filter {
+        grok {
+          match => [ "message", "%{WORD:foo}" ]
+        }
+        mutate {
+          lowercase => "foo"
+        }
+      }
+    CONFIG
+
+    sample "HELLO WORLD" do
+      insist { subject["foo"] } == "hello"
+    end
+  end
+
+  describe "LOGSTASH-757: rename should do nothing with a missing field" do
+    config <<-CONFIG
+      filter {
+        mutate {
+          rename => [ "nosuchfield", "hello" ]
+        }
+      }
+    CONFIG
+
+    sample "whatever" do
+      reject { subject }.include?("nosuchfield")
+      reject { subject }.include?("hello")
+    end
+  end
+
+  describe "convert should work on nested fields" do
+    config <<-CONFIG
+      filter {
+        mutate {
+          convert => [ "[foo][bar]", "integer" ]
+        }
+      }
+    CONFIG
+
+    sample({ "foo" => { "bar" => "1000" } }) do
+      insist { subject["[foo][bar]"] } == 1000
+      insist { subject["[foo][bar]"] }.is_a?(Fixnum)
+    end
+  end
+end
+