logstash-lib 1.3.2

data/spec/outputs/file.rb

@@ -0,0 +1,82 @@
+require "test_utils"
+require "logstash/outputs/file"
+require "tempfile"
+
+describe LogStash::Outputs::File do
+  extend LogStash::RSpec
+
+  describe "ship lots of events to a file" do
+    event_count = 10000 + rand(500)
+    tmp_file = Tempfile.new('logstash-spec-output-file')
+    
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "generator"
+        }
+      }
+      output {
+        file {
+          path => "#{tmp_file.path}"
+        }
+      }
+    CONFIG
+
+    agent do
+      line_num = 0
+      # Now check all events for order and correctness.
+      File.foreach(tmp_file) do |line|
+        event = LogStash::Event.new(JSON.parse(line))
+        insist {event["message"]} == "hello world"
+        insist {event["sequence"]} == line_num
+        line_num += 1
+      end
+      insist {line_num} == event_count
+    end # agent
+  end
+
+  describe "ship lots of events to a file gzipped" do
+    event_count = 10000 + rand(500)
+    tmp_file = Tempfile.new('logstash-spec-output-file')
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "generator"
+        }
+      }
+      output {
+        file {
+          path => "#{tmp_file.path}"
+          gzip => true
+        }
+      }
+    CONFIG
+
+    agent do
+      line_num = 0
+      # Now check all events for order and correctness.
+      Zlib::GzipReader.new(File.open(tmp_file)).each_line do |line|
+        event = LogStash::Event.new(JSON.parse(line))
+        insist {event["message"]} == "hello world"
+        insist {event["sequence"]} == line_num
+        line_num += 1
+      end
+      insist {line_num} == event_count
+
+      #LOGSTASH-997 confirm usage of zcat command on file
+      line_num = 0
+      `zcat #{tmp_file.path()}`.split("\n").each do |line|
+        event = LogStash::Event.new(JSON.parse(line))
+        insist {event["message"]} == "hello world"
+        insist {event["sequence"]} == line_num
+        line_num += 1
+      end
+      insist {line_num} == event_count
+    end # agent
+  end
+end

data/spec/outputs/graphite.rb

@@ -0,0 +1,236 @@
+require "test_utils"
+require "logstash/outputs/graphite"
+require "mocha/api"
+
+describe LogStash::Outputs::Graphite do
+  extend LogStash::RSpec
+
+  describe "defaults should include all metrics" do
+    port = 4939
+    config <<-CONFIG
+      input {
+        generator {
+          message => "foo=fancy bar=42"
+          count => 1
+          type => "generator"
+        }
+      }
+
+      filter {
+        kv { }
+      }
+
+      output {
+        graphite {
+          host => "localhost"
+          port => #{port}
+          metrics => [ "hurray.%{foo}", "%{bar}" ]
+        }
+      }
+    CONFIG
+
+    let(:queue) { Queue.new }
+    before :each do
+      server = TCPServer.new("127.0.0.1", port)
+      Thread.new do
+        client = server.accept
+        p client
+        while true
+          p :read
+          line = client.readline
+          p :done
+          queue << line
+          p line
+        end
+      end
+    end
+
+    agent do
+      lines = queue.pop
+
+      insist { lines.size } == 1
+      insist { lines }.any? { |l| l =~ /^hurray.fancy 42.0 \d{10,}\n$/ }
+    end
+  end
+
+  describe "fields_are_metrics => true" do
+    describe "metrics_format => ..." do
+      describe "match one key" do
+        config <<-CONFIG
+          input {
+            generator {
+              message => "foo=123"
+              count => 1
+              type => "generator"
+            }
+          }
+
+          filter {
+            kv { }
+          }
+
+          output {
+            graphite {
+                host => "localhost"
+                port => 2003
+                fields_are_metrics => true
+                include_metrics => ["foo"]
+                metrics_format => "foo.bar.sys.data.*"
+                debug => true
+            }
+          }
+        CONFIG
+
+        agent do
+          @mock.rewind
+          lines = @mock.readlines
+          insist { lines.size } == 1
+          insist { lines[0] } =~ /^foo.bar.sys.data.foo 123.0 \d{10,}\n$/
+        end
+      end
+
+      describe "match all keys" do
+        config <<-CONFIG
+          input {
+            generator {
+              message => "foo=123 bar=42"
+              count => 1
+              type => "generator"
+            }
+          }
+
+          filter {
+            kv { }
+          }
+
+          output {
+            graphite {
+                host => "localhost"
+                port => 2003
+                fields_are_metrics => true
+                include_metrics => [".*"]
+                metrics_format => "foo.bar.sys.data.*"
+                debug => true
+            }
+          }
+        CONFIG
+
+        agent do
+          @mock.rewind
+          lines = @mock.readlines.delete_if { |l| l =~ /\.sequence \d+/ }
+
+          insist { lines.size } == 2
+          insist { lines }.any? { |l| l =~ /^foo.bar.sys.data.foo 123.0 \d{10,}\n$/ }
+          insist { lines }.any? { |l| l =~ /^foo.bar.sys.data.bar 42.0 \d{10,}\n$/ }
+        end
+      end
+
+      describe "no match" do
+        config <<-CONFIG
+          input {
+            generator {
+              message => "foo=123 bar=42"
+              count => 1
+              type => "generator"
+            }
+          }
+
+          filter {
+            kv { }
+          }
+
+          output {
+            graphite {
+              host => "localhost"
+              port => 2003
+              fields_are_metrics => true
+              include_metrics => ["notmatchinganything"]
+              metrics_format => "foo.bar.sys.data.*"
+              debug => true
+            }
+          }
+        CONFIG
+
+        agent do
+          @mock.rewind
+          lines = @mock.readlines
+          insist { lines.size } == 0
+        end
+      end
+
+      describe "match one key with invalid metric_format" do
+        config <<-CONFIG
+          input {
+            generator {
+              message => "foo=123"
+              count => 1
+              type => "generator"
+            }
+          }
+
+          filter {
+            kv { }
+          }
+
+          output {
+            graphite {
+                host => "localhost"
+                port => 2003
+                fields_are_metrics => true
+                include_metrics => ["foo"]
+                metrics_format => "invalidformat"
+                debug => true
+            }
+          }
+        CONFIG
+
+        agent do
+          @mock.rewind
+          lines = @mock.readlines
+          insist { lines.size } == 1
+          insist { lines[0] } =~ /^foo 123.0 \d{10,}\n$/
+        end
+      end
+    end
+  end
+
+  describe "fields are metrics = false" do
+    describe "metrics_format not set" do
+      describe "match one key with metrics list" do
+        config <<-CONFIG
+          input {
+            generator {
+              message => "foo=123"
+              count => 1
+              type => "generator"
+            }
+          }
+
+          filter {
+            kv { }
+          }
+
+          output {
+            graphite {
+                host => "localhost"
+                port => 2003
+                fields_are_metrics => false
+                include_metrics => ["foo"]
+                metrics => [ "custom.foo", "%{foo}" ]
+                debug => true
+            }
+          }
+        CONFIG
+
+        agent do
+          @mock.rewind
+          lines = @mock.readlines
+
+          insist { lines.size } == 1
+          insist { lines[0] } =~ /^custom.foo 123.0 \d{10,}\n$/
+        end
+      end
+
+    end
+  end
+end

data/spec/outputs/redis.rb

@@ -0,0 +1,127 @@
+require "test_utils"
+require "logstash/outputs/redis"
+require "redis"
+
+describe LogStash::Outputs::Redis do
+  extend LogStash::RSpec
+
+  describe "ship lots of events to a list" do
+    key = 10.times.collect { rand(10).to_s }.join("")
+    event_count = 10000 + rand(500)
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "generator"
+        }
+      }
+      output {
+        redis {
+          host => "127.0.0.1"
+          key => "#{key}"
+          data_type => list
+        }
+      }
+    CONFIG
+
+    agent do
+      # Query redis directly and inspect the goodness.
+      redis = Redis.new(:host => "127.0.0.1")
+
+      # The list should contain the number of elements our agent pushed up.
+      insist { redis.llen(key) } == event_count
+
+      # Now check all events for order and correctness.
+      event_count.times do |value|
+        id, element = redis.blpop(key, 0)
+        event = LogStash::Event.new(JSON.parse(element))
+        insist { event["sequence"] } == value
+        insist { event["message"] } == "hello world"
+      end
+
+      # The list should now be empty
+      insist { redis.llen(key) } == 0
+    end # agent
+  end
+
+  describe "batch mode" do
+    key = 10.times.collect { rand(10).to_s }.join("")
+    event_count = 200000
+
+    config <<-CONFIG
+      input {
+        generator {
+          message => "hello world"
+          count => #{event_count}
+          type => "generator"
+        }
+      }
+      output {
+        redis {
+          host => "127.0.0.1"
+          key => "#{key}"
+          data_type => list
+          batch => true
+          batch_timeout => 5
+          timeout => 5
+        }
+      }
+    CONFIG
+
+    agent do
+      # we have to wait for teardown to execute & flush the last batch.
+      # otherwise we might start doing assertions before everything has been
+      # sent out to redis.
+      sleep 2
+
+      redis = Redis.new(:host => "127.0.0.1")
+
+      # The list should contain the number of elements our agent pushed up.
+      insist { redis.llen(key) } == event_count
+
+      # Now check all events for order and correctness.
+      event_count.times do |value|
+        id, element = redis.blpop(key, 0)
+        event = LogStash::Event.new(JSON.parse(element))
+        insist { event["sequence"] } == value
+        insist { event["message"] } == "hello world"
+      end
+
+      # The list should now be empty
+      insist { redis.llen(key) } == 0
+    end # agent
+  end
+
+  describe "converts US-ASCII to utf-8 without failures" do
+    key = 10.times.collect { rand(10).to_s }.join("")
+
+    config <<-CONFIG
+      input {
+        generator {
+          charset => "US-ASCII"
+          message => "\xAD\u0000"
+          count => 1
+          type => "generator"
+        }
+      }
+      output {
+        redis {
+          host => "127.0.0.1"
+          key => "#{key}"
+          data_type => list
+        }
+      }
+    CONFIG
+
+    agent do
+      # Query redis directly and inspect the goodness.
+      redis = Redis.new(:host => "127.0.0.1")
+
+      # The list should contain no elements.
+      insist { redis.llen(key) } == 1
+    end # agent
+  end
+end
+

data/spec/speed.rb

@@ -0,0 +1,20 @@
+require "test_utils"
+
+describe "speed tests" do
+  extend LogStash::RSpec
+  count = 1000000
+
+  config <<-CONFIG
+    input {
+      generator {
+        type => foo
+        count => #{count}
+      }
+    }
+    output { null { } }
+  CONFIG
+
+  agent do
+    puts "Rate: #{count / @duration}"
+  end
+end