logstash-lib 1.3.2

data/spec/inputs/redis.rb

@@ -0,0 +1,63 @@
+require "test_utils"
+require "redis"
+
+def populate(key, event_count)
+  require "logstash/event"
+  redis = Redis.new(:host => "localhost")
+  event_count.times do |value|
+    event = LogStash::Event.new("sequence" => value)
+    Stud::try(10.times) do
+      redis.rpush(key, event.to_json)
+    end
+  end
+end
+
+def process(pipeline, queue, event_count)
+  sequence = 0
+  Thread.new { pipeline.run }
+  event_count.times do |i|
+    event = queue.pop
+    insist { event["sequence"] } == i
+  end
+  pipeline.shutdown
+end # process
+
+describe "inputs/redis" do
+  extend LogStash::RSpec
+
+  describe "read events from a list" do
+    key = 10.times.collect { rand(10).to_s }.join("")
+    event_count = 1000 + rand(50)
+    config <<-CONFIG
+      input {
+        redis {
+          type => "blah"
+          key => "#{key}"
+          data_type => "list"
+        }
+      }
+    CONFIG
+
+    before(:each) { populate(key, event_count) }
+
+    input { |pipeline, queue| process(pipeline, queue, event_count) }
+  end
+
+  describe "read events from a list with batch_count=5" do
+    key = 10.times.collect { rand(10).to_s }.join("")
+    event_count = 1000 + rand(50)
+    config <<-CONFIG
+      input {
+        redis {
+          type => "blah"
+          key => "#{key}"
+          data_type => "list"
+          batch_count => #{rand(20)+1}
+        }
+      }
+    CONFIG
+
+    before(:each) { populate(key, event_count) }
+    input { |pipeline, queue| process(pipeline, queue, event_count) }
+  end
+end

data/spec/inputs/relp.rb

@@ -0,0 +1,70 @@
+# coding: utf-8
+require "test_utils"
+require "socket"
+require "logstash/util/relp"
+
+describe "inputs/relp" do
+  extend LogStash::RSpec
+
+  describe "Single client connection" do
+    event_count = 10
+    port = 5511
+    config <<-CONFIG
+    input {
+      relp {
+        type => "blah"
+        port => #{port}
+      }
+    }
+    CONFIG
+
+    input do |pipeline, queue|
+      th = Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      #Send events from clients
+      client = RelpClient.new("0.0.0.0", port, ["syslog"])
+      event_count.times do |value|
+        client.syslog_write("Hello #{value}")
+      end
+
+      events = event_count.times.collect { queue.pop }
+      event_count.times do |i|
+        insist { events[i]["message"] } == "Hello #{i}"
+      end
+
+      pipeline.shutdown
+      th.join
+    end # input
+  end
+  describe "Two client connection" do
+    event_count = 100
+    port = 5512
+    config <<-CONFIG
+    input {
+      relp {
+        type => "blah"
+        port => #{port}
+      }
+    }
+    CONFIG
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      #Send events from clients sockets
+      client = RelpClient.new("0.0.0.0", port, ["syslog"])
+      client2 = RelpClient.new("0.0.0.0", port, ["syslog"])
+
+      event_count.times do |value|
+        client.syslog_write("Hello from client")
+        client2.syslog_write("Hello from client 2")
+      end
+
+      events = (event_count*2).times.collect { queue.pop }
+      insist { events.select{|event| event["message"]=="Hello from client" }.size } == event_count
+      insist { events.select{|event| event["message"]=="Hello from client 2" }.size } == event_count
+    end # input
+  end
+end

data/spec/inputs/tcp.rb

@@ -0,0 +1,101 @@
+# coding: utf-8
+require "test_utils"
+require "socket"
+
+describe "inputs/tcp" do
+  extend LogStash::RSpec
+
+  describe "read json_event" do
+    event_count = 10
+    port = 5511
+    config <<-CONFIG
+      input {
+        tcp {
+          port => #{port}
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      event_count.times do |i|
+        # unicode smiley for testing unicode support!
+        socket.puts("#{i} ☹")
+      end
+      socket.close
+
+      events = event_count.times.collect { queue.pop }
+      event_count.times do |i|
+        insist { events[i]["message"] } == "#{i} ☹"
+      end
+    end # input
+  end
+
+  describe "read events with plain codec and ISO-8859-1 charset" do
+    port = 5513
+    charset = "ISO-8859-1"
+    config <<-CONFIG
+      input {
+        tcp {
+          port => #{port}
+          codec => plain { charset => "#{charset}" }
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      text = "\xA3" # the £ symbol in ISO-8859-1 aka Latin-1
+      text.force_encoding("ISO-8859-1")
+      socket.puts(text)
+      socket.close
+
+      event = queue.pop
+      # Make sure the 0xA3 latin-1 code converts correctly to UTF-8.
+      insist { event["message"].size } == 1
+      insist { event["message"].bytesize } == 2
+      insist { event["message"] } == "£"
+    end # input
+  end
+
+  describe "read events with json codec" do
+    port = 5514
+    config <<-CONFIG
+      input {
+        tcp {
+          port => #{port}
+          codec => json
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      data = {
+        "hello" => "world",
+        "foo" => [1,2,3],
+        "baz" => { "1" => "2" }
+      }
+
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      socket.puts(data.to_json)
+      socket.close
+
+      event = queue.pop
+      insist { event["hello"] } == data["hello"]
+      insist { event["foo"] } == data["foo"]
+      insist { event["baz"] } == data["baz"]
+    end # input
+  end
+end
+
+
+

data/spec/jar.rb

@@ -0,0 +1,21 @@
+require "insist"
+
+describe "logstash jar features", :if => (__FILE__ =~ /file:.*~/) do
+  before :each do 
+    @jar_root = __FILE__.split("!").first + "!"
+  end
+
+  it "must be only run from a jar" do
+    insist { __FILE__ } =~ /file:.*!/
+  end
+
+  it "must contain GeoLiteCity.dat" do
+    path = File.join(@jar_root, "GeoLiteCity.dat")
+    insist { File }.exists?(path)
+  end
+
+  it "must contain vendor/ua-parser/regexes.yaml" do
+    path = File.join(@jar_root, "vendor/ua-parser/regexes.yaml")
+    insist { File }.exists?(path)
+  end
+end

data/spec/outputs/csv.rb

@@ -0,0 +1,266 @@
+require "csv"
+require "tempfile"
+require "test_utils"
+require "logstash/outputs/csv"
+
+describe LogStash::Outputs::CSV do
+  extend LogStash::RSpec
+
+  describe "Write a single field to a csv file" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          add_field => ["foo","bar"]
+          count => 1
+        }
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => "foo"
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == "bar\n"
+    end
+  end
+
+  describe "write multiple fields and lines to a csv file" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          add_field => ["foo", "bar", "baz", "quux"]
+          count => 2
+        }
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 2
+      insist {lines[0]} == "bar,quux\n"
+      insist {lines[1]} == "bar,quux\n"
+    end
+  end
+
+  describe "missing event fields are empty in csv" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          add_field => ["foo","bar", "baz", "quux"]
+          count => 1
+        }
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "not_there", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == "bar,,quux\n"
+    end
+  end
+
+  describe "commas are quoted properly" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          add_field => ["foo","one,two", "baz", "quux"]
+          count => 1
+        }
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == "\"one,two\",quux\n"
+    end
+  end
+
+  describe "new lines are quoted properly" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          add_field => ["foo","one\ntwo", "baz", "quux"]
+          count => 1
+        }
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = CSV.read(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0][0]} == "one\ntwo"
+    end
+  end
+
+  describe "fields that are are objects are written as JSON" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":{"one":"two"},"baz": "quux"}'
+          count => 1
+        }
+      }
+      filter {
+        json { source => "message"}
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = CSV.read(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0][0]} == '{"one":"two"}'
+    end
+  end
+
+  describe "can address nested field using field reference syntax" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":{"one":"two"},"baz": "quux"}'
+          count => 1
+        }
+      }
+      filter {
+        json { source => "message"}
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["[foo][one]", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = CSV.read(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0][0]} == "two"
+      insist {lines[0][1]} == "quux"
+    end
+  end
+
+  describe "missing nested field is blank" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":{"one":"two"},"baz": "quux"}'
+          count => 1
+        }
+      }
+      filter {
+        json { source => "message"}
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["[foo][missing]", "baz"]
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == ",quux\n"
+    end
+  end
+
+  describe "can choose field seperator" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":"one","bar": "two"}'
+          count => 1
+        }
+      }
+      filter {
+        json { source => "message"}
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "bar"]
+          csv_options => {"col_sep" => "|"}
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == "one|two\n"
+    end
+  end
+  describe "can choose line seperator" do
+    tmpfile = Tempfile.new('logstash-spec-output-csv')
+    config <<-CONFIG
+      input {
+        generator {
+          message => '{"foo":"one","bar": "two"}'
+          count => 2
+        }
+      }
+      filter {
+        json { source => "message"}
+      }
+      output {
+        csv {
+          path => "#{tmpfile.path}"
+          fields => ["foo", "bar"]
+          csv_options => {"col_sep" => "|" "row_sep" => "\t"}
+        }
+      }
+    CONFIG
+
+    agent do
+      lines = File.readlines(tmpfile.path)
+      insist {lines.count} == 1
+      insist {lines[0]} == "one|two\tone|two\t"
+    end
+  end
+end