logstash-lib 1.3.2

data/spec/filters/useragent.rb

@@ -0,0 +1,42 @@
+require "test_utils"
+require "logstash/filters/useragent"
+
+describe LogStash::Filters::UserAgent do
+  extend LogStash::RSpec
+
+  describe "defaults" do
+    config <<-CONFIG
+      filter {
+        useragent {
+          source => "message"
+          target => "ua"
+        }
+      }
+    CONFIG
+
+    sample "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.63 Safari/537.31" do
+      insist { subject }.include?("ua")
+      insist { subject["ua"]["name"] } == "Chrome"
+      insist { subject["ua"]["os"] } == "Linux"
+      insist { subject["ua"]["major"] } == "26"
+      insist { subject["ua"]["minor"] } == "0"
+    end
+  end
+
+  describe "" do
+    config <<-CONFIG
+      filter {
+        useragent {
+          source => "message"
+        }
+      }
+    CONFIG
+
+    sample "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.63 Safari/537.31" do
+      insist { subject["name"] } == "Chrome"
+      insist { subject["os"] } == "Linux"
+      insist { subject["major"] } == "26"
+      insist { subject["minor"] } == "0"
+    end
+  end
+end

data/spec/filters/xml.rb

@@ -0,0 +1,157 @@
+require "test_utils"
+require "logstash/filters/xml"
+
+describe LogStash::Filters::Xml do
+  extend LogStash::RSpec
+
+  describe "parse standard xml (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "raw"
+        target => "data"
+      }
+    }
+    CONFIG
+
+    sample("raw" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => "value"}
+    end
+
+    #From parse xml with array as a value
+    sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+    end
+
+    #From parse xml with hash as a value
+    sample("raw" => '<foo><key1><key2>value</key2></key1></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+    end
+
+    #From bad xml
+    sample("raw" => '<foo /') do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+    end
+  end
+
+  describe "parse standard xml but do not store (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "raw"
+        target => "data"
+        store_xml => false
+      }
+    }
+    CONFIG
+
+    sample("raw" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == nil
+    end
+  end
+
+  describe "parse xml and store values with xpath (Deprecated checks)" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "raw"
+        target => "data"
+        xpath => [ "/foo/key/text()", "xpath_field" ]
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("raw" => '<foo><key>value</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value"]
+    end
+
+    #Multiple values
+    sample("raw" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value1","value2"]
+    end
+  end
+
+  ## New tests
+
+  describe "parse standard xml" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "xmldata"
+        target => "data"
+      }
+    }
+    CONFIG
+
+    sample("xmldata" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => "value"}
+    end
+
+    #From parse xml with array as a value
+    sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key" => ["value1", "value2"]}
+    end
+
+    #From parse xml with hash as a value
+    sample("xmldata" => '<foo><key1><key2>value</key2></key1></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == {"key1" => [{"key2" => ["value"]}]}
+    end
+
+    #From bad xml
+    sample("xmldata" => '<foo /') do
+      insist { subject["tags"] }.include?("_xmlparsefailure")
+    end
+  end
+
+  describe "parse standard xml but do not store" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "xmldata"
+        target => "data"
+        store_xml => false
+      }
+    }
+    CONFIG
+
+    sample("xmldata" => '<foo key="value"/>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["data"]} == nil
+    end
+  end
+
+  describe "parse xml and store values with xpath" do
+    config <<-CONFIG
+    filter {
+      xml {
+        source => "xmldata"
+        target => "data"
+        xpath => [ "/foo/key/text()", "xpath_field" ]
+      }
+    }
+    CONFIG
+
+    # Single value
+    sample("xmldata" => '<foo><key>value</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value"]
+    end
+
+    #Multiple values
+    sample("xmldata" => '<foo><key>value1</key><key>value2</key></foo>') do
+      insist { subject["tags"] }.nil?
+      insist { subject["xpath_field"]} == ["value1","value2"]
+    end
+  end
+
+end

data/spec/inputs/file.rb

@@ -0,0 +1,107 @@
+require "test_utils"
+require "tempfile"
+
+describe "inputs/file" do
+  extend LogStash::RSpec
+
+  describe "starts at the end of an existing file" do
+    tmp_file = Tempfile.new('logstash-spec-input-file')
+
+    config <<-CONFIG
+      input {
+        file {
+          type => "blah"
+          path => "#{tmp_file.path}"
+          sincedb_path => "/dev/null"
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      File.open(tmp_file, "a") do |fd|
+        fd.puts("ignore me")
+        fd.puts("ignore me 2")
+      end
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      File.open(tmp_file, "a") do |fd|
+        fd.puts("hello")
+        fd.puts("world")
+      end
+      events = 2.times.collect { queue.pop } 
+      insist { events[0]["message"] } == "hello"
+      insist { events[1]["message"] } == "world"
+    end
+  end
+
+  describe "can start at the beginning of an existing file" do
+    tmp_file = Tempfile.new('logstash-spec-input-file')
+
+    config <<-CONFIG
+      input {
+        file {
+          type => "blah"
+          path => "#{tmp_file.path}"
+          start_position => "beginning"
+          sincedb_path => "/dev/null"
+        }
+      }
+    CONFIG
+
+    before(:each) do
+      File.open(tmp_file, "w") do |fd|
+        fd.puts "hello"
+        fd.puts "world"
+      end
+    end
+
+    after(:each) do
+      tmp_file.close!
+    end
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      events = 2.times.collect { queue.pop } 
+      insist { events[0]["message"] } == "hello"
+      insist { events[1]["message"] } == "world"
+    end
+  end
+
+  describe "restarts at the sincedb value" do
+    tmp_file = Tempfile.new('logstash-spec-input-file')
+    tmp_sincedb = Tempfile.new('logstash-spec-input-file-sincedb')
+
+    config <<-CONFIG
+      input {
+        file {
+          type => "blah"
+          path => "#{tmp_file.path}"
+          start_position => "beginning"
+          sincedb_path => "#{tmp_sincedb.path}"
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      File.open(tmp_file, "a") do |fd|
+        fd.puts "hello"
+        fd.puts "world"
+      end
+      Thread.new { pipeline.run }
+      events = 2.times.collect { queue.pop } 
+      pipeline.shutdown
+
+      File.open(tmp_file, "a") do |fd|
+        fd.puts "foo"
+        fd.puts "bar"
+        fd.puts "baz"
+      end
+      Thread.new { pipeline.run }
+      events = 3.times.collect { queue.pop } 
+      insist { events[0]["message"] } == "foo"
+      insist { events[1]["message"] } == "bar"
+      insist { events[2]["message"] } == "baz"
+    end
+  end
+end

data/spec/inputs/gelf.rb

@@ -0,0 +1,52 @@
+
+require "test_utils"
+require "gelf"
+describe "inputs/gelf" do
+  extend LogStash::RSpec
+
+  describe "reads chunked gelf messages " do
+    port = 12209
+    host = "127.0.0.1"
+    chunksize = 1420
+    gelfclient = GELF::Notifier.new(host,port,chunksize)
+
+    config <<-CONFIG
+input {
+  gelf {
+    port => "#{port}"
+    host => "#{host}"
+  }
+}
+    CONFIG
+
+    input do |pipeline, queue|
+      Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+      
+      # generate random characters (message is zipped!) from printable ascii ( SPACE till ~ )  
+      # to trigger gelf chunking
+      s = StringIO.new
+      for i in 1..2000 
+          s << 32 + rand(126-32)
+      end
+      large_random = s.string
+      
+      [ "hello", 
+        "world", 
+        large_random, 
+        "we survived gelf!" 
+      ].each do |m| 
+  	gelfclient.notify!( "short_message" => m )
+        # poll at most 10 times 
+        waits = 0 
+        while waits < 10 and queue.size == 0
+           sleep 0.1 
+           waits += 1
+        end
+        insist { queue.size } > 0  
+        insist { queue.pop["message"] } == m
+      end
+
+    end
+  end
+end

data/spec/inputs/generator.rb

@@ -0,0 +1,30 @@
+require "test_utils"
+
+describe "inputs/generator" do
+  extend LogStash::RSpec
+
+  describe "generate events" do
+    event_count = 100000 + rand(50000)
+
+    config <<-CONFIG
+      input {
+        generator {
+          type => "blah"
+          count => #{event_count}
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      start = Time.now
+      Thread.new { pipeline.run }
+      event_count.times do |i|
+        event = queue.pop
+        insist { event["sequence"] } == i
+      end
+      duration = Time.now - start
+      puts "Rate: #{event_count / duration}"
+      pipeline.shutdown
+    end # input
+  end
+end

data/spec/inputs/imap.rb

@@ -0,0 +1,60 @@
+require "logstash/inputs/imap"
+require "mail"
+
+describe LogStash::Inputs::IMAP do
+  user = "logstash"
+  password = "secret"
+  msg_time = Time.new
+  msg_text = "foo\nbar\nbaz"
+  msg_html = "<p>a paragraph</p>\n\n"
+
+  msg = Mail.new do
+    from     "me@example.com"
+    to       "you@example.com"
+    subject  "logstash imap input test"
+    date     msg_time
+    body     msg_text
+    add_file :filename => "some.html", :content => msg_html
+  end
+
+  context "with both text and html parts" do
+    context "when no content-type selected" do
+      it "should select text/plain part" do
+        config = {"type" => "imap", "host" => "localhost",
+                  "user" => "#{user}", "password" => "#{password}"}
+
+        input = LogStash::Inputs::IMAP.new config
+        input.register
+        event = input.parse_mail(msg)
+        insist { event["message"] } == msg_text
+      end
+    end
+
+    context "when text/html content-type selected" do
+      it "should select text/html part" do
+        config = {"type" => "imap", "host" => "localhost",
+                  "user" => "#{user}", "password" => "#{password}",
+                  "content_type" => "text/html"}
+
+        input = LogStash::Inputs::IMAP.new config
+        input.register
+        event = input.parse_mail(msg)
+        insist { event["message"] } == msg_html
+      end
+    end
+  end
+
+  context "when subject is in RFC 2047 encoded-word format" do
+    it "should be decoded" do
+      msg.subject = "=?iso-8859-1?Q?foo_:_bar?="
+      config = {"type" => "imap", "host" => "localhost",
+                "user" => "#{user}", "password" => "#{password}"}
+
+      input = LogStash::Inputs::IMAP.new config
+      input.register
+      event = input.parse_mail(msg)
+      insist { event["subject"] } == "foo : bar"
+    end
+  end
+
+end